diff --git a/.github/workflows/base-image.yaml b/.github/workflows/base-image.yaml index 31f78ed127..fdbcd39b28 100644 --- a/.github/workflows/base-image.yaml +++ b/.github/workflows/base-image.yaml @@ -1,15 +1,14 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -# Build and push the sandbox base images to GHCR. +# Build and push sandbox base images, then publish validated managed images. # # Triggers: -# - Push to main when a base-image workflow input changes +# - Push to main when a base- or managed-image input changes # - Manual dispatch for ad-hoc rebuilds # -# The base image contains the expensive, rarely-changing layers (apt, gosu, -# user setup, openclaw CLI). The production Dockerfile layers PR-specific -# code on top via: FROM ghcr.io/nvidia/nemoclaw/sandbox-base: +# Base images contain the expensive, rarely-changing layers. Complete image +# publication consumes exact base digests from the same trusted workflow run. name: Images / Base Images @@ -22,6 +21,26 @@ on: # Re-run when this workflow gains or changes a publisher so the new path # takes effect immediately after merge instead of waiting for another tag. - ".github/workflows/base-image.yaml" + - ".github/workflows/managed-images.yaml" + - ".dockerignore" + # Complete managed-image inputs. Keep these reviewed families synchronized + # with tools/e2e/base-image-publication.mts. + - "Dockerfile" + - "agents/**" + - "nemoclaw/**" + - "nemoclaw-blueprint/**" + - "scripts/**" + - "src/lib/actions/sandbox/openshell-child-visible-credentials.v*.json" + - "src/lib/core/json-types.ts" + - "src/lib/core/ports.ts" + - "src/lib/messaging/**" + - "src/lib/onboard/managed-startup/**" + - "src/lib/security/credential-hash.ts" + - "src/lib/state/paths.ts" + - "src/lib/state/state-root.ts" + - "src/lib/tool-disclosure.ts" + - "tools/mcp-tool-discovery-runtime/**" + - "tsconfig.runtime-preloads.json" - "Dockerfile.base" - "agents/openclaw/openclaw-runtime/package.json" - "agents/openclaw/openclaw-runtime/package-lock.json" @@ -58,8 +77,8 @@ permissions: packages: write concurrency: - group: base-image - cancel-in-progress: true + group: base-image-${{ github.ref }} + cancel-in-progress: ${{ !startsWith(github.ref, 'refs/tags/v') }} env: REGISTRY: ghcr.io @@ -257,6 +276,7 @@ jobs: printf 'openclaw_build_arg=%s\n' "$openclaw_build_arg" >> "$GITHUB_OUTPUT" - name: Build and push + id: build uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 with: context: . @@ -269,6 +289,59 @@ jobs: cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ matrix.image }}:buildcache,mode=max build-args: ${{ steps.production-build-args.outputs.openclaw_build_arg }} + - name: Export managed base image contract + env: + AGENT: ${{ matrix.agent }} + DIGEST: ${{ steps.build.outputs.digest }} + IMAGE: ${{ env.REGISTRY }}/${{ matrix.image }} + run: | + set -euo pipefail + if [[ ! "$DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]]; then + echo "ERROR: base image build did not return a valid digest: $DIGEST" >&2 + exit 1 + fi + reference="${IMAGE}@${DIGEST}" + docker buildx imagetools inspect "$reference" >/dev/null + + contract_dir="$RUNNER_TEMP/managed-base-contract" + mkdir -p "$contract_dir" + jq -n \ + --arg agent "$AGENT" \ + --arg digest "$DIGEST" \ + --arg image "$IMAGE" \ + --arg reference "$reference" \ + --arg revision "$GITHUB_SHA" \ + --argjson runAttempt "$GITHUB_RUN_ATTEMPT" \ + --argjson runId "$GITHUB_RUN_ID" \ + '{ + contractVersion: 1, + agent: $agent, + image: $image, + digest: $digest, + reference: $reference, + platforms: ["linux/amd64", "linux/arm64"], + sourceRevision: $revision, + run: { + id: $runId, + attempt: $runAttempt + } + }' > "$contract_dir/contract.json" + jq -e \ + '.contractVersion == 1 + and (.sourceRevision | test("^[0-9a-f]{40}$")) + and (.digest | test("^sha256:[0-9a-f]{64}$")) + and .reference == (.image + "@" + .digest) + and .platforms == ["linux/amd64", "linux/arm64"]' \ + "$contract_dir/contract.json" >/dev/null + + - name: Upload managed base image contract + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: managed-base-${{ matrix.agent }} + path: ${{ runner.temp }}/managed-base-contract/contract.json + if-no-files-found: error + retention-days: 1 + # Preserve the established required-check name while making tag publication # contingent on both native platform builds succeeding. build-and-push-openclaw: @@ -308,6 +381,7 @@ jobs: type=sha,prefix=,format=short - name: Create and verify multi-platform manifest + id: manifest env: IMAGE: ${{ env.REGISTRY }}/nvidia/nemoclaw/sandbox-base TAGS: ${{ steps.meta.outputs.tags }} @@ -354,3 +428,83 @@ jobs: echo "ERROR: published manifest has unexpected platforms: $actual_platforms" >&2 exit 1 fi + manifest_inspect="$(docker buildx imagetools inspect "$first_tag")" + mapfile -t manifest_digests < <( + printf '%s\n' "$manifest_inspect" \ + | sed -nE 's/^Digest:[[:space:]]*(sha256:[0-9a-f]{64})$/\1/p' + ) + if [ "${#manifest_digests[@]}" -ne 1 ]; then + echo "ERROR: expected one published OpenClaw base digest." >&2 + exit 1 + fi + digest="${manifest_digests[0]}" + docker buildx imagetools inspect "$IMAGE@$digest" >/dev/null + printf 'digest=%s\n' "$digest" >> "$GITHUB_OUTPUT" + + - name: Export managed base image contract + env: + DIGEST: ${{ steps.manifest.outputs.digest }} + IMAGE: ${{ env.REGISTRY }}/nvidia/nemoclaw/sandbox-base + run: | + set -euo pipefail + if [[ ! "$DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]]; then + echo "ERROR: OpenClaw base manifest did not return a valid digest: $DIGEST" >&2 + exit 1 + fi + reference="${IMAGE}@${DIGEST}" + docker buildx imagetools inspect "$reference" >/dev/null + + contract_dir="$RUNNER_TEMP/managed-base-contract" + mkdir -p "$contract_dir" + jq -n \ + --arg digest "$DIGEST" \ + --arg image "$IMAGE" \ + --arg reference "$reference" \ + --arg revision "$GITHUB_SHA" \ + --argjson runAttempt "$GITHUB_RUN_ATTEMPT" \ + --argjson runId "$GITHUB_RUN_ID" \ + '{ + contractVersion: 1, + agent: "openclaw", + image: $image, + digest: $digest, + reference: $reference, + platforms: ["linux/amd64", "linux/arm64"], + sourceRevision: $revision, + run: { + id: $runId, + attempt: $runAttempt + } + }' > "$contract_dir/contract.json" + jq -e \ + '.contractVersion == 1 + and .agent == "openclaw" + and (.sourceRevision | test("^[0-9a-f]{40}$")) + and (.digest | test("^sha256:[0-9a-f]{64}$")) + and .reference == (.image + "@" + .digest) + and .platforms == ["linux/amd64", "linux/arm64"]' \ + "$contract_dir/contract.json" >/dev/null + + - name: Upload managed base image contract + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: managed-base-openclaw + path: ${{ runner.temp }}/managed-base-contract/contract.json + if-no-files-found: error + retention-days: 1 + + # The managed-image publisher consumes exact base digest contracts from the + # jobs above. Keep it in the same run so a release cannot race its bases. + publish-managed-images: + name: Publish complete managed images + if: >- + github.repository == 'NVIDIA/NemoClaw' && + (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')) && + (github.event_name != 'workflow_dispatch' || inputs.openclaw_version == '') + needs: + - build-and-push + - build-and-push-openclaw + permissions: + contents: read + packages: write + uses: ./.github/workflows/managed-images.yaml diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 485cbd1c65..6cd01a0cb9 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -15,7 +15,12 @@ on: default: "" type: string jobs: - description: "Optional comma-separated E2E test IDs. Empty runs default-enabled tests only when targets is also empty; explicit-only tests openshell-gateway-auth-contract, mcp-bridge-dev, hermes-gpu-startup, sandbox-rlimits-connect, jetson-nvmap-gpu, and staging-brev-launchable are skipped unless selected." + description: "Optional comma-separated E2E test IDs. Empty runs default-enabled tests only when targets is also empty; explicit-only tests openshell-gateway-auth-contract, mcp-bridge-dev, hermes-gpu-startup, sandbox-rlimits-connect, jetson-nvmap-gpu, podman-all-agents, and staging-brev-launchable are skipped unless selected." + required: false + default: "" + type: string + podman_managed_image_release: + description: "Published complete managed-image release for the explicit podman-all-agents lane (for example v0.0.98). A selected job fails visibly when this input is omitted." required: false default: "" type: string @@ -5776,6 +5781,369 @@ jobs: shell: bash run: bash .github/scripts/docker-auth-cleanup.sh + podman-all-agents: + name: Native Podman (${{ matrix.agent }}) + needs: generate-matrix + if: ${{ contains(format(',{0},', inputs.jobs), ',podman-all-agents,') || contains(format(',{0},', inputs.targets), ',podman-all-agents,') }} + runs-on: ubuntu-26.04 + timeout-minutes: 90 + strategy: + fail-fast: false + matrix: + agent: [openclaw, hermes, langchain-deepagents-code] + env: + E2E_JOB: "1" + E2E_DEFAULT_ENABLED: "0" + E2E_TARGET_ID: "podman-all-agents" + E2E_PODMAN_AGENT: ${{ matrix.agent }} + E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/podman-all-agents/${{ matrix.agent }} + NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js + NEMOCLAW_E2E_SHARD: ${{ matrix.agent }} + NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1" + NEMOCLAW_RUN_LIVE_E2E: "1" + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + NEMOCLAW_FRESH: "1" + NEMOCLAW_SANDBOX_GPU: "0" + OPENSHELL_GATEWAY: "nemoclaw" + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: ${{ inputs.checkout_repository || github.repository }} + ref: ${{ inputs.checkout_sha || github.sha }} + fetch-depth: 0 + persist-credentials: false + + - name: Install Docker invocation guard + shell: bash + run: | + set -euo pipefail + guard_dir="$RUNNER_TEMP/nemoclaw-podman-runtime-guard" + guard_log="$guard_dir/docker-invocations.log" + original_docker_bin="$(command -v docker || true)" + install -d -m 0700 "$guard_dir/bin" + : >"$guard_log" + cat >"$guard_dir/bin/docker" <<'DOCKER_GUARD' + #!/usr/bin/env bash + printf '%s\n' "$*" >> "${E2E_DOCKER_GUARD_LOG:?}" + printf 'Docker CLI use is forbidden in the native Podman E2E lane.\n' >&2 + exit 97 + DOCKER_GUARD + chmod 0700 "$guard_dir/bin/docker" + { + printf 'DOCKER_API_VERSION=\n' + printf 'DOCKER_CERT_PATH=\n' + printf 'DOCKER_CONFIG=\n' + printf 'DOCKER_CONTEXT=\n' + printf 'DOCKER_HOST=\n' + printf 'DOCKER_TLS_VERIFY=\n' + printf 'E2E_DOCKER_GUARD_LOG=%s\n' "$guard_log" + printf 'E2E_DOCKER_GUARD_BIN=%s\n' "$guard_dir/bin/docker" + printf 'E2E_ORIGINAL_DOCKER_BIN=%s\n' "$original_docker_bin" + printf 'PATH=%s:%s\n' "$guard_dir/bin" "$PATH" + } >>"$GITHUB_ENV" + + - name: Disable the Docker daemon for native Podman proof + shell: bash + run: | + set -euo pipefail + sudo systemctl stop docker.service docker.socket + sudo systemctl mask --runtime docker.service docker.socket + if systemctl is-active --quiet docker.service || systemctl is-active --quiet docker.socket; then + echo "::error::Docker service or socket unit remained active after it was disabled" >&2 + exit 1 + fi + if [ -S /var/run/docker.sock ]; then + echo "::error::Docker socket remained available after the daemon was disabled" >&2 + exit 1 + fi + docker_candidate="$(command -v docker || true)" + if [ "$docker_candidate" != "$E2E_DOCKER_GUARD_BIN" ]; then + echo "::error::Docker command resolution escaped the native Podman guard" >&2 + exit 1 + fi + install -d -m 0700 "$E2E_ARTIFACT_DIR" + jq -n \ + --arg candidateDockerBin "$docker_candidate" \ + --arg originalDockerBin "$E2E_ORIGINAL_DOCKER_BIN" \ + '{ + schemaVersion: 1, + dockerServiceActive: false, + dockerSocketActive: false, + dockerSocketPresent: false, + candidateDockerBin: $candidateDockerBin, + originalDockerBin: ($originalDockerBin | if length > 0 then . else null end) + }' >"$E2E_ARTIFACT_DIR/docker-absence-boundary.json" + + - id: podman_catalog + name: Resolve complete published managed-image release + env: + REQUESTED_RELEASE: ${{ inputs.podman_managed_image_release }} + shell: bash + run: | + set -euo pipefail + install -d -m 0700 "$E2E_ARTIFACT_DIR" + gate_evidence="$E2E_ARTIFACT_DIR/podman-release-gate.json" + catalog_evidence="$E2E_ARTIFACT_DIR/podman-managed-image-catalog.json" + release="${REQUESTED_RELEASE:-}" + if [ -z "$release" ]; then + jq -n \ + --arg reason "podman-all-agents is gated until a published complete managed-image release is supplied" \ + '{schemaVersion:1,status:"gated",reason:$reason}' >"$gate_evidence" + { + printf 'run=false\n' + printf 'catalog_evidence=%s\n' "$catalog_evidence" + } >>"$GITHUB_OUTPUT" + printf '### Native Podman E2E\n\nGated: no published managed-image release was supplied.\n' >>"$GITHUB_STEP_SUMMARY" + echo "::error::podman-all-agents was selected without podman_managed_image_release; reporting this lane as not passed" >&2 + exit 1 + fi + if [[ ! "$release" =~ ^v[0-9]+([.][0-9]+){2}([-.][0-9A-Za-z][0-9A-Za-z.-]*)?$ ]]; then + echo "::error::podman_managed_image_release must be a v-prefixed release version" >&2 + exit 1 + fi + + declare -A images=( + [openclaw]="ghcr.io/nvidia/nemoclaw/openclaw-sandbox" + [hermes]="ghcr.io/nvidia/nemoclaw/hermes-sandbox" + [langchain-deepagents-code]="ghcr.io/nvidia/nemoclaw/langchain-deepagents-code-sandbox" + ) + openclaw_json="$RUNNER_TEMP/podman-managed-openclaw.json" + skopeo inspect --override-os linux --override-arch amd64 \ + "docker://${images[openclaw]}:$release" >"$openclaw_json" + source_cohort="$( + jq -r '.Labels["io.nvidia.nemoclaw.managed-image.cohort"] // empty' "$openclaw_json" + )" + source_revision="$( + jq -r '.Labels["org.opencontainers.image.revision"] // empty' "$openclaw_json" + )" + if [[ ! "$source_cohort" =~ ^ghrun-[1-9][0-9]{0,19}-[1-9][0-9]{0,9}$ ]] || + [[ ! "$source_revision" =~ ^[0-9a-f]{40}$ ]]; then + echo "::error::OpenClaw release pointer lacks a valid cohort/revision identity" >&2 + exit 1 + fi + + rows="$RUNNER_TEMP/podman-managed-image-rows.jsonl" + : >"$rows" + for agent in openclaw hermes langchain-deepagents-code; do + image="${images[$agent]}" + if [ "$agent" = "openclaw" ]; then + root_reference="$release" + inspect_json="$openclaw_json" + else + root_reference="cohort-$source_cohort" + inspect_json="$RUNNER_TEMP/podman-managed-$agent.json" + skopeo inspect --override-os linux --override-arch amd64 \ + "docker://$image:$root_reference" >"$inspect_json" + fi + if ! jq -e \ + --arg agent "$agent" \ + --arg cohort "$source_cohort" \ + --arg revision "$source_revision" \ + ' + .Os == "linux" + and .Architecture == "amd64" + and .Labels["io.nvidia.nemoclaw.agent"] == $agent + and .Labels["io.nvidia.nemoclaw.managed-image.contract"] == "1" + and .Labels["io.nvidia.nemoclaw.managed-image.platform"] == "linux/amd64" + and .Labels["io.nvidia.nemoclaw.managed-image.startup-profile"] == "1" + and .Labels["io.nvidia.nemoclaw.managed-image.capabilities"] == "1" + and .Labels["io.nvidia.nemoclaw.managed-image.cohort"] == $cohort + and .Labels["org.opencontainers.image.revision"] == $revision + and .Labels["org.opencontainers.image.source"] == "https://github.com/NVIDIA/NemoClaw" + and (.Digest | test("^sha256:[0-9a-f]{64}$")) + ' "$inspect_json" >/dev/null + then + echo "::error::$agent managed image failed the complete release contract" >&2 + exit 1 + fi + digest="$(jq -r '.Digest' "$inspect_json")" + jq -n \ + --arg agent "$agent" \ + --arg digest "$digest" \ + --arg image "$image" \ + --arg release "$release" \ + --arg rootReference "$root_reference" \ + --arg sourceCohort "$source_cohort" \ + --arg sourceRevision "$source_revision" \ + '{ + agent: $agent, + release: $release, + image: $image, + digest: $digest, + reference: ($image + "@" + $digest), + rootReference: $rootReference, + sourceCohort: $sourceCohort, + sourceRevision: $sourceRevision + }' >>"$rows" + done + jq -s --arg release "$release" \ + '{schemaVersion:1,status:"resolved",release:$release,images:.}' \ + "$rows" >"$catalog_evidence" + jq -e \ + --arg cohort "$source_cohort" \ + --arg release "$release" \ + --arg revision "$source_revision" \ + ' + .status == "resolved" + and .release == $release + and (.images | length) == 3 + and ([.images[].agent] | sort) == [ + "hermes", + "langchain-deepagents-code", + "openclaw" + ] + and all(.images[]; + .release == $release + and .sourceCohort == $cohort + and .sourceRevision == $revision + and .reference == (.image + "@" + .digest) + ) + ' "$catalog_evidence" >/dev/null + if ! git merge-base --is-ancestor "$source_revision" HEAD; then + echo "::error::managed-image release revision is not an ancestor of the tested checkout" >&2 + exit 1 + fi + jq -n \ + --arg release "$release" \ + --arg sourceCohort "$source_cohort" \ + --arg sourceRevision "$source_revision" \ + '{schemaVersion:1,status:"ready",release:$release,sourceCohort:$sourceCohort,sourceRevision:$sourceRevision}' \ + >"$gate_evidence" + { + printf 'run=true\n' + printf 'release=%s\n' "$release" + printf 'catalog_evidence=%s\n' "$catalog_evidence" + } >>"$GITHUB_OUTPUT" + + - name: Bind the candidate CLI to the verified release identity + if: ${{ steps.podman_catalog.outputs.run == 'true' }} + env: + MANAGED_IMAGE_RELEASE: ${{ steps.podman_catalog.outputs.release }} + shell: bash + run: | + set -euo pipefail + git tag --force "$MANAGED_IMAGE_RELEASE" HEAD + resolved="$(git describe --tags --match 'v*')" + if [ "$resolved" != "$MANAGED_IMAGE_RELEASE" ]; then + echo "::error::candidate CLI did not resolve to the verified managed-image release" >&2 + exit 1 + fi + + - name: Start exact rootless Podman API socket + if: ${{ steps.podman_catalog.outputs.run == 'true' }} + shell: bash + run: | + set -euo pipefail + podman_version="$(podman --version)" + if [[ ! "$podman_version" =~ ^podman[[:space:]]version[[:space:]]([0-9]+)([.][0-9]+){1,3}$ ]] || + (( BASH_REMATCH[1] < 5 )); then + echo "::error::native Podman E2E requires Podman 5 or newer; got '$podman_version'" >&2 + exit 1 + fi + uid="$(id -u)" + runtime_dir="/run/user/$uid" + socket_path="$runtime_dir/podman/podman.sock" + if [ ! -d "$runtime_dir" ]; then + sudo install -d -o "$uid" -g "$(id -g)" -m 0700 "$runtime_dir" + fi + install -d -m 0700 "$runtime_dir" "$runtime_dir/podman" + service_log="$E2E_ARTIFACT_DIR/podman-system-service.log" + service_pid_file="$RUNNER_TEMP/nemoclaw-podman-service.pid" + if ! podman --url "unix://$socket_path" info --format json >/dev/null 2>&1; then + podman system service --time=0 "unix://$socket_path" >"$service_log" 2>&1 & + service_pid="$!" + printf '%s\n' "$service_pid" >"$service_pid_file" + fi + for attempt in $(seq 1 30); do + if podman --url "unix://$socket_path" info --format json \ + >"$E2E_ARTIFACT_DIR/podman-info.json" 2>>"$service_log"; then + break + fi + if [ "$attempt" = "30" ]; then + echo "::error::exact rootless Podman API socket did not become ready" >&2 + exit 1 + fi + sleep 1 + done + if ! jq -e '(.host.security.rootless // .Host.Security.Rootless) == true' \ + "$E2E_ARTIFACT_DIR/podman-info.json" >/dev/null; then + echo "::error::Podman API socket is not rootless" >&2 + exit 1 + fi + if [ ! -S "$socket_path" ]; then + echo "::error::Podman API endpoint is not a Unix socket: $socket_path" >&2 + exit 1 + fi + { + printf 'E2E_PODMAN_CATALOG_EVIDENCE=%s\n' "${{ steps.podman_catalog.outputs.catalog_evidence }}" + printf 'E2E_PODMAN_MANAGED_IMAGE_RELEASE=%s\n' "${{ steps.podman_catalog.outputs.release }}" + printf 'OPENSHELL_PODMAN_NETWORK_NAME=openshell-e2e-%s-%s\n' "$GITHUB_RUN_ID" "$GITHUB_RUN_ATTEMPT" + printf 'OPENSHELL_PODMAN_SOCKET=%s\n' "$socket_path" + printf 'XDG_RUNTIME_DIR=%s\n' "$runtime_dir" + } >>"$GITHUB_ENV" + jq -n \ + --arg podmanVersion "$podman_version" \ + --arg socketPath "$socket_path" \ + '{schemaVersion:1,driverName:"podman",rootless:true,podmanVersion:$podmanVersion,socketPath:$socketPath}' \ + >"$E2E_ARTIFACT_DIR/podman-runtime.json" + + - name: Prepare E2E workspace + uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75 + + - name: Run native Podman all-agent live test + if: ${{ steps.podman_catalog.outputs.run == 'true' }} + env: + NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }} + run: | + set -euo pipefail + npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/podman-all-agents.test.ts + + - name: Verify the lane never invoked Docker + if: ${{ always() }} + shell: bash + run: | + set -euo pipefail + if [ ! -f "$E2E_DOCKER_GUARD_LOG" ]; then + echo "::error::Docker invocation guard evidence is missing" >&2 + exit 1 + fi + if [ -s "$E2E_DOCKER_GUARD_LOG" ]; then + echo "::error::native Podman E2E invoked the Docker CLI" >&2 + sed -n '1,20p' "$E2E_DOCKER_GUARD_LOG" >&2 + exit 1 + fi + if [ "$(command -v docker || true)" != "$E2E_DOCKER_GUARD_BIN" ]; then + echo "::error::Docker command resolution escaped the native Podman guard" >&2 + exit 1 + fi + if systemctl is-active --quiet docker.service || systemctl is-active --quiet docker.socket; then + echo "::error::Docker service or socket unit became active during the native Podman lane" >&2 + exit 1 + fi + if [ -S /var/run/docker.sock ]; then + echo "::error::Docker socket became available during the native Podman lane" >&2 + exit 1 + fi + + - name: Stop rootless Podman API service + if: ${{ always() && steps.podman_catalog.outputs.run == 'true' }} + shell: bash + run: | + set -euo pipefail + service_pid_file="$RUNNER_TEMP/nemoclaw-podman-service.pid" + if [ -f "$service_pid_file" ]; then + service_pid="$(cat "$service_pid_file")" + if [[ "$service_pid" =~ ^[1-9][0-9]*$ ]]; then + kill "$service_pid" 2>/dev/null || true + wait "$service_pid" 2>/dev/null || true + fi + fi + + - name: Upload native Podman artifacts + if: always() + uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 + spark-install: needs: generate-matrix if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',spark-install,') || contains(format(',{0},', inputs.targets), ',spark-install,') }} @@ -5913,6 +6281,7 @@ jobs: openclaw-discord-pairing, openclaw-slack-pairing, channels-stop-start, + podman-all-agents, spark-install, ] if: ${{ always() && github.event_name == 'workflow_dispatch' && inputs.checkout_sha == '' }} diff --git a/.github/workflows/managed-images.yaml b/.github/workflows/managed-images.yaml new file mode 100644 index 0000000000..42d208f2e7 --- /dev/null +++ b/.github/workflows/managed-images.yaml @@ -0,0 +1,1238 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Publish complete managed images from exact base-image contracts. Every agent +# image applies the same startup-profile contract and is validated before the +# complete set receives a unique staged cohort alias. Only OpenClaw then receives +# public release or revision pointers for the complete cohort. The initial +# platform is Linux x86_64; the OCI image contract remains neutral across Podman, +# Docker, and future compute drivers. + +name: Images / Managed Images + +on: + workflow_call: + pull_request: + paths: + - ".github/workflows/managed-images.yaml" + - ".dockerignore" + - "Dockerfile" + - "agents/**" + - "ci/npm-audit-exceptions.json" + - "nemoclaw/**" + - "nemoclaw-blueprint/**" + - "scripts/**" + - "src/lib/actions/sandbox/openshell-child-visible-credentials.v*.json" + - "src/lib/core/json-types.ts" + - "src/lib/core/ports.ts" + - "src/lib/messaging/**" + - "src/lib/onboard/managed-startup/**" + - "src/lib/security/credential-hash.ts" + - "src/lib/state/paths.ts" + - "src/lib/state/state-root.ts" + - "src/lib/tool-disclosure.ts" + - "src/lib/**" + - "tools/mcp-tool-discovery-runtime/**" + - "tsconfig.runtime-preloads.json" + +permissions: + contents: read + packages: read + +env: + REGISTRY: ghcr.io + +jobs: + pr-build-and-entrypoint: + name: PR build, managed startup, and OpenShell (${{ matrix.display_name }}) + if: github.event_name == 'pull_request' + runs-on: ubuntu-24.04 + timeout-minutes: 90 + permissions: + contents: read + packages: read + strategy: + fail-fast: false + matrix: + include: + - agent: openclaw + display_name: OpenClaw + dockerfile: Dockerfile + base_alias: ghcr.io/nvidia/nemoclaw/sandbox-base:latest + base_repository: ghcr.io/nvidia/nemoclaw/sandbox-base + image: nemoclaw-managed-pr/openclaw + - agent: hermes + display_name: Hermes + dockerfile: agents/hermes/Dockerfile + base_alias: ghcr.io/nvidia/nemoclaw/hermes-sandbox-base:latest + base_repository: ghcr.io/nvidia/nemoclaw/hermes-sandbox-base + image: nemoclaw-managed-pr/hermes + - agent: langchain-deepagents-code + display_name: Deep Agents Code + dockerfile: agents/langchain-deepagents-code/Dockerfile + base_alias: ghcr.io/nvidia/nemoclaw/langchain-deepagents-code-sandbox-base:latest + base_repository: ghcr.io/nvidia/nemoclaw/langchain-deepagents-code-sandbox-base + image: nemoclaw-managed-pr/langchain-deepagents-code + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 + + - name: Set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: 22.19.0 + + - name: Resolve exact linux/amd64 PR base + id: base + shell: bash + env: + BASE_ALIAS: ${{ matrix.base_alias }} + BASE_REPOSITORY: ${{ matrix.base_repository }} + DISPLAY_NAME: ${{ matrix.display_name }} + run: | + set -euo pipefail + alias_raw="$RUNNER_TEMP/pr-base-alias.raw" + exact_raw="$RUNNER_TEMP/pr-base-exact.raw" + docker buildx imagetools inspect "$BASE_ALIAS" --raw > "$alias_raw" + if ! digest="$( + jq -er ' + if ( + .mediaType == "application/vnd.oci.image.index.v1+json" or + .mediaType == "application/vnd.docker.distribution.manifest.list.v2+json" + ) then + [ + .manifests[] + | select( + .platform.os == "linux" and + .platform.architecture == "amd64" + ) + ] + | if length == 1 then .[0].digest else error("not one linux/amd64 image") end + else + error("base alias is not a platform index") + end + ' "$alias_raw" + )"; then + echo "ERROR: PR base alias does not contain exactly one linux/amd64 image." >&2 + exit 1 + fi + if [[ ! "$digest" =~ ^sha256:[0-9a-f]{64}$ ]]; then + echo "ERROR: PR base alias returned an invalid linux/amd64 digest." >&2 + exit 1 + fi + reference="${BASE_REPOSITORY}@${digest}" + docker buildx imagetools inspect "$reference" --raw > "$exact_raw" + actual="sha256:$(sha256sum "$exact_raw" | awk '{print $1}')" + if [ "$actual" != "$digest" ]; then + echo "ERROR: exact PR base bytes do not match the selected descriptor digest." >&2 + exit 1 + fi + printf 'ref=%s\n' "$reference" >> "$GITHUB_OUTPUT" + printf '### %s PR base\n\n`%s`\n' "$DISPLAY_NAME" "$reference" \ + >> "$GITHUB_STEP_SUMMARY" + + - name: Validate PR managed-image build args + shell: bash + env: + BASE_REFERENCE: ${{ steps.base.outputs.ref }} + DOCKERFILE: ${{ matrix.dockerfile }} + run: | + set -euo pipefail + scripts/check-production-build-args.sh \ + -f "$DOCKERFILE" \ + --build-arg "BASE_IMAGE=${BASE_REFERENCE}" \ + --build-arg "NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION=1" \ + --build-arg "NEMOCLAW_MANAGED_IMAGE_RUNTIME_USER=root" + + - name: Build PR managed image locally + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 + with: + context: . + file: ${{ matrix.dockerfile }} + platforms: linux/amd64 + load: true + push: false + tags: ${{ matrix.image }}:${{ github.sha }} + labels: | + org.opencontainers.image.source=https://github.com/${{ github.repository }} + org.opencontainers.image.revision=${{ github.sha }} + io.nvidia.nemoclaw.agent=${{ matrix.agent }} + io.nvidia.nemoclaw.managed-image.contract=1 + io.nvidia.nemoclaw.managed-image.platform=linux/amd64 + io.nvidia.nemoclaw.managed-image.startup-profile=1 + io.nvidia.nemoclaw.managed-image.capabilities=1 + io.nvidia.nemoclaw.managed-image.cohort=ghrun-${{ github.run_id }}-${{ github.run_attempt }} + build-args: | + BASE_IMAGE=${{ steps.base.outputs.ref }} + NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION=1 + NEMOCLAW_MANAGED_IMAGE_RUNTIME_USER=root + cache-from: type=registry,ref=ghcr.io/nvidia/nemoclaw/${{ matrix.agent }}-sandbox:buildcache-linux-amd64 + provenance: false + sbom: false + + - name: Validate exact PR managed image contract + id: contract + shell: bash + env: + AGENT: ${{ matrix.agent }} + IMAGE_REFERENCE: ${{ matrix.image }}:${{ github.sha }} + PLATFORM: linux/amd64 + PUBLICATION_COHORT: ghrun-${{ github.run_id }}-${{ github.run_attempt }} + run: | + set -euo pipefail + image_json="$(docker image inspect "$IMAGE_REFERENCE")" + image_id="$( + jq -er ' + if length == 1 and (.[0].Id | type) == "string" + then .[0].Id + else error("expected one local image identity") + end + ' <<< "$image_json" + )" + if [[ ! "$image_id" =~ ^sha256:[0-9a-f]{64}$ ]]; then + echo "ERROR: PR managed image did not resolve to an immutable local image ID: $image_id" >&2 + exit 1 + fi + if [ "$(docker image inspect --format '{{.Id}}' "$image_id")" != "$image_id" ]; then + echo "ERROR: PR managed image ID did not resolve to itself." >&2 + exit 1 + fi + + if [[ ! "$PUBLICATION_COHORT" =~ ^ghrun-[1-9][0-9]{0,19}-[1-9][0-9]{0,9}$ ]] || + ! jq -e \ + --arg agent "$AGENT" \ + --arg cohort "$PUBLICATION_COHORT" \ + --arg image_id "$image_id" \ + --arg platform "$PLATFORM" \ + --arg revision "$GITHUB_SHA" ' + length == 1 and + .[0].Id == $image_id and + ((.[0].Config.User // "") as $user | + $user == "" or $user == "root" or $user == "0") and + .[0].Config.Labels["io.nvidia.nemoclaw.agent"] == $agent and + .[0].Config.Labels["io.nvidia.nemoclaw.managed-image.contract"] == "1" and + .[0].Config.Labels["io.nvidia.nemoclaw.managed-image.platform"] == $platform and + .[0].Config.Labels["io.nvidia.nemoclaw.managed-image.startup-profile"] == "1" and + .[0].Config.Labels["io.nvidia.nemoclaw.managed-image.capabilities"] == "1" and + .[0].Config.Labels["io.nvidia.nemoclaw.managed-image.cohort"] == $cohort and + .[0].Config.Labels["org.opencontainers.image.revision"] == $revision + ' <<< "$image_json" >/dev/null; then + echo "ERROR: PR managed image contract does not match the exact build identity." >&2 + exit 1 + fi + + printf 'reference=%s\n' "$image_id" >> "$GITHUB_OUTPUT" + printf '### %s exact PR managed image\n\n`%s`\n' "$AGENT" "$image_id" \ + >> "$GITHUB_STEP_SUMMARY" + + - name: Install managed-image OpenShell harness dependencies + run: | + npm ci --ignore-scripts + npm run build:policy-boundary + + - name: Exercise managed startup root stdin and hold + shell: bash + env: + AGENT: ${{ matrix.agent }} + IMAGE_REFERENCE: ${{ steps.contract.outputs.reference }} + run: | + set -euo pipefail + npx --no-install tsx \ + scripts/checks/run-managed-image-direct-e2e.ts \ + --agent "$AGENT" \ + --image "$IMAGE_REFERENCE" \ + --platform linux/amd64 + + - name: Install pinned OpenShell runtime + env: + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_OPENSHELL_MIN_VERSION: "0.0.85" + NEMOCLAW_OPENSHELL_MAX_VERSION: "0.0.85" + NEMOCLAW_OPENSHELL_PIN_VERSION: "0.0.85" + run: | + set -euo pipefail + env -u DOCKER_CONFIG \ + -u DOCKERHUB_USERNAME \ + -u DOCKERHUB_TOKEN \ + -u NVIDIA_API_KEY \ + -u NVIDIA_INFERENCE_API_KEY \ + -u GITHUB_TOKEN \ + -u GH_TOKEN \ + bash scripts/install-openshell.sh + + - name: Exercise exact PR image through real OpenShell + env: + AGENT: ${{ matrix.agent }} + IMAGE_REFERENCE: ${{ steps.contract.outputs.reference }} + SANDBOX_NAME: nemoclaw-pr-${{ matrix.agent }} + run: | + set -euo pipefail + export PATH="$HOME/.local/bin:$PATH" + npx tsx scripts/checks/run-managed-image-openshell-e2e.ts \ + --agent "$AGENT" \ + --image "$IMAGE_REFERENCE" \ + --sandbox "$SANDBOX_NAME" + + # Generic sandbox-create diagnostics can contain raw gateway or console + # output. Export only a bounded, fully redacted allowlist into a fresh + # runner-temporary tree before printing or uploading any evidence. + - name: Export sanitized managed-image OpenShell failure diagnostics + id: managed_image_openshell_diagnostics + if: failure() + shell: bash + run: | + set -euo pipefail + diagnostics_root="$HOME/.nemoclaw/onboard-failures" + sanitized_root="$(mktemp -d "$RUNNER_TEMP/managed-image-openshell-diagnostics.XXXXXX")" + npx --no-install tsx \ + scripts/checks/export-managed-image-failure-diagnostics.ts \ + --source "$diagnostics_root" \ + --output "$sanitized_root" + printf 'path=%s\n' "$sanitized_root" >> "$GITHUB_OUTPUT" + while IFS= read -r summary; do + printf '\n===== %s =====\n' "${summary#"$sanitized_root"/}" + sed -n '1,160p' "$summary" + done < <(find "$sanitized_root" -type f -name summary.txt | sort) + + - name: Upload managed-image OpenShell failure diagnostics + if: ${{ failure() && steps.managed_image_openshell_diagnostics.outcome == 'success' }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: managed-image-openshell-pr-${{ matrix.agent }}-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ steps.managed_image_openshell_diagnostics.outputs.path }} + include-hidden-files: true + if-no-files-found: ignore + retention-days: 14 + + build-and-validate: + name: Build and validate ${{ matrix.display_name }} managed image + if: github.event_name != 'pull_request' + runs-on: ubuntu-24.04 + timeout-minutes: 120 + permissions: + contents: read + packages: write + strategy: + fail-fast: false + matrix: + include: + - agent: openclaw + display_name: OpenClaw + dockerfile: Dockerfile + base_image: nvidia/nemoclaw/sandbox-base + image: nvidia/nemoclaw/openclaw-sandbox + platform: linux/amd64 + artifact_platform: linux-amd64 + required_binary: /usr/local/bin/openclaw + - agent: hermes + display_name: Hermes + dockerfile: agents/hermes/Dockerfile + base_image: nvidia/nemoclaw/hermes-sandbox-base + image: nvidia/nemoclaw/hermes-sandbox + platform: linux/amd64 + artifact_platform: linux-amd64 + required_binary: /usr/local/bin/hermes + - agent: langchain-deepagents-code + display_name: Deep Agents Code + dockerfile: agents/langchain-deepagents-code/Dockerfile + base_image: nvidia/nemoclaw/langchain-deepagents-code-sandbox-base + image: nvidia/nemoclaw/langchain-deepagents-code-sandbox + platform: linux/amd64 + artifact_platform: linux-amd64 + required_binary: /usr/local/bin/dcode + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: 22.19.0 + + - name: Download exact base image contract + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: managed-base-${{ matrix.agent }} + path: ${{ runner.temp }}/managed-base-contract + + - name: Validate exact base image contract + id: base + shell: bash + env: + AGENT: ${{ matrix.agent }} + BASE_IMAGE: ${{ env.REGISTRY }}/${{ matrix.base_image }} + CONTRACT: ${{ runner.temp }}/managed-base-contract/contract.json + PLATFORM: ${{ matrix.platform }} + run: | + set -euo pipefail + if [ ! -f "$CONTRACT" ] || [ -L "$CONTRACT" ]; then + echo "ERROR: exact base image contract is missing or is a symlink." >&2 + exit 1 + fi + if ! jq -e \ + --arg agent "$AGENT" \ + --arg image "$BASE_IMAGE" \ + --arg platform "$PLATFORM" \ + --arg revision "$GITHUB_SHA" \ + --argjson runAttempt "$GITHUB_RUN_ATTEMPT" \ + --argjson runId "$GITHUB_RUN_ID" \ + ' + (keys | sort) == [ + "agent", + "contractVersion", + "digest", + "image", + "platforms", + "reference", + "run", + "sourceRevision" + ] + and .contractVersion == 1 + and .agent == $agent + and .image == $image + and (.digest | test("^sha256:[0-9a-f]{64}$")) + and .reference == (.image + "@" + .digest) + and (.platforms | index($platform)) != null + and .sourceRevision == $revision + and .run == {id: $runId, attempt: $runAttempt} + ' "$CONTRACT" >/dev/null + then + echo "ERROR: exact base image contract failed closed validation." >&2 + exit 1 + fi + printf 'ref=%s\n' "$(jq -r '.reference' "$CONTRACT")" >> "$GITHUB_OUTPUT" + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 + + - name: Log in to GHCR + uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Validate production build args + env: + BASE_IMAGE: ${{ steps.base.outputs.ref }} + DOCKERFILE: ${{ matrix.dockerfile }} + run: | + set -euo pipefail + build_args=( + -f "$DOCKERFILE" + --build-arg "BASE_IMAGE=${BASE_IMAGE}" + --build-arg "NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION=1" + --build-arg "NEMOCLAW_MANAGED_IMAGE_RUNTIME_USER=root" + ) + scripts/check-production-build-args.sh "${build_args[@]}" + + # Push only the canonical digest. Consumer aliases do not exist until the + # exact image has passed anonymous-pull and runtime-contract validation. + - name: Build and push managed image by digest + id: build + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 + with: + context: . + file: ${{ matrix.dockerfile }} + platforms: ${{ matrix.platform }} + outputs: type=image,name=${{ env.REGISTRY }}/${{ matrix.image }},push-by-digest=true,name-canonical=true,push=true + labels: | + org.opencontainers.image.source=https://github.com/${{ github.repository }} + org.opencontainers.image.revision=${{ github.sha }} + io.nvidia.nemoclaw.agent=${{ matrix.agent }} + io.nvidia.nemoclaw.managed-image.contract=1 + io.nvidia.nemoclaw.managed-image.platform=${{ matrix.platform }} + io.nvidia.nemoclaw.managed-image.startup-profile=1 + io.nvidia.nemoclaw.managed-image.capabilities=1 + io.nvidia.nemoclaw.managed-image.cohort=ghrun-${{ github.run_id }}-${{ github.run_attempt }} + build-args: | + BASE_IMAGE=${{ steps.base.outputs.ref }} + NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION=1 + NEMOCLAW_MANAGED_IMAGE_RUNTIME_USER=root + cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ matrix.image }}:buildcache-${{ matrix.artifact_platform }} + cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ matrix.image }}:buildcache-${{ matrix.artifact_platform }},mode=max + provenance: mode=max + sbom: true + + - name: Install managed-image publication harness dependencies + run: | + npm ci --ignore-scripts + npm run build:policy-boundary + + - name: Validate exact managed image before promotion + id: validate + shell: bash + env: + AGENT: ${{ matrix.agent }} + DIGEST: ${{ steps.build.outputs.digest }} + IMAGE: ${{ env.REGISTRY }}/${{ matrix.image }} + PLATFORM: ${{ matrix.platform }} + PUBLICATION_COHORT: ghrun-${{ github.run_id }}-${{ github.run_attempt }} + REQUIRED_BINARY: ${{ matrix.required_binary }} + run: | + set -euo pipefail + if [[ ! "$DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]]; then + echo "ERROR: managed image build did not return a valid digest: $DIGEST" >&2 + exit 1 + fi + reference="${IMAGE}@${DIGEST}" + docker buildx imagetools inspect "$reference" >/dev/null + + anonymous_config="$(mktemp -d "$RUNNER_TEMP/anonymous-docker-XXXXXX")" + chmod 0700 "$anonymous_config" + if ! DOCKER_CONFIG="$anonymous_config" docker pull --platform "$PLATFORM" "$reference"; then + echo "::error::Anonymous exact-digest pull failed for ${reference}. Before aliases can be promoted, bootstrap the GHCR package ${IMAGE} with public visibility, then rerun this workflow." + exit 1 + fi + + image_id="$(docker image inspect --format '{{.Id}}' "$reference")" + if [[ ! "$image_id" =~ ^sha256:[0-9a-f]{64}$ ]] || + [ "$(docker image inspect --format '{{.Id}}' "$image_id")" != "$image_id" ]; then + echo "ERROR: exact managed image did not resolve to one immutable local image ID." >&2 + exit 1 + fi + image_user="$(docker image inspect --format '{{.Config.User}}' "$reference")" + if [ -n "$image_user" ] && [ "$image_user" != "root" ] && [ "$image_user" != "0" ]; then + echo "ERROR: managed image OCI user must be uid 0 for OpenShell supervisor initialization: $image_user" >&2 + exit 1 + fi + + agent_label="$( + docker image inspect \ + --format '{{index .Config.Labels "io.nvidia.nemoclaw.agent"}}' \ + "$reference" + )" + contract_label="$( + docker image inspect \ + --format '{{index .Config.Labels "io.nvidia.nemoclaw.managed-image.contract"}}' \ + "$reference" + )" + platform_label="$( + docker image inspect \ + --format '{{index .Config.Labels "io.nvidia.nemoclaw.managed-image.platform"}}' \ + "$reference" + )" + startup_profile_label="$( + docker image inspect \ + --format '{{index .Config.Labels "io.nvidia.nemoclaw.managed-image.startup-profile"}}' \ + "$reference" + )" + capabilities_label="$( + docker image inspect \ + --format '{{index .Config.Labels "io.nvidia.nemoclaw.managed-image.capabilities"}}' \ + "$reference" + )" + cohort_label="$( + docker image inspect \ + --format '{{index .Config.Labels "io.nvidia.nemoclaw.managed-image.cohort"}}' \ + "$reference" + )" + revision_label="$( + docker image inspect \ + --format '{{index .Config.Labels "org.opencontainers.image.revision"}}' \ + "$reference" + )" + if [[ ! "$PUBLICATION_COHORT" =~ ^ghrun-[1-9][0-9]{0,19}-[1-9][0-9]{0,9}$ ]] || + [ "$agent_label" != "$AGENT" ] || + [ "$contract_label" != "1" ] || + [ "$platform_label" != "$PLATFORM" ] || + [ "$startup_profile_label" != "1" ] || + [ "$capabilities_label" != "1" ] || + [ "$cohort_label" != "$PUBLICATION_COHORT" ] || + [ "$revision_label" != "$GITHUB_SHA" ]; then + echo "ERROR: managed image contract labels do not match the build identity." >&2 + exit 1 + fi + + DOCKER_CONFIG="$anonymous_config" docker run --rm -i \ + --platform "$PLATFORM" \ + --network none \ + --env "AGENT=$AGENT" \ + --env "REQUIRED_BINARY=$REQUIRED_BINARY" \ + --entrypoint /bin/sh \ + "$reference" -eu -s <<'VALIDATE_FILESYSTEM' + test -x /usr/local/bin/nemoclaw-start + test -f /usr/local/lib/nemoclaw/managed-startup-image-runtime.cjs + test ! -L /usr/local/lib/nemoclaw/managed-startup-image-runtime.cjs + test "$(stat -c '%u:%g:%a' /usr/local/lib/nemoclaw/managed-startup-image-runtime.cjs)" = "0:0:444" + test -x "$REQUIRED_BINARY" + test -r /opt/nemoclaw-blueprint/blueprint.yaml + test -r /usr/local/share/nemoclaw/node-tar-inventory.json + test ! -e /usr/local/share/nemoclaw/corporate-ca.pem + test ! -L /usr/local/share/nemoclaw/corporate-ca.pem + node <<'VALIDATE_NODE_TAR' + const fs = require("node:fs"); + const scan = JSON.parse( + fs.readFileSync("/usr/local/share/nemoclaw/node-tar-inventory.json", "utf8"), + ); + if ( + scan.schema !== 1 || + !Number.isInteger(scan.packageCount) || + scan.packageCount < 1 || + !Array.isArray(scan.packages) || + scan.packages.length !== scan.packageCount || + scan.packages.some((entry) => entry.status !== "fixed") + ) { + throw new Error("completed node-tar scan is missing or unsafe"); + } + VALIDATE_NODE_TAR + + test "$NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION" = "1" + case "$AGENT" in + openclaw) + node <<'VALIDATE_OPENCLAW_UNION' + const fs = require("node:fs"); + const path = require("node:path"); + const config = JSON.parse( + fs.readFileSync("/sandbox/.openclaw/openclaw.json", "utf8"), + ); + const packages = { + "diagnostics-otel": ["@openclaw/diagnostics-otel", "2026.7.1"], + brave: ["@openclaw/brave-plugin", "2026.7.1"], + discord: ["@openclaw/discord", "2026.7.1"], + "openclaw-weixin": ["@tencent-weixin/openclaw-weixin", "2.4.3"], + slack: ["@openclaw/slack", "2026.7.1"], + whatsapp: ["@openclaw/whatsapp", "2026.7.1"], + msteams: ["@openclaw/msteams", "2026.7.1"], + }; + for (const [id, [name, version]] of Object.entries(packages)) { + const install = config.plugins?.installs?.[id]; + if ( + !install || + typeof install.installPath !== "string" || + config.plugins?.entries?.[id]?.enabled !== false + ) { + throw new Error(`managed OpenClaw plugin ${id} is missing or active`); + } + const manifest = JSON.parse( + fs.readFileSync(path.join(install.installPath, "package.json"), "utf8"), + ); + if (manifest.name !== name || manifest.version !== version) { + throw new Error(`managed OpenClaw plugin ${id} has an unexpected package identity`); + } + } + for (const id of ["telegram", "tavily"]) { + if (config.plugins?.entries?.[id]?.enabled !== false) { + throw new Error(`bundled OpenClaw plugin ${id} is not explicitly disabled`); + } + } + for (const id of [ + "telegram", + "discord", + "openclaw-weixin", + "slack", + "whatsapp", + "msteams", + ]) { + if (config.channels?.[id]?.enabled !== false) { + throw new Error(`managed OpenClaw channel ${id} is not explicitly disabled`); + } + } + if (config.tools?.web?.search?.enabled !== false) { + throw new Error("managed OpenClaw web search is not explicitly disabled"); + } + VALIDATE_OPENCLAW_UNION + ;; + hermes) + /opt/hermes/.venv/bin/python -I <<'VALIDATE_HERMES_UNION' + import importlib.metadata as metadata + from pathlib import Path + import yaml + + config = yaml.safe_load(Path("/sandbox/.hermes/config.yaml").read_text()) + optional = ("telegram", "discord", "weixin", "slack", "whatsapp", "teams") + if any(config["platforms"].get(name) != {"enabled": False} for name in optional): + raise SystemExit("managed Hermes optional platform is not explicitly disabled") + if metadata.version("microsoft-teams-apps") != "2.0.13.4": + raise SystemExit("managed Hermes microsoft-teams-apps version is wrong") + if metadata.version("aiohttp") != "3.14.1": + raise SystemExit("managed Hermes aiohttp version is wrong") + VALIDATE_HERMES_UNION + ;; + langchain-deepagents-code) ;; + *) echo "ERROR: unsupported managed agent $AGENT" >&2; exit 1 ;; + esac + VALIDATE_FILESYSTEM + + version_output="$( + DOCKER_CONFIG="$anonymous_config" docker run --rm \ + --platform "$PLATFORM" \ + --network none \ + --entrypoint "$REQUIRED_BINARY" \ + "$reference" --version 2>&1 + )" + if [ -z "$version_output" ]; then + echo "ERROR: required agent binary returned an empty version." >&2 + exit 1 + fi + + DOCKER_CONFIG="$anonymous_config" \ + npx --no-install tsx \ + scripts/checks/run-managed-image-direct-e2e.ts \ + --agent "$AGENT" \ + --image "$reference" \ + --platform "$PLATFORM" + printf 'local_id=%s\n' "$image_id" >> "$GITHUB_OUTPUT" + printf 'reference=%s\n' "$reference" >> "$GITHUB_OUTPUT" + + - name: Install pinned OpenShell runtime for publication + env: + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_OPENSHELL_MIN_VERSION: "0.0.85" + NEMOCLAW_OPENSHELL_MAX_VERSION: "0.0.85" + NEMOCLAW_OPENSHELL_PIN_VERSION: "0.0.85" + run: | + set -euo pipefail + env -u DOCKER_CONFIG \ + -u DOCKERHUB_USERNAME \ + -u DOCKERHUB_TOKEN \ + -u NVIDIA_API_KEY \ + -u NVIDIA_INFERENCE_API_KEY \ + -u GITHUB_TOKEN \ + -u GH_TOKEN \ + bash scripts/install-openshell.sh + + - name: Exercise exact candidate through real OpenShell before promotion + env: + AGENT: ${{ matrix.agent }} + IMAGE_REFERENCE: ${{ steps.validate.outputs.local_id }} + SANDBOX_NAME: nemoclaw-publish-${{ matrix.agent }} + run: | + set -euo pipefail + export PATH="$HOME/.local/bin:$PATH" + npx tsx scripts/checks/run-managed-image-openshell-e2e.ts \ + --agent "$AGENT" \ + --image "$IMAGE_REFERENCE" \ + --sandbox "$SANDBOX_NAME" + + - name: Export sanitized managed-image OpenShell failure diagnostics + id: managed_image_openshell_diagnostics + if: failure() + shell: bash + run: | + set -euo pipefail + diagnostics_root="$HOME/.nemoclaw/onboard-failures" + sanitized_root="$(mktemp -d "$RUNNER_TEMP/managed-image-openshell-diagnostics.XXXXXX")" + npx --no-install tsx \ + scripts/checks/export-managed-image-failure-diagnostics.ts \ + --source "$diagnostics_root" \ + --output "$sanitized_root" + printf 'path=%s\n' "$sanitized_root" >> "$GITHUB_OUTPUT" + while IFS= read -r summary; do + printf '\n===== %s =====\n' "${summary#"$sanitized_root"/}" + sed -n '1,160p' "$summary" + done < <(find "$sanitized_root" -type f -name summary.txt | sort) + + - name: Upload managed-image OpenShell failure diagnostics + if: ${{ failure() && steps.managed_image_openshell_diagnostics.outcome == 'success' }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: managed-image-openshell-publish-${{ matrix.agent }}-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ steps.managed_image_openshell_diagnostics.outputs.path }} + include-hidden-files: true + if-no-files-found: ignore + retention-days: 14 + + # Matrix children publish immutable candidates only. No consumer alias can + # move until the aggregate job has accepted the complete three-agent set. + - name: Export validated managed image candidate + env: + AGENT: ${{ matrix.agent }} + BASE_REFERENCE: ${{ steps.base.outputs.ref }} + DIGEST: ${{ steps.build.outputs.digest }} + IMAGE: ${{ env.REGISTRY }}/${{ matrix.image }} + PLATFORM: ${{ matrix.platform }} + PUBLICATION_COHORT: ghrun-${{ github.run_id }}-${{ github.run_attempt }} + REFERENCE: ${{ steps.validate.outputs.reference }} + run: | + set -euo pipefail + if [[ ! "$GITHUB_SHA" =~ ^[0-9a-f]{40}$ ]]; then + echo "ERROR: source revision must be a full 40-character SHA." >&2 + exit 1 + fi + if [[ ! "$PUBLICATION_COHORT" =~ ^ghrun-[1-9][0-9]{0,19}-[1-9][0-9]{0,9}$ ]]; then + echo "ERROR: publication cohort is invalid: $PUBLICATION_COHORT" >&2 + exit 1 + fi + release_tag="" + if [[ "$GITHUB_REF" == refs/tags/* ]]; then + release_tag="${GITHUB_REF#refs/tags/}" + if [[ ! "$release_tag" =~ ^v[0-9]+([.][0-9]+){1,3}([-.][0-9A-Za-z][0-9A-Za-z.-]*)?$ ]]; then + echo "ERROR: release tag is not a supported version identity: $release_tag" >&2 + exit 1 + fi + elif [ "$GITHUB_REF" != "refs/heads/main" ]; then + echo "ERROR: managed images may only be published from main or a v* release tag." >&2 + exit 1 + fi + + contract_dir="$RUNNER_TEMP/managed-image-candidate" + mkdir -p "$contract_dir" + jq -n \ + --arg agent "$AGENT" \ + --arg baseReference "$BASE_REFERENCE" \ + --arg digest "$DIGEST" \ + --arg image "$IMAGE" \ + --arg platform "$PLATFORM" \ + --arg cohort "$PUBLICATION_COHORT" \ + --arg reference "$REFERENCE" \ + --arg ref "$GITHUB_REF" \ + --arg release "$release_tag" \ + --arg repository "$GITHUB_REPOSITORY" \ + --arg revision "$GITHUB_SHA" \ + --argjson runAttempt "$GITHUB_RUN_ATTEMPT" \ + --argjson runId "$GITHUB_RUN_ID" \ + '{ + contractVersion: 1, + phase: "candidate", + agent: $agent, + image: $image, + digest: $digest, + reference: $reference, + baseReference: $baseReference, + platform: $platform, + source: { + repository: $repository, + revision: $revision, + ref: $ref, + cohort: $cohort + }, + run: { + id: $runId, + attempt: $runAttempt + }, + release: (if $release == "" then null else $release end) + }' > "$contract_dir/contract.json" + jq -e \ + --arg agent "$AGENT" \ + --arg image "$IMAGE" \ + --arg platform "$PLATFORM" \ + --arg cohort "$PUBLICATION_COHORT" \ + --arg ref "$GITHUB_REF" \ + --arg release "$release_tag" \ + --arg repository "$GITHUB_REPOSITORY" \ + --arg revision "$GITHUB_SHA" \ + --argjson runAttempt "$GITHUB_RUN_ATTEMPT" \ + --argjson runId "$GITHUB_RUN_ID" \ + '(keys | sort) == [ + "agent", + "baseReference", + "contractVersion", + "digest", + "image", + "phase", + "platform", + "reference", + "release", + "run", + "source" + ] + and .contractVersion == 1 + and .phase == "candidate" + and .agent == $agent + and .image == $image + and .platform == $platform + and (.digest | test("^sha256:[0-9a-f]{64}$")) + and .reference == (.image + "@" + .digest) + and (.baseReference | test("@sha256:[0-9a-f]{64}$")) + and .source == { + repository: $repository, + revision: $revision, + ref: $ref, + cohort: $cohort + } + and .source.revision == $revision + and .run == {id: $runId, attempt: $runAttempt} + and .release == (if $release == "" then null else $release end) + and (has("aliases") | not)' \ + "$contract_dir/contract.json" >/dev/null + + - name: Upload validated managed image candidate + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: managed-image-candidate-${{ matrix.agent }}-${{ matrix.artifact_platform }} + path: ${{ runner.temp }}/managed-image-candidate/contract.json + if-no-files-found: error + retention-days: 1 + + promote: + name: Promote complete managed image set + if: github.event_name != 'pull_request' + needs: build-and-validate + runs-on: ubuntu-24.04 + timeout-minutes: 30 + permissions: + contents: read + packages: write + steps: + - name: Download all validated managed image candidates + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: managed-image-candidate-* + path: ${{ runner.temp }}/managed-image-candidates + merge-multiple: false + + # This is the all-three publication barrier. It fails closed on a missing, + # duplicate, stale, foreign, or differently-triggered candidate before + # registry authentication or any mutable alias operation. + - name: Validate complete managed image candidate set + id: candidates + shell: bash + env: + CANDIDATE_ROOT: ${{ runner.temp }}/managed-image-candidates + run: | + set -euo pipefail + if [[ ! "$GITHUB_SHA" =~ ^[0-9a-f]{40}$ ]]; then + echo "ERROR: source revision must be a full 40-character SHA." >&2 + exit 1 + fi + expected_cohort="ghrun-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + if [[ ! "$expected_cohort" =~ ^ghrun-[1-9][0-9]{0,19}-[1-9][0-9]{0,9}$ ]]; then + echo "ERROR: publication cohort is invalid: $expected_cohort" >&2 + exit 1 + fi + expected_release="" + if [[ "$GITHUB_REF" == refs/tags/* ]]; then + expected_release="${GITHUB_REF#refs/tags/}" + if [[ ! "$expected_release" =~ ^v[0-9]+([.][0-9]+){1,3}([-.][0-9A-Za-z][0-9A-Za-z.-]*)?$ ]]; then + echo "ERROR: release tag is not a supported version identity: $expected_release" >&2 + exit 1 + fi + elif [ "$GITHUB_REF" != "refs/heads/main" ]; then + echo "ERROR: managed images may only be promoted from main or a v* release tag." >&2 + exit 1 + fi + + expected_artifacts=( + managed-image-candidate-openclaw-linux-amd64 + managed-image-candidate-hermes-linux-amd64 + managed-image-candidate-langchain-deepagents-code-linux-amd64 + ) + if [ ! -d "$CANDIDATE_ROOT" ] || [ -L "$CANDIDATE_ROOT" ]; then + echo "ERROR: managed image candidate root is missing or unsafe." >&2 + exit 1 + fi + actual_entry_count="$( + find "$CANDIDATE_ROOT" ! -path "$CANDIDATE_ROOT" -prune -print | + wc -l | tr -d '[:space:]' + )" + if [ "$actual_entry_count" != "${#expected_artifacts[@]}" ]; then + echo "ERROR: expected exactly three managed image candidate artifacts." >&2 + exit 1 + fi + + candidate_files=() + expected_agents=(openclaw hermes langchain-deepagents-code) + for index in "${!expected_artifacts[@]}"; do + artifact="${expected_artifacts[$index]}" + expected_agent="${expected_agents[$index]}" + artifact_dir="$CANDIDATE_ROOT/$artifact" + contract="$artifact_dir/contract.json" + if [ ! -d "$artifact_dir" ] || [ -L "$artifact_dir" ] || + [ ! -f "$contract" ] || [ -L "$contract" ]; then + echo "ERROR: required managed image candidate is missing or unsafe: $artifact" >&2 + exit 1 + fi + artifact_entry_count="$( + find "$artifact_dir" ! -path "$artifact_dir" -prune -print | + wc -l | tr -d '[:space:]' + )" + if [ "$artifact_entry_count" != "1" ] || + ! jq -e --arg agent "$expected_agent" '.agent == $agent' "$contract" >/dev/null; then + echo "ERROR: managed image candidate artifact identity is invalid: $artifact" >&2 + exit 1 + fi + candidate_files+=("$contract") + done + + candidate_set="$RUNNER_TEMP/managed-image-candidate-set.json" + jq -s 'sort_by(.agent)' "${candidate_files[@]}" > "$candidate_set" + if ! jq -e \ + --arg ref "$GITHUB_REF" \ + --arg release "$expected_release" \ + --arg repository "$GITHUB_REPOSITORY" \ + --arg revision "$GITHUB_SHA" \ + --arg cohort "$expected_cohort" \ + --argjson runAttempt "$GITHUB_RUN_ATTEMPT" \ + --argjson runId "$GITHUB_RUN_ID" \ + ' + length == 3 + and ([.[].agent] | sort) == [ + "hermes", + "langchain-deepagents-code", + "openclaw" + ] + and all(.[]; + (keys | sort) == [ + "agent", + "baseReference", + "contractVersion", + "digest", + "image", + "phase", + "platform", + "reference", + "release", + "run", + "source" + ] + and .contractVersion == 1 + and .phase == "candidate" + and (.digest | test("^sha256:[0-9a-f]{64}$")) + and .reference == (.image + "@" + .digest) + and (.baseReference | test("@sha256:[0-9a-f]{64}$")) + and .platform == "linux/amd64" + and .source == { + repository: $repository, + revision: $revision, + ref: $ref, + cohort: $cohort + } + and .run == {id: $runId, attempt: $runAttempt} + and .release == (if $release == "" then null else $release end) + and (has("aliases") | not) + ) + and ([.[].source.repository] | unique) == [$repository] + and ([.[].source.revision] | unique) == [$revision] + and ([.[].source.ref] | unique) == [$ref] + and ([.[].source.cohort] | unique) == [$cohort] + and ([.[].run] | unique) == [{id: $runId, attempt: $runAttempt}] + and ( + map({key: .agent, value: .image}) | from_entries + ) == { + "openclaw": "ghcr.io/nvidia/nemoclaw/openclaw-sandbox", + "hermes": "ghcr.io/nvidia/nemoclaw/hermes-sandbox", + "langchain-deepagents-code": + "ghcr.io/nvidia/nemoclaw/langchain-deepagents-code-sandbox" + } + ' "$candidate_set" >/dev/null + then + echo "ERROR: complete managed image candidate set failed closed validation." >&2 + exit 1 + fi + printf 'candidate_set=%s\n' "$candidate_set" >> "$GITHUB_OUTPUT" + + - name: Verify release tag before managed image promotion + if: startsWith(github.ref, 'refs/tags/v') + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + RELEASE_TAG: ${{ github.ref_name }} + RELEASE_REVISION: ${{ github.sha }} + with: + script: | + const releaseTag = process.env.RELEASE_TAG ?? ''; + const releaseRevision = process.env.RELEASE_REVISION ?? ''; + if (!/^v\d+\.\d+\.\d+$/.test(releaseTag)) { + throw new Error(`Refusing to verify non-semver tag: ${releaseTag}`); + } + if (!/^[0-9a-f]{40}$/.test(releaseRevision)) { + throw new Error(`Refusing release promotion with invalid revision: ${releaseRevision}`); + } + + const { owner, repo } = context.repo; + const ref = await github.rest.git.getRef({ owner, repo, ref: `tags/${releaseTag}` }); + if (ref.data.object.type !== 'tag') { + throw new Error(`Release tag ${releaseTag} must be annotated`); + } + + const tagObjectSha = ref.data.object.sha; + let tagObject; + for (let attempt = 1; attempt <= 10; attempt += 1) { + ({ data: tagObject } = await github.rest.git.getTag({ + owner, + repo, + tag_sha: tagObjectSha, + })); + if (tagObject.verification?.verified === true) break; + if (attempt < 10) { + core.info(`Waiting for GitHub tag verification (${attempt}/10)`); + await new Promise((resolve) => setTimeout(resolve, 3000)); + } + } + + if (tagObject.tag !== releaseTag) { + throw new Error(`Tag object ${tagObjectSha} names ${tagObject.tag}, expected ${releaseTag}`); + } + if (tagObject.object.type !== 'commit') { + throw new Error(`Release tag ${releaseTag} must point directly to a commit`); + } + if (tagObject.object.sha !== releaseRevision) { + throw new Error( + `Release tag ${releaseTag} points to ${tagObject.object.sha}, expected ${releaseRevision}`, + ); + } + if (tagObject.verification?.verified !== true) { + throw new Error( + `Release tag ${releaseTag} is not GitHub-Verified (${tagObject.verification?.reason ?? 'unknown'})`, + ); + } + + core.info(`Verified signed release tag ${releaseTag} (${tagObjectSha})`); + + - name: Set up Docker Buildx for promotion + uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 + + - name: Log in to GHCR for promotion + uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Promote validated managed image aliases + shell: bash + env: + CANDIDATE_SET: ${{ steps.candidates.outputs.candidate_set }} + run: | + set -euo pipefail + contract_root="$RUNNER_TEMP/managed-image-contracts" + mkdir -p "$contract_root" + cohort="ghrun-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + if [[ ! "$cohort" =~ ^ghrun-[1-9][0-9]{0,19}-[1-9][0-9]{0,9}$ ]]; then + echo "ERROR: publication cohort is invalid: $cohort" >&2 + exit 1 + fi + + # Resolve every immutable reference before the first cohort alias is + # staged. These aliases are unique to this run attempt and are not + # consumer pointers. + while IFS= read -r candidate; do + reference="$(jq -r '.reference' <<<"$candidate")" + docker buildx imagetools inspect "$reference" >/dev/null + done < <(jq -c '.[]' "$CANDIDATE_SET") + + # Stage all three unique cohort aliases before any consumer pointer + # moves. A failure in this loop can leave only non-consumer cohort + # aliases; the existing OpenClaw SHA/release pointer remains intact. + while IFS= read -r candidate; do + image="$(jq -r '.image' <<<"$candidate")" + reference="$(jq -r '.reference' <<<"$candidate")" + candidate_cohort="$(jq -r '.source.cohort' <<<"$candidate")" + if [ "$candidate_cohort" != "$cohort" ]; then + echo "ERROR: candidate publication cohort changed after the aggregate barrier." >&2 + exit 1 + fi + cohort_alias="${image}:cohort-${cohort}" + docker buildx imagetools create --tag "$cohort_alias" "$reference" + done < <(jq -c '.[]' "$CANDIDATE_SET") + + # Verify every staged cohort alias against its exact validated + # manifest before the single OpenClaw consumer pointer moves. + while IFS= read -r candidate; do + agent="$(jq -r '.agent' <<<"$candidate")" + image="$(jq -r '.image' <<<"$candidate")" + reference="$(jq -r '.reference' <<<"$candidate")" + cohort_alias="${image}:cohort-${cohort}" + exact_raw="$RUNNER_TEMP/managed-image-${agent}-exact.raw" + cohort_raw="$RUNNER_TEMP/managed-image-${agent}-cohort.raw" + docker buildx imagetools inspect "$reference" --raw > "$exact_raw" + docker buildx imagetools inspect "$cohort_alias" --raw > "$cohort_raw" + if ! cmp -s "$exact_raw" "$cohort_raw"; then + echo "ERROR: staged cohort alias does not resolve to the validated raw manifest: $cohort_alias" >&2 + exit 1 + fi + done < <(jq -c '.[]' "$CANDIDATE_SET") + + openclaw_candidate="$(jq -ce '.[] | select(.agent == "openclaw")' "$CANDIDATE_SET")" + openclaw_image="$(jq -r '.image' <<<"$openclaw_candidate")" + openclaw_reference="$(jq -r '.reference' <<<"$openclaw_candidate")" + release_tag="$(jq -r '.release // empty' <<<"$openclaw_candidate")" + consumer_aliases=("${openclaw_image}:${GITHUB_SHA}") + if [ -n "$release_tag" ]; then + consumer_aliases+=("${openclaw_image}:${release_tag}") + fi + consumer_tag_args=() + for alias in "${consumer_aliases[@]}"; do + consumer_tag_args+=(--tag "$alias") + done + docker buildx imagetools create "${consumer_tag_args[@]}" "$openclaw_reference" + + openclaw_exact_raw="$RUNNER_TEMP/managed-image-openclaw-exact.raw" + docker buildx imagetools inspect "$openclaw_reference" --raw > "$openclaw_exact_raw" + for alias in "${consumer_aliases[@]}"; do + alias_raw="$RUNNER_TEMP/managed-image-openclaw-pointer-${alias##*:}.raw" + docker buildx imagetools inspect "$alias" --raw > "$alias_raw" + if ! cmp -s "$openclaw_exact_raw" "$alias_raw"; then + echo "ERROR: OpenClaw cohort pointer does not resolve to the validated raw manifest: $alias" >&2 + exit 1 + fi + done + + while IFS= read -r candidate; do + agent="$(jq -r '.agent' <<<"$candidate")" + image="$(jq -r '.image' <<<"$candidate")" + cohort_alias="${image}:cohort-${cohort}" + aliases=("$cohort_alias") + if [ "$agent" = "openclaw" ]; then + aliases+=("${consumer_aliases[@]}") + fi + aliases_json="$(printf '%s\n' "${aliases[@]}" | jq -R . | jq -s .)" + contract_dir="$contract_root/$agent" + mkdir -p "$contract_dir" + jq \ + --argjson aliases "$aliases_json" \ + '{ + contractVersion, + agent, + image, + digest, + reference, + baseReference, + platform, + source: { + repository: .source.repository, + revision: .source.revision, + release, + cohort: .source.cohort + }, + run, + aliases: $aliases + }' <<<"$candidate" > "$contract_dir/contract.json" + jq -e \ + --arg revision "$GITHUB_SHA" \ + --arg cohort "$cohort" \ + --argjson aliases "$aliases_json" \ + '.contractVersion == 1 + and (.digest | test("^sha256:[0-9a-f]{64}$")) + and .reference == (.image + "@" + .digest) + and (.baseReference | test("@sha256:[0-9a-f]{64}$")) + and .source.revision == $revision + and .source.cohort == $cohort + and .aliases == $aliases + and ( + (.image + ":cohort-" + $cohort) as $cohortAlias + | (.aliases | index($cohortAlias)) != null + )' \ + "$contract_dir/contract.json" >/dev/null + done < <(jq -c '.[]' "$CANDIDATE_SET") + + - name: Upload OpenClaw managed image contract + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: managed-image-openclaw-linux-amd64 + path: ${{ runner.temp }}/managed-image-contracts/openclaw/contract.json + if-no-files-found: error + retention-days: 90 + + - name: Upload Hermes managed image contract + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: managed-image-hermes-linux-amd64 + path: ${{ runner.temp }}/managed-image-contracts/hermes/contract.json + if-no-files-found: error + retention-days: 90 + + - name: Upload Deep Agents Code managed image contract + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: managed-image-langchain-deepagents-code-linux-amd64 + path: ${{ runner.temp }}/managed-image-contracts/langchain-deepagents-code/contract.json + if-no-files-found: error + retention-days: 90 diff --git a/Dockerfile b/Dockerfile index 2a52642546..24ebb70390 100644 --- a/Dockerfile +++ b/Dockerfile @@ -53,6 +53,21 @@ COPY tools/mcp-tool-discovery-runtime/package.json tools/mcp-tool-discovery-runt RUN ./install-reviewed-runtime.sh \ && rm -f ./install-reviewed-runtime.sh +# Bundle the driver-neutral startup-profile applicator into one CommonJS file. +# The final image needs no TypeScript loader or repository dependency tree. +FROM mcp-tool-discovery-runtime AS managed-startup-runtime-builder +WORKDIR /opt/nemoclaw-managed-startup-build +COPY src/lib/core/json-types.ts src/lib/core/ports.ts ./src/lib/core/ +COPY src/lib/messaging/ ./src/lib/messaging/ +COPY src/lib/onboard/managed-startup/ ./src/lib/onboard/managed-startup/ +COPY src/lib/security/credential-hash.ts ./src/lib/security/ +COPY src/lib/state/paths.ts src/lib/state/state-root.ts ./src/lib/state/ +RUN /opt/mcp-tool-discovery-runtime/node_modules/.bin/esbuild \ + src/lib/onboard/managed-startup/image-runtime.ts \ + --bundle --platform=node --format=cjs --target=node22 \ + --outfile=/out/managed-startup-image-runtime.cjs \ + && chmod 0444 /out/managed-startup-image-runtime.cjs + # Group repository-owned files outside the final image so both Docker builders # can collapse related payloads without invalidating earlier final-image work. FROM scratch AS openclaw-dependency-payload @@ -90,6 +105,7 @@ COPY scripts/verify-wechat-runtime-lock.mts /usr/local/lib/nemoclaw/verify-wecha FROM scratch AS openclaw-runtime-payload COPY scripts/lib/sandbox-init.sh /usr/local/lib/nemoclaw/sandbox-init.sh +COPY scripts/lib/entrypoint-env-wrapper.sh /usr/local/lib/nemoclaw/entrypoint-env-wrapper.sh COPY scripts/lib/gateway-supervisor.sh /usr/local/lib/nemoclaw/gateway-supervisor.sh COPY scripts/lib/sandbox-rlimits.sh /usr/local/lib/nemoclaw/sandbox-rlimits.sh COPY scripts/lib/openclaw_device_approval_policy.py /usr/local/lib/nemoclaw/openclaw_device_approval_policy.py @@ -99,12 +115,14 @@ COPY scripts/state-dir-guard.py /usr/local/lib/nemoclaw/state-dir-guard.py COPY scripts/openclaw-config-guard.py /usr/local/lib/nemoclaw/openclaw-config-guard.py COPY scripts/managed-gateway-control.py /usr/local/lib/nemoclaw/managed-gateway-control.py COPY scripts/nemoclaw-start.sh /usr/local/bin/nemoclaw-start +COPY scripts/managed-startup-hold.sh /usr/local/bin/nemoclaw-managed-startup-hold COPY scripts/gateway-control.sh /usr/local/bin/nemoclaw-gateway-control COPY nemoclaw-blueprint/scripts/*.js /usr/local/lib/nemoclaw/preloads/ COPY --from=runtime-preload-builder /opt/nemoclaw-root/dist/lib/messaging/channels/ /usr/local/lib/nemoclaw/preloads-compiled-channels/ COPY scripts/codex-acp-wrapper.sh /usr/local/bin/nemoclaw-codex-acp COPY scripts/generate-openclaw-config.mts /scripts/generate-openclaw-config.mts COPY scripts/validate-openclaw-tool-search.mts /scripts/validate-openclaw-tool-search.mts +COPY --from=managed-startup-runtime-builder /out/managed-startup-image-runtime.cjs /usr/local/lib/nemoclaw/managed-startup-image-runtime.cjs COPY src/lib/tool-disclosure.ts /src/lib/tool-disclosure.ts COPY src/lib/messaging/ /src/lib/messaging/ COPY nemoclaw-blueprint/openclaw-plugins/ /usr/local/share/nemoclaw/openclaw-plugins/ @@ -966,13 +984,18 @@ RUN discovery_contract="$(node /usr/local/lib/nemoclaw/mcp-tool-discovery-runtim && discovery_unsafe="$(find -L /usr/local/lib/nemoclaw/mcp-tool-discovery-runtime \( ! -user root -o -perm /022 \) -print -quit)" \ && test -z "$discovery_unsafe" -# Copy startup script and shared sandbox initialisation library -RUN chmod 755 /usr/local/bin/nemoclaw-start /usr/local/bin/nemoclaw-codex-acp \ +# Copy startup script and shared sandbox initialisation library. Precreate the +# root-owned managed-startup handoff so Landlock can bind its read-only rule +# before the host applies the profile. +RUN install -d -o root -g root -m 0755 /run/nemoclaw \ + && chmod 755 /usr/local/bin/nemoclaw-start /usr/local/bin/nemoclaw-codex-acp \ + /usr/local/bin/nemoclaw-managed-startup-hold \ /usr/local/lib/nemoclaw/sandbox-init.sh \ /scripts/generate-openclaw-config.mts \ /scripts/validate-openclaw-tool-search.mts \ /src/lib/messaging/applier/build/messaging-build-applier.mts \ && chmod 444 /src/lib/tool-disclosure.ts \ + /usr/local/lib/nemoclaw/entrypoint-env-wrapper.sh \ && chmod -R a+rX /src/lib/messaging \ && chown root:root /usr/local/bin/nemoclaw-gateway-control \ /usr/local/lib/nemoclaw/gateway-supervisor.sh \ @@ -984,6 +1007,7 @@ RUN chmod 755 /usr/local/bin/nemoclaw-start /usr/local/bin/nemoclaw-codex-acp \ /usr/local/lib/nemoclaw/openclaw-config-guard.py \ /usr/local/lib/nemoclaw/managed-gateway-control.py \ && chmod 444 /usr/local/lib/nemoclaw/gateway-supervisor.sh \ + /usr/local/lib/nemoclaw/entrypoint-env-wrapper.sh \ /usr/local/lib/nemoclaw/sandbox-rlimits.sh \ && chmod 644 /usr/local/lib/nemoclaw/openclaw_device_approval_policy.py \ /usr/local/lib/nemoclaw/clean_runtime_shell_env_shim.py \ @@ -1044,6 +1068,13 @@ ARG NEMOCLAW_INFERENCE_COMPAT_B64=e30= # rendering. The plan contains placeholders only; secrets are resolved at # runtime via OpenShell providers. ARG NEMOCLAW_MESSAGING_PLAN_B64= +# Release-image mode preinstalls the complete reviewed optional dependency +# union. It is inert by default and must never be enabled for a deployment- +# specific Dockerfile build carrying an active messaging plan. +ARG NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION=0 +# OpenClaw already uses a root supervisor; the explicit value keeps the managed +# image entry-user contract uniform with Hermes and DCode publication. +ARG NEMOCLAW_MANAGED_IMAGE_RUNTIME_USER=root # Base64-encoded JSON array of secondary OpenClaw agent config entries # (e.g. [{"id":"research","workspace":"/sandbox/.openclaw/workspace-research", # "agentDir":"/sandbox/.openclaw/agents/research", ...}]). @@ -1109,6 +1140,7 @@ ENV NEMOCLAW_MODEL=${NEMOCLAW_MODEL} \ NEMOCLAW_AGENT_HEARTBEAT_EVERY=${NEMOCLAW_AGENT_HEARTBEAT_EVERY} \ NEMOCLAW_INFERENCE_COMPAT_B64=${NEMOCLAW_INFERENCE_COMPAT_B64} \ NEMOCLAW_EXTRA_AGENTS_JSON_B64=${NEMOCLAW_EXTRA_AGENTS_JSON_B64} \ + NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION=${NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION} \ NEMOCLAW_OPENCLAW_WECHAT_PLUGIN_PREINSTALLED=1 \ NEMOCLAW_DASHBOARD_BIND=${NEMOCLAW_DASHBOARD_BIND} \ NEMOCLAW_WSL_DASHBOARD_EXPOSURE=${NEMOCLAW_WSL_DASHBOARD_EXPOSURE} \ @@ -1123,6 +1155,20 @@ ENV NEMOCLAW_MODEL=${NEMOCLAW_MODEL} \ NEMOCLAW_OPENCLAW_OTEL_SERVICE_NAME=${NEMOCLAW_OPENCLAW_OTEL_SERVICE_NAME} \ NEMOCLAW_OPENCLAW_OTEL_SAMPLE_RATE=${NEMOCLAW_OPENCLAW_OTEL_SAMPLE_RATE} +RUN case "$NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION" in \ + 0) ;; \ + 1) \ + test -z "$NEMOCLAW_MESSAGING_PLAN_B64" \ + || { echo "ERROR: managed-image capability union requires an empty messaging plan" >&2; exit 1; }; \ + test "$NEMOCLAW_WEB_SEARCH_ENABLED" = "0" \ + || { echo "ERROR: managed-image capability union requires web search disabled in the neutral image" >&2; exit 1; }; \ + test "$NEMOCLAW_OPENCLAW_OTEL" = "0" \ + || { echo "ERROR: managed-image capability union requires OTEL disabled in the neutral image" >&2; exit 1; } \ + ;; \ + *) echo "ERROR: NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION must be 0 or 1" >&2; exit 1 ;; \ + esac \ + && test "$NEMOCLAW_MANAGED_IMAGE_RUNTIME_USER" = "root" + # Bake reduced messaging runtime metadata for the entrypoint. The full # NEMOCLAW_MESSAGING_PLAN_B64 is a build input; OpenShell sandbox create only # forwards explicit runtime env, so nemoclaw-start reads this generic artifact @@ -1152,7 +1198,9 @@ USER sandbox # for child npm processes. During image build the OpenShell gateway is not # available at the runtime sandbox proxy address yet, so defer the final proxy # block until after build-time OpenClaw doctor/plugin commands complete. -RUN NEMOCLAW_OPENCLAW_MANAGED_PROXY=0 node --experimental-strip-types /scripts/generate-openclaw-config.mts +RUN NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION=0 \ + NEMOCLAW_OPENCLAW_MANAGED_PROXY=0 \ + node --experimental-strip-types /scripts/generate-openclaw-config.mts # Validate the patched OpenClaw tool-search contract against real generated # configs for both supported disclosure modes. This runs at image build time so @@ -1168,6 +1216,7 @@ RUN set -eu; \ NEMOCLAW_MODEL=test-model \ NEMOCLAW_PRIMARY_MODEL_REF=inference/test-model \ NEMOCLAW_TOOL_DISCLOSURE="$mode" \ + NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION=0 \ NEMOCLAW_OPENCLAW_MANAGED_PROXY=0 \ node --experimental-strip-types /scripts/generate-openclaw-config.mts; \ node --experimental-strip-types /scripts/validate-openclaw-tool-search.mts \ @@ -1188,6 +1237,7 @@ RUN set -eu; \ # openKeyedStore on OpenClaw >= 2026.6.10. # hadolint ignore=DL3059,DL4006 RUN set -eu; \ + managed_image_union="${NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION:-0}"; \ verify_openclaw_plugin_integrity() { \ plugin_spec="$1"; \ expected_integrity=""; \ @@ -1220,13 +1270,16 @@ RUN set -eu; \ openclaw plugins install "npm-pack:${plugin_install_archive}"; \ rm -rf "$plugin_root"; \ }; \ - if [ "$NEMOCLAW_OPENCLAW_OTEL" = "1" ] || [ "$NEMOCLAW_WEB_SEARCH_ENABLED" = "1" ]; then \ + if [ "$managed_image_union" = "1" ] || [ "$NEMOCLAW_OPENCLAW_OTEL" = "1" ] || [ "$NEMOCLAW_WEB_SEARCH_ENABLED" = "1" ]; then \ test -n "$OPENCLAW_VERSION"; \ fi; \ - if [ "$NEMOCLAW_OPENCLAW_OTEL" = "1" ]; then \ + if [ "$managed_image_union" = "1" ]; then \ + install_reviewed_openclaw_plugin "@openclaw/diagnostics-otel"; \ + install_reviewed_openclaw_plugin "@openclaw/brave-plugin"; \ + elif [ "$NEMOCLAW_OPENCLAW_OTEL" = "1" ]; then \ install_reviewed_openclaw_plugin "@openclaw/diagnostics-otel"; \ fi; \ - if [ "$NEMOCLAW_WEB_SEARCH_ENABLED" = "1" ]; then \ + if [ "$managed_image_union" != "1" ] && [ "$NEMOCLAW_WEB_SEARCH_ENABLED" = "1" ]; then \ case "${NEMOCLAW_WEB_SEARCH_PROVIDER:-brave}" in \ brave) \ install_reviewed_openclaw_plugin "@openclaw/brave-plugin"; \ @@ -1262,6 +1315,12 @@ RUN set -eu; \ NEMOCLAW_WECHAT_NPM_INSTALL_CACHE="$install_cache" \ OPENCLAW_VERSION="${OPENCLAW_VERSION}" \ node --experimental-strip-types /src/lib/messaging/applier/build/messaging-build-applier.mts --agent openclaw --phase agent-install; \ + if [ "$NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION" = "1" ]; then \ + NEMOCLAW_WECHAT_NPM_INSTALL_CACHE="$install_cache" \ + OPENCLAW_VERSION="${OPENCLAW_VERSION}" \ + node --experimental-strip-types /src/lib/messaging/applier/build/messaging-build-applier.mts \ + --agent openclaw --phase managed-image-capability-union; \ + fi; \ rm -rf "$install_cache"; \ trap - EXIT; \ test ! -e "$install_cache"; \ @@ -1305,6 +1364,17 @@ RUN NPM_CONFIG_IGNORE_SCRIPTS=true npm_config_ignore_scripts=true \ # hadolint ignore=DL3059,DL4006 RUN OPENCLAW_VERSION="${OPENCLAW_VERSION}" node --experimental-strip-types /src/lib/messaging/applier/build/messaging-build-applier.mts --agent openclaw --phase post-agent-install +# A managed image is a neutral capability carrier, not an all-channels-enabled +# deployment. Regenerate after every optional plugin is installed so OpenClaw's +# install registry survives while every optional plugin/channel remains inert. +# Validate the exact generated file through the pinned OpenClaw CLI. +# hadolint ignore=DL3059,DL4006 +RUN if [ "$NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION" = "1" ]; then \ + node --experimental-strip-types /scripts/generate-openclaw-config.mts; \ + validation="$(openclaw config validate --json)"; \ + node -e 'const result=JSON.parse(process.argv[1]); if (result.valid !== true) process.exit(1)' "$validation"; \ + fi + # Release the offline lock so the runtime sandbox can install MCP servers, # skills, and ad-hoc packages via the OpenShell L7 proxy. ENV NPM_CONFIG_OFFLINE=false diff --git a/agents/hermes/Dockerfile b/agents/hermes/Dockerfile index 843545fff2..aa326ba294 100644 --- a/agents/hermes/Dockerfile +++ b/agents/hermes/Dockerfile @@ -21,6 +21,19 @@ RUN ./install-reviewed-runtime.sh \ RUN chown -R root:root /opt/mcp-tool-discovery-runtime \ && chmod -R a=rX /opt/mcp-tool-discovery-runtime +FROM mcp-tool-discovery-runtime AS managed-startup-runtime-builder +WORKDIR /opt/nemoclaw-managed-startup-build +COPY src/lib/core/json-types.ts src/lib/core/ports.ts ./src/lib/core/ +COPY src/lib/messaging/ ./src/lib/messaging/ +COPY src/lib/onboard/managed-startup/ ./src/lib/onboard/managed-startup/ +COPY src/lib/security/credential-hash.ts ./src/lib/security/ +COPY src/lib/state/paths.ts src/lib/state/state-root.ts ./src/lib/state/ +RUN /opt/mcp-tool-discovery-runtime/node_modules/.bin/esbuild \ + src/lib/onboard/managed-startup/image-runtime.ts \ + --bundle --platform=node --format=cjs --target=node22 \ + --outfile=/out/managed-startup-image-runtime.cjs \ + && chmod 0444 /out/managed-startup-image-runtime.cjs + FROM scratch AS hermes-npm-patch-payload COPY scripts/lib/reviewed-npm-archive.mts /scripts/lib/reviewed-npm-archive.mts @@ -43,9 +56,12 @@ FROM scratch AS hermes-runtime-payload COPY --from=mcp-tool-discovery-runtime /opt/mcp-tool-discovery-runtime/dist/ /usr/local/lib/nemoclaw/mcp-tool-discovery-runtime/ COPY nemoclaw-blueprint/ /opt/nemoclaw-blueprint/ COPY scripts/lib/sandbox-init.sh /usr/local/lib/nemoclaw/sandbox-init.sh +COPY scripts/lib/entrypoint-env-wrapper.sh /usr/local/lib/nemoclaw/entrypoint-env-wrapper.sh COPY scripts/lib/gateway-supervisor.sh /usr/local/lib/nemoclaw/gateway-supervisor.sh COPY scripts/lib/sandbox-rlimits.sh /usr/local/lib/nemoclaw/sandbox-rlimits.sh COPY agents/hermes/start.sh /usr/local/bin/nemoclaw-start +COPY scripts/managed-startup-hold.sh /usr/local/bin/nemoclaw-managed-startup-hold +COPY --from=managed-startup-runtime-builder /out/managed-startup-image-runtime.cjs /usr/local/lib/nemoclaw/managed-startup-image-runtime.cjs COPY scripts/gateway-control.sh /usr/local/bin/nemoclaw-gateway-control COPY scripts/managed-gateway-control.py /usr/local/lib/nemoclaw/managed-gateway-control.py COPY agents/hermes/validate-env-secret-boundary.py /usr/local/lib/nemoclaw/validate-hermes-env-secret-boundary.py @@ -211,11 +227,12 @@ RUN chmod -R a+rX /opt/nemoclaw-blueprint/ # profile hook, bashrc hook, or root-owned helper mode. Remove it once the # minimum supported Hermes sandbox base tag guarantees those artifacts and # test/sandbox-rlimit-hooks.test.ts covers that base. -RUN chmod 755 /usr/local/bin/nemoclaw-start /usr/local/lib/nemoclaw/sandbox-init.sh /usr/local/lib/nemoclaw/validate-hermes-env-secret-boundary.py /usr/local/lib/nemoclaw/patch-hermes-session-list-preview.py /usr/local/lib/nemoclaw/patch-hermes-discord-recovery-permissions.py /usr/local/lib/nemoclaw/patch-hermes-profile-policy-defaults.py /usr/local/lib/nemoclaw/seed-hermes-dashboard-config.py /usr/local/lib/nemoclaw/hermes-runtime-config-guard.py /usr/local/lib/nemoclaw/finalize-tirith-marker.py /usr/local/lib/nemoclaw/hermes-mcp-config-transaction.py \ +RUN install -d -o root -g root -m 0755 /run/nemoclaw \ + && chmod 755 /usr/local/bin/nemoclaw-start /usr/local/bin/nemoclaw-managed-startup-hold /usr/local/lib/nemoclaw/sandbox-init.sh /usr/local/lib/nemoclaw/validate-hermes-env-secret-boundary.py /usr/local/lib/nemoclaw/patch-hermes-session-list-preview.py /usr/local/lib/nemoclaw/patch-hermes-discord-recovery-permissions.py /usr/local/lib/nemoclaw/patch-hermes-profile-policy-defaults.py /usr/local/lib/nemoclaw/seed-hermes-dashboard-config.py /usr/local/lib/nemoclaw/hermes-runtime-config-guard.py /usr/local/lib/nemoclaw/finalize-tirith-marker.py /usr/local/lib/nemoclaw/hermes-mcp-config-transaction.py \ && chown root:root /usr/local/bin/nemoclaw-gateway-control /usr/local/lib/nemoclaw/gateway-supervisor.sh /usr/local/lib/nemoclaw/state-dir-guard.py /usr/local/lib/nemoclaw/managed-gateway-control.py /usr/local/lib/nemoclaw/build-hermes-mcp-digest.py /usr/local/lib/nemoclaw/openshell-child-visible-credentials.v0.0.85.json \ && chmod 700 /usr/local/bin/nemoclaw-gateway-control \ && chmod 500 /usr/local/lib/nemoclaw/state-dir-guard.py /usr/local/lib/nemoclaw/managed-gateway-control.py \ - && chmod 444 /usr/local/lib/nemoclaw/gateway-supervisor.sh /usr/local/lib/nemoclaw/build-hermes-mcp-digest.py \ + && chmod 444 /usr/local/lib/nemoclaw/entrypoint-env-wrapper.sh /usr/local/lib/nemoclaw/gateway-supervisor.sh /usr/local/lib/nemoclaw/build-hermes-mcp-digest.py \ && chmod 444 /usr/local/lib/nemoclaw/patch-hermes-langfuse-credentials.mts \ && chmod 444 /usr/local/lib/nemoclaw/openshell-child-visible-credentials.v0.0.85.json \ && if [ -d /usr/local/lib/nemoclaw/preloads ]; then \ @@ -608,6 +625,10 @@ ARG NEMOCLAW_TOOL_DISCLOSURE=progressive # API remains exposed separately on port 8642. ARG CHAT_UI_URL=http://127.0.0.1:18789 ARG NEMOCLAW_MESSAGING_PLAN_B64= +# Release-image mode preinstalls every reviewed optional dependency while +# keeping the generated Hermes platform configuration disabled by default. +ARG NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION=0 +ARG NEMOCLAW_MANAGED_IMAGE_RUNTIME_USER=root ARG NEMOCLAW_WEB_SEARCH_ENABLED=0 ARG NEMOCLAW_WEB_SEARCH_PROVIDER=tavily ARG NEMOCLAW_HERMES_TOOL_GATEWAY_BROKER=0 @@ -638,11 +659,26 @@ ENV NEMOCLAW_MODEL=${NEMOCLAW_MODEL} \ NEMOCLAW_CONTEXT_WINDOW=${NEMOCLAW_CONTEXT_WINDOW} \ NEMOCLAW_TOOL_DISCLOSURE=${NEMOCLAW_TOOL_DISCLOSURE} \ CHAT_UI_URL=${CHAT_UI_URL} \ + NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION=${NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION} \ NEMOCLAW_WEB_SEARCH_ENABLED=${NEMOCLAW_WEB_SEARCH_ENABLED} \ NEMOCLAW_WEB_SEARCH_PROVIDER=${NEMOCLAW_WEB_SEARCH_PROVIDER} \ NEMOCLAW_HERMES_TOOL_GATEWAY_BROKER=${NEMOCLAW_HERMES_TOOL_GATEWAY_BROKER} \ NEMOCLAW_HERMES_TOOL_GATEWAY_PRESETS_B64=${NEMOCLAW_HERMES_TOOL_GATEWAY_PRESETS_B64} +RUN case "$NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION" in \ + 0) ;; \ + 1) \ + test -z "$NEMOCLAW_MESSAGING_PLAN_B64" \ + || { echo "ERROR: managed-image capability union requires an empty messaging plan" >&2; exit 1; }; \ + test "$NEMOCLAW_WEB_SEARCH_ENABLED" = "0" \ + || { echo "ERROR: managed-image capability union requires web search disabled in the neutral image" >&2; exit 1; }; \ + test "$NEMOCLAW_HERMES_TOOL_GATEWAY_BROKER" = "0" \ + || { echo "ERROR: managed-image capability union requires tool gateways disabled in the neutral image" >&2; exit 1; } \ + ;; \ + *) echo "ERROR: NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION must be 0 or 1" >&2; exit 1 ;; \ + esac \ + && test "$NEMOCLAW_MANAGED_IMAGE_RUNTIME_USER" = "root" + # Bake reduced messaging runtime metadata for the entrypoint. Hermes runtime # config refresh consumes generic manifest-owned aliases from this artifact. # hadolint ignore=DL3059 @@ -655,7 +691,11 @@ RUN node --experimental-strip-types /src/lib/messaging/applier/build/messaging-b # /opt/hermes/.venv before the runtime drops to the sandbox user. WORKDIR /opt/hermes # hadolint ignore=DL3059 -RUN node --experimental-strip-types /src/lib/messaging/applier/build/messaging-build-applier.mts --agent hermes --phase agent-install +RUN node --experimental-strip-types /src/lib/messaging/applier/build/messaging-build-applier.mts --agent hermes --phase agent-install \ + && if [ "$NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION" = "1" ]; then \ + node --experimental-strip-types /src/lib/messaging/applier/build/messaging-build-applier.mts \ + --agent hermes --phase managed-image-capability-union; \ + fi # Decode the host corporate-proxy CA (#6210) to a root-owned, read-only file # when onboard baked one in. No-op when NEMOCLAW_CORPORATE_CA_B64 is empty. The @@ -695,6 +735,9 @@ RUN mkdir -p /sandbox/.nemoclaw/blueprints/0.1.0 \ # code injection via build-arg interpolation (same concern as OpenClaw C-2). RUN HERMES_HOME=/sandbox/.hermes /usr/local/bin/hermes doctor --fix \ && node --experimental-strip-types /opt/nemoclaw-hermes-config/generate-config.ts \ + && if [ "$NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION" = "1" ]; then \ + /opt/hermes/.venv/bin/python -I -c 'import importlib.metadata as m, pathlib, yaml; config=yaml.safe_load(pathlib.Path("/sandbox/.hermes/config.yaml").read_text()); expected=("telegram","discord","weixin","slack","whatsapp","teams"); assert all(config["platforms"].get(name) == {"enabled": False} for name in expected); assert m.version("microsoft-teams-apps") == "2.0.13.4"; assert m.version("aiohttp") == "3.14.1"'; \ + fi \ && rm -rf /sandbox/.cache # Prove that the installed dashboard seeder carries every reviewed policy value @@ -1117,6 +1160,18 @@ RUN cp /etc/nemoclaw/hermes.config-hash /sandbox/.hermes/.config-hash \ && chmod 640 /sandbox/.hermes/.config-hash \ && chown sandbox:sandbox /sandbox/.hermes/.config-hash +# Keep the shared NemoClaw state root consistent across every shipped agent. +# The root-owned sticky directory protects the managed-startup transaction +# receipt while leaving the named plugin state directories sandbox-writable. +RUN chown root:root /sandbox/.nemoclaw \ + && chmod 1755 /sandbox/.nemoclaw \ + && chown -R root:root /sandbox/.nemoclaw/blueprints \ + && chmod -R 755 /sandbox/.nemoclaw/blueprints \ + && mkdir -p /sandbox/.nemoclaw/state /sandbox/.nemoclaw/migration /sandbox/.nemoclaw/snapshots /sandbox/.nemoclaw/staging \ + && chown sandbox:sandbox /sandbox/.nemoclaw/state /sandbox/.nemoclaw/migration /sandbox/.nemoclaw/snapshots /sandbox/.nemoclaw/staging \ + && printf '%s' '{}' > /sandbox/.nemoclaw/config.json \ + && chown sandbox:sandbox /sandbox/.nemoclaw/config.json + # OpenShell's macOS VM backend currently remaps extracted rootfs ownership to # the host uid/gid before startup. Hermes uses /sandbox as a read_write path, so # that repair walks the trusted rc files too; keep this Darwin-only so Linux @@ -1124,6 +1179,10 @@ RUN cp /etc/nemoclaw/hermes.config-hash /sandbox/.hermes/.config-hash \ RUN if [ "$NEMOCLAW_DARWIN_VM_COMPAT" = "1" ]; then \ chmod -R a+rwX /sandbox/.hermes; \ find /sandbox/.hermes -type d -exec chmod a+rwx {} +; \ + for p in /sandbox/.nemoclaw/state /sandbox/.nemoclaw/migration /sandbox/.nemoclaw/snapshots /sandbox/.nemoclaw/staging; do \ + chmod a+rwx "$p"; \ + done; \ + chmod a+rw /sandbox/.nemoclaw/config.json; \ chmod a+rw /sandbox/.bashrc /sandbox/.profile; \ fi @@ -1154,6 +1213,7 @@ RUN check_metadata() { \ && check_absent /root/.cache/electron \ && check_absent /root/.cache/node-gyp \ && check_absent /sandbox/.cache \ + && check_metadata /sandbox/.nemoclaw 'root:root 1755' \ && check_metadata /scripts/patch-bundled-npm-brace-expansion.mts 'root:root 444' \ && check_metadata /scripts/patch-bundled-npm-tar.mts 'root:root 444' \ && check_metadata /opt/nemoclaw-hermes-config/generate-config.ts 'root:root 444' \ diff --git a/agents/hermes/config/build-env.ts b/agents/hermes/config/build-env.ts index 139cbcba09..2f2b4e4f65 100644 --- a/agents/hermes/config/build-env.ts +++ b/agents/hermes/config/build-env.ts @@ -22,6 +22,7 @@ export type HermesBuildSettings = { contextWindow: number | null; toolDisclosure: "progressive" | "direct"; webSearchProvider: HermesWebSearchProvider | null; + managedImageCapabilityUnion: boolean; messagingCredentialPlaceholders: Array<{ envKey: string; placeholder: string; @@ -50,6 +51,10 @@ export function readHermesBuildSettings(env: NodeJS.ProcessEnv): HermesBuildSett contextWindow: readContextWindow(env), toolDisclosure: readToolDisclosureEnv(env), webSearchProvider: readWebSearchProvider(env), + managedImageCapabilityUnion: readBooleanBuildFlag( + env, + "NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION", + ), messagingCredentialPlaceholders: readMessagingCredentialPlaceholders(env), managedToolGateways: { brokerEnabled: env.NEMOCLAW_HERMES_TOOL_GATEWAY_BROKER === "1", @@ -58,6 +63,14 @@ export function readHermesBuildSettings(env: NodeJS.ProcessEnv): HermesBuildSett }; } +function readBooleanBuildFlag(env: NodeJS.ProcessEnv, name: string): boolean { + const value = env[name] ?? "0"; + if (value !== "0" && value !== "1") { + throw new Error(`${name} must be "0" or "1"`); + } + return value === "1"; +} + /** * Parse `NEMOCLAW_CONTEXT_WINDOW` for Hermes config generation. * diff --git a/agents/hermes/config/hermes-config.ts b/agents/hermes/config/hermes-config.ts index 0f84b539fb..9ba6a91bc4 100644 --- a/agents/hermes/config/hermes-config.ts +++ b/agents/hermes/config/hermes-config.ts @@ -27,6 +27,15 @@ const REMOTE_PLATFORM_TOOLSETS = [ "audio", ]; +export const MANAGED_IMAGE_HERMES_OPTIONAL_PLATFORMS = [ + "telegram", + "discord", + "weixin", + "slack", + "whatsapp", + "teams", +] as const; + function hermesApiMode(inferenceApi: string): string | null { switch (inferenceApi) { case "": @@ -103,6 +112,21 @@ export function buildHermesConfig( model: settings.model, }; + const platforms: Record = { + api_server: { + enabled: true, + extra: { + port: 18642, + host: "127.0.0.1", + }, + }, + }; + if (settings.managedImageCapabilityUnion) { + for (const platform of MANAGED_IMAGE_HERMES_OPTIONAL_PLATFORMS) { + platforms[platform] = { enabled: false }; + } + } + const config: Record = { _config_version: 33, _nemoclaw_upstream: upstream, @@ -211,15 +235,7 @@ export function buildHermesConfig( platform_toolsets: { api_server: remotePlatformToolsets, }, - platforms: { - api_server: { - enabled: true, - extra: { - port: 18642, - host: "127.0.0.1", - }, - }, - }, + platforms, }; const managedToolGatewayPresets = effectiveManagedToolGatewayPresets(settings); diff --git a/agents/hermes/policy-additions.yaml b/agents/hermes/policy-additions.yaml index 704b1a0757..95659ff1fc 100644 --- a/agents/hermes/policy-additions.yaml +++ b/agents/hermes/policy-additions.yaml @@ -23,6 +23,9 @@ filesystem_policy: - /app - /etc - /var/log + - /run/nemoclaw # Root-owned managed-startup handoff. Landlock + # grants reads only; file DAC still protects + # private runtime material in this directory. read_write: - /sandbox - /tmp diff --git a/agents/hermes/policy-permissive.yaml b/agents/hermes/policy-permissive.yaml index 3d0c51ac35..0d88abbe57 100644 --- a/agents/hermes/policy-permissive.yaml +++ b/agents/hermes/policy-permissive.yaml @@ -24,6 +24,8 @@ filesystem_policy: - /app - /etc - /var/log + - /run/nemoclaw # Mirrors the managed-startup handoff grant + # in the baseline create policy. read_write: - /sandbox - /tmp diff --git a/agents/hermes/start.sh b/agents/hermes/start.sh index 6231c200e1..e4bfacfaa0 100755 --- a/agents/hermes/start.sh +++ b/agents/hermes/start.sh @@ -20,6 +20,30 @@ set -euo pipefail # SECURITY: Lock down PATH before resolving or sourcing root startup helpers. export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" +# managed-entrypoint-env-wrapper begin +_NEMOCLAW_ENTRYPOINT_ENV_WRAPPER="/usr/local/lib/nemoclaw/entrypoint-env-wrapper.sh" +if [ ! -f "$_NEMOCLAW_ENTRYPOINT_ENV_WRAPPER" ]; then + _HERMES_ENTRYPOINT_SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + _NEMOCLAW_ENTRYPOINT_ENV_WRAPPER="${_HERMES_ENTRYPOINT_SOURCE_DIR}/../../scripts/lib/entrypoint-env-wrapper.sh" + unset _HERMES_ENTRYPOINT_SOURCE_DIR +fi +if [ ! -f "$_NEMOCLAW_ENTRYPOINT_ENV_WRAPPER" ]; then + printf '%s\n' '[SECURITY] Required entrypoint env-wrapper normalizer is missing.' >&2 + exit 1 +fi +# shellcheck source=scripts/lib/entrypoint-env-wrapper.sh +source "$_NEMOCLAW_ENTRYPOINT_ENV_WRAPPER" +nemoclaw_normalize_entrypoint_env_wrapper "$@" +if [ "$NEMOCLAW_ENTRYPOINT_NORMALIZED_ARGC" -eq 0 ]; then + set -- +else + set -- "${NEMOCLAW_ENTRYPOINT_NORMALIZED_ARGV[@]}" +fi +unset NEMOCLAW_ENTRYPOINT_NORMALIZED_ARGC NEMOCLAW_ENTRYPOINT_NORMALIZED_ARGV \ + _NEMOCLAW_ENTRYPOINT_ENV_WRAPPER +unset -f nemoclaw_normalize_entrypoint_env_wrapper +# managed-entrypoint-env-wrapper end + # ── Source shared sandbox initialisation library ───────────────── # Single source of truth for security-sensitive primitives shared with # scripts/nemoclaw-start.sh (OpenClaw). Ref: #2277 @@ -102,33 +126,6 @@ exec > >(tee -a "$_START_LOG") 2> >(tee -a "$_START_LOG" >&2) # ── Drop unnecessary Linux capabilities (shared) ──────────────── drop_capabilities /usr/local/bin/nemoclaw-start "$@" -# Normalize the self-wrapper bootstrap (same as OpenClaw entrypoint). -if [ "${1:-}" = "env" ]; then - _raw_args=("$@") - _self_wrapper_index="" - for ((i = 1; i < ${#_raw_args[@]}; i += 1)); do - case "${_raw_args[$i]}" in - *=*) ;; - nemoclaw-start | /usr/local/bin/nemoclaw-start) - _self_wrapper_index="$i" - break - ;; - *) - break - ;; - esac - done - if [ -n "$_self_wrapper_index" ]; then - for ((i = 1; i < _self_wrapper_index; i += 1)); do - export "${_raw_args[$i]}" - done - set -- "${_raw_args[@]:$((_self_wrapper_index + 1))}" - fi -fi - -case "${1:-}" in - nemoclaw-start | /usr/local/bin/nemoclaw-start) shift ;; -esac NEMOCLAW_CMD=("$@") _chat_ui_url_port() { @@ -1711,7 +1708,9 @@ merge_corporate_proxy_ca() { export _NEMOCLAW_CORPORATE_CA_MERGED=1 echo "[nemoclaw] merged corporate proxy CA into sandbox trust bundle (#6210)" >&2 } -merge_corporate_proxy_ca +if [ "${NEMOCLAW_MANAGED_STARTUP_APPLIED:-0}" != "1" ]; then + merge_corporate_proxy_ca +fi # OpenShell injects SSL_CERT_FILE/CURL_CA_BUNDLE for its L7 proxy CA. Persist # them into connect-session shells so Python Slack probes and Hermes tools trust @@ -1977,6 +1976,7 @@ refresh_hermes_runtime_config_hashes() { inspect_hermes_mcp_integrity() { local hash_file="${1:-}" local guard_status + local -a guard_command [ -n "$hash_file" ] || { if [ "$(id -u)" -eq 0 ]; then hash_file="$HERMES_HASH_FILE" @@ -1989,11 +1989,20 @@ inspect_hermes_mcp_integrity() { # exact-parent proof used by --startup-owner. State is returned only through # the kernel-owned exit status: 0=current, 10=pending, anything else=failure. # This avoids a same-UID writable result file or ambiguous shell byte parsing. - if "$_HERMES_PYTHON" -I "$_HERMES_RUNTIME_CONFIG_GUARD" inspect-mcp-integrity \ - --hermes-dir "$HERMES_DIR" \ - --hash-file "$hash_file" \ - --startup-owner \ - --mcp-state-exit-code >/dev/null; then + guard_command=( + "$_HERMES_PYTHON" -I "$_HERMES_RUNTIME_CONFIG_GUARD" inspect-mcp-integrity + --hermes-dir "$HERMES_DIR" + --hash-file "$hash_file" + --startup-owner + --mcp-state-exit-code + ) + if [ "$(id -u)" -eq 0 ]; then + # Hardened managed runtimes can remove root's DAC override before startup. + # Read the sandbox-owned mutable config through its owning identity; the + # step-down exec still leaves the guard as the startup owner's direct child. + guard_command=("${STEP_DOWN_PREFIX_SANDBOX[@]}" "${guard_command[@]}") + fi + if "${guard_command[@]}" >/dev/null; then guard_status=0 else guard_status=$? diff --git a/agents/langchain-deepagents-code/Dockerfile b/agents/langchain-deepagents-code/Dockerfile index 73147ad3c9..6e7e7ed9dd 100644 --- a/agents/langchain-deepagents-code/Dockerfile +++ b/agents/langchain-deepagents-code/Dockerfile @@ -20,10 +20,24 @@ RUN ./install-reviewed-runtime.sh \ RUN chown -R root:root /opt/mcp-tool-discovery-runtime \ && chmod -R a=rX /opt/mcp-tool-discovery-runtime +FROM mcp-tool-discovery-runtime AS managed-startup-runtime-builder +WORKDIR /opt/nemoclaw-managed-startup-build +COPY src/lib/core/json-types.ts src/lib/core/ports.ts ./src/lib/core/ +COPY src/lib/messaging/ ./src/lib/messaging/ +COPY src/lib/onboard/managed-startup/ ./src/lib/onboard/managed-startup/ +COPY src/lib/security/credential-hash.ts ./src/lib/security/ +COPY src/lib/state/paths.ts src/lib/state/state-root.ts ./src/lib/state/ +RUN /opt/mcp-tool-discovery-runtime/node_modules/.bin/esbuild \ + src/lib/onboard/managed-startup/image-runtime.ts \ + --bundle --platform=node --format=cjs --target=node22 \ + --outfile=/out/managed-startup-image-runtime.cjs \ + && chmod 0444 /out/managed-startup-image-runtime.cjs + # hadolint ignore=DL3006 FROM ${BASE_IMAGE} COPY --from=mcp-tool-discovery-runtime /opt/mcp-tool-discovery-runtime/dist/ /usr/local/lib/nemoclaw/mcp-tool-discovery-runtime/ +COPY --from=managed-startup-runtime-builder /out/managed-startup-image-runtime.cjs /usr/local/lib/nemoclaw/managed-startup-image-runtime.cjs RUN discovery_contract="$(node /usr/local/lib/nemoclaw/mcp-tool-discovery-runtime/mcp-tool-discovery.mjs)" \ && node -e 'const result = JSON.parse(process.argv[1]); if (result.protocol !== 1 || result.ok !== false || result.detail !== "tool discovery received invalid runtime arguments") process.exit(1);' "$discovery_contract" \ && discovery_unsafe="$(find -L /usr/local/lib/nemoclaw/mcp-tool-discovery-runtime \( ! -user root -o -perm /022 \) -print -quit)" \ @@ -69,7 +83,9 @@ COPY agents/langchain-deepagents-code/validate-observability.py /opt/nemoclaw-de COPY agents/langchain-deepagents-code/dcode-wrapper.sh /usr/local/lib/nemoclaw/dcode-wrapper.sh COPY agents/langchain-deepagents-code/dcode-launcher.sh /usr/local/lib/nemoclaw/dcode-launcher.sh COPY agents/langchain-deepagents-code/dcode-session-supervisor.py /usr/local/lib/nemoclaw/dcode-session-supervisor.py +COPY scripts/lib/entrypoint-env-wrapper.sh /usr/local/lib/nemoclaw/entrypoint-env-wrapper.sh COPY agents/langchain-deepagents-code/start.sh /usr/local/bin/nemoclaw-start +COPY scripts/managed-startup-hold.sh /usr/local/bin/nemoclaw-managed-startup-hold COPY nemoclaw-blueprint/ /opt/nemoclaw-blueprint/ # The first-party profile plugin uses Deep Agents' supported entry-point hook to # register managed aliases without modifying third-party package source. The @@ -82,8 +98,9 @@ COPY nemoclaw-blueprint/ /opt/nemoclaw-blueprint/ # regressionTest: the stripped-base gate must reach this marker, then fail import. # removalCondition: remove when installation validates dependencies atomically. # hadolint ignore=DL4006 -RUN chmod 444 /opt/nemoclaw-deepagents-code/generate-config.ts /opt/nemoclaw-deepagents-code/managed-dcode-runtime.py /opt/nemoclaw-deepagents-code/patch-managed-deepagents-code.py /opt/nemoclaw-deepagents-code/validate-nemotron-ultra-profile.py /opt/nemoclaw-deepagents-code/progressive_tool_disclosure.py /opt/nemoclaw-deepagents-code/nemoclaw_observability.py /opt/nemoclaw-deepagents-code/validate-progressive-tool-disclosure.py /opt/nemoclaw-deepagents-code/validate-observability.py \ - && chmod 755 /usr/local/bin/nemoclaw-start /usr/local/lib/nemoclaw/dcode-wrapper.sh /usr/local/lib/nemoclaw/dcode-launcher.sh /usr/local/lib/nemoclaw/dcode-session-supervisor.py \ +RUN install -d -o root -g root -m 0755 /run/nemoclaw \ + && chmod 444 /opt/nemoclaw-deepagents-code/generate-config.ts /opt/nemoclaw-deepagents-code/managed-dcode-runtime.py /opt/nemoclaw-deepagents-code/patch-managed-deepagents-code.py /opt/nemoclaw-deepagents-code/validate-nemotron-ultra-profile.py /opt/nemoclaw-deepagents-code/progressive_tool_disclosure.py /opt/nemoclaw-deepagents-code/nemoclaw_observability.py /opt/nemoclaw-deepagents-code/validate-progressive-tool-disclosure.py /opt/nemoclaw-deepagents-code/validate-observability.py /usr/local/lib/nemoclaw/entrypoint-env-wrapper.sh \ + && chmod 755 /usr/local/bin/nemoclaw-start /usr/local/bin/nemoclaw-managed-startup-hold /usr/local/lib/nemoclaw/dcode-wrapper.sh /usr/local/lib/nemoclaw/dcode-launcher.sh /usr/local/lib/nemoclaw/dcode-session-supervisor.py \ && test "$(stat -c '%u:%g:%a' /usr/local/lib/nemoclaw/dcode-session-supervisor.py)" = "0:0:755" \ && install -o root -g root -m 0755 /usr/local/lib/nemoclaw/dcode-launcher.sh /usr/local/lib/nemoclaw/dcode-managed-exec \ && test -f /usr/local/lib/nemoclaw/dcode-managed-exec \ @@ -119,6 +136,9 @@ ARG NEMOCLAW_INFERENCE_BASE_URL=https://inference.local/v1 ARG NEMOCLAW_INFERENCE_API=openai-completions ARG NEMOCLAW_TOOL_DISCLOSURE=progressive ARG NEMOCLAW_DCODE_AUTO_APPROVAL=disabled +# DCode has no extra optional packages today, but release images participate in +# the same managed-image capability contract as OpenClaw and Hermes. +ARG NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION=0 ARG NEMOCLAW_BUILD_ID=default ARG NEMOCLAW_DARWIN_VM_COMPAT=0 ARG NEMOCLAW_PROXY_HOST=10.200.0.1 @@ -131,13 +151,20 @@ RUN case "$NEMOCLAW_TOOL_DISCLOSURE" in \ && case "$NEMOCLAW_DCODE_AUTO_APPROVAL" in \ disabled|thread-opt-in) ;; \ *) echo "ERROR: NEMOCLAW_DCODE_AUTO_APPROVAL must be disabled or thread-opt-in" >&2; exit 1 ;; \ + esac \ + && case "$NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION" in \ + 0|1) ;; \ + *) echo "ERROR: NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION must be 0 or 1" >&2; exit 1 ;; \ esac # The launcher and startup script read these root-owned files instead of # trusting process-level environment overrides for inference routing. Invoking # each launcher validates the build args before the image can complete. The -# empty-prompt probe also exercises the installed launcher -> wrapper chain so a -# stale or misassembled image cannot silently lose the public exit-code contract. +# empty-prompt probe targets the installed wrapper directly: it validates the +# public parser contract without requiring the image builder's kernel to support +# the runtime-only child-subreaper supervisor. Keep that parser-only probe +# hermetic so base-image or builder observability variables cannot mask the +# diagnostic under test; the post-build workflow probes the real managed chain. RUN install -d -m 0755 /usr/local/share/nemoclaw \ && printf '%s\n' "$NEMOCLAW_PROXY_HOST" > /usr/local/share/nemoclaw/dcode-proxy-host \ && printf '%s\n' "$NEMOCLAW_PROXY_PORT" > /usr/local/share/nemoclaw/dcode-proxy-port \ @@ -146,14 +173,20 @@ RUN install -d -m 0755 /usr/local/share/nemoclaw \ && chown root:root /usr/local/share/nemoclaw/dcode-proxy-host /usr/local/share/nemoclaw/dcode-proxy-port /usr/local/share/nemoclaw/dcode-inference-base-url /usr/local/share/nemoclaw/dcode-auto-approval \ && chmod 0444 /usr/local/share/nemoclaw/dcode-proxy-host /usr/local/share/nemoclaw/dcode-proxy-port /usr/local/share/nemoclaw/dcode-inference-base-url /usr/local/share/nemoclaw/dcode-auto-approval \ && empty_prompt_log="$(mktemp)" \ - && if timeout 10 /usr/local/bin/dcode -n "" >"$empty_prompt_log" 2>&1; then empty_prompt_status=0; else empty_prompt_status=$?; fi \ - && test "$empty_prompt_status" -eq 2 \ - && test "$(cat "$empty_prompt_log")" = "NemoClaw: empty non-interactive prompt for -n; provide prompt text." \ + && if timeout 10 env -i /usr/local/lib/nemoclaw/dcode-wrapper.sh -n "" >"$empty_prompt_log" 2>&1; then empty_prompt_status=0; else empty_prompt_status=$?; fi \ + && empty_prompt_output="$(cat "$empty_prompt_log")" \ + && if [ "$empty_prompt_status" -ne 2 ] \ + || [ "$empty_prompt_output" != "NemoClaw: empty non-interactive prompt for -n; provide prompt text." ]; then \ + printf 'ERROR: managed dcode empty-prompt probe returned status %s:\n%s\n' \ + "$empty_prompt_status" "$empty_prompt_output" >&2; \ + rm -f "$empty_prompt_log"; \ + exit 1; \ + fi \ && rm -f "$empty_prompt_log" \ - && /usr/local/lib/nemoclaw/dcode-managed-exec /usr/bin/true \ - && /usr/local/bin/dcode --version \ - && /usr/local/bin/dcode.real --version \ - && /usr/local/bin/deepagents-code --version + && env -i /usr/local/lib/nemoclaw/dcode-managed-exec /usr/bin/true \ + && env -i /usr/local/bin/dcode --version \ + && env -i /usr/local/bin/dcode.real --version \ + && env -i /usr/local/bin/deepagents-code --version ENV HOME=/sandbox \ VIRTUAL_ENV=/opt/venv \ @@ -165,6 +198,7 @@ ENV HOME=/sandbox \ NEMOCLAW_INFERENCE_BASE_URL=${NEMOCLAW_INFERENCE_BASE_URL} \ NEMOCLAW_INFERENCE_API=${NEMOCLAW_INFERENCE_API} \ NEMOCLAW_TOOL_DISCLOSURE=${NEMOCLAW_TOOL_DISCLOSURE} \ + NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION=${NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION} \ NEMOCLAW_BUILD_ID=${NEMOCLAW_BUILD_ID} \ DEEPAGENTS_CODE_NO_UPDATE_CHECK=1 \ LANGGRAPH_NO_VERSION_CHECK=true \ @@ -262,6 +296,12 @@ RUN set -eu; \ test -z "$(dpkg --audit)" # End completed-image security package verification. -USER sandbox +ARG NEMOCLAW_MANAGED_IMAGE_RUNTIME_USER=sandbox +RUN case "$NEMOCLAW_MANAGED_IMAGE_RUNTIME_USER" in \ + root|sandbox) ;; \ + *) echo "ERROR: NEMOCLAW_MANAGED_IMAGE_RUNTIME_USER must be root or sandbox" >&2; exit 1 ;; \ + esac \ + && command -v setpriv >/dev/null 2>&1 +USER ${NEMOCLAW_MANAGED_IMAGE_RUNTIME_USER} ENTRYPOINT ["/usr/local/bin/nemoclaw-start"] CMD ["/bin/bash"] diff --git a/agents/langchain-deepagents-code/dcode-launcher.sh b/agents/langchain-deepagents-code/dcode-launcher.sh index ed16b26b3b..f447babf33 100755 --- a/agents/langchain-deepagents-code/dcode-launcher.sh +++ b/agents/langchain-deepagents-code/dcode-launcher.sh @@ -14,7 +14,12 @@ unset _nemoclaw_auto_approval_env readonly MANAGED_DCODE_WRAPPER="/usr/local/lib/nemoclaw/dcode-wrapper.sh" readonly MANAGED_EXEC_LAUNCHER="/usr/local/lib/nemoclaw/dcode-managed-exec" readonly MANAGED_OBSERVABILITY_MARKER="/sandbox/.deepagents/.nemoclaw-observability-enabled" -readonly MANAGED_FETCH_CA_BUNDLE_FILE="/etc/openshell-tls/ca-bundle.pem" +if [ -e /run/nemoclaw/managed-startup-ca-bundle.pem ] \ + || [ -L /run/nemoclaw/managed-startup-ca-bundle.pem ]; then + readonly MANAGED_FETCH_CA_BUNDLE_FILE="/run/nemoclaw/managed-startup-ca-bundle.pem" +else + readonly MANAGED_FETCH_CA_BUNDLE_FILE="/etc/openshell-tls/ca-bundle.pem" +fi readonly MANAGED_SESSION_SUPERVISOR="/usr/local/lib/nemoclaw/dcode-session-supervisor.py" export HOME=/sandbox export PATH="/usr/local/bin:/opt/venv/bin:/usr/local/sbin:/usr/sbin:/usr/bin:/sbin:/bin" diff --git a/agents/langchain-deepagents-code/managed-dcode-runtime.py b/agents/langchain-deepagents-code/managed-dcode-runtime.py index d4e36ea4d1..13267fa0e8 100644 --- a/agents/langchain-deepagents-code/managed-dcode-runtime.py +++ b/agents/langchain-deepagents-code/managed-dcode-runtime.py @@ -91,9 +91,20 @@ _FETCH_URL_TRUSTED_PROXY_ENV = ( "DEEPAGENTS_CODE_FETCH_URL_TRUSTED_PROXY_URL" ) -_MANAGED_FETCH_CA_BUNDLE_FILE = Path( +_MANAGED_STARTUP_CA_BUNDLE_FILE = Path( + "/run/nemoclaw/managed-startup-ca-bundle.pem" +) +_OPENSHELL_FETCH_CA_BUNDLE_FILE = Path( "/etc/openshell-tls/ca-bundle.pem" ) +# Startup-profile application runs before this module is imported. Prefer its +# root-owned merged bundle whenever the pathname exists at all; lexists keeps +# an unsafe symlink on the fail-closed path instead of silently falling back. +_MANAGED_FETCH_CA_BUNDLE_FILE = ( + _MANAGED_STARTUP_CA_BUNDLE_FILE + if os.path.lexists(_MANAGED_STARTUP_CA_BUNDLE_FILE) + else _OPENSHELL_FETCH_CA_BUNDLE_FILE +) # Keep this managed adapter allow-list in sync with generate-config.ts and the # patch-managed-deepagents-code.py provider guards injected into Deep Agents Code. _MANAGED_ADAPTER_PROVIDERS = frozenset({"openai", "openrouter"}) @@ -1080,7 +1091,7 @@ def _managed_fetch_proxy_url_from_files() -> str: def _managed_fetch_ca_bundle() -> tuple[int, str]: - """Open and validate fixed OpenShell TLS trust without a pathname race.""" + """Open and validate fixed managed TLS trust without a pathname race.""" path = _MANAGED_FETCH_CA_BUNDLE_FILE no_follow = getattr(os, "O_NOFOLLOW", None) if no_follow is None: diff --git a/agents/langchain-deepagents-code/policy-additions.yaml b/agents/langchain-deepagents-code/policy-additions.yaml index a0a739f47c..fc851e3c03 100644 --- a/agents/langchain-deepagents-code/policy-additions.yaml +++ b/agents/langchain-deepagents-code/policy-additions.yaml @@ -20,6 +20,9 @@ filesystem_policy: - /app - /etc - /var/log + - /run/nemoclaw # Root-owned managed-startup handoff. This + # directory must exist before strict Landlock + # prepares the sandbox workload ruleset. read_write: - /sandbox - /sandbox/.deepagents diff --git a/agents/langchain-deepagents-code/start.sh b/agents/langchain-deepagents-code/start.sh index 718771d4e1..6dfac535c4 100755 --- a/agents/langchain-deepagents-code/start.sh +++ b/agents/langchain-deepagents-code/start.sh @@ -6,13 +6,47 @@ set -euo pipefail unset BASH_ENV ENV + +export HOME=/sandbox +export PATH="/usr/local/bin:/opt/venv/bin:/usr/local/sbin:/usr/sbin:/usr/bin:/sbin:/bin" + +# managed-entrypoint-env-wrapper begin +_NEMOCLAW_ENTRYPOINT_ENV_WRAPPER="/usr/local/lib/nemoclaw/entrypoint-env-wrapper.sh" +if [ ! -f "$_NEMOCLAW_ENTRYPOINT_ENV_WRAPPER" ]; then + _DCODE_ENTRYPOINT_SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + _NEMOCLAW_ENTRYPOINT_ENV_WRAPPER="${_DCODE_ENTRYPOINT_SOURCE_DIR}/../../scripts/lib/entrypoint-env-wrapper.sh" + unset _DCODE_ENTRYPOINT_SOURCE_DIR +fi +if [ ! -f "$_NEMOCLAW_ENTRYPOINT_ENV_WRAPPER" ]; then + printf '%s\n' '[SECURITY] Required entrypoint env-wrapper normalizer is missing.' >&2 + exit 1 +fi +# shellcheck source=scripts/lib/entrypoint-env-wrapper.sh +source "$_NEMOCLAW_ENTRYPOINT_ENV_WRAPPER" +nemoclaw_normalize_entrypoint_env_wrapper "$@" +if [ "$NEMOCLAW_ENTRYPOINT_NORMALIZED_ARGC" -eq 0 ]; then + set -- +else + set -- "${NEMOCLAW_ENTRYPOINT_NORMALIZED_ARGV[@]}" +fi +unset NEMOCLAW_ENTRYPOINT_NORMALIZED_ARGC NEMOCLAW_ENTRYPOINT_NORMALIZED_ARGV \ + _NEMOCLAW_ENTRYPOINT_ENV_WRAPPER +unset -f nemoclaw_normalize_entrypoint_env_wrapper +# managed-entrypoint-env-wrapper end + +# The published managed image uses uid 0 as its OCI entry user so it can accept +# a future profile. Without one, drop immediately and execute the byte-for-byte +# legacy sandbox-user path. Ordinary non-managed builds still start as sandbox. +if [ "$(id -u)" -eq 0 ]; then + exec /usr/bin/setpriv --reuid=sandbox --regid=sandbox --init-groups -- \ + /usr/local/bin/nemoclaw-start "$@" +fi + while IFS= read -r _nemoclaw_auto_approval_env; do unset "$_nemoclaw_auto_approval_env" done < <(compgen -A variable NEMOCLAW_DCODE_AUTO_APPROVAL || true) unset _nemoclaw_auto_approval_env -export HOME=/sandbox -export PATH="/usr/local/bin:/opt/venv/bin:/usr/local/sbin:/usr/sbin:/usr/bin:/sbin:/bin" export DEEPAGENTS_CODE_NO_UPDATE_CHECK=1 export LANGGRAPH_NO_VERSION_CHECK=true export LANGGRAPH_CLI_NO_ANALYTICS=1 @@ -71,7 +105,12 @@ unset _NEMOCLAW_SANDBOX_RLIMITS # or when dcode no longer uses inference.local. readonly MANAGED_PROXY_HOST_FILE="/usr/local/share/nemoclaw/dcode-proxy-host" readonly MANAGED_PROXY_PORT_FILE="/usr/local/share/nemoclaw/dcode-proxy-port" -readonly MANAGED_FETCH_CA_BUNDLE_FILE="/etc/openshell-tls/ca-bundle.pem" +if [ -e /run/nemoclaw/managed-startup-ca-bundle.pem ] \ + || [ -L /run/nemoclaw/managed-startup-ca-bundle.pem ]; then + readonly MANAGED_FETCH_CA_BUNDLE_FILE="/run/nemoclaw/managed-startup-ca-bundle.pem" +else + readonly MANAGED_FETCH_CA_BUNDLE_FILE="/etc/openshell-tls/ca-bundle.pem" +fi readonly MANAGED_PROXY_OWNER_UID=0 managed_proxy_file_metadata() { diff --git a/agents/openclaw/policy-permissive.yaml b/agents/openclaw/policy-permissive.yaml index acb084f051..6a31fe24ea 100644 --- a/agents/openclaw/policy-permissive.yaml +++ b/agents/openclaw/policy-permissive.yaml @@ -20,6 +20,8 @@ filesystem_policy: - /app - /etc - /var/log + - /run/nemoclaw # Mirrors the managed-startup handoff grant + # in the baseline create policy. read_write: - /tmp - /dev/null diff --git a/ci/env-var-doc-allowlist.json b/ci/env-var-doc-allowlist.json index aed046f4cb..1c925733ed 100644 --- a/ci/env-var-doc-allowlist.json +++ b/ci/env-var-doc-allowlist.json @@ -7,6 +7,10 @@ "name": "NEMOCLAW_INVOKED_AS", "reason": "Internal launcher marker set by bin/nemohermes.js so the runtime knows which binary the user invoked. Never user-set; the value is constrained to a known-name allowlist in src/lib/cli/branding.ts." }, + { + "name": "NEMOCLAW_MANAGED_HERMES_HASH_B64", + "reason": "Internal one-shot base64 transport from the privileged managed-startup runtime to the sandbox-user Hermes compatibility-hash writer. NemoClaw synthesizes it for that constrained child action; users never set it." + }, { "name": "NEMOCLAW_MODEL_ROUTER_VENV", "reason": "Internal developer/test override for the managed Model Router virtualenv path. Production users should use the default ~/.nemoclaw/model-router-venv." @@ -90,5 +94,9 @@ { "name": "NEMOCLAW_DOCKER_GPU_PATCH_NETWORK", "reason": "Internal one-process handoff from Docker GPU patch preparation into sandbox creation. Rebuild scopes and restores it; users must not set it." + }, + { + "name": "NEMOCLAW_PODMAN_BIN", + "reason": "Internal test/runtime injection for the trusted Podman executable. Users select Podman with --compute-driver and must not override the executable path." } ] diff --git a/ci/platform-matrix.json b/ci/platform-matrix.json index 2e27ab585a..3b04ef5c66 100644 --- a/ci/platform-matrix.json +++ b/ci/platform-matrix.json @@ -1,7 +1,7 @@ { "$comment": "SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\nSPDX-License-Identifier: Apache-2.0\n\nSingle source of truth for NemoClaw launch claims and platform support. Covers platforms, inference providers, supported agents, messaging integrations, and deployment paths. Scripts read this to generate README and docs tables. QA/CI update platform/provider rows; the engineering owner reviews other rows. Docs are derived.", "version": "1.1", - "updated": "2026-07-21", + "updated": "2026-07-29", "project_status": { "stage": "alpha", @@ -33,6 +33,15 @@ "ci_tested": true, "notes": "Primary tested path. Ubuntu 24.04 has host-level onboarding validation. A digest-pinned Ubuntu 26.04 userspace lane builds the CLI and runs preflight, installer, and platform contracts on eligible main pushes; Docker-host, AppArmor, Landlock, and live onboarding validation on 26.04 remain pending. Other distros (Ubuntu 22.04, Fedora, Rocky, Alma, NixOS, Arch) may work but are not validated." }, + { + "name": "Linux (Podman, amd64)", + "runtimes": ["Podman 5+ (rootless)"], + "status": "caveated", + "prd_priority": "P0", + "ci_tested": false, + "prerequisites_notes": "Supported for CPU-only OpenClaw, Hermes, and LangChain Deep Agents Code sandboxes on Linux amd64 with rootless Podman 5 or later, cgroup v2, subordinate UID/GID ranges, and an active user Podman API socket. Select it explicitly with `--compute-driver podman` or `NEMOCLAW_COMPUTE_DRIVER=podman`. This path requires an immutable managed OCI image and does not use a local Dockerfile build.", + "notes": "Supported with limitations for OpenClaw, Hermes, and LangChain Deep Agents Code on Linux amd64. Requires rootless Podman 5 or later, cgroup v2, subordinate UID and GID ranges for the current user, and a responsive user Podman API socket. The first release supports CPU sandboxes with hosted inference only; GPU passthrough and host-local Ollama, NIM, or vLLM are not supported on this driver. Select Podman explicitly with `--compute-driver podman` or `NEMOCLAW_COMPUTE_DRIVER=podman`; Docker remains the default for new sandboxes. Podman onboarding requires a complete immutable managed OCI image, rejects `--from `, and does not fall back to a Dockerfile build or Docker daemon. NemoClaw persists the selected driver and exact managed runtime binding for status, rebuild, doctor, recovery, and `destroy --cleanup-gateway`. The explicit published-release Podman E2E lane must pass for all three agents before this row may claim dedicated CI qualification." + }, { "name": "macOS (Apple Silicon)", "runtimes": ["Colima", "Docker Desktop"], @@ -224,9 +233,9 @@ "out_of_scope": [ { - "name": "Podman / other container runtimes", + "name": "Other container runtimes", "status": "unsupported", - "notes": "Onboard surfaces an explicit unsupported-runtime error for Podman (`src/lib/onboard/fatal-runtime-preflight.ts` prints the rejection; `src/lib/onboard/preflight.ts` flags the unsupported runtime upstream). Only Docker Engine, Docker Desktop, and Colima are supported. See issue #420 (closed)." + "notes": "Docker Engine, Docker Desktop, and Colima remain supported, and the qualified Linux amd64 rootless Podman path is documented separately. Other container runtimes and drivers are not yet supported launch claims." }, { "name": "Intel Mac (macOS x86_64)", @@ -241,7 +250,7 @@ { "name": "Native Kubernetes or OpenShift deployments", "status": "unsupported", - "notes": "NemoClaw runs the sandbox as a Docker container, not a Kubernetes pod. The default Docker-driver topology does not embed k3s. Operator-managed K8s/OpenShift deployments are out of scope; see issue #407 (community OpenShift through agent-sandbox CRD)." + "notes": "NemoClaw runs the sandbox as an OCI container through a supported local runtime driver, not as an operator-managed Kubernetes pod. The default Docker-driver topology does not embed k3s. Operator-managed K8s/OpenShift deployments are out of scope; see issue #407 (community OpenShift through agent-sandbox CRD)." }, { "name": "Air-gapped / offline installs", @@ -322,7 +331,12 @@ { "name": "Local CLI onboard", "status": "tested", - "notes": "Run `$$nemoclaw onboard` on a tested platform with Docker available locally. Primary path." + "notes": "Run `$$nemoclaw onboard` on a tested platform. Docker is the default local container runtime and remains the primary path." + }, + { + "name": "Rootless Podman on Linux amd64", + "status": "caveated", + "notes": "Run `$$nemoclaw onboard --compute-driver podman` with Podman 5 or later, cgroup v2, subordinate UID/GID ranges, and an active user Podman API socket. Supports CPU-only OpenClaw, Hermes, and LangChain Deep Agents Code sandboxes from immutable managed OCI images; local Dockerfile builds, GPU passthrough, and host-local inference are not supported on this path." }, { "name": "Headless Linux server", diff --git a/ci/source-architecture-budget.json b/ci/source-architecture-budget.json index 3168eb519b..af98051336 100644 --- a/ci/source-architecture-budget.json +++ b/ci/source-architecture-budget.json @@ -5,7 +5,7 @@ "maxByFile": { "src/lib/actions/sandbox/mcp-bridge-contracts.ts": 26, "src/lib/actions/sandbox/process-recovery.ts": 26, - "src/lib/adapters/docker/index.ts": 45, + "src/lib/adapters/docker/index.ts": 44, "src/lib/adapters/openshell/client.ts": 22, "src/lib/adapters/openshell/resolve.ts": 28, "src/lib/adapters/openshell/runtime.ts": 50, @@ -22,7 +22,7 @@ "src/lib/credentials/store.ts": 44, "src/lib/inference/config.ts": 29, "src/lib/inference/web-search.ts": 21, - "src/lib/messaging/channels/index.ts": 25, + "src/lib/messaging/channels/index.ts": 23, "src/lib/onboard/gateway-binding.ts": 47, "src/lib/runner.ts": 89, "src/lib/security/redact.ts": 51, @@ -43,7 +43,7 @@ "src/lib/actions/sandbox/policy-channel.ts": 28, "src/lib/actions/sandbox/process-recovery.ts": 22, "src/lib/actions/sandbox/rebuild-pipeline.ts": 27, - "src/lib/actions/sandbox/snapshot.ts": 38, + "src/lib/actions/sandbox/snapshot.ts": 32, "src/lib/actions/uninstall/run-plan.ts": 25, "src/lib/inference/onboard-probes.ts": 21, "src/lib/inference/vllm.ts": 23, diff --git a/ci/test-file-size-budget.json b/ci/test-file-size-budget.json index 7fd8fc2d98..266f719363 100644 --- a/ci/test-file-size-budget.json +++ b/ci/test-file-size-budget.json @@ -7,7 +7,7 @@ "src/lib/onboard/preflight.test.ts": 1904, "test/generate-openclaw-config.test.ts": 1941, "test/install-preflight.test.ts": 3921, - "test/nemoclaw-start.test.ts": 4819, + "test/nemoclaw-start.test.ts": 4725, "test/onboard-messaging.test.ts": 2043, "test/onboard-selection.test.ts": 4769 } diff --git a/docs/about/how-it-works.mdx b/docs/about/how-it-works.mdx index 214cbb9e58..50f621e988 100644 --- a/docs/about/how-it-works.mdx +++ b/docs/about/how-it-works.mdx @@ -122,6 +122,8 @@ OpenShell-backed lifecycle Reproducible setup : Running setup again recreates the sandbox from the same blueprint and policy definitions. + Managed rebuilds resolve the current CLI release to an exact image digest. + Managed clones retain the source image digest and rebind target-specific configuration through the versioned startup-profile contract. ## CLI, Plugin, and Blueprint @@ -148,6 +150,13 @@ NemoClaw separates host orchestration from sandbox image contents. - The _blueprint_ is a versioned YAML package with the sandbox image, policy, inference profile, and supporting assets. The runner resolves and verifies the blueprint before applying it through OpenShell. +On supported `amd64` Docker hosts, stock onboarding prefers a complete immutable release image for the selected shipped agent. +NemoClaw accepts the release catalog only when one publication cohort contains OpenClaw, Hermes, and Deep Agents images. +It resolves the selected image to an exact digest and builds a secret-free startup profile for the current sandbox configuration. +The image startup applicator materializes that configuration when the sandbox starts. +If managed image resolution is unavailable, Docker uses the trusted canonical Dockerfile recipe as a fallback. +Explicit custom Dockerfile and custom-image workflows continue to build locally. + This separation keeps agent-specific sandbox assets focused and lets host orchestration and blueprint contents evolve on separate release cadences. ## Sandbox Creation @@ -156,16 +165,22 @@ When you run `$$nemoclaw onboard`, NemoClaw creates an OpenShell sandbox that ru The host CLI and blueprint runner orchestrate this process through the OpenShell CLI: 1. NemoClaw resolves the blueprint, checks version compatibility, and verifies the digest. -2. The onboarding flow determines which OpenShell resources to create or update, such as the gateway, inference providers, sandbox, and network policy. -3. The runner calls OpenShell CLI commands to create the sandbox and configure each resource. +2. NemoClaw selects an exact managed release image or the permitted Dockerfile build path. +3. The onboarding flow determines which OpenShell resources to create or update, such as the gateway, inference providers, sandbox, and network policy. +4. The runner starts the exact image with its startup profile or builds the selected Dockerfile locally. +5. The runner calls OpenShell CLI commands to configure the remaining resources. After the sandbox starts, the agent runs inside it with all network, filesystem, and inference controls in place. +Managed rebuilds resolve the complete image cohort for the current CLI release before they replace a sandbox. +Managed clones retain the source image digest and prepare a target-specific startup profile under the same contract. ## Inference Routing Inference requests from the agent never leave the sandbox directly. OpenShell intercepts every inference call and routes it to the configured provider. -During onboarding, NemoClaw validates the selected provider and model, configures the OpenShell route, and bakes the matching model reference into the sandbox image. +During onboarding, NemoClaw validates the selected provider and model and configures the OpenShell route. +For a managed release image, the startup profile supplies the matching model reference to the agent configuration. +The Dockerfile fallback supplies the same configuration through its local-build path. The sandbox then talks to `inference.local`, while the host owns the actual provider credential and upstream endpoint. When you select the Model Router provider, `inference.local` routes to a host-side router that chooses from the configured NVIDIA model pool for each request. diff --git a/docs/get-started/prerequisites.mdx b/docs/get-started/prerequisites.mdx index 41859d63a6..f46540a82e 100644 --- a/docs/get-started/prerequisites.mdx +++ b/docs/get-started/prerequisites.mdx @@ -27,24 +27,26 @@ If you cannot add memory, configure at least 8 GB of swap to work around the iss ## Software -| Dependency | Version | -|------------|----------------------------------| -| Node.js | 22.19 or later | -| npm | 10 or later | -| Docker | Docker Engine, Docker Desktop, or Colima on a tested platform | -| Platform | Refer to [Platforms](#platforms) below | - -On Linux, the installer checks Docker before it installs the NemoClaw CLI. -The installer can install Docker, start the Docker service, and add your user to the `docker` group. -When Docker is missing, the installer downloads Docker's official convenience script, prompts for `sudo`, and starts the service when systemd is available. +| Dependency | Version | +|------------|---------| +| Node.js | 22.19 or later | +| npm | 10 or later | +| Container runtime | Docker Engine, Docker Desktop, or Colima on a tested platform; or rootless Podman 5 or later on Linux amd64 | +| Platform | Refer to [Platforms](#platforms) below | + +Docker remains the default container runtime. +On Linux, the installer checks Docker before it installs the NemoClaw CLI unless you explicitly select Podman with `NEMOCLAW_COMPUTE_DRIVER=podman`. +For the default path, the installer can install Docker, start the Docker service, and add your user to the `docker` group. +On that default Docker path, when Docker is missing, the installer downloads Docker's official convenience script, prompts for `sudo`, and starts the service when systemd is available. In a non-interactive run, the installer can reactivate the group through `sg docker` and continue onboarding. If that path is unavailable or does not restore Docker access, the installer exits with `newgrp docker` guidance before it starts onboarding. +For the Podman path, the installer validates the existing rootless Podman service and does not install or start Docker. If you choose the native Linux Ollama install path, the onboard wizard also requires `zstd` for Ollama archive extraction. The installer also requires `strings` from `binutils` to verify the OpenShell binary before it continues with OpenShell install work. -NemoClaw needs Docker access. +The default Docker driver needs Docker access; the rootless Podman path does not. On personal Linux development machines, adding your user to the `docker` group is the standard way to run Docker without sudo. Members of the `docker` group can control the daemon with root-level impact. Grant this access only to trusted local accounts. @@ -59,6 +61,43 @@ If the installer reports that `strings` is missing, install `binutils` and rerun sudo apt-get install -y binutils ``` +### Rootless Podman on Linux amd64 + +The first native Podman release supports CPU-only OpenClaw, Hermes, and LangChain Deep Agents Code sandboxes on Linux amd64. +It requires Podman 5 or later, cgroup v2, subordinate UID and GID ranges for the current user, and a responsive rootless Podman API socket. +Verify the host before installation: + +```bash +podman --version +test "$(uname -s)" = Linux +test "$(uname -m)" = x86_64 +test "$(stat -fc %T /sys/fs/cgroup)" = cgroup2fs +podman unshare cat /proc/self/uid_map +podman unshare cat /proc/self/gid_map +systemctl --user enable --now podman.socket +test -S "${XDG_RUNTIME_DIR:-/run/user/$(id -u)}/podman/podman.sock" +``` + +Each `podman unshare` result must include a subordinate range longer than one ID. +If it does not, configure the current user in `/etc/subuid` and `/etc/subgid`, then restart the user Podman socket. +NemoClaw checks the canonical user socket automatically. +For a nonstandard absolute socket path, set `OPENSHELL_PODMAN_SOCKET` for the first onboarding run. + +Select Podman explicitly; `auto` continues to select Docker for a new sandbox: + + + +```bash +$$nemoclaw onboard --compute-driver podman --no-gpu +``` + + + +For scripted setup, `NEMOCLAW_COMPUTE_DRIVER=podman` is equivalent to the flag. +The Podman path uses the complete immutable managed OCI image for the selected agent. +It rejects `--from ` and stops if a complete managed-image cohort cannot be resolved; it never falls back to a local Dockerfile build or Docker daemon. +GPU passthrough and host-local Ollama, NIM, or vLLM are not supported on Podman in this release. + On macOS, NemoClaw uses the Docker-driver OpenShell gateway path with Docker Desktop or Colima. You do not need to install or sign a separate OpenShell VM driver helper for standard macOS onboarding. If you use Homebrew Colima, install the Docker CLI package with Colima because `brew install colima` does not provide the `docker` command: @@ -96,6 +135,7 @@ The table comes from [`ci/platform-matrix.json`](https://github.com/NVIDIA/NemoC | DGX OS (Spark) | Docker | Tested | Use the standard installer and `$$nemoclaw onboard`. For an end-to-end walkthrough with local inference, see the [NVIDIA Spark playbook](https://build.nvidia.com/spark/nemoclaw). | | DGX OS (Station) | Docker | Tested with limitations | Tested with limitations across qualified profiles on one physical DGX Station GB300; see [Additional Setup for DGX Station](additional-setup/dgx-station-preparation) for accepted profiles, the pending no-OTA DGX OS `7.6.x` end-to-end qualification, runtime gates, and current dual-Station and dedicated CI limitations. | | Linux | Docker | Tested | Primary tested path. Ubuntu 24.04 has host-level onboarding validation. A digest-pinned Ubuntu 26.04 userspace lane builds the CLI and runs preflight, installer, and platform contracts on eligible main pushes; Docker-host, AppArmor, Landlock, and live onboarding validation on 26.04 remain pending. Other distros (Ubuntu 22.04, Fedora, Rocky, Alma, NixOS, Arch) may work but are not validated. | +| Linux (Podman, amd64) | Podman 5+ (rootless) | Tested with limitations | Supported for CPU-only OpenClaw, Hermes, and LangChain Deep Agents Code sandboxes on Linux amd64 with rootless Podman 5 or later, cgroup v2, subordinate UID/GID ranges, and an active user Podman API socket. Select it explicitly with `--compute-driver podman` or `NEMOCLAW_COMPUTE_DRIVER=podman`. This path requires an immutable managed OCI image and does not use a local Dockerfile build. | | macOS (Apple Silicon) | Colima, Docker Desktop | Tested with limitations | Start the container runtime (Colima or Docker Desktop) before running the installer. When Homebrew is available, OpenShell uses its official formula and the gateway appears in `brew services list` as `openshell`; without Homebrew, NemoClaw uses the standalone OpenShell install and detached gateway fallback. Homebrew Colima users must install both Colima and the Docker CLI (`brew install colima docker`) before `docker info` can work. Xcode Command Line Tools (`xcode-select --install`) are typically required for Node native modules during install. NemoClaw recommends them but does not enforce them during preflight. | | Windows WSL2 | Docker Desktop (WSL backend) | Tested with limitations | Requires WSL2 with Docker Desktop backend. See [Additional Setup for Windows Machines](additional-setup/windows-preparation) before the Quickstart. | diff --git a/docs/get-started/quickstart-hermes.mdx b/docs/get-started/quickstart-hermes.mdx index d8592cddd6..f125f813bb 100644 --- a/docs/get-started/quickstart-hermes.mdx +++ b/docs/get-started/quickstart-hermes.mdx @@ -155,14 +155,15 @@ Use these details when your first-run path needs more control. DGX Spark, qualifying DGX Station hosts, and Windows WSL can offer an interactive express path after the third-party software notice. After you confirm the Express prompt, the installer switches the remaining onboarding to non-interactive mode and selects the managed local inference path for that platform. - The first Hermes build can take several minutes because NemoClaw builds the Hermes sandbox base image when it is not already cached. + On supported `amd64` Docker hosts, stock Hermes onboarding prefers the exact managed release image and applies current configuration through a secret-free startup profile. + Other supported Docker paths can use the trusted canonical Dockerfile fallback; its first local build can take several minutes when the base is not already cached. Refer to [Set Up vLLM](../inference/local-inference/set-up-vllm) for managed model profiles, Station choices, and headless setup. Refer to [Platform Support](../reference/platform-support) for current validation status. The wizard asks for an inference provider, model, required credential, and sandbox name before it prints the review summary. - After confirmation, NemoClaw registers inference, prompts for optional Tavily Search and supported messaging channels, builds and starts the sandbox, sets up Hermes, and applies the selected network policy tier and presets. + After confirmation, NemoClaw registers inference, prompts for optional Tavily Search and supported messaging channels, prepares and starts the sandbox, sets up Hermes, and applies the selected network policy tier and presets. At any prompt, press Enter to accept the default shown in `[brackets]`, type `back` to return to the previous prompt, or type `exit` to quit. The default Hermes sandbox name is `hermes`. @@ -185,8 +186,8 @@ Use these details when your first-run path needs more control. NemoClaw removes `nous-web` from the effective managed-tool selection while preserving selected Nous image, audio, browser, and code tools. API-key mode is inference-only and does not enable managed tool gateways. - After you select a provider and model, review the summary and confirm the build. - NemoClaw writes Hermes configuration into `/sandbox/.hermes`, routes model traffic through `inference.local`, and starts the Hermes gateway inside the sandbox. + After you select a provider and model, review and confirm the configuration. + NemoClaw materializes Hermes configuration in `/sandbox/.hermes`, routes model traffic through `inference.local`, and starts the Hermes gateway inside the sandbox. The Hermes image includes runtime dependencies for supported NemoClaw messaging integrations, the API service, and its health endpoint. The base image does not include unsupported Hermes integrations. @@ -213,7 +214,7 @@ Use these details when your first-run path needs more control. Use the provider variables from [Choose an Inference Provider](../inference/learn-and-choose/choose-inference-provider) when you choose another provider. Set `NEMOCLAW_WEB_SEARCH_PROVIDER=none` to disable web search explicitly. When the selector is unset, Hermes enables Tavily automatically when `TAVILY_API_KEY` is available and ignores `BRAVE_API_KEY`. - Changing or disabling Tavily requires sandbox recreation because the backend, credential attachment, and policy selection are build-time inputs. + Changing or disabling Tavily requires sandbox recreation because the backend, credential attachment, and policy selection are sandbox-creation and startup inputs. Rerun onboarding with the new selection and accept recreation, or pass `--recreate-sandbox`. diff --git a/docs/get-started/quickstart-langchain-deepagents-code.mdx b/docs/get-started/quickstart-langchain-deepagents-code.mdx index 30a3e837d9..258b2afde8 100644 --- a/docs/get-started/quickstart-langchain-deepagents-code.mdx +++ b/docs/get-started/quickstart-langchain-deepagents-code.mdx @@ -229,7 +229,7 @@ Use the interactive TUI when you need to inspect each destructive tool request b Managed Deep Agents sandboxes keep interactive thread auto-approval disabled by default. In this mode, the TUI auto-approval choice and `dcode -y` fail closed. -Enable the capability for a named sandbox through a transactional rebuild because NemoClaw bakes the capability into the managed image. +Enable the capability for a named sandbox through a transactional rebuild so NemoClaw materializes it in the replacement sandbox's managed startup profile. ```bash nemo-deepagents rebuild --dcode-auto-approval thread-opt-in --yes @@ -274,7 +274,7 @@ nemo-deepagents config get ``` NemoClaw parses `config.toml`, removes gateway auth data, and redacts credential-shaped values before printing it. -`config set` is not supported for this image-baked configuration; re-onboard the named sandbox to change its managed provider or model selection. +`config set` is not supported for this managed startup configuration; re-onboard the named sandbox to change its managed provider or model selection. @@ -301,9 +301,9 @@ The managed `.deepagents/.nemoclaw-mcp.json` projection is also excluded because Service credentials remain in OpenShell provider state. It also does not preserve `hooks.json`; executable Deep Agents Code hooks are disabled in the managed harness. If `.deepagents/.state/auth.json` contains upstream credentials, or `.deepagents/.state/chatgpt-auth.json` exists, the managed Deep Agents Code launch paths refuse to start until that credential state is removed. -Before a managed Deep Agents Code rebuild changes the sandbox, NemoClaw selects its recorded OpenShell gateway, tests the recorded inference route through `https://inference.local`, and prepares the replacement from the recorded provider, model, policy, and build inputs with a pinned base and fingerprinted context. +Before a managed Deep Agents Code rebuild changes the sandbox, NemoClaw selects its recorded OpenShell gateway, tests the recorded inference route through `https://inference.local`, resolves the current complete managed-image cohort to an exact Deep Agents Code digest, and stages a replacement startup profile from the current durable provider, model, policy, and agent options. Initial failures stop before backup. -After backup, NemoClaw rechecks the target, route, and retained build inputs before changing MCP state, then checks again after MCP preparation and before stopping inference or deleting the old sandbox. +After backup, NemoClaw rechecks the target, route, and retained catalog/profile handoff before changing MCP state, then checks again after MCP preparation and before stopping inference or deleting the old sandbox. If the final check fails, NemoClaw restores the previous MCP state and keeps the existing sandbox intact. Rebuild also preserves the standalone Deep Agents Code `tavily` preset, the recorded observability choice unless explicitly overridden, and recorded custom policies from their exact stored content. diff --git a/docs/get-started/quickstart.mdx b/docs/get-started/quickstart.mdx index 0e6391ec86..6f300dfbd8 100644 --- a/docs/get-started/quickstart.mdx +++ b/docs/get-started/quickstart.mdx @@ -133,7 +133,7 @@ Use these details when your first-run path needs more control. Export the relevant API key before starting the installer when you do not want the wizard to prompt for it. Refer to [Choose an Inference Provider](../inference/learn-and-choose/choose-inference-provider) for provider requirements, model choices, and local-server setup. - Web search and messaging are optional build-time choices. + Web search and messaging are optional onboarding choices. Add them when you need them, then rerun onboarding and accept sandbox recreation when you change those choices later. Refer to [Choose Messaging Channels](../manage-sandboxes/messaging-channels/choose-messaging-channels) and [Network Policies](../network-policy/approve-network-requests) before enabling them. @@ -144,6 +144,12 @@ Use these details when your first-run path needs more control. If it prints a `newgrp docker` command, run that command before you retry the installer. On macOS, start Docker Desktop or Colima first. + On supported `amd64` Docker hosts, stock onboarding prefers complete immutable release images for OpenClaw, Hermes, and Deep Agents. + NemoClaw resolves the selected release image to an exact digest. + It applies the current onboarding configuration through a secret-free startup profile when the sandbox starts. + If managed image resolution is unavailable, Docker uses the trusted canonical Dockerfile recipe as a fallback. + Explicit custom Dockerfile and custom-image workflows continue to use their local-build behavior. + When a coding agent reports that its execution sandbox blocked Docker, use its command-scoped approval flow, if available. Approve only the exact Docker-dependent command that the coding agent requests to rerun outside the sandbox. Do not change Docker socket permissions or grant broad host access only to bypass the restriction. @@ -208,14 +214,15 @@ Use these details when your first-run path needs more control. The wizard runs preflight checks, starts or reuses the OpenShell gateway, asks for an inference provider and model, collects required credentials, and asks for a sandbox name. It prints a review summary before it registers the provider with OpenShell. - After confirmation, NemoClaw registers inference, prompts for optional web search and messaging channels, builds and starts the sandbox, sets up OpenClaw, and applies the selected network policy tier and presets. + After confirmation, NemoClaw registers inference, prompts for optional web search and messaging channels, prepares and starts the sandbox, sets up the selected agent, and applies the selected network policy tier and presets. At any prompt, press Enter to accept the default shown in `[brackets]`, type `back` to return to the previous prompt, or type `exit` to quit. - Onboarding builds the sandbox image with a managed `NEMOCLAW_DISABLE_DEVICE_AUTH=1` compatibility setting so the dashboard is usable during setup. + Managed release-image onboarding puts the `NEMOCLAW_DISABLE_DEVICE_AUTH=1` compatibility setting in the secret-free startup profile. NemoClaw records that this value came from onboarding rather than reporting it as an operator-selected opt-out. - This build-time setting is baked into the image and setting it after onboarding does not affect an existing sandbox. + The trusted Dockerfile fallback applies the same setting during its local build. + Setting the variable after onboarding does not affect an existing sandbox. If registered sandboxes already exist, the installer prepares the current NemoClaw CLI without replacing OpenShell and requires a fresh backup of every registered sandbox before it changes the gateway. @@ -267,7 +274,7 @@ Use these details when your first-run path needs more control. For an OpenAI-compatible endpoint using HTTP on `localhost`, `127.0.0.1`, or `[::1]` and port `8000`, `11434`, or `11435`, press Enter at the API key prompt to select no authentication. - After you enter a sandbox name, the wizard asks for final confirmation before it registers the provider, prompts for integrations, and builds the sandbox image. + After you enter a sandbox name, the wizard asks for final confirmation before it registers the provider, prompts for integrations, and prepares the sandbox. ```text ────────────────────────────────────────────────── @@ -280,7 +287,6 @@ Use these details when your first-run path needs more control. Managed tools: none Messaging: none Sandbox name: my-gpt-claw - Note: Sandbox build typically takes 5–15 minutes on this host. ────────────────────────────────────────────────── Web search and messaging channels will be prompted next. Apply this configuration? [Y/n]: @@ -295,7 +301,8 @@ Use these details when your first-run path needs more control. After confirmation, NemoClaw registers the selected provider with the OpenShell gateway and sets the `inference.local` route. The wizard asks whether to enable web search and offers Brave Search or Tavily Search. Provide `BRAVE_API_KEY` for Brave Search or `TAVILY_API_KEY` for Tavily Search when prompted. - NemoClaw validates the selected key before it builds the sandbox, registers a sandbox-scoped OpenShell provider, and writes only an OpenShell resolver placeholder into the OpenClaw configuration. + NemoClaw validates the selected key before it prepares the sandbox. + It registers a sandbox-scoped OpenShell provider and writes only an OpenShell resolver placeholder into the OpenClaw configuration. OpenShell replaces the placeholder with the real key at egress. For non-interactive onboarding, select the provider explicitly and export its key. @@ -315,13 +322,13 @@ Use these details when your first-run path needs more control. The wizard also offers Telegram, Discord, Slack, WeChat, and WhatsApp. Press a channel number to toggle it, then press Enter to continue. Leave every channel unselected to skip messaging setup. - When you select a channel, NemoClaw validates the token format before it bakes the channel configuration into the sandbox. + When you select a channel, NemoClaw validates the token format before it prepares the sandbox configuration. For example, Slack bot tokens must start with `xoxb-`. WeChat and WhatsApp are experimental. Refer to [Choose Messaging Channels](../manage-sandboxes/messaging-channels/choose-messaging-channels) before enabling them. - After the sandbox image builds and OpenClaw starts, NemoClaw asks which network policy tier to apply. - Web search and messaging selections happen first so the sandbox image and policy suggestions stay aligned. + After the sandbox starts, NemoClaw asks which network policy tier to apply. + Web search and messaging selections happen first so the sandbox configuration and policy suggestions stay aligned. The default Balanced tier includes common development presets, such as npm, PyPI, Hugging Face, and Homebrew, plus the matching `brave` or `tavily` preset. Add the `weather` preset explicitly for read-only weather lookups. OpenClaw sandboxes also receive the `openclaw-pricing` preset automatically so session-cost records can populate without manual configuration. @@ -380,7 +387,7 @@ Use these details when your first-run path needs more control. The wizard starts a background dashboard port forward and prints its URL in the ready summary. The default host port is `18789`. When that port is occupied, NemoClaw uses the next free dashboard port, such as `18790`, and prints it in the final URL. - If the selected port becomes occupied after the sandbox build begins, onboarding rolls back the new sandbox and asks you to retry rather than print an unreachable URL. + If the selected port becomes occupied after sandbox creation begins, onboarding rolls back the new sandbox and asks you to retry rather than print an unreachable URL. The installation transcript does not print the gateway token. Use `nemoclaw my-gpt-claw dashboard-url --quiet` to print the complete authenticated URL explicitly. diff --git a/docs/manage-sandboxes/backup-restore.mdx b/docs/manage-sandboxes/backup-restore.mdx index a0cfab7835..deb5446cf9 100644 --- a/docs/manage-sandboxes/backup-restore.mdx +++ b/docs/manage-sandboxes/backup-restore.mdx @@ -118,6 +118,14 @@ $$nemoclaw my-assistant snapshot restore before-upgrade --to my-assistant-clone $$nemoclaw my-assistant snapshot restore before-upgrade --to my-assistant-clone --force --yes ``` +For a managed release-image source, NemoClaw validates the durable workload receipt and retains the exact source image digest. +It reconciles the source's current route and configuration before it creates a destination-specific secret-free startup profile and workload receipt. +Export the credential variables for every enabled messaging or web-search provider before you clone. +If the source receipt records an authenticated proxy, export the matching proxy environment variables again. +Managed Hermes sandboxes with enabled Nous tool gateways cannot be snapshot-cloned because the destination needs a fresh Nous OAuth credential and broker binding. +Disable those gateways before cloning, or re-onboard the destination. +NemoClaw completes these validations before it deletes an existing `--force` destination. + For dashboard-enabled agents, NemoClaw allocates the destination sandbox its own dashboard port instead of reusing the source port. If no port is available, restore stops before deleting an existing `--force` destination. diff --git a/docs/manage-sandboxes/runtime-controls.mdx b/docs/manage-sandboxes/runtime-controls.mdx index 4aca95e49c..c270fac248 100644 --- a/docs/manage-sandboxes/runtime-controls.mdx +++ b/docs/manage-sandboxes/runtime-controls.mdx @@ -4,7 +4,7 @@ title: "Understand Runtime Changes" sidebar-title: "Understand Runtime Changes" description: "Determine which NemoClaw sandbox changes apply at runtime and which require a rebuild or re-onboard." -description-agent: "Maps common OpenClaw and Hermes configuration changes to runtime updates, gateway restarts, rebuilds, or re-onboarding. Use when deciding how a sandbox change takes effect." +description-agent: "Maps supported agent configuration changes to runtime updates, gateway restarts, rebuilds, or re-onboarding. Use when deciding how a sandbox change takes effect." keywords: ["sandbox mutability", "sandbox runtime configuration", "sandbox rebuild"] content: type: "concept" @@ -13,7 +13,16 @@ skill: agent-variants: ["openclaw", "hermes"] --- Use this matrix to choose the operation that makes a sandbox change take effect. -NemoClaw applies its security posture in three layers: what onboarding writes into the sandbox image, what the running sandbox can hot-reload, and what requires a rebuild or re-onboard. +NemoClaw separates immutable image contents, startup configuration, hot-reloadable state, and changes that require a rebuild or re-onboard. + +## Managed Image and Startup Profile Changes + +On supported `amd64` Docker hosts, stock OpenClaw, Hermes, and Deep Agents onboarding prefers complete immutable release images. +NemoClaw resolves each selected image to an exact digest and applies current configuration through a secret-free startup profile. +Managed rebuilds resolve the current CLI release to an exact image digest and prepare a replacement startup profile. +Managed clones retain the source image digest and rebind target-specific configuration through the same startup-profile contract. +If managed image resolution is unavailable, Docker uses the trusted canonical Dockerfile recipe as a fallback. +Explicit custom Dockerfile and custom-image workflows continue to use their local-build behavior. @@ -21,18 +30,18 @@ NemoClaw applies its security posture in three layers: what onboarding writes in | Item | When the change takes effect | How to change it | |---|---|---| -| Inference provider | Runtime route and config update while shields are down; rebuild only if you need to recreate the image | Run `$$nemoclaw shields down`, then `$$nemoclaw inference set`, then restore shields | +| Inference provider | Runtime route and config update while shields are down; rebuild only if you need to recreate sandbox startup state | Run `$$nemoclaw shields down`, then `$$nemoclaw inference set`, then restore shields | | Inference model on the current provider | Runtime route and config update while shields are down | Run `$$nemoclaw shields down`, then `$$nemoclaw inference set`, then restore shields | -| Sub-agent | Re-onboard required because the sub-agent and workspace are baked at onboard | `$$nemoclaw onboard --recreate-sandbox` | +| Sub-agent | Re-onboard required because onboarding prepares the sub-agent and workspace together | `$$nemoclaw onboard --recreate-sandbox` | | Network policy preset | Runtime on the next request; rebuild only if the preset adds bind-mounted secrets | `$$nemoclaw policy add ` or `policy remove ` | | Network allowlist | Runtime on the next request | `openshell policy set` or the interactive approval prompt at the gateway | | Channel tokens | Rebuild required because the channel configuration and credential attachment are created during onboarding or rebuild | `$$nemoclaw channels add `, then accept the rebuild prompt | | Channel enable or disable | Rebuild required because `openclaw.json` is the runtime source of truth | `$$nemoclaw channels stop `, then rebuild | | Dashboard forward port | Runtime; the port is re-resolved on the next `connect` | `NEMOCLAW_DASHBOARD_PORT= $$nemoclaw connect` | -| Dashboard bind address | Build and runtime; an existing local-only sandbox must be recreated with the remote-bind opt-in | `NEMOCLAW_DASHBOARD_BIND=0.0.0.0 $$nemoclaw onboard --recreate-sandbox`, then use the same variable with `connect` | +| Dashboard bind address | Sandbox creation and runtime; an existing local-only sandbox must be recreated with the remote-bind opt-in | `NEMOCLAW_DASHBOARD_BIND=0.0.0.0 $$nemoclaw onboard --recreate-sandbox`, then use the same variable with `connect` | | Gateway process environment or startup-only plugin state | Runtime after gateway restart | `$$nemoclaw gateway restart` | -| Default workspace template seed | Locked at first sandbox boot; re-onboard required to change the bake-time choice | Set `NEMOCLAW_MINIMAL_BOOTSTRAP=1` before `$$nemoclaw onboard` to skip default template seeding for new or pristine workspaces; existing files are not deleted | -| Web search provider | Rebuild required because onboarding bakes the provider plugin configuration and credential attachment into the image | Set `NEMOCLAW_WEB_SEARCH_PROVIDER=brave`, `tavily`, or `none`, then rerun onboarding and recreate the sandbox | +| Default workspace template seed | Locked at first sandbox boot; re-onboard required to change the onboarding choice | Set `NEMOCLAW_MINIMAL_BOOTSTRAP=1` before `$$nemoclaw onboard` to skip default template seeding for new or pristine workspaces; existing files are not deleted | +| Web search provider | Rebuild required because onboarding prepares the provider configuration and credential attachment for sandbox creation | Set `NEMOCLAW_WEB_SEARCH_PROVIDER=brave`, `tavily`, or `none`, then rerun onboarding and recreate the sandbox | | Filesystem layout | Locked at creation | Re-onboard with `$$nemoclaw onboard --recreate-sandbox` | | Sandbox name | Locked at creation | Re-onboard with a different `--name` | | GPU passthrough or device selector | Locked at creation | Re-onboard with `--gpu` or `--sandbox-gpu-device` | @@ -43,7 +52,7 @@ For a new or pristine OpenClaw workspace, `NEMOCLAW_MINIMAL_BOOTSTRAP=1` avoids It does not delete existing workspace files. The runtime source of truth is `/sandbox/.openclaw/openclaw.json`. -The host registry caches metadata, but the image and OpenClaw read from the in-sandbox file. +The host registry caches metadata, but OpenClaw reads from the in-sandbox file. OpenClaw config and inference changes are refused while shields are up. Run `$$nemoclaw shields down` before the change, then restore lockdown with `$$nemoclaw shields up`. @@ -66,23 +75,24 @@ If preflight detects an unsafe path, invalid config, invalid ownership posture, | Item | When the change takes effect | How to change it | |---|---|---| -| Inference provider | Runtime route changes apply immediately; rebuild if you need to rebake model metadata into the image | `$$nemoclaw inference set` for route changes, or `$$nemoclaw rebuild` after changing build-time settings | +| Inference provider | Runtime route changes apply immediately; rebuild if you need to recreate the sandbox startup configuration | `$$nemoclaw inference set` for route changes, or `$$nemoclaw rebuild` after changing startup settings | | Inference model on the current provider | Hot-reloadable through the Hermes config sync path | `$$nemoclaw inference set` | -| Agent runtime | Re-onboard required because the agent and state layout are baked at onboard | `$$nemoclaw onboard --recreate-sandbox` or `nemoclaw onboard --agent openclaw --recreate-sandbox` | +| Agent runtime | Re-onboard required because onboarding selects the agent and state layout together | `$$nemoclaw onboard --recreate-sandbox` or `nemoclaw onboard --agent openclaw --recreate-sandbox` | | Network policy preset | Runtime on the next request; rebuild only if the preset adds bind-mounted secrets | `$$nemoclaw policy add ` or `policy remove ` | | Network allowlist | Runtime on the next request | `openshell policy set` or the interactive approval prompt at the gateway | | Channel tokens | Rebuild required because the channel configuration and credential attachment are created during onboarding or rebuild | `$$nemoclaw channels add `, then accept the rebuild prompt | -| Channel enable or disable | Rebuild required because `/sandbox/.hermes/.env` and Hermes config are baked at image build time | `$$nemoclaw channels stop `, then rebuild | +| Channel enable or disable | Rebuild required because sandbox creation materializes `/sandbox/.hermes/.env` and the Hermes configuration | `$$nemoclaw channels stop `, then rebuild | | API or dashboard forward port | Runtime; the host-side forward is re-resolved on the next `connect` | `$$nemoclaw connect` or `openshell forward start` | -| Hermes plugin code, Langfuse settings, or other startup-only runtime config | Runtime after a supported host-side update and gateway restart | Bake plugin code into the image or use a supported host config command, then run `$$nemoclaw gateway restart` | -| Web search provider | Rebuild required because onboarding bakes `web.backend`, the environment placeholder, and the credential attachment into the image | Set `NEMOCLAW_WEB_SEARCH_PROVIDER=tavily` or `none`, then rerun onboarding and recreate the sandbox | +| Hermes plugin code or dependencies | Rebuild required because custom plugin contents come from the custom Dockerfile image | Update the custom Dockerfile, then run `$$nemoclaw rebuild` | +| Langfuse settings or other supported startup-only runtime config | Runtime after a supported host-side update and gateway restart | Use a supported host config command, then run `$$nemoclaw gateway restart` | +| Web search provider | Rebuild required because onboarding prepares `web.backend`, the environment placeholder, and the credential attachment for sandbox creation | Set `NEMOCLAW_WEB_SEARCH_PROVIDER=tavily` or `none`, then rerun onboarding and recreate the sandbox | | Filesystem layout | Locked at creation | Re-onboard with `$$nemoclaw onboard --recreate-sandbox` | | Sandbox name | Locked at creation | Re-onboard with a different `--name` | | GPU passthrough or device selector | Locked at creation | Re-onboard with `--gpu` or `--sandbox-gpu-device` | | Hermes `config.yaml` keys | Mixed; inference and supported config keys can be patched by host commands, while image, policy, and channel changes still require rebuild | Use `$$nemoclaw inference set` or `$$nemoclaw config set` so the config and root-owned trust anchor change together | The runtime source of truth is `/sandbox/.hermes/config.yaml` plus `/sandbox/.hermes/.env`. -The host registry caches metadata, but the image and Hermes runtime read from the in-sandbox files. +The host registry caches metadata, but the Hermes runtime reads from the in-sandbox files. Do not edit those files or their hash files directly and then expect `gateway restart` to establish the bytes as trusted. Use supported host config and inference commands so NemoClaw updates the managed config metadata together. diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index c9e83b14dd..3958764fce 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -207,11 +207,11 @@ langchain-deepagents-code Terminal coding agent built on the Deep Agents SDK ### `$$nemoclaw onboard` Run the interactive setup wizard (recommended for new installs). -The wizard creates an OpenShell gateway, registers inference providers, builds the sandbox image, and creates the sandbox. +The wizard creates an OpenShell gateway, registers inference providers, prepares the selected workload, and creates the sandbox. Use this command for new installs and for recreating a sandbox after changes to policy or configuration. ```bash -$$nemoclaw onboard [--non-interactive] [--resume | --fresh] [--recreate-sandbox] [--gpu | --no-gpu] [--from ] [--name ] [--sandbox-gpu | --no-sandbox-gpu] [--sandbox-gpu-device ] [--agent ] [--agents ] [--tool-disclosure ] [--observability | --no-observability] [--control-ui-port ] [--events=jsonl] [--yes | -y] [--no-ollama-autostart] [--yes-i-accept-third-party-software] +$$nemoclaw onboard [--non-interactive] [--resume | --fresh] [--recreate-sandbox] [--compute-driver ] [--gpu | --no-gpu] [--from ] [--name ] [--sandbox-gpu | --no-sandbox-gpu] [--sandbox-gpu-device ] [--agent ] [--agents ] [--tool-disclosure ] [--observability | --no-observability] [--control-ui-port ] [--events=jsonl] [--yes | -y] [--no-ollama-autostart] [--yes-i-accept-third-party-software] ``` @@ -238,6 +238,31 @@ nemoclaw onboard --agent langchain-deepagents-code [options] `--agent` accepts the canonical manifest names from `$$nemoclaw agents list` plus common aliases. For example, `nemohermes` resolves to `hermes`, while `dcode`, `deepagents`, `deepagents-code`, and `langchain` resolve to `langchain-deepagents-code`. +#### `--compute-driver ` + +Select the local container-runtime driver for the NemoClaw-managed OpenShell gateway and sandbox. +`auto` is the default: it preserves the persisted driver for an existing sandbox or shared gateway and otherwise selects Docker. +Use `docker` to require the existing Docker Engine, Docker Desktop, or Colima path. +Use `podman` to require the qualified rootless Podman path on Linux amd64. + +The Podman path requires Podman 5 or later, cgroup v2, subordinate UID/GID ranges, and a responsive rootless user API socket. +It supports CPU-only sandboxes with hosted inference in this release. +It uses immutable managed OCI images and rejects `--from ` rather than building locally or falling back to Docker. +The same driver is available to every shipped agent: + + + +```bash +$$nemoclaw onboard --compute-driver podman --no-gpu +``` + + + +Set `NEMOCLAW_COMPUTE_DRIVER=podman` for the equivalent scripted selection. +An explicit flag takes precedence over the environment variable. +NemoClaw records the selected driver with the onboarding session, sandbox, and managed gateway runtime. +Resume, rebuild, status recovery, doctor, and destroy use that persisted binding; a later conflicting driver request fails instead of migrating the sandbox or silently falling back to Docker. + #### `--events=jsonl` Emit a read-only stream of canonical onboarding FSM events as JSON Lines on stdout. @@ -262,7 +287,7 @@ Without `--events=jsonl`, terminal output and behavior are unchanged. #### `--resume` and `--fresh` NemoClaw records onboarding progress so interrupted runs can continue. -Use `--resume` to continue a resumable onboarding session with the provider, model, sandbox name, agent, observability choice, and custom Dockerfile path recorded by the original run. +Use `--resume` to continue a resumable onboarding session with the provider, model, sandbox name, agent, compute driver, observability choice, and custom Dockerfile path recorded by the original run. @@ -286,7 +311,7 @@ It also bypasses locally recorded sandbox base-image resolution metadata and rer `--fresh` takes precedence over a base-image hint carried from a rebuild, so NemoClaw does not use that recorded hint. The installer also accepts `--fresh` and forwards it to `$$nemoclaw onboard`, which skips automatic resume detection. `--resume` and `--fresh` are mutually exclusive. -For an existing completed sandbox, use `--fresh --name --recreate-sandbox` when you intentionally want onboarding to replace that sandbox with a new provider, model, agent, or build-time setting. +For an existing completed sandbox, use `--fresh --name --recreate-sandbox` when you intentionally want onboarding to replace that sandbox with a new provider, model, agent, or startup setting. Use `$$nemoclaw rebuild` when you want NemoClaw to recreate the sandbox from its recorded registry metadata without changing those selections. #### `--tool-disclosure ` @@ -351,23 +376,35 @@ Review [Understand Deep Agents Trace Export](/user-guide/deepagents/monitoring/u -When Docker exposes the required identity metadata, NemoClaw records the base-image resolution on managed sandbox images. -During a warm recreate or rebuild, it validates the local image identity and platform, plus the exact repository digest for a published image and any active OpenShell ABI requirement, before reusing it. -A valid match avoids candidate discovery and a network pull. -Set `NEMOCLAW_SANDBOX_BASE_IMAGE_REFRESH=1` to bypass the recorded hint without changing onboarding session handling: +On supported `amd64` Docker hosts and qualified Linux amd64 Podman hosts, stock onboarding prefers a complete immutable release image for the selected shipped agent. +NemoClaw accepts the release catalog only when the same publication cohort contains OpenClaw, Hermes, and Deep Agents images. +It resolves the selected image to an exact repository digest and builds a secret-free startup profile for the current sandbox configuration. +The startup applicator verifies and materializes that profile when the sandbox starts. +Managed rebuilds resolve the current CLI release to an exact digest, while managed clones retain the source image digest and prepare a target-specific profile. + +On Docker, catalog or registry resolution can fall back to the trusted canonical Dockerfile recipe, and an explicit `--from ` request remains a local Dockerfile build. +Podman is buildless: catalog or registry resolution failure stops onboarding, and `--from ` is rejected before any sandbox mutation. +The Podman path never falls back to a Dockerfile build or Docker daemon. +NemoClaw shows a Dockerfile build-time estimate only after one of those local-build paths is known. +Stock managed-image launch does not show the estimate because it does not run a local Dockerfile build. + +The base-image resolution controls below apply only to Docker's trusted Dockerfile fallback and explicit custom Dockerfile workflows. +When Docker exposes the required identity metadata, NemoClaw records the resolved base identity on a locally built sandbox image. +During a warm recreate or rebuild of that path, it validates the local image identity and platform, the exact repository digest for a published base, and any active OpenShell ABI requirement before reusing the base. +Set `NEMOCLAW_SANDBOX_BASE_IMAGE_REFRESH=1` to bypass the recorded Dockerfile base hint without changing onboarding session handling: ```bash NEMOCLAW_SANDBOX_BASE_IMAGE_REFRESH=1 $$nemoclaw onboard --recreate-sandbox NEMOCLAW_SANDBOX_BASE_IMAGE_REFRESH=1 $$nemoclaw rebuild ``` -Base-image selection follows this precedence: +Dockerfile base-image selection follows this precedence: 1. `--fresh` or `NEMOCLAW_SANDBOX_BASE_IMAGE_REFRESH=1` bypasses recorded metadata and reruns normal candidate resolution. These controls are equivalent for base-image selection. 2. Without a bypass, NemoClaw validates and reuses the recorded hint when possible. 3. When the hint is absent or no longer valid, NemoClaw performs normal resolution. -After a cache miss, source checkouts require a fresh local build before candidate selection when base-image inputs have dirty or staged changes, or Git cannot inspect the worktree safely. +After a Dockerfile base cache miss, source checkouts require a fresh local base build before candidate selection when base-image inputs have dirty or staged changes, or Git cannot inspect the worktree safely. For a clean release checkout or versioned install, NemoClaw first accepts the exact release-version image. If that tag exists locally but fails compatibility validation, NemoClaw refreshes the same tag from the registry once and validates it again. If the release-version image is missing or still incompatible, NemoClaw builds a compatible local base instead of falling back to mutable `:latest`. @@ -379,7 +416,7 @@ The required-build path does not reuse an older local tag. If local builds are disabled or the build fails, resolution stops instead of selecting a stale image. When the OpenShell sandbox ABI is required, NemoClaw also rejects a built image that does not report a compatible glibc version. -Explicit base-image overrides are exact: NemoClaw validates the requested ref and fails closed when it cannot be pulled or does not satisfy required ABI, agent runtime, or dependency checks. +Explicit Dockerfile base-image overrides are exact: NemoClaw validates the requested ref and fails closed when it cannot be pulled or does not satisfy required ABI, agent runtime, or dependency checks. Otherwise, normal resolution checks compatible images in Docker's local image store before attempting to pull a missing published candidate. For warm-hint reuse and unversioned development resolution, NemoClaw can reuse another validated local fallback when published candidates are unavailable or incompatible. When the OpenShell sandbox ABI is required, that local fallback must be ABI-compatible. @@ -396,8 +433,8 @@ The final Hermes image accepts only the official published digest tracked by its -Bypassing the recorded hint does not clear Docker's local image store or require a network pull. -Only `--fresh` also discards the saved onboarding session; the refresh environment variable affects base-image selection only. +Bypassing the recorded Dockerfile base hint does not clear Docker's local image store or require a network pull. +Only `--fresh` also discards the saved onboarding session; the refresh environment variable affects Dockerfile base-image selection only. For NemoClaw-managed environments, use `$$nemoclaw onboard` when you need to create or recreate the OpenShell gateway or sandbox. @@ -439,10 +476,10 @@ The legacy `$$nemoclaw setup` command is deprecated; use `$$nemoclaw onboard` in After provider selection, the wizard reviews the provider, model, credential state, and sandbox name before registering inference. -It then prompts for optional web search and messaging channels, builds and starts the sandbox, and asks for a **policy tier** that controls the default set of network policy presets applied to the sandbox. +It then prompts for optional web search and messaging channels, prepares and starts the sandbox, and asks for a **policy tier** that controls the default set of network policy presets applied to the sandbox. -It then prompts for optional web search, builds and starts the sandbox, and asks for a **policy tier** that controls the default set of network policy presets applied to the sandbox. +It then prompts for optional web search, prepares and starts the sandbox, and asks for a **policy tier** that controls the default set of network policy presets applied to the sandbox. Three tiers are available: @@ -588,7 +625,8 @@ The `$$nemoclaw onboard --help` output lists installed runtime names inline, and -Use `--agents ` to declare secondary OpenClaw agents, `agents.defaults`, and main-agent overrides in a checked-in manifest that NemoClaw bakes into the sandbox image at build time. +Use `--agents ` to declare secondary OpenClaw agents, `agents.defaults`, and main-agent overrides in a checked-in manifest. +Managed-image onboarding applies the manifest through the startup profile, while a local Dockerfile path applies it during that build. Refer to [Declarative Multi-Agent Manifest](../configure-agents/declarative-agents-manifest) for the schema and OpenClaw-native sub-agent field semantics. @@ -604,10 +642,10 @@ For Hermes sandboxes, do not use port `8642`; NemoClaw reserves it for the Herme If you enable Slack during onboarding, the wizard collects both the Bot Token (`SLACK_BOT_TOKEN`) and the App-Level Token (`SLACK_APP_TOKEN`). Socket Mode requires both tokens. The app-level token is stored in a dedicated `slack-app` OpenShell provider and forwarded to the sandbox alongside the bot token. -The wizard also accepts optional `SLACK_ALLOWED_USERS` and `SLACK_ALLOWED_CHANNELS` values so you can restrict Slack DMs, channel `@mention` users, and channel IDs before the sandbox image is built. +The wizard also accepts optional `SLACK_ALLOWED_USERS` and `SLACK_ALLOWED_CHANNELS` values so you can restrict Slack DMs, channel `@mention` users, and channel IDs before NemoClaw prepares the sandbox configuration. If you enable Discord during onboarding, the wizard can also prompt for a Discord Server ID, whether the bot should reply only to `@mentions` or to all messages in that server, and an optional Discord User ID. -NemoClaw bakes those values into the sandbox image as Discord guild workspace config so the bot can respond in the selected server, not just in DMs. +NemoClaw includes those values in the Discord guild workspace configuration applied at sandbox startup so the bot can respond in the selected server, not just in DMs. If you leave the Discord User ID blank, the guild config omits the user allowlist and any member of the configured server can message the bot. Guild responses remain mention-gated by default unless you opt into all-message replies. If `DISCORD_SERVER_ID` is set and `DISCORD_REQUIRE_MENTION` is unset, NemoClaw records the existing mention-only default (`DISCORD_REQUIRE_MENTION=1`). @@ -650,7 +688,8 @@ Kanban backup does not include named boards, attachments, worker logs, scratch w Before creating the gateway, the wizard runs preflight checks. -It verifies that Docker is reachable, warns on untested runtimes such as Podman, and prints host remediation guidance when prerequisites are missing. +For the default Docker driver, it verifies that Docker is reachable and prints host remediation guidance when prerequisites are missing. +For an explicitly selected Podman driver, it verifies Linux amd64, Podman 5 or later, rootless operation, cgroup v2, subordinate UID/GID mappings, and the user API socket without probing Docker. The preflight also enforces the OpenShell version range declared in the blueprint (`min_openshell_version` and `max_openshell_version`). If the installed OpenShell version falls outside this range, onboarding exits with an actionable error and a link to compatible releases. For fresh OpenShell installs, NemoClaw queries published OpenShell releases and asks the installer to use a release that fits the blueprint range. @@ -670,8 +709,9 @@ The poll count is clamped to a minimum of `1` so the probe always runs at least #### `--from ` -Build the sandbox image from a custom Dockerfile instead of the stock NemoClaw image. +On the Docker driver, build the sandbox image from a custom Dockerfile instead of the stock NemoClaw image. The supplied Dockerfile defines the complete sandbox image, and NemoClaw does not layer it on top of the stock managed runtime. +The buildless Podman driver rejects this option and requires an immutable managed OCI image. The entire parent directory of the specified file is used as the Docker build context, so any files your Dockerfile references (scripts, config, etc.) must live alongside it. When the supplied path is the selected agent's own managed Dockerfile (for example, `agents/hermes/Dockerfile` in the NemoClaw checkout the CLI runs from), NemoClaw applies one exception and stages the repository root as the build context, exactly as the managed build does, because that Dockerfile copies repository-root paths. This lets you edit the managed Dockerfile in place (for example to add Python packages) and rebuild from it with `--from`. @@ -1113,7 +1153,7 @@ The command refuses to write `gateway` or any dotpath under `gateway.`, which ho -For Deep Agents sandboxes, `config set` is unavailable because the `dcode` configuration is baked into the sandbox image at build time. +For Deep Agents sandboxes, `config set` is unavailable because onboarding materializes the `dcode` configuration when the sandbox starts. Run `$$nemoclaw onboard --agent dcode --name --fresh` when you need to change it. Use `$$nemoclaw config get` to read the current values. @@ -1250,7 +1290,7 @@ Terminal agents do not have a gateway runtime and fail as unsupported. ### `$$nemoclaw stop` -Stop the sandbox's Docker container while preserving all of its state. +Stop the sandbox's local runtime container while preserving all of its state. Workspace files, credentials, network policies, the registry entry, and the OpenShell sandbox record stay in place. Use this to free CPU, memory, and GPU resources without destroying the sandbox; use [`$$nemoclaw destroy`](#$$nemoclaw-name-destroy) when you want to delete it instead. @@ -1271,7 +1311,9 @@ Stopping an already-stopped sandbox succeeds and attempts to remove any leftover Stopping an already-stopped sandbox succeeds without changes. -The command controls the local container directly, so it is available for local-container drivers (the default Docker driver and the vm driver) and unavailable for remote drivers such as kubernetes; if the Docker daemon itself is unreachable, the command reports the outage instead of guessing at container state. +The command controls the local container directly through the sandbox's recorded runtime adapter, so it is available for Docker, vm, and native Podman sandboxes and unavailable for remote or unregistered drivers such as kubernetes. +Docker keeps its daemon-outage guidance. +Podman resolves the exact protected rootless socket and refuses missing, ambiguous, or identity-mismatched managed containers instead of guessing which container to mutate. ### `$$nemoclaw start` @@ -1479,7 +1521,8 @@ The rebuild reuses the existing sandbox name and preserved manifest-defined stat ### `$$nemoclaw doctor` Run a focused health check for one sandbox and the host services it depends on. -The command checks the local CLI build, Docker daemon, OpenShell CLI, NemoClaw gateway container, gateway port mapping, live sandbox state, inference route, configured-provider model invocation, Ollama reachability, and the cloudflared tunnel state. +The command checks the local CLI build, selected container runtime, OpenShell CLI, NemoClaw-managed gateway runtime, gateway listener, live sandbox state, inference route, configured-provider model invocation, Ollama reachability, and the cloudflared tunnel state. +For a Podman sandbox, it validates the persisted rootless socket and managed runtime binding with Podman and does not probe or fall back to Docker. For gateway-based agents, it also reports messaging channel conflicts. @@ -1507,6 +1550,7 @@ Use `--json` for machine-readable output. For a `compatible-endpoint` route that uses `openai-completions`, the JSON report includes an informational `Inference` check labeled `Reasoning effort`. The check reports `low`, `medium`, `high`, or `endpoint-default` and never includes credentials. Because the check has `info` status, it does not change the command's exit status. +After a host reboot on Podman, start the user `podman.socket`, run `$$nemoclaw status` to recover the persisted managed gateway, and use `doctor --json` as the readiness check. @@ -1716,7 +1760,8 @@ Model traffic uses the OpenShell-managed `inference.local` route configured by N ### `$$nemoclaw destroy` -Stop the NIM container, remove the host-side Docker image built during onboard, and delete the sandbox. +Stop the NIM container when applicable, remove any sandbox-owned local Dockerfile image on Docker, and delete the sandbox. +Shared immutable managed images remain in the selected runtime's cache for other sandboxes. This removes the sandbox from the registry. For Ollama-backed sandboxes, `destroy` also asks Ollama to unload currently loaded models and clears stale auth proxy state on a best-effort basis. @@ -1748,6 +1793,8 @@ If deletion fails after hardening, the command keeps the surviving sandbox's loc By default, unattended final-sandbox destroys (`--yes`, `--force`, or `NEMOCLAW_NON_INTERACTIVE=1`) remove the shared NemoClaw gateway on macOS so the host listener is released, while Linux preserves it for reuse. Pass `--cleanup-gateway` to force removal, or `--no-cleanup-gateway` to force preservation. These flags always override both `NEMOCLAW_CLEANUP_GATEWAY` and the platform default. +For the last Podman sandbox, `--cleanup-gateway` uses the exact persisted driver, socket, network, and managed runtime binding to remove the gateway and its protected binding state. +It does not consult Docker or an ambient socket override that conflicts with the recorded binding. If the pre-delete workspace wipe cannot run, use a different sandbox name for a clean start. Cleaning up the gateway after the last sandbox also purges the shared cluster volume that retains the per-name persistent volume. If final gateway cleanup finds a live PID-file process whose command line does not prove it owns the target gateway, `destroy` exits non-zero after sandbox and registry deletion and skips gateway and volume removal. @@ -2074,7 +2121,7 @@ If you omit the required `` argument, the CLI prints the `channels add ### `$$nemoclaw channels remove ` -Clear the stored credentials for a messaging channel and rebuild the sandbox so the image drops the channel. +Clear the stored credentials for a messaging channel and rebuild the sandbox so its startup configuration drops the channel. Running `remove` for a channel that was never configured is a no-op against the credentials file and still triggers the rebuild prompt. When the bridge provider is attached to a live sandbox, NemoClaw detaches it before deleting the provider from the OpenShell gateway. If the matching built-in policy preset is applied, such as `telegram`, `discord`, `slack`, `wechat`, or `whatsapp`, NemoClaw also removes that preset so the upstream API is no longer allow-listed after the channel is gone. @@ -2098,7 +2145,8 @@ $$nemoclaw my-assistant channels remove telegram As with `channels add`, `NEMOCLAW_NON_INTERACTIVE=1` skips the rebuild prompt and queues the change for a manual `$$nemoclaw rebuild`. If you omit the required `` argument, the CLI prints the `channels remove ` usage with the supported channel list. -Host-side removal is the supported path because agent channel config is baked into the container image at build time (`/sandbox/.openclaw/openclaw.json` for OpenClaw and `/sandbox/.hermes/.env` for Hermes); agent-specific channel removals inside the sandbox would modify the running config but not persist changes across rebuilds. +Host-side removal is the supported path because NemoClaw materializes agent channel config when the sandbox starts (`/sandbox/.openclaw/openclaw.json` for OpenClaw and `/sandbox/.hermes/.env` for Hermes). +Agent-specific channel removals inside the sandbox would modify the running config but not persist changes across rebuilds. ### `$$nemoclaw channels stop ` @@ -2446,7 +2494,8 @@ $$nemoclaw my-assistant agents delete work --force --json Reconcile the live sandbox roster against a declarative [agents.yaml manifest](../configure-agents/declarative-agents-manifest). The verb lists current agents via `openclaw agents list --json`, diffs them against the manifest, and adds missing secondaries or deletes orphan ones through `openclaw agents add|delete`. -Per-agent `model`, `subagents.*`, `tools`, top-level `defaults`, and `main` overrides need a sandbox rebuild and are surfaced as warnings rather than silently dropped; rerun `$$nemoclaw onboard --agents --recreate-sandbox` to bake those fields. +Per-agent `model`, `subagents.*`, `tools`, top-level `defaults`, and `main` overrides need a sandbox rebuild and are surfaced as warnings rather than silently dropped. +Rerun `$$nemoclaw onboard --agents --recreate-sandbox` to apply those fields through the replacement sandbox's startup configuration. When the diff removes orphan agents, NemoClaw invokes OpenClaw's confirmation-skipping delete mode internally. `--non-interactive` controls the host-side `agents apply` prompt and is not forwarded to OpenClaw's delete command. @@ -2693,7 +2742,9 @@ $$nemoclaw my-assistant upload ./agent-skills/ /sandbox/.deepagents/agent/skills ### `$$nemoclaw rebuild` Upgrade a sandbox to the current agent version while preserving workspace state. -The command backs up workspace state, destroys the old sandbox (including its host-side Docker image), recreates it with the current image via `onboard --resume`, and restores workspace state into the new sandbox. +The command backs up workspace state, destroys the old sandbox, recreates it through `onboard --resume`, and restores workspace state into the new sandbox. +Managed rebuilds resolve the current CLI release to an exact image digest and a fresh startup profile. +Custom Dockerfile and trusted fallback rebuilds retain their local-build behavior. Credentials are stripped from backups before storage. Policy presets applied to the old sandbox are reapplied to the new one so your egress rules survive the rebuild. Before creating the replacement sandbox, NemoClaw prints the finalized create-time policy scope whenever presets are included. @@ -2805,10 +2856,9 @@ When the command is running from a source checkout, it reports that state and do ### `$$nemoclaw upgrade-sandboxes` -Rebuild sandboxes whose base image is older than the one currently pinned by NemoClaw. -NemoClaw resolves the digest of `ghcr.io/nvidia/nemoclaw/sandbox-base:latest` from the registry, then compares it against the digest each sandbox was created with. -Sandboxes that match the current digest are left alone. -NemoClaw also checks the build fingerprint recorded on each managed sandbox image. +Rebuild managed sandboxes whose agent version or recorded NemoClaw image fingerprint differs from the running CLI. +For a selected stale sandbox, rebuild resolves the current complete managed-image cohort and pins the replacement agent image to an exact digest before deleting the existing sandbox. +Sandboxes whose agent version and fingerprint match are left alone. A sandbox needs upgrade when its agent version is stale, when its recorded NemoClaw image fingerprint differs from the running CLI, or both. When the target version is older than the recorded one (for example after reinstalling with an older `NEMOCLAW_INSTALL_TAG`), the stale listing marks the change with a `(downgrade)` suffix instead of framing it as a routine upgrade. Custom Dockerfile sandboxes are not classified by image drift because rebuilding them onto the default image would drop the custom image. @@ -2826,7 +2876,8 @@ $$nemoclaw upgrade-sandboxes [--check] [--auto] [--yes|-y] | `--yes`, `-y` | Skip the confirmation prompt for the rebuild plan. | Each rebuild reuses the same workspace backup-and-restore flow as `$$nemoclaw rebuild`, so workspace files survive the upgrade. -If the registry is unreachable (offline or firewalled hosts), NemoClaw falls back to the unpinned `:latest` tag and reports that the digest could not be resolved instead of failing. +Managed-image rebuild requires the current complete release cohort and an exact replacement digest. +If catalog or registry resolution is unavailable, rebuild stops before deleting the existing sandbox; it does not fall back to a Dockerfile or an unpinned managed `:latest` image. During installer recovery, a registered sandbox that is not Ready can also be rebuilt from its validated latest backup. That recovery requires a NemoClaw-managed image fingerprint or the installer's explicit confirmation for a listed pre-fingerprint OpenClaw or Hermes entry. The legacy confirmation never overrides recorded custom-image evidence. @@ -2902,7 +2953,14 @@ The selector accepts any of: Pass `--to ` to restore the snapshot into a different sandbox instead of the source. When `dst` does not exist, it is auto-created by reusing the source sandbox's container image. -No re-onboarding is needed. +When the source passes the clone validations below, no re-onboarding is required. +For a managed release-image source, NemoClaw validates the durable workload receipt and retains the exact source image digest. +It reconciles the source's current route and configuration before it creates a destination-specific secret-free startup profile and workload receipt. +Export the credential variables for every enabled messaging or web-search provider before you clone. +If the source receipt records an authenticated proxy, export the matching proxy environment variables again. +Managed Hermes sandboxes with enabled Nous tool gateways cannot be snapshot-cloned because the destination needs a fresh Nous OAuth credential and broker binding. +Disable those gateways before cloning, or re-onboard the destination. +NemoClaw completes these validations before it deletes an existing `--force` destination. A cross-sandbox restore refuses to clone a source or replace an existing destination whose baseline exclusion transaction needs repair, before it creates or deletes anything. When `dst` already exists, `snapshot restore --to ` refuses by default to avoid silently mutating the destination's filesystem. To overwrite an existing destination, pass `--force`: the command deletes `dst`, then recreates it from the source's image and restores the snapshot into the fresh copy. @@ -3332,7 +3390,8 @@ $$nemoclaw credentials reset nvidia-prod ### `$$nemoclaw gc` Remove orphaned sandbox Docker images from the host. -Sandbox creation can build images in the gateway-managed `openshell/sandbox-from` repository or the locally prebuilt `nemoclaw-sandbox-local` repository. +Custom Dockerfile and trusted fallback creation can build sandbox-owned images in the gateway-managed `openshell/sandbox-from` repository or the locally prebuilt `nemoclaw-sandbox-local` repository. +Shared immutable managed release images are not sandbox-owned orphan candidates. The `destroy` and `rebuild` commands clean up the image automatically, but images from older NemoClaw versions or interrupted operations may remain. This command lists images from both repositories, cross-references the sandbox registry, and removes any that are no longer associated with a registered sandbox. @@ -3707,7 +3766,8 @@ Export the credential before running `$$nemoclaw onboard` for that profile. #### Extra OpenClaw agents -Set `NEMOCLAW_EXTRA_AGENTS_JSON` to either a JSON array of secondary-agent entries, or an object payload of the form `{"agents": [...], "defaults": {...}, "main": {...}}`, to bake them into `agents.list[]` at image build time. +Set `NEMOCLAW_EXTRA_AGENTS_JSON` to either a JSON array of secondary-agent entries, or an object payload of the form `{"agents": [...], "defaults": {...}, "main": {...}}`, to add them to `agents.list[]`. +Managed-image onboarding carries the validated values in the startup profile, while a local Dockerfile path carries them through that build. Each entry must declare `id` and `tools`; `workspace`, `agentDir`, `subagents`, `description`, and `model` are optional. The generator always writes the canonical `main` entry first with `default: true`, so secondary agents cannot displace the primary agent. Malformed JSON or invalid entries fail the image build with a structured error. @@ -3781,14 +3841,16 @@ Set them before running `$$nemoclaw onboard`. | Variable | Format | Effect | |----------|--------|--------| +| `NEMOCLAW_COMPUTE_DRIVER` | `auto`, `docker`, or `podman` | Selects the OpenShell compute driver for onboarding and installation. `auto` preserves a persisted driver and otherwise selects Docker. The `--compute-driver` flag takes precedence. A conflicting selection for an existing sandbox or shared gateway fails closed rather than migrating it. | +| `OPENSHELL_PODMAN_SOCKET` | Absolute Unix socket path | Selects a nonstandard rootless Podman API socket for the first Podman onboarding run. NemoClaw persists the qualified socket in the managed runtime binding for later recovery and lifecycle commands; a conflicting ambient override is rejected. | | `NEMOCLAW_YES` | `1` to enable | Auto-accepts confirmation prompts (`--yes` equivalent) including in helpers like the Ollama proxy auth setup, but does not change managed-vLLM storage-warning handling. Express and other non-interactive setup continue after a verified insufficient-capacity warning, interactive setup still requires an explicit `y` or `yes`, and an inconclusive model-cache check stops non-interactive setup with guidance to rerun interactively. | | `NEMOCLAW_OLLAMA_NO_AUTOSTART` | `1` to enable | Skips the wizard's eager Ollama auto-start during inference-provider selection (equivalent to passing `--no-ollama-autostart`). When set and Ollama is not running on `localhost:11434`, an agent that uses the legacy `16384`-token context floor, currently OpenClaw, prints a warning and selects the default fallback model instead of spawning `ollama serve`. An agent that requires a larger verified runtime context, currently Hermes at `64000` tokens, returns to interactive provider selection or exits when the Ollama provider is pinned or onboarding is non-interactive. The flag covers only the provider-selection step; later setup steps (auth proxy, validation, model warm) still expect a reachable Ollama. On Linux hosts with a systemd Ollama unit, the loopback-override path may still restart the daemon before this gate runs. | | `NEMOCLAW_NON_INTERACTIVE_SUDO_MODE` | `prompt` or empty/unset | When set to `prompt`, allows non-interactive onboarding to use prompt-capable `sudo` for host setup steps that require elevation, which can ask for a password. Empty/unset is the default and uses `sudo -n`, which fails instead of asking for a password. Any other value is rejected. | | `NEMOCLAW_NO_EXPRESS` | `1` to enable | Installer-only. Skips the DGX Spark, DGX Station, and Windows WSL express install prompt and continues with the normal interactive onboarding flow. | | `NEMOCLAW_EXPERIMENTAL` | `1` to enable | Surfaces experimental providers and flows in onboarding. | | `NEMOCLAW_IGNORE_RUNTIME_RESOURCES` | `1` to enable | Suppresses the under-provisioned runtime warning during preflight. Use only when you know the sandbox host meets the minimums. | -| `NEMOCLAW_DISABLE_OVERLAY_FIX` | `1` to enable | Skips the Docker overlay-fix step during sandbox build. For environments where the fix is incompatible. | -| `NEMOCLAW_OVERLAY_SNAPSHOTTER` | snapshotter name | Selects the containerd overlay snapshotter for sandbox builds. Empty (default) preserves containerd's choice. | +| `NEMOCLAW_DISABLE_OVERLAY_FIX` | `1` to enable | Skips the Docker overlay-fix step during a local Dockerfile sandbox build. For environments where the fix is incompatible. | +| `NEMOCLAW_OVERLAY_SNAPSHOTTER` | snapshotter name | Selects the containerd overlay snapshotter for local Dockerfile sandbox builds. Empty (default) preserves containerd's choice. | | `NEMOCLAW_SKIP_TELEGRAM_REACHABILITY` | `1` to enable | Skips the Telegram bot reachability probe during onboard (useful in restricted networks). | | `NEMOCLAW_SKIP_SLACK_AUTH_VALIDATION` | `1`, `true`, `yes`, or `on` to enable | Skips the live Slack `auth.test` and `apps.connections.open` credential probes during onboard and `channels add slack`. Use only in restricted networks or hermetic test environments; Slack token format checks still apply. | @@ -3798,8 +3860,8 @@ Set them before running `$$nemoclaw onboard`. | `NEMOCLAW_RAM` | percentage or Kubernetes memory quantity | Overrides the selected profile's memory size passed to OpenShell `--memory`. Percentages resolve against detected capacity. | | `NEMOCLAW_SANDBOX_GPU` | `auto`, `1`, or `0` | Controls sandbox GPU passthrough during onboarding. `auto` enables GPU passthrough when an NVIDIA GPU is detected, `1` requires GPU passthrough, and `0` forces CPU-only sandbox creation. | | `NEMOCLAW_SANDBOX_GPU_DEVICE` | OpenShell GPU device selector | Selects the GPU device passed with `openshell sandbox create --gpu-device`. Requires explicit sandbox GPU enablement with `NEMOCLAW_SANDBOX_GPU=1` (or `--sandbox-gpu` for CLI-driven onboarding); otherwise onboarding rejects the selector instead of treating it as an implicit opt-in. | -| `NEMOCLAW_SANDBOX_BASE_IMAGE_REFRESH` | `1`, `true`, `yes`, or `on` to enable | Bypasses recorded sandbox base-image resolution metadata during onboarding, recreation, and rebuild. NemoClaw reruns candidate resolution but can still use a compatible image from Docker's local image store. Versioned release candidates that exist locally but fail validation are refreshed from the registry once during normal resolution. This setting does not discard onboarding session state. | -| `NEMOCLAW_SANDBOX_BASE_LOCAL_BUILD` | unset or `auto` (default); `1`, `true`, `yes`, or `on` to enable; `0`, `false`, `no`, or `off` to disable | Controls whether base-image resolution may build a compatible image locally. The default allows builds during normal CLI runs and disables them when `NODE_ENV=test` or `VITEST=true`. When source inputs or a missing/incompatible release-version base require a fresh build, disabling local builds makes resolution fail instead of using an unproven image. | +| `NEMOCLAW_SANDBOX_BASE_IMAGE_REFRESH` | `1`, `true`, `yes`, or `on` to enable | Bypasses recorded base-image resolution metadata for the trusted Dockerfile fallback or explicit custom Dockerfile during onboarding, recreation, and rebuild. NemoClaw reruns candidate resolution but can still use a compatible base from Docker's local image store. Versioned release candidates that exist locally but fail validation are refreshed from the registry once during normal resolution. This setting does not discard onboarding session state and does not override managed complete-image selection. | +| `NEMOCLAW_SANDBOX_BASE_LOCAL_BUILD` | unset or `auto` (default); `1`, `true`, `yes`, or `on` to enable; `0`, `false`, `no`, or `off` to disable | Controls whether Dockerfile base-image resolution may build a compatible base locally. The default allows builds during normal CLI runs and disables them when `NODE_ENV=test` or `VITEST=true`. When source inputs or a missing or incompatible release-version base require a fresh build, disabling local builds makes Dockerfile resolution fail instead of using an unproven image. This setting does not override managed complete-image selection. | | `NEMOCLAW_DOCKER_GPU_PATCH` | unset, `auto`, `fallback`, `1`, or `0`; other legacy nonzero values remain accepted through `v0.0.x` and will be removed in `v0.1.0` | Selects Linux Docker-driver GPU routing. Unset, `auto`, or `0` uses native OpenShell GPU injection on ordinary native Linux. `fallback` explicitly opts into one native attempt followed by one bounded compatibility retry when trusted host evidence identifies a GPU-routing failure. `1` and legacy nonzero values select the compatibility patch from the outset. Docker Desktop WSL and Jetson/Tegra use the compatibility path by default; Docker Desktop WSL ignores `0`, while Jetson/Tegra accepts `0` only as a troubleshooting override that bypasses device-group propagation. | | `NEMOCLAW_OPENSHELL_GATEWAY_CONTAINER_PATCH` | `1` to enable; disabled by default | This setting explicitly opts into the Linux gateway compatibility container for an older host ABI or a diagnostic run; use it only on a trusted local host because it uses host networking and mounts the Docker socket read-only even though the socket still exposes the privileged Docker API; prefer OpenShell 0.0.85's directly supported glibc 2.28+ path; see the [OpenShell gateway compatibility review](/user-guide/openclaw/security/openshell-0.0.72-compatibility-review#source-of-truth-boundaries) for the unchanged container boundary. | | `NEMOCLAW_OPENSHELL_GATEWAY_BIN` | path | Advanced override for the `openshell-gateway` binary used by Linux Docker-driver startup. For the default port, the installer accepts the binary under an absolute `XDG_BIN_HOME` when set, otherwise `~/.local/bin/openshell-gateway`; it also accepts `/usr/local/bin/openshell-gateway` or `/usr/bin/openshell-gateway`. Another path fails service staging. The macOS Homebrew service uses the formula's binary. Defaults to the binary next to `openshell`, then common install paths. | @@ -3812,7 +3874,7 @@ Set them before running `$$nemoclaw onboard`. -Set `NEMOCLAW_SANDBOX_BASE_IMAGE_REF` to an OpenClaw sandbox-base tag or digest to override base-image resolution during onboarding. +Set `NEMOCLAW_SANDBOX_BASE_IMAGE_REF` to an OpenClaw sandbox-base tag or digest to override base-image resolution for a local Dockerfile workflow. Remote overrides must use the official NVIDIA sandbox-base repository and resolve to an immutable repository digest. Local bases are accepted only when NemoClaw builds and pins them during the current operation; a local image reference supplied through this environment variable is rejected because it has no trusted build capability. @@ -3820,7 +3882,7 @@ Local bases are accepted only when NemoClaw builds and pins them during the curr -Set `NEMOCLAW_LANGCHAIN_DEEPAGENTS_CODE_SANDBOX_BASE_IMAGE_REF` to a LangChain Deep Agents Code sandbox-base tag or digest to override base-image resolution during onboarding. +Set `NEMOCLAW_LANGCHAIN_DEEPAGENTS_CODE_SANDBOX_BASE_IMAGE_REF` to a LangChain Deep Agents Code sandbox-base tag or digest to override base-image resolution for a local Dockerfile workflow. NemoClaw requires environment overrides to use the official remote repository and resolve to an immutable repository digest, then validates the requested image against the manifest-required `deepagents-code` package version before using it. Local bases are accepted only when NemoClaw builds and pins them during the current operation. @@ -3828,7 +3890,7 @@ Local bases are accepted only when NemoClaw builds and pins them during the curr -Set `NEMOCLAW_HERMES_SANDBOX_BASE_IMAGE_REF` to a Hermes sandbox-base tag or digest to override base-image resolution during onboarding. +Set `NEMOCLAW_HERMES_SANDBOX_BASE_IMAGE_REF` to a Hermes sandbox-base tag or digest to override base-image resolution for a local Dockerfile workflow. NemoClaw requires environment overrides to use the official remote repository and resolve to an immutable repository digest, validates the requested image for the required MCP runtime, and keeps the final image bound to that trusted base. Local bases are accepted only when NemoClaw builds and pins them during the current operation. @@ -3846,7 +3908,7 @@ NEMOCLAW_TRACE_DIR=/tmp/nemoclaw-traces $$nemoclaw onboard NEMOCLAW_TRACE_FILE=/tmp/nemoclaw-onboard-trace.json $$nemoclaw onboard ``` -Trace artifacts include onboard phase timing, sandbox and service readiness waits, policy application, inference validation probes, curl probe results, and sandbox build progress events. +Trace artifacts include onboard phase timing, managed image resolution or local Dockerfile build progress, sandbox and service readiness waits, policy application, inference validation probes, and curl probe results. Secret-like metadata such as API keys, bearer tokens, cookies, and credentials is redacted before the file is written. @@ -3890,7 +3952,7 @@ NEMOCLAW_OPENCLAW_OTEL=1 $$nemoclaw onboard ``` Onboarding automatically applies the `openclaw-diagnostics-otel-local` preset at sandbox create and again during the policy step when `NEMOCLAW_OPENCLAW_OTEL=1`, so OTLP export is allowed before the gateway's first trace flush. -If you enabled OTEL after an existing sandbox was created, run `$$nemoclaw policy add openclaw-diagnostics-otel-local --yes` or recreate the sandbox with OTEL enabled at build time. +If you enabled OTEL after an existing sandbox was created, run `$$nemoclaw policy add openclaw-diagnostics-otel-local --yes` or recreate the sandbox with OTEL enabled in its startup configuration. Then open `http://localhost:16686` and select the `openclaw-gateway` service. The built-in `openclaw-diagnostics-otel-local` preset allows only `POST /v1/traces` (and subpaths) to `host.openshell.internal:4318` from `openclaw` and `node`. @@ -3923,7 +3985,7 @@ Set them before running `$$nemoclaw onboard` if a slow connection or large model |----------|---------|---------| | `NEMOCLAW_OLLAMA_PULL_TIMEOUT` | `1800` (30 minutes) | Wall-clock timeout for `ollama pull` during onboard, in seconds. Accepts integer or float values. Already-downloaded layers are kept; re-running the pull resumes them. | | `NEMOCLAW_LOCAL_INFERENCE_TIMEOUT` | `180` | Wall-clock timeout for the inference-server validation probe during onboard, in seconds. Raise on slow networks or for very large prompts. | -| `NEMOCLAW_SANDBOX_READY_TIMEOUT` | `180` | Wall-clock timeout for post-create readiness and OpenShell command re-registration after policy application, in seconds. Raise when the sandbox image build, gateway upload, in-sandbox boot, or post-policy re-registration exceeds the default (typical on 70B+ models, first-time gateway uploads over slow links, or DGX Station / remote-VM first runs). When the post-create deadline expires, onboarding deletes an orphaned sandbox and prints the retry hint. | +| `NEMOCLAW_SANDBOX_READY_TIMEOUT` | `180` | Wall-clock timeout for post-create readiness and OpenShell command re-registration after policy application, in seconds. Raise when managed image pull and startup, a local Dockerfile build and gateway upload, in-sandbox boot, or post-policy re-registration exceeds the default. When the post-create deadline expires, onboarding deletes an orphaned sandbox and prints the retry hint. | | `NEMOCLAW_SANDBOX_READY_ERROR_DEBOUNCE` | `30` | Consecutive `Error`-phase polls the post-create readiness wait tolerates before treating `Error` as terminal. Polling starts at 250ms and backs off to a 2-second cap, while `NEMOCLAW_SANDBOX_READY_TIMEOUT` remains the overall deadline. The gateway can briefly report a just-created sandbox in `Error` while it re-registers the sandbox (seen on DGX Spark); the debounce lets that transient recover to `Ready`. `Failed` and `CrashLoopBackOff` always fail immediately. Set to `1` to restore fast-fail on the first `Error` poll. | diff --git a/docs/reference/platform-support.mdx b/docs/reference/platform-support.mdx index e958d24f70..4b87b655f5 100644 --- a/docs/reference/platform-support.mdx +++ b/docs/reference/platform-support.mdx @@ -81,6 +81,7 @@ For install requirements and the shorter setup-oriented platform view, refer to | DGX OS (Spark) | Docker | Tested | P1 | Yes | Use the standard installer and `$$nemoclaw onboard`. For an end-to-end walkthrough with local inference, see the [NVIDIA Spark playbook](https://build.nvidia.com/spark/nemoclaw). | | DGX OS (Station) | Docker | Tested with limitations | P1 | No | The PRD marks this platform as P1. Physical validation on one DGX Station GB300 covers generic Ubuntu 24.04 ARM64, stock DGX OS `7.5.0`, the April 2026 NVIDIA Colossus BaseOS profile, and the June 2026 NVIDIA AI Developer Tools profile. A physical no-OTA DGX OS `7.6.0` host provided the release and hardware profile used for its stable workstation-family classifier and passed read-only eligibility and runtime-command preflight. Full Station Express end-to-end qualification for the accepted no-OTA DGX OS `7.6.x` profile is pending. The profile remains subject to the same physical GB300, driver, ECC, Docker, CDI, and container GPU validation. Clean-host end-to-end validation passed on generic Ubuntu and Colossus BaseOS; stock DGX OS and AI Developer Tools completed Station Express validation. The DGX OS `7.5.0` run used released OpenShell `0.0.85`, local Nemotron Ultra serving, sandbox `cuInit(0)`, and a Hermes write/read file-tool task. A dual-Station configuration has not been validated, and dedicated CI coverage is not available. Direct-GPU policies expose only the exact read-only BDF directory for each discovered display-class PCI device with NVIDIA vendor ID (`0x10de`) and GB300 device ID (`0x31c2` or `0x31c3`) plus required existing topology and module paths; they do not expose `/sys`, the PCI parent subtree, or sysfs write access. During physical validation, reads of `/sys/fs/cgroup/cgroup.controllers` and `/sys/class/net/lo/address` remained denied. For canonical hardware qualification, image requirements, preparation, repair limits, reboot handoff, and the explicit temporary metadata override, see [Prepare DGX Station to Install NemoClaw](../get-started/additional-setup/dgx-station-preparation). | | Linux | Docker | Tested | P0 | Yes | Primary tested path. Ubuntu 24.04 has host-level onboarding validation. A digest-pinned Ubuntu 26.04 userspace lane builds the CLI and runs preflight, installer, and platform contracts on eligible main pushes; Docker-host, AppArmor, Landlock, and live onboarding validation on 26.04 remain pending. Other distros (Ubuntu 22.04, Fedora, Rocky, Alma, NixOS, Arch) may work but are not validated. | +| Linux (Podman, amd64) | Podman 5+ (rootless) | Tested with limitations | P0 | No | Supported with limitations for OpenClaw, Hermes, and LangChain Deep Agents Code on Linux amd64. Requires rootless Podman 5 or later, cgroup v2, subordinate UID and GID ranges for the current user, and a responsive user Podman API socket. The first release supports CPU sandboxes with hosted inference only; GPU passthrough and host-local Ollama, NIM, or vLLM are not supported on this driver. Select Podman explicitly with `--compute-driver podman` or `NEMOCLAW_COMPUTE_DRIVER=podman`; Docker remains the default for new sandboxes. Podman onboarding requires a complete immutable managed OCI image, rejects `--from `, and does not fall back to a Dockerfile build or Docker daemon. NemoClaw persists the selected driver and exact managed runtime binding for status, rebuild, doctor, recovery, and `destroy --cleanup-gateway`. The explicit published-release Podman E2E lane must pass for all three agents before this row may claim dedicated CI qualification. | | macOS (Apple Silicon) | Colima, Docker Desktop | Tested with limitations | P0 | Yes | Start the container runtime (Colima or Docker Desktop) before running the installer. When Homebrew is available, OpenShell uses its official formula and the gateway appears in `brew services list` as `openshell`; without Homebrew, NemoClaw uses the standalone OpenShell install and detached gateway fallback. Homebrew Colima users must install both Colima and the Docker CLI (`brew install colima docker`) before `docker info` can work. Xcode Command Line Tools (`xcode-select --install`) are typically required for Node native modules during install. NemoClaw recommends them but does not enforce them during preflight. | | NVIDIA RTX (consumer and Pro workstation GPUs) | Docker | Deferred | P1 | No | The PRD marks this platform as P1. Covers RTX consumer cards and RTX Pro workstation cards on Linux hosts that meet the generic-Linux-GPU requirements (NVIDIA Container Toolkit + CDI present). The provider menu emits managed vLLM behind `NEMOCLAW_EXPERIMENTAL=1` or `NEMOCLAW_PROVIDER=install-vllm` for this host class today; the end-to-end onboard path on this hardware is not yet validated in CI. | | Windows WSL2 | Docker Desktop (WSL backend) | Tested with limitations | P1 | No | Requires WSL2 with Docker Desktop backend. | @@ -148,7 +149,8 @@ Pick the row that matches the target environment. {/* deployment-status:begin */} | Path | Status | Notes | |------|--------|-------| -| Local CLI onboard | Tested | Run `$$nemoclaw onboard` on a tested platform with Docker available locally. Primary path. | +| Local CLI onboard | Tested | Run `$$nemoclaw onboard` on a tested platform. Docker is the default local container runtime and remains the primary path. | +| Rootless Podman on Linux amd64 | Tested with limitations | Run `$$nemoclaw onboard --compute-driver podman` with Podman 5 or later, cgroup v2, subordinate UID/GID ranges, and an active user Podman API socket. Supports CPU-only OpenClaw, Hermes, and LangChain Deep Agents Code sandboxes from immutable managed OCI images; local Dockerfile builds, GPU passthrough, and host-local inference are not supported on this path. | | Headless Linux server | Tested with limitations | Provision a tested Linux host, connect over SSH, run the standard installer and `$$nemoclaw onboard`, and keep dashboards bound to loopback behind SSH port forwarding. Automatic recovery after a host reboot is not guaranteed; follow the documented manual recovery flow. | {/* deployment-status:end */} @@ -160,10 +162,10 @@ They are listed here so launch material, sales conversations, and support triage {/* out-of-scope:begin */} | Item | Status | Why | |------|--------|-----| -| Podman / other container runtimes | Unsupported | Onboard surfaces an explicit unsupported-runtime error for Podman (`src/lib/onboard/fatal-runtime-preflight.ts` prints the rejection; `src/lib/onboard/preflight.ts` flags the unsupported runtime upstream). Only Docker Engine, Docker Desktop, and Colima are supported. See issue #420 (closed). | +| Other container runtimes | Unsupported | Docker Engine, Docker Desktop, and Colima remain supported, and the qualified Linux amd64 rootless Podman path is documented separately. Other container runtimes and drivers are not yet supported launch claims. | | Intel Mac (macOS x86_64) | Unsupported | The supported OpenShell Homebrew gateway service path is Apple Silicon only, and OpenShell does not publish macOS x86_64 standalone gateway assets. The top-level installer rejects Intel Mac hosts before release-ref resolution or downloads (`install.sh:108`), and the OpenShell installer retains a downstream asset guard (`scripts/install-openshell.sh:689`). See issue #954 (closed). | | Non-Ubuntu/Debian Linux distros | Unsupported | Installer assumes `apt-get`. Fedora/Rocky/Alma/Arch/NixOS are not validated and the installer's package-manager probes do not cover them. See open issue #899 (Fedora hang). | -| Native Kubernetes or OpenShift deployments | Unsupported | NemoClaw runs the sandbox as a Docker container, not a Kubernetes pod. The default Docker-driver topology does not embed k3s. Operator-managed K8s/OpenShift deployments are out of scope; see issue #407 (community OpenShift through agent-sandbox CRD). | +| Native Kubernetes or OpenShift deployments | Unsupported | NemoClaw runs the sandbox as an OCI container through a supported local runtime driver, not as an operator-managed Kubernetes pod. The default Docker-driver topology does not embed k3s. Operator-managed K8s/OpenShift deployments are out of scope; see issue #407 (community OpenShift through agent-sandbox CRD). | | Air-gapped / offline installs | Unsupported | Onboard assumes network reachability for package fetches, container pulls, and provider validation. See open issues #4872 and #2218 (production-deployment epic covering air-gapped support, China network guidance, multi-host topology). | | Windows-on-ARM GPU passthrough | Unsupported | Windows-on-ARM CPU paths run under WSL2 'tested with limitations', but GPU passthrough on WOA is denylisted (`src/lib/onboard/wsl-docker-desktop-gpu.ts:188`, `src/lib/inference/gpu-trust.test.ts:70`). See closed issue #4565. | | Non-NVIDIA GPUs (AMD/ROCm, Intel Arc, Apple Metal) | Unsupported | Local vLLM and NIM paths assert NVIDIA CDI presence with `assertCdiNvidiaGpuSpecPresent` (`src/lib/onboard/fatal-runtime-preflight.ts`). NemoClaw does not install non-NVIDIA accelerator drivers. | diff --git a/docs/reference/troubleshooting.mdx b/docs/reference/troubleshooting.mdx index 6cad47f36f..feb7f4c0e2 100644 --- a/docs/reference/troubleshooting.mdx +++ b/docs/reference/troubleshooting.mdx @@ -62,11 +62,13 @@ nvm use 22 Then re-run the installer. -### Image push fails with out-of-memory errors +### Local Dockerfile image push fails with out-of-memory errors -The sandbox image is approximately 2.4 GB compressed. -During image push, the Docker daemon, k3s, and the OpenShell gateway run alongside the export pipeline, which buffers decompressed layers in memory. +This failure applies to the trusted Dockerfile fallback and explicit custom Dockerfile workflows. +Their locally built sandbox image can be approximately 2.4 GB compressed. +During local image push, the Docker daemon, k3s, and the OpenShell gateway run alongside the export pipeline, which buffers decompressed layers in memory. On machines with less than 8 GB of RAM, this combined usage can trigger the OOM killer. +Stock managed-image onboarding pulls an exact immutable image and does not run this local export pipeline. If you cannot add memory, configure at least 8 GB of swap to work around the issue at the cost of slower performance. @@ -148,7 +150,7 @@ To avoid these issues, install the prerequisites in the following order before r Homebrew Colima does not install the Docker CLI binary. If you install only Colima, `colima start` can succeed while later `docker` commands fail with `command not found`. -Install both packages, start Colima with enough resources for the sandbox image build, and verify Docker before onboarding: +Install both packages, start Colima with enough resources for the sandbox and any local Dockerfile fallback or custom build, and verify Docker before onboarding: ```bash brew install colima docker @@ -163,7 +165,8 @@ It installs Node.js via nvm and NemoClaw via npm, both into user-local directori The installer also handles OpenShell installation automatically using a pinned release. If you see permission errors during installation, they typically come from Docker, not the NemoClaw installer itself. -Docker must be installed and running before you run the installer, and installing Docker may require elevated privileges on Linux. +When you use the default Docker driver, Docker must be installed and running before you run the installer, and installing Docker may require elevated privileges on Linux. +The rootless Podman path instead requires the current user to own and reach the selected Podman API socket. ### npm install fails with permission errors @@ -192,11 +195,15 @@ Without this fix, sandbox pods fail DNS resolution against the in-cluster servic If the L4T version is not recognized, the setup step is skipped and the installer continues normally. -### DNS resolution from inside docker fails (corporate firewall) +### DNS resolution from inside Docker fails (corporate firewall) -Some corporate networks block outbound UDP port 53 to public DNS servers and force all host name resolution through DNS over TLS on TCP port 853. Containers do not inherit the host's DNS-over-TLS configuration, so the sandbox build's `npm ci` step times out trying to resolve `registry.npmjs.org` against `1.1.1.1` or `8.8.8.8`. +Some corporate networks block outbound UDP port 53 to public DNS servers and force all host name resolution through DNS over TLS on TCP port 853. +Containers do not inherit the host's DNS-over-TLS configuration. +This can block sandbox startup and, on a trusted fallback or explicit custom Dockerfile path, make dependency installation time out while resolving `registry.npmjs.org`. -NemoClaw's preflight runs a short `docker run --rm busybox nslookup nemoclaw-dns-probe-.invalid` probe before starting the sandbox build. The fresh `.invalid` name should return NXDOMAIN through a working resolver, so cached answers cannot hide blocked DNS egress. When the probe confirms a DNS failure, onboarding stops with platform-specific remediation instead of hanging for ~15 minutes and printing a cryptic `Exit handler never called`. +NemoClaw's preflight runs a short `docker run --rm busybox nslookup nemoclaw-dns-probe-.invalid` probe before it prepares the sandbox. +The fresh `.invalid` name should return NXDOMAIN through a working resolver, so cached answers cannot hide blocked DNS egress. +When the probe confirms a DNS failure, onboarding stops with platform-specific remediation before sandbox creation. Use the preflight headline to choose the recovery path: @@ -398,14 +405,22 @@ Do not guess or copy a binding from another sandbox because lifecycle commands u Older NemoClaw releases relied on a Docker cgroup workaround on Ubuntu 24.04, DGX Spark, and WSL2. Current OpenShell releases handle that behavior themselves, so NemoClaw no longer requires a Spark-specific setup step. -If onboarding reports that Docker is missing or unreachable, fix Docker first and retry onboarding: +If onboarding selected the default Docker driver and reports that Docker is missing or unreachable, fix Docker first and retry onboarding: ```bash $$nemoclaw onboard ``` -Podman is not a tested runtime. -If onboarding or sandbox lifecycle fails, switch to a tested runtime (Docker Desktop, Colima, or Docker Engine) and rerun onboarding. +If onboarding selected Podman and reports a cgroup error, confirm the host uses cgroup v2 and restart the rootless user socket: + +```bash +test "$(stat -fc %T /sys/fs/cgroup)" = cgroup2fs +systemctl --user enable --now podman.socket +podman --url "unix://${XDG_RUNTIME_DIR:-/run/user/$(id -u)}/podman/podman.sock" info +``` + +Podman support requires Linux amd64, rootless Podman 5 or later, and subordinate UID and GID ranges. +It does not use the older Docker cgroup workaround. ### Cluster fails with `overlayfs snapshotter cannot be enabled` on Docker 26+ @@ -536,7 +551,8 @@ If it does not, rerun onboarding with `NEMOCLAW_WEB_SEARCH_PROVIDER=tavily` and -Rerunning onboarding with a different provider recreates the sandbox because the provider configuration and credential attachment are build-time inputs. +Rerunning onboarding with a different provider recreates the sandbox because provider configuration and credential attachment are sandbox-creation inputs. +Managed images receive the provider configuration through a secret-free startup profile, while a local Dockerfile path applies the same configuration during its build. NemoClaw validates the replacement key before it removes the existing sandbox, then backs up and restores the supported workspace state during recreation. If the configuration is correct but the egress probe fails, keep the matching preset applied and inspect the blocked request with `openshell term` before widening any policy rule. @@ -678,15 +694,19 @@ If neither is found, verify that Colima is running: colima status ``` -### Sandbox build is slow or hangs (under-provisioned container runtime) +### Sandbox startup or local Dockerfile build is slow (under-provisioned container runtime) -Default Colima ships with 2 vCPU and 2 GiB of memory, which is not enough headroom for the BuildKit-driven sandbox image build. -On macOS Apple Silicon, the build can stall part-way through with no progress and no error, leaving the wizard waiting indefinitely. +Default Colima ships with 2 vCPU and 2 GiB of memory, which is not enough headroom for a NemoClaw sandbox. +Stock onboarding on a supported `amd64` Docker host prefers an exact managed image and avoids a local Dockerfile build. +A trusted Dockerfile fallback or explicit custom Dockerfile can still stall part-way through a local BuildKit build on an under-provisioned runtime. Preflight inspects `docker info` for `NCPU` and `MemTotal` and prints a warning when the runtime falls below 4 vCPU or 8 GiB. -In interactive onboarding, the warning prompt defaults to abort, so pressing Enter stops the run before the sandbox build reaches the likely stall point. +In interactive onboarding, the warning prompt defaults to abort, so pressing Enter stops the run before sandbox preparation reaches the likely failure point. Type `y` only when you intentionally want to continue on the smaller runtime. Non-interactive onboarding prints the warning and continues. +The configuration review shows a local build-time estimate only for an explicit custom Dockerfile. +If NemoClaw selects the trusted Dockerfile fallback later during sandbox creation, it prints the fallback notice and build estimate at that point. +Managed-image launch does not show a Dockerfile build estimate. On Colima, raise the resources before re-running onboard: ```bash @@ -1588,8 +1608,7 @@ that these backends do not emit. For the compatible-endpoint provider, NemoClaw now defaults to `/v1/chat/completions` and skips the Responses API probe entirely unless you opt in. -If you onboarded an older release that selected `/v1/responses`, re-run -onboarding so the wizard rebuilds the image with chat completions: +If you onboarded an older release that selected `/v1/responses`, rerun onboarding so the replacement sandbox starts with Chat Completions: ```bash $$nemoclaw onboard @@ -1602,8 +1621,8 @@ When you enable Telegram messaging with an OpenAI-compatible endpoint, onboardin If that smoke check fails, fix the compatible-endpoint base URL, credentials, model, or network route before testing the Telegram bot again. Do not rely on `NEMOCLAW_INFERENCE_API_OVERRIDE` alone. -It patches the config at container startup but does not update the Dockerfile ARG baked into the image. -A fresh `$$nemoclaw onboard` is the reliable fix. +It patches the running config but does not update the recorded startup configuration. +A fresh `$$nemoclaw onboard` regenerates that configuration for either the managed-image or local Dockerfile path. @@ -1674,10 +1693,11 @@ Streaming defects then surface at runtime instead of during onboarding. ### `NEMOCLAW_DISABLE_DEVICE_AUTH=1` does not change an existing sandbox This is expected behavior. -`NEMOCLAW_DISABLE_DEVICE_AUTH` is a build-time setting used when NemoClaw creates the sandbox image. -Changing or exporting it later does not rewrite the baked `openclaw.json` inside an existing sandbox. +`NEMOCLAW_DISABLE_DEVICE_AUTH` is a sandbox-creation setting. +Managed-image onboarding includes it in the secret-free startup profile, while the trusted Dockerfile fallback applies it during the local build. +Changing or exporting it later does not rewrite `openclaw.json` inside an existing sandbox. -If you need a different device-auth setting, rerun onboarding so NemoClaw rebuilds the sandbox image with the desired configuration. +If you need a different device-auth setting, rerun onboarding so NemoClaw recreates the sandbox with the desired configuration. For the security trade-offs, refer to [Security Best Practices](../security/best-practices). ### `openclaw.json` is empty after changing inference @@ -1743,7 +1763,9 @@ To discard the quarantined settings, upgrade the CLI and run `$$nemoclaw ### `openclaw channels add` or `remove` is blocked inside the sandbox This is expected. -The messaging channel list is frozen into the sandbox's container image when the image is built during `$$nemoclaw onboard` or `$$nemoclaw rebuild` (the selected channel names are passed to the `docker build` as `NEMOCLAW_MESSAGING_CHANNELS_B64` and written into agent config such as `/sandbox/.openclaw/openclaw.json` for OpenClaw or `/sandbox/.hermes/.env` for Hermes). +The messaging channel list is materialized when the sandbox starts during `$$nemoclaw onboard` or `$$nemoclaw rebuild`. +Managed-image onboarding carries the selected channel names in the startup profile, while a local Dockerfile build uses its build arguments. +The startup path writes agent config such as `/sandbox/.openclaw/openclaw.json` for OpenClaw or `/sandbox/.hermes/.env` for Hermes. Changes made inside the running sandbox do not persist across rebuilds, so `openclaw channels` commands that mutate the config are intercepted. NemoClaw's sandbox entrypoint installs a guard that intercepts `openclaw channels ` and prints an actionable error pointing at the host-side commands below, instead of letting the call fail deep in the binary with a raw `EACCES` trace. @@ -1756,7 +1778,7 @@ $$nemoclaw channels remove ``` `channels add` registers credentials with the OpenShell gateway and `channels remove` clears them. -Both offer to rebuild the sandbox so the image reflects the new channel set. +Both offer to rebuild the sandbox so its startup configuration reflects the new channel set. In non-interactive mode (`NEMOCLAW_NON_INTERACTIVE=1`), the commands stage the change and leave the rebuild to a follow-up `$$nemoclaw rebuild`. WeChat and WhatsApp are experimental. Review [Choose Messaging Channels](../manage-sandboxes/messaging-channels/choose-messaging-channels) before enabling them. @@ -1816,15 +1838,15 @@ Host-side `config set` validates any HTTP or HTTPS URLs in the new value, includ ### `openclaw doctor --fix` cannot repair Discord channel config inside the sandbox This is expected in NemoClaw-managed sandboxes. -NemoClaw bakes channel entries into `/sandbox/.openclaw/openclaw.json` at image build time. +NemoClaw materializes channel entries in `/sandbox/.openclaw/openclaw.json` when the sandbox starts. -As a result, commands that try to rewrite the baked config from inside the sandbox, including `openclaw doctor --fix`, cannot repair Discord, Telegram, or Slack channel entries in place. +As a result, commands that try to rewrite the managed config from inside the sandbox, including `openclaw doctor --fix`, cannot repair Discord, Telegram, or Slack channel entries in place. -If your Discord channel config is wrong, rerun onboarding so NemoClaw rebuilds the sandbox image with the correct messaging selection. +If your Discord channel config is wrong, rerun onboarding so NemoClaw recreates the sandbox with the correct messaging selection. Do not treat a failed `doctor --fix` run as proof that the Discord gateway path itself is broken. If `openclaw doctor` reports that it moved Telegram single-account values under `channels.telegram.accounts.default`, rerun onboarding and rebuild the sandbox rather than trying to patch `openclaw.json` in place. -Current NemoClaw rebuilds bake Telegram in the account-based layout and set Telegram group chats to `groupPolicy: open`, which avoids the empty `groupAllowFrom` warning path for default group-chat access. +Current NemoClaw rebuilds materialize Telegram in the account-based layout and set Telegram group chats to `groupPolicy: open`, which avoids the empty `groupAllowFrom` warning path for default group-chat access. ### `openclaw doctor --fix` tightened config permissions and the gateway can no longer save config @@ -1875,7 +1897,7 @@ A NemoClaw sandbox has two intentional permission states for `/sandbox/.openclaw Separate the problem into two parts: -1. Baked config and provider wiring +1. Managed config and provider wiring Check that onboarding selected Discord and that the sandbox was created with the Discord messaging provider attached. If Discord was skipped during onboarding, rerun onboarding and select Discord again. @@ -1884,7 +1906,7 @@ Separate the problem into two parts: Successful login alone does not prove that Discord works end to end. Discord also needs a working gateway connection to `gateway.discord.gg`. - If logs show errors such as `getaddrinfo EAI_AGAIN gateway.discord.gg`, repeated reconnect loops, or a `400` response while probing the gateway path, the problem is usually in the native gateway/proxy path rather than in the baked config. + If logs show errors such as `getaddrinfo EAI_AGAIN gateway.discord.gg`, repeated reconnect loops, or a `400` response while probing the gateway path, the problem is usually in the native gateway/proxy path rather than in the managed config. Common signs of a native gateway-path failure: @@ -2051,8 +2073,9 @@ export NEMOCLAW_PROXY_PORT=8080 $$nemoclaw onboard ``` -These are build-time settings baked into the sandbox image. -Changing them after onboarding requires re-running `$$nemoclaw onboard` to rebuild the image. +These are sandbox-startup settings. +Managed-image onboarding includes them in the startup profile, while a local Dockerfile path applies them through that build. +Changing them after onboarding requires rerunning `$$nemoclaw onboard` to recreate the sandbox. When `HTTP_PROXY` or `HTTPS_PROXY` is set on the host, NemoClaw adds `localhost`, `127.0.0.1`, `::1`, `0.0.0.0`, the container-host aliases `host.docker.internal` and `host.containers.internal`, and the managed inference hostname `inference.local` to `NO_PROXY` for host-side subprocesses and for the env forwarded into `openshell sandbox create`. This keeps local Ollama health checks, model pulls, and managed inference traffic from being chained through a corporate or desktop proxy at the sandbox-create boundary, while preserving the proxy for external hosts. @@ -2662,9 +2685,78 @@ For first-time Hermes setup, refer to [Quickstart with Hermes](../get-started/qu ## Podman -Podman is not a tested runtime. -OpenShell officially documents Docker-based runtimes only. -If you encounter issues with Podman, switch to a tested runtime (Docker Engine, Docker Desktop, or Colima) and rerun onboarding. +NemoClaw supports rootless Podman 5 or later for CPU-only OpenClaw, Hermes, and LangChain Deep Agents Code sandboxes on Linux amd64. +Docker remains the default for new sandboxes. +Select Podman explicitly with `--compute-driver podman` or `NEMOCLAW_COMPUTE_DRIVER=podman`. + +### Podman preflight fails + +Verify the complete first-release host contract: + +```bash +podman --version +test "$(uname -s)" = Linux +test "$(uname -m)" = x86_64 +test "$(stat -fc %T /sys/fs/cgroup)" = cgroup2fs +podman unshare cat /proc/self/uid_map +podman unshare cat /proc/self/gid_map +systemctl --user enable --now podman.socket +test -S "${XDG_RUNTIME_DIR:-/run/user/$(id -u)}/podman/podman.sock" +podman --url "unix://${XDG_RUNTIME_DIR:-/run/user/$(id -u)}/podman/podman.sock" info +``` + +The UID and GID maps must each contain a subordinate range longer than one ID. +If either map does not, configure the current user in `/etc/subuid` and `/etc/subgid`, log in again if required by your distribution, and restart `podman.socket`. +For a nonstandard rootless socket, set `OPENSHELL_PODMAN_SOCKET` to its absolute path for the first onboarding run. + +### Podman onboarding tries to use a Dockerfile or local inference + +The Podman driver is buildless. +It requires the complete immutable managed OCI image for the selected agent and fails closed when the catalog, cohort, digest, or registry pull cannot be verified. +It rejects `--from ` and never falls back to a local Dockerfile build or Docker daemon. + +The first release supports CPU sandboxes with hosted inference only. +Remove `--gpu` or `--sandbox-gpu`, use `--no-gpu`, and select a hosted inference provider. +Host-local Ollama, NIM, and vLLM are not supported on the Podman driver. +These limits apply equally to OpenClaw, Hermes, and LangChain Deep Agents Code. + +### Recover a Podman sandbox after a reboot + +Start the rootless Podman API socket, then let NemoClaw recover the exact persisted runtime: + +```bash +systemctl --user start podman.socket +$$nemoclaw my-assistant status +$$nemoclaw my-assistant doctor --json +``` + +NemoClaw records the selected driver, socket, network, and managed gateway runtime during onboarding. +Later status, rebuild, doctor, and destroy operations use that protected binding even in a fresh shell. +You do not need to re-export `OPENSHELL_PODMAN_SOCKET` when the recorded socket still exists. +If an ambient socket override conflicts with the recorded binding, NemoClaw stops instead of switching runtimes or falling back to Docker. +Restore the original user socket path, run `doctor --json`, and retry. + +To recreate the agent sandbox while preserving its recorded Podman driver, use: + +```bash +$$nemoclaw my-assistant rebuild --yes +``` + +To remove the final sandbox and its NemoClaw-managed Podman gateway state, use: + +```bash +$$nemoclaw my-assistant destroy --yes --cleanup-gateway +``` + +The cleanup validates the exact persisted Podman binding before removing the gateway, network, and protected runtime state. +If cleanup cannot prove ownership, it fails closed and retains the binding for diagnosis. + +### A Podman sandbox reports Docker runtime state + +A sandbox created with Podman must retain `podman` as its persisted compute driver. +Run `$$nemoclaw my-assistant doctor --json` and inspect the compute-driver and gateway checks. +Do not delete or rewrite the registry to force a driver change. +Re-onboard under a new sandbox name if you intentionally need the default Docker path. diff --git a/docs/security/configure-corporate-ca-trust.mdx b/docs/security/configure-corporate-ca-trust.mdx index 0f0291aa0d..58598cfbe4 100644 --- a/docs/security/configure-corporate-ca-trust.mdx +++ b/docs/security/configure-corporate-ca-trust.mdx @@ -3,8 +3,8 @@ # SPDX-License-Identifier: Apache-2.0 title: "Configure Corporate Certificate Authority Trust" sidebar-title: "Configure Corporate CA Trust" -description: "Import a bounded corporate proxy CA chain into NemoClaw sandbox runtime trust and supported build-time flows." -description-agent: "Configures corporate proxy CA trust for runtime TLS and supported build-time operations. Use when a corporate MITM proxy re-signs external TLS or certificate verification fails behind an enterprise proxy." +description: "Import a bounded corporate proxy CA chain for managed image resolution, sandbox runtime trust, and supported local builds." +description-agent: "Configures corporate proxy CA trust for managed image resolution, runtime TLS, and supported local builds. Use when a corporate MITM proxy re-signs external TLS or certificate verification fails behind an enterprise proxy." keywords: ["nemoclaw corporate ca", "corporate proxy tls", "certificate verification"] content: type: "how_to" @@ -28,18 +28,28 @@ Use `$$nemoclaw rebuild` after adding or changing the CA for an existing ## Understand Image and Runtime Trust -NemoClaw validates the selected bundle and bakes it into the sandbox image as `NEMOCLAW_CORPORATE_CA_B64`. -The managed Dockerfile decodes it to the root-owned, read-only file `/usr/local/share/nemoclaw/corporate-ca.pem`. +NemoClaw validates the selected bundle before it prepares the sandbox workload. +Managed release-image catalog requests use the validated corporate CA with the public root set and configured HTTP(S) proxy environment. +The request-scoped registry transport does not replace the process-wide network dispatcher. + +For a managed release image, NemoClaw passes the bounded CA bundle separately when the sandbox starts. +The secret-free startup profile contains only the bundle's SHA-256 digest. +The root-owned startup applicator verifies that digest and installs `/usr/local/share/nemoclaw/corporate-ca.pem` as a read-only file. + +For the trusted Dockerfile fallback, NemoClaw encodes the bundle as `NEMOCLAW_CORPORATE_CA_B64`. +The canonical Dockerfile decodes it to the same root-owned, read-only file. -It sets `NODE_EXTRA_CA_CERTS` before build-time Node.js dependency verification, including `npm audit signatures` requests that cross a TLS-inspecting proxy. +The OpenClaw Dockerfile sets `NODE_EXTRA_CA_CERTS` before build-time Node.js dependency verification. +This includes `npm audit signatures` requests that cross a TLS-inspecting proxy. -The Hermes Dockerfile decodes the bundle after its managed build-time dependency steps, so the corporate CA does not apply to those earlier operations. +The Hermes Dockerfile decodes the bundle after its local build-time dependency steps. +The corporate CA does not apply to those earlier operations. @@ -65,16 +75,15 @@ A corporate root that exists only as a hand-edited entry in a merged trust store ## Verify the Selected Source -Check onboarding build output for the source and path that NemoClaw selected. -A successful selection logs `baking corporate proxy CA from ...` without printing certificate contents. +For a managed release image, sandbox startup verifies the CA digest before the agent launches. +For the Dockerfile fallback, build output logs `baking corporate proxy CA from ...` without printing certificate contents. If a conventional variable points at a missing or invalid file, onboarding logs that the source was skipped for corporate CA import. An administrator anchor directory that contains candidates but no valid CA logs a similar warning. -If no selection message appears, no source passed validation. ## Use a Custom Dockerfile -Managed NemoClaw Dockerfiles implement the corporate CA build contract automatically. +The trusted canonical Dockerfile fallback implements the corporate CA build contract automatically. A custom Dockerfile must declare `ARG NEMOCLAW_CORPORATE_CA_B64` and decode it into `/usr/local/share/nemoclaw/corporate-ca.pem` for runtime trust. If its build needs network access behind the corporate proxy, establish build-time trust before the dependency operations that require TLS. diff --git a/install.sh b/install.sh index add4c49fb8..e12a1e43d9 100755 --- a/install.sh +++ b/install.sh @@ -131,6 +131,8 @@ bootstrap_usage() { printf " --station-deepseek Use DeepSeek V4 Flash for DGX Station express install\n" printf " --yes-i-accept-third-party-software Accept the third-party software notice without prompting\n" printf " --fresh Discard any failed/interrupted onboarding session and start over\n" + printf " --compute-driver \n" + printf " Container runtime for OpenShell sandboxes (default: auto)\n" printf " --version, -v Print installer version and exit\n" printf " --help, -h Show this help message and exit\n\n" printf " Environment:\n" @@ -140,6 +142,7 @@ bootstrap_usage() { printf " Example: curl -fsSL https://www.nvidia.com/nemoclaw.sh | NEMOCLAW_INSTALL_TAG=%s bash\n" "$INSTALL_TAG_EXAMPLE" printf " NEMOCLAW_NON_INTERACTIVE=1 Same as --non-interactive\n" printf " NEMOCLAW_FRESH=1 Same as --fresh\n" + printf " NEMOCLAW_COMPUTE_DRIVER auto | docker | podman (flag takes precedence)\n" printf " NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 Same as --yes-i-accept-third-party-software\n" printf " NEMOCLAW_NO_EXPRESS=1 Skip express install prompt on supported platforms\n" printf " NEMOCLAW_SANDBOX_NAME Sandbox name to create/use\n" diff --git a/nemoclaw-blueprint/policies/openclaw-sandbox-permissive.yaml b/nemoclaw-blueprint/policies/openclaw-sandbox-permissive.yaml index 6dd338464d..f34ac3efd7 100644 --- a/nemoclaw-blueprint/policies/openclaw-sandbox-permissive.yaml +++ b/nemoclaw-blueprint/policies/openclaw-sandbox-permissive.yaml @@ -25,6 +25,8 @@ filesystem_policy: - /app - /etc - /var/log + - /run/nemoclaw # Mirrors the managed-startup handoff grant + # in the baseline create policy. read_write: - /tmp - /dev/null diff --git a/nemoclaw-blueprint/policies/openclaw-sandbox.yaml b/nemoclaw-blueprint/policies/openclaw-sandbox.yaml index 23700e740e..e0256fd8a4 100644 --- a/nemoclaw-blueprint/policies/openclaw-sandbox.yaml +++ b/nemoclaw-blueprint/policies/openclaw-sandbox.yaml @@ -28,6 +28,9 @@ filesystem_policy: - /app - /etc - /var/log + - /run/nemoclaw # Root-owned managed-startup handoff. Landlock + # grants reads only; file DAC still protects + # private runtime material in this directory. read_write: - /tmp - /dev/null diff --git a/package-lock.json b/package-lock.json index 3b7137d256..a3bf6c6e29 100644 --- a/package-lock.json +++ b/package-lock.json @@ -21,6 +21,7 @@ "p-retry": "^4.6.2", "qrcode-terminal": "^0.12.0", "smol-toml": "1.7.0", + "undici": "8.5.0", "yaml": "2.8.3" }, "bin": { @@ -40,7 +41,6 @@ "tsx": "^4.21.0", "typebox": "1.1.38", "typescript": "6.0.3", - "undici": "8.5.0", "vitest": "^4.1.9" }, "engines": { @@ -6811,7 +6811,6 @@ "version": "8.5.0", "resolved": "https://registry.npmjs.org/undici/-/undici-8.5.0.tgz", "integrity": "sha512-xamtWoB1EshgjpmlXd7GGm2VfdDtw1+rD8uhry8pSNW3If6S8E0m2T2+orSKeZXEn/aPJMviCpDBA65WJt8zhg==", - "dev": true, "license": "MIT", "engines": { "node": ">=22.19.0" diff --git a/package.json b/package.json index 1c1d368976..e1e916c559 100644 --- a/package.json +++ b/package.json @@ -98,6 +98,7 @@ "p-retry": "^4.6.2", "qrcode-terminal": "^0.12.0", "smol-toml": "1.7.0", + "undici": "8.5.0", "yaml": "2.8.3" }, "bundleDependencies": [ @@ -138,7 +139,6 @@ "tsx": "^4.21.0", "typebox": "1.1.38", "typescript": "6.0.3", - "undici": "8.5.0", "vitest": "^4.1.9" } } diff --git a/scripts/check-dcode-profile-import-gate.sh b/scripts/check-dcode-profile-import-gate.sh index b65b31b8be..a8bb7b19d0 100755 --- a/scripts/check-dcode-profile-import-gate.sh +++ b/scripts/check-dcode-profile-import-gate.sh @@ -38,7 +38,7 @@ for dockerfile in \ agents/langchain-deepagents-code/Dockerfile; do while IFS= read -r arg_name; do case "${arg_name}" in - BASE_IMAGE | NEMOCLAW_CORPORATE_CA_B64 | NEMOCLAW_MODEL | NEMOCLAW_INFERENCE_PROVIDER_ID | NEMOCLAW_PROVIDER_KEY | NEMOCLAW_UPSTREAM_PROVIDER | NEMOCLAW_UPSTREAM_ENDPOINT_URL | NEMOCLAW_INFERENCE_BASE_URL | NEMOCLAW_INFERENCE_API | NEMOCLAW_TOOL_DISCLOSURE | NEMOCLAW_DCODE_AUTO_APPROVAL | NEMOCLAW_BUILD_ID | NEMOCLAW_DARWIN_VM_COMPAT | NEMOCLAW_PROXY_HOST | NEMOCLAW_PROXY_PORT) ;; + BASE_IMAGE | NEMOCLAW_CORPORATE_CA_B64 | NEMOCLAW_MODEL | NEMOCLAW_INFERENCE_PROVIDER_ID | NEMOCLAW_PROVIDER_KEY | NEMOCLAW_UPSTREAM_PROVIDER | NEMOCLAW_UPSTREAM_ENDPOINT_URL | NEMOCLAW_INFERENCE_BASE_URL | NEMOCLAW_INFERENCE_API | NEMOCLAW_TOOL_DISCLOSURE | NEMOCLAW_DCODE_AUTO_APPROVAL | NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION | NEMOCLAW_MANAGED_IMAGE_RUNTIME_USER | NEMOCLAW_BUILD_ID | NEMOCLAW_DARWIN_VM_COMPAT | NEMOCLAW_PROXY_HOST | NEMOCLAW_PROXY_PORT) ;; *) echo "ERROR: plain-progress build refuses unreviewed ARG ${arg_name} in ${dockerfile}" >&2 exit 1 diff --git a/scripts/checks/export-managed-image-failure-diagnostics.ts b/scripts/checks/export-managed-image-failure-diagnostics.ts new file mode 100644 index 0000000000..92ecca712b --- /dev/null +++ b/scripts/checks/export-managed-image-failure-diagnostics.ts @@ -0,0 +1,250 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; + +import { createDockerGpuDiagnosticRedactor } from "../../src/lib/onboard/docker-gpu-diagnostic-redaction.ts"; +import { isCredentialField } from "../../src/lib/security/credential-filter.ts"; + +const EXPORTED_DIAGNOSTIC_FILES = new Set([ + // Docker patch diagnostics redact logs at their source boundary after + // learning opaque credential values from the exact inspected containers. + // Redact them again here before they cross the CI artifact boundary. + "docker-logs.txt", + "openshell-gateway-relevant.log", + "openshell-gateway-tail.log", + "rootfs-console.log", + "summary.txt", +]); + +export const MANAGED_IMAGE_DIAGNOSTIC_EXPORT_LIMITS = { + maxBundles: 4, + maxFiles: 16, + maxOutputFileBytes: 32 * 1024, + maxSourceFileBytes: 64 * 1024, + maxTotalOutputBytes: 128 * 1024, +} as const; + +export type ManagedImageDiagnosticExportOptions = { + readonly env?: NodeJS.ProcessEnv; + readonly outputRoot: string; + readonly sourceRoot: string; +}; + +export type ManagedImageDiagnosticExportResult = { + readonly bundles: number; + readonly bytes: number; + readonly files: number; +}; + +type PreparedFile = { + readonly bundle: number; + readonly contents: string; + readonly name: string; +}; + +function requiredArgument(argv: readonly string[], name: string): string { + const index = argv.indexOf(name); + const value = index >= 0 ? argv[index + 1] : undefined; + if (!value || value.startsWith("--")) throw new Error(`${name} is required`); + return value; +} + +function parseArguments( + argv: readonly string[], +): Pick { + if (argv.length !== 4) { + throw new Error("usage: --source --output "); + } + return { + sourceRoot: requiredArgument(argv, "--source"), + outputRoot: requiredArgument(argv, "--output"), + }; +} + +function assertPlainDirectory(directory: string, label: string): void { + const stat = fs.lstatSync(directory); + if (stat.isSymbolicLink() || !stat.isDirectory()) { + throw new Error(`${label} must be a real directory`); + } +} + +function assertOutputIsSeparate(sourceRoot: string, outputRoot: string): void { + const source = path.resolve(sourceRoot); + const output = path.resolve(outputRoot); + if ( + source === output || + output.startsWith(`${source}${path.sep}`) || + source.startsWith(`${output}${path.sep}`) + ) { + throw new Error("diagnostic source and sanitized output must be separate"); + } +} + +function knownSecretValues(env: NodeJS.ProcessEnv): string[] { + return [ + ...new Set( + Object.entries(env) + .filter(([name, value]) => Boolean(value) && isCredentialField(name)) + .map(([, value]) => value as string) + // Avoid treating a short runner setting as a global text replacement. + // Production credentials and the canaries used by this exporter are + // substantially longer than this lower bound. + .filter((value) => Buffer.byteLength(value, "utf8") >= 8), + ), + ].sort((left, right) => right.length - left.length); +} + +function truncateRedactedText(text: string, maxBytes: number): string { + const encoded = Buffer.from(text, "utf8"); + if (encoded.byteLength <= maxBytes) return text; + + const marker = Buffer.from("\n[truncated after redaction]\n", "utf8"); + const utf8Prefix = (value: Buffer, byteLimit: number): Buffer => { + let end = Math.min(value.byteLength, byteLimit); + while (end > 0 && end < value.byteLength && (value[end] & 0xc0) === 0x80) end--; + return value.subarray(0, end); + }; + if (maxBytes <= marker.byteLength) return utf8Prefix(marker, maxBytes).toString("utf8"); + return Buffer.concat([utf8Prefix(encoded, maxBytes - marker.byteLength), marker]).toString( + "utf8", + ); +} + +function readRegularFileNoFollow(filePath: string): string | null { + const before = fs.lstatSync(filePath); + if (before.isSymbolicLink() || !before.isFile()) { + throw new Error(`diagnostic entry is not a regular file: ${filePath}`); + } + if (before.size > MANAGED_IMAGE_DIAGNOSTIC_EXPORT_LIMITS.maxSourceFileBytes) return null; + + const noFollow = typeof fs.constants.O_NOFOLLOW === "number" ? fs.constants.O_NOFOLLOW : 0; + const descriptor = fs.openSync(filePath, fs.constants.O_RDONLY | noFollow); + try { + const opened = fs.fstatSync(descriptor); + if ( + !opened.isFile() || + opened.dev !== before.dev || + opened.ino !== before.ino || + opened.size !== before.size + ) { + throw new Error(`diagnostic entry changed while being read: ${filePath}`); + } + return fs.readFileSync(descriptor, "utf8"); + } finally { + fs.closeSync(descriptor); + } +} + +function prepareFiles( + sourceRoot: string, + env: NodeJS.ProcessEnv, +): { bundles: number; files: PreparedFile[] } { + if (!fs.existsSync(sourceRoot)) return { bundles: 0, files: [] }; + assertPlainDirectory(sourceRoot, "diagnostic source"); + + const rootEntries = fs.readdirSync(sourceRoot, { withFileTypes: true }); + for (const entry of rootEntries) { + const entryPath = path.join(sourceRoot, entry.name); + const stat = fs.lstatSync(entryPath); + if (stat.isSymbolicLink() || !stat.isDirectory()) { + throw new Error(`diagnostic bundle entry is not a real directory: ${entryPath}`); + } + } + + const selectedBundles = rootEntries + .map((entry) => entry.name) + .sort((left, right) => right.localeCompare(left)) + .slice(0, MANAGED_IMAGE_DIAGNOSTIC_EXPORT_LIMITS.maxBundles); + const redactText = createDockerGpuDiagnosticRedactor(knownSecretValues(env)).redactText; + const prepared: PreparedFile[] = []; + let totalBytes = 0; + + for (const [bundleIndex, bundleName] of selectedBundles.entries()) { + const bundlePath = path.join(sourceRoot, bundleName); + const entries = fs.readdirSync(bundlePath, { withFileTypes: true }); + for (const entry of entries) { + const sourcePath = path.join(bundlePath, entry.name); + const stat = fs.lstatSync(sourcePath); + if (stat.isSymbolicLink() || !stat.isFile()) { + throw new Error(`diagnostic entry is not a regular file: ${sourcePath}`); + } + } + + for (const name of [...EXPORTED_DIAGNOSTIC_FILES].sort()) { + if (prepared.length >= MANAGED_IMAGE_DIAGNOSTIC_EXPORT_LIMITS.maxFiles) break; + const sourcePath = path.join(bundlePath, name); + if (!fs.existsSync(sourcePath)) continue; + + const raw = readRegularFileNoFollow(sourcePath); + const redacted = + raw === null + ? `[omitted: source exceeded ${MANAGED_IMAGE_DIAGNOSTIC_EXPORT_LIMITS.maxSourceFileBytes} bytes]\n` + : redactText(raw); + let contents = truncateRedactedText( + redacted, + MANAGED_IMAGE_DIAGNOSTIC_EXPORT_LIMITS.maxOutputFileBytes, + ); + const remaining = MANAGED_IMAGE_DIAGNOSTIC_EXPORT_LIMITS.maxTotalOutputBytes - totalBytes; + if (remaining <= 0) break; + contents = truncateRedactedText(contents, remaining); + const byteLength = Buffer.byteLength(contents, "utf8"); + if (byteLength === 0) break; + prepared.push({ bundle: bundleIndex + 1, contents, name }); + totalBytes += byteLength; + } + if ( + prepared.length >= MANAGED_IMAGE_DIAGNOSTIC_EXPORT_LIMITS.maxFiles || + totalBytes >= MANAGED_IMAGE_DIAGNOSTIC_EXPORT_LIMITS.maxTotalOutputBytes + ) { + break; + } + } + + return { bundles: new Set(prepared.map((file) => file.bundle)).size, files: prepared }; +} + +export function exportManagedImageFailureDiagnostics( + options: ManagedImageDiagnosticExportOptions, +): ManagedImageDiagnosticExportResult { + assertOutputIsSeparate(options.sourceRoot, options.outputRoot); + assertPlainDirectory(options.outputRoot, "sanitized diagnostic output"); + if (fs.readdirSync(options.outputRoot).length !== 0) { + throw new Error("sanitized diagnostic output must be empty"); + } + + const prepared = prepareFiles(options.sourceRoot, options.env ?? process.env); + let bytes = 0; + for (const file of prepared.files) { + const bundlePath = path.join( + options.outputRoot, + `bundle-${String(file.bundle).padStart(2, "0")}`, + ); + fs.mkdirSync(bundlePath, { recursive: true, mode: 0o700 }); + const destination = path.join(bundlePath, file.name); + fs.writeFileSync(destination, file.contents, { encoding: "utf8", mode: 0o600 }); + bytes += Buffer.byteLength(file.contents, "utf8"); + } + return { bundles: prepared.bundles, bytes, files: prepared.files.length }; +} + +function isMainModule(): boolean { + const entrypoint = process.argv[1]; + return Boolean(entrypoint) && pathToFileURL(path.resolve(entrypoint)).href === import.meta.url; +} + +if (isMainModule()) { + try { + const result = exportManagedImageFailureDiagnostics(parseArguments(process.argv.slice(2))); + process.stdout.write( + `sanitized_bundles=${result.bundles} sanitized_files=${result.files} sanitized_bytes=${result.bytes}\n`, + ); + } catch (error) { + process.stderr.write( + `managed-image diagnostic export failed closed: ${(error as Error).message}\n`, + ); + process.exitCode = 1; + } +} diff --git a/scripts/checks/extract-installer-pins.mts b/scripts/checks/extract-installer-pins.mts index 71c7646c20..7143dd427d 100644 --- a/scripts/checks/extract-installer-pins.mts +++ b/scripts/checks/extract-installer-pins.mts @@ -52,7 +52,7 @@ const MAX_INSTALLER_INPUT_BYTES = 1024 * 1024; // revokes that temporary trust after success or failure, and removes inherited // trust around an unverified dev install. const TRUSTED_INSTALLER_TEMPLATE_SHA256_ALLOWLIST = [ - "0fa737a64cf2a7a6a437dc5f203dad81f66f191dc316214c2f343f762ad9b0a5", + "1d6b2ff3375533786a9cf880cc284ff894707299266b3f68fbb30324ea756214", ] as const; const TRUSTED_BREV_TEMPLATE_SHA256_ALLOWLIST = [ "c0a4ddf25a02a9fe02b2df53a60942ea887610f04d4ce16a121b6e79a5aeff1a", diff --git a/scripts/checks/generate-managed-startup-profile-fixture.mts b/scripts/checks/generate-managed-startup-profile-fixture.mts new file mode 100755 index 0000000000..72499b1cfb --- /dev/null +++ b/scripts/checks/generate-managed-startup-profile-fixture.mts @@ -0,0 +1,229 @@ +#!/usr/bin/env -S node --experimental-strip-types + +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; +import { resolve } from "node:path"; +import { pathToFileURL } from "node:url"; +import { + encodeManagedStartupProfile, + MANAGED_STARTUP_PROFILE_SCHEMA_VERSION, + type ManagedStartupAgent, + type ManagedStartupProfile, +} from "../../src/lib/onboard/managed-startup/profile.ts"; + +const AGENTS = new Set(["openclaw", "hermes", "langchain-deepagents-code"]); + +export const MANAGED_STARTUP_E2E_HTTP_PROXY = "http://fixture-http-proxy.example.test:18080"; +export const MANAGED_STARTUP_E2E_HTTPS_PROXY = "http://fixture-https-proxy.example.test:18443"; +export const MANAGED_STARTUP_E2E_NO_PROXY = ["localhost", "127.0.0.1", ".example.test"] as const; + +// Real self-signed X.509 CA used by the no-network managed-image lifecycle +// gate. DCode additionally proves its hardened fetch transport selects the +// root-owned merged bundle containing these exact bytes. +export const MANAGED_STARTUP_E2E_CORPORATE_CA_PEM = `-----BEGIN CERTIFICATE----- +MIIDKzCCAhOgAwIBAgIUL3YNpyohvjOEzlwisLKfyiU3dRwwDQYJKoZIhvcNAQEL +BQAwJTEjMCEGA1UEAwwaTmVtb0NsYXcgVGVzdCBDb3Jwb3JhdGUgQ0EwHhcNMjYw +NzA2MDQwMjM2WhcNMzYwNzAzMDQwMjM2WjAlMSMwIQYDVQQDDBpOZW1vQ2xhdyBU +ZXN0IENvcnBvcmF0ZSBDQTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB +ALVbV5tyMc65jEH39ejvQvBk7dvI8rz8rSZl+5BWSK2a4TzKm3jD3U+qCDZPicrA +ETCDcO09bN6YIAgpB6rYg5BIURJWxFuljBIBMCZEdO6AVlbURPaGsw6RKLA3cmhx +ZekT0qMcoOKm3N+Hb5MHXsWZ8EUf0co2LsWwJgDZrdwY26gF6w+9wr3iGLE92ZbO +LHhjHUYR1oWXmkXS3YW8MN2h5I+oyL71jBiwLHUi59wogxA/LTAD97/GqwJ6DC4C +UERbIpGYhZfrbiKmT+ASJuKRXaUp/0My3IzH90RqqY70d1E/pkAsd5M8SQ332qAZ +OgW4GgO3n7gAlaN/ILwunZ8CAwEAAaNTMFEwHQYDVR0OBBYEFMa5M8bvDm85eFQi +1D5fNATE/rawMB8GA1UdIwQYMBaAFMa5M8bvDm85eFQi1D5fNATE/rawMA8GA1Ud +EwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAB8NR/0HBUH1WbbDOmGNDzge +o+4Pz0KWR5fPDSx9CrmvUk8ijKpJQcSjQcmrXuhCoRs6aExXLh+wImKkOyMIVXfd +YFWjCffSJzeBQfDlMVW+wiAjUh7xaIqpA6Z8EmpdfyoNWd30AuHjs9m8dAa8M/lP +0qhzCbjDiHNHfYSrAuBHlMJ5RsUrNVtSZGpg1dtaSBa+8XFWWNBeJrUANxb8i7Ax +MAhrfNQcxSkZH2lVY+TA2JO83v12nKXzaW1dC94SlsFf0tVSvM3QTeWVgijpr0q+ +J0N7VBg2CdK6jRjKLQOSOPq3ySCicHhVRI8hxIWotif7mK3jj6D8NRalwmlHgNM= +-----END CERTIFICATE----- +`; + +export function managedStartupE2eProfile( + agent: ManagedStartupAgent, + changed = false, + withCorporateCa = false, + withoutHostProxy = false, +): ManagedStartupProfile { + const model = changed ? "nvidia/nemotron-3-super-120b-a12b" : "nvidia/nemotron-3-ultra-550b-a55b"; + const common = { + schemaVersion: MANAGED_STARTUP_PROFILE_SCHEMA_VERSION, + inference: { + routeProvider: "inference", + upstreamProvider: "nvidia", + model, + routedBaseUrl: "https://inference.local/v1", + upstreamEndpointUrl: null, + api: "openai-completions" as const, + }, + proxy: { + managedHost: "10.200.0.1", + managedPort: 3128, + hostHttpUrl: withoutHostProxy ? null : MANAGED_STARTUP_E2E_HTTP_PROXY, + hostHttpsUrl: withoutHostProxy ? null : MANAGED_STARTUP_E2E_HTTPS_PROXY, + hostNoProxy: withoutHostProxy ? [] : MANAGED_STARTUP_E2E_NO_PROXY, + }, + tools: { + disclosure: "progressive" as const, + enabledGateways: [], + }, + messaging: { plan: null }, + corporateCa: { + bundleSha256: withCorporateCa + ? createHash("sha256").update(MANAGED_STARTUP_E2E_CORPORATE_CA_PEM).digest("hex") + : null, + }, + }; + + switch (agent) { + case "openclaw": + return { + ...common, + agent, + agentConfig: { + agent, + webSearch: { enabled: false, provider: "brave" }, + otel: { + enabled: false, + endpointUrl: "http://host.openshell.internal:4318", + serviceName: "openclaw-gateway", + sampleRate: 1, + }, + agentTimeoutSeconds: 600, + heartbeatEvery: null, + extraAgents: { agents: [], defaults: {}, main: {} }, + deviceAuth: { disabled: true, optOutSource: "managed-onboard" }, + minimalBootstrap: true, + }, + inference: { + ...common.inference, + primaryModelRef: `inference/${model}`, + compatibility: {}, + inputModalities: ["text"], + }, + dashboard: { + agent, + mode: "loopback", + url: "http://127.0.0.1:18789", + port: 18_789, + bindAddress: "127.0.0.1", + wslExposure: false, + }, + tuning: { + contextWindow: 131_072, + maxTokens: 8192, + reasoning: false, + reasoningEffort: "default", + }, + }; + case "hermes": + return { + ...common, + agent, + agentConfig: { + agent, + webSearch: { enabled: false, provider: "tavily" }, + }, + inference: { + ...common.inference, + primaryModelRef: null, + compatibility: null, + inputModalities: null, + }, + dashboard: { + agent, + mode: "disabled", + url: "http://127.0.0.1:18789", + publicPort: null, + internalPort: null, + tuiEnabled: false, + }, + tuning: { + contextWindow: 131_072, + maxTokens: null, + reasoning: null, + reasoningEffort: null, + }, + }; + case "langchain-deepagents-code": + return { + ...common, + agent, + agentConfig: { + agent, + autoApprovalMode: "disabled", + observabilityEnabled: false, + }, + inference: { + ...common.inference, + upstreamEndpointUrl: "https://integrate.api.nvidia.com/v1", + primaryModelRef: null, + compatibility: null, + inputModalities: null, + }, + dashboard: { + agent, + mode: "disabled", + }, + tuning: { + contextWindow: null, + maxTokens: null, + reasoning: null, + reasoningEffort: null, + }, + }; + } +} + +function readAgent(value: string | undefined): ManagedStartupAgent { + if (value && AGENTS.has(value as ManagedStartupAgent)) { + return value as ManagedStartupAgent; + } + throw new Error("--agent must identify a shipped managed-image agent"); +} + +function main(argv: readonly string[]): void { + if (argv.length === 1 && argv[0] === "--corporate-ca-b64") { + process.stdout.write( + `${Buffer.from(MANAGED_STARTUP_E2E_CORPORATE_CA_PEM, "utf8").toString("base64")}\n`, + ); + return; + } + const agentIndex = argv.indexOf("--agent"); + if (agentIndex < 0) throw new Error("--agent is required"); + const unexpected = argv.filter( + (value, index) => + index !== agentIndex && + index !== agentIndex + 1 && + value !== "--changed" && + value !== "--corporate-ca" && + value !== "--without-host-proxy", + ); + if (unexpected.length > 0) { + throw new Error(`unsupported arguments: ${unexpected.join(" ")}`); + } + const agent = readAgent(argv[agentIndex + 1]); + process.stdout.write( + `${encodeManagedStartupProfile( + managedStartupE2eProfile( + agent, + argv.includes("--changed"), + argv.includes("--corporate-ca"), + argv.includes("--without-host-proxy"), + ), + )}\n`, + ); +} + +if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) { + try { + main(process.argv.slice(2)); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + } +} diff --git a/scripts/checks/run-managed-image-direct-e2e.ts b/scripts/checks/run-managed-image-direct-e2e.ts new file mode 100755 index 0000000000..7786fc75f4 --- /dev/null +++ b/scripts/checks/run-managed-image-direct-e2e.ts @@ -0,0 +1,491 @@ +#!/usr/bin/env -S node --no-warnings --experimental-strip-types + +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import { resolve } from "node:path"; +import { pathToFileURL } from "node:url"; +import { + encodeManagedStartupProfile, + MANAGED_STARTUP_AGENTS, + type ManagedStartupAgent, +} from "../../src/lib/onboard/managed-startup/profile.ts"; +import { + createManagedStartupRootApplyRequest, + type ManagedStartupRootApplyRequest, + serializeManagedStartupRootApplyRequest, +} from "../../src/lib/onboard/managed-startup/root-apply.ts"; +import { + MANAGED_STARTUP_E2E_CORPORATE_CA_PEM, + managedStartupE2eProfile, +} from "./generate-managed-startup-profile-fixture.mts"; + +const CONTAINER_ID_RE = /^[a-f0-9]{64}$/u; +const IMMUTABLE_IMAGE_RE = /^sha256:[a-f0-9]{64}$/u; +const IMMUTABLE_REFERENCE_RE = /^(?:sha256:[a-f0-9]{64}|[^\s@]+@sha256:[a-f0-9]{64})$/u; +const RUNTIME = "/usr/local/lib/nemoclaw/managed-startup-image-runtime.cjs"; +const HOLD = "/usr/local/bin/nemoclaw-managed-startup-hold"; +const FIXED_ROOT_ENV = [ + "HOME=/root", + "LANG=C.UTF-8", + "LC_ALL=C.UTF-8", + "NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION=1", + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", +] as const; + +export interface ManagedImageDirectE2eInputs { + readonly agent: ManagedStartupAgent; + readonly image: string; + readonly platform: "linux/amd64"; +} + +interface CommandResult { + readonly status: number; + readonly stdout: string; + readonly stderr: string; +} + +interface ContainerInspect { + readonly Id?: string; + readonly Image?: string; + readonly Config?: { + readonly Env?: readonly string[] | null; + } | null; + readonly State?: { + readonly Running?: boolean; + } | null; +} + +function requiredArgument(argv: readonly string[], flag: string): string { + const index = argv.indexOf(flag); + const value = index >= 0 ? argv[index + 1] : undefined; + if (!value || value.startsWith("--")) throw new Error(`${flag} is required`); + return value; +} + +export function parseManagedImageDirectE2eInputs( + argv: readonly string[], +): ManagedImageDirectE2eInputs { + const agent = requiredArgument(argv, "--agent"); + const image = requiredArgument(argv, "--image"); + const platform = requiredArgument(argv, "--platform"); + const knownFlags = new Set(["--agent", "--image", "--platform"]); + if (argv.length !== 6 || argv.some((value, index) => index % 2 === 0 && !knownFlags.has(value))) { + throw new Error("usage: --agent --image --platform linux/amd64"); + } + if (!(MANAGED_STARTUP_AGENTS as readonly string[]).includes(agent)) { + throw new Error("--agent must identify a shipped managed-image agent"); + } + if (!IMMUTABLE_REFERENCE_RE.test(image)) { + throw new Error("--image must be an immutable image ID or digest reference"); + } + if (platform !== "linux/amd64") { + throw new Error("--platform must be linux/amd64"); + } + return { agent: agent as ManagedStartupAgent, image, platform }; +} + +function commandDetail(result: CommandResult): string { + return `${result.stderr} ${result.stdout}`.trim().slice(-2000); +} + +function docker( + args: readonly string[], + options: { + readonly ignoreError?: boolean; + readonly input?: string; + readonly timeout?: number; + } = {}, +): CommandResult { + const result = spawnSync("docker", [...args], { + encoding: "utf8", + input: options.input, + maxBuffer: 16 * 1024 * 1024, + timeout: options.timeout ?? 120_000, + }); + const normalized = { + status: Number(result.status ?? 1), + stdout: String(result.stdout ?? ""), + stderr: String(result.stderr ?? result.error?.message ?? ""), + }; + if (normalized.status !== 0 && options.ignoreError !== true) { + throw new Error(`docker ${args[0] ?? "command"} failed: ${commandDetail(normalized)}`); + } + return normalized; +} + +function requestFor(agent: ManagedStartupAgent, changed = false): ManagedStartupRootApplyRequest { + return createManagedStartupRootApplyRequest({ + agent, + encodedProfile: encodeManagedStartupProfile( + managedStartupE2eProfile(agent, changed, true, true), + ), + corporateCaB64: Buffer.from(MANAGED_STARTUP_E2E_CORPORATE_CA_PEM, "utf8").toString("base64"), + }); +} + +function rootRuntimeArgs( + containerId: string, + agent: ManagedStartupAgent, + action: "--apply-root-stdin" | "--commit-shared-state-transaction", + user = "0:0", +): string[] { + return [ + "exec", + ...(action === "--apply-root-stdin" ? ["--interactive"] : []), + "--user", + user, + "--workdir", + "/", + containerId, + "/usr/bin/env", + "-i", + ...FIXED_ROOT_ENV, + "/usr/local/bin/node", + RUNTIME, + action, + "--agent", + agent, + ]; +} + +function managedConfig(agent: ManagedStartupAgent): string { + switch (agent) { + case "openclaw": + return "/sandbox/.openclaw/openclaw.json"; + case "hermes": + return "/sandbox/.hermes/config.yaml"; + case "langchain-deepagents-code": + return "/sandbox/.deepagents/config.toml"; + } +} + +function waitForAgentCommand(containerId: string): void { + const deadline = Date.now() + 120_000; + while (Date.now() < deadline) { + const ready = docker( + [ + "exec", + "--user", + "0:0", + containerId, + "/bin/sh", + "-eu", + "-c", + "test -s /tmp/nemoclaw-managed-command-uid && test -s /run/nemoclaw/managed-startup-complete.json", + ], + { ignoreError: true, timeout: 15_000 }, + ); + if (ready.status === 0) return; + const running = docker(["inspect", "--format", "{{.State.Running}}", containerId], { + ignoreError: true, + timeout: 15_000, + }); + if (running.status !== 0 || running.stdout.trim() !== "true") { + const logs = docker(["logs", containerId], { ignoreError: true, timeout: 15_000 }); + throw new Error(`managed image exited before agent command: ${commandDetail(logs)}`); + } + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 500); + } + throw new Error("managed image did not reach the forwarded sandbox command"); +} + +function exactProxyEnvironment(): string { + return [ + "HTTP_PROXY=http://10.200.0.1:3128", + "HTTPS_PROXY=http://10.200.0.1:3128", + "NO_PROXY=localhost,127.0.0.1,::1,10.200.0.1", + "http_proxy=http://10.200.0.1:3128", + "https_proxy=http://10.200.0.1:3128", + "no_proxy=localhost,127.0.0.1,::1,10.200.0.1", + ].join("\n"); +} + +export function runManagedImageDirectE2e(input: ManagedImageDirectE2eInputs): void { + const request = requestFor(input.agent); + const payload = serializeManagedStartupRootApplyRequest(request); + const expectedImageId = docker([ + "image", + "inspect", + "--format", + "{{.Id}}", + input.image, + ]).stdout.trim(); + if (!IMMUTABLE_IMAGE_RE.test(expectedImageId)) { + throw new Error("managed image reference did not resolve to one immutable local image ID"); + } + const finalCommand = [ + "if [ -x /usr/local/bin/dcode ]; then", + ' timeout 10 /usr/local/bin/dcode -n "" > /tmp/nemoclaw-managed-dcode-empty-prompt-output 2>&1', + ' printf "%s\\n" "$?" > /tmp/nemoclaw-managed-dcode-empty-prompt-status', + "fi", + "id -u > /tmp/nemoclaw-managed-command-uid", + "{", + ' printf "HTTP_PROXY=%s\\n" "${HTTP_PROXY-__UNSET__}"', + ' printf "HTTPS_PROXY=%s\\n" "${HTTPS_PROXY-__UNSET__}"', + ' printf "NO_PROXY=%s\\n" "${NO_PROXY-__UNSET__}"', + ' printf "http_proxy=%s\\n" "${http_proxy-__UNSET__}"', + ' printf "https_proxy=%s\\n" "${https_proxy-__UNSET__}"', + ' printf "no_proxy=%s\\n" "${no_proxy-__UNSET__}"', + "} > /tmp/nemoclaw-managed-command-proxy-env", + "exec /usr/bin/tail -f /dev/null", + ].join("\n"); + let containerId = ""; + try { + // Mirror the supported OpenShell split: the image OCI user is root for its + // supervisor, while the supervisor enters the sandbox startup command as + // sandbox:sandbox. + containerId = docker([ + "run", + "-d", + "--platform", + input.platform, + "--network", + "none", + "--user", + "sandbox", + "--env", + "HTTP_PROXY=http://upper-http:upper-secret@upper-http.example.test:18080", + "--env", + "HTTPS_PROXY=http://upper-https:upper-secret@upper-https.example.test:18443", + "--env", + "NO_PROXY=upper.internal", + "--env", + "http_proxy=http://lower-http:lower-secret@lower-http.example.test:28080", + "--env", + "https_proxy=http://lower-https:lower-secret@lower-https.example.test:28443", + "--env", + "no_proxy=lower.internal", + "--entrypoint", + HOLD, + input.image, + "--agent", + input.agent, + "--profile-fingerprint", + request.profileFingerprint, + "/bin/sh", + "-c", + finalCommand, + ]).stdout.trim(); + if (!CONTAINER_ID_RE.test(containerId)) { + throw new Error("docker run did not return one exact container identity"); + } + + const inspectOutput = docker(["inspect", "--type", "container", containerId]).stdout; + const parsed = JSON.parse(inspectOutput) as unknown; + if (!Array.isArray(parsed) || parsed.length !== 1) { + throw new Error("docker inspect did not return one exact container"); + } + const inspect = parsed[0] as ContainerInspect; + if ( + inspect.Id !== containerId || + inspect.Image !== expectedImageId || + inspect.State?.Running !== true + ) { + throw new Error("managed startup did not pin one running exact-image container"); + } + const inspectText = JSON.stringify(inspect); + if ( + inspectText.includes(request.encodedProfile) || + inspectText.includes(String(request.corporateCaB64)) || + (inspect.Config?.Env ?? []).some((value) => + /^(?:NEMOCLAW_STARTUP_PROFILE_B64|NEMOCLAW_CORPORATE_CA_B64)=/u.test(value), + ) + ) { + throw new Error("managed profile or corporate CA entered Docker argv/env metadata"); + } + + const applied = docker(rootRuntimeArgs(containerId, input.agent, "--apply-root-stdin"), { + input: payload, + timeout: 300_000, + }); + if (!applied.stdout.includes("transaction pending")) { + throw new Error("root application did not leave a pending shared-state transaction"); + } + waitForAgentCommand(containerId); + + const sandboxUid = docker([ + "exec", + "--user", + "0:0", + containerId, + "id", + "-u", + "sandbox", + ]).stdout.trim(); + const commandUid = docker([ + "exec", + "--user", + "0:0", + containerId, + "cat", + "/tmp/nemoclaw-managed-command-uid", + ]).stdout.trim(); + if (commandUid !== sandboxUid) { + throw new Error( + "managed hold or legacy entrypoint did not preserve the sandbox command identity", + ); + } + const proxyEnvironment = docker([ + "exec", + "--user", + "sandbox", + containerId, + "cat", + "/tmp/nemoclaw-managed-command-proxy-env", + ]).stdout.trim(); + if (proxyEnvironment !== exactProxyEnvironment()) { + throw new Error("managed hold did not replace hostile inherited proxy material"); + } + + const config = docker([ + "exec", + "--user", + "sandbox", + containerId, + "cat", + managedConfig(input.agent), + ]).stdout; + if (!config.includes("nvidia/nemotron-3-ultra-550b-a55b")) { + throw new Error("managed agent configuration does not contain the requested model"); + } + const runtimeEnvironment = docker([ + "exec", + "--user", + "0:0", + containerId, + "cat", + "/run/nemoclaw/managed-startup-runtime.env", + ]).stdout; + if ( + runtimeEnvironment.includes("upper-secret") || + runtimeEnvironment.includes("lower-secret") || + runtimeEnvironment.includes(request.encodedProfile) || + runtimeEnvironment.includes(String(request.corporateCaB64)) + ) { + throw new Error("managed runtime handoff contains a forbidden transport or credential"); + } + const installedCa = docker([ + "exec", + "--user", + "0:0", + containerId, + "cat", + "/usr/local/share/nemoclaw/corporate-ca.pem", + ]).stdout; + const mergedCa = docker([ + "exec", + "--user", + "0:0", + containerId, + "cat", + "/run/nemoclaw/managed-startup-ca-bundle.pem", + ]).stdout; + if ( + installedCa !== MANAGED_STARTUP_E2E_CORPORATE_CA_PEM || + !mergedCa.endsWith(MANAGED_STARTUP_E2E_CORPORATE_CA_PEM) + ) { + throw new Error("managed corporate CA was not installed and merged exactly"); + } + docker([ + "exec", + "--user", + "0:0", + containerId, + "/bin/sh", + "-eu", + "-c", + [ + 'test "$(stat -c "%u:%g:%a" /run/nemoclaw/managed-startup-runtime.env)" = "0:0:444"', + 'test "$(stat -c "%u:%g:%a" /run/nemoclaw/managed-startup-complete.json)" = "0:0:444"', + 'test "$(stat -c "%u:%g:%a" /usr/local/share/nemoclaw/corporate-ca.pem)" = "0:0:444"', + 'test "$(stat -c "%u:%g:%a" /run/nemoclaw/managed-startup-ca-bundle.pem)" = "0:0:444"', + "test -d /var/lib/nemoclaw/managed-startup-shared-state-transaction-v1", + ].join("\n"), + ]); + + if (input.agent === "langchain-deepagents-code") { + const status = docker([ + "exec", + "--user", + "0:0", + containerId, + "cat", + "/tmp/nemoclaw-managed-dcode-empty-prompt-status", + ]).stdout.trim(); + const output = docker([ + "exec", + "--user", + "0:0", + containerId, + "cat", + "/tmp/nemoclaw-managed-dcode-empty-prompt-output", + ]).stdout.trim(); + if ( + status !== "2" || + output !== "NemoClaw: empty non-interactive prompt for -n; provide prompt text." + ) { + throw new Error("managed DCode launcher/supervisor empty-prompt contract failed"); + } + } + + docker(rootRuntimeArgs(containerId, input.agent, "--commit-shared-state-transaction")); + docker([ + "exec", + "--user", + "0:0", + containerId, + "test", + "!", + "-e", + "/var/lib/nemoclaw/managed-startup-shared-state-transaction-v1", + ]); + + const replay = docker(rootRuntimeArgs(containerId, input.agent, "--apply-root-stdin"), { + input: payload, + timeout: 300_000, + }); + if (!replay.stdout.includes("was already complete")) { + throw new Error("same-profile root application was not idempotent"); + } + const changed = docker(rootRuntimeArgs(containerId, input.agent, "--apply-root-stdin"), { + ignoreError: true, + input: serializeManagedStartupRootApplyRequest(requestFor(input.agent, true)), + timeout: 300_000, + }); + if ( + changed.status === 0 || + !commandDetail(changed).includes("completion marker does not match the requested profile") + ) { + throw new Error("changed profile did not require a fresh sandbox"); + } + const nonroot = docker( + rootRuntimeArgs(containerId, input.agent, "--apply-root-stdin", "sandbox"), + { ignoreError: true, input: payload, timeout: 300_000 }, + ); + if ( + nonroot.status === 0 || + !commandDetail(nonroot).includes("requires container effective uid 0") + ) { + throw new Error("sandbox account bypassed root-only profile application"); + } + + process.stdout.write( + `Validated exact ${input.agent} managed image ${input.image} through root stdin and sandbox hold.\n`, + ); + } finally { + if (CONTAINER_ID_RE.test(containerId)) { + docker(["rm", "-f", containerId], { ignoreError: true, timeout: 30_000 }); + } + } +} + +if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) { + try { + runManagedImageDirectE2e(parseManagedImageDirectE2eInputs(process.argv.slice(2))); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + } +} diff --git a/scripts/checks/run-managed-image-openshell-e2e.ts b/scripts/checks/run-managed-image-openshell-e2e.ts new file mode 100644 index 0000000000..72370722ea --- /dev/null +++ b/scripts/checks/run-managed-image-openshell-e2e.ts @@ -0,0 +1,685 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { type ChildProcess, spawn, spawnSync } from "node:child_process"; +import fs from "node:fs"; +import net from "node:net"; +import os from "node:os"; +import path from "node:path"; +import { collectDockerGpuPatchDiagnostics } from "../../src/lib/onboard/docker-gpu-patch.ts"; +import { createDockerGpuSandboxCreatePatch } from "../../src/lib/onboard/docker-gpu-sandbox-create.ts"; +import { resolveDockerStartupCommandPatch } from "../../src/lib/onboard/docker-startup-command-agent.ts"; +import { + type InitialSandboxPolicy, + prepareInitialSandboxCreatePolicy, +} from "../../src/lib/onboard/initial-policy.ts"; +import { + encodeManagedStartupProfile, + type ManagedStartupAgent, +} from "../../src/lib/onboard/managed-startup/profile.ts"; +import { collectSandboxCreateFailureDiagnostics } from "../../src/lib/onboard/sandbox-create-failure.ts"; +import { + MANAGED_STARTUP_E2E_CORPORATE_CA_PEM, + managedStartupE2eProfile, +} from "./generate-managed-startup-profile-fixture.mts"; + +const MANAGED_AGENTS = new Set([ + "openclaw", + "hermes", + "langchain-deepagents-code", +]); +const MODEL = "nvidia/nemotron-3-ultra-550b-a55b"; +const GATEWAY_PORT = 8080; +const MANAGED_AGENT_BASE_POLICIES: Record = { + openclaw: ["nemoclaw-blueprint", "policies", "openclaw-sandbox.yaml"], + hermes: ["agents", "hermes", "policy-additions.yaml"], + "langchain-deepagents-code": ["agents", "langchain-deepagents-code", "policy-additions.yaml"], +}; + +type Inputs = { + agent: ManagedStartupAgent; + image: string; + sandbox: string; +}; + +type OnboardModule = { + openshellArgv(args: string[]): string[]; + runOpenshell(args: string[], opts?: Record): ReturnType; + runCaptureOpenshell(args: string[], opts?: Record): string; + sleepSeconds(seconds: number): void; + startGatewayForRecovery(options: { gatewayName: string; gatewayPort: number }): Promise; +}; + +type SandboxCreateLaunchModule = { + prepareSandboxCreateManagedImageLaunch(input: Record): { + createArgv: string[]; + prebuild: { createArgs: string[]; imageId: null; imageRef: null }; + sandboxEnv: Record; + sandboxStartupCommand: string[]; + startupRequirement: "sandbox-command" | "trusted-image-init"; + managedStartupRootApplyRequest: Parameters< + typeof createDockerGpuSandboxCreatePatch + >[0]["managedStartupRootApplyRequest"]; + }; +}; + +type ManagedStartupPatch = ReturnType; + +function requiredValue(argv: readonly string[], flag: string): string { + const index = argv.indexOf(flag); + const value = index >= 0 ? argv[index + 1] : undefined; + if (!value || value.startsWith("--")) throw new Error(`${flag} is required`); + return value; +} + +export function parseManagedImageOpenShellE2eInputs(argv: readonly string[]): Inputs { + const unexpected = argv.filter( + (value, index) => + !["--agent", "--image", "--sandbox"].includes(value) && + !["--agent", "--image", "--sandbox"].includes(argv[index - 1] ?? ""), + ); + if (unexpected.length > 0) { + throw new Error(`unsupported arguments: ${unexpected.join(" ")}`); + } + const agentValue = requiredValue(argv, "--agent"); + if (!MANAGED_AGENTS.has(agentValue as ManagedStartupAgent)) { + throw new Error("--agent must identify a shipped managed-image agent"); + } + const image = requiredValue(argv, "--image"); + if (!/^sha256:[0-9a-f]{64}$/u.test(image)) { + throw new Error("--image must be an immutable local sha256 image ID"); + } + const sandbox = requiredValue(argv, "--sandbox"); + if (!/^[a-z0-9](?:[a-z0-9.-]{0,61}[a-z0-9])?$/u.test(sandbox)) { + throw new Error("--sandbox must be a valid RFC 1123 label"); + } + return { agent: agentValue as ManagedStartupAgent, image, sandbox }; +} + +export function managedImageOpenShellBasePolicyPath(agent: ManagedStartupAgent): string { + return path.resolve(import.meta.dirname, "..", "..", ...MANAGED_AGENT_BASE_POLICIES[agent]); +} + +function commandResult(argv: readonly string[], env: NodeJS.ProcessEnv, timeout = 20_000) { + const [command, ...args] = argv; + if (!command) throw new Error("command argv must not be empty"); + return spawnSync(command, args, { + encoding: "utf8", + env, + killSignal: "SIGKILL", + stdio: ["ignore", "pipe", "pipe"], + timeout, + }); +} + +function commandDetail(result: ReturnType): string { + return `${result.error?.message ?? ""}\n${result.stdout ?? ""}\n${result.stderr ?? ""}` + .trim() + .slice(-8_000); +} + +function isDockerNotFound(result: ReturnType): boolean { + return ( + result.status !== 0 && + /(?:no such (?:container|network|object)|not found)/iu.test(commandDetail(result)) + ); +} + +function readGatewayPid(stateDir: string): number | null { + try { + const value = Number.parseInt( + fs.readFileSync(path.join(stateDir, "openshell-gateway.pid"), "utf8").trim(), + 10, + ); + return Number.isSafeInteger(value) && value > 1 ? value : null; + } catch { + return null; + } +} + +function stopProcess(pid: number | null): void { + if (!pid) return; + try { + process.kill(pid, "SIGTERM"); + } catch { + return; + } + for (let attempt = 0; attempt < 50; attempt += 1) { + try { + process.kill(pid, 0); + } catch { + return; + } + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 100); + } + try { + process.kill(pid, "SIGKILL"); + } catch { + // The process exited between the liveness probe and the final signal. + } +} + +function waitForChildExit(child: ChildProcess, timeoutMs: number): Promise { + if (child.exitCode !== null || child.signalCode !== null) return Promise.resolve(true); + return new Promise((resolve) => { + let settled = false; + const finish = (exited: boolean) => { + if (settled) return; + settled = true; + clearTimeout(timer); + child.off("close", onClose); + child.off("error", onError); + resolve(exited); + }; + const onClose = () => finish(true); + const onError = () => finish(false); + const timer = setTimeout(() => finish(false), timeoutMs); + child.once("close", onClose); + child.once("error", onError); + }); +} + +async function stopChild(child: ChildProcess | null): Promise { + if (!child || child.pid === undefined || child.exitCode !== null || child.signalCode !== null) { + return; + } + child.kill("SIGTERM"); + if (await waitForChildExit(child, 5_000)) return; + child.kill("SIGKILL"); + if (!(await waitForChildExit(child, 5_000))) { + throw new Error("OpenShell sandbox create child did not exit after SIGKILL"); + } +} + +async function assertGatewayPortAvailable(): Promise { + await new Promise((resolve, reject) => { + const server = net.createServer(); + server.unref(); + server.once("error", () => { + reject( + new Error( + `refusing to disturb an existing listener on the managed-image E2E gateway port ${GATEWAY_PORT}`, + ), + ); + }); + server.listen(GATEWAY_PORT, "127.0.0.1", () => { + server.close((error) => { + if (error) reject(error); + else resolve(); + }); + }); + }); +} + +function managedConfigPath(agent: ManagedStartupAgent): string { + switch (agent) { + case "openclaw": + return "/sandbox/.openclaw/openclaw.json"; + case "hermes": + return "/sandbox/.hermes/config.yaml"; + case "langchain-deepagents-code": + return "/sandbox/.deepagents/config.toml"; + } +} + +export function managedImageOpenShellProbe(agent: ManagedStartupAgent): string { + const healthProbe = + agent === "openclaw" + ? "/usr/bin/curl -fsS --max-time 5 http://127.0.0.1:18789/health >/dev/null" + : agent === "hermes" + ? "/usr/bin/curl -fsS --max-time 5 http://127.0.0.1:8642/health >/dev/null" + : "/usr/local/bin/dcode --version >/dev/null"; + return [ + "set -eu", + `test -x ${ + agent === "openclaw" + ? "/usr/local/bin/openclaw" + : agent === "hermes" + ? "/usr/local/bin/hermes" + : "/usr/local/bin/dcode" + }`, + `grep -F ${JSON.stringify(MODEL)} ${JSON.stringify(managedConfigPath(agent))} >/dev/null`, + "test ! -L /run/nemoclaw/managed-startup-runtime.env", + 'test "$(stat -c "%u:%g:%a" /run/nemoclaw/managed-startup-runtime.env)" = "0:0:444"', + "test ! -L /run/nemoclaw/managed-startup-complete.json", + 'test "$(stat -c "%u:%g:%a" /run/nemoclaw/managed-startup-complete.json)" = "0:0:444"', + "test -s /usr/local/share/nemoclaw/corporate-ca.pem", + 'test "$(stat -c "%u:%g:%a" /usr/local/share/nemoclaw/corporate-ca.pem)" = "0:0:444"', + "test -s /run/nemoclaw/managed-startup-ca-bundle.pem", + 'test "$(stat -c "%u:%g:%a" /run/nemoclaw/managed-startup-ca-bundle.pem)" = "0:0:444"', + healthProbe, + ].join("\n"); +} + +export function managedImageOpenShellCommittedProbe(): string { + return [ + "set -eu", + "test ! -e /var/lib/nemoclaw/managed-startup-shared-state-transaction-v1", + ].join("\n"); +} + +function printLogTail(file: string): void { + try { + const lines = fs.readFileSync(file, "utf8").split("\n"); + process.stderr.write(`${lines.slice(-160).join("\n")}\n`); + } catch { + // The create process may fail before opening its log. + } +} + +function captureFailureDiagnosticsBeforeRollback( + onboard: OnboardModule, + input: Inputs, + createLog: string, + startupPatch: ManagedStartupPatch, + error: unknown, +): void { + const diagnosticDirectories: string[] = []; + try { + const dockerDiagnostics = collectDockerGpuPatchDiagnostics( + input.sandbox, + { + error, + selectedMode: startupPatch.selectedMode(), + }, + { runCaptureOpenshell: onboard.runCaptureOpenshell }, + ); + if (dockerDiagnostics) diagnosticDirectories.push(dockerDiagnostics.dir); + } catch { + // Diagnostics must never mask the primary live failure or block rollback. + } + try { + const gatewayDiagnostics = collectSandboxCreateFailureDiagnostics(input.sandbox, { + gatewayLogPath: path.join(path.dirname(createLog), "openshell-gateway.log"), + }); + if (gatewayDiagnostics) diagnosticDirectories.push(gatewayDiagnostics.dir); + } catch { + // The Docker snapshot above remains useful when gateway collection fails. + } + for (const directory of diagnosticDirectories) { + process.stderr.write(`Managed-image OpenShell failure diagnostics staged: ${directory}\n`); + } +} + +async function waitForSandboxProbe( + onboard: OnboardModule, + input: Inputs, + env: NodeJS.ProcessEnv, + createChild: ChildProcess, + createLog: string, + startupPatch: ManagedStartupPatch, +): Promise { + const healthProbe = managedImageOpenShellProbe(input.agent); + const committedProbe = managedImageOpenShellCommittedProbe(); + const deadline = Date.now() + 240_000; + let createSpawnErrorMessage: string | null = null; + createChild.once("error", (error) => { + createSpawnErrorMessage = error.message; + }); + const applyStartupPatch = () => { + startupPatch.maybeApplyDuringCreate(); + startupPatch.exitOnPatchError(); + // Reconnect is a reversible precondition. The receipt and any recreation + // backup remain available until the exact live health probe passes. + startupPatch.waitForSupervisorReconnectIfNeeded(); + }; + const runProbe = (probe: string, timeoutMs: number) => + commandResult( + onboard.openshellArgv([ + "sandbox", + "exec", + "--name", + input.sandbox, + "--", + "/bin/sh", + "-eu", + "-c", + probe, + ]), + env, + timeoutMs, + ); + try { + while (Date.now() < deadline) { + applyStartupPatch(); + const remainingMs = deadline - Date.now(); + const result = runProbe(healthProbe, Math.max(1, Math.min(15_000, remainingMs))); + applyStartupPatch(); + if (result.status === 0) { + startupPatch.commitAfterReady(); + const committed = runProbe( + committedProbe, + Math.max(1, Math.min(15_000, deadline - Date.now())), + ); + if (committed.status !== 0) { + throw new Error( + `managed startup committed, but transaction cleanup was not observable through the exact sandbox: ${commandDetail(committed)}`, + ); + } + return; + } + if (createSpawnErrorMessage) { + printLogTail(createLog); + throw new Error(`could not spawn OpenShell sandbox create: ${createSpawnErrorMessage}`); + } + if (createChild.signalCode !== null) { + printLogTail(createLog); + throw new Error( + `OpenShell sandbox create exited on signal ${createChild.signalCode} before the live probe passed: ${commandDetail(result)}`, + ); + } + if (createChild.exitCode !== null && createChild.exitCode !== 0) { + printLogTail(createLog); + throw new Error( + `OpenShell sandbox create exited ${String(createChild.exitCode)} before the live probe passed: ${commandDetail(result)}`, + ); + } + const sleepMs = Math.min(2_000, Math.max(0, deadline - Date.now())); + if (sleepMs > 0) { + await new Promise((resolve) => setTimeout(resolve, sleepMs)); + } + } + printLogTail(createLog); + throw new Error("OpenShell sandbox did not pass the exact-image startup probe within 240s"); + } catch (error) { + // Capture the recreated container and gateway while they still exist. The + // rollback below intentionally destroys that transient failure evidence. + captureFailureDiagnosticsBeforeRollback(onboard, input, createLog, startupPatch, error); + startupPatch.rollbackManagedStartupAfterCreateFailure(); + throw error; + } +} + +function exactHarnessContainerIds( + input: Inputs, + networkName: string, + env: NodeJS.ProcessEnv, +): { candidateCount: number; exactIds: string[] } { + const list = commandResult( + [ + "docker", + "ps", + "-aq", + "--no-trunc", + "--filter", + "label=openshell.ai/managed-by=openshell", + "--filter", + `label=openshell.ai/sandbox-name=${input.sandbox}`, + ], + env, + ); + if (list.status !== 0) { + throw new Error(`could not resolve the OpenShell sandbox container: ${commandDetail(list)}`); + } + const candidates = String(list.stdout ?? "") + .trim() + .split(/\s+/u) + .filter(Boolean); + const exactIds: string[] = []; + for (const candidate of candidates) { + const inspect = commandResult(["docker", "inspect", candidate], env); + if (inspect.status !== 0) continue; + try { + const records = JSON.parse(String(inspect.stdout ?? "")) as Array<{ + Config?: { Labels?: Record }; + Image?: string; + NetworkSettings?: { Networks?: Record }; + }>; + const record = records.length === 1 ? records[0] : undefined; + if ( + record?.Config?.Labels?.["openshell.ai/managed-by"] === "openshell" && + record.Config.Labels["openshell.ai/sandbox-name"] === input.sandbox && + record.Image === input.image && + Object.hasOwn(record.NetworkSettings?.Networks ?? {}, networkName) + ) { + exactIds.push(candidate); + } + } catch { + // An unparseable inspection result cannot establish cleanup ownership. + } + } + return { candidateCount: candidates.length, exactIds }; +} + +function assertExactSandboxImage( + input: Inputs, + networkName: string, + env: NodeJS.ProcessEnv, +): string { + const resolved = exactHarnessContainerIds(input, networkName, env); + if (resolved.candidateCount !== 1 || resolved.exactIds.length !== 1) { + throw new Error( + `OpenShell did not launch exactly one harness-owned PR image container: found ${resolved.candidateCount} labeled and ${resolved.exactIds.length} exact`, + ); + } + return resolved.exactIds[0] ?? ""; +} + +async function run(input: Inputs): Promise { + const stateParent = process.env.RUNNER_TEMP || os.tmpdir(); + const stateDir = fs.mkdtempSync(path.join(stateParent, "nemoclaw-managed-openshell-")); + const createLog = path.join(stateDir, "sandbox-create.log"); + const networkName = `nemoclaw-managed-pr-${process.pid}-${Date.now().toString(36)}`; + process.env.NEMOCLAW_NON_INTERACTIVE = "1"; + process.env.NEMOCLAW_OPENSHELL_GATEWAY_STATE_DIR = stateDir; + process.env.NEMOCLAW_GATEWAY_PORT = String(GATEWAY_PORT); + process.env.NEMOCLAW_DOCKER_GPU_SUPERVISOR_RECONNECT_TIMEOUT = "240"; + process.env.OPENSHELL_DOCKER_NETWORK_NAME = networkName; + process.env.XDG_CONFIG_HOME = path.join(stateDir, "xdg-config"); + process.env.XDG_DATA_HOME = path.join(stateDir, "xdg-data"); + process.env.XDG_STATE_HOME = path.join(stateDir, "xdg-state"); + process.env.PATH = `${path.join(os.homedir(), ".local", "bin")}:${process.env.PATH ?? ""}`; + + let onboard: OnboardModule | null = null; + let createChild: ChildProcess | null = null; + let createLogFd: number | null = null; + let ownedContainerId: string | null = null; + let initialSandboxPolicy: InitialSandboxPolicy | null = null; + try { + await assertGatewayPortAvailable(); + const localImage = commandResult( + ["docker", "image", "inspect", "--format", "{{.Id}}", input.image], + process.env, + ); + if (localImage.status !== 0 || String(localImage.stdout ?? "").trim() !== input.image) { + throw new Error("--image does not resolve to the exact local PR image ID"); + } + + const onboardImport = (await import("../../src/lib/onboard.ts")) as unknown as + | OnboardModule + | { default: OnboardModule }; + onboard = "default" in onboardImport ? onboardImport.default : onboardImport; + await onboard.startGatewayForRecovery({ + gatewayName: "nemoclaw", + gatewayPort: GATEWAY_PORT, + }); + + const launchImport = (await import( + "../../src/lib/onboard/sandbox-create-launch.ts" + )) as unknown as SandboxCreateLaunchModule | { default: SandboxCreateLaunchModule }; + const launchModule = "default" in launchImport ? launchImport.default : launchImport; + const profile = encodeManagedStartupProfile( + managedStartupE2eProfile(input.agent, false, true, true), + ); + initialSandboxPolicy = prepareInitialSandboxCreatePolicy( + managedImageOpenShellBasePolicyPath(input.agent), + [], + { agentName: input.agent }, + ); + const createArgs = [ + "--from", + input.image, + "--name", + input.sandbox, + "--policy", + initialSandboxPolicy.policyPath, + ]; + const launch = launchModule.prepareSandboxCreateManagedImageLaunch({ + agent: { + name: input.agent, + ...(input.agent === "openclaw" ? { configPaths: { dir: "/sandbox/.openclaw" } } : {}), + }, + sandboxName: input.sandbox, + chatUiUrl: "", + createArgs, + env: {}, + extraPlaceholderKeys: [], + getDashboardForwardPort: () => "0", + hermesDashboardState: { config: null, enabled: false }, + manageDashboard: false, + openshellShellCommand: (args: string[]) => args.map((arg) => JSON.stringify(arg)).join(" "), + openshellArgv: onboard.openshellArgv, + managedStartupProfile: { + encodedProfile: profile, + corporateCaB64: Buffer.from(MANAGED_STARTUP_E2E_CORPORATE_CA_PEM, "utf8").toString( + "base64", + ), + }, + }); + if ( + launch.prebuild.imageId !== null || + launch.prebuild.imageRef !== null || + launch.prebuild.createArgs.join("\0") !== createArgs.join("\0") || + launch.createArgv.filter((value) => value === "--from").length !== 1 || + launch.createArgv[launch.createArgv.indexOf("--from") + 1] !== input.image || + launch.createArgv.filter((value) => value === "--policy").length !== 1 || + launch.createArgv[launch.createArgv.indexOf("--policy") + 1] !== + initialSandboxPolicy.policyPath + ) { + throw new Error("managed-image launch renderer altered the exact PR image identity"); + } + const startupPlan = resolveDockerStartupCommandPatch( + { name: input.agent } as Parameters[0], + true, + ); + if (!launch.managedStartupRootApplyRequest) { + throw new Error("managed-image launch did not retain the bounded root-apply request"); + } + const startupPatch = createDockerGpuSandboxCreatePatch({ + route: "none", + persistStartupCommand: startupPlan.persistStartupCommand, + managedStartupRootApplyRequest: launch.managedStartupRootApplyRequest, + sandboxName: input.sandbox, + openshellSandboxCommand: launch.sandboxStartupCommand, + requiredUlimits: startupPlan.requiredUlimits, + timeoutSecs: 240, + deps: { + runOpenshell: onboard.runOpenshell, + runCaptureOpenshell: onboard.runCaptureOpenshell, + sleep: onboard.sleepSeconds, + }, + }); + + createLogFd = fs.openSync(createLog, "a", 0o600); + const [createExecutable, ...renderedCreateArgs] = launch.createArgv; + if (!createExecutable) throw new Error("managed-image launch renderer returned empty argv"); + createChild = spawn(createExecutable, renderedCreateArgs, { + env: launch.sandboxEnv, + stdio: ["ignore", createLogFd, createLogFd], + }); + fs.closeSync(createLogFd); + createLogFd = null; + + await waitForSandboxProbe( + onboard, + input, + launch.sandboxEnv, + createChild, + createLog, + startupPatch, + ); + ownedContainerId = assertExactSandboxImage(input, networkName, launch.sandboxEnv); + process.stdout.write( + `OpenShell launched exact ${input.agent} PR image ${input.image} and applied its managed startup profile.\n`, + ); + } finally { + const cleanupErrors: string[] = []; + if (createLogFd !== null) fs.closeSync(createLogFd); + try { + await stopChild(createChild); + } catch (error) { + cleanupErrors.push(error instanceof Error ? error.message : String(error)); + } + if (onboard) { + commandResult( + onboard.openshellArgv(["sandbox", "delete", input.sandbox]), + process.env, + 15_000, + ); + } + stopProcess(readGatewayPid(stateDir)); + if (onboard) { + commandResult(onboard.openshellArgv(["gateway", "remove", "nemoclaw"]), process.env, 15_000); + } + try { + const resolved = exactHarnessContainerIds(input, networkName, process.env); + const cleanupContainerId = + resolved.exactIds.length === 1 ? (resolved.exactIds[0] ?? null) : null; + if (cleanupContainerId) { + const remove = commandResult( + ["docker", "rm", "-f", cleanupContainerId], + process.env, + 15_000, + ); + const verify = commandResult( + ["docker", "container", "inspect", cleanupContainerId], + process.env, + 15_000, + ); + if (verify.status === 0 || !isDockerNotFound(verify)) { + cleanupErrors.push( + `exact harness container ${cleanupContainerId} was not removed: ${commandDetail(remove)} ${commandDetail(verify)}`.trim(), + ); + } + } else if (resolved.exactIds.length > 1) { + cleanupErrors.push( + `refusing ambiguous exact harness container cleanup: ${resolved.exactIds.length} matches`, + ); + } else if (ownedContainerId) { + const verify = commandResult( + ["docker", "container", "inspect", ownedContainerId], + process.env, + 15_000, + ); + if (verify.status === 0 || !isDockerNotFound(verify)) { + cleanupErrors.push( + `could not prove exact harness container ${ownedContainerId} was removed: ${commandDetail(verify)}`, + ); + } + } + } catch (error) { + cleanupErrors.push(error instanceof Error ? error.message : String(error)); + } + const removeNetwork = commandResult( + ["docker", "network", "rm", networkName], + process.env, + 15_000, + ); + const verifyNetwork = commandResult( + ["docker", "network", "inspect", networkName], + process.env, + 15_000, + ); + if (verifyNetwork.status === 0 || !isDockerNotFound(verifyNetwork)) { + cleanupErrors.push( + `harness network ${networkName} was not removed: ${commandDetail(removeNetwork)} ${commandDetail(verifyNetwork)}`.trim(), + ); + } + try { + initialSandboxPolicy?.cleanup?.(); + } catch (error) { + cleanupErrors.push(error instanceof Error ? error.message : String(error)); + } + fs.rmSync(stateDir, { recursive: true, force: true }); + if (cleanupErrors.length > 0) { + throw new Error(`managed-image OpenShell cleanup failed: ${cleanupErrors.join("; ")}`); + } + } +} + +if (require.main === module) { + run(parseManagedImageOpenShellE2eInputs(process.argv.slice(2))).catch((error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + }); +} diff --git a/scripts/checks/vitest-project-overlap.mts b/scripts/checks/vitest-project-overlap.mts index e7519110f1..f9b914dce9 100644 --- a/scripts/checks/vitest-project-overlap.mts +++ b/scripts/checks/vitest-project-overlap.mts @@ -43,8 +43,10 @@ const INSTALLER_INTEGRATION_TESTS = new Set([ "test/install-clone-ref.test.ts", "test/install-express-prompt.test.ts", "test/install-express-wsl-ollama.test.ts", + "test/install-openshell-podman-components.test.ts", "test/install-openshell-version-pin.test.ts", "test/install-openshell-version-check.test.ts", + "test/install-podman-runtime.test.ts", "test/install-preflight-docker-bootstrap.test.ts", "test/install-preflight.test.ts", "test/install-station-controller-binding.test.ts", diff --git a/scripts/generate-openclaw-config.mts b/scripts/generate-openclaw-config.mts index 18f3a557a5..0b8ad52088 100755 --- a/scripts/generate-openclaw-config.mts +++ b/scripts/generate-openclaw-config.mts @@ -24,7 +24,8 @@ // NEMOCLAW_OPENCLAW_MANAGED_PROXY, NEMOCLAW_WEB_SEARCH_ENABLED, // NEMOCLAW_WEB_SEARCH_PROVIDER, // NEMOCLAW_OPENCLAW_OTEL, NEMOCLAW_OPENCLAW_OTEL_ENDPOINT, -// NEMOCLAW_OPENCLAW_OTEL_SERVICE_NAME, NEMOCLAW_OPENCLAW_OTEL_SAMPLE_RATE. +// NEMOCLAW_OPENCLAW_OTEL_SERVICE_NAME, NEMOCLAW_OPENCLAW_OTEL_SAMPLE_RATE, +// NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION. import { chmodSync, @@ -122,6 +123,28 @@ const WEB_SEARCH_PROVIDERS = { type WebSearchProvider = keyof typeof WEB_SEARCH_PROVIDERS; const DEFAULT_OPENCLAW_OTEL_ENDPOINT = "http://host.openshell.internal:4318"; const DEFAULT_OPENCLAW_OTEL_SERVICE_NAME = "openclaw-gateway"; +// Runtime-facing IDs declared by the built-in messaging manifests. Package +// selection remains manifest-derived in messaging-build-applier.mts; these IDs +// describe only the neutral config written after those packages are installed. +const MANAGED_IMAGE_OPENCLAW_CHANNEL_IDS = [ + "telegram", + "discord", + "openclaw-weixin", + "slack", + "whatsapp", + "msteams", +] as const; +const MANAGED_IMAGE_OPENCLAW_PLUGIN_IDS = [ + "telegram", + "discord", + "openclaw-weixin", + "slack", + "whatsapp", + "msteams", + "diagnostics-otel", + "brave", + "tavily", +] as const; const SCRIPT_PATH = fileURLToPath(import.meta.url); const SCRIPT_DIR = dirname(SCRIPT_PATH); @@ -1386,6 +1409,15 @@ export function buildConfig(env: Env = process.env): JsonObject { const pluginEntries: JsonObject = { bonjour: { enabled: false }, }; + const managedImageCapabilityUnion = readBooleanBuildFlag( + env, + "NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION", + ); + if (managedImageCapabilityUnion) { + for (const pluginId of MANAGED_IMAGE_OPENCLAW_PLUGIN_IDS) { + pluginEntries[pluginId] = { enabled: false }; + } + } const openclawOtel = buildOpenClawOtelConfig(env); if (openclawOtel) { pluginEntries["diagnostics-otel"] = { enabled: true }; @@ -1431,13 +1463,20 @@ export function buildConfig(env: Env = process.env): JsonObject { agentDefaults.compaction = managedInferenceCompaction; } + const channels: JsonObject = { defaults: {} }; + if (managedImageCapabilityUnion) { + for (const channelId of MANAGED_IMAGE_OPENCLAW_CHANNEL_IDS) { + channels[channelId] = { enabled: false }; + } + } + const config: JsonObject = { agents: { defaults: agentDefaults, list: buildAgentsList(extraAgents, extraAgentsPayload.main), }, models: { mode: "merge", providers }, - channels: { defaults: {} }, + channels, tools: openclawTools, update: { checkOnStart: false }, ...(securityAuditSuppressions.length > 0 @@ -1488,6 +1527,9 @@ export function buildConfig(env: Env = process.env): JsonObject { const tools = config.tools; tools.web ??= {}; tools.web.fetch = { enabled: true, useTrustedEnvProxy: true }; + if (managedImageCapabilityUnion) { + tools.web.search = { enabled: false }; + } if (env.NEMOCLAW_WEB_SEARCH_ENABLED === "1") { // OpenClaw 2026.5.x keeps provider-owned credentials under diff --git a/scripts/install-openshell.sh b/scripts/install-openshell.sh index 11751eac1f..6fb441dc6c 100755 --- a/scripts/install-openshell.sh +++ b/scripts/install-openshell.sh @@ -33,6 +33,20 @@ esac info "Detected $OS_LABEL ($ARCH_LABEL)" +COMPUTE_DRIVER_REQUEST="${NEMOCLAW_COMPUTE_DRIVER:-auto}" +COMPUTE_DRIVER_REQUEST="${COMPUTE_DRIVER_REQUEST#"${COMPUTE_DRIVER_REQUEST%%[![:space:]]*}"}" +COMPUTE_DRIVER_REQUEST="${COMPUTE_DRIVER_REQUEST%"${COMPUTE_DRIVER_REQUEST##*[![:space:]]}"}" +COMPUTE_DRIVER_REQUEST="$(printf '%s' "${COMPUTE_DRIVER_REQUEST:-auto}" | tr '[:upper:]' '[:lower:]')" +case "$COMPUTE_DRIVER_REQUEST" in + auto | docker) EFFECTIVE_COMPUTE_DRIVER="docker" ;; + podman) EFFECTIVE_COMPUTE_DRIVER="podman" ;; + *) fail "NEMOCLAW_COMPUTE_DRIVER must be one of: auto, docker, podman." ;; +esac +if [ "$EFFECTIVE_COMPUTE_DRIVER" = "podman" ] \ + && { [ "$OS" != "Linux" ] || [ "$ARCH_LABEL" != "x86_64" ]; }; then + fail "Native Podman support currently requires Linux x86_64; detected ${OS_LABEL} ${ARCH_LABEL}." +fi + # Minimum version required for native messaging credential rewrite and # round-trippable base policies: WebSocket text frames, provider-shaped # aliases, REST request bodies, MCP/JSON-RPC L7 enforcement, and @@ -225,6 +239,7 @@ installed_component_path() { selected_sandbox_component_path() { local openshell_bin="$1" local explicit_path="${NEMOCLAW_OPENSHELL_SANDBOX_BIN:-}" + [ "$EFFECTIVE_COMPUTE_DRIVER" = "podman" ] && return 0 # Darwin uses the VM driver and ships no standalone sandbox supervisor. # Ignore a leftover sibling unless the operator explicitly selected it. if [ "$OS" = "Darwin" ] && [ -z "$explicit_path" ]; then @@ -233,6 +248,10 @@ selected_sandbox_component_path() { installed_component_path "$openshell_bin" openshell-sandbox "$explicit_path" } +driver_requires_sandbox_component() { + [ "$OS" = "Linux" ] && [ "$EFFECTIVE_COMPUTE_DRIVER" = "docker" ] +} + canonical_file_path() { local target="$1" local link dir @@ -364,8 +383,10 @@ required_driver_bins_present() { sandbox_bin="$(selected_sandbox_component_path "$openshell_bin")" case "$OS" in Linux) - [ -f "$gateway_bin" ] && [ -x "$gateway_bin" ] \ - && [ -f "$sandbox_bin" ] && [ -x "$sandbox_bin" ] + [ -f "$gateway_bin" ] && [ -x "$gateway_bin" ] || return 1 + if driver_requires_sandbox_component; then + [ -f "$sandbox_bin" ] && [ -x "$sandbox_bin" ] + fi ;; Darwin) [ -f "$gateway_bin" ] && [ -x "$gateway_bin" ] @@ -380,7 +401,10 @@ required_driver_bins_installed_in_dir() { local dir="$1" case "$OS" in Linux) - [ -x "$dir/openshell-gateway" ] && [ -x "$dir/openshell-sandbox" ] + [ -x "$dir/openshell-gateway" ] || return 1 + if driver_requires_sandbox_component; then + [ -x "$dir/openshell-sandbox" ] + fi ;; Darwin) [ -x "$dir/openshell-gateway" ] @@ -450,7 +474,8 @@ openshell_has_required_messaging_features() { OPENSHELL_FEATURE_CHECK_ERROR="The explicit OpenShell gateway binary '$gateway_bin' is missing, unreadable, or not executable." return 1 fi - if [ -n "${NEMOCLAW_OPENSHELL_SANDBOX_BIN:-}" ] \ + if [ "$EFFECTIVE_COMPUTE_DRIVER" != "podman" ] \ + && [ -n "${NEMOCLAW_OPENSHELL_SANDBOX_BIN:-}" ] \ && { [ ! -f "$sandbox_bin" ] || [ ! -r "$sandbox_bin" ] || [ ! -x "$sandbox_bin" ]; }; then OPENSHELL_FEATURE_CHECK_ERROR="The explicit OpenShell sandbox binary '$sandbox_bin' is missing, unreadable, or not executable." return 1 @@ -753,7 +778,9 @@ install_macos_homebrew_formula() { } validate_explicit_component_override gateway "${NEMOCLAW_OPENSHELL_GATEWAY_BIN:-}" -validate_explicit_component_override sandbox "${NEMOCLAW_OPENSHELL_SANDBOX_BIN:-}" +if [ "$EFFECTIVE_COMPUTE_DRIVER" != "podman" ]; then + validate_explicit_component_override sandbox "${NEMOCLAW_OPENSHELL_SANDBOX_BIN:-}" +fi ACTIVE_OPENSHELL_BIN="" if command -v openshell >/dev/null 2>&1; then @@ -792,7 +819,11 @@ if command -v openshell >/dev/null 2>&1; then if ! version_gte "$MAX_VERSION" "$INSTALLED_VERSION"; then warn "openshell $INSTALLED_VERSION is above the maximum ($MAX_VERSION) supported by this NemoClaw release — reinstalling pinned OpenShell ${PIN_VERSION}..." elif ! required_driver_bins_present "$ACTIVE_OPENSHELL_BIN"; then - warn "openshell $INSTALLED_VERSION is missing Docker-driver binaries — reinstalling pinned OpenShell ${PIN_VERSION}..." + if [ "$EFFECTIVE_COMPUTE_DRIVER" = "docker" ]; then + warn "openshell $INSTALLED_VERSION is missing Docker-driver binaries — reinstalling pinned OpenShell ${PIN_VERSION}..." + else + warn "openshell $INSTALLED_VERSION is missing Podman-driver binaries — reinstalling pinned OpenShell ${PIN_VERSION}..." + fi elif ! openshell_has_required_messaging_features "$ACTIVE_OPENSHELL_BIN"; then fail "${OPENSHELL_FEATURE_CHECK_ERROR:-openshell $INSTALLED_VERSION is missing required messaging credential rewrite and MCP L7 policy support. Install an OpenShell build that includes provider aliases, WebSocket text rewrite, request-body credential rewrite, and MCP/JSON-RPC L7 policy enforcement.}" elif [ "$OS" = "Darwin" ] && command -v brew >/dev/null 2>&1 && ! macos_homebrew_formula_installed; then @@ -853,15 +884,19 @@ case "$OS" in case "$ARCH_LABEL" in x86_64) ASSETS+=("openshell-gateway-x86_64-unknown-linux-gnu.tar.gz") - ASSETS+=("openshell-sandbox-x86_64-unknown-linux-gnu.tar.gz") ;; aarch64) ASSETS+=("openshell-gateway-aarch64-unknown-linux-gnu.tar.gz") - ASSETS+=("openshell-sandbox-aarch64-unknown-linux-gnu.tar.gz") ;; esac CHECKSUM_FILES+=("openshell-gateway-checksums-sha256.txt") - CHECKSUM_FILES+=("openshell-sandbox-checksums-sha256.txt") + if driver_requires_sandbox_component; then + case "$ARCH_LABEL" in + x86_64) ASSETS+=("openshell-sandbox-x86_64-unknown-linux-gnu.tar.gz") ;; + aarch64) ASSETS+=("openshell-sandbox-aarch64-unknown-linux-gnu.tar.gz") ;; + esac + CHECKSUM_FILES+=("openshell-sandbox-checksums-sha256.txt") + fi ;; esac @@ -994,7 +1029,7 @@ else fi required_driver_bins_installed_in_dir "$target_dir" \ - || fail "OpenShell release '$RELEASE_TAG' did not install the required Docker-driver binaries." + || fail "OpenShell release '$RELEASE_TAG' did not install the required ${EFFECTIVE_COMPUTE_DRIVER}-driver binaries." require_openshell_messaging_features "$target_dir/openshell" info "$("$target_dir/openshell" --version 2>&1 || echo openshell) installed" diff --git a/scripts/install.sh b/scripts/install.sh index a3997b792b..ae7d2412fb 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -50,6 +50,7 @@ resolve_repo_root() { DEFAULT_NEMOCLAW_VERSION="0.1.0" DEFAULT_INSTALL_REF="lkg" INSTALL_TAG_EXAMPLE="vX.Y.Z" +MIN_NATIVE_PODMAN_VERSION_MAJOR=5 TOTAL_STEPS=3 is_mutable_install_ref() { @@ -788,6 +789,8 @@ usage() { printf " --non-interactive Skip prompts (uses env vars / defaults)\n" printf " --yes-i-accept-third-party-software Accept the third-party software notice without prompting\n" printf " --fresh Discard any failed/interrupted onboarding session and start over\n" + printf " --compute-driver \n" + printf " Container runtime for OpenShell sandboxes (default: auto)\n" printf " --station-deepseek Use DeepSeek V4 Flash for DGX Station express install (interactive terminal required)\n" printf " --force-station-install Bypass only the DGX release-metadata allowlist for Station GB300 express install\n" printf " --version, -v Print installer version and exit\n" @@ -798,6 +801,7 @@ usage() { printf " NEMOCLAW_NON_INTERACTIVE=1 Same as --non-interactive\n" printf " NEMOCLAW_NON_INTERACTIVE_SUDO_MODE=prompt Allow sudo prompts during non-interactive onboarding\n" printf " NEMOCLAW_FRESH=1 Same as --fresh\n" + printf " NEMOCLAW_COMPUTE_DRIVER auto | docker | podman (flag takes precedence)\n" printf " NEMOCLAW_NO_EXPRESS=1 Skip express install prompt on supported platforms\n" printf " NEMOCLAW_SANDBOX_NAME Sandbox name to create/use\n" printf " HF_TOKEN Optional Hugging Face read token for managed-vLLM downloads\n" @@ -2773,7 +2777,18 @@ repair_installer_nvidia_cdi_spec() { fi } +installer_uses_native_podman() { + [ "${_INSTALLER_CONTAINER_RUNTIME:-docker}" = "podman" ] +} + run_installer_host_preflight() { + # Native Podman was qualified before installation and the CLI repeats the + # exact-socket runtime preflight during onboarding. This legacy remediation + # pass is Docker-specific (including CDI repair), so it must not discover or + # invoke Docker for a Podman-selected install. + if installer_uses_native_podman; then + return 0 + fi local preflight_module="${NEMOCLAW_SOURCE_ROOT}/dist/lib/onboard/preflight.js" if ! command_exists node || [[ ! -f "$preflight_module" ]]; then return 0 @@ -2925,6 +2940,10 @@ run_onboard() { show_usage_notice info "Running ${_CLI_BIN} onboard…" local -a onboard_cmd=(onboard) + if [ -n "${_INSTALLER_COMPUTE_DRIVER_REQUEST:-}" ] \ + && [ "$_INSTALLER_COMPUTE_DRIVER_REQUEST" != "auto" ]; then + onboard_cmd+=(--compute-driver "$_INSTALLER_COMPUTE_DRIVER_REQUEST") + fi local installer_auto_fresh_receipt_generation="" local session_file session_file="$(nemoclaw_state_dir)/onboard-session.json" @@ -3131,6 +3150,160 @@ station_express_receipt_retirement_pending() { ' "$session_file" >/dev/null 2>&1 } +installer_station_reconciliation_pending() { + installer_uses_native_podman && return 1 + [[ "${_SELECTED_EXPRESS_PLATFORM:-}" == "DGX Station" ]] \ + || [[ "${_STATION_EXPRESS_RESUME_LOADED:-}" == "1" ]] \ + || station_express_receipt_retirement_pending +} + +# Resolve the installer's runtime request through the same public values and +# precedence as `nemoclaw onboard`: a non-empty flag wins over the environment, +# matching the CLI resolver, and auto preserves the existing Docker bootstrap. +resolve_installer_compute_driver() { + local requested="${INSTALLER_COMPUTE_DRIVER_FLAG:-}" + requested="${requested#"${requested%%[![:space:]]*}"}" + requested="${requested%"${requested##*[![:space:]]}"}" + if [ -z "$requested" ]; then + requested="${NEMOCLAW_COMPUTE_DRIVER:-auto}" + requested="${requested#"${requested%%[![:space:]]*}"}" + requested="${requested%"${requested##*[![:space:]]}"}" + fi + requested="$(printf '%s' "${requested:-auto}" | tr '[:upper:]' '[:lower:]')" + case "$requested" in + auto | docker | podman) ;; + *) + error "NEMOCLAW_COMPUTE_DRIVER and --compute-driver must be one of: auto, docker, podman." + ;; + esac + + _INSTALLER_COMPUTE_DRIVER_REQUEST="$requested" + case "$requested" in + podman) _INSTALLER_CONTAINER_RUNTIME="podman" ;; + # Linux and supported macOS installs have always bootstrapped Docker. + # Keep that default stable until another runtime is explicitly selected. + auto | docker) _INSTALLER_CONTAINER_RUNTIME="docker" ;; + esac + export NEMOCLAW_COMPUTE_DRIVER="$_INSTALLER_COMPUTE_DRIVER_REQUEST" +} + +validate_installer_compute_driver_platform() { + installer_uses_native_podman || return 0 + local host_os host_arch + host_os="$(uname -s)" + host_arch="$(uname -m)" + if [ "$host_os" != "Linux" ] || { + [ "$host_arch" != "x86_64" ] && [ "$host_arch" != "amd64" ] + }; then + error "Native Podman support currently requires Linux x86_64; detected ${host_os} ${host_arch}." + fi + if [ "${STATION_DEEPSEEK:-}" = "1" ] || [ "${FORCE_STATION_INSTALL:-}" = "1" ]; then + error "DGX Station express-install flags currently require the Docker compute driver." + fi +} + +podman_socket_is_owned_by_current_user() { + local socket_path="$1" socket_uid + [ -S "$socket_path" ] && [ ! -L "$socket_path" ] || return 1 + socket_uid="$(stat -Lc '%u' -- "$socket_path" 2>/dev/null)" || return 1 + [ "$socket_uid" = "$(id -u)" ] +} + +podman_mapping_has_subordinate_range() { + awk ' + NF == 3 && $1 ~ /^[0-9]+$/ && $2 ~ /^[0-9]+$/ && $3 ~ /^[0-9]+$/ && $3 > 1 { + found = 1 + } + END { exit(found ? 0 : 1) } + ' +} + +ensure_native_podman() { + validate_installer_compute_driver_platform + command_exists podman \ + || error "Native Podman support requires Podman ${MIN_NATIVE_PODMAN_VERSION_MAJOR}.0 or newer. Install Podman and enable its rootless API socket, then rerun." + + local version_output version major + version_output="$(podman --version 2>/dev/null || true)" + version="$(printf '%s\n' "$version_output" | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1 || true)" + major="${version%%.*}" + if [ -z "$version" ] || ! [[ "$major" =~ ^[0-9]+$ ]] \ + || [ "$major" -lt "$MIN_NATIVE_PODMAN_VERSION_MAJOR" ]; then + error "Native Podman support requires Podman ${MIN_NATIVE_PODMAN_VERSION_MAJOR}.0 or newer; detected '${version:-unavailable}'." + fi + + local explicit_socket="${OPENSHELL_PODMAN_SOCKET:-}" + explicit_socket="${explicit_socket#"${explicit_socket%%[![:space:]]*}"}" + explicit_socket="${explicit_socket%"${explicit_socket##*[![:space:]]}"}" + local -a candidates=() + if [ -n "$explicit_socket" ]; then + candidates=("$explicit_socket") + else + if [ -n "${XDG_RUNTIME_DIR:-}" ]; then + candidates+=("${XDG_RUNTIME_DIR%/}/podman/podman.sock") + fi + candidates+=("/run/user/$(id -u)/podman/podman.sock") + if [ -n "${HOME:-}" ]; then + candidates+=("${HOME%/}/.local/share/containers/podman/podman.sock") + fi + fi + + local socket_path="" candidate + for candidate in "${candidates[@]}"; do + case "$candidate" in + /*) ;; + *) + if [ -n "$explicit_socket" ]; then + error "OPENSHELL_PODMAN_SOCKET must be an absolute path." + fi + continue + ;; + esac + [[ "$candidate" != *$'\n'* && "$candidate" != *$'\r'* ]] || continue + if podman_socket_is_owned_by_current_user "$candidate"; then + socket_path="$candidate" + break + fi + [ -z "$explicit_socket" ] \ + || error "OPENSHELL_PODMAN_SOCKET must name a current-user-owned, non-symlink Unix socket: ${candidate}" + done + [ -n "$socket_path" ] \ + || error "No current-user-owned rootless Podman API socket was found. Enable podman.socket or set OPENSHELL_PODMAN_SOCKET to its absolute path." + + local info_json compact_info + if ! info_json="$(podman --url "unix://${socket_path}" info --format json 2>/dev/null)"; then + error "The rootless Podman API socket is not responsive: ${socket_path}" + fi + compact_info="$(printf '%s' "$info_json" | tr -d '[:space:]')" + printf '%s' "$compact_info" \ + | grep -Eqi '"rootless":true' \ + || error "Native Podman support requires a rootless Podman API service." + printf '%s' "$compact_info" \ + | grep -Eqi '"cgroups?version":"v?2"' \ + || error "Native Podman support requires cgroups v2." + printf '%s' "$compact_info" \ + | grep -Eqi '"os":"linux"' \ + || error "The selected Podman API service is not reporting Linux." + printf '%s' "$compact_info" \ + | grep -Eqi '"arch":"(amd64|x86_64)"' \ + || error "The selected Podman API service is not reporting x86_64." + + local map_name map_output + for map_name in uid_map gid_map; do + if ! map_output="$(podman unshare cat "/proc/self/${map_name}" 2>/dev/null)" \ + || ! printf '%s\n' "$map_output" | podman_mapping_has_subordinate_range; then + if [ "$map_name" = "uid_map" ]; then + error "Rootless Podman requires a subordinate UID range for the current user (configure /etc/subuid)." + fi + error "Rootless Podman requires a subordinate GID range for the current user (configure /etc/subgid)." + fi + done + + OPENSHELL_PODMAN_SOCKET="$socket_path" + export OPENSHELL_PODMAN_SOCKET + ok "Rootless Podman ${version} is ready at ${socket_path}" +} + # Make sure Docker is installed and the current user can run it without # sudo. If we install Docker or add the user to the docker group, exit with # instructions to relogin/newgrp — Linux only loads group membership at @@ -4478,6 +4651,12 @@ clear_station_dual_pair_resume() { } prepare_installer_host() { + if installer_uses_native_podman; then + info "Using native Podman; Docker and DGX express host preparation are skipped." + ensure_native_podman + ensure_openshell_build_deps + return 0 + fi maybe_offer_express_install # Reject conflicting explicit Station selections and pending-pair bypasses # before the local host-preparation helper can mutate packages or Docker. @@ -4496,6 +4675,11 @@ prepare_installer_host() { ensure_openshell_build_deps } +run_installer_platform_setup() { + installer_uses_native_podman && return 0 + bash "${SCRIPT_DIR}/setup-jetson.sh" +} + # Prompt the user to opt into express install on supported platforms. Sets the # non-interactive + provider/model env vars when accepted. Skipped when # the user already passed --non-interactive, set NEMOCLAW_PROVIDER, or has @@ -4751,7 +4935,10 @@ main() { FRESH="" STATION_DEEPSEEK="" FORCE_STATION_INSTALL="" - for arg in "$@"; do + INSTALLER_COMPUTE_DRIVER_FLAG="" + while [ "$#" -gt 0 ]; do + local arg="$1" + shift case "$arg" in --non-interactive) NON_INTERACTIVE=1 @@ -4759,6 +4946,14 @@ main() { ;; --yes-i-accept-third-party-software) ACCEPT_THIRD_PARTY_SOFTWARE=1 ;; --fresh) FRESH=1 ;; + --compute-driver) + [ "$#" -gt 0 ] || error "--compute-driver requires one of: auto, docker, podman." + INSTALLER_COMPUTE_DRIVER_FLAG="$1" + shift + ;; + --compute-driver=*) + INSTALLER_COMPUTE_DRIVER_FLAG="${arg#*=}" + ;; --station-deepseek) STATION_DEEPSEEK=1 ;; --force-station-install) FORCE_STATION_INSTALL=1 ;; --version | -v) @@ -4784,6 +4979,8 @@ main() { fi ACCEPT_THIRD_PARTY_SOFTWARE="${ACCEPT_THIRD_PARTY_SOFTWARE:-${NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE:-}}" FRESH="${FRESH:-${NEMOCLAW_FRESH:-}}" + resolve_installer_compute_driver + validate_installer_compute_driver_platform # If the user explicitly accepted the third-party-software notice, treat # that as non-interactive intent for the rest of the run too — show_usage_notice @@ -4806,9 +5003,11 @@ main() { export NEMOCLAW_NON_INTERACTIVE="${NON_INTERACTIVE}" export NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE="${ACCEPT_THIRD_PARTY_SOFTWARE}" - load_station_vllm_conflict_helpers - if consume_station_local_vllm_resume; then - info "Resuming the selected manual Local vLLM setup." + if ! installer_uses_native_podman; then + load_station_vllm_conflict_helpers + if consume_station_local_vllm_resume; then + info "Resuming the selected manual Local vLLM setup." + fi fi # Validate the gateway port before the banner, notice acceptance, downloads, @@ -4819,7 +5018,9 @@ main() { # dependencies, or any other host mutation. maybe_offer_express_install # repeats the same authoritative validation at the prompt boundary because # it is also exercised directly by sourced-installer callers and tests. - preflight_explicit_express_flags + if ! installer_uses_native_podman; then + preflight_explicit_express_flags + fi print_banner @@ -4838,12 +5039,14 @@ main() { prepare_installer_host _INSTALL_START=$SECONDS - bash "${SCRIPT_DIR}/setup-jetson.sh" + run_installer_platform_setup step 1 "Node.js" install_nodejs ensure_supported_runtime - ensure_station_express_pair + if ! installer_uses_native_podman; then + ensure_station_express_pair + fi step 2 "${_CLI_DISPLAY} CLI" # Ollama and vLLM install/upgrade and model pulls are owned by @@ -4888,9 +5091,7 @@ main() { if [[ "${_PREEXISTING_SANDBOX_ORPHANED:-false}" == true ]]; then # #6520: do not claim recovery when recorded sandboxes are stranded. warn "Some recorded sandboxes could not be recovered; skipping generic onboarding." - elif [[ "${_SELECTED_EXPRESS_PLATFORM:-}" == "DGX Station" ]] \ - || [[ "${_STATION_EXPRESS_RESUME_LOADED:-}" == "1" ]] \ - || station_express_receipt_retirement_pending; then + elif installer_station_reconciliation_pending; then info "Existing sandboxes recovered; reconciling DGX Station Express onboarding state." run_onboard || error "Onboarding did not complete successfully." ONBOARD_RAN=true @@ -4912,7 +5113,9 @@ main() { fi finalize_install - clear_station_resume_after_completed_onboarding + if ! installer_uses_native_podman; then + clear_station_resume_after_completed_onboarding + fi } clear_station_resume_after_completed_onboarding() { @@ -4934,7 +5137,7 @@ finalize_install() { if [[ "${_UPGRADE_SANDBOXES_FAILED:-false}" == true ]]; then error "Installation incomplete: one or more existing sandboxes failed to upgrade. See the recovery guidance above." fi - if [[ "${_STATION_LOCAL_VLLM_SELECTED:-}" == "1" ]]; then + if ! installer_uses_native_podman && [[ "${_STATION_LOCAL_VLLM_SELECTED:-}" == "1" ]]; then clear_station_local_vllm_resume fi } diff --git a/scripts/lib/entrypoint-env-wrapper.sh b/scripts/lib/entrypoint-env-wrapper.sh new file mode 100755 index 0000000000..095c3f597c --- /dev/null +++ b/scripts/lib/entrypoint-env-wrapper.sh @@ -0,0 +1,138 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Normalize OpenShell's sandbox-create command when an OCI runtime invokes the +# image ENTRYPOINT with the literal argv: +# +# env NAME=value ... nemoclaw-start [agent command...] +# +# This runs before any managed-startup gate. Only environment names emitted by +# NemoClaw's launch renderer are promoted into the root entrypoint process; +# interpreter/loader variables such as NODE_OPTIONS, BASH_ENV, PATH, and +# LD_PRELOAD therefore cannot be smuggled into the trusted profile applicator. +# +# Result: NEMOCLAW_ENTRYPOINT_NORMALIZED_ARGV contains the command tail. +nemoclaw_normalize_entrypoint_env_wrapper() { + NEMOCLAW_ENTRYPOINT_NORMALIZED_ARGV=("$@") + NEMOCLAW_ENTRYPOINT_NORMALIZED_ARGC="$#" + [ "$#" -gt 0 ] || return 0 + + case "$1" in + nemoclaw-start | /usr/local/bin/nemoclaw-start) + shift + NEMOCLAW_ENTRYPOINT_NORMALIZED_ARGV=("$@") + NEMOCLAW_ENTRYPOINT_NORMALIZED_ARGC="$#" + return 0 + ;; + env) ;; + *) return 0 ;; + esac + + local -a _nemoclaw_original_argv=("$@") + local -a _nemoclaw_assignments=() + local _nemoclaw_self_index=-1 + local _nemoclaw_index + local _nemoclaw_token + local _nemoclaw_name + local _nemoclaw_seen_names="|" + + # Locate only the exact self-wrapper grammar. A normal explicit command such + # as `env FOO=bar printenv` remains a user command and is not interpreted by + # this root-side bootstrap. + for ((_nemoclaw_index = 1; _nemoclaw_index < ${#_nemoclaw_original_argv[@]}; _nemoclaw_index += 1)); do + _nemoclaw_token="${_nemoclaw_original_argv[$_nemoclaw_index]}" + case "$_nemoclaw_token" in + nemoclaw-start | /usr/local/bin/nemoclaw-start) + _nemoclaw_self_index="$_nemoclaw_index" + break + ;; + *=*) ;; + *) break ;; + esac + done + + if [ "$_nemoclaw_self_index" -lt 0 ]; then + # A managed handoff must never silently degrade into an unmanaged command + # because the self-wrapper was absent or malformed. + for _nemoclaw_token in "${_nemoclaw_original_argv[@]:1}"; do + case "$_nemoclaw_token" in + NEMOCLAW_STARTUP_PROFILE_B64=* | NEMOCLAW_CORPORATE_CA_B64=*) + printf '%s\n' \ + '[SECURITY] Malformed managed startup env wrapper; expected nemoclaw-start after assignments.' >&2 + return 1 + ;; + esac + done + return 0 + fi + + if [ "$_nemoclaw_self_index" -gt 65 ]; then + printf '%s\n' '[SECURITY] Managed startup env wrapper has too many assignments.' >&2 + return 1 + fi + + for ((_nemoclaw_index = 1; _nemoclaw_index < _nemoclaw_self_index; _nemoclaw_index += 1)); do + _nemoclaw_token="${_nemoclaw_original_argv[$_nemoclaw_index]}" + _nemoclaw_name="${_nemoclaw_token%%=*}" + if [ "${#_nemoclaw_token}" -gt 122880 ] \ + || [[ ! "$_nemoclaw_name" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]] \ + || [[ "$_nemoclaw_token" == *$'\n'* ]] \ + || [[ "$_nemoclaw_token" == *$'\r'* ]]; then + printf '%s\n' '[SECURITY] Managed startup env wrapper contains a malformed assignment.' >&2 + return 1 + fi + case "$_nemoclaw_name" in + AWS_EC2_METADATA_DISABLED | \ + CHAT_UI_URL | \ + HTTP_PROXY | HTTPS_PROXY | NO_PROXY | \ + http_proxy | https_proxy | no_proxy | \ + OPENCLAW_HOME | OPENCLAW_STATE_DIR | OPENCLAW_WORKSPACE_DIR | \ + NEMOCLAW_AUTO_PAIR_DEADLINE_SECS | \ + NEMOCLAW_AUTO_PAIR_FAST_DEADLINE_SECS | \ + NEMOCLAW_AUTO_PAIR_RUN_TIMEOUT_SECS | \ + NEMOCLAW_AUTO_PAIR_SLOW_INTERVAL_SECS | \ + NEMOCLAW_CORPORATE_CA_B64 | \ + NEMOCLAW_DASHBOARD_BIND | NEMOCLAW_DASHBOARD_PORT | \ + NEMOCLAW_EXTRA_PLACEHOLDER_KEYS | \ + NEMOCLAW_HERMES_DASHBOARD | \ + NEMOCLAW_HERMES_DASHBOARD_INTERNAL_PORT | \ + NEMOCLAW_HERMES_DASHBOARD_PORT | \ + NEMOCLAW_HERMES_DASHBOARD_TUI | \ + NEMOCLAW_MINIMAL_BOOTSTRAP | \ + NEMOCLAW_OBSERVABILITY | \ + NEMOCLAW_PROXY_HOST | NEMOCLAW_PROXY_PORT | \ + NEMOCLAW_SANDBOX_NAME | \ + NEMOCLAW_STARTUP_PROFILE_B64) ;; + *) + printf '%s\n' \ + "[SECURITY] Managed startup env wrapper contains unsupported variable '${_nemoclaw_name}'." >&2 + return 1 + ;; + esac + case "$_nemoclaw_seen_names" in + *"|${_nemoclaw_name}|"*) + printf '%s\n' \ + "[SECURITY] Managed startup env wrapper repeats variable '${_nemoclaw_name}'." >&2 + return 1 + ;; + esac + _nemoclaw_assignments+=("$_nemoclaw_token") + _nemoclaw_seen_names="${_nemoclaw_seen_names}${_nemoclaw_name}|" + done + + # Export only after the complete vector has passed validation so malformed + # input cannot leave a partially mutated root process. + if [ "$_nemoclaw_self_index" -gt 1 ]; then + for _nemoclaw_token in "${_nemoclaw_assignments[@]}"; do + export "${_nemoclaw_token?}" + done + fi + # shellcheck disable=SC2034 # output array is consumed by the sourcing entrypoint + NEMOCLAW_ENTRYPOINT_NORMALIZED_ARGV=( + "${_nemoclaw_original_argv[@]:$((_nemoclaw_self_index + 1))}" + ) + # shellcheck disable=SC2034 # output count is consumed by the sourcing entrypoint + NEMOCLAW_ENTRYPOINT_NORMALIZED_ARGC=$((\ + ${#_nemoclaw_original_argv[@]} - _nemoclaw_self_index - 1)) +} diff --git a/scripts/managed-startup-hold.sh b/scripts/managed-startup-hold.sh new file mode 100755 index 0000000000..d8020b64da --- /dev/null +++ b/scripts/managed-startup-hold.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Trusted image-owned hold for managed startup. The image OCI user remains root +# for the OpenShell supervisor, which deliberately drops the sandbox startup +# command to sandbox:sandbox before entering this hold. The host separately +# applies one bounded profile as root to the exact final container. No agent +# process starts until the root-owned marker authenticates the exact +# runtime-environment handoff. + +set -euo pipefail +unset BASH_ENV ENV NODE_OPTIONS NODE_PATH +export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + +fail() { + printf '[SECURITY] Managed startup hold: %s\n' "$*" >&2 + exit 1 +} + +_nemoclaw_sandbox_uid="$(id -u sandbox)" || fail "sandbox uid is unavailable" +_nemoclaw_sandbox_gid="$(id -g sandbox)" || fail "sandbox gid is unavailable" +if [ "$(id -u)" -ne "$_nemoclaw_sandbox_uid" ] \ + || [ "$(id -g)" -ne "$_nemoclaw_sandbox_gid" ]; then + fail "must run as the sandbox account" +fi +[ "$#" -ge 4 ] || fail "expected --agent --profile-fingerprint " +[ "$1" = "--agent" ] || fail "agent argument is missing" +_nemoclaw_agent="$2" +[ "$3" = "--profile-fingerprint" ] || fail "profile fingerprint argument is missing" +_nemoclaw_fingerprint="$4" +shift 4 + +case "$_nemoclaw_agent" in + openclaw | hermes | langchain-deepagents-code) ;; + *) fail "agent is unsupported" ;; +esac +case "$_nemoclaw_fingerprint" in + *[!0-9a-f]* | "") fail "profile fingerprint must be lowercase SHA-256" ;; +esac +[ "${#_nemoclaw_fingerprint}" -eq 64 ] \ + || fail "profile fingerprint must be lowercase SHA-256" + +_nemoclaw_runtime="/usr/local/lib/nemoclaw/managed-startup-image-runtime.cjs" +_nemoclaw_runtime_env="/run/nemoclaw/managed-startup-runtime.env" +[ -f "$_nemoclaw_runtime" ] || fail "managed startup runtime is missing" + +/usr/local/bin/node "$_nemoclaw_runtime" \ + --wait-for-completion \ + --agent "$_nemoclaw_agent" \ + --profile-fingerprint "$_nemoclaw_fingerprint" + +if [ -L "$_nemoclaw_runtime_env" ] \ + || [ ! -f "$_nemoclaw_runtime_env" ] \ + || [ "$(stat -c '%u:%g:%a' "$_nemoclaw_runtime_env")" != "0:0:444" ]; then + fail "runtime environment failed root ownership validation" +fi + +# shellcheck disable=SC1090 # fixed root-owned path authenticated by exact digest above +. "$_nemoclaw_runtime_env" +unset NEMOCLAW_STARTUP_PROFILE_B64 NEMOCLAW_CORPORATE_CA_B64 +unset _nemoclaw_agent _nemoclaw_fingerprint _nemoclaw_runtime _nemoclaw_runtime_env +unset _nemoclaw_sandbox_uid _nemoclaw_sandbox_gid +unset -f fail +exec /usr/local/bin/nemoclaw-start "$@" diff --git a/scripts/nemoclaw-start.sh b/scripts/nemoclaw-start.sh index c4354d7ac1..28448fa692 100755 --- a/scripts/nemoclaw-start.sh +++ b/scripts/nemoclaw-start.sh @@ -38,6 +38,30 @@ set -euo pipefail # cannot resolve id/chown/chmod/tee from an attacker-controlled location. export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" +# managed-entrypoint-env-wrapper begin +_NEMOCLAW_ENTRYPOINT_ENV_WRAPPER="/usr/local/lib/nemoclaw/entrypoint-env-wrapper.sh" +if [ ! -f "$_NEMOCLAW_ENTRYPOINT_ENV_WRAPPER" ]; then + _NEMOCLAW_ENTRYPOINT_SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + _NEMOCLAW_ENTRYPOINT_ENV_WRAPPER="${_NEMOCLAW_ENTRYPOINT_SOURCE_DIR}/lib/entrypoint-env-wrapper.sh" + unset _NEMOCLAW_ENTRYPOINT_SOURCE_DIR +fi +if [ ! -f "$_NEMOCLAW_ENTRYPOINT_ENV_WRAPPER" ]; then + printf '%s\n' '[SECURITY] Required entrypoint env-wrapper normalizer is missing.' >&2 + exit 1 +fi +# shellcheck source=scripts/lib/entrypoint-env-wrapper.sh +source "$_NEMOCLAW_ENTRYPOINT_ENV_WRAPPER" +nemoclaw_normalize_entrypoint_env_wrapper "$@" +if [ "$NEMOCLAW_ENTRYPOINT_NORMALIZED_ARGC" -eq 0 ]; then + set -- +else + set -- "${NEMOCLAW_ENTRYPOINT_NORMALIZED_ARGV[@]}" +fi +unset NEMOCLAW_ENTRYPOINT_NORMALIZED_ARGC NEMOCLAW_ENTRYPOINT_NORMALIZED_ARGV \ + _NEMOCLAW_ENTRYPOINT_ENV_WRAPPER +unset -f nemoclaw_normalize_entrypoint_env_wrapper +# managed-entrypoint-env-wrapper end + # Reject an invalid explicit dashboard port before installing the tee/fd startup # capture below. Some CI Docker runners can drop very early fd4 output from # short-lived containers, and this validation is meant to be fail-fast and @@ -196,41 +220,6 @@ fi # ── Drop unnecessary Linux capabilities (shared) ──────────────── drop_capabilities /usr/local/bin/nemoclaw-start "$@" -# Normalize the sandbox-create bootstrap wrapper. Onboard launches the -# container as `env CHAT_UI_URL=... nemoclaw-start`, but this script is already -# the ENTRYPOINT. If we treat that wrapper as a real command, the root path will -# try `gosu sandbox env ... nemoclaw-start`, which fails on Spark/arm64 when -# no-new-privileges blocks gosu. Consume only the self-wrapper form and promote -# the env assignments into the current process. -if [ "${1:-}" = "env" ]; then - _raw_args=("$@") - _self_wrapper_index="" - for ((i = 1; i < ${#_raw_args[@]}; i += 1)); do - case "${_raw_args[$i]}" in - *=*) ;; - nemoclaw-start | /usr/local/bin/nemoclaw-start) - _self_wrapper_index="$i" - break - ;; - *) - break - ;; - esac - done - if [ -n "$_self_wrapper_index" ]; then - for ((i = 1; i < _self_wrapper_index; i += 1)); do - export "${_raw_args[$i]}" - done - set -- "${_raw_args[@]:$((_self_wrapper_index + 1))}" - fi -fi - -# Filter out direct self-invocation too. Since this script is the ENTRYPOINT, -# receiving our own name as $1 would otherwise recurse via the NEMOCLAW_CMD -# exec path. Only strip from $1 — later args with this name are legitimate. -case "${1:-}" in - nemoclaw-start | /usr/local/bin/nemoclaw-start) shift ;; -esac NEMOCLAW_CMD=("$@") # OpenShell blocks the link-local EC2 Instance Metadata Service. Force this @@ -3234,7 +3223,9 @@ merge_corporate_proxy_ca() { export _NEMOCLAW_CORPORATE_CA_MERGED=1 echo "[nemoclaw] merged corporate proxy CA into sandbox trust bundle (#6210)" >&2 } -merge_corporate_proxy_ca +if [ "${NEMOCLAW_MANAGED_STARTUP_APPLIED:-0}" != "1" ]; then + merge_corporate_proxy_ca +fi # Git TLS CA bundle fix (NemoClaw#2270). # OpenShell's L7 proxy does MITM TLS termination and re-signs with its own CA. diff --git a/src/lib/actions/sandbox/agents/managed-workload-rebuild-profile.ts b/src/lib/actions/sandbox/agents/managed-workload-rebuild-profile.ts new file mode 100644 index 0000000000..c97fd6b2b9 --- /dev/null +++ b/src/lib/actions/sandbox/agents/managed-workload-rebuild-profile.ts @@ -0,0 +1,135 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { resolveContextWindowForModel } from "../../../inference/context-window"; +import type { SandboxMessagingPlan } from "../../../messaging"; +import { shouldManageDashboardForAgent } from "../../../onboard/dashboard-runtime"; +import { resolveHermesDashboardOnboardState } from "../../../onboard/hermes-dashboard"; +import { resolveManagedStartupInferenceRoute } from "../../../onboard/inference-route"; +import { + type ManagedWorkloadRebuildCatalogHandoff, + type ManagedWorkloadRebuildHandoff, + stageManagedWorkloadRebuildProfile, +} from "../../../onboard/workload/rebuild"; +import type { RebuildRecreateOnboardOpts } from "../rebuild-gpu-opt-out"; +import type { RebuildTargetConfig } from "../rebuild-target-config"; + +export const managedRebuildProfileDependencies = { + resolveContextWindowForModel, +}; + +export function resolveManagedRebuildOpenClawReasoning( + provider: string, + compatibleEndpointReasoning: "true" | "false" | null, +): boolean { + return provider === "compatible-endpoint" && compatibleEndpointReasoning === "true"; +} + +export function resolveManagedRebuildOpenClawReasoningEffort( + provider: string, + inferenceApi: string, + compatibleEndpointReasoningEffort: "low" | "medium" | "high" | null, +): "default" | "low" | "medium" | "high" { + return provider === "compatible-endpoint" && inferenceApi === "openai-completions" + ? (compatibleEndpointReasoningEffort ?? "default") + : "default"; +} + +/** + * Overlay mutable authoritative rebuild state onto receipt-owned managed-image + * affordances and render the exact replacement profile before destructive work. + */ +export function prepareManagedRebuildProfileHandoff(input: { + readonly catalogHandoff: ManagedWorkloadRebuildCatalogHandoff; + readonly targetConfig: RebuildTargetConfig; + readonly recreateOptions: RebuildRecreateOnboardOpts; + readonly messagingPlan: SandboxMessagingPlan | null; + readonly environment?: NodeJS.ProcessEnv; +}): ManagedWorkloadRebuildHandoff { + const { catalogHandoff, targetConfig, recreateOptions, messagingPlan } = input; + const agent = catalogHandoff.agent; + const { resumeConfig, durableConfig } = targetConfig; + const manageDashboard = shouldManageDashboardForAgent(targetConfig.agentDefinition); + const effectiveDashboardPort = manageDashboard ? (recreateOptions.controlUiPort ?? 0) : 0; + const chatUiUrl = manageDashboard ? `http://127.0.0.1:${String(effectiveDashboardPort)}` : ""; + const inference = resolveManagedStartupInferenceRoute( + agent, + resumeConfig.provider, + resumeConfig.model, + resumeConfig.preferredInferenceApi, + ); + const previousDashboard = catalogHandoff.previousProfile.dashboard; + const currentOpenClawContextWindow = + agent === "openclaw" + ? managedRebuildProfileDependencies.resolveContextWindowForModel( + resumeConfig.provider, + resumeConfig.model, + ) + : null; + if ( + agent === "openclaw" && + currentOpenClawContextWindow === null && + (catalogHandoff.previousProfile.inference.model !== resumeConfig.model || + catalogHandoff.previousProfile.inference.upstreamProvider !== resumeConfig.provider) + ) { + throw new Error( + `Cannot determine a context window for the current OpenClaw target '${resumeConfig.provider}/${resumeConfig.model}'.`, + ); + } + + return stageManagedWorkloadRebuildProfile( + catalogHandoff, + { + inference: { + routeProvider: inference.providerKey, + upstreamProvider: resumeConfig.provider, + model: resumeConfig.model, + routedBaseUrl: inference.inferenceBaseUrl, + upstreamEndpointUrl: + agent === "langchain-deepagents-code" ? resumeConfig.endpointUrl : null, + api: inference.inferenceApi as + | "openai-completions" + | "openai-responses" + | "anthropic-messages", + primaryModelRef: agent === "openclaw" ? inference.primaryModelRef : null, + compatibility: agent === "openclaw" ? (inference.inferenceCompat ?? {}) : null, + }, + chatUiUrl, + effectiveDashboardPort, + manageDashboard, + dashboardBindAddress: + previousDashboard.agent === "openclaw" && previousDashboard.bindAddress === "0.0.0.0" + ? "0.0.0.0" + : undefined, + wslExposure: previousDashboard.agent === "openclaw" && previousDashboard.wslExposure, + hermesDashboardState: resolveHermesDashboardOnboardState({ + agentName: agent, + effectivePort: effectiveDashboardPort, + env: input.environment ?? process.env, + }), + webSearch: durableConfig.webSearchConfig, + toolDisclosure: recreateOptions.toolDisclosure, + hermesToolGateways: targetConfig.hermesToolGateways, + messagingPlan, + dcodeAutoApprovalMode: recreateOptions.dcodeAutoApprovalMode, + observabilityEnabled: recreateOptions.observabilityEnabled, + }, + input.environment, + agent === "openclaw" + ? { + ...(currentOpenClawContextWindow === null + ? {} + : { openClawContextWindow: currentOpenClawContextWindow }), + openClawReasoning: resolveManagedRebuildOpenClawReasoning( + resumeConfig.provider, + resumeConfig.compatibleEndpointReasoning, + ), + openClawReasoningEffort: resolveManagedRebuildOpenClawReasoningEffort( + resumeConfig.provider, + inference.inferenceApi, + resumeConfig.compatibleEndpointReasoningEffort, + ), + } + : {}, + ); +} diff --git a/src/lib/actions/sandbox/connect-route-repair.test.ts b/src/lib/actions/sandbox/connect-route-repair.test.ts index c6092b14e3..f2aaaf74f8 100644 --- a/src/lib/actions/sandbox/connect-route-repair.test.ts +++ b/src/lib/actions/sandbox/connect-route-repair.test.ts @@ -199,6 +199,23 @@ describe("sandbox connect route repair unit flow", () => { expect(calls.logs).toContain(" inference.local route repaired."); }); + it("uses inference route reapply without Docker repair for Podman sandboxes", () => { + const { calls, deps } = makeRepairDeps([broken(), healthy()]); + + const result = repairSandboxInferenceRouteWithDeps( + "podman-box", + sandbox({ openshellDriver: "podman" }), + {}, + deps, + ); + + expect(result.healthy).toBe(true); + expect(result.repairAttempted).toBe(true); + expect(calls.legacyRepairs).toEqual([]); + expect(calls.reapplications).toEqual(["podman-box"]); + expect(calls.logs).toContain(" inference.local route repaired."); + }); + it("lets the VM monkeypatch satisfy the route before inference reapply", () => { const { calls, deps } = makeRepairDeps([broken(), healthy()], { shouldApplyVmDnsMonkeypatch: vi.fn(() => true), diff --git a/src/lib/actions/sandbox/connect.ts b/src/lib/actions/sandbox/connect.ts index 6b8c97c336..e6b0eed1ba 100644 --- a/src/lib/actions/sandbox/connect.ts +++ b/src/lib/actions/sandbox/connect.ts @@ -405,14 +405,13 @@ function probeSandboxInferenceRoute( function shouldUseLegacyDnsProxyRepair(sb: SandboxEntry | null): boolean { // The legacy repair patches CoreDNS inside an `openshell-cluster-` - // container, which only the k3s/kubernetes gateway runs. The docker driver - // runs the gateway as `nemoclaw-openshell-gateway` with host networking, and - // the vm driver has no cluster container either, so both recover the route via - // `openshell inference set` instead of the cluster CoreDNS patch. Mirrors - // usesGatewayMetadataProbe (snapshot.ts) and the `!== "docker"` guard on the - // snapshot DNS-proxy step. (#3403) - const driver = sb?.openshellDriver; - return driver !== "vm" && driver !== "docker"; + // container, which only the legacy k3s/kubernetes gateway runs. Docker, + // Podman, vm, and future named drivers have no cluster container and must + // recover through their driver-neutral inference route instead of inheriting + // the Docker/kubectl CoreDNS patch. A missing legacy driver remains on the + // historical cluster path. (#3403) + const driver = sb?.openshellDriver?.trim().toLowerCase(); + return !driver || driver === "kubernetes"; } function reapplyVmInferenceRoute( diff --git a/src/lib/actions/sandbox/destroy-flow.test.ts b/src/lib/actions/sandbox/destroy-flow.test.ts index 0f34afa94d..ebb305ddf6 100644 --- a/src/lib/actions/sandbox/destroy-flow.test.ts +++ b/src/lib/actions/sandbox/destroy-flow.test.ts @@ -55,6 +55,24 @@ describe("destroySandbox flow", () => { expectSuccessfulLiveDestroy(harness, exitSpy); }); + it("carries the native Podman driver into final gateway cleanup without probing Docker", async () => { + const harness = createDestroyHarness({ openshellDriver: "podman" }); + harness.dockerCaptureSpy.mockImplementation(() => { + throw new Error("Docker must not be invoked for Podman destroy"); + }); + + await expect( + harness.destroySandbox("alpha", { yes: true, cleanupGateway: true }), + ).resolves.toBeUndefined(); + + expect(harness.dockerCaptureSpy).not.toHaveBeenCalled(); + expect(harness.cleanupGatewaySpy).toHaveBeenCalledWith( + "nemoclaw-19080", + harness.runOpenshellSpy, + { openshellDriver: "podman" }, + ); + }); + it("revokes the prior HTTPS-pin route only after confirmed deletion and registry removal", async () => { const routeId = "a".repeat(64); const harness = createDestroyHarness({ diff --git a/src/lib/actions/sandbox/destroy-gateway-cleanup.test.ts b/src/lib/actions/sandbox/destroy-gateway-cleanup.test.ts index 85e69d9d10..703c76e66f 100644 --- a/src/lib/actions/sandbox/destroy-gateway-cleanup.test.ts +++ b/src/lib/actions/sandbox/destroy-gateway-cleanup.test.ts @@ -1,14 +1,62 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { describe, expect, it, vi } from "vitest"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; import { hasNoLiveSandboxes } from "../../domain/sandbox/destroy"; +import { buildPodmanDriverGatewayEnv } from "../../onboard/compute/podman/gateway-env"; +import { + DOCKER_DRIVER_GATEWAY_CONFIG_NAME, + MANAGED_GATEWAY_RUNTIME_BINDING_NAME, +} from "../../onboard/docker-driver-gateway-config"; import { collectLiveSandboxProbeSnapshot, + type SandboxRuntimeContainerProbeAdapterRegistry, shouldCleanupGatewayAfterConfirmedFinalDestroy, } from "./destroy-gateway-cleanup"; +const runtimeBindingDirectories: string[] = []; + +function podmanSocketAuthority(socketPath: string) { + return { + directoryChain: [ + { + device: "8", + inode: "7000", + mode: "448", + ownerUid: "1000", + path: path.dirname(socketPath), + }, + ], + device: "8", + inode: "9001", + ownerUid: "1000", + socketPath, + } as const; +} + +function createPersistedPodmanBinding(socketPath: string): string { + const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-destroy-podman-binding-")); + runtimeBindingDirectories.push(stateDir); + buildPodmanDriverGatewayEnv({ + gatewayPort: 8080, + stateDir, + podmanSocketPath: socketPath, + supervisorImage: "ghcr.io/nvidia/openshell/supervisor@sha256:test", + }); + return stateDir; +} + +afterEach(() => { + vi.restoreAllMocks(); + for (const directory of runtimeBindingDirectories.splice(0)) { + fs.rmSync(directory, { force: true, recursive: true }); + } +}); + describe("shouldCleanupGatewayAfterConfirmedFinalDestroy", () => { it("defers live probes until the local registry is empty", () => { const liveSandboxProbe = vi.fn(() => true); @@ -80,7 +128,7 @@ describe("shouldCleanupGatewayAfterConfirmedFinalDestroy", () => { expect(events).toEqual(["registry-empty", "live-sandbox-observed"]); }); - it("collects OpenShell and Docker live-sandbox snapshots in the action layer", () => { + it("collects OpenShell and legacy Docker live-sandbox snapshots in the action layer", () => { const captureOpenshell = vi.fn(() => ({ status: 0, output: @@ -104,6 +152,7 @@ describe("shouldCleanupGatewayAfterConfirmedFinalDestroy", () => { timeout: 1_000, }, ); + expect(snapshot.runtimeContainersBySandboxName.get("npmtest")).toEqual({ present: true }); expect(hasNoLiveSandboxes(snapshot)).toBe(false); }); @@ -122,12 +171,268 @@ describe("shouldCleanupGatewayAfterConfirmedFinalDestroy", () => { }); expect(hasNoLiveSandboxes(snapshot)).toBe(false); - expect(snapshot.dockerContainersBySandboxName.get("npmtest")).toEqual({ - output: "", + expect(snapshot.runtimeContainersBySandboxName.get("npmtest")).toEqual({ + present: false, probeFailed: true, }); expect(warn).toHaveBeenCalledWith( "Docker container probe failed for sandbox 'npmtest'; preserving shared gateway: Error: docker unavailable", ); }); + + it("uses the exact Podman socket and OpenShell labels without touching Docker", () => { + const socketPath = "/run/user/1000/podman/podman.sock"; + const stateDir = createPersistedPodmanBinding(socketPath); + const dockerCapture = vi.fn(() => { + throw new Error("Docker must not be invoked for Podman"); + }); + const captureHostCommand = vi.fn(() => ({ + status: 0, + stdout: '[{"Id":"podman-container"}]\n', + stderr: "", + })); + const authority = podmanSocketAuthority(socketPath); + + const snapshot = collectLiveSandboxProbeSnapshot({ + assertPodmanSocketAuthority: vi.fn(), + captureOpenshell: () => ({ + status: 0, + output: + "NAME CREATED PHASE\nnpmtest now Error\n", + }), + captureHostCommand, + capturePodmanSocketAuthority: vi.fn(() => authority), + dockerCapture, + environment: { + OPENSHELL_PODMAN_SOCKET: socketPath, + }, + gatewayStateDir: stateDir, + openshellDriver: "podman", + timeoutMs: 1_000, + }); + + expect(dockerCapture).not.toHaveBeenCalled(); + expect(captureHostCommand).toHaveBeenCalledWith( + "podman", + [ + "--url", + "unix:///run/user/1000/podman/podman.sock", + "ps", + "--all", + "--filter", + "label=openshell.ai/managed-by=openshell", + "--filter", + "label=openshell.ai/sandbox-name=npmtest", + "--format", + "json", + ], + 1_000, + ); + expect(snapshot.runtimeContainersBySandboxName.get("npmtest")).toEqual({ present: true }); + expect(hasNoLiveSandboxes(snapshot)).toBe(false); + }); + + it("recovers the exact Podman socket from the target gateway binding in a fresh shell", () => { + const socketPath = "/run/user/1000/podman/persisted.sock"; + const stateDir = createPersistedPodmanBinding(socketPath); + const dockerCapture = vi.fn(() => { + throw new Error("Docker must not be invoked for persisted Podman cleanup"); + }); + const captureHostCommand = vi.fn(() => ({ + status: 0, + stdout: "[]\n", + stderr: "", + })); + const authority = podmanSocketAuthority(socketPath); + + const snapshot = collectLiveSandboxProbeSnapshot({ + assertPodmanSocketAuthority: vi.fn(), + captureOpenshell: () => ({ + status: 0, + output: + "NAME CREATED PHASE\nnpmtest now Error\n", + }), + captureHostCommand, + capturePodmanSocketAuthority: vi.fn(() => authority), + dockerCapture, + environment: {}, + gatewayStateDir: stateDir, + openshellDriver: "podman", + timeoutMs: 1_000, + }); + + expect(dockerCapture).not.toHaveBeenCalled(); + expect(captureHostCommand).toHaveBeenCalledWith( + "podman", + [ + "--url", + `unix://${socketPath}`, + "ps", + "--all", + "--filter", + "label=openshell.ai/managed-by=openshell", + "--filter", + "label=openshell.ai/sandbox-name=npmtest", + "--format", + "json", + ], + 1_000, + ); + expect(hasNoLiveSandboxes(snapshot)).toBe(true); + }); + + it("preserves the shared gateway when Podman socket authority changes before its probe", () => { + const socketPath = "/run/user/1000/podman/persisted.sock"; + const stateDir = createPersistedPodmanBinding(socketPath); + const captureHostCommand = vi.fn(); + const assertPodmanSocketAuthority = vi + .fn() + .mockImplementationOnce(() => {}) + .mockImplementationOnce(() => { + throw new Error("Podman socket authority changed after it was qualified."); + }); + const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); + + const snapshot = collectLiveSandboxProbeSnapshot({ + assertPodmanSocketAuthority, + captureOpenshell: () => ({ + status: 0, + output: + "NAME CREATED PHASE\nnpmtest now Error\n", + }), + captureHostCommand, + capturePodmanSocketAuthority: vi.fn(() => podmanSocketAuthority(socketPath)), + environment: {}, + gatewayStateDir: stateDir, + openshellDriver: "podman", + timeoutMs: 1_000, + }); + + expect(captureHostCommand).not.toHaveBeenCalled(); + expect(hasNoLiveSandboxes(snapshot)).toBe(false); + expect(snapshot.runtimeContainersBySandboxName.get("npmtest")).toEqual({ + present: false, + probeFailed: true, + }); + expect(warn).toHaveBeenCalledWith(expect.stringContaining("socket authority changed")); + }); + + it.each([ + [ + "driver mismatch", + (stateDir: string) => { + const bindingPath = path.join(stateDir, MANAGED_GATEWAY_RUNTIME_BINDING_NAME); + const binding = JSON.parse(fs.readFileSync(bindingPath, "utf8")) as Record; + binding.driverName = "docker"; + fs.writeFileSync(bindingPath, `${JSON.stringify(binding, null, 2)}\n`, { mode: 0o600 }); + }, + /declares driver 'docker', not 'podman'/, + ], + [ + "config tampering", + (stateDir: string) => { + fs.appendFileSync( + path.join(stateDir, DOCKER_DRIVER_GATEWAY_CONFIG_NAME), + "\n# injected drift\n", + ); + }, + /does not match its gateway configuration/, + ], + ] as const)("fails closed on persisted Podman binding %s without touching Docker", (_case, tamper, expected) => { + const stateDir = createPersistedPodmanBinding("/run/user/1000/podman/persisted.sock"); + tamper(stateDir); + const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); + const dockerCapture = vi.fn(() => { + throw new Error("Docker must not be the binding recovery fallback"); + }); + const captureHostCommand = vi.fn(() => { + throw new Error("Podman must not run with untrusted binding evidence"); + }); + + const snapshot = collectLiveSandboxProbeSnapshot({ + captureOpenshell: () => ({ + status: 0, + output: + "NAME CREATED PHASE\nnpmtest now Failed\n", + }), + captureHostCommand, + dockerCapture, + environment: {}, + gatewayStateDir: stateDir, + openshellDriver: "podman", + timeoutMs: 1_000, + }); + + expect(dockerCapture).not.toHaveBeenCalled(); + expect(captureHostCommand).not.toHaveBeenCalled(); + expect(hasNoLiveSandboxes(snapshot)).toBe(false); + expect(warn).toHaveBeenCalledWith(expect.stringMatching(expected)); + }); + + it("accepts an injected MXC container probe without inheriting Docker or Podman behavior", () => { + const createProbe = vi.fn(() => (sandboxName: string) => ({ + present: sandboxName === "npmtest", + })); + const runtimeContainerProbeAdapters = { + mxc: { + displayName: "MXC", + driverName: "mxc", + createProbe, + }, + } satisfies SandboxRuntimeContainerProbeAdapterRegistry; + const dockerCapture = vi.fn(() => { + throw new Error("MXC must not inherit Docker probing"); + }); + const captureHostCommand = vi.fn(() => { + throw new Error("MXC must not inherit Podman probing"); + }); + + const snapshot = collectLiveSandboxProbeSnapshot({ + captureOpenshell: () => ({ + status: 0, + output: + "NAME CREATED PHASE\nnpmtest now Error\n", + }), + captureHostCommand, + dockerCapture, + openshellDriver: "mxc", + runtimeContainerProbeAdapters, + timeoutMs: 1_000, + }); + + expect(createProbe).toHaveBeenCalledOnce(); + expect(dockerCapture).not.toHaveBeenCalled(); + expect(captureHostCommand).not.toHaveBeenCalled(); + expect(snapshot.runtimeContainersBySandboxName.get("npmtest")).toEqual({ present: true }); + expect(hasNoLiveSandboxes(snapshot)).toBe(false); + }); + + it("does not inherit Docker probing for an unregistered future runtime", () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); + const dockerCapture = vi.fn(() => { + throw new Error("Docker must not be the future-runtime fallback"); + }); + const captureHostCommand = vi.fn(() => { + throw new Error("No runtime adapter is registered"); + }); + + const snapshot = collectLiveSandboxProbeSnapshot({ + captureOpenshell: () => ({ + status: 0, + output: + "NAME CREATED PHASE\nnpmtest now Failed\n", + }), + captureHostCommand, + dockerCapture, + openshellDriver: "mxc", + timeoutMs: 1_000, + }); + + expect(dockerCapture).not.toHaveBeenCalled(); + expect(captureHostCommand).not.toHaveBeenCalled(); + expect(hasNoLiveSandboxes(snapshot)).toBe(false); + expect(warn).toHaveBeenCalledWith( + "mxc container probe failed for sandbox 'npmtest'; preserving shared gateway: no runtime container probe is registered", + ); + }); }); diff --git a/src/lib/actions/sandbox/destroy-gateway-cleanup.ts b/src/lib/actions/sandbox/destroy-gateway-cleanup.ts index d6d2fc7ed4..23bcdd3c74 100644 --- a/src/lib/actions/sandbox/destroy-gateway-cleanup.ts +++ b/src/lib/actions/sandbox/destroy-gateway-cleanup.ts @@ -7,10 +7,32 @@ import { dockerSandboxContainerNamePrefix, getLiveSandboxNames, hasNoLiveSandboxes, + hasRunningDockerSandboxContainer, type LiveSandboxListSnapshot, + type SandboxRuntimeContainerSnapshot, shouldCleanupGatewayAfterDestroy, } from "../../domain/sandbox/destroy"; +import { + CURRENT_MANAGED_GATEWAY_DRIVER_PROFILES, + type ManagedGatewayDriverProfile, + type ManagedGatewayDriverProfileRegistry, +} from "../../onboard/compute/managed-gateway-profile"; +import { + assertPodmanSocketAuthority, + capturePodmanSocketAuthority, + type PodmanSocketAuthority, +} from "../../onboard/compute/podman/socket-authority"; +import { + OPENSHELL_MANAGED_BY_LABEL, + OPENSHELL_MANAGED_BY_VALUE, + OPENSHELL_SANDBOX_NAME_LABEL, +} from "../../onboard/docker-driver-sandbox-recovery"; import * as registry from "../../state/registry"; +import { + type CommandCapture, + captureHostCommand, + resolvePodmanRuntimeSocket, +} from "./doctor-host-command"; type SandboxListProvider = () => { sandboxes: unknown[] }; @@ -20,15 +42,55 @@ type LiveSandboxListProbe = ( ) => LiveSandboxListSnapshot; type DockerCaptureProbe = (args: string[], opts?: Record) => string; +type HostCommandCaptureProbe = ( + command: string, + args: string[], + timeout?: number, +) => CommandCapture; + +export interface SandboxRuntimeContainerProbeContext { + readonly assertPodmanSocketAuthority: typeof assertPodmanSocketAuthority; + readonly captureDocker: DockerCaptureProbe; + readonly captureHostCommand: HostCommandCaptureProbe; + readonly capturePodmanSocketAuthority: typeof capturePodmanSocketAuthority; + readonly environment: NodeJS.ProcessEnv; + readonly gatewayStateDir?: string | null; + readonly liveSandboxNames: readonly string[]; + readonly resolvePodmanSocket: typeof resolvePodmanRuntimeSocket; + readonly timeoutMs: number; +} + +export interface SandboxRuntimeContainerProbeAdapter { + readonly displayName: string; + readonly driverName: string; + createProbe( + context: SandboxRuntimeContainerProbeContext, + ): (sandboxName: string) => SandboxRuntimeContainerSnapshot; +} + +export type SandboxRuntimeContainerProbeAdapterRegistry = Readonly< + Record +>; type LiveSandboxProbe = (deps?: { + assertPodmanSocketAuthority?: typeof assertPodmanSocketAuthority; captureOpenshell?: LiveSandboxListProbe; + captureHostCommand?: HostCommandCaptureProbe; + capturePodmanSocketAuthority?: typeof capturePodmanSocketAuthority; dockerCapture?: DockerCaptureProbe; + environment?: NodeJS.ProcessEnv; + gatewayStateDir?: string | null; + managedGatewayProfiles?: ManagedGatewayDriverProfileRegistry; + openshellDriver?: string | null; + resolvePodmanSocket?: typeof resolvePodmanRuntimeSocket; + runtimeContainerProbeAdapters?: SandboxRuntimeContainerProbeAdapterRegistry; timeoutMs?: number; }) => boolean; type FinalDestroyGatewayCleanupInput = { deleteSucceededOrAlreadyGone: boolean; + gatewayStateDir?: string | null; + openshellDriver?: string | null; removedRegistryEntry: boolean; }; @@ -52,10 +114,167 @@ function captureDockerContainers(...args: Parameters) { return dockerCapture(...args); } +function managedGatewayProfile( + driverName: string, + profiles: ManagedGatewayDriverProfileRegistry = CURRENT_MANAGED_GATEWAY_DRIVER_PROFILES, +): ManagedGatewayDriverProfile | null { + const profile = Object.hasOwn(profiles, driverName) ? profiles[driverName] : undefined; + return profile?.driverName === driverName ? profile : null; +} + +function runtimeProbeDriverName( + openshellDriver: string | null | undefined, + profiles: ManagedGatewayDriverProfileRegistry, +): string { + const driver = openshellDriver?.trim().toLowerCase(); + // Entries written before openshellDriver was persisted are legacy managed + // Docker entries. Unknown future drivers must fail closed instead of + // inheriting Docker compatibility behavior. + if (!driver) return "docker"; + const profile = managedGatewayProfile(driver, profiles); + return profile?.capabilities.legacyDockerGatewayCleanup === true ? "docker" : driver; +} + +function podmanContainerPresent(output: string): boolean { + const parsed = JSON.parse(output) as unknown; + if (!Array.isArray(parsed)) { + throw new Error("Podman container probe returned a non-array JSON payload"); + } + return parsed.length > 0; +} + +function failedRuntimeSnapshot( + sandboxName: string, + runtimeName: string, + error: unknown, +): SandboxRuntimeContainerSnapshot { + console.warn( + `${runtimeName} container probe failed for sandbox '${sandboxName}'; preserving shared gateway: ${String(error)}`, + ); + return { present: false, probeFailed: true }; +} + +export const CURRENT_SANDBOX_RUNTIME_CONTAINER_PROBE_ADAPTERS = { + docker: { + displayName: "Docker", + driverName: "docker", + createProbe(context: SandboxRuntimeContainerProbeContext) { + return (sandboxName: string): SandboxRuntimeContainerSnapshot => { + try { + const snapshot: DockerSandboxContainerSnapshot = { + output: context.captureDocker( + [ + "ps", + "--filter", + `name=${dockerSandboxContainerNamePrefix(sandboxName)}`, + "--format", + "{{.Names}}", + ], + { + timeout: context.timeoutMs, + }, + ), + }; + return { + present: hasRunningDockerSandboxContainer( + sandboxName, + snapshot, + context.liveSandboxNames, + ), + }; + } catch (error) { + // This probe follows a terminal OpenShell row and must attest that + // its backing container is absent. Unknown state preserves the + // shared gateway. + return failedRuntimeSnapshot(sandboxName, "Docker", error); + } + }; + }, + }, + podman: { + displayName: "Podman", + driverName: "podman", + createProbe(context: SandboxRuntimeContainerProbeContext) { + let socketPath = ""; + let socketAuthority: PodmanSocketAuthority | null = null; + let socketError: unknown; + try { + socketPath = context.resolvePodmanSocket(context.gatewayStateDir, context.environment); + socketAuthority = context.capturePodmanSocketAuthority(socketPath); + if (socketAuthority.socketPath !== socketPath) { + throw new Error("Podman cleanup probe authority does not match its runtime binding."); + } + context.assertPodmanSocketAuthority(socketAuthority); + } catch (error) { + socketError = error; + } + return (sandboxName: string): SandboxRuntimeContainerSnapshot => { + if (socketError) return failedRuntimeSnapshot(sandboxName, "Podman", socketError); + if (!socketAuthority) { + return failedRuntimeSnapshot( + sandboxName, + "Podman", + "runtime socket authority was not captured", + ); + } + try { + context.assertPodmanSocketAuthority(socketAuthority); + const result = context.captureHostCommand( + "podman", + [ + "--url", + `unix://${socketPath}`, + "ps", + "--all", + "--filter", + `label=${OPENSHELL_MANAGED_BY_LABEL}=${OPENSHELL_MANAGED_BY_VALUE}`, + "--filter", + `label=${OPENSHELL_SANDBOX_NAME_LABEL}=${sandboxName}`, + "--format", + "json", + ], + context.timeoutMs, + ); + context.assertPodmanSocketAuthority(socketAuthority); + if (result.status !== 0) { + return failedRuntimeSnapshot( + sandboxName, + "Podman", + result.stderr || result.error?.message || `podman exited ${String(result.status)}`, + ); + } + return { present: podmanContainerPresent(result.stdout) }; + } catch (error) { + return failedRuntimeSnapshot(sandboxName, "Podman", error); + } + }; + }, + }, +} as const satisfies SandboxRuntimeContainerProbeAdapterRegistry; + +export function resolveSandboxRuntimeContainerProbeAdapter( + openshellDriver: string | null | undefined, + adapters: SandboxRuntimeContainerProbeAdapterRegistry = CURRENT_SANDBOX_RUNTIME_CONTAINER_PROBE_ADAPTERS, + profiles: ManagedGatewayDriverProfileRegistry = CURRENT_MANAGED_GATEWAY_DRIVER_PROFILES, +): SandboxRuntimeContainerProbeAdapter | null { + const driverName = runtimeProbeDriverName(openshellDriver, profiles); + const adapter = Object.hasOwn(adapters, driverName) ? adapters[driverName] : undefined; + return adapter?.driverName === driverName ? adapter : null; +} + export function collectLiveSandboxProbeSnapshot( deps: { + assertPodmanSocketAuthority?: typeof assertPodmanSocketAuthority; captureOpenshell?: LiveSandboxListProbe; + captureHostCommand?: HostCommandCaptureProbe; + capturePodmanSocketAuthority?: typeof capturePodmanSocketAuthority; dockerCapture?: DockerCaptureProbe; + environment?: NodeJS.ProcessEnv; + gatewayStateDir?: string | null; + managedGatewayProfiles?: ManagedGatewayDriverProfileRegistry; + openshellDriver?: string | null; + resolvePodmanSocket?: typeof resolvePodmanRuntimeSocket; + runtimeContainerProbeAdapters?: SandboxRuntimeContainerProbeAdapterRegistry; timeoutMs?: number; } = {}, ): Parameters[0] { @@ -63,43 +282,50 @@ export function collectLiveSandboxProbeSnapshot( // after the registry check and before the cleanup decision. const captureOpenshell = deps.captureOpenshell ?? captureLiveSandboxes; const dockerCapture = deps.dockerCapture ?? captureDockerContainers; + const captureRuntimeCommand = deps.captureHostCommand ?? captureHostCommand; const timeoutMs = deps.timeoutMs ?? OPENSHELL_PROBE_TIMEOUT_MS; const liveList = captureOpenshell(["sandbox", "list"], { ignoreError: true, timeout: timeoutMs, }); - const dockerContainersBySandboxName = new Map(); - for (const sandboxName of getLiveSandboxNames(liveList)) { - try { - dockerContainersBySandboxName.set(sandboxName, { - output: dockerCapture( - [ - "ps", - "--filter", - `name=${dockerSandboxContainerNamePrefix(sandboxName)}`, - "--format", - "{{.Names}}", - ], - { - timeout: timeoutMs, - }, + const sandboxNames = getLiveSandboxNames(liveList); + const runtimeContainersBySandboxName = new Map(); + const adapter = resolveSandboxRuntimeContainerProbeAdapter( + deps.openshellDriver, + deps.runtimeContainerProbeAdapters, + deps.managedGatewayProfiles, + ); + const probe = + sandboxNames.length > 0 + ? adapter?.createProbe({ + assertPodmanSocketAuthority: + deps.assertPodmanSocketAuthority ?? assertPodmanSocketAuthority, + captureDocker: dockerCapture, + captureHostCommand: captureRuntimeCommand, + capturePodmanSocketAuthority: + deps.capturePodmanSocketAuthority ?? capturePodmanSocketAuthority, + environment: deps.environment ?? process.env, + gatewayStateDir: deps.gatewayStateDir, + liveSandboxNames: sandboxNames, + resolvePodmanSocket: deps.resolvePodmanSocket ?? resolvePodmanRuntimeSocket, + timeoutMs, + }) + : undefined; + for (const sandboxName of sandboxNames) { + if (!probe || !adapter) { + runtimeContainersBySandboxName.set( + sandboxName, + failedRuntimeSnapshot( + sandboxName, + deps.openshellDriver?.trim() || "Unknown runtime", + "no runtime container probe is registered", ), - }); - } catch (error) { - // SOURCE_OF_TRUTH: this host Docker CLI probe follows a terminal OpenShell - // row and must attest that its backing container is absent. An exception - // leaves live-sandbox state unknown, so preserve the shared gateway. - // NemoClaw cannot manufacture that container-runtime attestation here; - // destroy-gateway-cleanup.test.ts locks this fail-closed behavior. Remove - // it only when final cleanup has one authoritative sandbox/container state - // source; see the OpenShell listener-removal boundary tracked in #6639. - console.warn( - `Docker container probe failed for sandbox '${sandboxName}'; preserving shared gateway: ${String(error)}`, ); - dockerContainersBySandboxName.set(sandboxName, { output: "", probeFailed: true }); + continue; } + runtimeContainersBySandboxName.set(sandboxName, probe(sandboxName)); } - return { liveList, dockerContainersBySandboxName }; + return { liveList, runtimeContainersBySandboxName }; } function hasNoLiveSandboxesFromHost(deps?: Parameters[0]): boolean { @@ -119,6 +345,8 @@ export function shouldCleanupGatewayAfterConfirmedFinalDestroy( input.removedRegistryEntry && noRegisteredSandboxes && liveSandboxProbe({ + gatewayStateDir: input.gatewayStateDir, + openshellDriver: input.openshellDriver, timeoutMs, }); diff --git a/src/lib/actions/sandbox/destroy-gateway-runtime-evidence.test.ts b/src/lib/actions/sandbox/destroy-gateway-runtime-evidence.test.ts index 6868bbc3b2..8229084cdb 100644 --- a/src/lib/actions/sandbox/destroy-gateway-runtime-evidence.test.ts +++ b/src/lib/actions/sandbox/destroy-gateway-runtime-evidence.test.ts @@ -6,6 +6,7 @@ import os from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { MANAGED_GATEWAY_RUNTIME_BINDING_NAME } from "../../onboard/docker-driver-gateway-config"; const mocks = vi.hoisted(() => ({ dockerRemoveVolumesByPrefix: vi.fn(), @@ -61,8 +62,10 @@ describe("cleanupGatewayAfterLastSandbox runtime evidence", () => { let pidIsAlive = true; const pidFile = path.join(stateDir, "openshell-gateway.pid"); const runtimeMarker = path.join(stateDir, "runtime.json"); + const managedRuntimeBinding = path.join(stateDir, MANAGED_GATEWAY_RUNTIME_BINDING_NAME); fs.writeFileSync(pidFile, `${pid}\n`); fs.writeFileSync(runtimeMarker, '{"evidence":"keep-until-safe"}\n'); + fs.writeFileSync(managedRuntimeBinding, '{"driverName":"podman"}\n'); vi.spyOn(process, "platform", "get").mockReturnValue("linux"); const missingProcess = () => ({ status: 1, stdout: "", stderr: "" }); const processResponses = new Map([ @@ -86,6 +89,7 @@ describe("cleanupGatewayAfterLastSandbox runtime evidence", () => { ); expect(fs.readFileSync(pidFile, "utf-8")).toBe(`${pid}\n`); expect(fs.readFileSync(runtimeMarker, "utf-8")).toContain("keep-until-safe"); + expect(fs.readFileSync(managedRuntimeBinding, "utf-8")).toContain("podman"); expect(runOpenshell).not.toHaveBeenCalledWith( ["gateway", "remove", "nemoclaw-8081"], expect.anything(), @@ -96,6 +100,7 @@ describe("cleanupGatewayAfterLastSandbox runtime evidence", () => { expect(() => cleanupGatewayAfterLastSandbox("nemoclaw-8081", runOpenshell)).not.toThrow(); expect(fs.existsSync(pidFile)).toBe(false); expect(fs.existsSync(runtimeMarker)).toBe(false); + expect(fs.existsSync(managedRuntimeBinding)).toBe(false); expect(runOpenshell).toHaveBeenCalledWith(["gateway", "remove", "nemoclaw-8081"], { ignoreError: true, stdio: ["ignore", "pipe", "pipe"], diff --git a/src/lib/actions/sandbox/destroy-gateway.test.ts b/src/lib/actions/sandbox/destroy-gateway.test.ts index 7f10d13bf3..a79ea3fb34 100644 --- a/src/lib/actions/sandbox/destroy-gateway.test.ts +++ b/src/lib/actions/sandbox/destroy-gateway.test.ts @@ -117,7 +117,7 @@ describe("cleanupGatewayAfterLastSandbox", () => { afterEach(() => { vi.restoreAllMocks(); - vi.clearAllMocks(); + vi.resetAllMocks(); delete process.env.NEMOCLAW_OPENSHELL_GATEWAY_STATE_DIR; }); @@ -209,6 +209,62 @@ describe("cleanupGatewayAfterLastSandbox", () => { }); }); + it("never invokes legacy Docker cleanup for the native Podman profile", () => { + vi.spyOn(process, "platform", "get").mockReturnValue("linux"); + mocks.dockerRemoveVolumesByPrefix.mockImplementation(() => { + throw new Error("Docker volume cleanup must not run for Podman"); + }); + const runOpenshell = vi.fn((args: string[]) => + args[1] === "remove" + ? { status: 2, stdout: "", stderr: "injected removal failure" } + : { status: 0, stdout: "", stderr: "" }, + ); + const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); + const clearManagedGatewayRuntimeBinding = vi.fn(() => { + throw new Error("Ambiguous cleanup must retain the runtime binding"); + }); + + expect(() => + cleanupGatewayAfterLastSandbox("nemoclaw-8081", runOpenshell, { + clearManagedGatewayRuntimeBinding, + openshellDriver: "podman", + }), + ).not.toThrow(); + + expect(runOpenshell).toHaveBeenCalledWith(["gateway", "remove", "nemoclaw-8081"], { + ignoreError: true, + stdio: ["ignore", "pipe", "pipe"], + }); + expect(runOpenshell).not.toHaveBeenCalledWith( + ["gateway", "destroy", "-g", "nemoclaw-8081"], + expect.anything(), + ); + expect(mocks.dockerRemoveVolumesByPrefix).not.toHaveBeenCalled(); + expect(clearManagedGatewayRuntimeBinding).not.toHaveBeenCalled(); + expect(warn).toHaveBeenCalledWith( + expect.stringContaining("legacy Docker gateway cleanup is disabled"), + ); + }); + + it("clears the durable runtime binding only after successful Podman gateway removal", () => { + vi.spyOn(process, "platform", "get").mockReturnValue("linux"); + vi.stubEnv("NEMOCLAW_OPENSHELL_GATEWAY_STATE_DIR", "/runtime/nemoclaw-8081"); + mocks.dockerRemoveVolumesByPrefix.mockImplementation(() => { + throw new Error("Docker volume cleanup must not run for Podman"); + }); + const runOpenshell = vi.fn(() => ({ status: 0, stdout: "", stderr: "" })); + const clearManagedGatewayRuntimeBinding = vi.fn(); + + cleanupGatewayAfterLastSandbox("nemoclaw-8081", runOpenshell, { + clearManagedGatewayRuntimeBinding, + openshellDriver: "podman", + }); + + expect(mocks.dockerRemoveVolumesByPrefix).not.toHaveBeenCalled(); + expect(clearManagedGatewayRuntimeBinding).toHaveBeenCalledOnce(); + expect(clearManagedGatewayRuntimeBinding).toHaveBeenCalledWith("/runtime/nemoclaw-8081"); + }); + it("fails before gateway and volume removal when the owned host listener survives (#4662)", () => { vi.spyOn(process, "platform", "get").mockReturnValue("darwin"); vi.spyOn(os, "homedir").mockReturnValue("/home/tester"); diff --git a/src/lib/actions/sandbox/destroy-gateway.ts b/src/lib/actions/sandbox/destroy-gateway.ts index 32c495fe2c..00e4f3d099 100644 --- a/src/lib/actions/sandbox/destroy-gateway.ts +++ b/src/lib/actions/sandbox/destroy-gateway.ts @@ -7,6 +7,12 @@ import path from "node:path"; import { dockerRemoveVolumesByPrefix } from "../../adapters/docker/volume"; import { OPENSHELL_OPERATION_TIMEOUT_MS } from "../../adapters/openshell/timeouts"; import { DASHBOARD_PORT } from "../../core/ports"; +import { + CURRENT_MANAGED_GATEWAY_DRIVER_PROFILES, + type ManagedGatewayDriverCapabilities, + type ManagedGatewayDriverProfileRegistry, +} from "../../onboard/compute/managed-gateway-profile"; +import { clearManagedGatewayRuntimeBinding } from "../../onboard/docker-driver-gateway-config"; import { resolveGatewayPortFromName, resolveGatewayStateDirName, @@ -27,16 +33,36 @@ export type DestroyRunOpenshell = ( const DASHBOARD_FORWARD_PORT = String(DASHBOARD_PORT); export interface CleanupGatewayDeps { + clearManagedGatewayRuntimeBinding?: typeof clearManagedGatewayRuntimeBinding; + openshellDriver?: string | null; resolveGatewayTeardownAuthority?: GatewayTeardownAuthorityResolver; } +function permitsLegacyDockerCleanup( + openshellDriver: string | null | undefined, + capability: keyof Pick< + ManagedGatewayDriverCapabilities, + "legacyDockerGatewayCleanup" | "legacyDockerVolumeCleanup" + >, + profiles: ManagedGatewayDriverProfileRegistry = CURRENT_MANAGED_GATEWAY_DRIVER_PROFILES, +): boolean { + const driver = openshellDriver?.trim().toLowerCase(); + // Missing driver metadata predates the pluggable runtime registry and is + // therefore the one compatibility case that retains Docker cleanup. + if (!driver) return true; + const profile = Object.hasOwn(profiles, driver) ? profiles[driver] : undefined; + return profile?.driverName === driver && profile.capabilities[capability] === true; +} + // Compute the Docker-driver gateway state directory that belongs to // `gatewayName`. `stopHostGatewayProcesses` defaults to the bare leaf // `openshell-docker-gateway`, so without this override a destroy of a // `nemoclaw-` sandbox would read the default instance's pid file and // stop the wrong host gateway process. Returns null when the gateway name is // outside the NemoClaw namespace (the caller then keeps the defaults). -function resolvePerGatewayState(gatewayName: string): { port: number; stateDir: string } | null { +export function resolvePerGatewayState( + gatewayName: string, +): { port: number; stateDir: string } | null { const port = resolveGatewayPortFromName(gatewayName); if (port === null) return null; const configured = process.env.NEMOCLAW_OPENSHELL_GATEWAY_STATE_DIR; @@ -91,6 +117,16 @@ export function cleanupGatewayAfterLastSandbox( { env: process.env }, ); const externallySupervised = isExternallySupervised(owner); + const legacyDockerGatewayCleanup = permitsLegacyDockerCleanup( + deps.openshellDriver, + "legacyDockerGatewayCleanup", + ); + const legacyDockerVolumeCleanup = permitsLegacyDockerCleanup( + deps.openshellDriver, + "legacyDockerVolumeCleanup", + ); + const clearRuntimeBinding = + deps.clearManagedGatewayRuntimeBinding ?? clearManagedGatewayRuntimeBinding; const openshell = runOpenshell ?? (require("../../adapters/openshell/runtime") as { runOpenshell: DestroyRunOpenshell }) @@ -175,23 +211,34 @@ export function cleanupGatewayAfterLastSandbox( ignoreError: true, stdio: ["ignore", "pipe", "pipe"], }); + let gatewayRegistrationRemoved = removeResult.status === 0; if (removeResult.status !== 0) { if (externallySupervised) { console.warn( `Could not remove local registration for externally supervised gateway '${gatewayName}'. ` + "NemoClaw will not use the legacy gateway destroy command for an externally supervised gateway.", ); - } else { - openshell(["gateway", "destroy", "-g", gatewayName], { + } else if (legacyDockerGatewayCleanup) { + const legacyDestroyResult = openshell(["gateway", "destroy", "-g", gatewayName], { ignoreError: true, stdio: ["ignore", "pipe", "pipe"], }); + gatewayRegistrationRemoved = legacyDestroyResult.status === 0; + } else { + console.warn( + `Could not remove gateway registration '${gatewayName}'; legacy Docker gateway cleanup is disabled for OpenShell driver '${deps.openshellDriver || "unknown"}'.`, + ); } } if (externallySupervised) { return; } - dockerRemoveVolumesByPrefix(`openshell-cluster-${gatewayName}`, { - ignoreError: true, - }); + if (legacyDockerVolumeCleanup) { + dockerRemoveVolumesByPrefix(`openshell-cluster-${gatewayName}`, { + ignoreError: true, + }); + } + if (gatewayRegistrationRemoved) { + clearRuntimeBinding(perGatewayState.stateDir); + } } diff --git a/src/lib/actions/sandbox/destroy.ts b/src/lib/actions/sandbox/destroy.ts index 78a9483f8c..03041f986d 100644 --- a/src/lib/actions/sandbox/destroy.ts +++ b/src/lib/actions/sandbox/destroy.ts @@ -32,7 +32,7 @@ import { resolveNemoclawStateDir } from "../../state/paths"; import * as registry from "../../state/registry"; import { confirmSandboxDestroy } from "./destroy-confirmation"; import { executeSandboxDestroy } from "./destroy-execution"; -import { cleanupGatewayAfterLastSandbox } from "./destroy-gateway"; +import { cleanupGatewayAfterLastSandbox, resolvePerGatewayState } from "./destroy-gateway"; import { shouldCleanupGatewayAfterConfirmedFinalDestroy } from "./destroy-gateway-cleanup"; import { prepareSandboxDestroy } from "./destroy-preflight"; import { type WipeSandboxStateDeps, wipeSandboxState } from "./wipe-state"; @@ -216,7 +216,8 @@ export function removeShieldsState( } /** - * Remove the host-side Docker image that was built for a sandbox during onboard. + * Remove the host-side Docker image that was built only for this sandbox. + * Shared immutable managed-image cohorts remain in the runtime cache. * Must be called before registry.removeSandbox() since the imageTag is stored there. */ export function removeSandboxImage(sandboxName: string, deps: RemoveSandboxImageDeps = {}): void { @@ -224,7 +225,7 @@ export function removeSandboxImage(sandboxName: string, deps: RemoveSandboxImage const removeImage = deps.dockerRmi ?? (require("../../adapters/docker") as { dockerRmi: DockerRmi }).dockerRmi; const sb = getSandbox(sandboxName); - if (!sb?.imageTag) return; + if (!sb?.imageTag || sb.workload?.shared === true) return; const result = removeImage(sb.imageTag, { ignoreError: true }); if (result.status === 0) { console.log(` Removed Docker image ${sb.imageTag}`); @@ -440,12 +441,20 @@ async function destroySandboxUnlocked( if ( shouldCleanupGatewayAfterConfirmedFinalDestroy({ deleteSucceededOrAlreadyGone, + gatewayStateDir: resolvePerGatewayState(cleanupGatewayName)?.stateDir ?? null, + openshellDriver: sandbox?.openshellDriver, removedRegistryEntry: removed, }) ) { const shouldCleanupGateway = await resolveCleanupGatewayDecision(normalized); if (shouldCleanupGateway) { - cleanupGatewayAfterLastSandbox(cleanupGatewayName, runOpenshell); + if (sandbox?.openshellDriver) { + cleanupGatewayAfterLastSandbox(cleanupGatewayName, runOpenshell, { + openshellDriver: sandbox.openshellDriver, + }); + } else { + cleanupGatewayAfterLastSandbox(cleanupGatewayName, runOpenshell); + } } else { // `gateway remove ` is the modern OpenShell subcommand on every // platform; the old `gateway destroy -g` was pre-0.0.44 only and current diff --git a/src/lib/actions/sandbox/doctor-flow.test.ts b/src/lib/actions/sandbox/doctor-flow.test.ts index 468444e153..cfa5bf2fcd 100644 --- a/src/lib/actions/sandbox/doctor-flow.test.ts +++ b/src/lib/actions/sandbox/doctor-flow.test.ts @@ -3,6 +3,7 @@ import fs from "node:fs"; import { createRequire } from "node:module"; +import os from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, type MockInstance, vi } from "vitest"; @@ -12,8 +13,26 @@ type RunSandboxDoctor = typeof import("./doctor")["runSandboxDoctor"]; const requireDist = createRequire(import.meta.url); const doctorModulePath = "./doctor.js"; +const runtimeBindingDirectories: string[] = []; + +function createPersistedPodmanBinding(socketPath: string): string { + const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-doctor-podman-binding-")); + runtimeBindingDirectories.push(stateDir); + const { buildPodmanDriverGatewayEnv } = requireDist( + "../../onboard/compute/podman/gateway-env.js", + ) as typeof import("../../onboard/compute/podman/gateway-env"); + buildPodmanDriverGatewayEnv({ + gatewayPort: 8080, + stateDir, + podmanSocketPath: socketPath, + supervisorImage: "ghcr.io/nvidia/openshell/supervisor@sha256:test", + }); + return stateDir; +} -function createDoctorHarness(provider = "ollama-local"): { +function createDoctorHarness( + options: { openshellDriver?: string | null; provider?: string } = {}, +): { buildToolScopeChecksSpy: MockInstance; captureOpenShellSpy: MockInstance; captureHostCommandSpy: MockInstance; @@ -35,6 +54,7 @@ function createDoctorHarness(provider = "ollama-local"): { resolveSandboxGatewayNameSpy: MockInstance; runSandboxDoctor: RunSandboxDoctor; } { + const provider = options.provider ?? "ollama-local"; delete require.cache[requireDist.resolve(doctorModulePath)]; const logSpy = vi.spyOn(console, "log").mockImplementation(() => undefined); @@ -64,7 +84,7 @@ function createDoctorHarness(provider = "ollama-local"): { agent: "openclaw", model: "registry-model", provider, - openshellDriver: "docker", + openshellDriver: options.openshellDriver ?? "docker", openshellVersion: "0.0.72", nemoclawVersion: "0.0.83", fromDockerfile: null, @@ -244,7 +264,11 @@ describe("runSandboxDoctor flow", () => { afterEach(() => { vi.restoreAllMocks(); + vi.unstubAllEnvs(); delete require.cache[requireDist.resolve(doctorModulePath)]; + for (const directory of runtimeBindingDirectories.splice(0)) { + fs.rmSync(directory, { force: true, recursive: true }); + } }); it( @@ -298,7 +322,7 @@ describe("runSandboxDoctor flow", () => { ["high", "high"], [null, "endpoint-default"], ] as const)("reports effective reasoning effort in doctor JSON (%s) (#7659)", async (stored, expected) => { - const harness = createDoctorHarness("compatible-endpoint"); + const harness = createDoctorHarness({ provider: "compatible-endpoint" }); harness.getSandboxSpy.mockReturnValue({ name: "alpha", agent: "openclaw", @@ -326,6 +350,81 @@ describe("runSandboxDoctor flow", () => { }); }); + it( + "probes a Podman sandbox through its exact socket without invoking Docker", + testTimeoutOptions(30_000), + async () => { + const socketPath = "/run/user/1000/podman/podman.sock"; + const stateDir = createPersistedPodmanBinding(socketPath); + vi.stubEnv("OPENSHELL_PODMAN_SOCKET", socketPath); + vi.stubEnv("NEMOCLAW_OPENSHELL_GATEWAY_STATE_DIR", stateDir); + const harness = createDoctorHarness({ openshellDriver: "podman" }); + harness.captureHostCommandSpy.mockImplementation((command: unknown) => { + if (command === "docker") { + throw new Error("Docker must not be invoked for Podman doctor"); + } + return { status: 0, stdout: "{}\n", stderr: "" }; + }); + + const report = await harness.runSandboxDoctor("alpha", ["--json"], { quietJson: true }); + + expect(harness.captureHostCommandSpy).toHaveBeenCalledWith( + "podman", + ["--url", `unix://${socketPath}`, "info", "--format", "json"], + 8000, + ); + expect(harness.captureHostCommandSpy).not.toHaveBeenCalledWith( + "docker", + expect.anything(), + expect.anything(), + ); + expect(report?.checks).toEqual( + expect.arrayContaining([ + expect.objectContaining({ group: "Host", label: "Podman service", status: "ok" }), + ]), + ); + expect(report?.checks).not.toEqual( + expect.arrayContaining([expect.objectContaining({ label: "Docker daemon" })]), + ); + }, + ); + + it( + "recovers the exact Podman socket from the target gateway binding in a fresh shell", + testTimeoutOptions(30_000), + async () => { + const socketPath = "/run/user/1000/podman/persisted.sock"; + const stateDir = createPersistedPodmanBinding(socketPath); + vi.stubEnv("OPENSHELL_PODMAN_SOCKET", ""); + vi.stubEnv("NEMOCLAW_OPENSHELL_GATEWAY_STATE_DIR", stateDir); + const harness = createDoctorHarness({ openshellDriver: "podman" }); + harness.captureHostCommandSpy.mockImplementation((command: unknown) => { + if (command === "docker") { + throw new Error("Docker must not be invoked for persisted Podman doctor"); + } + return { status: 0, stdout: "{}\n", stderr: "" }; + }); + + const report = await harness.runSandboxDoctor("alpha", ["--json"], { quietJson: true }); + + expect(harness.captureHostCommandSpy).toHaveBeenCalledWith( + "podman", + ["--url", `unix://${socketPath}`, "info", "--format", "json"], + 8000, + ); + expect(report?.checks).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + group: "Host", + label: "Podman service", + status: "ok", + detail: `connected via ${socketPath}`, + }), + ]), + ); + }, + ); + it( "reports baseline exclusions and flags content drift since approval (#7194)", testTimeoutOptions(30_000), diff --git a/src/lib/actions/sandbox/doctor-host-command.test.ts b/src/lib/actions/sandbox/doctor-host-command.test.ts index b6db4ba716..2812435a41 100644 --- a/src/lib/actions/sandbox/doctor-host-command.test.ts +++ b/src/lib/actions/sandbox/doctor-host-command.test.ts @@ -1,8 +1,13 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { describe, expect, it } from "vitest"; -import { captureHostCommand } from "./doctor-host-command"; +import { describe, expect, it, vi } from "vitest"; +import { + captureHostCommand, + collectDoctorHostRuntimeCheck, + type DoctorHostRuntimeAdapterRegistry, + resolveDoctorHostRuntimeAdapter, +} from "./doctor-host-command"; describe("captureHostCommand", () => { it("treats signal-terminated processes as failed", () => { @@ -13,4 +18,130 @@ describe("captureHostCommand", () => { expect(result.status).not.toBe(0); }); + + it("routes an injected MXC driver to only its registered doctor adapter", () => { + const inspect = vi.fn(() => ({ + group: "Host" as const, + label: "MXC runtime", + status: "ok" as const, + detail: "connected to the injected MXC control plane", + })); + const adapters: DoctorHostRuntimeAdapterRegistry = { + mxc: { + driverName: "mxc", + inspect, + }, + }; + + expect( + collectDoctorHostRuntimeCheck( + { + driverName: "mxc", + managedGatewayStateDirectory: "/state/mxc", + }, + adapters, + ), + ).toEqual({ + group: "Host", + label: "MXC runtime", + status: "ok", + detail: "connected to the injected MXC control plane", + }); + expect(inspect).toHaveBeenCalledExactlyOnceWith( + { + driverName: "mxc", + managedGatewayStateDirectory: "/state/mxc", + }, + { + captureHostCommand: expect.any(Function), + }, + ); + }); + + it("preserves the Docker adapter only for empty legacy driver metadata", () => { + const inspect = vi.fn(() => ({ + group: "Host" as const, + label: "legacy Docker", + status: "ok" as const, + detail: "legacy", + })); + const adapters: DoctorHostRuntimeAdapterRegistry = { + docker: { driverName: "docker", inspect }, + }; + + expect( + collectDoctorHostRuntimeCheck( + { driverName: null, managedGatewayStateDirectory: null }, + adapters, + ), + ).toMatchObject({ label: "legacy Docker" }); + expect(inspect).toHaveBeenCalledExactlyOnceWith( + { + driverName: "docker", + managedGatewayStateDirectory: null, + }, + { + captureHostCommand: expect.any(Function), + }, + ); + }); + + it("requires Podman doctor checks to recover the exact managed runtime binding", () => { + expect( + collectDoctorHostRuntimeCheck({ + driverName: "podman", + managedGatewayStateDirectory: null, + }), + ).toMatchObject({ + group: "Host", + label: "Podman service", + status: "fail", + detail: expect.stringContaining( + "Podman runtime socket recovery requires a managed gateway state directory", + ), + }); + }); + + it("does not let an unknown future driver inherit Docker or Podman checks", () => { + const dockerInspect = vi.fn(); + const podmanInspect = vi.fn(); + const adapters: DoctorHostRuntimeAdapterRegistry = { + docker: { + driverName: "docker", + inspect: dockerInspect, + }, + podman: { + driverName: "podman", + inspect: podmanInspect, + }, + }; + + expect( + collectDoctorHostRuntimeCheck( + { + driverName: "mxc", + managedGatewayStateDirectory: "/state/mxc", + }, + adapters, + ), + ).toBeNull(); + expect(dockerInspect).not.toHaveBeenCalled(); + expect(podmanInspect).not.toHaveBeenCalled(); + }); + + it("rejects a registry entry whose adapter claims another driver identity", () => { + expect(() => + resolveDoctorHostRuntimeAdapter("mxc", { + mxc: { + driverName: "docker", + inspect: () => ({ + group: "Host", + label: "wrong", + status: "ok", + detail: "wrong", + }), + }, + }), + ).toThrow("does not match its registered driver identity"); + }); }); diff --git a/src/lib/actions/sandbox/doctor-host-command.ts b/src/lib/actions/sandbox/doctor-host-command.ts index 058d93232d..deac2fdee1 100644 --- a/src/lib/actions/sandbox/doctor-host-command.ts +++ b/src/lib/actions/sandbox/doctor-host-command.ts @@ -3,6 +3,9 @@ import { spawnSync } from "node:child_process"; import { ROOT } from "../../state/paths"; +import { resolvePodmanRuntimeSocket } from "./runtime/podman-socket"; + +export { resolvePodmanRuntimeSocket }; export type CommandCapture = { status: number; @@ -11,6 +14,33 @@ export type CommandCapture = { error?: Error; }; +export type DoctorHostRuntimeCheck = { + group: "Host"; + label: string; + status: "ok" | "warn" | "fail" | "info"; + detail: string; + hint?: string; +}; + +export interface DoctorHostRuntimeAdapterInput { + readonly driverName: string; + readonly managedGatewayStateDirectory: string | null; +} + +export interface DoctorHostRuntimeAdapterDeps { + readonly captureHostCommand: typeof captureHostCommand; +} + +export interface DoctorHostRuntimeAdapter { + readonly driverName: string; + inspect( + input: DoctorHostRuntimeAdapterInput, + deps: DoctorHostRuntimeAdapterDeps, + ): DoctorHostRuntimeCheck; +} + +export type DoctorHostRuntimeAdapterRegistry = Readonly>; + export function captureHostCommand( command: string, args: string[], @@ -30,3 +60,114 @@ export function captureHostCommand( error: result.error, }; } + +function oneLine(value = ""): string { + return String(value).replace(/\s+/g, " ").trim(); +} + +const DOCKER_DOCTOR_ADAPTER: DoctorHostRuntimeAdapter = { + driverName: "docker", + inspect(_input, deps) { + const dockerInfo = deps.captureHostCommand( + "docker", + ["info", "--format", "{{.ServerVersion}}"], + 8000, + ); + return { + group: "Host", + label: "Docker daemon", + status: dockerInfo.status === 0 ? "ok" : "fail", + detail: + dockerInfo.status === 0 + ? `server ${dockerInfo.stdout.trim() || "unknown"}` + : oneLine(dockerInfo.stderr || dockerInfo.error?.message || "docker info failed"), + hint: + dockerInfo.status === 0 + ? undefined + : "start Docker and verify your user can access the daemon", + }; + }, +}; + +const PODMAN_DOCTOR_ADAPTER: DoctorHostRuntimeAdapter = { + driverName: "podman", + inspect(input, deps) { + let socketPath: string; + try { + socketPath = resolvePodmanRuntimeSocket(input.managedGatewayStateDirectory); + } catch (error) { + return { + group: "Host", + label: "Podman service", + status: "fail", + detail: oneLine(error instanceof Error ? error.message : String(error)), + hint: "restore the managed runtime binding or set an absolute OPENSHELL_PODMAN_SOCKET", + }; + } + + const podmanInfo = deps.captureHostCommand( + "podman", + ["--url", `unix://${socketPath}`, "info", "--format", "json"], + 8000, + ); + return { + group: "Host", + label: "Podman service", + status: podmanInfo.status === 0 ? "ok" : "fail", + detail: + podmanInfo.status === 0 + ? `connected via ${socketPath}` + : oneLine(podmanInfo.stderr || podmanInfo.error?.message || "podman info failed"), + hint: + podmanInfo.status === 0 + ? undefined + : "start the rootless Podman API service and verify OPENSHELL_PODMAN_SOCKET", + }; + }, +}; + +export const CURRENT_DOCTOR_HOST_RUNTIME_ADAPTERS = { + docker: DOCKER_DOCTOR_ADAPTER, + podman: PODMAN_DOCTOR_ADAPTER, +} as const satisfies DoctorHostRuntimeAdapterRegistry; + +/** + * Resolve a doctor adapter by persisted compute-driver identity. Empty legacy + * entries retain the historical Docker probe, while every named future driver + * must register its own adapter and cannot inherit Docker or Podman behavior. + */ +export function resolveDoctorHostRuntimeAdapter( + driverName: string | null | undefined, + adapters: DoctorHostRuntimeAdapterRegistry = CURRENT_DOCTOR_HOST_RUNTIME_ADAPTERS, +): DoctorHostRuntimeAdapter | null { + const normalized = driverName?.trim().toLowerCase() || "docker"; + const adapter = Object.hasOwn(adapters, normalized) ? adapters[normalized] : undefined; + if (!adapter) return null; + if (adapter.driverName !== normalized) { + throw new Error( + `Doctor host runtime adapter '${normalized}' does not match its registered driver identity.`, + ); + } + return adapter; +} + +export function collectDoctorHostRuntimeCheck( + input: { + readonly driverName: string | null | undefined; + readonly managedGatewayStateDirectory: string | null; + }, + adapters: DoctorHostRuntimeAdapterRegistry = CURRENT_DOCTOR_HOST_RUNTIME_ADAPTERS, + deps: Partial = {}, +): DoctorHostRuntimeCheck | null { + const adapter = resolveDoctorHostRuntimeAdapter(input.driverName, adapters); + if (!adapter) return null; + return adapter.inspect( + { + driverName: adapter.driverName, + managedGatewayStateDirectory: input.managedGatewayStateDirectory, + }, + { + captureHostCommand: deps.captureHostCommand ?? captureHostCommand, + }, + ); +} diff --git a/src/lib/actions/sandbox/doctor-system-checks.test.ts b/src/lib/actions/sandbox/doctor-system-checks.test.ts index 704d73adf8..acb9c51310 100644 --- a/src/lib/actions/sandbox/doctor-system-checks.test.ts +++ b/src/lib/actions/sandbox/doctor-system-checks.test.ts @@ -31,4 +31,10 @@ describe("doctor system checks", () => { hint: "expected host port 19080 for this sandbox gateway", }); }); + + it("does not inspect the legacy Docker gateway container for Podman sandboxes", () => { + const { shouldInspectLegacyGatewayContainer } = requireDist(modulePath); + + expect(shouldInspectLegacyGatewayContainer({ openshellDriver: "podman" })).toBe(false); + }); }); diff --git a/src/lib/actions/sandbox/doctor-system-checks.ts b/src/lib/actions/sandbox/doctor-system-checks.ts index 1d3fb564e1..ca690619c0 100644 --- a/src/lib/actions/sandbox/doctor-system-checks.ts +++ b/src/lib/actions/sandbox/doctor-system-checks.ts @@ -194,7 +194,7 @@ export function ollamaDoctorCheck(currentProvider: string): DoctorCheck { */ export function shouldInspectLegacyGatewayContainer(sb: SandboxEntry | null | undefined): boolean { const driver = sb?.openshellDriver; - if (driver === "docker" || driver === "vm") return false; + if (driver === "docker" || driver === "podman" || driver === "vm") return false; if (driver === "kubernetes") return true; return !isLinuxDockerDriverGatewayEnabled(); } diff --git a/src/lib/actions/sandbox/doctor.ts b/src/lib/actions/sandbox/doctor.ts index 8c07a63f1b..add06face3 100644 --- a/src/lib/actions/sandbox/doctor.ts +++ b/src/lib/actions/sandbox/doctor.ts @@ -17,7 +17,11 @@ import { } from "../../gateway-runtime-action"; import { parseGatewayInference } from "../../inference/config"; import { shouldManageDashboardForAgent } from "../../onboard/dashboard-runtime"; -import { resolveGatewayName, resolveSandboxGatewayName } from "../../onboard/gateway-binding"; +import { + resolveGatewayName, + resolveManagedGatewayStateDirectory, + resolveSandboxGatewayName, +} from "../../onboard/gateway-binding"; import { executeSandboxCommandForVerification } from "../../onboard/sandbox-verification-exec"; import { getBaselineExclusionRuntimeStatus } from "../../policy"; import { @@ -32,7 +36,7 @@ import type { SandboxEntry } from "../../state/registry"; import * as registry from "../../state/registry"; import { runSandboxAutoPairApprovalPass } from "./auto-pair-approval"; import { buildConfigPermsCheck } from "./doctor-config-perms"; -import { captureHostCommand } from "./doctor-host-command"; +import { captureHostCommand, collectDoctorHostRuntimeCheck } from "./doctor-host-command"; import { collectInferenceChecks, type DoctorInferenceRoute, @@ -125,29 +129,29 @@ function cliBuildCheck(): DoctorCheck { }; } -function collectHostChecks(): { +function collectHostChecks( + sb: SandboxEntry | null | undefined, + gatewayName: string | null, +): { checks: DoctorCheck[]; openshellBin: ReturnType; } { const cli = cliBuildCheck(); - const dockerInfo = captureHostCommand("docker", ["info", "--format", "{{.ServerVersion}}"], 8000); + const runtime = collectDoctorHostRuntimeCheck( + { + driverName: sb?.openshellDriver, + managedGatewayStateDirectory: gatewayName + ? resolveManagedGatewayStateDirectory(gatewayName) + : null, + }, + undefined, + { captureHostCommand }, + ); const openshellBin = resolveOpenshell(); return { checks: [ cli, - { - group: "Host", - label: "Docker daemon", - status: dockerInfo.status === 0 ? "ok" : "fail", - detail: - dockerInfo.status === 0 - ? `server ${dockerInfo.stdout.trim() || "unknown"}` - : oneLine(dockerInfo.stderr || dockerInfo.error?.message || "docker info failed"), - hint: - dockerInfo.status === 0 - ? undefined - : "start Docker and verify your user can access the daemon", - }, + ...(runtime ? [runtime] : []), { group: "Host", label: "OpenShell CLI", @@ -506,7 +510,7 @@ async function collectDoctorChecks( gatewayName: string | null, intent: DoctorIntent, ): Promise { - const host = collectHostChecks(); + const host = collectHostChecks(sb, gatewayName); const gateway: GatewayProbe = gatewayName ? await collectGatewayChecks(gatewayName, sb, host.openshellBin, !intent.asJson) : { diff --git a/src/lib/actions/sandbox/gateway-failure-classifier.test.ts b/src/lib/actions/sandbox/gateway-failure-classifier.test.ts index e102364017..f3d246b776 100644 --- a/src/lib/actions/sandbox/gateway-failure-classifier.test.ts +++ b/src/lib/actions/sandbox/gateway-failure-classifier.test.ts @@ -10,7 +10,11 @@ vi.mock("../../state/registry", () => ({ listSandboxes: vi.fn(() => ({ sandboxes: [] })), })); -import { classifyGatewayFailure, type GatewayFailureRunners } from "./gateway-failure-classifier"; +import { + classifyGatewayFailure, + type GatewayFailureRunners, + isDockerRuntimeDown, +} from "./gateway-failure-classifier"; function runners(overrides: Partial = {}): GatewayFailureRunners { return { @@ -93,3 +97,37 @@ describe("classifyGatewayFailure (#7348)", () => { expect(getSandboxMock).not.toHaveBeenCalled(); }); }); + +describe("runtime-specific Docker outage preflight", () => { + it.each([ + "vm", + "podman", + "mxc", + ])("does not inherit a Docker probe for the %s driver", (openshellDriver) => { + const dockerInfo = vi.fn(() => false); + + expect( + isDockerRuntimeDown("sb-1", { + getSandbox: () => ({ name: "sb-1", openshellDriver }), + runners: { dockerInfo }, + }), + ).toBe(false); + expect(dockerInfo).not.toHaveBeenCalled(); + }); + + it.each([ + null, + "docker", + "kubernetes", + ])("keeps the Docker outage probe for driver evidence %s", (openshellDriver) => { + const dockerInfo = vi.fn(() => false); + + expect( + isDockerRuntimeDown("sb-1", { + getSandbox: () => ({ name: "sb-1", openshellDriver }), + runners: { dockerInfo }, + }), + ).toBe(true); + expect(dockerInfo).toHaveBeenCalledOnce(); + }); +}); diff --git a/src/lib/actions/sandbox/gateway-failure-classifier.ts b/src/lib/actions/sandbox/gateway-failure-classifier.ts index e6124db4c3..e343a17c5c 100644 --- a/src/lib/actions/sandbox/gateway-failure-classifier.ts +++ b/src/lib/actions/sandbox/gateway-failure-classifier.ts @@ -247,24 +247,19 @@ type SandboxDriverLookup = (name: string) => { openshellDriver?: string | null } // `docker` driver and the `kubernetes`/k3s driver (k3s-in-Docker, or Docker // Desktop's Kubernetes — selected by `isLinuxDockerDriverGatewayEnabled()` for // non-Linux/non-arm64 hosts) both depend on a reachable local Docker daemon. A -// `vm` sandbox runs in a real VM with no local Docker daemon, so a failing -// `docker info` is normal and must not trigger the outage preflight. -const NON_DOCKER_DRIVERS = new Set(["vm"]); +const DOCKER_BACKED_DRIVERS = new Set(["docker", "kubernetes"]); /** - * Whether a sandbox's runtime depends on the local Docker daemon. Only the - * explicit `vm` driver is excluded. The `docker` and `kubernetes` drivers are - * Docker-backed, and legacy/recovered registry entries that predate - * `openshellDriver` metadata (field omitted/null) are also treated as - * Docker-backed so the outage guard still protects the Linux/Docker sandboxes - * #4428 targets — the historical default driver was Docker. The narrow cost is - * that a recovered `vm` entry that lost its driver metadata could see Docker - * guidance on a Docker-less host; that is preferable to silently regressing - * every legacy Docker sandbox. (#4428) + * Whether a sandbox's runtime explicitly depends on the local Docker daemon. + * The `docker` and `kubernetes` drivers are Docker-backed, and legacy entries + * without driver metadata retain that historical default. Podman, vm, and + * future named drivers do not inherit Docker probes merely because their + * runtime adapter has not been loaded in this module. */ function isDockerBackedSandbox(sandboxName: string, getSandbox: SandboxDriverLookup): boolean { const driver = getSandbox(sandboxName)?.openshellDriver; - return !(typeof driver === "string" && NON_DOCKER_DRIVERS.has(driver.toLowerCase())); + if (typeof driver !== "string" || !driver.trim()) return true; + return DOCKER_BACKED_DRIVERS.has(driver.trim().toLowerCase()); } /** @@ -272,9 +267,10 @@ function isDockerBackedSandbox(sandboxName: string, getSandbox: SandboxDriverLoo * `docker_unreachable` layer of {@link classifyGatewayFailure}). Sandbox * commands use this as a fast preflight so a transient Docker daemon outage is * classified as a host runtime problem rather than a stuck sandbox phase or a - * connect timeout (#4428). Returns `false` for VM sandboxes so they are never - * misclassified. `docker info` is a `spawnSync` call, so this stays synchronous - * and can run from non-async call sites such as `logs` and `policy-list`. + * connect timeout (#4428). Returns `false` for every named non-Docker runtime + * so Podman, vm, and future adapters are never misclassified. `docker info` is + * a `spawnSync` call, so this stays synchronous and can run from non-async call + * sites such as `logs` and `policy-list`. */ export function isDockerRuntimeDown( sandboxName: string, diff --git a/src/lib/actions/sandbox/gateway-state-drift.test.ts b/src/lib/actions/sandbox/gateway-state-drift.test.ts index feab961715..22f2ef9682 100644 --- a/src/lib/actions/sandbox/gateway-state-drift.test.ts +++ b/src/lib/actions/sandbox/gateway-state-drift.test.ts @@ -41,6 +41,7 @@ describe("sandbox gateway state drift guard", () => { let captureOpenshellSpy: MockInstance; let captureOpenshellForStatusSpy: MockInstance; let detectPreflightIssueSpy: MockInstance; + let recoverDockerDriverSandboxSpy: MockInstance; let getNamedGatewayLifecycleStateSpy: MockInstance; let getSandboxSpy: MockInstance; let gatewaySelectSpy: MockInstance; @@ -102,9 +103,9 @@ describe("sandbox gateway state drift guard", () => { getSandboxSpy, gatewaySelectSpy, recoverNamedGatewayRuntimeSpy, - vi + (recoverDockerDriverSandboxSpy = vi .spyOn(dockerDriverRecovery, "recoverDockerDriverSandbox") - .mockReturnValue({ recovered: false, via: null }), + .mockReturnValue({ recovered: false, via: null })), removeSandboxSpy, ); }); @@ -178,6 +179,28 @@ describe("sandbox gateway state drift guard", () => { expect(removeSandboxSpy).not.toHaveBeenCalled(); }); + it("never invokes Docker-side recovery for a Podman sandbox missing from a healthy gateway", () => { + detectPreflightIssueSpy.mockReturnValue(null); + getSandboxSpy.mockReturnValue({ + name: "alpha", + gatewayName: "nemoclaw", + gatewayPort: 8080, + openshellDriver: "podman", + }); + getNamedGatewayLifecycleStateSpy.mockReturnValue({ + state: "healthy_named", + status: "Gateway: nemoclaw\nStatus: Connected", + }); + + const lookup = gatewayState.reconcileMissingAgainstNamedGateway("alpha", { + state: "missing", + output: "NotFound", + }); + + expect(lookup.state).toBe("missing"); + expect(recoverDockerDriverSandboxSpy).not.toHaveBeenCalled(); + }); + it("preserves registry state and prints deterministic guidance after targeting the owning gateway (#2276)", async () => { vi.stubEnv("NEMOCLAW_NON_INTERACTIVE", "1"); detectPreflightIssueSpy.mockReturnValue(null); diff --git a/src/lib/actions/sandbox/gateway-state.ts b/src/lib/actions/sandbox/gateway-state.ts index 37e35a75d4..6f5bdb8390 100644 --- a/src/lib/actions/sandbox/gateway-state.ts +++ b/src/lib/actions/sandbox/gateway-state.ts @@ -15,6 +15,7 @@ import { isTerminalSandboxPhase, parseSandboxPhase } from "../../state/gateway"; import { selectSandboxOwningGateway } from "./gateway-select"; import { gatewayNamePattern, + getKnownSandboxOpenShellDriver, getKnownSandboxTargetGatewayName, getSandboxTargetGatewayName, } from "./gateway-target"; @@ -77,6 +78,11 @@ type SandboxGatewayStateLookup = ( gatewayName?: string, ) => SandboxGatewayState | Promise; +function sandboxUsesDockerLifecycle(sandboxName: string): boolean { + const driver = getKnownSandboxOpenShellDriver(sandboxName)?.toLowerCase(); + return driver === undefined || driver === "" || driver === "docker" || driver === "vm"; +} + function gatewayScopedArgs(args: string[], gatewayName?: string): string[] { if (!gatewayName) return args; return [...args.slice(0, 2), "-g", gatewayName, ...args.slice(2)]; @@ -373,7 +379,9 @@ export function reconcileMissingAgainstNamedGateway( // no sandbox memory, but Docker may still have the labeled // container. Attempt active Docker-side recovery before falling // through to non-destructive guidance. - return tryRecoverDockerDriverSandbox(sandboxName, missingLookup); + return sandboxUsesDockerLifecycle(sandboxName) + ? tryRecoverDockerDriverSandbox(sandboxName, missingLookup) + : missingLookup; } return missingLookup; } @@ -556,7 +564,11 @@ export async function getReconciledSandboxGatewayState( return lookup; } const recoveryGatewayName = targetGatewayName ?? getSandboxTargetGatewayName(); - const recovery = await recoverNamedGatewayRuntime({ gatewayName: recoveryGatewayName }); + const computeDriver = getKnownSandboxOpenShellDriver(sandboxName); + const recovery = await recoverNamedGatewayRuntime({ + ...(computeDriver ? { computeDriver } : {}), + gatewayName: recoveryGatewayName, + }); if (recovery.recovered) { const retried = await getState(sandboxName, recoveryGatewayName); if (retried.state === "present" || retried.state === "missing") { @@ -619,11 +631,13 @@ export async function ensureLiveSandboxOrExit( // sandbox is fine and recreating it cannot succeed until Docker is back // (#4428). Terminal phases (Failed/Error/...) are settled failures and // keep the rebuild guidance so a genuine failure is never masked. - if (!isTerminalSandboxPhase(phase) && isDockerRuntimeDown(sandboxName)) { + const dockerLifecycle = sandboxUsesDockerLifecycle(sandboxName); + if (dockerLifecycle && !isTerminalSandboxPhase(phase) && isDockerRuntimeDown(sandboxName)) { printDockerRuntimeDownGuidance(sandboxName); process.exit(1); } - const dockerRuntime = phase === "Error" ? getSandboxDockerRuntime(sandboxName) : null; + const dockerRuntime = + dockerLifecycle && phase === "Error" ? getSandboxDockerRuntime(sandboxName) : null; if (dockerRuntime?.paused && dockerRuntime.containerName) { console.error(` Sandbox '${sandboxName}' is stuck in '${phase}' phase.`); console.error(""); diff --git a/src/lib/actions/sandbox/gateway-target.ts b/src/lib/actions/sandbox/gateway-target.ts index cd125ceb8b..a6d6b3e3be 100644 --- a/src/lib/actions/sandbox/gateway-target.ts +++ b/src/lib/actions/sandbox/gateway-target.ts @@ -2,7 +2,12 @@ // SPDX-License-Identifier: Apache-2.0 import { GATEWAY_PORT } from "../../core/ports"; -import { resolveGatewayName, resolveSandboxGatewayName } from "../../onboard/gateway-binding"; +import { + resolveGatewayName, + resolveManagedGatewayStateDirectory, + resolveSandboxGatewayName, + type SandboxGatewayComputeBinding, +} from "../../onboard/gateway-binding"; import * as registry from "../../state/registry"; export function getKnownSandboxTargetGatewayName(sandboxName = ""): string | null { @@ -14,6 +19,23 @@ export function getSandboxTargetGatewayName(sandboxName = ""): string { return getKnownSandboxTargetGatewayName(sandboxName) ?? resolveGatewayName(GATEWAY_PORT); } +export function getKnownSandboxOpenShellDriver(sandboxName = ""): string | null { + return sandboxName ? (registry.getSandbox(sandboxName)?.openshellDriver?.trim() ?? null) : null; +} + +export function listSandboxGatewayComputeBindings(): readonly SandboxGatewayComputeBinding[] { + return registry.listSandboxes().sandboxes; +} + +export function resolveSandboxManagedGatewayStateDirectory( + sandbox: SandboxGatewayComputeBinding, + environment: NodeJS.ProcessEnv = process.env, +): string { + return resolveManagedGatewayStateDirectory(resolveSandboxGatewayName(sandbox), { + env: environment, + }); +} + export function gatewayNamePattern(gatewayName: string): RegExp { return new RegExp( `Gateway:\\s+${gatewayName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}(?=\\s|$)`, diff --git a/src/lib/actions/sandbox/mcp-bridge-state.ts b/src/lib/actions/sandbox/mcp-bridge-state.ts index bafb6335c8..2decfe027a 100644 --- a/src/lib/actions/sandbox/mcp-bridge-state.ts +++ b/src/lib/actions/sandbox/mcp-bridge-state.ts @@ -234,8 +234,11 @@ export function removeBridgeEntry(sandboxName: string, server: string): void { } export async function ensureSandboxGatewaySelected(sandboxName: string): Promise { + const sandbox = getSandboxOrThrow(sandboxName); const gatewayName = getSandboxTargetGatewayName(sandboxName); + const computeDriver = sandbox.openshellDriver?.trim(); const recovery = await recoverNamedGatewayRuntime({ + ...(computeDriver ? { computeDriver } : {}), gatewayName, }); if (!recovery.recovered || recovery.after.state !== "healthy_named") { diff --git a/src/lib/actions/sandbox/policy-channel.ts b/src/lib/actions/sandbox/policy-channel.ts index 14eb9f6af8..9fe4c98216 100644 --- a/src/lib/actions/sandbox/policy-channel.ts +++ b/src/lib/actions/sandbox/policy-channel.ts @@ -713,7 +713,11 @@ async function applyChannelAddToGatewayAndRegistry( })); if (tokenDefs.length > 0) { const gatewayName = getSandboxTargetGatewayName(sandboxName); - const recovery = await recoverNamedGatewayRuntime({ gatewayName }); + const computeDriver = registry.getSandbox(sandboxName)?.openshellDriver?.trim(); + const recovery = await recoverNamedGatewayRuntime({ + ...(computeDriver ? { computeDriver } : {}), + gatewayName, + }); if (!recovery.recovered) { console.error( ` Could not reach the ${CLI_DISPLAY_NAME} OpenShell gateway. Tokens were staged`, @@ -742,7 +746,11 @@ async function applyChannelRemoveToGatewayAndRegistry( if (channelTokenKeys.length > 0) { const gatewayName = getSandboxTargetGatewayName(sandboxName); - const recovery = await recoverNamedGatewayRuntime({ gatewayName }); + const computeDriver = registry.getSandbox(sandboxName)?.openshellDriver?.trim(); + const recovery = await recoverNamedGatewayRuntime({ + ...(computeDriver ? { computeDriver } : {}), + gatewayName, + }); if (!recovery.recovered) { console.error( ` Could not reach the ${CLI_DISPLAY_NAME} OpenShell gateway to delete the bridge.`, diff --git a/src/lib/actions/sandbox/rebuild-dcode-orchestrator.ts b/src/lib/actions/sandbox/rebuild-dcode-orchestrator.ts index ca0e3d4f2f..f7c18ca8a7 100644 --- a/src/lib/actions/sandbox/rebuild-dcode-orchestrator.ts +++ b/src/lib/actions/sandbox/rebuild-dcode-orchestrator.ts @@ -12,6 +12,7 @@ import { type PreparedDcodeReplacement, prepareDcodeReplacementBeforeMutation, revalidateDcodeReplacementAtMutationEdge, + revalidateManagedDcodeWorkloadAtMutationEdge, } from "./rebuild-dcode-preflight"; import { DCODE_AGENT_NAME } from "./rebuild-dcode-target"; import type { RebuildAgentBaseImageOptions, RebuildSandboxEntry } from "./rebuild-flow-helpers"; @@ -36,6 +37,7 @@ type CreateDcodeRebuildOrchestratorOptions = { sandboxName: string; entry: RebuildSandboxEntry; rebuildAgent: string | null; + managedWorkloadRebuild?: boolean; log(message: string): void; bail: DcodeRebuildPreflightBail; deps: DcodeRebuildOrchestratorDeps; @@ -97,7 +99,15 @@ export function isDcodeRebuildAgent(agentName: string | null): boolean { export function createDcodeRebuildOrchestrator( options: CreateDcodeRebuildOrchestratorOptions, ): DcodeRebuildOrchestrator { - const { sandboxName, entry, rebuildAgent, log, bail, deps } = options; + const { + sandboxName, + entry, + rebuildAgent, + managedWorkloadRebuild = false, + log, + bail, + deps, + } = options; const scope = createDcodeRebuildPreflightScope(isDcodeRebuildAgent(rebuildAgent), bail); const run = async (action: () => Promise): Promise => { @@ -150,6 +160,20 @@ export function createDcodeRebuildOrchestrator( if (!scope.enabled) { return deps.ensureAgentBaseImage(rebuildAgent, scope.bail, baseImageOptions); } + if (managedWorkloadRebuild) { + return revalidateManagedDcodeWorkloadAtMutationEdge({ + sandboxName, + entry, + resumeConfig, + toolDisclosure, + dcodeAutoApprovalMode, + skipLiveRoute, + gatewayPort, + log, + bail: scope.bail, + checkGatewaySchema: () => deps.checkGatewaySchema(sandboxName, scope.bail), + }); + } const replacement = await prepareDcodeReplacementBeforeMutation({ sandboxName, entry, @@ -179,6 +203,20 @@ export function createDcodeRebuildOrchestrator( ) => run(async () => { if (!scope.enabled) return true; + if (managedWorkloadRebuild) { + return revalidateManagedDcodeWorkloadAtMutationEdge({ + sandboxName, + entry, + resumeConfig, + toolDisclosure, + dcodeAutoApprovalMode, + skipLiveRoute, + gatewayPort, + log, + bail: scope.bail, + checkGatewaySchema: () => deps.checkGatewaySchema(sandboxName, scope.bail), + }); + } const replacement = scope.preparedReplacement; if (!replacement) return scope.bail("DCode replacement preflight was not retained."); return revalidateDcodeReplacementAtMutationEdge({ @@ -204,26 +242,39 @@ export function createDcodeRebuildOrchestrator( ) => { if (!scope.enabled) return { ok: true }; const replacement = scope.preparedReplacement; - if (!replacement) { + if (!managedWorkloadRebuild && !replacement) { return { ok: false, message: "DCode replacement preflight was not retained." }; } const capturedBail = (message: string, code?: number): never => { throw new CapturedDcodeRebuildBail(message, code); }; try { - const valid = await revalidateDcodeReplacementAtMutationEdge({ - sandboxName, - entry, - resumeConfig, - toolDisclosure, - dcodeAutoApprovalMode, - skipLiveRoute, - gatewayPort, - log, - bail: capturedBail, - checkGatewaySchema: () => deps.checkGatewaySchema(sandboxName, capturedBail), - replacement, - }); + const valid = await (managedWorkloadRebuild + ? revalidateManagedDcodeWorkloadAtMutationEdge({ + sandboxName, + entry, + resumeConfig, + toolDisclosure, + dcodeAutoApprovalMode, + skipLiveRoute, + gatewayPort, + log, + bail: capturedBail, + checkGatewaySchema: () => deps.checkGatewaySchema(sandboxName, capturedBail), + }) + : revalidateDcodeReplacementAtMutationEdge({ + sandboxName, + entry, + resumeConfig, + toolDisclosure, + dcodeAutoApprovalMode, + skipLiveRoute, + gatewayPort, + log, + bail: capturedBail, + checkGatewaySchema: () => deps.checkGatewaySchema(sandboxName, capturedBail), + replacement: replacement!, + })); if (!valid) { scope.cleanup(); return { diff --git a/src/lib/actions/sandbox/rebuild-dcode-preflight.ts b/src/lib/actions/sandbox/rebuild-dcode-preflight.ts index 660ecd8894..3466f7d19e 100644 --- a/src/lib/actions/sandbox/rebuild-dcode-preflight.ts +++ b/src/lib/actions/sandbox/rebuild-dcode-preflight.ts @@ -19,8 +19,8 @@ import { getResumeSandboxGpuOverrides, resolveSandboxGpuConfig, } from "../../onboard/sandbox-gpu-mode"; -import { redact } from "../../security/redact"; import type { TrustedLocalBaseImageOverride } from "../../sandbox-base-image"; +import { redact } from "../../security/redact"; import * as onboardSession from "../../state/onboard-session"; import * as registry from "../../state/registry"; import * as sandboxState from "../../state/sandbox"; @@ -170,7 +170,9 @@ export async function ensureDcodeRebuildTargetGatewaySelected( fail(error instanceof Error ? error.message : String(error), bail); } + const computeDriver = entry.openshellDriver?.trim(); const recovery = await recoverNamedGatewayRuntime({ + ...(computeDriver ? { computeDriver } : {}), gatewayName, recoverableStates: ["missing_named", "named_unhealthy", "named_unreachable", "connected_other"], }); @@ -464,3 +466,22 @@ export async function revalidateDcodeReplacementAtMutationEdge( } return true; } + +/** + * Revalidate DCode's live route and registry target when the replacement is an + * immutable managed image. No Dockerfile/base-image artifact exists to retain; + * the generic managed-workload handoff owns exact image/profile validation. + */ +export async function revalidateManagedDcodeWorkloadAtMutationEdge( + input: DcodeReplacementPreflightInput, +): Promise { + const { sandboxName, entry, resumeConfig, skipLiveRoute, gatewayPort, log, bail } = input; + const target = resolveTarget(entry, resumeConfig, bail, gatewayPort); + if (!(await ensureDcodeRebuildTargetGatewaySelected(sandboxName, entry, log, bail))) { + return false; + } + if (!input.checkGatewaySchema()) return false; + if (!skipLiveRoute) requireInferenceRoute(sandboxName, target, bail); + requireCurrentTarget(sandboxName, entry, target, resumeConfig, bail, gatewayPort); + return true; +} diff --git a/src/lib/actions/sandbox/rebuild-flow-helpers.ts b/src/lib/actions/sandbox/rebuild-flow-helpers.ts index b69f2d5905..7cf27a2ee7 100644 --- a/src/lib/actions/sandbox/rebuild-flow-helpers.ts +++ b/src/lib/actions/sandbox/rebuild-flow-helpers.ts @@ -133,7 +133,11 @@ export async function ensureRebuildTargetGatewaySelected( bail: (message: string, code?: number) => never, ): Promise { const gatewayName = resolveSandboxGatewayName(sb); - const recovery = await recoverNamedGatewayRuntime({ gatewayName }); + const computeDriver = sb.openshellDriver?.trim(); + const recovery = await recoverNamedGatewayRuntime({ + ...(computeDriver ? { computeDriver } : {}), + gatewayName, + }); if (!recovery.recovered || recovery.after.state !== "healthy_named") { console.error(""); console.error( @@ -158,8 +162,10 @@ export async function resolveRebuildLiveState( bail: (msg: string, code?: number) => never, ): Promise { const recordedGateway = resolveSandboxGatewayName(sb); + const computeDriver = sb.openshellDriver?.trim(); log(`Checking sandbox liveness on ${recordedGateway}: openshell sandbox list`); const liveRecovery = await captureSandboxListWithGatewayRecovery({ + ...(computeDriver ? { computeDriver } : {}), gatewayName: recordedGateway, }); const isLive = liveRecovery.result; diff --git a/src/lib/actions/sandbox/rebuild-gpu-opt-out.test.ts b/src/lib/actions/sandbox/rebuild-gpu-opt-out.test.ts index ccb5abd920..8ff8e10a51 100644 --- a/src/lib/actions/sandbox/rebuild-gpu-opt-out.test.ts +++ b/src/lib/actions/sandbox/rebuild-gpu-opt-out.test.ts @@ -203,6 +203,20 @@ describe("buildRebuildRecreateOnboardOpts", () => { expect(opts.toolDisclosure).toBe("direct"); }); + it("preserves the concrete compute driver through destructive recreate", () => { + const recorded = buildRebuildRecreateOnboardOpts({ + ...baseArgs, + sb: { ...dashboard, openshellDriver: "podman" }, + }); + const legacy = buildRebuildRecreateOnboardOpts({ + ...baseArgs, + sb: dashboard, + }); + + expect(recorded.computeDriver).toBe("podman"); + expect(legacy.computeDriver).not.toBe("auto"); + }); + it("preserves only recognized endpoint provenance across authoritative rebuild", () => { const onboard = buildRebuildRecreateOnboardOpts({ ...baseArgs, diff --git a/src/lib/actions/sandbox/rebuild-gpu-opt-out.ts b/src/lib/actions/sandbox/rebuild-gpu-opt-out.ts index eb95181606..c80f5b2e64 100644 --- a/src/lib/actions/sandbox/rebuild-gpu-opt-out.ts +++ b/src/lib/actions/sandbox/rebuild-gpu-opt-out.ts @@ -6,6 +6,7 @@ import { type InferenceEndpointSource, normalizeInferenceEndpointSource, } from "../../inference/selection"; +import { resolveCurrentOpenShellComputePlan } from "../../onboard/compute/plan"; import { shouldManageDashboardForAgent } from "../../onboard/dashboard-runtime"; import { type DcodeAutoApprovalMode, @@ -26,6 +27,7 @@ import type { RebuildRouteHandoff, } from "../../onboard/rebuild-route-handoff"; import { normalizeSandboxGpuMode } from "../../onboard/sandbox-gpu-mode"; +import type { ManagedWorkloadRebuildHandoff } from "../../onboard/workload/rebuild"; import { getTier } from "../../policy/tiers"; import type { SandboxBaseImageResolutionMetadata } from "../../sandbox-base-image"; import { type ToolDisclosure, toolDisclosureOrDefault } from "../../tool-disclosure"; @@ -43,6 +45,7 @@ export type RebuildGpuOptOutEntry = { observabilityEnabled?: boolean; policyTier?: string | null; endpointSource?: InferenceEndpointSource | null; + openshellDriver?: string | null; }; // Modern source of truth is the persisted `sandboxGpuMode` string ("0" / "1" / @@ -110,11 +113,13 @@ export type RebuildRecreateOnboardOpts = { targetGatewayName: string; targetGatewayPort: number; onboardLockAlreadyHeld: true; + computeDriver: string; preparedDcodeRebuild?: PreparedDcodeRebuildHandoff; rebuildRegistryInferenceRoute?: RebuildRouteHandoff; rebuildProviderReconfigure?: RebuildProviderReconfigureHandoff; providerRecoveryReceipt?: ProviderRecoveryReceipt; preparedImageRebuild?: PreparedImageRebuildHandoff; + managedWorkloadRebuild?: ManagedWorkloadRebuildHandoff; autoYes: boolean; toolDisclosure: ToolDisclosure; dcodeAutoApprovalMode: DcodeAutoApprovalMode; @@ -164,6 +169,8 @@ export function buildRebuildRecreateOnboardOpts(args: { const managesDashboard = shouldManageDashboardForAgent( loadAgent(args.rebuildAgent || "openclaw"), ); + const computeDriver = + args.sb?.openshellDriver?.trim() || resolveCurrentOpenShellComputePlan().driverName; if (managesDashboard && (!dashboardPort || dashboardPort < 1)) { throw new Error( "Cannot recreate a dashboard-managed sandbox without its persisted dashboard port.", @@ -184,6 +191,7 @@ export function buildRebuildRecreateOnboardOpts(args: { targetGatewayName, targetGatewayPort, onboardLockAlreadyHeld: true, + computeDriver, ...(args.preparedDcodeRebuild ? { preparedDcodeRebuild: args.preparedDcodeRebuild } : {}), autoYes: args.autoYes, toolDisclosure: toolDisclosureOrDefault(args.sb?.toolDisclosure), diff --git a/src/lib/actions/sandbox/rebuild-managed-workload-buildless.test.ts b/src/lib/actions/sandbox/rebuild-managed-workload-buildless.test.ts new file mode 100644 index 0000000000..1cf8303a28 --- /dev/null +++ b/src/lib/actions/sandbox/rebuild-managed-workload-buildless.test.ts @@ -0,0 +1,303 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; + +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +import { managedStartupE2eProfile } from "../../../../scripts/checks/generate-managed-startup-profile-fixture.mts"; +import { + createRebuildFlowHarness, + resetRebuildFlowTestEnvironment, + restoreRebuildFlowTestEnvironment, +} from "../../../../test/helpers/rebuild-flow-harness"; +import { + MANAGED_IMAGE_CAPABILITY_CONTRACT_VERSION, + MANAGED_IMAGE_CONTRACT_VERSION, + MANAGED_IMAGE_PLATFORM, + MANAGED_IMAGE_REPOSITORIES, + MANAGED_IMAGE_SOURCE_REPOSITORY, + MANAGED_IMAGE_STARTUP_PROFILE_CONTRACT_VERSION, + type ManagedImageContractV1, + type ShippedManagedImageAgent, +} from "../../onboard/managed-image/contract"; +import { encodeManagedStartupProfile } from "../../onboard/managed-startup/profile"; +import { + resolveManagedRebuildOpenClawReasoning, + resolveManagedRebuildOpenClawReasoningEffort, +} from "./agents/managed-workload-rebuild-profile"; + +const OLD_RELEASE = "v0.0.97"; +const NEW_RELEASE = "v0.0.98"; + +function managedContract( + agent: ShippedManagedImageAgent, + generation: "old" | "new", +): ManagedImageContractV1 { + const image = MANAGED_IMAGE_REPOSITORIES[agent]; + const digest: `sha256:${string}` = `sha256:${(generation === "old" ? "a" : "b").repeat(64)}`; + return { + contractVersion: MANAGED_IMAGE_CONTRACT_VERSION, + agent, + platform: MANAGED_IMAGE_PLATFORM, + image, + digest, + reference: `${image}@${digest}` as ManagedImageContractV1["reference"], + source: { + repository: MANAGED_IMAGE_SOURCE_REPOSITORY, + revision: (generation === "old" ? "1" : "2").repeat(40), + release: generation === "old" ? OLD_RELEASE : NEW_RELEASE, + cohort: generation === "old" ? "ghrun-100-1" : "ghrun-200-2", + }, + startupProfileContractVersion: MANAGED_IMAGE_STARTUP_PROFILE_CONTRACT_VERSION, + capabilityContractVersion: MANAGED_IMAGE_CAPABILITY_CONTRACT_VERSION, + }; +} + +function managedSandboxEntry(agent: ShippedManagedImageAgent): Record { + const old = managedContract(agent, "old"); + const baseProfile = managedStartupE2eProfile(agent); + const profile = + baseProfile.agent === "openclaw" + ? { + ...baseProfile, + inference: { + ...baseProfile.inference, + upstreamProvider: "compatible-endpoint", + }, + tuning: { + ...baseProfile.tuning, + reasoning: true, + }, + } + : baseProfile; + const encodedProfile = encodeManagedStartupProfile(profile); + return { + agent: agent === "openclaw" ? null : agent, + provider: "ollama-local", + model: "nvidia/nemotron", + fromDockerfile: null, + imageTag: old.reference, + dashboardPort: agent === "langchain-deepagents-code" ? 0 : 18_789, + gatewayName: "nemoclaw", + gatewayPort: 8_080, + toolDisclosure: "progressive", + dcodeAutoApprovalMode: "disabled", + observabilityEnabled: false, + workload: { + schemaVersion: 1, + kind: "managed-image", + reference: old.reference, + release: old.source.release, + sourceRevision: old.source.revision, + sourceCohort: old.source.cohort, + capabilityContractVersion: old.capabilityContractVersion, + startupProfileContractVersion: old.startupProfileContractVersion, + encodedProfile, + startupProfileSha256: createHash("sha256").update(encodedProfile, "utf8").digest("hex"), + credentialProxyReplayRequired: false, + shared: true, + }, + }; +} + +function replacement(agent: ShippedManagedImageAgent): Record { + const contract = managedContract(agent, "new"); + return { + source: { + kind: "managed-image", + reference: contract.reference, + contract, + }, + release: NEW_RELEASE, + fallbackDiagnostic: null, + }; +} + +describe("managed workload rebuild buildless boundary", () => { + beforeEach(resetRebuildFlowTestEnvironment); + afterEach(restoreRebuildFlowTestEnvironment); + + it.each([ + "openclaw", + "hermes", + "langchain-deepagents-code", + ] as const)("upgrades %s through one staged catalog/profile handoff without any build path", async (agent) => { + const oldEntry = managedSandboxEntry(agent); + const replacementWorkload = replacement(agent); + const harness = createRebuildFlowHarness({ + agentName: agent, + sandboxEntry: oldEntry, + managedWorkloadReplacement: replacementWorkload, + ...(agent === "openclaw" ? { managedContextWindow: 65_536 } : {}), + }); + harness.ensureAgentBaseImageSpy.mockImplementation(() => { + throw new Error("managed rebuild touched agent base-image preparation"); + }); + harness.preflightRebuildImageSpy.mockImplementation(() => { + throw new Error("managed rebuild touched generic image preflight"); + }); + harness.prepareManagedDcodeRebuildImageSpy.mockImplementation(() => { + throw new Error("managed rebuild touched DCode image preparation"); + }); + harness.dockerBuildSpy.mockImplementation(() => { + throw new Error("managed rebuild touched docker build"); + }); + + await expect( + harness.rebuildSandbox( + "alpha", + [ + "--yes", + "--tool-disclosure", + "direct", + ...(agent === "langchain-deepagents-code" + ? ["--dcode-auto-approval", "thread-opt-in", "--observability"] + : []), + ], + { throwOnError: true }, + ), + ).resolves.toBeUndefined(); + + expect(harness.prepareManagedWorkloadSourceSpy).toHaveBeenCalledWith( + expect.objectContaining({ + agentName: agent, + policy: "require-managed", + }), + ); + expect(harness.prepareManagedRebuildProfileHandoffSpy).toHaveBeenCalledOnce(); + expect(harness.ensureAgentBaseImageSpy).not.toHaveBeenCalled(); + expect(harness.preflightRebuildImageSpy).not.toHaveBeenCalled(); + expect(harness.prepareManagedDcodeRebuildImageSpy).not.toHaveBeenCalled(); + expect(harness.dockerBuildSpy).not.toHaveBeenCalled(); + expect(harness.onboardSpy).toHaveBeenCalledWith( + expect.objectContaining({ + agent: agent === "openclaw" ? null : agent, + fromDockerfile: null, + toolDisclosure: "direct", + managedWorkloadRebuild: expect.objectContaining({ + previousReceipt: expect.objectContaining({ + reference: oldEntry.imageTag, + }), + replacement: expect.objectContaining({ + source: expect.objectContaining({ + reference: (replacementWorkload.source as Record).reference, + }), + }), + replacementProfile: expect.objectContaining({ + encodedProfile: expect.any(String), + profile: expect.objectContaining({ + agent, + inference: expect.objectContaining({ + model: "nvidia/nemotron", + }), + tools: expect.objectContaining({ disclosure: "direct" }), + ...(agent === "openclaw" + ? { + tuning: expect.objectContaining({ + contextWindow: 65_536, + reasoning: false, + }), + } + : {}), + }), + }), + }), + ...(agent === "langchain-deepagents-code" + ? { + dcodeAutoApprovalMode: "thread-opt-in", + observabilityEnabled: true, + } + : {}), + }), + ); + + const profileOrder = harness.prepareManagedRebuildProfileHandoffSpy.mock.invocationCallOrder[0]; + const registryOrder = harness.registryUpdateSpy.mock.invocationCallOrder[0]; + const shieldsOrder = harness.openShieldsSpy.mock.invocationCallOrder[0]; + const backupOrder = harness.backupSandboxStateSpy.mock.invocationCallOrder[0]; + const onboardOrder = harness.onboardSpy.mock.invocationCallOrder[0]; + expect(profileOrder).toBeLessThan(registryOrder); + expect(registryOrder).toBeLessThan(shieldsOrder); + expect(shieldsOrder).toBeLessThan(backupOrder); + expect(backupOrder).toBeLessThan(onboardOrder); + }, 20_000); + + it.each([ + ["compatible-endpoint", "false", false], + ["compatible-endpoint", "true", true], + ["nvidia-prod", "true", false], + ] as const)("derives current OpenClaw reasoning for %s/%s instead of retaining receipt state", (provider, compatibleReasoning, expected) => { + expect(resolveManagedRebuildOpenClawReasoning(provider, compatibleReasoning)).toBe(expected); + }); + + it.each([ + ["compatible-endpoint", "openai-completions", "high", "high"], + ["compatible-endpoint", "openai-completions", null, "default"], + ["compatible-endpoint", "openai-responses", "high", "default"], + ["nvidia-prod", "openai-completions", "high", "default"], + ] as const)("derives current OpenClaw reasoning effort for %s/%s/%s", (provider, inferenceApi, compatibleReasoningEffort, expected) => { + expect( + resolveManagedRebuildOpenClawReasoningEffort( + provider, + inferenceApi, + compatibleReasoningEffort, + ), + ).toBe(expected); + }); + + it("fails profile rendering before registry update, shields, backup, or deletion", async () => { + const harness = createRebuildFlowHarness({ + agentName: "openclaw", + sandboxEntry: managedSandboxEntry("openclaw"), + managedWorkloadReplacement: replacement("openclaw"), + prepareManagedRebuildProfileHandoff: () => { + throw new Error("poisoned fresh profile"); + }, + }); + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), + ).rejects.toThrow("poisoned fresh profile"); + + expect(harness.prepareManagedRebuildProfileHandoffSpy).toHaveBeenCalledOnce(); + expect(harness.registryUpdateSpy).not.toHaveBeenCalled(); + expect(harness.openShieldsSpy).not.toHaveBeenCalled(); + expect(harness.backupSandboxStateSpy).not.toHaveBeenCalled(); + expect(harness.removeSandboxRegistryEntrySpy).not.toHaveBeenCalled(); + expect(harness.onboardSpy).not.toHaveBeenCalled(); + }); + + it("restores the old immutable receipt when recreation fails after delete", async () => { + const oldEntry = managedSandboxEntry("hermes"); + const harness = createRebuildFlowHarness({ + agentName: "hermes", + sandboxEntry: oldEntry, + managedWorkloadReplacement: replacement("hermes"), + onboard: () => { + throw new Error("replacement launch failed"); + }, + }); + + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), + ).rejects.toThrow("Recreate failed"); + + expect(harness.onboardSpy).toHaveBeenCalledOnce(); + expect(harness.restoreRegistryEntryIfMissingSpy).toHaveBeenCalledWith( + expect.objectContaining({ + entry: expect.objectContaining({ + imageTag: oldEntry.imageTag, + workload: oldEntry.workload, + }), + }), + ); + expect( + ( + harness.restoreRegistryEntryIfMissingSpy.mock.calls[0]?.[0] as { + entry: { imageTag: string }; + } + ).entry.imageTag, + ).not.toBe((replacement("hermes").source as Record).reference); + }); +}); diff --git a/src/lib/actions/sandbox/rebuild-pipeline.ts b/src/lib/actions/sandbox/rebuild-pipeline.ts index 1f97334062..18dbacd66c 100644 --- a/src/lib/actions/sandbox/rebuild-pipeline.ts +++ b/src/lib/actions/sandbox/rebuild-pipeline.ts @@ -20,7 +20,10 @@ import { disposeRebuildAgentBaseImagePreflight } from "./rebuild-flow-helpers"; import { stageMessagingManifestPlanForRebuild } from "./rebuild-messaging-phase"; import { runRebuildPostRestorePhase } from "./rebuild-post-restore-phase"; import { printRebuildPreflightFailure } from "./rebuild-preflight-error"; -import { blockRebuildOnPendingBaselineTransition } from "./rebuild-preflight-guards"; +import { + blockRebuildOnPendingBaselineTransition, + managedWorkloadRebuildHandoffMatchesRegisteredEntry, +} from "./rebuild-preflight-guards"; import { runRebuildPreflightPhase } from "./rebuild-preflight-phase"; import { disposePreparedBuildContext, @@ -135,6 +138,21 @@ async function rebuildSandboxUnlocked( getRecoveryRegistrySnapshot: () => recoveryRegistrySnapshot, log, }); + const validateManagedWorkloadHandoff = (): boolean => { + const handoff = recreateOptions.managedWorkloadRebuild; + if (!handoff) return true; + if (managedWorkloadRebuildHandoffMatchesRegisteredEntry(sandboxName, handoff)) { + return true; + } + printRebuildPreflightFailure( + "the managed workload receipt changed after rebuild preflight.", + "Retry the rebuild so the immutable image and startup profile can be validated again.", + "Managed workload receipt changed before delete", + bail, + ); + return false; + }; + if (!validateManagedWorkloadHandoff()) return; const shieldsPhase = runRebuildShieldsPhase( sandboxName, recoveryRecreate, @@ -181,6 +199,7 @@ async function rebuildSandboxUnlocked( relockShieldsIfNeeded, }); if (!backup) return; + if (!validateManagedWorkloadHandoff()) return; // The post-delete create must consume the exact context that passed the // image preflight. Revalidate at the last safe point so mutation of the diff --git a/src/lib/actions/sandbox/rebuild-preflight-guards.ts b/src/lib/actions/sandbox/rebuild-preflight-guards.ts index 9e086abfc4..77ac62b31f 100644 --- a/src/lib/actions/sandbox/rebuild-preflight-guards.ts +++ b/src/lib/actions/sandbox/rebuild-preflight-guards.ts @@ -7,6 +7,10 @@ import { } from "../../adapters/openshell/gateway-drift"; import { CLI_NAME } from "../../cli/branding"; import { resolveSandboxGatewayName } from "../../onboard/gateway-binding"; +import { + type ManagedWorkloadRebuildHandoff, + managedWorkloadRebuildHandoffMatchesEntry, +} from "../../onboard/workload/rebuild"; import * as onboardSession from "../../state/onboard-session"; import * as registry from "../../state/registry"; import type { RebuildBail } from "./rebuild-credential-preflight"; @@ -54,6 +58,13 @@ export function getRebuildSandboxEntryOrBail( return sb; } +export function managedWorkloadRebuildHandoffMatchesRegisteredEntry( + sandboxName: string, + handoff: ManagedWorkloadRebuildHandoff, +): boolean { + return managedWorkloadRebuildHandoffMatchesEntry(handoff, registry.getSandbox(sandboxName)); +} + /** Keep the pending baseline-policy transaction guard identical at every rebuild boundary. */ export function blockRebuildOnPendingBaselineTransition( sandboxEntry: RebuildSandboxEntry, diff --git a/src/lib/actions/sandbox/rebuild-preflight-phase.ts b/src/lib/actions/sandbox/rebuild-preflight-phase.ts index 03a9c3b129..a147dccaf1 100644 --- a/src/lib/actions/sandbox/rebuild-preflight-phase.ts +++ b/src/lib/actions/sandbox/rebuild-preflight-phase.ts @@ -178,6 +178,7 @@ export async function runRebuildPreflightPhase( sandboxName, entry: expectedSandboxEntry, rebuildAgent, + managedWorkloadRebuild: expectedSandboxEntry.workload?.kind === "managed-image", log, bail, deps: { @@ -235,8 +236,11 @@ export async function runRebuildPreflightPhase( recoveryRecreate, preparedTarget.recreateOptions.targetGatewayPort, ); - if (!imageReady || !dcodePreflight.preparedReplacement) return null; - preparedTarget.recreateOptions.preparedDcodeRebuild = dcodePreflight.preparedReplacement; + if (!imageReady) return null; + if (!preparedTarget.recreateOptions.managedWorkloadRebuild) { + if (!dcodePreflight.preparedReplacement) return null; + preparedTarget.recreateOptions.preparedDcodeRebuild = dcodePreflight.preparedReplacement; + } } // Keep credential-reuse validation after DCode's live-route/image proofs, // but before shields, backup, or any destructive rebuild work begins. diff --git a/src/lib/actions/sandbox/rebuild-preflight-target-phase.test.ts b/src/lib/actions/sandbox/rebuild-preflight-target-phase.test.ts index 38fe099572..2dba6e200e 100644 --- a/src/lib/actions/sandbox/rebuild-preflight-target-phase.test.ts +++ b/src/lib/actions/sandbox/rebuild-preflight-target-phase.test.ts @@ -9,6 +9,7 @@ import type { } from "../../onboard/rebuild-route-handoff"; import type { SandboxBaseImageResolutionMetadata } from "../../sandbox-base-image"; import { + resolveRebuildOpenShellComputePlan, stageRebuildBaseImageResolutionHandoff, stageRegistryProviderRecoveryReceipt, } from "./rebuild-preflight-target-phase"; @@ -59,6 +60,22 @@ describe("stageRegistryProviderRecoveryReceipt", () => { }); }); +describe("resolveRebuildOpenShellComputePlan", () => { + it("preserves the durable Podman driver for managed workload rebuilds", () => { + expect(resolveRebuildOpenShellComputePlan("podman")).toEqual({ + driverName: "podman", + gatewayLauncher: "nemoclaw", + }); + }); + + it("normalizes a legacy VM receipt to its existing Docker-managed plan", () => { + expect(resolveRebuildOpenShellComputePlan("vm")).toEqual({ + driverName: "docker", + gatewayLauncher: "nemoclaw", + }); + }); +}); + describe("stageRebuildBaseImageResolutionHandoff", () => { it("binds outer resolver provenance to its immutable local handoff (#7144)", () => { const imageId = `sha256:${"a".repeat(64)}`; diff --git a/src/lib/actions/sandbox/rebuild-preflight-target-phase.ts b/src/lib/actions/sandbox/rebuild-preflight-target-phase.ts index 3d396ddea1..443be0e2c2 100644 --- a/src/lib/actions/sandbox/rebuild-preflight-target-phase.ts +++ b/src/lib/actions/sandbox/rebuild-preflight-target-phase.ts @@ -5,7 +5,12 @@ import { randomUUID } from "node:crypto"; import { CLI_NAME } from "../../cli/branding"; import type { SandboxMessagingPlan } from "../../messaging"; import { isSandboxBaseImageRefreshRequested } from "../../onboard/base-image-resolution-flow"; +import { + resolveCurrentOpenShellComputePlan, + resolveOpenShellComputeSelection, +} from "../../onboard/compute/plan"; import type { DcodeAutoApprovalMode } from "../../onboard/dcode-auto-approval"; +import { credentialHostProxyReplayEnvArgs } from "../../onboard/host-proxy-env"; import { createRebuildProviderReconfigureHandoff, @@ -13,9 +18,15 @@ import { type ProviderRecoveryReceipt, type RegistryInferenceRoute, } from "../../onboard/rebuild-route-handoff"; +import { + prepareManagedWorkloadRebuildHandoff, + prepareSandboxWorkloadSourceFromRebuildHandoff, +} from "../../onboard/workload/rebuild"; +import { resolveSandboxWorkloadRuntimeCapabilities } from "../../onboard/workload/runtime"; import { readSandboxBaseImageResolutionMetadata } from "../../sandbox-base-image"; import * as registry from "../../state/registry"; import type { ToolDisclosure } from "../../tool-disclosure"; +import { prepareManagedRebuildProfileHandoff } from "./agents/managed-workload-rebuild-profile"; import { getSandboxTargetGatewayName } from "./gateway-target"; import type { RebuildBail, RebuildLog } from "./rebuild-credential-preflight"; import type { PreparedRebuildImage } from "./rebuild-custom-image-preflight"; @@ -77,6 +88,13 @@ export interface RebuildPreparedTarget { preparedImage: PreparedRebuildImage | null; } +export function resolveRebuildOpenShellComputePlan(computeDriver: string) { + return resolveOpenShellComputeSelection({ + requestedDriver: computeDriver === "vm" ? "docker" : computeDriver, + autoPlan: resolveCurrentOpenShellComputePlan(), + }); +} + /** Carry the outer resolver's verified provenance into the inner onboard build. */ export function stageRebuildBaseImageResolutionHandoff( recreateOptions: Pick, @@ -160,6 +178,26 @@ export async function prepareRebuildTargetPreflights(args: { bail, ); if (!recreateOptions) return null; + let managedWorkloadRebuildCatalog: Awaited< + ReturnType + > = null; + try { + const runtime = resolveSandboxWorkloadRuntimeCapabilities( + resolveRebuildOpenShellComputePlan(recreateOptions.computeDriver), + ); + managedWorkloadRebuildCatalog = await prepareManagedWorkloadRebuildHandoff(sandboxEntry, { + runtime, + }); + if (managedWorkloadRebuildCatalog) { + prepareSandboxWorkloadSourceFromRebuildHandoff(managedWorkloadRebuildCatalog, runtime); + if (managedWorkloadRebuildCatalog.previousReceipt.credentialProxyReplayRequired) { + credentialHostProxyReplayEnvArgs(process.env); + } + } + } catch (error) { + bail(error instanceof Error ? error.message : String(error)); + return null; + } // The durable resolver may recover a legacy row's choice from its matching // session. Use that authoritative value for both preflight and inner onboard, // never the raw registry fallback used while constructing generic options. @@ -188,6 +226,19 @@ export async function prepareRebuildTargetPreflights(args: { log, bail, ); + if (managedWorkloadRebuildCatalog) { + try { + recreateOptions.managedWorkloadRebuild = prepareManagedRebuildProfileHandoff({ + catalogHandoff: managedWorkloadRebuildCatalog, + targetConfig, + recreateOptions, + messagingPlan, + }); + } catch (error) { + bail(error instanceof Error ? error.message : String(error)); + return null; + } + } // Detect cross-sandbox credential conflicts immediately after staging the // exact rebuild plan, before host/runtime probes and every destructive phase. await preflightRebuildMessagingConflicts(messagingPlan, { @@ -225,12 +276,14 @@ export async function prepareRebuildTargetPreflights(args: { if (!checkRebuildGatewaySchemaPreflight(sandboxName, sandboxEntry, bail)) return null; const rebuildsDcodeSandbox = isDcodeRebuildAgent(rebuildAgent); - const baseImagePreflight = rebuildsDcodeSandbox - ? { ok: true, imageRef: null, overrideEnvVar: null } - : ensureRebuildAgentBaseImage(rebuildAgent, bail, { - resolutionHint: baseImageResolutionHint, - forceBaseImageRefresh, - }); + const rebuildsManagedWorkload = recreateOptions.managedWorkloadRebuild !== undefined; + const baseImagePreflight = + rebuildsDcodeSandbox || rebuildsManagedWorkload + ? { ok: true, imageRef: null, overrideEnvVar: null } + : ensureRebuildAgentBaseImage(rebuildAgent, bail, { + resolutionHint: baseImageResolutionHint, + forceBaseImageRefresh, + }); if (!baseImagePreflight.ok) return null; let retainBaseImagePreflight = false; try { @@ -248,7 +301,7 @@ export async function prepareRebuildTargetPreflights(args: { bail, { allowMissingGatewayProviderWithHostCredential: preparedBackupRecovery, - skipImagePreflight: rebuildsDcodeSandbox, + skipImagePreflight: rebuildsDcodeSandbox || rebuildsManagedWorkload, }, ); } finally { diff --git a/src/lib/actions/sandbox/rebuild-recreate-observability.test.ts b/src/lib/actions/sandbox/rebuild-recreate-observability.test.ts index dde431881a..beca298781 100644 --- a/src/lib/actions/sandbox/rebuild-recreate-observability.test.ts +++ b/src/lib/actions/sandbox/rebuild-recreate-observability.test.ts @@ -56,6 +56,7 @@ const recreateOptions: RebuildRecreateOnboardOpts = { targetGatewayName: "nemoclaw", targetGatewayPort: 8080, onboardLockAlreadyHeld: true, + computeDriver: "docker", autoYes: true, toolDisclosure: "progressive", dcodeAutoApprovalMode: "disabled", diff --git a/src/lib/actions/sandbox/rebuild-recreate-phase.ts b/src/lib/actions/sandbox/rebuild-recreate-phase.ts index 2bf430b00a..190cd5122b 100644 --- a/src/lib/actions/sandbox/rebuild-recreate-phase.ts +++ b/src/lib/actions/sandbox/rebuild-recreate-phase.ts @@ -123,6 +123,7 @@ export async function runRebuildRecreatePhase(input: RebuildRecreatePhaseInput): metadata: { gatewayName: recreateOptions.targetGatewayName, fromDockerfile: storedFromDockerfile, + openshellDriver: recreateOptions.computeDriver, }, }), ); diff --git a/src/lib/actions/sandbox/rebuild-registry-rollback.test.ts b/src/lib/actions/sandbox/rebuild-registry-rollback.test.ts index aa1f279d99..2878c133f7 100644 --- a/src/lib/actions/sandbox/rebuild-registry-rollback.test.ts +++ b/src/lib/actions/sandbox/rebuild-registry-rollback.test.ts @@ -101,6 +101,46 @@ describe("createRebuildRegistryRollback", () => { expect(log).toHaveBeenCalledWith("Recreate failed: restored registry metadata for retry"); }); + it("preserves the image tag when restoring a shared managed workload receipt", () => { + const removed = sandboxEntry({ + workload: { + schemaVersion: 1, + kind: "managed-image", + reference: "ghcr.io/nvidia/nemoclaw-openclaw@sha256:managed", + release: "v0.0.99", + sourceRevision: "0123456789abcdef0123456789abcdef01234567", + sourceCohort: "ghrun-123456-1", + capabilityContractVersion: 1, + startupProfileContractVersion: 1, + encodedProfile: "e30", + startupProfileSha256: "beab987bef9c00dfc301b490ddb45321517e7d6a6bb3d31d259898b7d46393d8", + credentialProxyReplayRequired: false, + shared: true, + }, + }); + const restoreSandboxEntryIfMissing = vi.fn(() => true); + const rollback = createRebuildRegistryRollback( + { + sandboxName: "alpha", + preparedBackupRecovery: false, + staleRecovery: false, + getRecoveryRegistrySnapshot: () => null, + log: vi.fn(), + }, + { restoreSandboxEntryIfMissing }, + ); + rollback.recordRemoval(removalReceipt(removed)); + + rollback.restoreForRetry(); + + expect(restoreSandboxEntryIfMissing).toHaveBeenCalledWith({ + entry: removed, + wasDefault: true, + fallbackDefault: "beta", + postRemovalDefaultSelectionRevision: 17, + }); + }); + it("keeps a replacement registered by failed onboarding", () => { const restoreSandboxEntryIfMissing = vi.fn(() => false); const log = vi.fn(); diff --git a/src/lib/actions/sandbox/rebuild-registry-rollback.ts b/src/lib/actions/sandbox/rebuild-registry-rollback.ts index 209e77d5e4..479853e916 100644 --- a/src/lib/actions/sandbox/rebuild-registry-rollback.ts +++ b/src/lib/actions/sandbox/rebuild-registry-rollback.ts @@ -76,12 +76,17 @@ export function createRebuildRegistryRollback( if (!registryEntryRemoved || !removedRegistryReceipt) return; rollbackAttempted = true; try { + const removedEntry = removedRegistryReceipt.entry; + const restoresSharedManagedImage = + removedEntry.workload?.kind === "managed-image" && removedEntry.workload.shared === true; const restored = restoreSandboxEntryIfMissing({ ...removedRegistryReceipt, - entry: { - ...removedRegistryReceipt.entry, - imageTag: null, - }, + entry: restoresSharedManagedImage + ? removedEntry + : { + ...removedEntry, + imageTag: null, + }, }); const recreateLabel = options.staleRecovery ? "Stale-recovery recreate" : "Recreate"; options.log( diff --git a/src/lib/actions/sandbox/rebuild-target-runtime.test.ts b/src/lib/actions/sandbox/rebuild-target-runtime.test.ts index 928f0abcec..572dc3bc8a 100644 --- a/src/lib/actions/sandbox/rebuild-target-runtime.test.ts +++ b/src/lib/actions/sandbox/rebuild-target-runtime.test.ts @@ -8,7 +8,6 @@ const mocks = vi.hoisted(() => ({ enforceDockerGpuPatchPreserveNetwork: vi.fn(), ensureValidatedWebSearchCredential: vi.fn(), isDockerDesktopWslRuntime: vi.fn(), - isLinuxDockerDriverGatewayEnabled: vi.fn(), preflightRebuildCredentials: vi.fn(), readGatewayProviderMetadata: vi.fn(), runOpenshell: vi.fn(), @@ -33,10 +32,6 @@ vi.mock("./rebuild-onboard-dependencies", () => ({ }, })); -vi.mock("../../onboard/docker-driver-platform", () => ({ - isLinuxDockerDriverGatewayEnabled: mocks.isLinuxDockerDriverGatewayEnabled, -})); - vi.mock("../../onboard/docker-gpu-local-inference", () => ({ enforceDockerGpuPatchPreserveNetwork: mocks.enforceDockerGpuPatchPreserveNetwork, })); @@ -75,6 +70,7 @@ const RECREATE_OPTIONS = { sandboxGpuDevice: null, controlUiPort: 18789, targetGatewayPort: 8080, + computeDriver: "docker", } as RebuildRecreateOnboardOpts; describe("preflightRebuildTargetRuntime GPU route", () => { @@ -90,7 +86,6 @@ describe("preflightRebuildTargetRuntime GPU route", () => { nimCapable: true, platform: "linux", }); - mocks.isLinuxDockerDriverGatewayEnabled.mockReturnValue(true); mocks.isDockerDesktopWslRuntime.mockReturnValue(false); mocks.enforceDockerGpuPatchPreserveNetwork.mockResolvedValue(false); mocks.preflightRebuildCredentials.mockReturnValue(true); @@ -143,6 +138,51 @@ describe("preflightRebuildTargetRuntime GPU route", () => { expect(bail).not.toHaveBeenCalled(); }); + it("accepts persisted Podman GPU state without entering Docker network preflight", async () => { + const bail = vi.fn(); + + await expect( + preflightRebuildTargetRuntime( + TARGET, + ENTRY, + { ...RECREATE_OPTIONS, computeDriver: "podman" }, + vi.fn(), + bail as never, + { skipImagePreflight: true }, + ), + ).resolves.toEqual({ + ok: true, + preparedImage: null, + requiresGatewayProviderReconfigure: false, + }); + + expect(bail).not.toHaveBeenCalled(); + expect(mocks.enforceDockerGpuPatchPreserveNetwork).not.toHaveBeenCalled(); + }); + + it("keeps a CPU-only Podman rebuild out of Docker-driver GPU routing", async () => { + const log = vi.fn(); + const bail = vi.fn(); + + await expect( + preflightRebuildTargetRuntime( + TARGET, + ENTRY, + { ...RECREATE_OPTIONS, computeDriver: "podman", sandboxGpu: "disable" }, + log, + bail as never, + { skipImagePreflight: true }, + ), + ).resolves.toEqual({ + ok: true, + preparedImage: null, + requiresGatewayProviderReconfigure: false, + }); + + expect(mocks.enforceDockerGpuPatchPreserveNetwork).not.toHaveBeenCalled(); + expect(bail).not.toHaveBeenCalled(); + }); + it("passes the immutable base provenance into replacement image preflight (#7144)", async () => { const metadata = { schema: 1, @@ -215,7 +255,6 @@ describe("preflightRebuildTargetRuntime web search credential", () => { nimCapable: true, platform: "linux", }); - mocks.isLinuxDockerDriverGatewayEnabled.mockReturnValue(true); mocks.isDockerDesktopWslRuntime.mockReturnValue(false); mocks.enforceDockerGpuPatchPreserveNetwork.mockResolvedValue(false); mocks.preflightRebuildCredentials.mockReturnValue(true); diff --git a/src/lib/actions/sandbox/rebuild-target-runtime.ts b/src/lib/actions/sandbox/rebuild-target-runtime.ts index e92f7d3da0..f5cfe82668 100644 --- a/src/lib/actions/sandbox/rebuild-target-runtime.ts +++ b/src/lib/actions/sandbox/rebuild-target-runtime.ts @@ -11,7 +11,6 @@ import { webSearchProviderForConfig, } from "../../inference/web-search"; import { shouldManageDashboardForAgent } from "../../onboard/dashboard-runtime"; -import { isLinuxDockerDriverGatewayEnabled } from "../../onboard/docker-driver-platform"; import { enforceDockerGpuPatchPreserveNetwork } from "../../onboard/docker-gpu-local-inference"; import { initialDockerGpuRoute, resolveDockerGpuRoutePlan } from "../../onboard/docker-gpu-route"; import { isDockerDesktopWslRuntime } from "../../onboard/docker-gpu-sandbox-create"; @@ -175,19 +174,23 @@ export async function preflightRebuildTargetRuntime( return { ok: false }; } try { - const dockerDriverGateway = isLinuxDockerDriverGatewayEnabled(); + // Rebuild is bound to the persisted compute driver before this preflight. + // Only the Docker profile may inherit Docker GPU routing semantics. + const dockerDriverGateway = recreateOptions.computeDriver === "docker"; const selectedRoute = initialDockerGpuRoute( resolveDockerGpuRoutePlan(sandboxGpuConfig, { dockerDriverGateway, dockerDesktopWsl: isDockerDesktopWslRuntime(), }), ); - await enforceDockerGpuPatchPreserveNetwork(target.resumeConfig.provider, sandboxGpuConfig, { - dockerDriverGateway, - selectedRoute, - gatewayPort: recreateOptions.targetGatewayPort, - log, - }); + if (dockerDriverGateway) { + await enforceDockerGpuPatchPreserveNetwork(target.resumeConfig.provider, sandboxGpuConfig, { + dockerDriverGateway, + selectedRoute, + gatewayPort: recreateOptions.targetGatewayPort, + log, + }); + } } catch (err) { printRebuildPreflightFailure( "the recorded GPU network path is not reachable.", diff --git a/src/lib/actions/sandbox/runtime/lifecycle-runtime-actions.test.ts b/src/lib/actions/sandbox/runtime/lifecycle-runtime-actions.test.ts new file mode 100644 index 0000000000..78a34d3128 --- /dev/null +++ b/src/lib/actions/sandbox/runtime/lifecycle-runtime-actions.test.ts @@ -0,0 +1,329 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import { + PODMAN_MANAGED_LABEL, + PODMAN_SANDBOX_ID_LABEL, + PODMAN_SANDBOX_NAME_LABEL, + PODMAN_SANDBOX_NAMESPACE_LABEL, +} from "../../../onboard/compute/podman/sandbox-recreate-spec"; +import type { SandboxEntry } from "../../../state/registry"; +import { startSandbox } from "../start"; +import { stopSandbox } from "../stop"; +import type { + SandboxLifecycleRuntimeAdapter, + SandboxLifecycleRuntimeAdapterRegistry, +} from "./lifecycle-runtime"; + +const AGENTS = ["openclaw", "hermes", "langchain-deepagents-code"] as const; +const SANDBOX_NAME = "my-sandbox"; +const CONTAINER_NAME = `openshell-sandbox-${SANDBOX_NAME}`; +const CONTAINER_ID = "a".repeat(64); +const IMAGE_ID = "b".repeat(64); +const SOCKET_PATH = "/run/user/1000/podman/podman.sock"; +const SOCKET_AUTHORITY = { + directoryChain: [ + { + device: "8", + inode: "7000", + mode: "448", + ownerUid: "1000", + path: "/run/user/1000/podman", + }, + ], + device: "8", + inode: "9001", + ownerUid: "1000", + socketPath: SOCKET_PATH, +} as const; + +function podmanSandbox(agent: (typeof AGENTS)[number]): SandboxEntry { + return { + agent, + gatewayName: "nemoclaw", + name: SANDBOX_NAME, + openshellDriver: "podman", + }; +} + +function podmanInspect(running: boolean, status?: string): string { + return JSON.stringify([ + { + Config: { + Labels: { + [PODMAN_MANAGED_LABEL]: "true", + [PODMAN_SANDBOX_ID_LABEL]: "sandbox-id", + [PODMAN_SANDBOX_NAME_LABEL]: SANDBOX_NAME, + [PODMAN_SANDBOX_NAMESPACE_LABEL]: "default", + }, + }, + Id: CONTAINER_ID, + Image: `sha256:${IMAGE_ID}`, + Name: CONTAINER_NAME, + State: { + Paused: false, + Running: running, + Status: status ?? (running ? "running" : "exited"), + }, + }, + ]); +} + +function podmanLifecycleHarness( + agent: (typeof AGENTS)[number], + options: { + initialRunning?: boolean; + initialStatus?: string; + lookupRows?: string; + } = {}, +) { + let running = options.initialRunning ?? true; + const captureHostCommand = vi.fn((_command: string, args: string[]) => { + const operationIndex = args.indexOf("--url") + 2; + const operation = args[operationIndex]; + if (operation === "ps") { + return { + status: 0, + stderr: "", + stdout: options.lookupRows ?? `${CONTAINER_ID}\t${CONTAINER_NAME}\n`, + }; + } + if (operation === "container" && args[operationIndex + 1] === "inspect") { + return { + status: 0, + stderr: "", + stdout: podmanInspect(running, options.initialStatus), + }; + } + if (operation === "stop") { + running = false; + return { status: 0, stderr: "", stdout: CONTAINER_ID }; + } + if (operation === "start") { + running = true; + return { status: 0, stderr: "", stdout: CONTAINER_ID }; + } + return { status: 125, stderr: `unexpected Podman operation: ${String(operation)}`, stdout: "" }; + }); + const getSandbox = vi.fn(() => podmanSandbox(agent)); + const assertPodmanSocketAuthority = vi.fn(); + const capturePodmanSocketAuthority = vi.fn(() => SOCKET_AUTHORITY); + const resolvePodmanRuntimeSocket = vi.fn(() => SOCKET_PATH); + const resolveSandboxManagedGatewayStateDirectory = vi.fn(() => "/state/podman"); + const stopSandboxChannels = vi.fn(); + const teardownSandboxDashboardForward = vi.fn(); + const probeSandbox = vi.fn(async () => {}); + const log = vi.fn(); + return { + assertPodmanSocketAuthority, + captureHostCommand, + capturePodmanSocketAuthority, + getSandbox, + log, + probeSandbox, + resolvePodmanRuntimeSocket, + resolveSandboxManagedGatewayStateDirectory, + stopSandboxChannels, + teardownSandboxDashboardForward, + }; +} + +describe("sandbox lifecycle runtime actions", () => { + it.each(AGENTS)("stops and restarts the exact native Podman container for %s", async (agent) => { + const h = podmanLifecycleHarness(agent); + const sharedDeps = { + assertPodmanSocketAuthority: h.assertPodmanSocketAuthority, + captureHostCommand: h.captureHostCommand, + capturePodmanSocketAuthority: h.capturePodmanSocketAuthority, + environment: { NEMOCLAW_PODMAN_BIN: "podman" }, + getSandbox: h.getSandbox, + log: h.log, + resolvePodmanRuntimeSocket: h.resolvePodmanRuntimeSocket, + resolveSandboxManagedGatewayStateDirectory: h.resolveSandboxManagedGatewayStateDirectory, + }; + + expect( + stopSandbox(SANDBOX_NAME, { + ...sharedDeps, + stopSandboxChannels: h.stopSandboxChannels, + teardownSandboxDashboardForward: h.teardownSandboxDashboardForward, + }), + ).toEqual({ exitCode: 0 }); + expect(h.stopSandboxChannels).toHaveBeenCalledWith( + SANDBOX_NAME, + expect.objectContaining({ + allowDockerGatewayExec: false, + info: expect.any(Function), + warn: expect.any(Function), + }), + ); + expect(h.teardownSandboxDashboardForward).toHaveBeenCalledWith(SANDBOX_NAME); + + await expect( + startSandbox(SANDBOX_NAME, { + ...sharedDeps, + probeSandbox: h.probeSandbox, + }), + ).resolves.toEqual({ exitCode: 0 }); + expect(h.probeSandbox).toHaveBeenCalledWith(SANDBOX_NAME); + + const calls = h.captureHostCommand.mock.calls; + expect(calls.every(([command]) => command === "podman")).toBe(true); + expect(calls.map(([, args]) => args)).toContainEqual([ + "--url", + `unix://${SOCKET_PATH}`, + "stop", + "--time", + "30", + CONTAINER_ID, + ]); + expect(calls.map(([, args]) => args)).toContainEqual([ + "--url", + `unix://${SOCKET_PATH}`, + "start", + CONTAINER_ID, + ]); + expect(JSON.stringify(calls)).not.toContain("docker"); + expect(h.resolvePodmanRuntimeSocket).toHaveBeenCalledTimes(2); + expect(h.assertPodmanSocketAuthority).toHaveBeenCalledTimes(calls.length * 2 + 2); + }); + + it("routes a future MXC driver only through its injected lifecycle adapter", async () => { + const events: string[] = []; + const adapter: SandboxLifecycleRuntimeAdapter = { + channelStopTransport: "openshell", + displayName: "MXC", + driverName: "mxc", + preflight: vi.fn(() => null), + start: vi.fn(() => { + events.push("start"); + return { exitCode: 0 }; + }), + stop: vi.fn((_input, _deps, hooks) => { + hooks.beforeStop(); + events.push("stop"); + return { exitCode: 0, state: "stopped" as const }; + }), + }; + const runtimeAdapters: SandboxLifecycleRuntimeAdapterRegistry = { mxc: adapter }; + const entry: SandboxEntry = { + agent: "hermes", + name: SANDBOX_NAME, + openshellDriver: "mxc", + }; + const stopSandboxChannels = vi.fn(); + const probeSandbox = vi.fn(async () => {}); + + expect( + stopSandbox(SANDBOX_NAME, { + getSandbox: () => entry, + runtimeAdapters, + stopSandboxChannels, + teardownSandboxDashboardForward: vi.fn(), + }), + ).toEqual({ exitCode: 0 }); + await expect( + startSandbox(SANDBOX_NAME, { + getSandbox: () => entry, + probeSandbox, + runtimeAdapters, + }), + ).resolves.toEqual({ exitCode: 0 }); + + expect(events).toEqual(["stop", "start"]); + expect(stopSandboxChannels).toHaveBeenCalledOnce(); + expect(probeSandbox).toHaveBeenCalledWith(SANDBOX_NAME); + }); + + it("refuses ambiguous or unknown Podman container identity without side effects", () => { + const ambiguous = podmanLifecycleHarness("openclaw", { + lookupRows: `${CONTAINER_ID}\t${CONTAINER_NAME}\n` + `${"c".repeat(64)}\t${CONTAINER_NAME}\n`, + }); + expect( + stopSandbox(SANDBOX_NAME, { + assertPodmanSocketAuthority: ambiguous.assertPodmanSocketAuthority, + captureHostCommand: ambiguous.captureHostCommand, + capturePodmanSocketAuthority: ambiguous.capturePodmanSocketAuthority, + getSandbox: ambiguous.getSandbox, + resolvePodmanRuntimeSocket: ambiguous.resolvePodmanRuntimeSocket, + resolveSandboxManagedGatewayStateDirectory: + ambiguous.resolveSandboxManagedGatewayStateDirectory, + stopSandboxChannels: ambiguous.stopSandboxChannels, + teardownSandboxDashboardForward: ambiguous.teardownSandboxDashboardForward, + }), + ).toMatchObject({ + exitCode: 1, + message: expect.stringContaining("2 managed containers"), + }); + expect(ambiguous.stopSandboxChannels).not.toHaveBeenCalled(); + expect(ambiguous.teardownSandboxDashboardForward).not.toHaveBeenCalled(); + + const unknownState = podmanLifecycleHarness("openclaw", { + initialRunning: false, + initialStatus: "unknown", + }); + expect( + stopSandbox(SANDBOX_NAME, { + assertPodmanSocketAuthority: unknownState.assertPodmanSocketAuthority, + captureHostCommand: unknownState.captureHostCommand, + capturePodmanSocketAuthority: unknownState.capturePodmanSocketAuthority, + getSandbox: unknownState.getSandbox, + resolvePodmanRuntimeSocket: unknownState.resolvePodmanRuntimeSocket, + resolveSandboxManagedGatewayStateDirectory: + unknownState.resolveSandboxManagedGatewayStateDirectory, + stopSandboxChannels: unknownState.stopSandboxChannels, + teardownSandboxDashboardForward: unknownState.teardownSandboxDashboardForward, + }), + ).toMatchObject({ + exitCode: 1, + message: expect.stringContaining("not safely stoppable"), + }); + expect(unknownState.stopSandboxChannels).not.toHaveBeenCalled(); + expect(unknownState.teardownSandboxDashboardForward).not.toHaveBeenCalled(); + }); + + it("revalidates after stop preparation and sends no mutation through a replaced socket", () => { + const h = podmanLifecycleHarness("openclaw"); + let validations = 0; + h.assertPodmanSocketAuthority.mockImplementation(() => { + validations += 1; + if (validations === 6) { + throw new Error("Podman socket authority changed after it was qualified."); + } + }); + + expect( + stopSandbox(SANDBOX_NAME, { + assertPodmanSocketAuthority: h.assertPodmanSocketAuthority, + captureHostCommand: h.captureHostCommand, + capturePodmanSocketAuthority: h.capturePodmanSocketAuthority, + getSandbox: h.getSandbox, + resolvePodmanRuntimeSocket: h.resolvePodmanRuntimeSocket, + resolveSandboxManagedGatewayStateDirectory: h.resolveSandboxManagedGatewayStateDirectory, + stopSandboxChannels: h.stopSandboxChannels, + teardownSandboxDashboardForward: h.teardownSandboxDashboardForward, + }), + ).toMatchObject({ + exitCode: 1, + message: expect.stringContaining("socket authority changed"), + }); + expect(h.captureHostCommand.mock.calls.some(([, args]) => args.includes("stop"))).toBe(false); + }); + + it("fails closed for an unregistered named runtime", async () => { + const entry: SandboxEntry = { + name: SANDBOX_NAME, + openshellDriver: "unregistered", + }; + expect(stopSandbox(SANDBOX_NAME, { getSandbox: () => entry })).toMatchObject({ + exitCode: 1, + message: expect.stringContaining("unregistered"), + }); + await expect(startSandbox(SANDBOX_NAME, { getSandbox: () => entry })).resolves.toMatchObject({ + exitCode: 1, + message: expect.stringContaining("unregistered"), + }); + }); +}); diff --git a/src/lib/actions/sandbox/runtime/lifecycle-runtime.ts b/src/lib/actions/sandbox/runtime/lifecycle-runtime.ts new file mode 100644 index 0000000000..095f4d2210 --- /dev/null +++ b/src/lib/actions/sandbox/runtime/lifecycle-runtime.ts @@ -0,0 +1,568 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { CLI_NAME } from "../../../cli/branding"; +import { + PODMAN_MANAGED_LABEL, + PODMAN_SANDBOX_CONTAINER_PREFIX, + PODMAN_SANDBOX_NAME_LABEL, + parsePodmanManagedSandboxInspect, +} from "../../../onboard/compute/podman/sandbox-recreate-spec"; +import { + assertPodmanSocketAuthority, + capturePodmanSocketAuthority, + type PodmanSocketAuthority, +} from "../../../onboard/compute/podman/socket-authority"; +import { + findLabeledSandboxContainers, + recoverDockerDriverSandbox, +} from "../../../onboard/docker-driver-sandbox-recovery"; +import type { SandboxEntry } from "../../../state/registry"; +import { type CommandCapture, captureHostCommand } from "../doctor-host-command"; +import { isDockerRuntimeDown, printDockerRuntimeDownGuidance } from "../gateway-failure-classifier"; +import { resolveSandboxManagedGatewayStateDirectory } from "../gateway-target"; +import { resolvePodmanRuntimeSocket } from "./podman-socket"; + +const DOCKER_OPERATION_TIMEOUT_MS = 30_000; +const PODMAN_OPERATION_TIMEOUT_MS = 40_000; +const PODMAN_PROBE_TIMEOUT_MS = 5_000; +const PODMAN_STOP_GRACE_SECONDS = 30; +const AT_REST_DOCKER_STATUS_PREFIXES = ["Exited", "Created", "Dead"] as const; +const AT_REST_PODMAN_STATES = new Set(["configured", "created", "dead", "exited", "stopped"]); +const FULL_CONTAINER_ID_RE = /^(?:sha256:)?[0-9a-f]{64}$/iu; + +type DockerOpResult = { status?: number | null }; +type DockerStopFn = (name: string, opts?: Record) => DockerOpResult; +type DockerUnpauseFn = (name: string, opts?: Record) => DockerOpResult; + +function loadDockerStop(): DockerStopFn { + return (require("../../../adapters/docker") as { dockerStop: DockerStopFn }).dockerStop; +} + +function loadDockerUnpause(): DockerUnpauseFn { + return (require("../../../adapters/docker") as { dockerUnpause: DockerUnpauseFn }).dockerUnpause; +} + +export type SandboxLifecycleResult = { + exitCode: number; + message?: string; +}; + +export interface SandboxLifecycleRuntimeInput { + readonly environment: NodeJS.ProcessEnv; + readonly log: (message: string) => void; + readonly sandbox: SandboxEntry; + readonly sandboxName: string; +} + +export interface SandboxLifecycleStopHooks { + /** Called only after a runtime container is proved stoppable and before mutation. */ + readonly beforeStop: () => void; +} + +export type SandboxLifecycleStopOutcome = SandboxLifecycleResult & { + readonly state?: "already-stopped" | "stopped"; +}; + +export interface SandboxLifecycleRuntimeDependencies { + readonly captureHostCommand: typeof captureHostCommand; + readonly assertPodmanSocketAuthority: typeof assertPodmanSocketAuthority; + readonly capturePodmanSocketAuthority: typeof capturePodmanSocketAuthority; + readonly dockerStop: DockerStopFn; + readonly dockerUnpause: DockerUnpauseFn; + readonly findLabeledSandboxContainers: typeof findLabeledSandboxContainers; + readonly isDockerRuntimeDown: typeof isDockerRuntimeDown; + readonly printDockerRuntimeDownGuidance: typeof printDockerRuntimeDownGuidance; + readonly recoverDockerDriverSandbox: typeof recoverDockerDriverSandbox; + readonly resolvePodmanRuntimeSocket: typeof resolvePodmanRuntimeSocket; + readonly resolveSandboxManagedGatewayStateDirectory: typeof resolveSandboxManagedGatewayStateDirectory; +} + +export interface SandboxLifecycleRuntimeAdapter { + readonly channelStopTransport: "docker-kubectl-first" | "openshell"; + readonly displayName: string; + readonly driverName: string; + preflight( + action: "start" | "stop", + input: SandboxLifecycleRuntimeInput, + deps: SandboxLifecycleRuntimeDependencies, + ): SandboxLifecycleResult | null; + start( + input: SandboxLifecycleRuntimeInput, + deps: SandboxLifecycleRuntimeDependencies, + ): SandboxLifecycleResult; + stop( + input: SandboxLifecycleRuntimeInput, + deps: SandboxLifecycleRuntimeDependencies, + hooks: SandboxLifecycleStopHooks, + ): SandboxLifecycleStopOutcome; +} + +export type SandboxLifecycleRuntimeAdapterRegistry = Readonly< + Record +>; + +export function resolveSandboxLifecycleRuntimeDependencies( + overrides: Partial = {}, +): SandboxLifecycleRuntimeDependencies { + return { + assertPodmanSocketAuthority: + overrides.assertPodmanSocketAuthority ?? assertPodmanSocketAuthority, + captureHostCommand: overrides.captureHostCommand ?? captureHostCommand, + capturePodmanSocketAuthority: + overrides.capturePodmanSocketAuthority ?? capturePodmanSocketAuthority, + dockerStop: overrides.dockerStop ?? ((name, options) => loadDockerStop()(name, options)), + dockerUnpause: + overrides.dockerUnpause ?? ((name, options) => loadDockerUnpause()(name, options)), + findLabeledSandboxContainers: + overrides.findLabeledSandboxContainers ?? findLabeledSandboxContainers, + isDockerRuntimeDown: overrides.isDockerRuntimeDown ?? isDockerRuntimeDown, + printDockerRuntimeDownGuidance: + overrides.printDockerRuntimeDownGuidance ?? printDockerRuntimeDownGuidance, + recoverDockerDriverSandbox: overrides.recoverDockerDriverSandbox ?? recoverDockerDriverSandbox, + resolvePodmanRuntimeSocket: overrides.resolvePodmanRuntimeSocket ?? resolvePodmanRuntimeSocket, + resolveSandboxManagedGatewayStateDirectory: + overrides.resolveSandboxManagedGatewayStateDirectory ?? + resolveSandboxManagedGatewayStateDirectory, + }; +} + +function dockerRuntimePreflight( + action: "start" | "stop", + input: SandboxLifecycleRuntimeInput, + deps: SandboxLifecycleRuntimeDependencies, +): SandboxLifecycleResult | null { + if (!deps.isDockerRuntimeDown(input.sandboxName)) return null; + deps.printDockerRuntimeDownGuidance(input.sandboxName, { retryCommand: action }); + return { exitCode: 1 }; +} + +function isPausedDockerStatus(status: string): boolean { + return status.startsWith("Up") && status.endsWith("(Paused)"); +} + +function isAtRestDockerStatus(status: string): boolean { + return AT_REST_DOCKER_STATUS_PREFIXES.some((prefix) => status.startsWith(prefix)); +} + +const DOCKER_LIFECYCLE_ADAPTER: SandboxLifecycleRuntimeAdapter = { + channelStopTransport: "docker-kubectl-first", + displayName: "Docker", + driverName: "docker", + preflight: dockerRuntimePreflight, + start(input, deps) { + const containers = deps.findLabeledSandboxContainers(input.sandboxName); + const paused = containers.find((container) => isPausedDockerStatus(container.status)); + if (paused) { + const result = deps.dockerUnpause(paused.name, { + ignoreError: true, + timeout: DOCKER_OPERATION_TIMEOUT_MS, + }); + if (result.status !== 0) { + return { + exitCode: 1, + message: ` docker unpause ${paused.name} failed (exit ${result.status ?? "unknown"}).`, + }; + } + input.log(` Container '${paused.name}' unpaused.`); + return { exitCode: 0 }; + } + + const recovery = deps.recoverDockerDriverSandbox(input.sandboxName); + if (!recovery.recovered) { + return { + exitCode: 1, + message: + ` Could not start sandbox '${input.sandboxName}': ${recovery.detail ?? "unknown failure"}. ` + + `If the container was removed, run '${CLI_NAME} ${input.sandboxName} rebuild' to recreate it.`, + }; + } + if (recovery.via === "started-running-original") { + input.log(` Sandbox '${input.sandboxName}' is already running.`); + } else { + input.log(` Container '${recovery.containerName ?? input.sandboxName}' started.`); + } + return { exitCode: 0 }; + }, + stop(input, deps, hooks) { + const containers = deps.findLabeledSandboxContainers(input.sandboxName); + if (containers.length === 0) { + return { + exitCode: 1, + message: + ` No Docker container found for sandbox '${input.sandboxName}'. ` + + `If the container was removed, run '${CLI_NAME} ${input.sandboxName} rebuild' to recreate it.`, + }; + } + + const stoppable = containers.filter((container) => !isAtRestDockerStatus(container.status)); + if (stoppable.length === 0) return { exitCode: 0, state: "already-stopped" }; + + hooks.beforeStop(); + const failures: string[] = []; + for (const container of stoppable) { + input.log(` Stopping container '${container.name}'…`); + const result = deps.dockerStop(container.name, { + ignoreError: true, + timeout: DOCKER_OPERATION_TIMEOUT_MS, + }); + if (result.status !== 0) { + failures.push(`${container.name} (exit ${result.status ?? "unknown"})`); + } + } + if (failures.length > 0) { + return { + exitCode: 1, + message: ` docker stop failed for: ${failures.join(", ")}.`, + }; + } + return { exitCode: 0, state: "stopped" }; + }, +}; + +type PodmanManagedContainer = { + readonly bin: string; + readonly containerId: string; + readonly name: string; + readonly paused: boolean; + readonly running: boolean; + readonly socketAuthority: PodmanSocketAuthority; + readonly socketPath: string; + readonly status: string; +}; + +function podmanCommandDetail(result: CommandCapture): string { + return (result.stderr || result.stdout || result.error?.message || "unknown failure") + .replace(/\s+/gu, " ") + .trim() + .slice(-500); +} + +function podmanFailure(operation: string, result: CommandCapture): SandboxLifecycleResult { + const detail = podmanCommandDetail(result); + return { + exitCode: 1, + message: ` podman ${operation} failed (exit ${String(result.status)})${ + detail ? `: ${detail}` : "." + }`, + }; +} + +function requirePodmanContainerState( + raw: Readonly>, +): Pick { + const value = raw.State; + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error("Podman managed sandbox inspect has no object State."); + } + const state = value as Record; + const paused = state.Paused; + if (paused !== undefined && paused !== null && typeof paused !== "boolean") { + throw new Error("Podman managed sandbox inspect State.Paused must be a boolean."); + } + const status = state.Status; + if (typeof status !== "string" || !status.trim()) { + throw new Error("Podman managed sandbox inspect State.Status must be a non-empty string."); + } + return { + paused: paused === true, + status: status.trim().toLowerCase(), + }; +} + +function podmanRuntimeIdentity( + input: SandboxLifecycleRuntimeInput, + deps: SandboxLifecycleRuntimeDependencies, +): { bin: string; socketAuthority: PodmanSocketAuthority; socketPath: string } { + const stateDir = deps.resolveSandboxManagedGatewayStateDirectory( + input.sandbox, + input.environment, + ); + const socketPath = deps.resolvePodmanRuntimeSocket(stateDir, input.environment); + const socketAuthority = deps.capturePodmanSocketAuthority(socketPath); + if (socketAuthority.socketPath !== socketPath) { + throw new Error( + "Managed Podman lifecycle socket authority does not match its runtime binding.", + ); + } + deps.assertPodmanSocketAuthority(socketAuthority); + return { + bin: input.environment.NEMOCLAW_PODMAN_BIN?.trim() || "podman", + socketAuthority, + socketPath, + }; +} + +function captureAuthorizedPodmanCommand( + deps: SandboxLifecycleRuntimeDependencies, + runtime: Pick, + args: string[], + timeoutMs: number, +): CommandCapture { + if (runtime.socketAuthority.socketPath !== runtime.socketPath) { + throw new Error("Managed Podman lifecycle authority changed its socket path."); + } + deps.assertPodmanSocketAuthority(runtime.socketAuthority); + const result = deps.captureHostCommand(runtime.bin, args, timeoutMs); + deps.assertPodmanSocketAuthority(runtime.socketAuthority); + return result; +} + +function inspectPodmanManagedSandbox( + input: SandboxLifecycleRuntimeInput, + deps: SandboxLifecycleRuntimeDependencies, +): PodmanManagedContainer | SandboxLifecycleResult { + let runtime: ReturnType; + try { + runtime = podmanRuntimeIdentity(input, deps); + } catch (error) { + return { + exitCode: 1, + message: ` Could not resolve the managed Podman runtime: ${ + error instanceof Error ? error.message : String(error) + }`, + }; + } + + const socketUrl = `unix://${runtime.socketPath}`; + let lookup: CommandCapture; + try { + lookup = captureAuthorizedPodmanCommand( + deps, + runtime, + [ + "--url", + socketUrl, + "ps", + "--all", + "--no-trunc", + "--filter", + `label=${PODMAN_MANAGED_LABEL}=true`, + "--filter", + `label=${PODMAN_SANDBOX_NAME_LABEL}=${input.sandboxName}`, + "--format", + "{{.ID}}\t{{.Names}}", + ], + PODMAN_PROBE_TIMEOUT_MS, + ); + } catch (error) { + return { + exitCode: 1, + message: ` Refusing Podman lifecycle mutation: ${ + error instanceof Error ? error.message : String(error) + }`, + }; + } + if (lookup.status !== 0) return podmanFailure("container lookup", lookup); + + let candidates: Array<{ containerId: string; name: string }>; + try { + candidates = lookup.stdout + .split(/\r?\n/gu) + .map((line) => line.trim()) + .filter(Boolean) + .map((line) => { + const columns = line.split("\t"); + if (columns.length !== 2 || !columns[0] || !columns[1]) { + throw new Error("Podman managed sandbox lookup returned a malformed row."); + } + return { containerId: columns[0], name: columns[1] }; + }); + } catch (error) { + return { + exitCode: 1, + message: ` Refusing Podman lifecycle mutation: ${ + error instanceof Error ? error.message : String(error) + }`, + }; + } + if (candidates.length === 0) { + return { + exitCode: 1, + message: + ` No Podman container found for sandbox '${input.sandboxName}'. ` + + `If the container was removed, run '${CLI_NAME} ${input.sandboxName} rebuild' to recreate it.`, + }; + } + if (candidates.length !== 1) { + return { + exitCode: 1, + message: + ` Refusing Podman lifecycle mutation: sandbox '${input.sandboxName}' has ` + + `${candidates.length} managed containers.`, + }; + } + + const candidate = candidates[0]; + const expectedName = `${PODMAN_SANDBOX_CONTAINER_PREFIX}${input.sandboxName}`; + if ( + !candidate || + candidate.name !== expectedName || + !FULL_CONTAINER_ID_RE.test(candidate.containerId) + ) { + return { + exitCode: 1, + message: ` Refusing Podman lifecycle mutation: sandbox '${input.sandboxName}' has an unexpected container identity.`, + }; + } + let inspect: CommandCapture; + try { + inspect = captureAuthorizedPodmanCommand( + deps, + runtime, + ["--url", socketUrl, "container", "inspect", candidate.containerId], + PODMAN_PROBE_TIMEOUT_MS, + ); + } catch (error) { + return { + exitCode: 1, + message: ` Refusing Podman lifecycle mutation: ${ + error instanceof Error ? error.message : String(error) + }`, + }; + } + if (inspect.status !== 0) return podmanFailure("container inspect", inspect); + + try { + const parsed = parsePodmanManagedSandboxInspect(inspect.stdout, { + containerId: candidate.containerId, + name: expectedName, + sandboxName: input.sandboxName, + }); + const state = requirePodmanContainerState(parsed.raw); + return { + bin: runtime.bin, + containerId: parsed.containerId, + name: parsed.name, + paused: state.paused, + running: parsed.running, + socketAuthority: runtime.socketAuthority, + socketPath: runtime.socketPath, + status: state.status, + }; + } catch (error) { + return { + exitCode: 1, + message: ` Refusing Podman lifecycle mutation: ${ + error instanceof Error ? error.message : String(error) + }`, + }; + } +} + +function isLifecycleFailure( + value: PodmanManagedContainer | SandboxLifecycleResult, +): value is SandboxLifecycleResult { + return Object.hasOwn(value, "exitCode"); +} + +function runPodmanContainerMutation( + deps: SandboxLifecycleRuntimeDependencies, + runtime: Pick, + operation: "start" | "stop" | "unpause", + containerId: string, +): SandboxLifecycleResult { + const args = [ + "--url", + `unix://${runtime.socketPath}`, + operation, + ...(operation === "stop" ? ["--time", String(PODMAN_STOP_GRACE_SECONDS)] : []), + containerId, + ]; + let result: CommandCapture; + try { + result = captureAuthorizedPodmanCommand(deps, runtime, args, PODMAN_OPERATION_TIMEOUT_MS); + } catch (error) { + return { + exitCode: 1, + message: ` Refusing Podman ${operation}: ${ + error instanceof Error ? error.message : String(error) + }`, + }; + } + return result.status === 0 ? { exitCode: 0 } : podmanFailure(operation, result); +} + +const PODMAN_LIFECYCLE_ADAPTER: SandboxLifecycleRuntimeAdapter = { + channelStopTransport: "openshell", + displayName: "Podman", + driverName: "podman", + preflight() { + return null; + }, + start(input, deps) { + const container = inspectPodmanManagedSandbox(input, deps); + if (isLifecycleFailure(container)) return container; + if (container.running && !container.paused) { + input.log(` Sandbox '${input.sandboxName}' is already running.`); + return { exitCode: 0 }; + } + if (!container.paused && !AT_REST_PODMAN_STATES.has(container.status)) { + return { + exitCode: 1, + message: + ` Refusing Podman start for sandbox '${input.sandboxName}': ` + + `container state '${container.status}' is not safely restartable.`, + }; + } + const operation = container.paused ? "unpause" : "start"; + const result = runPodmanContainerMutation(deps, container, operation, container.containerId); + if (result.exitCode !== 0) return result; + input.log( + ` Container '${container.name}' ${operation === "unpause" ? "unpaused" : "started"}.`, + ); + return result; + }, + stop(input, deps, hooks) { + const container = inspectPodmanManagedSandbox(input, deps); + if (isLifecycleFailure(container)) return container; + const stoppable = + container.running || + container.paused || + container.status === "restarting" || + container.status === "stopping"; + if (!stoppable) { + if (AT_REST_PODMAN_STATES.has(container.status)) { + return { exitCode: 0, state: "already-stopped" }; + } + return { + exitCode: 1, + message: + ` Refusing Podman stop for sandbox '${input.sandboxName}': ` + + `container state '${container.status}' is not safely stoppable.`, + }; + } + + hooks.beforeStop(); + input.log(` Stopping container '${container.name}'…`); + const result = runPodmanContainerMutation(deps, container, "stop", container.containerId); + return result.exitCode === 0 ? { ...result, state: "stopped" } : result; + }, +}; + +export const CURRENT_SANDBOX_LIFECYCLE_RUNTIME_ADAPTERS = { + docker: DOCKER_LIFECYCLE_ADAPTER, + podman: PODMAN_LIFECYCLE_ADAPTER, +} as const satisfies SandboxLifecycleRuntimeAdapterRegistry; + +/** + * Resolve persisted runtime identity without letting new drivers inherit an + * existing runtime's lifecycle behavior. Empty and `vm` entries retain the + * historical Docker implementation; every named future driver must register + * an exact adapter. + */ +export function resolveSandboxLifecycleRuntimeAdapter( + driverName: string | null | undefined, + adapters: SandboxLifecycleRuntimeAdapterRegistry = CURRENT_SANDBOX_LIFECYCLE_RUNTIME_ADAPTERS, +): SandboxLifecycleRuntimeAdapter | null { + const normalized = driverName?.trim().toLowerCase(); + const runtimeDriver = !normalized || normalized === "vm" ? "docker" : normalized; + const adapter = Object.hasOwn(adapters, runtimeDriver) ? adapters[runtimeDriver] : undefined; + if (!adapter) return null; + if (adapter.driverName !== runtimeDriver) { + throw new Error( + `Sandbox lifecycle runtime adapter '${runtimeDriver}' does not match its registered driver identity.`, + ); + } + return adapter; +} diff --git a/src/lib/actions/sandbox/runtime/podman-socket.ts b/src/lib/actions/sandbox/runtime/podman-socket.ts new file mode 100644 index 0000000000..deee25b77a --- /dev/null +++ b/src/lib/actions/sandbox/runtime/podman-socket.ts @@ -0,0 +1,66 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import path from "node:path"; + +import { + type ManagedGatewayRuntimeBinding, + readManagedGatewayRuntimeBinding, +} from "../../../onboard/docker-driver-gateway-config"; + +export interface ResolvePodmanRuntimeSocketDeps { + readRuntimeBinding?: (stateDir: string) => ManagedGatewayRuntimeBinding | null; +} + +function requireAbsoluteSocketPath(socketPath: string, source: string): string { + const normalized = socketPath.trim(); + if (!normalized || !path.isAbsolute(normalized)) { + throw new Error(`${source} must contain an absolute Podman socket path.`); + } + return normalized; +} + +/** + * Resolve the exact socket bound to a Podman-backed managed gateway. + * + * An explicit environment override remains useful for recovery, but fresh + * lifecycle commands normally recover the value from the protected, + * config-bound runtime sidecar written during onboarding. + */ +export function resolvePodmanRuntimeSocket( + stateDir: string | null | undefined, + environment: NodeJS.ProcessEnv = process.env, + deps: ResolvePodmanRuntimeSocketDeps = {}, +): string { + if (!stateDir) { + throw new Error("Podman runtime socket recovery requires a managed gateway state directory."); + } + + const binding = (deps.readRuntimeBinding ?? readManagedGatewayRuntimeBinding)(stateDir); + if (!binding) { + throw new Error(`Managed runtime binding is missing in '${stateDir}'.`); + } + if (binding.driverName !== "podman") { + throw new Error( + `Managed runtime binding in '${stateDir}' declares driver '${binding.driverName}', not 'podman'.`, + ); + } + const socketPath = binding.values.socket_path; + if (typeof socketPath !== "string") { + throw new Error(`Managed Podman runtime binding in '${stateDir}' has no string socket_path.`); + } + const persisted = requireAbsoluteSocketPath( + socketPath, + `Managed Podman runtime binding in '${stateDir}'`, + ); + const explicit = environment.OPENSHELL_PODMAN_SOCKET?.trim(); + if (explicit) { + const requested = requireAbsoluteSocketPath(explicit, "OPENSHELL_PODMAN_SOCKET"); + if (requested !== persisted) { + throw new Error( + `OPENSHELL_PODMAN_SOCKET does not match the managed Podman runtime binding in '${stateDir}'.`, + ); + } + } + return persisted; +} diff --git a/src/lib/actions/sandbox/sandbox-gateway-routing.test.ts b/src/lib/actions/sandbox/sandbox-gateway-routing.test.ts index b1cbc20ec1..8e0067e345 100644 --- a/src/lib/actions/sandbox/sandbox-gateway-routing.test.ts +++ b/src/lib/actions/sandbox/sandbox-gateway-routing.test.ts @@ -60,6 +60,10 @@ describe("sandbox gateway routing helpers", () => { ); }); + it("routes Podman through OpenShell metadata instead of a Docker cluster-container probe", () => { + expect(routing.usesGatewayMetadataProbe("podman")).toBe(true); + }); + it("selects the persisted gateway before sandbox-scoped OpenShell commands", () => { routing.selectSandboxGatewayIfRegistered("alpha"); diff --git a/src/lib/actions/sandbox/sandbox-gateway-routing.ts b/src/lib/actions/sandbox/sandbox-gateway-routing.ts index 22bf2b3f68..82e30c40f0 100644 --- a/src/lib/actions/sandbox/sandbox-gateway-routing.ts +++ b/src/lib/actions/sandbox/sandbox-gateway-routing.ts @@ -47,7 +47,7 @@ export function probeGatewayMetadataHealth(gatewayName: string): boolean { } export function usesGatewayMetadataProbe(driver: string | null | undefined): boolean { - return driver === "docker" || driver === "vm"; + return driver === "docker" || driver === "podman" || driver === "vm"; } /** diff --git a/src/lib/actions/sandbox/snapshot-managed-profile-lifecycle.test.ts b/src/lib/actions/sandbox/snapshot-managed-profile-lifecycle.test.ts new file mode 100644 index 0000000000..feef050cb1 --- /dev/null +++ b/src/lib/actions/sandbox/snapshot-managed-profile-lifecycle.test.ts @@ -0,0 +1,851 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import type { SandboxMessagingPlan } from "../../messaging/manifest"; +import { decodeManagedStartupProfile } from "../../onboard/managed-startup/profile"; +import { buildManagedStartupProfile } from "../../onboard/managed-startup/profile-builder"; +import { SANDBOX_CREATE_MAX_ARGUMENT_BYTES } from "../../onboard/sandbox-create/transport"; +import { withSandboxMutationLock } from "../../state/mcp-lifecycle-lock"; +import * as s from "./snapshot/lifecycle-test-support"; +import * as f from "./snapshot-restore-test-fixture"; + +const tempHomes: string[] = []; +const PROXY_ENV_NAMES = [ + "HTTP_PROXY", + "HTTPS_PROXY", + "NO_PROXY", + "http_proxy", + "https_proxy", + "no_proxy", +] as const; + +function managedMessagingPlan( + agent: "openclaw" | "hermes", + sandboxName: string, + allowedId: string, + credentialHash?: string, +): SandboxMessagingPlan { + return { + schemaVersion: 1, + sandboxName, + agent, + workflow: "onboard", + channels: [ + { + channelId: "telegram", + configured: true, + active: true, + disabled: false, + inputs: [ + { inputId: "botToken", credentialAvailable: true }, + { inputId: "allowedIds", value: [allowedId] }, + ], + }, + ], + disabledChannels: [], + credentialBindings: credentialHash + ? [ + { + channelId: "telegram", + providerEnvKey: "TELEGRAM_BOT_TOKEN", + credentialAvailable: true, + credentialHash, + }, + ] + : [], + networkPolicy: { presets: [], entries: [] }, + agentRender: [], + buildSteps: [], + runtimeSetup: { nodePreloads: [], envAliases: [], secretScans: [] }, + stateUpdates: [], + healthChecks: [], + } as unknown as SandboxMessagingPlan; +} + +function providerMetadata(name: string, type: string, credentialEnv: string): string { + return [ + `Name: ${name}`, + `Type: ${type}`, + `Credential keys: ${credentialEnv}`, + "Config keys: ", + "", + ].join("\n"); +} + +function managedOpenClawProfile() { + return buildManagedStartupProfile({ + agent: "openclaw", + inference: { + routeProvider: "inference", + upstreamProvider: "openai-api", + model: "gpt-5.4", + routedBaseUrl: "https://inference.local/v1", + upstreamEndpointUrl: null, + api: "openai-responses", + primaryModelRef: "inference/gpt-5.4", + compatibility: {}, + }, + dashboard: { + agent: "openclaw", + mode: "loopback", + url: "http://127.0.0.1:18789", + port: 18_789, + bindAddress: "127.0.0.1", + wslExposure: false, + }, + webSearch: null, + toolDisclosure: "progressive", + hermesToolGateways: [], + messagingPlan: null, + dcodeAutoApprovalMode: null, + observabilityEnabled: null, + environment: {}, + corporateCa: null, + }); +} + +function capturedManagedStartupRootApplyRequest(): { + readonly agent: "openclaw" | "hermes" | "langchain-deepagents-code"; + readonly encodedProfile: string; + readonly profileFingerprint: string; +} { + const options = f.createDockerGpuSandboxCreatePatchMock.mock.calls.at(-1)?.[0] as + | { + managedStartupRootApplyRequest?: { + agent: "openclaw" | "hermes" | "langchain-deepagents-code"; + encodedProfile: string; + profileFingerprint: string; + }; + } + | undefined; + const request = options?.managedStartupRootApplyRequest; + expect(request).toBeDefined(); + return request!; +} + +beforeEach(() => { + f.resetSnapshotRestoreMocks(); +}); +afterEach(() => { + f.cleanupSnapshotRestoreMocks(); + for (const tempHome of tempHomes.splice(0)) { + fs.rmSync(tempHome, { recursive: true, force: true }); + } +}); +describe("runSandboxSnapshot restore: lifecycle and destination safety", () => { + it("refuses provider replacement when an unrelated sandbox is attached", async () => { + const commands: Array<{ args: string[]; options?: Record }> = []; + const runner = vi.fn( + s.recordingCommandRouter(commands, { + "provider get beta-telegram-bridge": () => ({ + status: 0, + stdout: providerMetadata("beta-telegram-bridge", "generic", "TELEGRAM_BOT_TOKEN"), + }), + "provider delete beta-telegram-bridge": () => ({ + status: 1, + stderr: "provider 'beta-telegram-bridge' is attached to sandbox(es): beta, gamma", + }), + }), + ); + const { provisionManagedCloneProviders } = await import("./snapshot/managed-clone-providers"); + + expect(() => + provisionManagedCloneProviders( + [ + { + providerName: "beta-telegram-bridge", + providerType: "generic", + providerEnvKey: "TELEGRAM_BOT_TOKEN", + source: "messaging", + replaceExistingCredential: true, + }, + ], + { + environment: { TELEGRAM_BOT_TOKEN: "replacement-secret" }, + runOpenshell: runner, + rollbackSandboxName: "beta", + }, + ), + ).toThrow("is still attached outside destination 'beta'"); + expect(commands.some(({ args }) => args[0] === "sandbox")).toBe(false); + expect(commands.some(({ args }) => ["create", "update"].includes(args[1] ?? ""))).toBe(false); + expect(commands.every(({ options }) => options?.env === undefined)).toBe(true); + }); + + it("starts and registers a managed DCode clone with its receipt-bound profile transport", async () => { + const built = buildManagedStartupProfile({ + agent: "langchain-deepagents-code", + inference: { + routeProvider: "inference", + upstreamProvider: "openrouter", + model: "openai/gpt-5.4", + routedBaseUrl: "https://inference.local/v1", + upstreamEndpointUrl: "https://openrouter.ai/api/v1", + api: "openai-completions", + primaryModelRef: null, + compatibility: null, + }, + dashboard: { agent: "langchain-deepagents-code", mode: "disabled" }, + webSearch: null, + toolDisclosure: "progressive", + hermesToolGateways: [], + messagingPlan: null, + dcodeAutoApprovalMode: "disabled", + observabilityEnabled: false, + environment: {}, + corporateCa: null, + }); + const reference = `ghcr.io/nvidia/nemoclaw/langchain-deepagents-code-sandbox@sha256:${"a".repeat( + 64, + )}`; + const source = { + name: "alpha", + agent: "langchain-deepagents-code", + imageTag: reference, + openshellDriver: "docker", + provider: "openrouter", + model: "openai/gpt-5.4", + endpointUrl: "https://openrouter.ai/api/v1", + preferredInferenceApi: "openai-completions", + toolDisclosure: "progressive", + dcodeAutoApprovalMode: "disabled", + observabilityEnabled: false, + workload: { + schemaVersion: 1, + kind: "managed-image", + reference, + release: "v0.0.99", + sourceRevision: "b".repeat(40), + sourceCohort: "ghrun-123456-1", + capabilityContractVersion: 1, + startupProfileContractVersion: 1, + encodedProfile: built.encodedProfile, + startupProfileSha256: built.startupProfileSha256, + credentialProxyReplayRequired: false, + shared: true, + }, + } as const; + let registeredClone: f.SandboxRecord | null = null; + f.registerSandboxMock.mockImplementation( + (entry) => (registeredClone = entry as f.SandboxRecord), + ); + f.getSandboxMock.mockImplementation((name) => + name === "alpha" ? (source as never) : registeredClone, + ); + f.parseLiveSandboxNamesMock.mockReturnValue(new Set(["alpha"])); + f.captureOpenshellMock.mockImplementation((args) => + f.openshellResponses(args, { + "sandbox exec": { status: 0, output: f.dcodeProbeOutput("idle") }, + "sandbox list": { status: 0, output: "alpha Ready\nbeta Ready\n" }, + }), + ); + f.getLatestBackupMock.mockReturnValue({ ...f.latestBackupFixture }); + f.restoreSandboxStateMock.mockReturnValue({ + success: true, + restoredDirs: ["workspace"], + restoredFiles: [], + failedDirs: [], + failedFiles: [], + }); + vi.stubEnv("NEMOCLAW_STARTUP_PROFILE_B64", "ambient-profile"); + const { runSandboxSnapshot } = await import("./snapshot"); + + await runSandboxSnapshot("alpha", { kind: "restore", to: "beta" }); + + const createCall = f.streamSandboxCreateMock.mock.calls[0] ?? []; + const createExecutable = createCall[0] as string; + const createArgs = createCall[1] as readonly string[]; + const createEnv = createCall[2] as NodeJS.ProcessEnv | undefined; + expect( + [createExecutable, ...createArgs].every( + (argument) => Buffer.byteLength(argument, "utf8") + 1 <= SANDBOX_CREATE_MAX_ARGUMENT_BYTES, + ), + ).toBe(true); + const rootApplyRequest = capturedManagedStartupRootApplyRequest(); + expect(rootApplyRequest.encodedProfile).toBe(built.encodedProfile); + expect(createArgs.slice(createArgs.lastIndexOf("--") + 1)).toEqual([ + "env", + "/usr/local/bin/nemoclaw-managed-startup-hold", + "--agent", + "langchain-deepagents-code", + "--profile-fingerprint", + rootApplyRequest.profileFingerprint, + ]); + expect(createArgs.join(" ")).not.toContain("NEMOCLAW_STARTUP_PROFILE_B64"); + expect(createEnv?.NEMOCLAW_STARTUP_PROFILE_B64).toBeUndefined(); + expect(f.registerSandboxMock).toHaveBeenCalledWith( + expect.objectContaining({ + name: "beta", + workload: source.workload, + messaging: undefined, + }), + ); + }); + + it("routes managed root-apply failures through snapshot clone cleanup", async () => { + const built = managedOpenClawProfile(); + const reference = `ghcr.io/nvidia/nemoclaw/openclaw-sandbox@sha256:${"a".repeat(64)}`; + const source = { + name: "alpha", + agent: "openclaw", + dashboardPort: 18_789, + imageTag: reference, + openshellDriver: "docker", + provider: "openai-api", + model: "gpt-5.4", + endpointUrl: null, + preferredInferenceApi: "openai-responses", + compatibleEndpointReasoning: null, + toolDisclosure: "progressive", + workload: { + schemaVersion: 1, + kind: "managed-image", + reference, + release: "v0.0.99", + sourceRevision: "b".repeat(40), + sourceCohort: "ghrun-123456-1", + capabilityContractVersion: 1, + startupProfileContractVersion: 1, + encodedProfile: built.encodedProfile, + startupProfileSha256: built.startupProfileSha256, + credentialProxyReplayRequired: false, + shared: true, + }, + } as const; + f.getSandboxMock.mockImplementation((name) => (name === "alpha" ? (source as never) : null)); + f.parseLiveSandboxNamesMock.mockReturnValue(new Set(["alpha"])); + f.captureOpenshellMock.mockImplementation((args) => + f.openshellResponses(args, { + "sandbox exec": { status: 0, output: f.dcodeProbeOutput("no-runtime") }, + "sandbox list": { status: 0, output: "alpha Ready\nbeta Ready\n" }, + }), + ); + f.getLatestBackupMock.mockReturnValue({ ...f.latestBackupFixture }); + f.createDockerGpuSandboxCreatePatchMock.mockImplementation((rawOptions) => { + const options = rawOptions as { + overrides?: { + onPatchFailureExit?: ( + sandboxName: string, + error: unknown, + deps: Record, + ) => void; + }; + }; + return { + maybeApplyDuringCreate: vi.fn(), + createFailureMessage: vi.fn(() => null), + exitOnPatchError: vi.fn(), + rollbackManagedStartupAfterCreateFailure: vi.fn(), + ensureApplied: vi.fn(() => + options.overrides?.onPatchFailureExit?.( + "beta", + new Error("managed root apply failed"), + {}, + ), + ), + revalidateBeforeMutation: vi.fn(), + waitForSupervisorReconnectIfNeeded: vi.fn(), + commitAfterReady: vi.fn(), + selectedMode: vi.fn(() => null), + printReadinessFailureIfEnabled: vi.fn(), + verifyGpuOrExit: vi.fn(), + }; + }); + const { runSandboxSnapshot } = await import("./snapshot"); + + await expect(runSandboxSnapshot("alpha", { kind: "restore", to: "beta" })).rejects.toThrow( + "managed root apply failed", + ); + + expect(f.runOpenshellMock).toHaveBeenCalledWith( + ["sandbox", "delete", "beta"], + expect.objectContaining({ ignoreError: true }), + ); + expect(f.registerSandboxMock).not.toHaveBeenCalled(); + }); + + it("reconciles a stale OpenClaw receipt to current inference and messaging before clone launch", async () => { + const oldMessaging = managedMessagingPlan("openclaw", "alpha", "111111"); + const staleSourceCredentialHash = "f".repeat(64); + const currentMessaging = managedMessagingPlan( + "openclaw", + "alpha", + "222222", + staleSourceCredentialHash, + ); + const built = buildManagedStartupProfile({ + agent: "openclaw", + inference: { + routeProvider: "openai", + upstreamProvider: "openai-api", + model: "gpt-5.4", + routedBaseUrl: "https://inference.local/v1", + upstreamEndpointUrl: null, + api: "openai-responses", + primaryModelRef: "openai/gpt-5.4", + compatibility: {}, + }, + dashboard: { + agent: "openclaw", + mode: "loopback", + url: "http://127.0.0.1:18789", + port: 18_789, + bindAddress: "127.0.0.1", + wslExposure: false, + }, + webSearch: { fetchEnabled: false, provider: "brave" }, + toolDisclosure: "progressive", + hermesToolGateways: [], + messagingPlan: oldMessaging, + dcodeAutoApprovalMode: null, + observabilityEnabled: null, + environment: { NEMOCLAW_CONTEXT_WINDOW: "65536" }, + corporateCa: null, + }); + const reference = `ghcr.io/nvidia/nemoclaw/openclaw-sandbox@sha256:${"a".repeat(64)}`; + const source = { + name: "alpha", + agent: "openclaw", + dashboardPort: 18_789, + dashboardRemoteBindPrepared: false, + imageTag: reference, + openshellDriver: "docker", + provider: "compatible-endpoint", + model: "gpt-5.5", + endpointUrl: "https://compatible.example.test/v1", + endpointSource: "explicit", + preferredInferenceApi: "openai-completions", + compatibleEndpointReasoning: "true", + compatibleEndpointReasoningEffort: "high", + toolDisclosure: "direct", + webSearchEnabled: true, + webSearchProvider: "brave", + messaging: { schemaVersion: 1, plan: currentMessaging }, + workload: { + schemaVersion: 1, + kind: "managed-image", + reference, + release: "v0.0.99", + sourceRevision: "b".repeat(40), + sourceCohort: "ghrun-123456-1", + capabilityContractVersion: 1, + startupProfileContractVersion: 1, + encodedProfile: built.encodedProfile, + startupProfileSha256: built.startupProfileSha256, + credentialProxyReplayRequired: false, + shared: true, + }, + } as const; + let registeredClone: f.SandboxRecord | null = null; + f.registerSandboxMock.mockImplementation( + (entry) => (registeredClone = entry as f.SandboxRecord), + ); + f.getSandboxMock.mockImplementation((name) => + name === "alpha" ? (source as never) : registeredClone, + ); + f.parseLiveSandboxNamesMock.mockReturnValue(new Set(["alpha"])); + f.captureOpenshellMock.mockImplementation((args) => + f.openshellResponses(args, { + "forward list": { + status: 0, + output: "alpha 127.0.0.1 18789 23189 running\n", + }, + "sandbox exec": { status: 0, output: f.dcodeProbeOutput("no-runtime") }, + "sandbox list": { status: 0, output: "alpha Ready\nbeta Ready\n" }, + }), + ); + f.getLatestBackupMock.mockReturnValue({ ...f.latestBackupFixture }); + vi.stubEnv("TELEGRAM_BOT_TOKEN", "clone-only-token"); + vi.stubEnv("BRAVE_API_KEY", "clone-only-brave-key"); + f.runOpenshellMock.mockImplementation( + s.managedProviderCreationRunner({ + "beta-telegram-bridge": { + type: "generic", + credential: "TELEGRAM_BOT_TOKEN", + }, + "beta-brave-search": { + type: "brave", + credential: "BRAVE_API_KEY", + }, + }), + ); + const { runSandboxSnapshot } = await import("./snapshot"); + + await runSandboxSnapshot("alpha", { kind: "restore", to: "beta" }); + + const createArgs = f.streamSandboxCreateMock.mock.calls[0]?.[1] as readonly string[]; + expect(createArgs).toContain("beta-telegram-bridge"); + expect( + createArgs.some( + (argument, index) => + argument === "--provider" && createArgs[index + 1] === "beta-telegram-bridge", + ), + ).toBe(true); + expect( + createArgs.some( + (argument, index) => + argument === "--provider" && createArgs[index + 1] === "beta-brave-search", + ), + ).toBe(true); + expect(f.runOpenshellMock).toHaveBeenCalledWith( + [ + "provider", + "create", + "--name", + "beta-telegram-bridge", + "--type", + "generic", + "--credential", + "TELEGRAM_BOT_TOKEN", + ], + expect.objectContaining({ + env: { TELEGRAM_BOT_TOKEN: "clone-only-token" }, + }), + ); + expect(f.runOpenshellMock).toHaveBeenCalledWith( + [ + "provider", + "create", + "--name", + "beta-brave-search", + "--type", + "brave", + "--credential", + "BRAVE_API_KEY", + ], + expect.objectContaining({ + env: { BRAVE_API_KEY: "clone-only-brave-key" }, + }), + ); + expect(createArgs.join(" ")).not.toContain("NEMOCLAW_STARTUP_PROFILE_B64"); + const encodedProfile = capturedManagedStartupRootApplyRequest().encodedProfile; + expect(encodedProfile).not.toBe(built.encodedProfile); + const profile = decodeManagedStartupProfile(encodedProfile); + expect(profile.inference).toMatchObject({ + upstreamProvider: "compatible-endpoint", + model: "gpt-5.5", + api: "openai-completions", + }); + expect(profile.tuning.contextWindow).toBe(131_072); + expect(profile.tuning).toMatchObject({ + reasoning: true, + reasoningEffort: "high", + }); + expect(profile.tools.disclosure).toBe("direct"); + expect(profile.agentConfig).toMatchObject({ + agent: "openclaw", + webSearch: { enabled: true, provider: "brave" }, + }); + const plan = profile.messaging.plan as unknown as SandboxMessagingPlan; + expect(plan.sandboxName).toBe("beta"); + expect(plan.channels[0]?.inputs).toContainEqual( + expect.objectContaining({ + inputId: "allowedIds", + value: ["222222"], + }), + ); + expect(JSON.stringify(plan)).not.toContain("111111"); + expect(JSON.stringify(plan)).not.toContain(staleSourceCredentialHash); + expect(plan.credentialBindings[0]).not.toHaveProperty("credentialHash"); + expect(encodedProfile).not.toContain("clone-only-token"); + expect(f.registerSandboxMock).toHaveBeenCalledWith( + expect.objectContaining({ + name: "beta", + provider: "compatible-endpoint", + model: "gpt-5.5", + preferredInferenceApi: "openai-completions", + compatibleEndpointReasoning: "true", + compatibleEndpointReasoningEffort: "high", + toolDisclosure: "direct", + webSearchEnabled: true, + webSearchProvider: "brave", + workload: expect.objectContaining({ encodedProfile }), + messaging: expect.objectContaining({ + schemaVersion: 1, + plan: expect.objectContaining({ sandboxName: "beta" }), + }), + }), + ); + }); + + it("reconciles a stale Hermes receipt to current route, tools, dashboard, and messaging", async () => { + const oldMessaging = managedMessagingPlan("hermes", "alpha", "111111"); + const currentMessaging = managedMessagingPlan("hermes", "alpha", "333333"); + const built = buildManagedStartupProfile({ + agent: "hermes", + inference: { + routeProvider: "inference", + upstreamProvider: "compatible-anthropic-endpoint", + model: "old-model", + routedBaseUrl: "https://inference.local/v1", + upstreamEndpointUrl: null, + api: "openai-completions", + primaryModelRef: null, + compatibility: null, + }, + dashboard: { + agent: "hermes", + mode: "loopback-forwarded", + url: "http://127.0.0.1:19189", + publicPort: 19_189, + internalPort: 29_000, + tuiEnabled: true, + }, + webSearch: { fetchEnabled: false, provider: "tavily" }, + toolDisclosure: "progressive", + hermesToolGateways: [], + messagingPlan: oldMessaging, + dcodeAutoApprovalMode: null, + observabilityEnabled: null, + environment: {}, + corporateCa: null, + }); + const reference = `ghcr.io/nvidia/nemoclaw/hermes-sandbox@sha256:${"a".repeat(64)}`; + const source = { + name: "alpha", + agent: "hermes", + dashboardPort: 19_189, + hermesDashboardEnabled: true, + hermesDashboardPort: 19_189, + hermesDashboardInternalPort: 29_189, + hermesDashboardTui: false, + imageTag: reference, + openshellDriver: "docker", + provider: "hermes-provider", + model: "new-model", + endpointUrl: null, + preferredInferenceApi: "openai-completions", + toolDisclosure: "direct", + webSearchEnabled: true, + webSearchProvider: "tavily", + hermesToolGateways: [], + messaging: { schemaVersion: 1, plan: currentMessaging }, + workload: { + schemaVersion: 1, + kind: "managed-image", + reference, + release: "v0.0.99", + sourceRevision: "b".repeat(40), + sourceCohort: "ghrun-123456-1", + capabilityContractVersion: 1, + startupProfileContractVersion: 1, + encodedProfile: built.encodedProfile, + startupProfileSha256: built.startupProfileSha256, + credentialProxyReplayRequired: false, + shared: true, + }, + } as const; + let registeredClone: f.SandboxRecord | null = null; + f.registerSandboxMock.mockImplementation( + (entry) => (registeredClone = entry as f.SandboxRecord), + ); + f.getSandboxMock.mockImplementation((name) => + name === "alpha" ? (source as never) : registeredClone, + ); + f.parseLiveSandboxNamesMock.mockReturnValue(new Set(["alpha"])); + f.captureOpenshellMock.mockImplementation((args) => + f.openshellResponses(args, { + "forward list": { + status: 0, + output: "alpha 127.0.0.1 19189 23189 running\n", + }, + "sandbox exec": { status: 0, output: f.dcodeProbeOutput("no-runtime") }, + "sandbox list": { status: 0, output: "alpha Ready\nbeta Ready\n" }, + }), + ); + f.getLatestBackupMock.mockReturnValue({ ...f.latestBackupFixture }); + vi.stubEnv("TELEGRAM_BOT_TOKEN", "clone-only-hermes-token"); + vi.stubEnv("TAVILY_API_KEY", "clone-only-tavily-key"); + f.runOpenshellMock.mockImplementation( + s.managedProviderCreationRunner({ + "beta-telegram-bridge": { + type: "generic", + credential: "TELEGRAM_BOT_TOKEN", + }, + "beta-tavily-search": { + type: "tavily-hermes-v1", + credential: "TAVILY_API_KEY", + }, + }), + ); + const { runSandboxSnapshot } = await import("./snapshot"); + + await runSandboxSnapshot("alpha", { kind: "restore", to: "beta" }); + + const createArgs = f.streamSandboxCreateMock.mock.calls[0]?.[1] as readonly string[]; + expect(createArgs.join(" ")).not.toContain("NEMOCLAW_STARTUP_PROFILE_B64"); + const encodedProfile = capturedManagedStartupRootApplyRequest().encodedProfile; + const profile = decodeManagedStartupProfile(encodedProfile); + expect(profile.inference).toMatchObject({ + upstreamProvider: "hermes-provider", + model: "new-model", + api: "openai-completions", + }); + expect(profile.tools).toEqual({ + disclosure: "direct", + enabledGateways: [], + }); + expect(profile.agentConfig).toMatchObject({ + agent: "hermes", + webSearch: { enabled: true, provider: "tavily" }, + }); + expect(profile.dashboard).toMatchObject({ + agent: "hermes", + mode: "loopback-forwarded", + internalPort: 29_189, + tuiEnabled: false, + }); + const plan = profile.messaging.plan as unknown as SandboxMessagingPlan; + expect(plan.sandboxName).toBe("beta"); + expect(plan.channels[0]?.inputs).toContainEqual( + expect.objectContaining({ + inputId: "allowedIds", + value: ["333333"], + }), + ); + expect(JSON.stringify(plan)).not.toContain("111111"); + expect( + createArgs.some( + (argument, index) => + argument === "--provider" && createArgs[index + 1] === "beta-tavily-search", + ), + ).toBe(true); + expect(f.runOpenshellMock).toHaveBeenCalledWith( + [ + "provider", + "create", + "--name", + "beta-tavily-search", + "--type", + "tavily-hermes-v1", + "--credential", + "TAVILY_API_KEY", + ], + expect.objectContaining({ + env: { TAVILY_API_KEY: "clone-only-tavily-key" }, + }), + ); + expect(f.registerSandboxMock).toHaveBeenCalledWith( + expect.objectContaining({ + name: "beta", + provider: "hermes-provider", + model: "new-model", + preferredInferenceApi: "openai-completions", + toolDisclosure: "direct", + webSearchEnabled: true, + webSearchProvider: "tavily", + hermesToolGateways: undefined, + hermesDashboardInternalPort: 29_189, + hermesDashboardTui: undefined, + workload: expect.objectContaining({ encodedProfile }), + }), + ); + }); + + it("replays launch-only authenticated proxies when cloning a managed OpenClaw sandbox", async () => { + const built = managedOpenClawProfile(); + const reference = `ghcr.io/nvidia/nemoclaw/openclaw-sandbox@sha256:${"a".repeat(64)}`; + const source = { + name: "alpha", + agent: "openclaw", + dashboardPort: 18_789, + imageTag: reference, + openshellDriver: "docker", + provider: "openai-api", + model: "gpt-5.4", + workload: { + schemaVersion: 1, + kind: "managed-image", + reference, + release: "v0.0.99", + sourceRevision: "b".repeat(40), + sourceCohort: "ghrun-123456-1", + capabilityContractVersion: 1, + startupProfileContractVersion: 1, + encodedProfile: built.encodedProfile, + startupProfileSha256: built.startupProfileSha256, + credentialProxyReplayRequired: true, + shared: true, + }, + } as const; + let registeredClone: f.SandboxRecord | null = null; + f.registerSandboxMock.mockImplementation( + (entry) => (registeredClone = entry as f.SandboxRecord), + ); + f.getSandboxMock.mockImplementation((name) => + name === "alpha" ? (source as never) : registeredClone, + ); + f.parseLiveSandboxNamesMock.mockReturnValue(new Set(["alpha"])); + f.captureOpenshellMock.mockImplementation((args) => + f.openshellResponses(args, { + "forward list": { + status: 0, + output: "alpha 127.0.0.1 18789 23189 running\n", + }, + "sandbox exec": { status: 0, output: f.dcodeProbeOutput("no-runtime") }, + "sandbox list": { status: 0, output: "alpha Ready\nbeta Ready\n" }, + }), + ); + f.getLatestBackupMock.mockReturnValue({ ...f.latestBackupFixture }); + f.restoreSandboxStateMock.mockReturnValue({ + success: true, + restoredDirs: ["workspace"], + restoredFiles: [], + failedDirs: [], + failedFiles: [], + }); + const credentialProxyEnvironment = { + HTTP_PROXY: "http://upper-http:upper-pass@upper-http.example.test:18080", + HTTPS_PROXY: "http://upper-https:upper-pass@upper-https.example.test:18443", + NO_PROXY: "upper.internal", + http_proxy: "http://lower-http:lower-pass@lower-http.example.test:28080", + https_proxy: "http://lower-https:lower-pass@lower-https.example.test:28443", + no_proxy: "lower.internal", + } as const; + for (const [name, value] of Object.entries(credentialProxyEnvironment)) { + vi.stubEnv(name, value); + } + const { runSandboxSnapshot } = await import("./snapshot"); + + await runSandboxSnapshot("alpha", { kind: "restore", to: "beta" }); + + const createCall = f.streamSandboxCreateMock.mock.calls[0] ?? []; + const createArgs = createCall[1] as readonly string[]; + const startupArgs = createArgs.slice(createArgs.lastIndexOf("--") + 1); + const rootApplyRequest = capturedManagedStartupRootApplyRequest(); + expect(startupArgs).toEqual([ + "env", + "HTTP_PROXY=http://upper-http:upper-pass@upper-http.example.test:18080", + "HTTPS_PROXY=http://upper-https:upper-pass@upper-https.example.test:18443", + expect.stringMatching(/^NO_PROXY=upper\.internal,localhost,/u), + "http_proxy=http://lower-http:lower-pass@lower-http.example.test:28080", + "https_proxy=http://lower-https:lower-pass@lower-https.example.test:28443", + expect.stringMatching(/^no_proxy=lower\.internal,localhost,/u), + "/usr/local/bin/nemoclaw-managed-startup-hold", + "--agent", + "openclaw", + "--profile-fingerprint", + rootApplyRequest.profileFingerprint, + ]); + expect(startupArgs.join(" ")).not.toContain("NEMOCLAW_STARTUP_PROFILE_B64"); + const encodedProfile = rootApplyRequest.encodedProfile; + const reboundDashboard = decodeManagedStartupProfile(encodedProfile).dashboard; + expect(reboundDashboard).toMatchObject({ agent: "openclaw" }); + s.assertOpenClawDashboard(reboundDashboard); + expect(reboundDashboard.port).not.toBe(18_789); + expect(new URL(reboundDashboard.url).port).toBe(String(reboundDashboard.port)); + const registration = f.registerSandboxMock.mock.calls[0]?.[0] as + | { workload?: { credentialProxyReplayRequired?: boolean; encodedProfile?: string } } + | undefined; + expect(registration?.workload?.credentialProxyReplayRequired).toBe(true); + expect(registration?.workload?.encodedProfile).toBe(encodedProfile); + const durableReceipt = JSON.stringify(registration?.workload); + expect(durableReceipt).not.toContain("upper-pass"); + expect(durableReceipt).not.toContain("lower-pass"); + }); +}); diff --git a/src/lib/actions/sandbox/snapshot-restore-lifecycle.test.ts b/src/lib/actions/sandbox/snapshot-restore-lifecycle.test.ts index d1fbc6da9b..1529c651de 100644 --- a/src/lib/actions/sandbox/snapshot-restore-lifecycle.test.ts +++ b/src/lib/actions/sandbox/snapshot-restore-lifecycle.test.ts @@ -6,10 +6,109 @@ import os from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { SandboxMessagingPlan } from "../../messaging/manifest"; +import { decodeManagedStartupProfile } from "../../onboard/managed-startup/profile"; +import { buildManagedStartupProfile } from "../../onboard/managed-startup/profile-builder"; +import { SANDBOX_CREATE_MAX_ARGUMENT_BYTES } from "../../onboard/sandbox-create/transport"; import { withSandboxMutationLock } from "../../state/mcp-lifecycle-lock"; +import * as s from "./snapshot/lifecycle-test-support"; import * as f from "./snapshot-restore-test-fixture"; const tempHomes: string[] = []; +const PROXY_ENV_NAMES = [ + "HTTP_PROXY", + "HTTPS_PROXY", + "NO_PROXY", + "http_proxy", + "https_proxy", + "no_proxy", +] as const; + +function managedMessagingPlan( + agent: "openclaw" | "hermes", + sandboxName: string, + allowedId: string, + credentialHash?: string, +): SandboxMessagingPlan { + return { + schemaVersion: 1, + sandboxName, + agent, + workflow: "onboard", + channels: [ + { + channelId: "telegram", + configured: true, + active: true, + disabled: false, + inputs: [ + { inputId: "botToken", credentialAvailable: true }, + { inputId: "allowedIds", value: [allowedId] }, + ], + }, + ], + disabledChannels: [], + credentialBindings: credentialHash + ? [ + { + channelId: "telegram", + providerEnvKey: "TELEGRAM_BOT_TOKEN", + credentialAvailable: true, + credentialHash, + }, + ] + : [], + networkPolicy: { presets: [], entries: [] }, + agentRender: [], + buildSteps: [], + runtimeSetup: { nodePreloads: [], envAliases: [], secretScans: [] }, + stateUpdates: [], + healthChecks: [], + } as unknown as SandboxMessagingPlan; +} + +function providerMetadata(name: string, type: string, credentialEnv: string): string { + return [ + `Name: ${name}`, + `Type: ${type}`, + `Credential keys: ${credentialEnv}`, + "Config keys: ", + "", + ].join("\n"); +} + +function managedOpenClawProfile() { + return buildManagedStartupProfile({ + agent: "openclaw", + inference: { + routeProvider: "inference", + upstreamProvider: "openai-api", + model: "gpt-5.4", + routedBaseUrl: "https://inference.local/v1", + upstreamEndpointUrl: null, + api: "openai-responses", + primaryModelRef: "inference/gpt-5.4", + compatibility: {}, + }, + dashboard: { + agent: "openclaw", + mode: "loopback", + url: "http://127.0.0.1:18789", + port: 18_789, + bindAddress: "127.0.0.1", + wslExposure: false, + }, + webSearch: null, + toolDisclosure: "progressive", + hermesToolGateways: [], + messagingPlan: null, + dcodeAutoApprovalMode: null, + observabilityEnabled: null, + environment: {}, + corporateCa: null, + }); +} + beforeEach(() => { f.resetSnapshotRestoreMocks(); }); @@ -20,6 +119,825 @@ afterEach(() => { } }); describe("runSandboxSnapshot restore: lifecycle and destination safety", () => { + it("requires proxy re-onboarding before mutating a forced managed-clone destination", async () => { + const consoleError = vi.spyOn(console, "error").mockImplementation(() => {}); + const built = managedOpenClawProfile(); + const reference = `ghcr.io/nvidia/nemoclaw/openclaw-sandbox@sha256:${"a".repeat(64)}`; + f.getSandboxMock.mockImplementation( + s.valueByName({ + alpha: { + name: "alpha", + agent: "openclaw", + dashboardPort: 18_789, + imageTag: reference, + openshellDriver: "docker", + provider: "openai-api", + model: "gpt-5.4", + workload: { + schemaVersion: 1, + kind: "managed-image", + reference, + release: "v0.0.99", + sourceRevision: "b".repeat(40), + sourceCohort: "ghrun-123456-1", + capabilityContractVersion: 1, + startupProfileContractVersion: 1, + encodedProfile: built.encodedProfile, + startupProfileSha256: built.startupProfileSha256, + credentialProxyReplayRequired: true, + shared: true, + }, + } as never, + beta: { + name: "beta", + agent: "openclaw", + dashboardPort: 19_789, + imageTag: "nemoclaw-beta:test", + openshellDriver: "docker", + provider: "openai-api", + model: "gpt-5.4", + }, + }), + ); + f.parseLiveSandboxNamesMock.mockReturnValue(new Set(["alpha", "beta"])); + f.captureOpenshellMock.mockImplementation((args) => + f.openshellResponses(args, { + "forward list": { + status: 0, + output: "alpha 127.0.0.1 18789 23189 running\nbeta 127.0.0.1 19789 24189 running\n", + }, + "sandbox exec": { status: 0, output: f.dcodeProbeOutput("no-runtime") }, + "sandbox list": { status: 0, output: "alpha Ready\nbeta Ready\n" }, + }), + ); + f.getLatestBackupMock.mockReturnValue({ ...f.latestBackupFixture }); + for (const name of PROXY_ENV_NAMES) vi.stubEnv(name, ""); + const { runSandboxSnapshot } = await import("./snapshot"); + + await expect( + runSandboxSnapshot("alpha", { + kind: "restore", + to: "beta", + force: true, + yes: true, + }), + ).rejects.toMatchObject({ exitCode: 1 }); + + const output = consoleError.mock.calls.flat().join("\n"); + expect(output).toContain("requires a credential-bearing proxy"); + expect(output).toContain("Re-onboard the source before retrying this restore"); + expect(f.lifecycleMock.events).not.toContain("delete"); + expect(f.streamSandboxCreateMock).not.toHaveBeenCalled(); + expect(f.registerSandboxMock).not.toHaveBeenCalled(); + }); + + it("fails Podman runtime authority before forced destination or provider mutation", async () => { + const consoleError = vi.spyOn(console, "error").mockImplementation(() => {}); + const built = managedOpenClawProfile(); + const reference = `ghcr.io/nvidia/nemoclaw/openclaw-sandbox@sha256:${"a".repeat(64)}`; + f.getSandboxMock.mockImplementation( + s.valueByName({ + alpha: { + name: "alpha", + agent: "openclaw", + dashboardPort: 18_789, + dashboardRemoteBindPrepared: false, + imageTag: reference, + openshellDriver: "podman", + provider: "openai-api", + model: "gpt-5.4", + endpointUrl: null, + preferredInferenceApi: "openai-responses", + compatibleEndpointReasoning: null, + toolDisclosure: "progressive", + webSearchEnabled: false, + webSearchProvider: null, + workload: { + schemaVersion: 1, + kind: "managed-image", + reference, + release: "v0.0.99", + sourceRevision: "b".repeat(40), + sourceCohort: "ghrun-123456-1", + capabilityContractVersion: 1, + startupProfileContractVersion: 1, + encodedProfile: built.encodedProfile, + startupProfileSha256: built.startupProfileSha256, + credentialProxyReplayRequired: false, + shared: true, + }, + } as never, + beta: { + name: "beta", + agent: "openclaw", + dashboardPort: 19_789, + imageTag: "nemoclaw-beta:test", + openshellDriver: "podman", + provider: "openai-api", + model: "gpt-5.4", + }, + }), + ); + f.parseLiveSandboxNamesMock.mockReturnValue(new Set(["alpha", "beta"])); + f.captureOpenshellMock.mockImplementation((args) => + f.openshellResponses(args, { + "forward list": { + status: 0, + output: "alpha 127.0.0.1 18789 23189 running\nbeta 127.0.0.1 19789 24189 running\n", + }, + "sandbox exec": { status: 0, output: f.dcodeProbeOutput("no-runtime") }, + "sandbox list": { status: 0, output: "alpha Ready\nbeta Ready\n" }, + }), + ); + f.resolveManagedSnapshotRuntimeAuthorityMock.mockImplementation(() => { + throw new Error("socket authority changed"); + }); + f.getLatestBackupMock.mockReturnValue({ ...f.latestBackupFixture }); + const { runSandboxSnapshot } = await import("./snapshot"); + + await expect( + runSandboxSnapshot("alpha", { + kind: "restore", + to: "beta", + force: true, + yes: true, + }), + ).rejects.toMatchObject({ exitCode: 1 }); + + expect(consoleError.mock.calls.flat().join("\n")).toContain("socket authority changed"); + expect(f.lifecycleMock.events).not.toContain("delete"); + expect( + f.runOpenshellMock.mock.calls.some( + ([args]) => + args[0] === "provider" || + (args[0] === "sandbox" && args[1] === "provider") || + (args[0] === "sandbox" && args[1] === "delete"), + ), + ).toBe(false); + expect(f.streamSandboxCreateMock).not.toHaveBeenCalled(); + expect(f.registerSandboxMock).not.toHaveBeenCalled(); + }); + + it("bounds a managed clone launch before deleting a forced destination", async () => { + const built = buildManagedStartupProfile({ + agent: "langchain-deepagents-code", + inference: { + routeProvider: "inference", + upstreamProvider: "openrouter", + model: "openai/gpt-5.4", + routedBaseUrl: "https://inference.local/v1", + upstreamEndpointUrl: "https://openrouter.ai/api/v1", + api: "openai-completions", + primaryModelRef: null, + compatibility: null, + }, + dashboard: { agent: "langchain-deepagents-code", mode: "disabled" }, + webSearch: null, + toolDisclosure: "progressive", + hermesToolGateways: [], + messagingPlan: null, + dcodeAutoApprovalMode: "disabled", + observabilityEnabled: false, + environment: {}, + corporateCa: null, + }); + const reference = `registry.example.test/${"a".repeat(SANDBOX_CREATE_MAX_ARGUMENT_BYTES)}`; + f.getSandboxMock.mockImplementation( + s.valueByName({ + alpha: { + name: "alpha", + agent: "langchain-deepagents-code", + imageTag: reference, + openshellDriver: "docker", + provider: "nvidia-nim", + model: "nvidia/model-a", + workload: { + schemaVersion: 1, + kind: "managed-image", + reference, + release: "v0.0.99", + sourceRevision: "b".repeat(40), + sourceCohort: "ghrun-123456-1", + capabilityContractVersion: 1, + startupProfileContractVersion: 1, + encodedProfile: built.encodedProfile, + startupProfileSha256: built.startupProfileSha256, + credentialProxyReplayRequired: false, + shared: true, + }, + } as never, + beta: { + name: "beta", + agent: "langchain-deepagents-code", + imageTag: "nemoclaw-beta:test", + openshellDriver: "docker", + provider: "nvidia-nim", + model: "nvidia/model-a", + }, + }), + ); + f.parseLiveSandboxNamesMock.mockReturnValue(new Set(["alpha", "beta"])); + f.captureOpenshellMock.mockImplementation((args) => + f.openshellResponses(args, { + "sandbox exec": { status: 0, output: f.dcodeProbeOutput("idle") }, + "sandbox list": { status: 0, output: "alpha Ready\nbeta Ready\n" }, + }), + ); + f.getLatestBackupMock.mockReturnValue({ ...f.latestBackupFixture }); + const { runSandboxSnapshot } = await import("./snapshot"); + + await expect( + runSandboxSnapshot("alpha", { + kind: "restore", + to: "beta", + force: true, + yes: true, + }), + ).rejects.toThrow(/safe per-argument transport limit/u); + + expect(f.lifecycleMock.events).not.toContain("delete"); + expect(f.streamSandboxCreateMock).not.toHaveBeenCalled(); + expect(f.registerSandboxMock).not.toHaveBeenCalled(); + }); + + it("rejects an invalid managed profile before deleting a forced destination", async () => { + const consoleError = vi.spyOn(console, "error").mockImplementation(() => {}); + const reference = `ghcr.io/nvidia/nemoclaw/langchain-deepagents-code-sandbox@sha256:${"a".repeat( + 64, + )}`; + f.getSandboxMock.mockImplementation( + s.valueByName({ + alpha: { + name: "alpha", + agent: "langchain-deepagents-code", + imageTag: reference, + openshellDriver: "docker", + provider: "nvidia-nim", + model: "nvidia/model-a", + workload: { + schemaVersion: 1, + kind: "managed-image", + reference, + release: "v0.0.99", + sourceRevision: "b".repeat(40), + sourceCohort: "ghrun-123456-1", + capabilityContractVersion: 1, + startupProfileContractVersion: 1, + encodedProfile: "e30", + startupProfileSha256: + "beab987bef9c00dfc301b490ddb45321517e7d6a6bb3d31d259898b7d46393d8", + credentialProxyReplayRequired: false, + shared: true, + }, + } as never, + beta: { + name: "beta", + agent: "langchain-deepagents-code", + imageTag: "nemoclaw-beta:test", + openshellDriver: "docker", + provider: "nvidia-nim", + model: "nvidia/model-a", + }, + }), + ); + f.parseLiveSandboxNamesMock.mockReturnValue(new Set(["alpha", "beta"])); + f.captureOpenshellMock.mockImplementation((args) => + f.openshellResponses(args, { + "sandbox exec": { status: 0, output: f.dcodeProbeOutput("idle") }, + "sandbox list": { status: 0, output: "alpha Ready\nbeta Ready\n" }, + }), + ); + f.getLatestBackupMock.mockReturnValue({ ...f.latestBackupFixture }); + const { runSandboxSnapshot } = await import("./snapshot"); + + await expect( + runSandboxSnapshot("alpha", { + kind: "restore", + to: "beta", + force: true, + yes: true, + }), + ).rejects.toMatchObject({ exitCode: 1 }); + + expect(consoleError.mock.calls.flat().join("\n")).toContain( + "source profile transport is not canonical and valid", + ); + expect(f.lifecycleMock.events).not.toContain("delete"); + expect(f.streamSandboxCreateMock).not.toHaveBeenCalled(); + expect(f.registerSandboxMock).not.toHaveBeenCalled(); + }); + + it.each([ + [ + "malformed messaging state", + { messaging: { schemaVersion: 2, plan: {} } }, + "current source messaging state is invalid", + ], + [ + "unknown tool disclosure", + { toolDisclosure: "automatic" }, + "current source tool disclosure is invalid", + ], + ])("rejects %s before deleting a forced managed-clone destination", async (_label, currentOverride, expectedError) => { + const consoleError = vi.spyOn(console, "error").mockImplementation(() => {}); + const built = managedOpenClawProfile(); + const reference = `ghcr.io/nvidia/nemoclaw/openclaw-sandbox@sha256:${"a".repeat(64)}`; + f.getSandboxMock.mockImplementation( + s.valueByName({ + alpha: { + name: "alpha", + agent: "openclaw", + dashboardPort: 18_789, + dashboardRemoteBindPrepared: false, + imageTag: reference, + openshellDriver: "docker", + provider: "openai-api", + model: "gpt-5.4", + endpointUrl: null, + preferredInferenceApi: "openai-responses", + compatibleEndpointReasoning: null, + toolDisclosure: "progressive", + webSearchEnabled: false, + webSearchProvider: null, + ...currentOverride, + workload: { + schemaVersion: 1, + kind: "managed-image", + reference, + release: "v0.0.99", + sourceRevision: "b".repeat(40), + sourceCohort: "ghrun-123456-1", + capabilityContractVersion: 1, + startupProfileContractVersion: 1, + encodedProfile: built.encodedProfile, + startupProfileSha256: built.startupProfileSha256, + credentialProxyReplayRequired: false, + shared: true, + }, + } as never, + beta: { + name: "beta", + agent: "openclaw", + dashboardPort: 19_789, + imageTag: "nemoclaw-beta:test", + openshellDriver: "docker", + provider: "openai-api", + model: "gpt-5.4", + }, + }), + ); + f.parseLiveSandboxNamesMock.mockReturnValue(new Set(["alpha", "beta"])); + f.captureOpenshellMock.mockImplementation((args) => + f.openshellResponses(args, { + "forward list": { + status: 0, + output: "alpha 127.0.0.1 18789 23189 running\nbeta 127.0.0.1 19789 24189 running\n", + }, + "sandbox exec": { status: 0, output: f.dcodeProbeOutput("no-runtime") }, + "sandbox list": { status: 0, output: "alpha Ready\nbeta Ready\n" }, + }), + ); + f.getLatestBackupMock.mockReturnValue({ ...f.latestBackupFixture }); + const { runSandboxSnapshot } = await import("./snapshot"); + + await expect( + runSandboxSnapshot("alpha", { + kind: "restore", + to: "beta", + force: true, + yes: true, + }), + ).rejects.toMatchObject({ exitCode: 1 }); + + expect(consoleError.mock.calls.flat().join("\n")).toContain(expectedError); + expect(f.lifecycleMock.events).not.toContain("delete"); + expect(f.streamSandboxCreateMock).not.toHaveBeenCalled(); + expect(f.registerSandboxMock).not.toHaveBeenCalled(); + }); + + it("rejects an incompatible destination provider binding before force-delete", async () => { + const consoleError = vi.spyOn(console, "error").mockImplementation(() => {}); + const built = managedOpenClawProfile(); + const currentMessaging = managedMessagingPlan("openclaw", "alpha", "222222"); + const reference = `ghcr.io/nvidia/nemoclaw/openclaw-sandbox@sha256:${"a".repeat(64)}`; + f.getSandboxMock.mockImplementation( + s.valueByName({ + alpha: { + name: "alpha", + agent: "openclaw", + dashboardPort: 18_789, + dashboardRemoteBindPrepared: false, + imageTag: reference, + openshellDriver: "docker", + provider: "openai-api", + model: "gpt-5.4", + endpointUrl: null, + preferredInferenceApi: "openai-responses", + compatibleEndpointReasoning: null, + toolDisclosure: "progressive", + webSearchEnabled: false, + webSearchProvider: null, + messaging: { schemaVersion: 1, plan: currentMessaging }, + workload: { + schemaVersion: 1, + kind: "managed-image", + reference, + release: "v0.0.99", + sourceRevision: "b".repeat(40), + sourceCohort: "ghrun-123456-1", + capabilityContractVersion: 1, + startupProfileContractVersion: 1, + encodedProfile: built.encodedProfile, + startupProfileSha256: built.startupProfileSha256, + credentialProxyReplayRequired: false, + shared: true, + }, + } as never, + beta: { + name: "beta", + agent: "openclaw", + dashboardPort: 19_789, + imageTag: "nemoclaw-beta:test", + openshellDriver: "docker", + provider: "openai-api", + model: "gpt-5.4", + }, + }), + ); + f.parseLiveSandboxNamesMock.mockReturnValue(new Set(["alpha", "beta"])); + f.captureOpenshellMock.mockImplementation((args) => + f.openshellResponses(args, { + "forward list": { + status: 0, + output: "alpha 127.0.0.1 18789 23189 running\nbeta 127.0.0.1 19789 24189 running\n", + }, + "sandbox exec": { status: 0, output: f.dcodeProbeOutput("no-runtime") }, + "sandbox list": { status: 0, output: "alpha Ready\nbeta Ready\n" }, + }), + ); + f.runOpenshellMock.mockImplementation( + s.commandRouter({ + "provider get beta-telegram-bridge": () => ({ + status: 0, + stdout: providerMetadata("beta-telegram-bridge", "brave", "TELEGRAM_BOT_TOKEN"), + stderr: "", + output: "", + }), + }), + ); + f.getLatestBackupMock.mockReturnValue({ ...f.latestBackupFixture }); + const { runSandboxSnapshot } = await import("./snapshot"); + + await expect( + runSandboxSnapshot("alpha", { + kind: "restore", + to: "beta", + force: true, + yes: true, + }), + ).rejects.toMatchObject({ exitCode: 1 }); + + expect(consoleError.mock.calls.flat().join("\n")).toContain( + "exists with an incompatible type or credential binding", + ); + expect(f.lifecycleMock.events).not.toContain("delete"); + expect(f.streamSandboxCreateMock).not.toHaveBeenCalled(); + expect(f.registerSandboxMock).not.toHaveBeenCalled(); + }); + + it("detaches and replaces an exact forced-destination provider with the explicit clone credential", async () => { + const built = managedOpenClawProfile(); + const currentMessaging = managedMessagingPlan("openclaw", "alpha", "222222"); + const reference = `ghcr.io/nvidia/nemoclaw/openclaw-sandbox@sha256:${"a".repeat(64)}`; + let registeredClone: f.SandboxRecord | null = null; + f.registerSandboxMock.mockImplementation((entry) => { + registeredClone = entry as f.SandboxRecord; + }); + f.getSandboxMock.mockImplementation( + s.valueFactoryByName({ + alpha: () => + ({ + name: "alpha", + agent: "openclaw", + dashboardPort: 18_789, + dashboardRemoteBindPrepared: false, + imageTag: reference, + openshellDriver: "docker", + provider: "openai-api", + model: "gpt-5.4", + endpointUrl: null, + preferredInferenceApi: "openai-responses", + compatibleEndpointReasoning: null, + toolDisclosure: "progressive", + webSearchEnabled: false, + webSearchProvider: null, + messaging: { schemaVersion: 1, plan: currentMessaging }, + workload: { + schemaVersion: 1, + kind: "managed-image", + reference, + release: "v0.0.99", + sourceRevision: "b".repeat(40), + sourceCohort: "ghrun-123456-1", + capabilityContractVersion: 1, + startupProfileContractVersion: 1, + encodedProfile: built.encodedProfile, + startupProfileSha256: built.startupProfileSha256, + credentialProxyReplayRequired: false, + shared: true, + }, + }) as never, + beta: () => + registeredClone ?? { + name: "beta", + agent: "openclaw", + dashboardPort: 19_789, + imageTag: "nemoclaw-beta:test", + openshellDriver: "docker", + provider: "openai-api", + model: "gpt-5.4", + }, + }), + ); + f.parseLiveSandboxNamesMock.mockReturnValue(new Set(["alpha", "beta"])); + f.captureOpenshellMock.mockImplementation((args) => + f.openshellResponses(args, { + "forward list": { + status: 0, + output: "alpha 127.0.0.1 18789 23189 running\nbeta 127.0.0.1 19789 24189 running\n", + }, + "sandbox exec": { status: 0, output: f.dcodeProbeOutput("no-runtime") }, + "sandbox list": { status: 0, output: "alpha Ready\nbeta Ready\n" }, + }), + ); + vi.stubEnv("TELEGRAM_BOT_TOKEN", "new-clone-token"); + const providerHarness = s.createReplacementProviderHarness((options) => { + expect(options?.env).toEqual({ TELEGRAM_BOT_TOKEN: "new-clone-token" }); + }); + const { events } = providerHarness; + f.runOpenshellMock.mockImplementation(providerHarness.runner); + f.streamSandboxCreateMock.mockImplementation(async () => { + events.push("sandbox-create"); + return { + status: 0, + output: "", + sawProgress: false, + forcedReady: false, + }; + }); + f.getLatestBackupMock.mockReturnValue({ ...f.latestBackupFixture }); + const { runSandboxSnapshot } = await import("./snapshot"); + + await runSandboxSnapshot("alpha", { + kind: "restore", + to: "beta", + force: true, + yes: true, + }); + + expect(events.indexOf("detach")).toBeLessThan(events.indexOf("sandbox-delete")); + expect(events.indexOf("sandbox-delete")).toBeLessThan(events.indexOf("provider-delete")); + expect(events.indexOf("provider-delete")).toBeLessThan( + events.indexOf("provider-create:new-clone-token"), + ); + expect(events.indexOf("provider-create:new-clone-token")).toBeLessThan( + events.indexOf("sandbox-create"), + ); + }); + + it("removes a recreated forced-target provider when downstream sandbox create fails", async () => { + const built = managedOpenClawProfile(); + const currentMessaging = managedMessagingPlan("openclaw", "alpha", "222222"); + const reference = `ghcr.io/nvidia/nemoclaw/openclaw-sandbox@sha256:${"a".repeat(64)}`; + const providerHarness = s.createFailedReplacementProviderHarness((options) => { + expect(options?.env).toEqual({ TELEGRAM_BOT_TOKEN: "rotated-clone-token" }); + }); + const { events } = providerHarness; + f.getSandboxMock.mockImplementation( + s.valueByName({ + alpha: { + name: "alpha", + agent: "openclaw", + dashboardPort: 18_789, + dashboardRemoteBindPrepared: false, + imageTag: reference, + openshellDriver: "docker", + provider: "openai-api", + model: "gpt-5.4", + endpointUrl: null, + preferredInferenceApi: "openai-responses", + compatibleEndpointReasoning: null, + toolDisclosure: "progressive", + webSearchEnabled: false, + webSearchProvider: null, + messaging: { schemaVersion: 1, plan: currentMessaging }, + workload: { + schemaVersion: 1, + kind: "managed-image", + reference, + release: "v0.0.99", + sourceRevision: "b".repeat(40), + sourceCohort: "ghrun-123456-1", + capabilityContractVersion: 1, + startupProfileContractVersion: 1, + encodedProfile: built.encodedProfile, + startupProfileSha256: built.startupProfileSha256, + credentialProxyReplayRequired: false, + shared: true, + }, + } as never, + beta: { + name: "beta", + agent: "openclaw", + dashboardPort: 19_789, + imageTag: "nemoclaw-beta:test", + openshellDriver: "docker", + provider: "openai-api", + model: "gpt-5.4", + }, + }), + ); + f.parseLiveSandboxNamesMock.mockReturnValue(new Set(["alpha", "beta"])); + f.captureOpenshellMock.mockImplementation((args) => + f.openshellResponses(args, { + "forward list": { + status: 0, + output: "alpha 127.0.0.1 18789 23189 running\nbeta 127.0.0.1 19789 24189 running\n", + }, + "sandbox exec": { status: 0, output: f.dcodeProbeOutput("no-runtime") }, + "sandbox list": { status: 0, output: "alpha Ready\n" }, + }), + ); + vi.stubEnv("TELEGRAM_BOT_TOKEN", "rotated-clone-token"); + f.runOpenshellMock.mockImplementation(providerHarness.runner); + f.streamSandboxCreateMock.mockImplementation(async () => { + providerHarness.markSandboxCreateFailed(); + return { + status: 1, + output: "synthetic create failure", + sawProgress: false, + forcedReady: false, + }; + }); + f.getLatestBackupMock.mockReturnValue({ ...f.latestBackupFixture }); + const { runSandboxSnapshot } = await import("./snapshot"); + + await expect( + runSandboxSnapshot("alpha", { + kind: "restore", + to: "beta", + force: true, + yes: true, + }), + ).rejects.toMatchObject({ exitCode: 1 }); + + const expectedOrder = [ + "provider-delete:survived", + "replacement-provider-delete", + "provider-create:rotated-clone-token", + "sandbox-create:failed", + "partial-provider-detach:failed", + "partial-sandbox-delete:failed", + "provider-delete:blocked-attached", + "rollback-provider-detach:recovered", + "provider-delete:rollback", + ]; + const observedOrder = expectedOrder.map((event) => events.indexOf(event)); + expect(observedOrder).not.toContain(-1); + expect(observedOrder).toEqual([...observedOrder].sort((left, right) => left - right)); + expect(providerHarness.state()).toEqual({ + destinationProviderExists: false, + partialSandboxExists: true, + }); + expect(f.registerSandboxMock).not.toHaveBeenCalled(); + }); + + it("refuses rollback recovery when provider attachment names an unrelated sandbox", async () => { + const consoleWarn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const rollbackRunner = vi.fn( + s.commandRouter({ + "provider delete beta-telegram-bridge": () => ({ + status: 1, + stdout: "", + stderr: "provider 'beta-telegram-bridge' is attached to sandbox(es): beta, gamma", + }), + }), + ); + const { cleanupManagedCloneProviders } = await import("./snapshot/managed-clone-providers"); + + cleanupManagedCloneProviders(["beta-telegram-bridge"], rollbackRunner, "beta"); + + expect( + rollbackRunner.mock.calls.some( + ([args]) => args[0] === "sandbox" && args[1] === "provider" && args[2] === "detach", + ), + ).toBe(false); + expect(consoleWarn.mock.calls.flat().join("\n")).toContain( + "could not clean up managed clone provider 'beta-telegram-bridge'", + ); + }); + + it("fails before force-delete when Hermes tool gateways need a fresh broker binding", async () => { + const consoleError = vi.spyOn(console, "error").mockImplementation(() => {}); + const built = buildManagedStartupProfile({ + agent: "hermes", + inference: { + routeProvider: "inference", + upstreamProvider: "hermes-provider", + model: "new-model", + routedBaseUrl: "https://inference.local/v1", + upstreamEndpointUrl: null, + api: "openai-completions", + primaryModelRef: null, + compatibility: null, + }, + dashboard: { + agent: "hermes", + mode: "disabled", + url: "http://127.0.0.1:19189", + publicPort: null, + internalPort: null, + tuiEnabled: false, + }, + webSearch: { fetchEnabled: false, provider: "tavily" }, + toolDisclosure: "direct", + hermesToolGateways: [], + messagingPlan: null, + dcodeAutoApprovalMode: null, + observabilityEnabled: null, + environment: {}, + corporateCa: null, + }); + const reference = `ghcr.io/nvidia/nemoclaw/hermes-sandbox@sha256:${"a".repeat(64)}`; + f.getSandboxMock.mockImplementation( + s.valueByName({ + alpha: { + name: "alpha", + agent: "hermes", + imageTag: reference, + openshellDriver: "docker", + provider: "hermes-provider", + model: "new-model", + endpointUrl: null, + preferredInferenceApi: "openai-completions", + toolDisclosure: "direct", + webSearchEnabled: false, + webSearchProvider: null, + hermesToolGateways: ["nous-web"], + hermesDashboardEnabled: false, + messaging: undefined, + workload: { + schemaVersion: 1, + kind: "managed-image", + reference, + release: "v0.0.99", + sourceRevision: "b".repeat(40), + sourceCohort: "ghrun-123456-1", + capabilityContractVersion: 1, + startupProfileContractVersion: 1, + encodedProfile: built.encodedProfile, + startupProfileSha256: built.startupProfileSha256, + credentialProxyReplayRequired: false, + shared: true, + }, + } as never, + beta: { + name: "beta", + agent: "hermes", + imageTag: "nemoclaw-beta:test", + openshellDriver: "docker", + provider: "hermes-provider", + model: "new-model", + }, + }), + ); + f.parseLiveSandboxNamesMock.mockReturnValue(new Set(["alpha", "beta"])); + f.captureOpenshellMock.mockImplementation((args) => + f.openshellResponses(args, { + "sandbox exec": { status: 0, output: f.dcodeProbeOutput("no-runtime") }, + "sandbox list": { status: 0, output: "alpha Ready\nbeta Ready\n" }, + }), + ); + f.getLatestBackupMock.mockReturnValue({ ...f.latestBackupFixture }); + const { runSandboxSnapshot } = await import("./snapshot"); + + await expect( + runSandboxSnapshot("alpha", { + kind: "restore", + to: "beta", + force: true, + yes: true, + }), + ).rejects.toMatchObject({ exitCode: 1 }); + + expect(consoleError.mock.calls.flat().join("\n")).toContain( + "fresh Nous OAuth refresh credential and destination broker binding", + ); + expect(f.lifecycleMock.events).not.toContain("delete"); + expect(f.streamSandboxCreateMock).not.toHaveBeenCalled(); + expect(f.registerSandboxMock).not.toHaveBeenCalled(); + }); + it("restores the latest snapshot into the source sandbox", async () => { const consoleLog = vi.spyOn(console, "log").mockImplementation(() => {}); f.getLatestBackupMock.mockReturnValue({ diff --git a/src/lib/actions/sandbox/snapshot-restore-test-fixture.ts b/src/lib/actions/sandbox/snapshot-restore-test-fixture.ts index 07810b81f0..2d98466294 100644 --- a/src/lib/actions/sandbox/snapshot-restore-test-fixture.ts +++ b/src/lib/actions/sandbox/snapshot-restore-test-fixture.ts @@ -166,6 +166,9 @@ export const getSandboxMock = vi.fn<(name?: string) => SandboxRecord | null>(() export const isGatewayHealthyMock = vi.fn(() => true); export const listBackupsMock = vi.fn<() => Array>>(() => []); export const parseLiveSandboxNamesMock = vi.fn(() => new Set(["alpha"])); +export const resolveManagedSnapshotRuntimeAuthorityMock = vi.fn((driverName: string) => + driverName === "podman" ? { driverName: "podman" } : null, +); export const prepareInitialSandboxCreatePolicyMock = vi.fn( ( policyPath: string, @@ -177,7 +180,7 @@ export const prepareInitialSandboxCreatePolicyMock = vi.fn( export const registerSandboxMock = vi.fn(); export const updateSandboxMock = vi.fn(); export const restoreSandboxStateMock = vi.fn(); -export const runOpenshellMock = vi.fn((args: string[]) => { +export const runOpenshellMock = vi.fn((args: string[], _opts?: Record) => { args[0] === "sandbox" && args[1] === "delete" && lifecycleMock.events.push("delete"); return { status: 0, output: "" }; }); @@ -187,6 +190,26 @@ export const streamSandboxCreateMock = vi.fn(as sawProgress: false, forcedReady: false, })); +function managedStartupPatchFixture() { + return { + maybeApplyDuringCreate: vi.fn(), + createFailureMessage: vi.fn(() => null), + exitOnPatchError: vi.fn(), + rollbackManagedStartupAfterCreateFailure: vi.fn(), + ensureApplied: vi.fn(), + revalidateBeforeMutation: vi.fn(), + waitForSupervisorReconnectIfNeeded: vi.fn(), + commitAfterReady: vi.fn(), + selectedMode: vi.fn(() => null), + printReadinessFailureIfEnabled: vi.fn(), + verifyGpuOrExit: vi.fn((verifyDirectSandboxGpu: (sandboxName: string) => unknown) => + verifyDirectSandboxGpu("beta"), + ), + }; +} +export const createDockerGpuSandboxCreatePatchMock = vi.fn((_options?: unknown) => + managedStartupPatchFixture(), +); export const latestBackupFixture = { timestamp: "2026-06-15T00:00:00.000Z", backupPath: "/tmp/backup-alpha", @@ -196,7 +219,13 @@ export { lifecycleMock, shieldsMock }; vi.mock("../../adapters/docker", () => ({ dockerCapture: vi.fn(() => ""), + dockerForceRm: vi.fn(() => ({ status: 0, stdout: "", stderr: "" })), dockerInspect: dockerInspectMock, + dockerRunDetached: vi.fn(() => ({ status: 0, stdout: "", stderr: "" })), +})); + +vi.mock("../../onboard/docker-gpu-sandbox-create", () => ({ + createDockerGpuSandboxCreatePatch: createDockerGpuSandboxCreatePatchMock, })); vi.mock("../../agent/defs", () => ({ @@ -210,7 +239,9 @@ vi.mock("../../adapters/openshell/runtime", () => ({ })); vi.mock("../../credentials/store", () => ({ + getCredential: vi.fn(), prompt: vi.fn(), + saveCredential: vi.fn(), })); vi.mock("../../domain/sandbox/destroy", () => ({ @@ -267,6 +298,14 @@ vi.mock("../../sandbox/create-stream", () => ({ streamSandboxCreate: streamSandboxCreateMock, })); +vi.mock("./snapshot/runtime-authority", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + resolveManagedSnapshotRuntimeAuthority: resolveManagedSnapshotRuntimeAuthorityMock, + }; +}); + vi.mock("../../state/gateway", () => ({ isGatewayHealthy: isGatewayHealthyMock, isSandboxReady: vi.fn((output: string, sandboxName: string) => @@ -354,13 +393,22 @@ export function resetSnapshotRestoreMocks(): void { failedDirs: [], failedFiles: [], }); + runOpenshellMock.mockImplementation((args) => { + args[0] === "sandbox" && args[1] === "delete" && lifecycleMock.events.push("delete"); + return { status: 0, output: "" }; + }); streamSandboxCreateMock.mockImplementation(async () => ({ status: 0, output: "", sawProgress: false, forcedReady: false, })); + createDockerGpuSandboxCreatePatchMock.mockReset(); + createDockerGpuSandboxCreatePatchMock.mockImplementation(() => managedStartupPatchFixture()); parseLiveSandboxNamesMock.mockReturnValue(new Set(["alpha"])); + resolveManagedSnapshotRuntimeAuthorityMock.mockImplementation((driverName: string) => + driverName === "podman" ? { driverName: "podman" } : null, + ); } export function cleanupSnapshotRestoreMocks(): void { diff --git a/src/lib/actions/sandbox/snapshot.test.ts b/src/lib/actions/sandbox/snapshot.test.ts index 97b37d0fae..92eb4b6062 100644 --- a/src/lib/actions/sandbox/snapshot.test.ts +++ b/src/lib/actions/sandbox/snapshot.test.ts @@ -348,7 +348,7 @@ describe("runSandboxSnapshot", () => { expect(consoleError.mock.calls.flat().join("\n")).toContain( "Cannot verify shields state. Refusing to create snapshot.", ); - }); + }, 10_000); it("creates a named snapshot after gateway, liveness, and shields checks pass", async () => { const consoleLog = vi.spyOn(console, "log").mockImplementation(() => {}); diff --git a/src/lib/actions/sandbox/snapshot.ts b/src/lib/actions/sandbox/snapshot.ts index 447fcd9ba1..b4d376d3db 100644 --- a/src/lib/actions/sandbox/snapshot.ts +++ b/src/lib/actions/sandbox/snapshot.ts @@ -3,38 +3,17 @@ import fs from "node:fs"; import path from "node:path"; -import { dockerCapture } from "../../adapters/docker"; -import { - captureOpenshell, - getOpenshellBinary, - runOpenshell, -} from "../../adapters/openshell/runtime"; import { OPENSHELL_PROBE_TIMEOUT_MS } from "../../adapters/openshell/timeouts"; import { CLI_NAME } from "../../cli/branding"; import { prompt as askPrompt } from "../../credentials/store"; import { formatFailedBackupItems } from "../../domain/backup-failure"; import { getSandboxDeleteOutcome } from "../../domain/sandbox/destroy"; -import { - HERMES_DASHBOARD_ENABLE_ENV, - HERMES_DASHBOARD_INTERNAL_PORT_ENV, - HERMES_DASHBOARD_PORT_ENV, - HERMES_DASHBOARD_TUI_ENV, -} from "../../hermes-dashboard"; import { checkGatewayRouteCompatibility, formatGatewayRouteConflict, } from "../../inference/gateway-route-compatibility"; import { withGatewayRouteMutationLock } from "../../inference/gateway-route-mutation-lock"; import * as nim from "../../inference/nim"; -import { listMessagingProviderSuffixes } from "../../messaging/channels"; -import { - findAvailableDashboardPort, - getRegistryOccupiedDashboardPorts, - withDashboardPortReservationLock, -} from "../../onboard/dashboard-port"; -import { isValidForwardPort } from "../../onboard/dashboard-runtime"; -import { resolveSandboxGatewayName } from "../../onboard/gateway-binding"; -import { resolveHermesDashboardOnboardState } from "../../onboard/hermes-dashboard"; import { isDcodeAgent, OBSERVABILITY_OTLP_LOCAL_POLICY_PRESET, @@ -44,7 +23,6 @@ import { normalizePolicyTierName } from "../../onboard/policy-tier-suppression"; import * as policies from "../../policy"; import { ROOT, run, validateName } from "../../runner"; import { parseLiveSandboxNames } from "../../runtime-recovery"; -import { streamSandboxCreate } from "../../sandbox/create-stream"; import * as shields from "../../shields"; import { withTimerBoundShieldsMutationLock } from "../../shields/timer-bound-lock"; import { readTimerMarker } from "../../shields/timer-control"; @@ -72,8 +50,42 @@ import { selectSandboxGatewayIfRegistered, usesGatewayMetadataProbe, } from "./sandbox-gateway-routing"; -import { formatSnapshotBaselineExclusionSummary } from "./snapshot-baseline-exclusion-summary"; -import { printHermesGatewayRestoreHint } from "./snapshot-hermes-gateway-hint"; +import { + assertSandboxCreateArgvWithinTransportLimit, + captureOpenshell, + createDockerGpuSandboxCreatePatch, + createManagedSnapshotRuntimePatch, + createManagedStartupRootApplyRequest, + dockerCapture, + findAvailableDashboardPort, + formatSnapshotBaselineExclusionSummary, + getOpenshellBinary, + getRegistryOccupiedDashboardPorts, + HERMES_DASHBOARD_ENABLE_ENV, + HERMES_DASHBOARD_INTERNAL_PORT_ENV, + HERMES_DASHBOARD_PORT_ENV, + HERMES_DASHBOARD_TUI_ENV, + isValidForwardPort, + MANAGED_STARTUP_CA_ENV, + MANAGED_STARTUP_HOLD_EXECUTABLE, + MANAGED_STARTUP_PROFILE_ENV, + type ManagedStartupAgent, + type ManagedStartupProfile, + type ManagedStartupRootApplyRequest, + type PreparedManagedCloneProvider, + printHermesGatewayRestoreHint, + resolveHermesDashboardOnboardState, + resolveManagedSnapshotRuntimeAuthority, + resolveSandboxGatewayName, + runAuthorizedManagedSnapshotDestinationDelete, + runOpenshell, + runSandboxProviderPreDeleteCleanup, + SANDBOX_PROVIDER_SUFFIXES, + type SandboxCreateRuntimePatch, + sleepSeconds, + streamSandboxCreate, + withDashboardPortReservationLock, +} from "./snapshot/dependencies"; const useColor = !process.env.NO_COLOR && !!process.stdout.isTTY; const trueColor = @@ -275,6 +287,153 @@ function resolveCloneDashboardEnvArgs( return envArgs; } +interface PreparedManagedSnapshotClone { + readonly envArgs: readonly string[]; + readonly rootApplyRequest: ManagedStartupRootApplyRequest; + readonly workload: Extract, { kind: "managed-image" }>; + readonly messaging: SandboxEntry["messaging"]; + readonly registryFields: Partial; + readonly credentialProviders: readonly PreparedManagedCloneProvider[]; +} + +type SnapshotMessagingPlan = NonNullable["plan"]; + +function managedCloneRegistryFields( + profile: ManagedStartupProfile, + source: SandboxEntry, +): Partial { + const webSearch = + profile.agentConfig.agent === "langchain-deepagents-code" + ? null + : profile.agentConfig.webSearch; + const hermesDashboard = profile.dashboard.agent === "hermes" ? profile.dashboard : null; + const dcodeConfig = + profile.agentConfig.agent === "langchain-deepagents-code" ? profile.agentConfig : null; + return { + provider: profile.inference.upstreamProvider, + model: profile.inference.model, + endpointUrl: source.endpointUrl ?? null, + endpointSource: source.endpointSource ?? null, + credentialEnv: source.credentialEnv ?? null, + preferredInferenceApi: profile.inference.api, + compatibleEndpointReasoning: + profile.agent === "openclaw" && profile.inference.upstreamProvider === "compatible-endpoint" + ? profile.tuning.reasoning === true + ? "true" + : "false" + : null, + compatibleEndpointReasoningEffort: + profile.agent === "openclaw" && + profile.inference.upstreamProvider === "compatible-endpoint" && + profile.inference.api === "openai-completions" && + profile.tuning.reasoningEffort !== "default" + ? profile.tuning.reasoningEffort + : null, + toolDisclosure: profile.tools.disclosure, + webSearchEnabled: webSearch?.enabled === true, + webSearchProvider: webSearch?.enabled === true ? webSearch.provider : null, + observabilityEnabled: dcodeConfig?.observabilityEnabled === true, + ...(dcodeConfig ? { dcodeAutoApprovalMode: dcodeConfig.autoApprovalMode } : {}), + hermesToolGateways: + profile.agent === "hermes" && profile.tools.enabledGateways.length > 0 + ? [...profile.tools.enabledGateways] + : undefined, + hermesDashboardEnabled: hermesDashboard?.mode === "loopback-forwarded" ? true : undefined, + hermesDashboardPort: + hermesDashboard?.mode === "loopback-forwarded" ? hermesDashboard.publicPort : undefined, + hermesDashboardInternalPort: + hermesDashboard?.mode === "loopback-forwarded" ? hermesDashboard.internalPort : undefined, + hermesDashboardTui: + hermesDashboard?.mode === "loopback-forwarded" && hermesDashboard.tuiEnabled + ? true + : undefined, + dashboardRemoteBindPrepared: + profile.dashboard.agent === "openclaw" && profile.dashboard.bindAddress === "0.0.0.0", + }; +} + +async function prepareManagedSnapshotClone( + sourceSandboxName: string, + destinationSandboxName: string, + source: SandboxEntry, + fromImage: string, + destinationDashboardPort: number | null, + destinationWillBeReplaced: boolean, +): Promise { + const workload = source.workload; + if (workload?.kind !== "managed-image") return null; + if (workload.reference !== fromImage || source.imageTag !== workload.reference) { + throw new Error("managed workload receipt does not match the source sandbox image reference"); + } + const { MANAGED_STARTUP_AGENTS, MANAGED_STARTUP_PROFILE_SCHEMA_VERSION } = await import( + "../../onboard/managed-startup/profile" + ); + if ( + typeof source.agent !== "string" || + !(MANAGED_STARTUP_AGENTS as readonly string[]).includes(source.agent) + ) { + throw new Error("managed workload receipt has no exact shipped-agent identity"); + } + if (workload.startupProfileContractVersion !== MANAGED_STARTUP_PROFILE_SCHEMA_VERSION) { + throw new Error( + `managed workload receipt uses unsupported startup profile contract ${String( + workload.startupProfileContractVersion, + )}`, + ); + } + const { rebindManagedStartupProfileForClone } = await import( + "../../onboard/managed-startup/clone-rebinder" + ); + const rebound = rebindManagedStartupProfileForClone({ + sourceSandboxName, + destinationSandboxName, + expectedAgent: source.agent as ManagedStartupAgent, + destinationDashboardPort, + encodedProfile: workload.encodedProfile, + startupProfileSha256: workload.startupProfileSha256, + ...(workload.corporateCaB64 === undefined ? {} : { corporateCaB64: workload.corporateCaB64 }), + currentSource: source, + }); + const targetWorkload = { + ...workload, + encodedProfile: rebound.encodedProfile, + startupProfileSha256: rebound.startupProfileSha256, + ...(rebound.corporateCaB64 === undefined ? {} : { corporateCaB64: rebound.corporateCaB64 }), + } as const; + const credentialProxyEnvArgs = workload.credentialProxyReplayRequired + ? (await import("../../onboard/host-proxy-env")).credentialHostProxyReplayEnvArgs(process.env) + : []; + const messaging = + rebound.profile.messaging.plan === null + ? undefined + : { + schemaVersion: 1 as const, + plan: rebound.profile.messaging.plan as unknown as NonNullable< + SandboxEntry["messaging"] + >["plan"], + }; + const { prepareManagedCloneProviders } = await import("./snapshot/managed-clone-providers"); + return { + envArgs: credentialProxyEnvArgs, + rootApplyRequest: createManagedStartupRootApplyRequest({ + agent: rebound.profile.agent, + encodedProfile: rebound.encodedProfile, + ...(rebound.corporateCaB64 === undefined ? {} : { corporateCaB64: rebound.corporateCaB64 }), + }), + workload: targetWorkload, + registryFields: managedCloneRegistryFields(rebound.profile, source), + messaging, + credentialProviders: prepareManagedCloneProviders({ + profile: rebound.profile, + messagingPlan: (messaging?.plan ?? null) as SnapshotMessagingPlan | null, + destinationSandboxName, + destinationWillBeReplaced, + root: ROOT, + runOpenshell, + }), + }; +} + async function prepareSnapshotClonePolicy(srcEntry: SandboxEntry): Promise<{ policyPath: string; cleanup?: () => boolean; @@ -307,26 +466,48 @@ async function prepareSnapshotClonePolicy(srcEntry: SandboxEntry): Promise<{ // Used by `snapshot restore --to ` when dst does not exist yet: reuses // the source's baked image so the user does not have to re-run onboarding. // Returns true on success; on failure, logs and throws SnapshotCommandError. -async function autoCreateSandboxFromSource( - srcName: string, +interface PreparedSnapshotCloneLaunch { + readonly command: string; + readonly commandArgs: readonly string[]; + readonly createEnv: NodeJS.ProcessEnv; + readonly sourceEntry: SandboxEntry | { name: string }; + readonly sourceObservabilityEnabled: boolean; + readonly managedClone: PreparedManagedSnapshotClone | null; + readonly destinationDashboardPort: number | null; + readonly startupCommand: readonly string[]; +} + +function prepareSnapshotCloneLaunch( dstName: string, srcEntry: SandboxEntry | { name: string }, fromImage: string, createPolicyPath: string, dstDashboardPort: number | null, dashboardEnvArgs: readonly string[], -): Promise { + managedClone: PreparedManagedSnapshotClone | null, +): PreparedSnapshotCloneLaunch { const openshellBin = getOpenshellBinary(); const sourceObservabilityEnabled = (srcEntry as { observabilityEnabled?: boolean }).observabilityEnabled === true; const startupCommand = [ "env", - `NEMOCLAW_OBSERVABILITY=${sourceObservabilityEnabled ? "1" : "0"}`, - ...dashboardEnvArgs, - "nemoclaw-start", + ...(managedClone + ? managedClone.envArgs + : [`NEMOCLAW_OBSERVABILITY=${sourceObservabilityEnabled ? "1" : "0"}`, ...dashboardEnvArgs]), + ...(managedClone + ? [ + MANAGED_STARTUP_HOLD_EXECUTABLE, + "--agent", + managedClone.rootApplyRequest.agent, + "--profile-fingerprint", + managedClone.rootApplyRequest.profileFingerprint, + ] + : ["nemoclaw-start"]), ]; const createEnv = { ...process.env }; delete createEnv.NEMOCLAW_OBSERVABILITY; + delete createEnv[MANAGED_STARTUP_PROFILE_ENV]; + delete createEnv[MANAGED_STARTUP_CA_ENV]; const command = openshellBin; const commandArgs = [ @@ -339,10 +520,95 @@ async function autoCreateSandboxFromSource( "--policy", createPolicyPath, "--auto-providers", + ...(managedClone + ? managedClone.credentialProviders.flatMap((provider) => [ + "--provider", + provider.providerName, + ]) + : []), "--", ...startupCommand, ]; + assertSandboxCreateArgvWithinTransportLimit([command, ...commandArgs]); + return { + command, + commandArgs, + createEnv, + sourceEntry: srcEntry, + sourceObservabilityEnabled, + managedClone, + destinationDashboardPort: dstDashboardPort, + startupCommand, + }; +} + +function createManagedSnapshotStartupPatch( + dstName: string, + launch: PreparedSnapshotCloneLaunch, +): SandboxCreateRuntimePatch | null { + const managedClone = launch.managedClone; + if (!managedClone) return null; + return createManagedSnapshotRuntimePatch( + { + destinationSandboxName: dstName, + sourceEntry: launch.sourceEntry as SandboxEntry, + lifecycle: { + managedStartupRootApplyRequest: managedClone.rootApplyRequest, + openshellSandboxCommand: launch.startupCommand, + sandboxName: dstName, + timeoutSecs: Math.ceil(OPENSHELL_PROBE_TIMEOUT_MS / 1000), + deps: { + runOpenshell, + runCaptureOpenshell: (args, options) => + captureOpenshell(args, options as { ignoreError?: boolean }).output ?? "", + sleep: sleepSeconds, + }, + }, + createDockerPatch: (lifecycle) => + createDockerGpuSandboxCreatePatch({ + route: "native", + managedStartupRootApplyRequest: lifecycle.managedStartupRootApplyRequest, + persistStartupCommand: lifecycle.persistStartupCommand, + sandboxName: lifecycle.sandboxName, + openshellSandboxCommand: lifecycle.openshellSandboxCommand, + requiredUlimits: lifecycle.requiredUlimits, + timeoutSecs: lifecycle.timeoutSecs, + backend: "generic", + deps: { + dockerCapture, + runOpenshell, + runCaptureOpenshell: lifecycle.deps.runCaptureOpenshell, + sleep: sleepSeconds, + }, + overrides: { + onPatchFailureExit: (_sandboxName, error) => { + throw error instanceof Error ? error : new Error(String(error)); + }, + }, + }), + }, + { + resolveRuntimeAuthority: resolveManagedSnapshotRuntimeAuthority, + }, + ); +} +async function autoCreateSandboxFromSource( + srcName: string, + dstName: string, + fromImage: string, + launch: PreparedSnapshotCloneLaunch, + managedStartupPatch: SandboxCreateRuntimePatch | null, +): Promise { + const { + command, + commandArgs, + createEnv, + sourceEntry, + sourceObservabilityEnabled, + managedClone, + destinationDashboardPort, + } = launch; console.log(` '${dstName}' does not exist. Creating from '${srcName}' image (${fromImage})...`); const createResult = await streamSandboxCreate(command, commandArgs, createEnv, { @@ -356,28 +622,40 @@ async function autoCreateSandboxFromSource( if (list.status !== 0) return false; return isSandboxReady(list.output || "", dstName); }, + ...(managedStartupPatch + ? { + onPoll: managedStartupPatch.maybeApplyDuringCreate, + failureCheck: managedStartupPatch.createFailureMessage, + } + : {}), }); + managedStartupPatch?.exitOnPatchError(); if (createResult.status !== 0 && !createResult.forcedReady) { + managedStartupPatch?.rollbackManagedStartupAfterCreateFailure(); console.error(` Failed to create sandbox '${dstName}' (exit ${createResult.status}).`); const tail = (createResult.output || "").slice(-600); if (tail) console.error(tail); snapshotExit(1); } + managedStartupPatch?.ensureApplied(); + managedStartupPatch?.waitForSupervisorReconnectIfNeeded(); // Double-check Ready after stream exit. const verify = captureOpenshell(["sandbox", "list"], { ignoreError: true }); if (verify.status !== 0 || !isSandboxReady(verify.output || "", dstName)) { + managedStartupPatch?.rollbackManagedStartupAfterCreateFailure(); console.error(` Sandbox '${dstName}' did not reach Ready state after create.`); snapshotExit(1); } + managedStartupPatch?.commitAfterReady(); // DNS proxy is only meaningful for the kubernetes driver (matches onboard.ts). const dnsScript = path.join(ROOT, "scripts", "setup-dns-proxy.sh"); - const srcDriver = (srcEntry as { openshellDriver?: string | null }).openshellDriver; + const srcDriver = (sourceEntry as { openshellDriver?: string | null }).openshellDriver; if (srcDriver === "kubernetes" && fs.existsSync(dnsScript)) { const srcGatewayName = resolveSandboxGatewayName( - srcEntry as { gatewayName?: string | null; gatewayPort?: number | null }, + sourceEntry as { gatewayName?: string | null; gatewayPort?: number | null }, ); run(["bash", dnsScript, srcGatewayName, dstName], { ignoreError: true }); } @@ -386,11 +664,14 @@ async function autoCreateSandboxFromSource( // Policies are cleared here — the caller replays them from the snapshot // manifest after the restore succeeds and writes them back into this entry. registry.registerSandbox({ - ...srcEntry, + ...sourceEntry, + ...(managedClone ? managedClone.registryFields : {}), name: dstName, createdAt: new Date().toISOString(), policies: [], observabilityEnabled: sourceObservabilityEnabled, + workload: managedClone ? managedClone.workload : (sourceEntry as SandboxEntry).workload, + messaging: managedClone ? managedClone.messaging : (sourceEntry as SandboxEntry).messaging, // dst has its own lifecycle; don't inherit src's local NIM container // reference, or destroying dst would stop src's NIM. nimContainer: null, @@ -398,14 +679,14 @@ async function autoCreateSandboxFromSource( // so clear src's proof rather than inheriting it — otherwise dst could show // `Sandbox GPU: enabled (CUDA verified)` based on another sandbox's run (#4231). sandboxGpuProof: null, - dashboardPort: dstDashboardPort, + dashboardPort: destinationDashboardPort, // The shared image keeps Hermes' image-baked internal listener port, but // the public WebUI port is a per-sandbox host resource and must follow the // clone's newly allocated dashboard port so rebuild validation converges. hermesDashboardPort: - (srcEntry as SandboxEntry).hermesDashboardEnabled === true - ? dstDashboardPort - : (srcEntry as SandboxEntry).hermesDashboardPort, + (sourceEntry as SandboxEntry).hermesDashboardEnabled === true + ? destinationDashboardPort + : (sourceEntry as SandboxEntry).hermesDashboardPort, }); console.log(` ${G}\u2713${R} Sandbox '${dstName}' created`); @@ -439,6 +720,19 @@ function deleteSandboxForRestore(name: string): void { allowLegacyHermesProtocol: true, }); } + const providerCleanup = runSandboxProviderPreDeleteCleanup(name, { + runOpenshell, + // Snapshot restore treats detach failures as fatal below and emits only + // generated provider names, so never echo untrusted gateway diagnostics. + warn: () => {}, + }); + if (providerCleanup.failures.length > 0) { + console.error( + ` Failed to detach destination provider(s) before deleting '${name}': ` + + providerCleanup.failures.map((failure) => failure.name).join(", "), + ); + snapshotExit(1); + } const deleteResult = runOpenshell(["sandbox", "delete", name], { ignoreError: true, stdio: ["ignore", "pipe", "pipe"], @@ -466,8 +760,8 @@ function deleteSandboxForRestore(name: string): void { } catch { // PID dir may not exist \u2014 ignore. } - for (const suffix of listMessagingProviderSuffixes()) { - runOpenshell(["provider", "delete", `${name}${suffix}`], { + for (const suffix of SANDBOX_PROVIDER_SUFFIXES) { + runOpenshell(["provider", "delete", `${name}-${suffix}`], { ignoreError: true, stdio: ["ignore", "ignore", "ignore"], }); @@ -478,6 +772,30 @@ function deleteSandboxForRestore(name: string): void { console.log(` ${G}\u2713${R} '${name}' deleted`); } +function cleanupFailedSnapshotCloneTarget(name: string): void { + const providerCleanup = runSandboxProviderPreDeleteCleanup(name, { + runOpenshell, + tolerateMissingSandbox: true, + warn: () => {}, + }); + if (providerCleanup.failures.length > 0) { + console.warn( + ` Warning: could not detach all providers from failed clone '${name}': ` + + providerCleanup.failures.map((failure) => failure.name).join(", "), + ); + } + const deleteResult = runOpenshell(["sandbox", "delete", name], { + ignoreError: true, + stdio: ["ignore", "pipe", "pipe"], + }); + const { alreadyGone } = getSandboxDeleteOutcome(deleteResult); + if (deleteResult.status !== 0 && !alreadyGone) { + console.warn( + ` Warning: could not remove incomplete clone '${name}' before credential-provider rollback.`, + ); + } +} + function listLiveSandboxesOnSandboxGateway(sandboxName: string): Set | null { if (!selectSandboxGatewayIfRegistered(sandboxName)) return null; if (!probeGatewayRunning(sandboxName)) return null; @@ -1081,27 +1399,104 @@ async function runSnapshotRestoreUnlocked( // validation the image and gateway-route checks above already do (#3756). const dstDashboardPort = allocateCloneDashboardPort(targetSandbox, lockedSourceEntry); const dashboardEnvArgs = resolveCloneDashboardEnvArgs(lockedSourceEntry, dstDashboardPort); - const clonePolicy = await prepareSnapshotClonePolicy(lockedSourceEntry); + let managedClone: PreparedManagedSnapshotClone | null; try { - if (targetExists) { - if (targetEntry) { - verifyRestoreDestinationOnOwnGateway(targetSandbox); - } - deleteSandboxForRestore(targetSandbox); - requireLiveSandboxesOnSandboxGateway( - sandboxName, - " Failed to re-select source sandbox gateway after deleting destination.", - ); - } - await autoCreateSandboxFromSource( + managedClone = await prepareManagedSnapshotClone( sandboxName, + targetSandbox, + lockedSourceEntry, + lockedFromImage, + dstDashboardPort, + targetExists, + ); + } catch (error) { + console.error( + ` Cannot prepare managed clone '${targetSandbox}': ${ + error instanceof Error ? error.message : String(error) + }.`, + ); + console.error( + ` Destination '${targetSandbox}' was not changed. Re-onboard the source before retrying this restore.`, + ); + snapshotExit(1); + } + const clonePolicy = await prepareSnapshotClonePolicy(lockedSourceEntry); + try { + // Render and bound the complete argv before a forced restore can delete + // the destination. Managed clone profile/CA transports therefore use + // the same exec-safe argument gate as ordinary managed onboarding. + const cloneLaunch = prepareSnapshotCloneLaunch( targetSandbox, lockedSourceEntry, lockedFromImage, clonePolicy.policyPath, dstDashboardPort, dashboardEnvArgs, + managedClone, ); + let managedStartupPatch: SandboxCreateRuntimePatch | null; + try { + // Resolve driver authority before provider mutation or a forced + // destination delete. Native runtimes fail closed here while the + // original destination is still intact. + managedStartupPatch = createManagedSnapshotStartupPatch(targetSandbox, cloneLaunch); + } catch (error) { + console.error( + ` Cannot prepare ${lockedSourceEntry.openshellDriver || "legacy"} runtime clone authority: ${ + error instanceof Error ? error.message : String(error) + }.`, + ); + console.error(` Destination '${targetSandbox}' was not changed.`); + snapshotExit(1); + } + // Keep the managed credential-provider graph lazy. Legacy/custom + // snapshot operations must retain their narrow runtime and test seam. + const managedProviderLifecycle = managedClone + ? await import("./snapshot/managed-clone-providers") + : null; + let mutatedCredentialProviders: string[] = []; + let cloneCreateAttempted = false; + try { + if (targetExists) { + if (targetEntry) { + verifyRestoreDestinationOnOwnGateway(targetSandbox); + } + runAuthorizedManagedSnapshotDestinationDelete(managedStartupPatch, () => + deleteSandboxForRestore(targetSandbox), + ); + requireLiveSandboxesOnSandboxGateway( + sandboxName, + " Failed to re-select source sandbox gateway after deleting destination.", + ); + } + mutatedCredentialProviders = + managedProviderLifecycle?.provisionManagedCloneProviders( + managedClone?.credentialProviders ?? [], + { + runOpenshell, + ...(targetExists ? { rollbackSandboxName: targetSandbox } : {}), + }, + ) ?? []; + cloneCreateAttempted = true; + await autoCreateSandboxFromSource( + sandboxName, + targetSandbox, + lockedFromImage, + cloneLaunch, + managedStartupPatch, + ); + // The mutated providers are now attached to the successfully created + // destination and owned by its rebound messaging plan. + mutatedCredentialProviders = []; + } catch (error) { + if (cloneCreateAttempted) cleanupFailedSnapshotCloneTarget(targetSandbox); + managedProviderLifecycle?.cleanupManagedCloneProviders( + mutatedCredentialProviders, + runOpenshell, + targetSandbox, + ); + throw error; + } } finally { clonePolicy.cleanup?.(); } diff --git a/src/lib/actions/sandbox/snapshot/dependencies.test.ts b/src/lib/actions/sandbox/snapshot/dependencies.test.ts new file mode 100644 index 0000000000..1f24211921 --- /dev/null +++ b/src/lib/actions/sandbox/snapshot/dependencies.test.ts @@ -0,0 +1,17 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { sleepSeconds } from "./dependencies"; + +describe("snapshot runtime dependencies", () => { + afterEach(() => vi.restoreAllMocks()); + + it("provides the synchronous sleep contract required by runtime lifecycle polling", () => { + const wait = vi.spyOn(Atomics, "wait").mockReturnValue("timed-out"); + + expect(sleepSeconds(0.25)).toBeUndefined(); + expect(wait).toHaveBeenCalledExactlyOnceWith(expect.any(Int32Array), 0, 0, 250); + }); +}); diff --git a/src/lib/actions/sandbox/snapshot/dependencies.ts b/src/lib/actions/sandbox/snapshot/dependencies.ts new file mode 100644 index 0000000000..c530d51d6a --- /dev/null +++ b/src/lib/actions/sandbox/snapshot/dependencies.ts @@ -0,0 +1,115 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import { + captureOpenshell, + getOpenshellBinary, + runOpenshell, +} from "../../../adapters/openshell/runtime"; +import { + resolveGatewayPortFromName, + resolveManagedGatewayStateDirectory, + resolveSandboxGatewayName, +} from "../../../onboard/gateway-binding"; +import { + createPodmanManagedSnapshotRuntimeAuthority, + currentManagedSnapshotRuntimeAuthorityAdapters, + resolveManagedSnapshotRuntimeAuthority as resolveManagedSnapshotRuntimeAuthorityWithAdapters, +} from "./runtime-authority"; +import type { ManagedSnapshotRuntimePatchContext } from "./runtime-patch"; + +/** + * Narrow dependency facade for the large snapshot lifecycle coordinator. + * + * Snapshot restore spans runtime, dashboard, profile, and launch boundaries. + * Keep those leaf imports here so snapshot.ts depends on one explicit + * integration seam instead of growing direct fan-out for every clone feature. + */ +export { dockerCapture } from "../../../adapters/docker"; +export { + HERMES_DASHBOARD_ENABLE_ENV, + HERMES_DASHBOARD_INTERNAL_PORT_ENV, + HERMES_DASHBOARD_PORT_ENV, + HERMES_DASHBOARD_TUI_ENV, +} from "../../../hermes-dashboard"; +export { + findAvailableDashboardPort, + getRegistryOccupiedDashboardPorts, + withDashboardPortReservationLock, +} from "../../../onboard/dashboard-port"; +export { isValidForwardPort } from "../../../onboard/dashboard-runtime"; +export { createDockerGpuSandboxCreatePatch } from "../../../onboard/docker-gpu-sandbox-create"; +export { resolveHermesDashboardOnboardState } from "../../../onboard/hermes-dashboard"; +export { MANAGED_STARTUP_HOLD_EXECUTABLE } from "../../../onboard/managed-startup/hold"; +export type { + ManagedStartupAgent, + ManagedStartupProfile, +} from "../../../onboard/managed-startup/profile"; +export { + createManagedStartupRootApplyRequest, + type ManagedStartupRootApplyRequest, +} from "../../../onboard/managed-startup/root-apply"; +export { + MANAGED_STARTUP_CA_ENV, + MANAGED_STARTUP_PROFILE_ENV, +} from "../../../onboard/managed-startup/transport"; +export { assertSandboxCreateArgvWithinTransportLimit } from "../../../onboard/sandbox-create/transport"; +export type { SandboxCreateRuntimePatch } from "../../../onboard/sandbox-create-runtime/types"; +export { + runSandboxProviderPreDeleteCleanup, + SANDBOX_PROVIDER_SUFFIXES, +} from "../../../onboard/sandbox-provider-cleanup"; +export { streamSandboxCreate } from "../../../sandbox/create-stream"; +export { formatSnapshotBaselineExclusionSummary } from "../snapshot-baseline-exclusion-summary"; +export { printHermesGatewayRestoreHint } from "../snapshot-hermes-gateway-hint"; +export type { PreparedManagedCloneProvider } from "./managed-clone-providers"; +export { + createManagedSnapshotRuntimePatch, + runAuthorizedManagedSnapshotDestinationDelete, +} from "./runtime-patch"; +export { captureOpenshell, getOpenshellBinary, resolveSandboxGatewayName, runOpenshell }; + +const SNAPSHOT_SLEEP_BUFFER = new Int32Array(new SharedArrayBuffer(4)); + +export function sleepSeconds(seconds: number): void { + Atomics.wait(SNAPSHOT_SLEEP_BUFFER, 0, 0, Math.max(0, seconds) * 1000); +} + +function runSnapshotHostCapture(args: string[], options: { ignoreError?: boolean } = {}): string { + const [file, ...fileArgs] = args; + if (!file) throw new Error("Snapshot runtime capture requires a command"); + const result = spawnSync(file, fileArgs, { + encoding: "utf8", + env: process.env, + shell: false, + stdio: ["ignore", "pipe", "pipe"], + }); + if (!result.error && result.status === 0) return (result.stdout ?? "").trim(); + if (options.ignoreError) return ""; + if (result.error) throw result.error; + throw new Error(`Snapshot runtime command failed with status ${String(result.status)}`); +} + +const MANAGED_SNAPSHOT_RUNTIME_AUTHORITY_ADAPTERS = currentManagedSnapshotRuntimeAuthorityAdapters( + (context) => + createPodmanManagedSnapshotRuntimeAuthority(context, { + captureOpenshell, + getOpenshellBinary, + resolveGatewayPortFromName, + resolveManagedGatewayStateDirectory, + resolveSandboxGatewayName, + runCapture: runSnapshotHostCapture, + }), +); + +export function resolveManagedSnapshotRuntimeAuthority( + driverName: string, + context: ManagedSnapshotRuntimePatchContext, +): unknown { + return resolveManagedSnapshotRuntimeAuthorityWithAdapters( + driverName, + context, + MANAGED_SNAPSHOT_RUNTIME_AUTHORITY_ADAPTERS, + ); +} diff --git a/src/lib/actions/sandbox/snapshot/lifecycle-test-support.ts b/src/lib/actions/sandbox/snapshot/lifecycle-test-support.ts new file mode 100644 index 0000000000..963069dcc5 --- /dev/null +++ b/src/lib/actions/sandbox/snapshot/lifecycle-test-support.ts @@ -0,0 +1,257 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export type TestCommandOptions = Record | undefined; +export type TestCommandResult = { + readonly status: number; + readonly output: string; + readonly stdout?: string; + readonly stderr?: string; +}; +type TestCommandRouteResult = Omit & { + readonly output?: string; +}; +type TestCommandRoute = (args: string[], options?: TestCommandOptions) => TestCommandRouteResult; +export type TestCommandHandler = ( + args: string[], + options?: TestCommandOptions, +) => TestCommandResult; + +const SUCCESSFUL_COMMAND: TestCommandResult = { status: 0, output: "" }; +const MISSING_COMMAND: TestCommandResult = { + status: 1, + stdout: "", + stderr: "", + output: "", +}; + +export function providerMetadata(name: string, type: string, credentialEnv: string): string { + return [ + `Name: ${name}`, + `Type: ${type}`, + `Credential keys: ${credentialEnv}`, + "Config keys: ", + "", + ].join("\n"); +} + +export function valueByName(values: Readonly>): (name?: string) => T | null { + return (name) => { + if (name === undefined) return null; + return values[name] ?? null; + }; +} + +export function valueFactoryByName( + values: Readonly T | null>>, +): (name?: string) => T | null { + return (name) => { + if (name === undefined) return null; + return values[name]?.() ?? null; + }; +} + +export function commandRouter( + routes: Readonly>, + fallback: TestCommandRoute = () => SUCCESSFUL_COMMAND, +): TestCommandHandler { + return (args, options) => { + const handler = routes[args.join(" ")] ?? fallback; + return { output: "", ...handler(args, options) }; + }; +} + +export function recordingCommandRouter( + calls: Array<{ args: string[]; options?: Record }>, + routes: Readonly>, +): TestCommandHandler { + const route = commandRouter(routes); + return (args, options) => { + calls.push({ args, options }); + return route(args, options); + }; +} + +export function managedProviderCreationRunner( + bindings: Readonly>, +): TestCommandHandler { + const createdProviders = new Set(); + return (args) => { + if (args[0] === "provider" && args[1] === "get") { + const providerName = args[2] ?? ""; + const binding = bindings[providerName]; + return binding !== undefined && createdProviders.has(providerName) + ? { + status: 0, + stdout: providerMetadata(providerName, binding.type, binding.credential), + stderr: "", + output: "", + } + : MISSING_COMMAND; + } + if (args[0] === "provider" && args[1] === "create") { + createdProviders.add(args[3] ?? ""); + return { status: 0, stdout: "", stderr: "", output: "" }; + } + return SUCCESSFUL_COMMAND; + }; +} + +export function assertOpenClawDashboard(dashboard: { + readonly agent: string; +}): asserts dashboard is { + readonly agent: "openclaw"; + readonly port: number; + readonly url: string; +} { + if (dashboard.agent !== "openclaw") throw new Error("fixture mismatch"); +} + +export function createReplacementProviderHarness( + validateCreateOptions: (options?: TestCommandOptions) => void, +): { readonly events: string[]; readonly runner: TestCommandHandler } { + const events: string[] = []; + let destinationProviderExists = true; + const runner = commandRouter({ + "provider get beta-telegram-bridge": () => + destinationProviderExists + ? { + status: 0, + stdout: providerMetadata("beta-telegram-bridge", "generic", "TELEGRAM_BOT_TOKEN"), + stderr: "", + output: "", + } + : MISSING_COMMAND, + "sandbox provider detach beta beta-telegram-bridge": () => { + events.push("detach"); + return SUCCESSFUL_COMMAND; + }, + "sandbox delete beta": () => { + events.push("sandbox-delete"); + return SUCCESSFUL_COMMAND; + }, + "provider delete beta-telegram-bridge": () => { + destinationProviderExists = false; + events.push("provider-delete"); + return SUCCESSFUL_COMMAND; + }, + "provider create --name beta-telegram-bridge --type generic --credential TELEGRAM_BOT_TOKEN": ( + _args, + options, + ) => { + validateCreateOptions(options); + destinationProviderExists = true; + events.push("provider-create:new-clone-token"); + return SUCCESSFUL_COMMAND; + }, + }); + return { events, runner }; +} + +export interface FailedReplacementProviderHarness { + readonly events: string[]; + readonly runner: TestCommandHandler; + markSandboxCreateFailed(): void; + state(): { + readonly destinationProviderExists: boolean; + readonly partialSandboxExists: boolean; + }; +} + +export function createFailedReplacementProviderHarness( + validateCreateOptions: (options?: TestCommandOptions) => void, +): FailedReplacementProviderHarness { + const events: string[] = []; + let destinationProviderExists = true; + let providerWasCreated = false; + let partialSandboxExists = false; + let providerAttached = true; + let providerDeleteCount = 0; + let sandboxDeleteCount = 0; + let telegramDetachCount = 0; + + const runner = commandRouter({ + "provider get beta-telegram-bridge": () => + destinationProviderExists + ? { + status: 0, + stdout: providerMetadata("beta-telegram-bridge", "generic", "TELEGRAM_BOT_TOKEN"), + stderr: "", + output: "", + } + : MISSING_COMMAND, + "sandbox provider detach beta beta-telegram-bridge": () => { + telegramDetachCount += 1; + if (telegramDetachCount === 2) { + events.push("partial-provider-detach:failed"); + return { + status: 1, + stderr: "synthetic transient detach failure", + stdout: "", + output: "", + }; + } + events.push( + telegramDetachCount === 1 + ? "initial-provider-detach" + : "rollback-provider-detach:recovered", + ); + providerAttached = false; + return SUCCESSFUL_COMMAND; + }, + "sandbox delete beta": () => { + sandboxDeleteCount += 1; + if (sandboxDeleteCount === 1) { + events.push("initial-sandbox-delete"); + return SUCCESSFUL_COMMAND; + } + events.push("partial-sandbox-delete:failed"); + return { status: 1, stderr: "synthetic partial delete failure", output: "" }; + }, + "provider delete beta-telegram-bridge": () => { + providerDeleteCount += 1; + if (!providerWasCreated && destinationProviderExists) { + if (providerDeleteCount > 1 && !providerAttached) { + destinationProviderExists = false; + events.push("replacement-provider-delete"); + return SUCCESSFUL_COMMAND; + } + events.push("provider-delete:survived"); + return { status: 1, output: "" }; + } + if (providerAttached) { + events.push("provider-delete:blocked-attached"); + return { + status: 1, + stderr: "provider 'beta-telegram-bridge' is attached to sandbox(es): beta", + stdout: "", + output: "", + }; + } + destinationProviderExists = false; + events.push("provider-delete:rollback"); + return SUCCESSFUL_COMMAND; + }, + "provider create --name beta-telegram-bridge --type generic --credential TELEGRAM_BOT_TOKEN": ( + _args, + options, + ) => { + validateCreateOptions(options); + providerWasCreated = true; + destinationProviderExists = true; + events.push("provider-create:rotated-clone-token"); + return SUCCESSFUL_COMMAND; + }, + }); + + return { + events, + runner, + markSandboxCreateFailed() { + partialSandboxExists = true; + providerAttached = true; + events.push("sandbox-create:failed"); + }, + state: () => ({ destinationProviderExists, partialSandboxExists }), + }; +} diff --git a/src/lib/actions/sandbox/snapshot/managed-clone-providers.ts b/src/lib/actions/sandbox/snapshot/managed-clone-providers.ts new file mode 100644 index 0000000000..1ed3fede35 --- /dev/null +++ b/src/lib/actions/sandbox/snapshot/managed-clone-providers.ts @@ -0,0 +1,366 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { SandboxMessagingPlan } from "../../../messaging/manifest"; +import { + ensureWebSearchProviderProfiles, + HERMES_TAVILY_PROVIDER_PROFILE_ID, +} from "../../../onboard/brave-provider-profile"; +import { + matchesGatewayCredentialOnlyProviderBinding, + parseGatewayProviderMetadata, +} from "../../../onboard/gateway-provider-metadata"; +import type { ManagedStartupProfile } from "../../../onboard/managed-startup/profile"; +import { deleteProviderWithRecovery } from "../../../onboard/sandbox-provider-cleanup"; + +type ManagedCloneProviderSource = "messaging" | "web-search"; + +type ManagedCloneProviderCommandResult = { + readonly status: number | null; + readonly stdout?: string | Buffer | null; + readonly stderr?: string | Buffer | null; +}; + +export type ManagedCloneProviderRunner = ( + args: string[], + options?: { + readonly [key: string]: unknown; + readonly ignoreError?: boolean; + readonly env?: NodeJS.ProcessEnv; + readonly stdio?: ["ignore", "ignore" | "pipe", "ignore" | "pipe"]; + }, +) => ManagedCloneProviderCommandResult; + +export interface PreparedManagedCloneProvider { + readonly providerName: string; + readonly providerType: string; + readonly providerEnvKey: string; + readonly source: ManagedCloneProviderSource; + /** + * Only a force-replaced destination authorizes replacement of an existing + * destination-named provider. Provisioning must still prove that no other + * sandbox is attached before it deletes and recreates the provider. + */ + readonly replaceExistingCredential: boolean; +} + +type ProviderInspection = + | { readonly kind: "missing" } + | { readonly kind: "exact" } + | { readonly kind: "collision" }; + +function commandStreamText(value: string | Buffer | null | undefined): string { + return Buffer.isBuffer(value) ? value.toString("utf8") : (value ?? ""); +} + +function inspectProvider( + provider: Pick, + runOpenshell: ManagedCloneProviderRunner, +): ProviderInspection { + const inspection = runOpenshell(["provider", "get", provider.providerName], { + ignoreError: true, + stdio: ["ignore", "pipe", "pipe"], + }); + if (inspection.status !== 0) return { kind: "missing" }; + + const metadata = parseGatewayProviderMetadata( + `${commandStreamText(inspection.stdout)}\n${commandStreamText(inspection.stderr)}`, + ); + return matchesGatewayCredentialOnlyProviderBinding(metadata, { + name: provider.providerName, + type: provider.providerType, + credentialKey: provider.providerEnvKey, + }) + ? { kind: "exact" } + : { kind: "collision" }; +} + +function hasCredential(environment: NodeJS.ProcessEnv, name: string): boolean { + const value = environment[name]; + return typeof value === "string" && value.replace(/\r/gu, "").trim().length > 0; +} + +function activeMessagingCredentialBindings( + plan: SandboxMessagingPlan, +): readonly SandboxMessagingPlan["credentialBindings"][number][] { + const activeChannels = new Set( + plan.channels + .filter( + (channel) => + channel.active && !channel.disabled && !plan.disabledChannels.includes(channel.channelId), + ) + .map((channel) => channel.channelId), + ); + return plan.credentialBindings.filter((binding) => activeChannels.has(binding.channelId)); +} + +function addProvider( + providers: Map>, + provider: Omit, +): void { + const existing = providers.get(provider.providerName); + if ( + existing && + (existing.providerType !== provider.providerType || + existing.providerEnvKey !== provider.providerEnvKey) + ) { + throw new Error( + `managed clone provider '${provider.providerName}' has conflicting credential bindings`, + ); + } + if (!existing) providers.set(provider.providerName, provider); +} + +function addMessagingProviders( + providers: Map>, + plan: SandboxMessagingPlan | null, +): void { + if (!plan) return; + for (const binding of activeMessagingCredentialBindings(plan)) { + addProvider(providers, { + providerName: binding.providerName, + providerType: "generic", + providerEnvKey: binding.providerEnvKey, + source: "messaging", + }); + } +} + +function addWebSearchProvider( + providers: Map>, + profile: ManagedStartupProfile, + destinationSandboxName: string, +): void { + if (profile.agentConfig.agent === "langchain-deepagents-code") return; + const webSearch = profile.agentConfig.webSearch; + if (!webSearch.enabled) return; + const providerType = + profile.agent === "hermes" && webSearch.provider === "tavily" + ? HERMES_TAVILY_PROVIDER_PROFILE_ID + : webSearch.provider; + addProvider(providers, { + providerName: `${destinationSandboxName}-${webSearch.provider}-search`, + providerType, + providerEnvKey: webSearch.provider === "tavily" ? "TAVILY_API_KEY" : "BRAVE_API_KEY", + source: "web-search", + }); +} + +function assertHermesToolGatewayCloneIsRecredentialable(profile: ManagedStartupProfile): void { + if (profile.agent !== "hermes" || profile.tools.enabledGateways.length === 0) return; + throw new Error( + "managed Hermes tool gateways cannot yet be cloned without a fresh Nous OAuth refresh " + + "credential and destination broker binding; disable the gateways or re-onboard the clone", + ); +} + +function ensureRequiredWebSearchProfiles( + providers: readonly Omit[], + root: string, + runOpenshell: ManagedCloneProviderRunner, +): void { + const webSearchProviders = providers.filter((provider) => provider.source === "web-search"); + if (webSearchProviders.length === 0) return; + ensureWebSearchProviderProfiles( + webSearchProviders.map((provider) => ({ + providerType: provider.providerType, + // Profile registration only needs the non-secret fact that this provider + // will be attached; never pass the actual credential through this layer. + token: "credential-binding-present", + })), + { + root, + runOpenshell, + // Diagnostics are intentionally suppressed below. Do not introduce a + // second secret-bearing logging path merely to satisfy the profile + // helper's redaction dependency. + redact: () => "", + log: () => {}, + exit: (code = 1): never => { + throw new Error(`failed to register managed web-search provider profile (exit ${code})`); + }, + }, + ); +} + +/** + * Build and validate every credential-provider binding the rebound profile + * will attach. This runs before destination deletion so absent credentials, + * colliding provider identities, unsupported broker state, and provider + * profile import failures all leave a force-restore destination untouched. + */ +export function prepareManagedCloneProviders(input: { + readonly profile: ManagedStartupProfile; + readonly messagingPlan: SandboxMessagingPlan | null; + readonly destinationSandboxName: string; + readonly destinationWillBeReplaced: boolean; + readonly environment?: NodeJS.ProcessEnv; + readonly root: string; + readonly runOpenshell: ManagedCloneProviderRunner; +}): readonly PreparedManagedCloneProvider[] { + assertHermesToolGatewayCloneIsRecredentialable(input.profile); + + const pending = new Map< + string, + Omit + >(); + addMessagingProviders(pending, input.messagingPlan); + addWebSearchProvider(pending, input.profile, input.destinationSandboxName); + const environment = input.environment ?? process.env; + const prepared: PreparedManagedCloneProvider[] = []; + + for (const provider of pending.values()) { + const inspection = inspectProvider(provider, input.runOpenshell); + if (inspection.kind === "collision") { + throw new Error( + `managed clone provider '${provider.providerName}' exists with an incompatible ` + + `type or credential binding`, + ); + } + if (inspection.kind === "exact" && !input.destinationWillBeReplaced) { + throw new Error( + `managed clone provider '${provider.providerName}' already exists without a ` + + "replaceable destination; refusing unproven credential reuse", + ); + } + if (!hasCredential(environment, provider.providerEnvKey)) { + throw new Error( + `managed ${provider.source} provider '${provider.providerName}' requires an explicit ` + + `clone credential; export ${provider.providerEnvKey} before retrying`, + ); + } + prepared.push({ + ...provider, + replaceExistingCredential: input.destinationWillBeReplaced, + }); + } + + // Import immutable custom profiles while the destination is still intact. + // Provisioning below can then fail only on provider CRUD/races, not on a + // missing Brave/Tavily/Hermes-Tavily profile artifact. + ensureRequiredWebSearchProfiles([...pending.values()], input.root, input.runOpenshell); + return prepared; +} + +function cleanupCreatedProviders( + providerNames: readonly string[], + runOpenshell: ManagedCloneProviderRunner, + authorizedSandboxName?: string, +): void { + for (const providerName of [...providerNames].reverse()) { + const result = authorizedSandboxName + ? deleteProviderWithRecovery(providerName, { + runOpenshell, + allowedSandboxes: [authorizedSandboxName], + }) + : runOpenshell(["provider", "delete", providerName], { + ignoreError: true, + stdio: ["ignore", "ignore", "ignore"], + }); + const deleted = "ok" in result ? result.ok : result.status === 0; + if (!deleted) { + console.warn( + ` Warning: could not clean up managed clone provider '${providerName}' after failure.`, + ); + } + } +} + +/** + * Materialize any preflighted provider that is not already exact, then prove + * the resulting metadata before its name is placed on sandbox-create argv. + */ +export function provisionManagedCloneProviders( + prepared: readonly PreparedManagedCloneProvider[], + input: { + readonly environment?: NodeJS.ProcessEnv; + readonly runOpenshell: ManagedCloneProviderRunner; + readonly rollbackSandboxName?: string; + }, +): string[] { + const environment = input.environment ?? process.env; + // A force-replaced destination authorizes deletion and recreation, never a + // gateway-wide credential update. The bounded delete helper proves that no + // unrelated sandbox is attached before the new credential is introduced. + const mutated: string[] = []; + try { + for (const provider of prepared) { + const current = inspectProvider(provider, input.runOpenshell); + if (current.kind === "collision") { + throw new Error( + `managed clone provider '${provider.providerName}' changed to an incompatible binding`, + ); + } + if (current.kind === "exact" && !provider.replaceExistingCredential) { + throw new Error( + `managed clone provider '${provider.providerName}' appeared before launch; ` + + "refusing unproven credential reuse", + ); + } + if (current.kind === "exact") { + const authorizedSandboxName = input.rollbackSandboxName; + if (!authorizedSandboxName) { + throw new Error( + `managed clone provider '${provider.providerName}' cannot be replaced without ` + + "an exact destination sandbox", + ); + } + const replacementDelete = deleteProviderWithRecovery(provider.providerName, { + runOpenshell: input.runOpenshell, + allowedSandboxes: [authorizedSandboxName], + }); + if (!replacementDelete.ok) { + throw new Error( + `managed clone provider '${provider.providerName}' is still attached outside ` + + `destination '${authorizedSandboxName}'`, + ); + } + } + + const credential = environment[provider.providerEnvKey]?.replace(/\r/gu, "").trim(); + if (!credential) { + throw new Error( + `managed clone credential ${provider.providerEnvKey} disappeared before provider creation`, + ); + } + const result = input.runOpenshell( + [ + "provider", + "create", + "--name", + provider.providerName, + "--type", + provider.providerType, + "--credential", + provider.providerEnvKey, + ], + { + ignoreError: true, + env: { [provider.providerEnvKey]: credential }, + stdio: ["ignore", "pipe", "pipe"], + }, + ); + if (result.status !== 0) { + throw new Error(`failed to create managed clone provider '${provider.providerName}'`); + } + mutated.push(provider.providerName); + if (inspectProvider(provider, input.runOpenshell).kind !== "exact") { + throw new Error( + `managed clone provider '${provider.providerName}' did not retain its exact binding`, + ); + } + } + return mutated; + } catch (error) { + cleanupCreatedProviders(mutated, input.runOpenshell, input.rollbackSandboxName); + throw error; + } +} + +export function cleanupManagedCloneProviders( + providerNames: readonly string[], + runOpenshell: ManagedCloneProviderRunner, + authorizedSandboxName?: string, +): void { + cleanupCreatedProviders(providerNames, runOpenshell, authorizedSandboxName); +} diff --git a/src/lib/actions/sandbox/snapshot/runtime-authority.test.ts b/src/lib/actions/sandbox/snapshot/runtime-authority.test.ts new file mode 100644 index 0000000000..0c3288c84f --- /dev/null +++ b/src/lib/actions/sandbox/snapshot/runtime-authority.test.ts @@ -0,0 +1,191 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, it, vi } from "vitest"; + +import { buildPodmanDriverGatewayEnv } from "../../../onboard/compute/podman/gateway-env"; +import { createPodmanSandboxCreateRuntimeAuthority } from "../../../onboard/compute/podman/sandbox-create-authority"; +import { createPodmanOpenShellWatcherController } from "../../../onboard/compute/podman/sandbox-recreate"; +import type { PodmanSocketAuthority } from "../../../onboard/compute/podman/socket-authority"; +import type { SandboxRuntimeAuthorityAdapterRegistry } from "../../../onboard/compute/runtime-authority"; +import type { SandboxEntry } from "../../../state/registry"; +import { + createPodmanManagedSnapshotRuntimeAuthority, + currentManagedSnapshotRuntimeAuthorityAdapters, + resolveManagedSnapshotRuntimeAuthority, +} from "./runtime-authority"; +import type { ManagedSnapshotRuntimePatchContext } from "./runtime-patch"; + +function context(driverName: string): ManagedSnapshotRuntimePatchContext { + return { + destinationSandboxName: "beta", + sourceEntry: { + gatewayName: "nemoclaw-8443", + gatewayPort: 8443, + name: "alpha", + openshellDriver: driverName, + } as SandboxEntry, + }; +} + +describe("managed snapshot runtime authority", () => { + it("routes exact source context to the Podman authority adapter", () => { + const authority = { + socketAuthority: { + directoryChain: [], + device: "8", + inode: "9001", + ownerUid: "1001", + socketPath: "/run/user/1001/podman/podman.sock", + }, + socketPath: "/run/user/1001/podman/podman.sock", + watcherController: createPodmanOpenShellWatcherController({ + assertStopped: vi.fn(), + resumeAndProve: vi.fn(), + stopAndProve: vi.fn(() => ({ stopped: true })), + }), + }; + const createPodman = vi.fn(() => authority); + const input = context("podman"); + + expect( + resolveManagedSnapshotRuntimeAuthority( + "podman", + input, + currentManagedSnapshotRuntimeAuthorityAdapters(createPodman), + ), + ).toBe(authority); + expect(createPodman).toHaveBeenCalledWith(input); + }); + + it("keeps direct drivers authority-free", () => { + const adapters = currentManagedSnapshotRuntimeAuthorityAdapters(vi.fn()); + expect( + resolveManagedSnapshotRuntimeAuthority("docker", context("docker"), adapters), + ).toBeNull(); + expect( + resolveManagedSnapshotRuntimeAuthority("kubernetes", context("kubernetes"), adapters), + ).toBeNull(); + }); + + it("accepts an independently registered MXC authority adapter", () => { + const authority = { endpoint: "unix:///run/mxc/runtime.sock" }; + const adapters: SandboxRuntimeAuthorityAdapterRegistry = { + mxc: { driverName: "mxc", resolve: vi.fn(() => authority) }, + }; + + expect(resolveManagedSnapshotRuntimeAuthority("mxc", context("mxc"), adapters)).toBe(authority); + }); + + it("qualifies the recovered Podman API before composing mutable snapshot authority", () => { + const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-snapshot-authority-")); + try { + const config = 'compute_drivers = ["podman"]\n'; + fs.writeFileSync(path.join(stateDir, "openshell-gateway.toml"), config, { mode: 0o600 }); + fs.writeFileSync( + path.join(stateDir, "managed-runtime.json"), + `${JSON.stringify({ + version: 1, + driverName: "podman", + configSha256: createHash("sha256").update(config).digest("hex"), + values: { + network_name: "openshell", + socket_path: "/run/user/1000/podman/podman.sock", + supervisor_image: `ghcr.io/nvidia/openshell/supervisor@sha256:${"a".repeat(64)}`, + }, + })}\n`, + { mode: 0o600 }, + ); + const qualifyRecoveryRuntime = vi.fn(() => { + throw new Error("rootless Podman API is unavailable"); + }); + const getOpenshellBinary = vi.fn(() => "/usr/bin/openshell"); + + expect(() => + createPodmanManagedSnapshotRuntimeAuthority(context("podman"), { + captureOpenshell: vi.fn(() => ({ output: "" })), + getOpenshellBinary, + qualifyRecoveryRuntime, + resolveGatewayPortFromName: vi.fn(() => 8443), + resolveManagedGatewayStateDirectory: vi.fn(() => stateDir), + resolveSandboxGatewayName: vi.fn(() => "nemoclaw-8443"), + runCapture: vi.fn(() => ""), + }), + ).toThrow("rootless Podman API is unavailable"); + expect(qualifyRecoveryRuntime).toHaveBeenCalledOnce(); + expect(getOpenshellBinary).not.toHaveBeenCalled(); + } finally { + fs.rmSync(stateDir, { force: true, recursive: true }); + } + }); + + it("threads the qualified socket receipt and rejects replacement before watcher composition", () => { + const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-snapshot-authority-")); + const previousGatewayBin = process.env.NEMOCLAW_OPENSHELL_GATEWAY_BIN; + try { + const socketPath = "/run/user/1000/podman/podman.sock"; + buildPodmanDriverGatewayEnv({ + gatewayPort: 8443, + podmanSocketPath: socketPath, + stateDir, + supervisorImage: `ghcr.io/nvidia/openshell/supervisor@sha256:${"a".repeat(64)}`, + }); + const socketAuthority: PodmanSocketAuthority = { + directoryChain: [], + device: "8", + inode: "9001", + ownerUid: "1000", + socketPath, + }; + const qualifyRecoveryRuntime = vi.fn(() => ({ + architecture: "amd64" as const, + cgroupVersion: "v2" as const, + driverName: "podman" as const, + networkBackend: "netavark", + os: "linux" as const, + rootless: true as const, + socketAuthority, + socketPath, + version: "5.0.0", + })); + const assertSocketAuthority = vi.fn(() => { + throw new Error("Podman socket authority changed after it was qualified."); + }); + const createWatcherController = vi.fn(() => { + throw new Error("watcher composition must not run after authority replacement"); + }); + process.env.NEMOCLAW_OPENSHELL_GATEWAY_BIN = "/bin/sh"; + + expect(() => + createPodmanManagedSnapshotRuntimeAuthority(context("podman"), { + captureOpenshell: vi.fn(() => ({ output: "" })), + createRuntimeAuthority: (input) => + createPodmanSandboxCreateRuntimeAuthority(input, { + assertSocketAuthority, + createWatcherController, + }), + getOpenshellBinary: vi.fn(() => "/usr/bin/openshell"), + qualifyRecoveryRuntime, + resolveGatewayPortFromName: vi.fn(() => 8443), + resolveManagedGatewayStateDirectory: vi.fn(() => stateDir), + resolveSandboxGatewayName: vi.fn(() => "nemoclaw-8443"), + runCapture: vi.fn(() => ""), + }), + ).toThrow("Podman socket authority changed after it was qualified."); + expect(qualifyRecoveryRuntime).toHaveBeenCalledOnce(); + expect(assertSocketAuthority).toHaveBeenCalledExactlyOnceWith(socketAuthority); + expect(createWatcherController).not.toHaveBeenCalled(); + } finally { + if (previousGatewayBin === undefined) { + delete process.env.NEMOCLAW_OPENSHELL_GATEWAY_BIN; + } else { + process.env.NEMOCLAW_OPENSHELL_GATEWAY_BIN = previousGatewayBin; + } + fs.rmSync(stateDir, { force: true, recursive: true }); + } + }); +}); diff --git a/src/lib/actions/sandbox/snapshot/runtime-authority.ts b/src/lib/actions/sandbox/snapshot/runtime-authority.ts new file mode 100644 index 0000000000..5b472c27b9 --- /dev/null +++ b/src/lib/actions/sandbox/snapshot/runtime-authority.ts @@ -0,0 +1,200 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + CURRENT_MANAGED_GATEWAY_DRIVER_PROFILES, + resolveManagedGatewayDriverProfile, +} from "../../../onboard/compute/managed-gateway-profile"; +import { buildPersistedPodmanDriverGatewayEnv } from "../../../onboard/compute/podman/gateway-env"; +import { + createPodmanSandboxCreateRuntimeAuthority, + type PodmanSandboxCreateRuntimeAuthority, + type PodmanSandboxCreateRuntimeAuthorityInput, +} from "../../../onboard/compute/podman/sandbox-create-authority"; +import type { PodmanSocketAuthority } from "../../../onboard/compute/podman/socket-authority"; +import { + qualifyManagedGatewayRecoveryRuntime, + resolveManagedGatewayRecoveryRuntime, +} from "../../../onboard/compute/recovery-runtime"; +import { + resolveSandboxRuntimeAuthority, + type SandboxRuntimeAuthorityAdapterRegistry, +} from "../../../onboard/compute/runtime-authority"; +import { readManagedGatewayRuntimeBinding } from "../../../onboard/docker-driver-gateway-config"; +import { buildHostManagedGatewayRuntimeIdentity } from "../../../onboard/docker-driver-gateway-launch"; +import { createDockerDriverGatewayRuntimeHelpers } from "../../../onboard/docker-driver-gateway-runtime"; +import { + getBlueprintMaxOpenshellVersion, + getInstalledOpenshellVersion, + isOpenshellDevVersion, + SUPPORTED_OPENSHELL_FALLBACK_VERSION, + shouldUseOpenshellDevChannel, +} from "../../../onboard/openshell-version"; +import { isGatewayHealthy } from "../../../state/gateway"; +import type { SandboxEntry } from "../../../state/registry"; +import type { ManagedSnapshotRuntimePatchContext } from "./runtime-patch"; + +type SnapshotAuthorityRegistry = + SandboxRuntimeAuthorityAdapterRegistry; + +export interface PodmanManagedSnapshotRuntimeAuthorityDependencies { + captureOpenshell( + args: string[], + options: { ignoreError: true }, + ): { readonly output?: string | null }; + getOpenshellBinary(): string; + resolveGatewayPortFromName(gatewayName: string): number | null; + resolveManagedGatewayStateDirectory(gatewayName: string): string; + resolveSandboxGatewayName(sourceEntry: SandboxEntry): string; + runCapture(args: string[], options?: { ignoreError?: boolean }): string; + qualifyRecoveryRuntime?: typeof qualifyManagedGatewayRecoveryRuntime; + createRuntimeAuthority?: ( + input: PodmanSandboxCreateRuntimeAuthorityInput, + ) => PodmanSandboxCreateRuntimeAuthority; +} + +function captureOutput( + deps: PodmanManagedSnapshotRuntimeAuthorityDependencies, + args: string[], +): string { + return deps.captureOpenshell(args, { ignoreError: true }).output ?? ""; +} + +function requireQualifiedPodmanSocketAuthority( + qualification: unknown, + expectedSocketPath: string, +): PodmanSocketAuthority { + if (!qualification || typeof qualification !== "object") { + throw new Error("Managed Podman runtime qualification returned no socket authority."); + } + const receipt = qualification as { + readonly driverName?: unknown; + readonly socketAuthority?: Partial; + readonly socketPath?: unknown; + }; + const authority = receipt.socketAuthority; + if ( + receipt.driverName !== "podman" || + receipt.socketPath !== expectedSocketPath || + !authority || + authority.socketPath !== expectedSocketPath || + !Array.isArray(authority.directoryChain) || + typeof authority.device !== "string" || + typeof authority.inode !== "string" || + typeof authority.ownerUid !== "string" + ) { + throw new Error("Managed Podman runtime qualification returned mismatched socket authority."); + } + return authority as PodmanSocketAuthority; +} + +export function createPodmanManagedSnapshotRuntimeAuthority( + context: ManagedSnapshotRuntimePatchContext, + deps: PodmanManagedSnapshotRuntimeAuthorityDependencies, +): PodmanSandboxCreateRuntimeAuthority { + const sourceEntry = context.sourceEntry as SandboxEntry; + const gatewayName = deps.resolveSandboxGatewayName(sourceEntry); + const gatewayPort = deps.resolveGatewayPortFromName(gatewayName); + if (gatewayPort === null) { + throw new Error(`Managed Podman snapshot source has invalid gateway '${gatewayName}'.`); + } + const stateDir = deps.resolveManagedGatewayStateDirectory(gatewayName); + const binding = readManagedGatewayRuntimeBinding(stateDir); + if (!binding) { + throw new Error(`Managed Podman runtime binding is missing in '${stateDir}'.`); + } + const recovery = resolveManagedGatewayRecoveryRuntime( + { driverName: "podman", stateDir }, + undefined, + () => binding, + ); + const socketPath = recovery.environment.OPENSHELL_PODMAN_SOCKET; + const supervisorImage = recovery.environment.OPENSHELL_SUPERVISOR_IMAGE; + if (!socketPath || !supervisorImage) { + throw new Error("Managed Podman runtime binding is missing its socket or supervisor image."); + } + const qualification = (deps.qualifyRecoveryRuntime ?? qualifyManagedGatewayRecoveryRuntime)( + recovery, + ); + const socketAuthority = requireQualifiedPodmanSocketAuthority(qualification, socketPath); + const runtimeHelpers = createDockerDriverGatewayRuntimeHelpers({ + gatewayPort, + getCachedOpenshellBinary: deps.getOpenshellBinary, + getBlueprintMaxOpenshellVersion, + getInstalledOpenshellVersion, + isOpenshellDevVersion, + runCapture: deps.runCapture, + shouldUseOpenshellDevChannel, + stateDir, + supportedOpenshellFallbackVersion: SUPPORTED_OPENSHELL_FALLBACK_VERSION, + }); + const gatewayBin = runtimeHelpers.resolveOpenShellGatewayBinary(); + if (!gatewayBin) { + throw new Error("Managed Podman snapshot clone could not resolve the gateway binary."); + } + const profile = resolveManagedGatewayDriverProfile( + { driverName: "podman", gatewayLauncher: "nemoclaw" }, + CURRENT_MANAGED_GATEWAY_DRIVER_PROFILES, + ); + if (!profile) throw new Error("Managed Podman gateway profile is unavailable."); + const gatewayEnv = buildPersistedPodmanDriverGatewayEnv({ + configSha256: binding.configSha256, + gatewayPort, + podmanSocketPath: socketPath, + stateDir, + supervisorImage, + }); + const runtimeIdentity = buildHostManagedGatewayRuntimeIdentity({ + gatewayBin, + gatewayEnv, + gatewayName, + removeEnvironmentKeys: profile.incompatibleRuntimeEnvironmentKeys, + runtimeEnvironmentKeys: profile.runtimeEnvironmentKeys, + }); + return (deps.createRuntimeAuthority ?? createPodmanSandboxCreateRuntimeAuthority)({ + driverLabel: profile.displayName, + gatewayBin, + gatewayName, + gatewayPort, + getRememberedGatewayPid: runtimeHelpers.getDockerDriverGatewayPid, + getRuntimeDrift: (pid, desiredEnv, driftGatewayBin, trustedServicePid) => + runtimeHelpers.getDockerDriverGatewayReuseDrift( + pid, + { ...desiredEnv }, + driftGatewayBin, + trustedServicePid, + ), + isGatewayHealthy: () => + isGatewayHealthy( + captureOutput(deps, ["status"]), + captureOutput(deps, ["gateway", "info", "-g", gatewayName]), + captureOutput(deps, ["gateway", "info"]), + ), + isPidAlive: runtimeHelpers.isPidAlive, + rememberGatewayPid: runtimeHelpers.rememberDockerDriverGatewayPid, + runtimeIdentity, + socketAuthority, + socketPath, + stateDir, + }); +} + +export function currentManagedSnapshotRuntimeAuthorityAdapters( + createPodman: ( + context: ManagedSnapshotRuntimePatchContext, + ) => PodmanSandboxCreateRuntimeAuthority, +): SnapshotAuthorityRegistry { + return { + docker: { driverName: "docker", resolve: () => null }, + kubernetes: { driverName: "kubernetes", resolve: () => null }, + podman: { driverName: "podman", resolve: createPodman }, + }; +} + +export function resolveManagedSnapshotRuntimeAuthority( + driverName: string, + context: ManagedSnapshotRuntimePatchContext, + adapters: SnapshotAuthorityRegistry, +): unknown { + return resolveSandboxRuntimeAuthority(driverName, context, adapters); +} diff --git a/src/lib/actions/sandbox/snapshot/runtime-patch.test.ts b/src/lib/actions/sandbox/snapshot/runtime-patch.test.ts new file mode 100644 index 0000000000..34aadbe8d2 --- /dev/null +++ b/src/lib/actions/sandbox/snapshot/runtime-patch.test.ts @@ -0,0 +1,245 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; +import { DCODE_MANAGED_RUNTIME_ULIMITS } from "../../../onboard/compute/managed-startup-runtime-requirements"; +import { createPodmanOpenShellWatcherController } from "../../../onboard/compute/podman/sandbox-recreate"; +import type { SandboxCreateRuntimePatchRequest } from "../../../onboard/sandbox-create-runtime/registry"; +import type { SandboxCreateRuntimePatch } from "../../../onboard/sandbox-create-runtime/types"; +import type { SandboxEntry } from "../../../state/registry"; +import { + createManagedSnapshotRuntimePatch, + runAuthorizedManagedSnapshotDestinationDelete, +} from "./runtime-patch"; + +function patch(): SandboxCreateRuntimePatch { + return { + maybeApplyDuringCreate: vi.fn(), + createFailureMessage: vi.fn(() => null), + exitOnPatchError: vi.fn(), + rollbackManagedStartupAfterCreateFailure: vi.fn(), + ensureApplied: vi.fn(), + revalidateBeforeMutation: vi.fn(), + waitForSupervisorReconnectIfNeeded: vi.fn(), + commitAfterReady: vi.fn(), + }; +} + +function lifecycle() { + return { + deps: { + runCaptureOpenshell: vi.fn(() => ""), + runOpenshell: vi.fn(() => ({ status: 0 })), + sleep: vi.fn(), + }, + openshellSandboxCommand: ["nemoclaw-start"], + sandboxName: "beta", + timeoutSecs: 30, + }; +} + +function sourceEntry(openshellDriver?: string | null, agent?: string): SandboxEntry { + return { + name: "alpha", + ...(openshellDriver !== undefined ? { openshellDriver } : {}), + ...(agent ? { agent } : {}), + }; +} + +describe("managed snapshot runtime patch selection", () => { + it("revalidates after preparation and prevents forced destination deletion on drift", () => { + const selected = patch(); + const deleteDestination = vi.fn(); + vi.mocked(selected.revalidateBeforeMutation).mockImplementation(() => { + throw new Error("Podman socket authority changed before destination delete"); + }); + + expect(() => + runAuthorizedManagedSnapshotDestinationDelete(selected, deleteDestination), + ).toThrow("socket authority changed"); + expect(selected.revalidateBeforeMutation).toHaveBeenCalledOnce(); + expect(deleteDestination).not.toHaveBeenCalled(); + }); + + it.each([ + ["an unrecorded legacy source", undefined], + ["an explicit Docker source", "docker"], + ["a legacy VM source", "vm"], + ])("preserves the Docker patch path for %s", (_label, openshellDriver) => { + const selected = patch(); + const createRuntimePatch = vi.fn(() => selected); + const runtimeLifecycle = lifecycle(); + + expect( + createManagedSnapshotRuntimePatch( + { + destinationSandboxName: "beta", + lifecycle: runtimeLifecycle, + sourceEntry: sourceEntry(openshellDriver), + }, + { createRuntimePatch }, + ), + ).toBe(selected); + expect(createRuntimePatch).toHaveBeenCalledWith({ + driverName: "docker", + lifecycle: { + ...runtimeLifecycle, + persistStartupCommand: false, + requiredUlimits: null, + sandboxGpuEnabled: false, + }, + runtimeAuthority: undefined, + }); + }); + + it("injects exact Podman runtime authority into the shared patch factory", () => { + const selected = patch(); + const createRuntimePatch = vi.fn(() => selected); + const watcherController = createPodmanOpenShellWatcherController({ + assertStopped: vi.fn(), + resumeAndProve: vi.fn(), + stopAndProve: vi.fn(() => ({ owner: "gateway" })), + }); + const runtimeAuthority = { + socketPath: "/run/user/1000/podman/podman.sock", + watcherController, + }; + const resolveRuntimeAuthority = vi.fn(() => runtimeAuthority); + const source = sourceEntry("podman"); + const runtimeLifecycle = lifecycle(); + + expect( + createManagedSnapshotRuntimePatch( + { + destinationSandboxName: "beta", + lifecycle: runtimeLifecycle, + sourceEntry: source, + }, + { createRuntimePatch, resolveRuntimeAuthority }, + ), + ).toBe(selected); + expect(resolveRuntimeAuthority).toHaveBeenCalledWith("podman", { + destinationSandboxName: "beta", + sourceEntry: source, + }); + expect(createRuntimePatch).toHaveBeenCalledWith({ + driverName: "podman", + lifecycle: { + ...runtimeLifecycle, + persistStartupCommand: false, + requiredUlimits: null, + sandboxGpuEnabled: false, + }, + runtimeAuthority, + }); + }); + + it("carries DCode's exact persistence and ulimits into a Podman snapshot clone", () => { + const selected = patch(); + const createRuntimePatch = vi.fn(() => selected); + const runtimeAuthority = { + socketPath: "/run/user/1000/podman/podman.sock", + watcherController: createPodmanOpenShellWatcherController({ + assertStopped: vi.fn(), + resumeAndProve: vi.fn(), + stopAndProve: vi.fn(() => ({ owner: "gateway" })), + }), + }; + const runtimeLifecycle = lifecycle(); + + expect( + createManagedSnapshotRuntimePatch( + { + destinationSandboxName: "beta", + lifecycle: runtimeLifecycle, + sourceEntry: sourceEntry("podman", "langchain-deepagents-code"), + }, + { + createRuntimePatch, + resolveRuntimeAuthority: () => runtimeAuthority, + }, + ), + ).toBe(selected); + expect(createRuntimePatch).toHaveBeenCalledWith({ + driverName: "podman", + lifecycle: { + ...runtimeLifecycle, + persistStartupCommand: true, + requiredUlimits: DCODE_MANAGED_RUNTIME_ULIMITS, + sandboxGpuEnabled: false, + }, + runtimeAuthority, + }); + }); + + it("fails closed before factory dispatch without Podman lifecycle authority", () => { + expect(() => + createManagedSnapshotRuntimePatch({ + destinationSandboxName: "beta", + lifecycle: lifecycle(), + sourceEntry: sourceEntry("podman"), + }), + ).toThrow("requires its qualified socket and watcher controller"); + }); + + it("rejects a managed Kubernetes snapshot clone before destination mutation", () => { + expect(() => + createManagedSnapshotRuntimePatch({ + destinationSandboxName: "beta", + lifecycle: { + ...lifecycle(), + managedStartupRootApplyRequest: { agent: "openclaw" } as never, + }, + sourceEntry: sourceEntry("kubernetes", "openclaw"), + }), + ).toThrow("has no managed-startup runtime adapter"); + }); + + it("delegates future runtime authority and patch adapters without coordinator changes", () => { + const selected = patch(); + const createRuntimePatch = vi.fn((_request: SandboxCreateRuntimePatchRequest) => selected); + const runtimeAuthority = { endpoint: "unix:///run/mxc.sock" }; + const resolveRuntimeAuthority = vi.fn(() => runtimeAuthority); + const runtimeRequirements = { + persistStartupCommand: true, + requiredUlimits: [{ name: "nofile", soft: 8192, hard: 8192 }], + } as const; + const resolveRuntimeRequirements = vi.fn(() => runtimeRequirements); + const source = sourceEntry("mxc", "hermes"); + const runtimeLifecycle = lifecycle(); + + expect( + createManagedSnapshotRuntimePatch( + { + destinationSandboxName: "beta", + lifecycle: runtimeLifecycle, + sourceEntry: source, + }, + { createRuntimePatch, resolveRuntimeAuthority, resolveRuntimeRequirements }, + ), + ).toBe(selected); + expect(resolveRuntimeAuthority).toHaveBeenCalledWith("mxc", { + destinationSandboxName: "beta", + sourceEntry: source, + }); + expect(resolveRuntimeRequirements).toHaveBeenCalledWith( + "mxc", + { + destinationSandboxName: "beta", + sourceEntry: source, + }, + { managedGatewayOwned: true }, + ); + const [request] = createRuntimePatch.mock.calls[0] ?? []; + expect(request).toEqual({ + driverName: "mxc", + lifecycle: { + ...runtimeLifecycle, + ...runtimeRequirements, + sandboxGpuEnabled: false, + }, + runtimeAuthority, + }); + expect(request).not.toHaveProperty("docker"); + }); +}); diff --git a/src/lib/actions/sandbox/snapshot/runtime-patch.ts b/src/lib/actions/sandbox/snapshot/runtime-patch.ts new file mode 100644 index 0000000000..846365631a --- /dev/null +++ b/src/lib/actions/sandbox/snapshot/runtime-patch.ts @@ -0,0 +1,105 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + type ManagedStartupRuntimeRequirements, + resolveManagedStartupRuntimeRequirements, +} from "../../../onboard/compute/managed-startup-runtime-requirements"; +import { + createSandboxCreateRuntimePatch, + currentSandboxCreateRuntimePatchAdapters, + type SandboxCreateRuntimeLifecycleContext, + type SandboxCreateRuntimePatchRequest, +} from "../../../onboard/sandbox-create-runtime/registry"; +import type { SandboxCreateRuntimePatch } from "../../../onboard/sandbox-create-runtime/types"; +import type { SandboxEntry } from "../../../state/registry"; + +export interface ManagedSnapshotRuntimePatchContext { + readonly destinationSandboxName: string; + readonly sourceEntry: SandboxEntry; +} + +export interface ManagedSnapshotRuntimePatchDependencies { + readonly createRuntimePatch?: ( + request: SandboxCreateRuntimePatchRequest, + ) => SandboxCreateRuntimePatch; + readonly resolveRuntimeAuthority?: ( + driverName: string, + context: ManagedSnapshotRuntimePatchContext, + ) => unknown; + readonly resolveRuntimeRequirements?: ( + driverName: string, + context: ManagedSnapshotRuntimePatchContext, + options: { readonly managedGatewayOwned: boolean }, + ) => ManagedStartupRuntimeRequirements; +} + +export interface ManagedSnapshotRuntimePatchInput extends ManagedSnapshotRuntimePatchContext { + readonly createDockerPatch?: ( + lifecycle: SandboxCreateRuntimeLifecycleContext, + ) => SandboxCreateRuntimePatch; + readonly lifecycle: Omit< + SandboxCreateRuntimeLifecycleContext, + "persistStartupCommand" | "requiredUlimits" | "sandboxGpuEnabled" + >; +} + +function snapshotRuntimeDriver(sourceEntry: SandboxEntry): string { + const recorded = sourceEntry.openshellDriver?.trim(); + // Legacy VM records use the same Docker-compatible managed-startup patch + // path as unrecorded and explicit Docker sandboxes. + return !recorded || recorded === "vm" ? "docker" : recorded; +} + +/** + * Route managed snapshot clones through the shared runtime-patch registry. + * + * A native runtime may require lifecycle authority that snapshot.ts cannot + * safely infer. Resolve those dependencies in the owning runtime composition + * and inject the exact result here. Missing dependencies fail before sandbox + * creation begins; Docker keeps its existing dependency-free selection path. + */ +export function createManagedSnapshotRuntimePatch( + input: ManagedSnapshotRuntimePatchInput, + dependencies: ManagedSnapshotRuntimePatchDependencies = {}, +): SandboxCreateRuntimePatch { + const driverName = snapshotRuntimeDriver(input.sourceEntry); + const context = { + destinationSandboxName: input.destinationSandboxName, + sourceEntry: input.sourceEntry, + } as const; + const runtimeAuthority = dependencies.resolveRuntimeAuthority?.(driverName, context); + const managedGatewayOwned = driverName !== "docker" && runtimeAuthority != null; + const runtimeRequirements = dependencies.resolveRuntimeRequirements + ? dependencies.resolveRuntimeRequirements(driverName, context, { managedGatewayOwned }) + : resolveManagedStartupRuntimeRequirements( + typeof input.sourceEntry.agent === "string" ? { name: input.sourceEntry.agent } : null, + driverName, + { managedGatewayOwned }, + ); + + const lifecycle: SandboxCreateRuntimeLifecycleContext = { + ...input.lifecycle, + ...runtimeRequirements, + sandboxGpuEnabled: false, + }; + const request: SandboxCreateRuntimePatchRequest = { + driverName, + lifecycle, + runtimeAuthority, + }; + if (dependencies.createRuntimePatch) return dependencies.createRuntimePatch(request); + const dockerPatch = driverName === "docker" ? input.createDockerPatch?.(lifecycle) : undefined; + return createSandboxCreateRuntimePatch( + request, + currentSandboxCreateRuntimePatchAdapters(dockerPatch), + ); +} + +export function runAuthorizedManagedSnapshotDestinationDelete( + runtimePatch: SandboxCreateRuntimePatch | null, + deleteDestination: () => void, +): void { + runtimePatch?.revalidateBeforeMutation(); + deleteDestination(); +} diff --git a/src/lib/actions/sandbox/start.ts b/src/lib/actions/sandbox/start.ts index 0f6b05884b..cb4e14d0c7 100644 --- a/src/lib/actions/sandbox/start.ts +++ b/src/lib/actions/sandbox/start.ts @@ -1,22 +1,17 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { CLI_NAME } from "../../cli/branding"; -import { - findLabeledSandboxContainers, - recoverDockerDriverSandbox, -} from "../../onboard/docker-driver-sandbox-recovery"; import * as registry from "../../state/registry"; import { - gateDirectDriverLifecycle, - gateDockerRuntimeUp, - type SandboxLifecycleResult, - type SandboxStopDeps, -} from "./stop"; + resolveSandboxLifecycleRuntimeAdapter, + resolveSandboxLifecycleRuntimeDependencies, + type SandboxLifecycleRuntimeAdapterRegistry, + type SandboxLifecycleRuntimeDependencies, +} from "./runtime/lifecycle-runtime"; +import { gateDirectDriverLifecycle, type SandboxLifecycleResult } from "./stop"; -// Lazy requires keep the heavy connect module (and the docker adapter's -// transitive imports) out of this module's load path; tests inject -// `deps.probeSandbox` / `deps.dockerUnpause`. +// Lazy require keeps the heavy connect module out of this module's load path; +// tests inject `deps.probeSandbox`. function loadConnectProbe(): (sandboxName: string) => Promise { const { connectSandbox } = require("./connect") as { connectSandbox: (sandboxName: string, options?: { probeOnly?: boolean }) => Promise; @@ -24,29 +19,10 @@ function loadConnectProbe(): (sandboxName: string) => Promise { return (sandboxName) => connectSandbox(sandboxName, { probeOnly: true }); } -type DockerOpResult = { status?: number | null }; -type DockerUnpauseFn = (name: string, opts?: Record) => DockerOpResult; - -function loadDockerUnpause(): DockerUnpauseFn { - return (require("../../adapters/docker") as { dockerUnpause: DockerUnpauseFn }).dockerUnpause; -} - -const DOCKER_UNPAUSE_TIMEOUT_MS = 30_000; - -// Paused containers report `Up N minutes (Paused)` from `docker ps`, so the -// recovery classifier counts them as running and would no-op — while -// `docker start` on them fails outright. `docker unpause` is the only verb -// that resumes them (#6026). -function isPausedStatus(status: string): boolean { - return status.startsWith("Up") && status.endsWith("(Paused)"); -} - -export interface SandboxStartDeps - extends Pick { +export interface SandboxStartDeps extends Partial { + environment?: NodeJS.ProcessEnv; getSandbox?: typeof registry.getSandbox; - findLabeledSandboxContainers?: typeof findLabeledSandboxContainers; - recoverDockerDriverSandbox?: typeof recoverDockerDriverSandbox; - dockerUnpause?: DockerUnpauseFn; + runtimeAdapters?: SandboxLifecycleRuntimeAdapterRegistry; /** Gateway/forward health probe; defaults to the `recover` action body. */ probeSandbox?: (sandboxName: string) => Promise; log?: (message: string) => void; @@ -56,10 +32,10 @@ export interface SandboxStartDeps * Restart a stopped sandbox container and bring its gateway and host * forwards back up (#6026). Counterpart to `stopSandbox`. * - * Container restart reuses the #4423 recovery module (handles the stopped - * original and the gpu-backup-sibling rename) plus a paused-container - * unpause branch; the health probe reuses the `recover` action body so - * forwards and the in-sandbox gateway come back exactly as they would after + * The selected compute-runtime adapter restores the exact container (the + * Docker adapter retains #4423 backup-sibling recovery), then the shared + * health probe reuses the `recover` action body so forwards and the + * in-sandbox gateway come back exactly as they would after * `nemoclaw recover`. */ export async function startSandbox( @@ -67,51 +43,34 @@ export async function startSandbox( deps: SandboxStartDeps = {}, ): Promise { const log = deps.log ?? console.log; + const getSandbox = deps.getSandbox ?? registry.getSandbox; + const sandbox = getSandbox(sandboxName); - const gate = gateDirectDriverLifecycle( - sandboxName, - "start", - deps.getSandbox ?? registry.getSandbox, - ); + const gate = gateDirectDriverLifecycle(sandboxName, "start", () => sandbox, deps.runtimeAdapters); if (gate) return gate; - const runtimeGate = gateDockerRuntimeUp(sandboxName, "start", deps); - if (runtimeGate) return runtimeGate; - - const containers = (deps.findLabeledSandboxContainers ?? findLabeledSandboxContainers)( - sandboxName, + const adapter = resolveSandboxLifecycleRuntimeAdapter( + sandbox?.openshellDriver, + deps.runtimeAdapters, ); - const paused = containers.find((container) => isPausedStatus(container.status)); - if (paused) { - const dockerUnpause = deps.dockerUnpause ?? ((name, opts) => loadDockerUnpause()(name, opts)); - const result = dockerUnpause(paused.name, { - ignoreError: true, - timeout: DOCKER_UNPAUSE_TIMEOUT_MS, - }); - if (result.status !== 0) { - return { - exitCode: 1, - message: ` docker unpause ${paused.name} failed (exit ${result.status ?? "unknown"}).`, - }; - } - log(` Container '${paused.name}' unpaused.`); - } else { - const recovery = (deps.recoverDockerDriverSandbox ?? recoverDockerDriverSandbox)(sandboxName); - if (!recovery.recovered) { - return { - exitCode: 1, - message: - ` Could not start sandbox '${sandboxName}': ${recovery.detail ?? "unknown failure"}. ` + - `If the container was removed, run '${CLI_NAME} ${sandboxName} rebuild' to recreate it.`, - }; - } - - if (recovery.via === "started-running-original") { - log(` Sandbox '${sandboxName}' is already running.`); - } else { - log(` Container '${recovery.containerName ?? sandboxName}' started.`); - } + if (!adapter || !sandbox) { + return { + exitCode: 1, + message: ` No lifecycle runtime adapter is available for sandbox '${sandboxName}'.`, + }; } + const runtimeDeps = resolveSandboxLifecycleRuntimeDependencies(deps); + const input = { + environment: deps.environment ?? process.env, + log, + sandbox, + sandboxName, + }; + const runtimeGate = adapter.preflight("start", input, runtimeDeps); + if (runtimeGate) return runtimeGate; + + const result = adapter.start(input, runtimeDeps); + if (result.exitCode !== 0) return result; log(" Checking gateway health and host forwards…"); await (deps.probeSandbox ?? loadConnectProbe())(sandboxName); diff --git a/src/lib/actions/sandbox/stop.test.ts b/src/lib/actions/sandbox/stop.test.ts index 77ec0b8519..d5b9f11d16 100644 --- a/src/lib/actions/sandbox/stop.test.ts +++ b/src/lib/actions/sandbox/stop.test.ts @@ -138,7 +138,11 @@ describe("stopSandbox", () => { expect(result.exitCode).toBe(0); expect(h.stopSandboxChannels).toHaveBeenCalledWith( "my-sandbox", - expect.objectContaining({ info: expect.any(Function), warn: expect.any(Function) }), + expect.objectContaining({ + allowDockerGatewayExec: true, + info: expect.any(Function), + warn: expect.any(Function), + }), ); expect(h.dockerStop).toHaveBeenCalledWith("openshell-my-sandbox", { ignoreError: true, diff --git a/src/lib/actions/sandbox/stop.ts b/src/lib/actions/sandbox/stop.ts index 53cc1489ec..24db730a9a 100644 --- a/src/lib/actions/sandbox/stop.ts +++ b/src/lib/actions/sandbox/stop.ts @@ -2,32 +2,18 @@ // SPDX-License-Identifier: Apache-2.0 import { CLI_NAME } from "../../cli/branding"; -import { findLabeledSandboxContainers } from "../../onboard/docker-driver-sandbox-recovery"; import * as registry from "../../state/registry"; import { stopSandboxChannels } from "../../tunnel/sandbox-gateway-stop"; import { teardownSandboxDashboardForward } from "./forward-recovery"; import { isDockerRuntimeDown, printDockerRuntimeDownGuidance } from "./gateway-failure-classifier"; - -// Lazy adapter accessor, same pattern as docker-driver-sandbox-recovery.ts: -// tests inject `deps.dockerStop` so the lazy require never fires. -type DockerOpResult = { status?: number | null }; -type DockerStopFn = (name: string, opts?: Record) => DockerOpResult; - -function loadDockerStop(): DockerStopFn { - return (require("../../adapters/docker") as { dockerStop: DockerStopFn }).dockerStop; -} - -const DOCKER_STOP_TIMEOUT_MS = 30_000; - -// Docker `Status` strings for containers that hold no resources: nothing to -// stop. Everything else — `Up`, `Up … (Paused)`, and crash-looping -// `Restarting (N) …` — is stoppable; `docker stop` also disarms an armed -// restart policy, which is exactly how a crash loop is silenced (#6026). -const AT_REST_STATUS_PREFIXES = ["Exited", "Created", "Dead"] as const; - -function isAtRest(status: string): boolean { - return AT_REST_STATUS_PREFIXES.some((prefix) => status.startsWith(prefix)); -} +import { + CURRENT_SANDBOX_LIFECYCLE_RUNTIME_ADAPTERS, + resolveSandboxLifecycleRuntimeAdapter, + resolveSandboxLifecycleRuntimeDependencies, + type SandboxLifecycleResult, + type SandboxLifecycleRuntimeAdapterRegistry, + type SandboxLifecycleRuntimeDependencies, +} from "./runtime/lifecycle-runtime"; function teardownDashboardForwardBestEffort( sandboxName: string, @@ -42,19 +28,14 @@ function teardownDashboardForwardBestEffort( } } -export type SandboxLifecycleResult = { - exitCode: number; - message?: string; -}; +export type { SandboxLifecycleResult } from "./runtime/lifecycle-runtime"; -export interface SandboxStopDeps { +export interface SandboxStopDeps extends Partial { + environment?: NodeJS.ProcessEnv; getSandbox?: typeof registry.getSandbox; - isDockerRuntimeDown?: typeof isDockerRuntimeDown; - printDockerRuntimeDownGuidance?: typeof printDockerRuntimeDownGuidance; - findLabeledSandboxContainers?: typeof findLabeledSandboxContainers; + runtimeAdapters?: SandboxLifecycleRuntimeAdapterRegistry; stopSandboxChannels?: typeof stopSandboxChannels; teardownSandboxDashboardForward?: typeof teardownSandboxDashboardForward; - dockerStop?: DockerStopFn; log?: (message: string) => void; warn?: (message: string) => void; } @@ -64,14 +45,15 @@ function normalizeDriver(driver: unknown): string | null { } /** - * Refuse lifecycle control for sandboxes we cannot reach through the local - * Docker daemon. Mirrors the gate `privilegedSandboxExecArgv` applies before - * direct-container mutation (src/lib/sandbox/privileged-exec.ts). + * Refuse direct lifecycle mutation unless the persisted compute driver has an + * exact registered adapter. A future runtime never inherits Docker or Podman + * behavior merely because NemoClaw launches its gateway. */ export function gateDirectDriverLifecycle( sandboxName: string, action: "stop" | "start", getSandbox: typeof registry.getSandbox, + adapters: SandboxLifecycleRuntimeAdapterRegistry = CURRENT_SANDBOX_LIFECYCLE_RUNTIME_ADAPTERS, ): SandboxLifecycleResult | null { const entry = getSandbox(sandboxName); if (!entry) { @@ -83,12 +65,12 @@ export function gateDirectDriverLifecycle( }; } const driver = normalizeDriver(entry.openshellDriver); - if (driver !== null && driver !== "docker" && driver !== "vm") { + if (!resolveSandboxLifecycleRuntimeAdapter(driver, adapters)) { return { exitCode: 1, message: - ` '${CLI_NAME} ${sandboxName} ${action}' controls the local Docker container ` + - `directly and is unavailable for driver '${driver}'.`, + ` '${CLI_NAME} ${sandboxName} ${action}' has no registered local lifecycle ` + + `adapter for driver '${driver}'.`, }; } return null; @@ -114,11 +96,11 @@ export function gateDockerRuntimeUp( } /** - * Stop a sandbox's Docker container while preserving every piece of state + * Stop a sandbox's runtime container while preserving every piece of state * destroy would remove: the workspace volume, registry entry, OpenShell * sandbox record, credentials, and images all stay in place (#6026). * - * The shared host gateway, tunnel, and any NIM inference container are + * The shared host gateway, tunnel, and any inference service are * gateway-scoped and serve other sandboxes — deliberately untouched. */ export function stopSandbox( @@ -127,31 +109,57 @@ export function stopSandbox( ): SandboxLifecycleResult { const log = deps.log ?? console.log; const warn = deps.warn ?? console.warn; + const getSandbox = deps.getSandbox ?? registry.getSandbox; + const sandbox = getSandbox(sandboxName); - const gate = gateDirectDriverLifecycle( - sandboxName, - "stop", - deps.getSandbox ?? registry.getSandbox, - ); + const gate = gateDirectDriverLifecycle(sandboxName, "stop", () => sandbox, deps.runtimeAdapters); if (gate) return gate; - const runtimeGate = gateDockerRuntimeUp(sandboxName, "stop", deps); - if (runtimeGate) return runtimeGate; - - const containers = (deps.findLabeledSandboxContainers ?? findLabeledSandboxContainers)( - sandboxName, + const adapter = resolveSandboxLifecycleRuntimeAdapter( + sandbox?.openshellDriver, + deps.runtimeAdapters, ); - if (containers.length === 0) { + if (!adapter || !sandbox) { return { exitCode: 1, - message: - ` No Docker container found for sandbox '${sandboxName}'. ` + - `If the container was removed, run '${CLI_NAME} ${sandboxName} rebuild' to recreate it.`, + message: ` No lifecycle runtime adapter is available for sandbox '${sandboxName}'.`, }; } + const runtimeDeps = resolveSandboxLifecycleRuntimeDependencies(deps); + const input = { + environment: deps.environment ?? process.env, + log, + sandbox, + sandboxName, + }; + const runtimeGate = adapter.preflight("stop", input, runtimeDeps); + if (runtimeGate) return runtimeGate; - const stoppable = containers.filter((container) => !isAtRest(container.status)); - if (stoppable.length === 0) { + let channelsStopped = false; + const outcome = adapter.stop(input, runtimeDeps, { + beforeStop() { + if (channelsStopped) return; + channelsStopped = true; + // Graceful in-sandbox gateway shutdown first, so channels disconnect + // cleanly instead of dying with the container's SIGTERM. Best-effort: + // a stop must still free resources when the gateway is unreachable. + // Agent-managed gateways (e.g. Hermes) are supervised inside the sandbox + // and shut down with the container's stop signal instead. + try { + (deps.stopSandboxChannels ?? stopSandboxChannels)(sandboxName, { + allowDockerGatewayExec: adapter.channelStopTransport === "docker-kubectl-first", + info: (message) => log(` ${message}`), + warn: (message) => warn(` ${message}`), + }); + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + warn(` Warning: could not stop in-sandbox channels gracefully: ${detail}`); + } + }, + }); + if (outcome.exitCode !== 0) return outcome; + + if (outcome.state === "already-stopped") { log(` Sandbox '${sandboxName}' is already stopped.`); // Idempotent teardown: an earlier stop may have left the dashboard forward // alive (e.g. openshell was unreachable then, or the forward was orphaned by @@ -166,45 +174,6 @@ export function stopSandbox( return { exitCode: 0 }; } - // Graceful in-sandbox gateway shutdown first, so channels disconnect - // cleanly instead of dying with the container's SIGTERM. Best-effort: - // a stop must still free resources when the gateway is unreachable. - // Agent-managed gateways (e.g. Hermes) are supervised inside the sandbox - // and shut down with the container's stop signal instead. - try { - (deps.stopSandboxChannels ?? stopSandboxChannels)(sandboxName, { - info: (message) => log(` ${message}`), - warn: (message) => warn(` ${message}`), - }); - } catch (error) { - const detail = error instanceof Error ? error.message : String(error); - warn(` Warning: could not stop in-sandbox channels gracefully: ${detail}`); - } - - // Attempt every stoppable container even if one fails: a sandbox can have - // more than one labeled container (e.g. a gpu-backup sibling), and aborting - // on the first failure would leave the rest running — the opposite of what - // stop is for. Collect failures and report them together. - const dockerStop = deps.dockerStop ?? ((name, opts) => loadDockerStop()(name, opts)); - const failures: string[] = []; - for (const container of stoppable) { - log(` Stopping container '${container.name}'…`); - const result = dockerStop(container.name, { - ignoreError: true, - timeout: DOCKER_STOP_TIMEOUT_MS, - }); - if (result.status !== 0) { - failures.push(`${container.name} (exit ${result.status ?? "unknown"})`); - } - } - - if (failures.length > 0) { - return { - exitCode: 1, - message: ` docker stop failed for: ${failures.join(", ")}.`, - }; - } - // Release the host-side dashboard port-forward this sandbox created. Without // this, the `ssh -L` listener stays alive after the container is stopped, so // `status` misreports the cleanly-stopped sandbox as a foreign diff --git a/src/lib/adapters/container-engine.ts b/src/lib/adapters/container-engine.ts new file mode 100644 index 0000000000..2467edb19c --- /dev/null +++ b/src/lib/adapters/container-engine.ts @@ -0,0 +1,106 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * Host container-engine command selected for the current operation. + * + * The existing Docker adapter surface is intentionally retained for callers + * while this command seam keeps the executable, endpoint, argument + * translation, and authority proof runtime-pluggable. Docker remains the + * default. Native Podman onboarding installs a scoped override and restores it + * when the onboarding transaction ends. + */ +export interface HostContainerEngineCommand { + readonly driverName: string; + readonly executable: string; + readonly prefixArgs?: readonly string[]; + /** OCI architecture proved by the selected runtime, when runtime-owned. */ + readonly runtimeArchitecture?: "amd64" | "arm64"; + /** Exact sandbox network used for host-service route probes. */ + readonly sandboxNetworkName?: string; + /** Runtime-native target accepted by `--add-host :`. */ + readonly hostGatewayTarget?: string; + readonly assertAuthority?: () => void; + readonly translateArgs?: (args: readonly string[]) => readonly string[]; +} + +const DEFAULT_DOCKER_COMMAND: HostContainerEngineCommand = Object.freeze({ + driverName: "docker", + executable: "docker", +}); + +let activeCommand: HostContainerEngineCommand = DEFAULT_DOCKER_COMMAND; + +function safeCommandToken(value: string, label: string): string { + if (!value || value.includes("\0") || /[\r\n]/u.test(value)) { + throw new Error(`${label} must be a non-empty command token`); + } + return value; +} + +function validateCommand(command: HostContainerEngineCommand): HostContainerEngineCommand { + safeCommandToken(command.driverName, "Container-engine driver name"); + safeCommandToken(command.executable, "Container-engine executable"); + const prefixArgs = [...(command.prefixArgs ?? [])]; + for (const [index, value] of prefixArgs.entries()) { + safeCommandToken(value, `Container-engine prefix argument ${String(index)}`); + } + if (command.sandboxNetworkName !== undefined) { + safeCommandToken(command.sandboxNetworkName, "Container-engine sandbox network name"); + } + if (command.hostGatewayTarget !== undefined) { + safeCommandToken(command.hostGatewayTarget, "Container-engine host-gateway target"); + } + return Object.freeze({ + ...command, + ...(prefixArgs.length > 0 ? { prefixArgs: Object.freeze(prefixArgs) } : {}), + }); +} + +export function currentHostContainerEngineCommand(): HostContainerEngineCommand { + return activeCommand; +} + +export function hostContainerEngineDisplayName(): string { + return activeCommand.driverName === "podman" ? "Podman" : "Docker"; +} + +export function hostContainerEngineExecutable(): string { + return activeCommand.executable; +} + +export function hostContainerEngineArgv(args: readonly string[]): string[] { + const command = activeCommand; + command.assertAuthority?.(); + const translated = command.translateArgs?.(args) ?? args; + return [ + safeCommandToken(command.executable, "Container-engine executable"), + ...(command.prefixArgs ?? []).map((value, index) => + safeCommandToken(value, `Container-engine prefix argument ${String(index)}`), + ), + ...translated.map((value, index) => + safeCommandToken(String(value), `Container-engine argument ${String(index)}`), + ), + ]; +} + +/** + * Install one process-scoped command override and return an exact restoration + * closure. NemoClaw's CLI executes one onboarding transaction per process, so + * the override cannot cross concurrent user requests; tests restore it in + * `finally`/`afterEach`. + */ +export function configureHostContainerEngine(command: HostContainerEngineCommand): () => void { + const previous = activeCommand; + activeCommand = validateCommand(command); + let restored = false; + return () => { + if (restored) return; + restored = true; + activeCommand = previous; + }; +} + +export function resetHostContainerEngineForTests(): void { + activeCommand = DEFAULT_DOCKER_COMMAND; +} diff --git a/src/lib/adapters/docker/container.ts b/src/lib/adapters/docker/container.ts index 9dba35695a..9bc236dc63 100644 --- a/src/lib/adapters/docker/container.ts +++ b/src/lib/adapters/docker/container.ts @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { hostContainerEngineArgv } from "../container-engine"; import { type DockerCaptureOptions, type DockerRunOptions, @@ -74,5 +75,5 @@ export function dockerPort( } export function dockerExecArgv(containerName: string, cmd: readonly string[]): string[] { - return ["docker", "exec", containerName, ...cmd]; + return hostContainerEngineArgv(["exec", containerName, ...cmd]); } diff --git a/src/lib/adapters/docker/exec.ts b/src/lib/adapters/docker/exec.ts index ea44e8392a..a17022673c 100644 --- a/src/lib/adapters/docker/exec.ts +++ b/src/lib/adapters/docker/exec.ts @@ -2,11 +2,12 @@ // SPDX-License-Identifier: Apache-2.0 import { + type ExecFileSyncOptionsWithStringEncoding, execFileSync, spawn, spawnSync, - type ExecFileSyncOptionsWithStringEncoding, } from "node:child_process"; +import { hostContainerEngineArgv } from "../container-engine"; export type DockerExecFileSyncOptions = Omit; export type DockerSpawnSyncOptions = Parameters[2]; @@ -16,19 +17,22 @@ export function dockerExecFileSync( args: readonly string[], opts: DockerExecFileSyncOptions = {}, ): string { - return String(execFileSync("docker", [...args], { encoding: "utf-8", ...opts })); + const [command, ...commandArgs] = hostContainerEngineArgv(args); + return String(execFileSync(command, commandArgs, { encoding: "utf-8", ...opts })); } export function dockerSpawnSync( args: readonly string[], opts: DockerSpawnSyncOptions = {}, ): DockerSpawnSyncResult { - return spawnSync("docker", [...args], opts); + const [command, ...commandArgs] = hostContainerEngineArgv(args); + return spawnSync(command, commandArgs, opts); } export function dockerSpawn( args: readonly string[], opts: Parameters[2] = {}, ): ReturnType { - return spawn("docker", [...args], opts); + const [command, ...commandArgs] = hostContainerEngineArgv(args); + return spawn(command, commandArgs, opts); } diff --git a/src/lib/adapters/docker/index.test.ts b/src/lib/adapters/docker/index.test.ts index a54f7774ca..243d49e080 100644 --- a/src/lib/adapters/docker/index.test.ts +++ b/src/lib/adapters/docker/index.test.ts @@ -13,6 +13,7 @@ vi.mock("../../runner", () => ({ })); import { + configureHostContainerEngine, dockerBuild, dockerContainerInspectFormat, dockerInfoFormat, @@ -24,16 +25,58 @@ import { dockerRmi, dockerRunDetached, dockerTag, + resetHostContainerEngineForTests, } from "./index"; describe("docker helpers", () => { beforeEach(() => { + resetHostContainerEngineForTests(); runMock.mockReset(); runCaptureMock.mockReset(); runMock.mockReturnValue({ status: 0, stdout: "", stderr: "" }); runCaptureMock.mockReturnValue(""); }); + it("routes legacy helper call sites through a scoped pluggable container engine", () => { + const restore = configureHostContainerEngine({ + driverName: "podman", + executable: "podman", + prefixArgs: ["--url", "unix:///run/user/1000/podman/podman.sock"], + }); + try { + dockerPull("nvcr.io/nvidia/nim:latest"); + dockerRunDetached(["--device", "nvidia.com/gpu=all", "nvcr.io/nvidia/nim:latest"]); + } finally { + restore(); + } + + expect(runMock.mock.calls).toEqual([ + [ + [ + "podman", + "--url", + "unix:///run/user/1000/podman/podman.sock", + "pull", + "nvcr.io/nvidia/nim:latest", + ], + {}, + ], + [ + [ + "podman", + "--url", + "unix:///run/user/1000/podman/podman.sock", + "run", + "-d", + "--device", + "nvidia.com/gpu=all", + "nvcr.io/nvidia/nim:latest", + ], + {}, + ], + ]); + }); + it("prefixes docker argv for pull/build/run/rmi helpers", () => { dockerPull("ghcr.io/example/image:latest"); dockerBuild("Dockerfile", "example:tag", "/tmp/build"); diff --git a/src/lib/adapters/docker/index.ts b/src/lib/adapters/docker/index.ts index fc6d4250ba..81d1fc5a64 100644 --- a/src/lib/adapters/docker/index.ts +++ b/src/lib/adapters/docker/index.ts @@ -1,13 +1,21 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -export * from "./run"; +export { + configureHostContainerEngine, + currentHostContainerEngineCommand, + type HostContainerEngineCommand, + hostContainerEngineDisplayName, + hostContainerEngineExecutable, + resetHostContainerEngineForTests, +} from "../container-engine"; +export * from "./container"; export * from "./exec"; -export * from "./pull"; +export * from "./image"; export * from "./info"; -export * from "./runtime"; export * from "./inspect"; -export * from "./image"; -export * from "./container"; -export * from "./volume"; export * from "./login"; +export * from "./pull"; +export * from "./run"; +export * from "./runtime"; +export * from "./volume"; diff --git a/src/lib/adapters/docker/pull.test.ts b/src/lib/adapters/docker/pull.test.ts index aa6066a944..65e434226e 100644 --- a/src/lib/adapters/docker/pull.test.ts +++ b/src/lib/adapters/docker/pull.test.ts @@ -60,6 +60,12 @@ describe("docker pull progress watchdog", () => { expect(dockerPullProgressSignature("b39b21d4717d: Download complete ")).toBe( "layer:b39b21d4717d:Download complete", ); + expect(dockerPullProgressSignature("Copying blob sha256:abc123def 48.1 MiB / 1.2 GiB")).toBe( + "podman:copying blob:sha256:abc123def:48.1 mib / 1.2 gib", + ); + expect(dockerPullProgressSignature("Writing manifest to image destination")).toBe( + "podman:writing manifest to image destination", + ); expect(dockerPullProgressSignature("#12 sha256:abc123 25.4MB/100MB 4.0s 10.1MB/s")).toBeNull(); }); diff --git a/src/lib/adapters/docker/pull.ts b/src/lib/adapters/docker/pull.ts index 8daa153493..a64cf0a9ad 100644 --- a/src/lib/adapters/docker/pull.ts +++ b/src/lib/adapters/docker/pull.ts @@ -6,7 +6,7 @@ import type { SpawnOptions } from "node:child_process"; import { ROOT } from "../../runner"; import { buildSubprocessEnv } from "../../subprocess-env"; import { dockerSpawn } from "./exec"; -import { dockerRun, type DockerRunOptions, type DockerRunResult } from "./run"; +import { type DockerRunOptions, type DockerRunResult, dockerRun } from "./run"; export function dockerPull(imageRef: string, opts: DockerRunOptions = {}): DockerRunResult { return dockerRun(["pull", imageRef], opts); @@ -78,9 +78,22 @@ export function dockerPullProgressSignature(line: string): string | null { if (/^Status: /.test(normalized)) { return `status:${normalized}`; } + const podmanCopy = normalized.match( + /^(Copying (?:blob|config))\s+([a-f0-9]{6,}|sha256:[a-f0-9]{6,})\s*(.*)$/iu, + ); + if (podmanCopy) { + return `podman:${podmanCopy[1].toLowerCase()}:${podmanCopy[2].toLowerCase()}:${podmanCopy[3].trim().toLowerCase()}`; + } + if ( + /^(?:Trying to pull|Getting image source signatures|Writing manifest to image destination|Storing signatures)\b/iu.test( + normalized, + ) + ) { + return `podman:${normalized.toLowerCase()}`; + } - // This vLLM watchdog intentionally recognizes observed `docker pull` - // layer/status output only; BuildKit-style build output is not pull progress. + // This vLLM watchdog intentionally recognizes observed Docker and Podman + // pull output only; BuildKit-style build output is not pull progress. return null; } diff --git a/src/lib/adapters/docker/run.ts b/src/lib/adapters/docker/run.ts index 4dcf6cc78f..e32def91f9 100644 --- a/src/lib/adapters/docker/run.ts +++ b/src/lib/adapters/docker/run.ts @@ -2,12 +2,13 @@ // SPDX-License-Identifier: Apache-2.0 import { run, runCapture } from "../../runner"; +import { hostContainerEngineArgv } from "../container-engine"; export type DockerRunOptions = Parameters[1]; export type DockerCaptureOptions = Parameters[1]; export type DockerRunResult = ReturnType; export function dockerArgv(args: readonly string[]): string[] { - return ["docker", ...args]; + return hostContainerEngineArgv(args); } export function dockerRun(args: readonly string[], opts: DockerRunOptions = {}): DockerRunResult { diff --git a/src/lib/domain/sandbox/destroy.test.ts b/src/lib/domain/sandbox/destroy.test.ts index da887b51d9..42df7cf606 100644 --- a/src/lib/domain/sandbox/destroy.test.ts +++ b/src/lib/domain/sandbox/destroy.test.ts @@ -8,6 +8,7 @@ import { getSandboxDeleteOutcome, hasNoLiveSandboxes, hasRunningDockerSandboxContainer, + hasRuntimeSandboxContainer, isGatewayUnreachableDeleteOutput, isMissingSandboxDeleteOutput, resolveDestroyGatewayCleanupDecision, @@ -140,15 +141,13 @@ describe("sandbox destroy helpers", () => { expect( hasNoLiveSandboxes({ liveList: { status: 0, output: liveListOutput }, - dockerContainersBySandboxName: new Map([["npmtest", { output: "" }]]), + runtimeContainersBySandboxName: new Map([["npmtest", { present: false }]]), }), ).toBe(true); expect( hasNoLiveSandboxes({ liveList: { status: 0, output: liveListOutput }, - dockerContainersBySandboxName: new Map([ - ["npmtest", { output: "openshell-npmtest-e487d1bd\n" }], - ]), + runtimeContainersBySandboxName: new Map([["npmtest", { present: true }]]), }), ).toBe(false); expect( @@ -158,7 +157,7 @@ describe("sandbox destroy helpers", () => { output: "NAME CREATED PHASE\nnpmtest now Ready\n", }, - dockerContainersBySandboxName: new Map([["npmtest", { output: "" }]]), + runtimeContainersBySandboxName: new Map([["npmtest", { present: false }]]), }), ).toBe(false); }); @@ -175,7 +174,9 @@ describe("sandbox destroy helpers", () => { output: "NAME CREATED PHASE\nnpmtest now Failed\n", }, - dockerContainersBySandboxName: new Map([["npmtest", { output: "", probeFailed: true }]]), + runtimeContainersBySandboxName: new Map([ + ["npmtest", { present: false, probeFailed: true }], + ]), }), ).toBe(false); }); @@ -184,7 +185,7 @@ describe("sandbox destroy helpers", () => { expect( hasNoLiveSandboxes({ liveList: { status: 1, output: "" }, - dockerContainersBySandboxName: new Map(), + runtimeContainersBySandboxName: new Map(), }), ).toBe(false); }); @@ -209,4 +210,11 @@ describe("sandbox destroy helpers", () => { ), ).toBe(false); }); + + it("treats missing and failed pluggable-runtime evidence as live", () => { + expect(hasRuntimeSandboxContainer(undefined)).toBe(true); + expect(hasRuntimeSandboxContainer({ present: false, probeFailed: true })).toBe(true); + expect(hasRuntimeSandboxContainer({ present: true })).toBe(true); + expect(hasRuntimeSandboxContainer({ present: false })).toBe(false); + }); }); diff --git a/src/lib/domain/sandbox/destroy.ts b/src/lib/domain/sandbox/destroy.ts index 4f02020c7e..d999bc62b0 100644 --- a/src/lib/domain/sandbox/destroy.ts +++ b/src/lib/domain/sandbox/destroy.ts @@ -39,9 +39,14 @@ export type DockerSandboxContainerSnapshot = { probeFailed?: boolean; }; +export type SandboxRuntimeContainerSnapshot = { + present: boolean; + probeFailed?: boolean; +}; + export type LiveSandboxProbeSnapshot = { liveList: LiveSandboxListSnapshot; - dockerContainersBySandboxName: ReadonlyMap; + runtimeContainersBySandboxName: ReadonlyMap; }; export function isMissingSandboxDeleteOutput(output = ""): boolean { @@ -170,6 +175,14 @@ export function hasRunningDockerSandboxContainer( ); } +export function hasRuntimeSandboxContainer( + snapshot: SandboxRuntimeContainerSnapshot | undefined, +): boolean { + // Missing or failed runtime evidence cannot prove absence. Preserve the + // shared gateway until the driver-specific action-layer probe succeeds. + return !snapshot || snapshot.probeFailed === true || snapshot.present; +} + export function getLiveSandboxNames(liveList: LiveSandboxListSnapshot): string[] { if (liveList.status !== 0) { return []; @@ -179,7 +192,7 @@ export function getLiveSandboxNames(liveList: LiveSandboxListSnapshot): string[] export function hasNoLiveSandboxes({ liveList, - dockerContainersBySandboxName, + runtimeContainersBySandboxName, }: LiveSandboxProbeSnapshot): boolean { // Fail closed: if OpenShell cannot report authoritative sandbox state, // preserve the shared gateway so a sandbox never loses its listener. @@ -187,13 +200,8 @@ export function hasNoLiveSandboxes({ return false; } const entries = parseLiveSandboxEntries(liveList.output); - const sandboxNames = entries.map((entry) => entry.name); return entries.every((entry) => { if (!TERMINAL_OPEN_SHELL_SANDBOX_PHASES.has(entry.phase ?? "")) return false; - return !hasRunningDockerSandboxContainer( - entry.name, - dockerContainersBySandboxName.get(entry.name), - sandboxNames, - ); + return !hasRuntimeSandboxContainer(runtimeContainersBySandboxName.get(entry.name)); }); } diff --git a/src/lib/gateway-runtime-action.test.ts b/src/lib/gateway-runtime-action.test.ts index 4af0e8a79a..d341d88908 100644 --- a/src/lib/gateway-runtime-action.test.ts +++ b/src/lib/gateway-runtime-action.test.ts @@ -13,6 +13,19 @@ describe("gateway-runtime-action per-sandbox gateway routing", () => { beforeEach(() => { captureSpy = vi.spyOn(gatewayRuntime.gatewayRuntimeDependencies, "captureOpenshell"); runSpy = vi.spyOn(gatewayRuntime.gatewayRuntimeDependencies, "runOpenshell"); + vi.spyOn( + gatewayRuntime.gatewayRuntimeDependencies, + "listSandboxDriverBindings", + ).mockReturnValue([ + { name: "default-agent", gatewayName: "nemoclaw", openshellDriver: "docker" }, + { name: "port-agent", gatewayPort: 8090, openshellDriver: "docker" }, + ]); + vi.spyOn(gatewayRuntime.gatewayRuntimeDependencies, "legacyComputeDriverName").mockReturnValue( + "docker", + ); + vi.spyOn(gatewayRuntime.gatewayRuntimeDependencies, "readGatewayRuntimeDriver").mockReturnValue( + null, + ); startGatewaySpy = vi .spyOn(gatewayRuntime.gatewayRuntimeDependencies, "startGatewayForRecovery") .mockResolvedValue(undefined as never); @@ -224,6 +237,7 @@ describe("gateway-runtime-action per-sandbox gateway routing", () => { }); expect(startGatewaySpy).toHaveBeenCalledWith({ + computeDriver: "docker", gatewayName: "nemoclaw-8090", gatewayPort: 8090, }); @@ -231,5 +245,97 @@ describe("gateway-runtime-action per-sandbox gateway routing", () => { expect(result.via).toBe("start"); expect(process.env.OPENSHELL_GATEWAY).toBe("nemoclaw-8090"); }); + + it("forwards the durable Podman driver when a missing gateway must be started", async () => { + vi.mocked( + gatewayRuntime.gatewayRuntimeDependencies.listSandboxDriverBindings, + ).mockReturnValue([{ name: "podman-agent", gatewayPort: 8090, openshellDriver: "podman" }]); + captureSpy + .mockReturnValueOnce({ status: 0, output: "Status: Disconnected\nGateway: nemoclaw\n" }) + .mockReturnValueOnce({ status: 0, output: "" }) + .mockReturnValueOnce({ status: 0, output: "Status: Disconnected\nGateway: nemoclaw\n" }) + .mockReturnValueOnce({ status: 0, output: "" }) + .mockReturnValueOnce({ + status: 0, + output: "Status: Connected\nGateway: nemoclaw-8090\n", + }) + .mockReturnValueOnce({ status: 0, output: "Gateway: nemoclaw-8090\n" }); + runSpy.mockReturnValue({ status: 0 } as never); + + const result = await gatewayRuntime.recoverNamedGatewayRuntime({ + gatewayName: "nemoclaw-8090", + }); + + expect(startGatewaySpy).toHaveBeenCalledWith({ + computeDriver: "podman", + gatewayName: "nemoclaw-8090", + gatewayPort: 8090, + }); + expect(result.recovered).toBe(true); + }); + + it("does not start a gateway without durable driver evidence", async () => { + vi.mocked( + gatewayRuntime.gatewayRuntimeDependencies.listSandboxDriverBindings, + ).mockReturnValue([]); + captureSpy.mockReturnValue({ status: 0, output: "Status: Disconnected\nGateway: other\n" }); + runSpy.mockReturnValue({ status: 0 } as never); + + const result = await gatewayRuntime.recoverNamedGatewayRuntime({ + gatewayName: "nemoclaw-8090", + }); + + expect(startGatewaySpy).not.toHaveBeenCalled(); + expect(result.recovered).toBe(false); + }); + + it("uses a durable gateway-runtime binding when no sandbox row remains", async () => { + vi.mocked( + gatewayRuntime.gatewayRuntimeDependencies.listSandboxDriverBindings, + ).mockReturnValue([]); + vi.mocked(gatewayRuntime.gatewayRuntimeDependencies.readGatewayRuntimeDriver).mockReturnValue( + "podman", + ); + captureSpy + .mockReturnValueOnce({ status: 0, output: "Status: Disconnected\nGateway: nemoclaw\n" }) + .mockReturnValueOnce({ status: 0, output: "" }) + .mockReturnValueOnce({ status: 0, output: "Status: Disconnected\nGateway: nemoclaw\n" }) + .mockReturnValueOnce({ status: 0, output: "" }) + .mockReturnValueOnce({ + status: 0, + output: "Status: Connected\nGateway: nemoclaw-8090\n", + }) + .mockReturnValueOnce({ status: 0, output: "Gateway: nemoclaw-8090\n" }); + runSpy.mockReturnValue({ status: 0 } as never); + + const result = await gatewayRuntime.recoverNamedGatewayRuntime({ + gatewayName: "nemoclaw-8090", + }); + + expect(startGatewaySpy).toHaveBeenCalledWith({ + computeDriver: "podman", + gatewayName: "nemoclaw-8090", + gatewayPort: 8090, + }); + expect(result.recovered).toBe(true); + }); + + it("does not start a gateway with conflicting durable driver evidence", async () => { + vi.mocked( + gatewayRuntime.gatewayRuntimeDependencies.listSandboxDriverBindings, + ).mockReturnValue([ + { name: "docker-agent", gatewayPort: 8090, openshellDriver: "docker" }, + { name: "podman-agent", gatewayPort: 8090, openshellDriver: "podman" }, + ]); + captureSpy.mockReturnValue({ status: 0, output: "Status: Disconnected\nGateway: other\n" }); + runSpy.mockReturnValue({ status: 0 } as never); + + const result = await gatewayRuntime.recoverNamedGatewayRuntime({ + gatewayName: "nemoclaw-8090", + }); + + expect(startGatewaySpy).not.toHaveBeenCalled(); + expect(result.recovered).toBe(false); + }); }); }); diff --git a/src/lib/gateway-runtime-action.ts b/src/lib/gateway-runtime-action.ts index 453973d999..cd7783704c 100644 --- a/src/lib/gateway-runtime-action.ts +++ b/src/lib/gateway-runtime-action.ts @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { listSandboxGatewayComputeBindings } from "./actions/sandbox/gateway-target"; import { stripAnsi } from "./adapters/openshell/client"; import * as openshellRuntime from "./adapters/openshell/runtime"; import { @@ -8,9 +9,17 @@ import { OPENSHELL_PROBE_TIMEOUT_MS, } from "./adapters/openshell/timeouts"; import { GATEWAY_PORT } from "./core/ports"; -import { resolveGatewayName, resolveGatewayPortFromName } from "./onboard/gateway-binding"; +import { resolveCurrentOpenShellComputePlan } from "./onboard/compute/plan"; +import { readManagedGatewayRuntimeBinding } from "./onboard/docker-driver-gateway-config"; +import { + resolveGatewayComputeDriverBinding, + resolveGatewayName, + resolveGatewayPortFromName, + resolveManagedGatewayStateDirectory, +} from "./onboard/gateway-binding"; type StartGatewayForRecoveryOptions = { + computeDriver?: string; gatewayName?: string; gatewayPort?: number; }; @@ -35,6 +44,18 @@ export const gatewayRuntimeDependencies = { const onboard = (await import("./onboard")) as unknown as LegacyOnboardModule; return onboard.startGatewayForRecovery(options); }, + listSandboxDriverBindings() { + return listSandboxGatewayComputeBindings(); + }, + legacyComputeDriverName() { + return resolveCurrentOpenShellComputePlan().driverName; + }, + readGatewayRuntimeDriver(gatewayName: string) { + return ( + readManagedGatewayRuntimeBinding(resolveManagedGatewayStateDirectory(gatewayName)) + ?.driverName ?? null + ); + }, }; /** Whether `gateway info` output names the given NemoClaw gateway. */ @@ -139,10 +160,42 @@ export function getNamedGatewayLifecycleState( type NamedGatewayLifecycleStateName = ReturnType["state"]; export type RecoverNamedGatewayRuntimeOptions = { + computeDriver?: string; recoverableStates?: readonly NamedGatewayLifecycleStateName[]; gatewayName?: string; }; +export function resolveGatewayRecoveryComputeDriver(options: { + computeDriver?: string; + gatewayName: string; +}): string { + const requested = options.computeDriver?.trim() || null; + const registryDriver = resolveGatewayComputeDriverBinding({ + gatewayName: options.gatewayName, + sandboxes: gatewayRuntimeDependencies.listSandboxDriverBindings(), + legacyDriverName: gatewayRuntimeDependencies.legacyComputeDriverName(), + }); + const runtimeDriver = gatewayRuntimeDependencies.readGatewayRuntimeDriver(options.gatewayName); + if (registryDriver && runtimeDriver && registryDriver !== runtimeDriver) { + throw new Error( + `Gateway '${options.gatewayName}' has conflicting registry and runtime drivers '${registryDriver}' and '${runtimeDriver}'.`, + ); + } + const persisted = registryDriver ?? runtimeDriver; + if (requested && persisted && requested !== persisted) { + throw new Error( + `Requested OpenShell compute driver '${requested}' does not match gateway '${options.gatewayName}' driver '${persisted}'.`, + ); + } + const driverName = requested ?? persisted; + if (!driverName) { + throw new Error( + `No durable OpenShell compute driver is bound to gateway '${options.gatewayName}'; refusing ambiguous recovery.`, + ); + } + return driverName; +} + /** Attempt to recover the named NemoClaw gateway after a restart or connectivity loss. */ export async function recoverNamedGatewayRuntime(options: RecoverNamedGatewayRuntimeOptions = {}) { const gatewayName = options.gatewayName ?? resolveGatewayName(GATEWAY_PORT); @@ -181,7 +234,12 @@ export async function recoverNamedGatewayRuntime(options: RecoverNamedGatewayRun if (shouldStartGateway) { try { + const computeDriver = resolveGatewayRecoveryComputeDriver({ + computeDriver: options.computeDriver, + gatewayName, + }); await gatewayRuntimeDependencies.startGatewayForRecovery({ + computeDriver, gatewayName, gatewayPort: resolveGatewayPortFromName(gatewayName) ?? undefined, }); diff --git a/src/lib/inference/nim.ts b/src/lib/inference/nim.ts index 852bd500da..bf225562c3 100644 --- a/src/lib/inference/nim.ts +++ b/src/lib/inference/nim.ts @@ -6,6 +6,11 @@ import fs from "node:fs"; import nimImages from "../../../bin/lib/nim-images.json"; import { + currentHostContainerEngineCommand, + hostContainerEngineDisplayName, +} from "../adapters/container-engine"; +import { + dockerCapture, dockerContainerInspectFormat, dockerForceRm, dockerLoginPasswordStdin, @@ -687,6 +692,14 @@ export function detectGpu(deps: DetectGpuDeps = {}): GpuDetection | null { // leaves an empty marker entry { "nvcr.io": {} } in auths after a successful // login. That marker plus a global credsStore is treated as logged in. export function isNgcLoggedIn(): boolean { + if (currentHostContainerEngineCommand().driverName === "podman") { + return Boolean( + dockerCapture(["login", "--get-login", "nvcr.io"], { + ignoreError: true, + timeout: NIM_STATUS_PROBE_TIMEOUT_MS, + }).trim(), + ); + } try { const os = require("os"); const fs = require("fs"); @@ -708,12 +721,13 @@ export function isNgcLoggedIn(): boolean { // NGC expects literal "$oauthtoken" as the username for API key authentication. export function dockerLoginNgc(apiKey: string): boolean { const result = dockerLoginPasswordStdin("nvcr.io", "$oauthtoken", apiKey); + const engine = hostContainerEngineDisplayName(); if (result.error) { - console.error(` Docker error: ${result.error.message}`); + console.error(` ${engine} error: ${result.error.message}`); return false; } if (result.status !== 0 && result.stderr) { - console.error(` Docker login error: ${String(result.stderr).trim()}`); + console.error(` ${engine} login error: ${String(result.stderr).trim()}`); } return result.status === 0; } diff --git a/src/lib/inference/vllm.test.ts b/src/lib/inference/vllm.test.ts index 42d7cc3b5b..07306ef5be 100644 --- a/src/lib/inference/vllm.test.ts +++ b/src/lib/inference/vllm.test.ts @@ -53,6 +53,10 @@ vi.mock("./vllm-storage", async (importOriginal) => { }; }); +import { + configureHostContainerEngine, + resetHostContainerEngineForTests, +} from "../adapters/container-engine"; import { currentPhaseActivityLabel } from "../core/phase-activity"; import { assertVllmRegistryDigestRef, @@ -95,6 +99,10 @@ beforeEach(() => { }); }); +afterEach(() => { + resetHostContainerEngineForTests(); +}); + function currentHostIdentity(): string | null { const uid = process.getuid?.(); const gid = process.getgid?.(); @@ -622,6 +630,75 @@ describe("managed vLLM ownership", () => { }); expect(isNemoClawManagedVllmRunning()).toBe(false); }); + + it("uses Podman JSON ownership inspection without Docker-only templates", () => { + const assertAuthority = vi.fn(); + configureHostContainerEngine({ + assertAuthority, + driverName: "podman", + executable: "podman", + prefixArgs: ["--url", "unix:///run/user/1000/podman/podman.sock"], + }); + mocks.dockerCapture.mockReturnValue( + JSON.stringify([ + { + Id: MANAGED_CONTAINER_ID, + Labels: { + [NEMOCLAW_VLLM_MANAGED_LABEL]: "true", + }, + Names: [NEMOCLAW_VLLM_CONTAINER_NAME], + State: "running", + }, + ]), + ); + + expect(isNemoClawManagedVllmRunning()).toBe(true); + expect(mocks.dockerCapture).toHaveBeenCalledWith( + [ + "container", + "ls", + "--all", + "--no-trunc", + "--filter", + `name=^${NEMOCLAW_VLLM_CONTAINER_NAME}$`, + "--format", + "json", + ], + expect.objectContaining({ timeout: 10_000 }), + ); + expect(assertAuthority).not.toHaveBeenCalled(); + }); + + it.each([ + "", + "malformed", + "{}", + JSON.stringify([ + { + Id: MANAGED_CONTAINER_ID, + Labels: {}, + Names: [NEMOCLAW_VLLM_CONTAINER_NAME], + State: "running", + }, + ]), + JSON.stringify([ + { + Id: MANAGED_CONTAINER_ID, + Labels: { [NEMOCLAW_VLLM_MANAGED_LABEL]: "true" }, + Names: ["lookalike"], + State: "running", + }, + ]), + ])("fails closed for Podman ownership output %j", (output) => { + configureHostContainerEngine({ + driverName: "podman", + executable: "podman", + prefixArgs: ["--url", "unix:///run/user/1000/podman/podman.sock"], + }); + mocks.dockerCapture.mockReturnValue(output); + + expect(isNemoClawManagedVllmRunning()).toBe(false); + }); }); describe("installVllm model resolution", () => { diff --git a/src/lib/inference/vllm.ts b/src/lib/inference/vllm.ts index 4a6b74b8f2..eea97b917f 100644 --- a/src/lib/inference/vllm.ts +++ b/src/lib/inference/vllm.ts @@ -10,6 +10,11 @@ import os from "node:os"; import path from "node:path"; import { StringDecoder } from "node:string_decoder"; import { isDeepStrictEqual } from "node:util"; +import { + currentHostContainerEngineCommand, + hostContainerEngineDisplayName, + hostContainerEngineExecutable, +} from "../adapters/container-engine"; import { dockerCapture, dockerForceRm, @@ -465,8 +470,14 @@ function formatElapsed(ms: number): string { } function dockerPrereqsOk(): { ok: boolean; reason?: string } { - if (!runCapture(["sh", "-c", "command -v docker"], { ignoreError: true }).trim()) { - return { ok: false, reason: "docker not found on PATH" }; + const executable = hostContainerEngineExecutable(); + const displayName = hostContainerEngineDisplayName(); + if ( + !runCapture(["sh", "-c", 'command -v "$1"', "--", executable], { + ignoreError: true, + }).trim() + ) { + return { ok: false, reason: `${displayName} not found on PATH` }; } if (!runCapture(["sh", "-c", "command -v nvidia-smi"], { ignoreError: true }).trim()) { return { ok: false, reason: "nvidia-smi not found — vLLM requires NVIDIA drivers" }; @@ -496,16 +507,17 @@ export async function pullImage( logLine: emit, }); if (result.status !== 0) { + const engine = currentHostContainerEngineCommand().driverName; if (result.timeoutKind === "stall") { - return { ok: false, reason: "docker pull stalled with no progress" }; + return { ok: false, reason: `${engine} pull stalled with no progress` }; } if (result.timeoutKind === "max") { return { ok: false, - reason: `docker pull exceeded ${String(profile.pullTimeoutSec)}s safety budget`, + reason: `${engine} pull exceeded ${String(profile.pullTimeoutSec)}s safety budget`, }; } - return { ok: false, reason: `docker pull failed (exit ${String(result.status)})` }; + return { ok: false, reason: `${engine} pull failed (exit ${String(result.status)})` }; } return { ok: true }; } @@ -777,6 +789,44 @@ type VllmContainerOwnership = | { kind: "managed"; containerId: string; running: boolean } | { kind: "unknown" }; +interface VllmContainerOwnershipFields { + containerId: string; + expectedName: string; + observedName: string; + state: string; + labels: Readonly>; +} + +function classifyVllmContainerOwnership({ + containerId, + expectedName, + labels, + observedName, + state, +}: VllmContainerOwnershipFields): VllmContainerOwnership { + if (observedName !== expectedName) return { kind: "unknown" }; + if (!DOCKER_CONTAINER_ID_PATTERN.test(containerId)) return { kind: "unknown" }; + + const managedLabel = labels[NEMOCLAW_VLLM_MANAGED_LABEL] ?? ""; + const dualRole = labels[DUAL_STATION_VLLM_ROLE_LABEL] ?? ""; + const dualEndpoint = labels[DUAL_STATION_VLLM_ENDPOINT_LABEL] ?? ""; + const dualCluster = labels[DUAL_STATION_VLLM_CLUSTER_LABEL] ?? ""; + if (managedLabel !== "true") return { kind: "foreign" }; + const hasAnyDualLabel = Boolean(dualRole || dualEndpoint || dualCluster); + if (hasAnyDualLabel) { + const exactDualHead = + dualRole === "head" && + /^http:\/\/192\.168\.|^http:\/\/10\.|^http:\/\/172\.(?:1[6-9]|2[0-9]|3[01])\./.test( + dualEndpoint, + ) && + /^[a-f0-9]{64}$/.test(dualCluster); + return exactDualHead + ? { kind: "dual-managed", containerId, running: state === "running" } + : { kind: "unknown" }; + } + return { kind: "managed", containerId, running: state === "running" }; +} + function inspectVllmContainerOwnershipInDockerEnv( containerName: string, env: Record, @@ -812,29 +862,93 @@ function inspectVllmContainerOwnershipInDockerEnv( if (fields.length !== 7) return { kind: "unknown" }; const [containerId, observedName, state, managedLabel, dualRole, dualEndpoint, dualCluster] = fields; - if (observedName !== containerName || !DOCKER_CONTAINER_ID_PATTERN.test(containerId)) { + if (observedName !== containerName) { return { kind: "unknown" }; } - if (managedLabel !== "true") return { kind: "foreign" }; - const hasAnyDualLabel = Boolean(dualRole || dualEndpoint || dualCluster); - if (hasAnyDualLabel) { - const exactDualHead = - dualRole === "head" && - /^http:\/\/192\.168\.|^http:\/\/10\.|^http:\/\/172\.(?:1[6-9]|2[0-9]|3[01])\./.test( - dualEndpoint, - ) && - /^[a-f0-9]{64}$/.test(dualCluster); - return exactDualHead - ? { kind: "dual-managed", containerId, running: state === "running" } - : { kind: "unknown" }; + return classifyVllmContainerOwnership({ + containerId, + expectedName: containerName, + labels: { + [DUAL_STATION_VLLM_CLUSTER_LABEL]: dualCluster, + [DUAL_STATION_VLLM_ENDPOINT_LABEL]: dualEndpoint, + [DUAL_STATION_VLLM_ROLE_LABEL]: dualRole, + [NEMOCLAW_VLLM_MANAGED_LABEL]: managedLabel, + }, + observedName, + state, + }); + } catch { + return { kind: "unknown" }; + } +} + +function stringRecord(value: unknown): Record | null { + if (!value || typeof value !== "object" || Array.isArray(value)) return null; + const result: Record = {}; + for (const [key, entry] of Object.entries(value)) { + if (typeof entry !== "string") return null; + result[key] = entry; + } + return result; +} + +function inspectVllmContainerOwnershipInPodmanEnv( + containerName: string, + env: Record, +): VllmContainerOwnership { + try { + const output = dockerCapture( + [ + "container", + "ls", + "--all", + "--no-trunc", + "--filter", + `name=^${containerName}$`, + "--format", + "json", + ], + { env, timeout: 10_000 }, + ).trim(); + if (!output) return { kind: "unknown" }; + + const parsed: unknown = JSON.parse(output); + if (!Array.isArray(parsed) || parsed.length > 1) return { kind: "unknown" }; + if (parsed.length === 0) return { kind: "absent" }; + const row = parsed[0]; + if (!row || typeof row !== "object" || Array.isArray(row)) return { kind: "unknown" }; + const record = row as Record; + const containerId = record.Id ?? record.ID; + const names = record.Names; + const state = record.State; + const labels = stringRecord(record.Labels); + if ( + typeof containerId !== "string" || + !Array.isArray(names) || + names.length !== 1 || + names[0] !== containerName || + typeof state !== "string" || + !labels + ) { + return { kind: "unknown" }; } - return { kind: "managed", containerId, running: state === "running" }; + return classifyVllmContainerOwnership({ + containerId, + expectedName: containerName, + labels, + observedName: names[0], + state, + }); } catch { return { kind: "unknown" }; } } function inspectVllmContainerOwnership(containerName: string): VllmContainerOwnership { + if (currentHostContainerEngineCommand().driverName === "podman") { + return inspectVllmContainerOwnershipInPodmanEnv(containerName, buildVllmDockerEnv()); + } + // A managed dual-Station head always lives on the physical host's default // daemon. Inspect it before following ambient single-host Docker routing so // DOCKER_HOST, DOCKER_CONTEXT, or Docker's persisted currentContext cannot @@ -861,9 +975,10 @@ function vllmContainerReplacementTarget( }; } if (ownership.kind === "unknown") { + const engine = hostContainerEngineDisplayName(); return { ok: false, - reason: `Could not verify ownership of Docker container "${containerName}". NemoClaw will not remove it. Check Docker access and retry.`, + reason: `Could not verify ownership of ${engine} container "${containerName}". NemoClaw will not remove it. Check ${engine} access and retry.`, }; } if (ownership.kind === "dual-managed") { @@ -968,7 +1083,10 @@ function startContainer( suppressOutput: true, }); if (result.status !== 0) { - return { ok: false, reason: `docker run failed (exit ${String(result.status)})` }; + return { + ok: false, + reason: `${currentHostContainerEngineCommand().driverName} run failed (exit ${String(result.status)})`, + }; } return { ok: true }; } @@ -1211,7 +1329,7 @@ function managedStorageRequirements({ const requirements: ManagedStorageRequirement[] = []; if (includeImage && dockerProbe) { requirements.push({ - label: "Docker image storage", + label: `${hostContainerEngineDisplayName()} image storage`, probe: dockerProbe, requiredBytes: estimate.imageCompressedBytes + estimate.imageUnpackedBytes, }); @@ -1351,8 +1469,13 @@ function printManagedStorageWarning({ } console.error(" Useful diagnostics:"); if (includeImage) { - console.error(" docker system df"); - console.error(" docker info --format '{{.DockerRootDir}}'"); + const executable = hostContainerEngineExecutable(); + console.error(` ${executable} system df`); + console.error( + currentHostContainerEngineCommand().driverName === "podman" + ? ` ${executable} info --format json` + : ` ${executable} info --format '{{.DockerRootDir}}'`, + ); } console.error(' df -h "$HOME/.cache/huggingface"'); } @@ -1415,8 +1538,11 @@ async function managedStorageAccepted( (requirement) => requirement.label === "Model cache storage" && !requirement.probe.ok, ); if (problem.kind === "unknown") { - if (problem.check.label === "Docker image storage") { - console.error(" Continuing because Docker storage capacity could not be verified."); + const engineStorageLabel = `${hostContainerEngineDisplayName()} image storage`; + if (problem.check.label === engineStorageLabel) { + console.error( + ` Continuing because ${hostContainerEngineDisplayName()} storage capacity could not be verified.`, + ); return true; } if (opts.nonInteractive) { diff --git a/src/lib/messaging/applier/build/messaging-build-applier.mts b/src/lib/messaging/applier/build/messaging-build-applier.mts index e6cfdab783..f30362ba5f 100755 --- a/src/lib/messaging/applier/build/messaging-build-applier.mts +++ b/src/lib/messaging/applier/build/messaging-build-applier.mts @@ -28,11 +28,17 @@ import { telegramManifest } from "../../channels/telegram/manifest.ts"; import { wechatManifest } from "../../channels/wechat/manifest.ts"; import { whatsappManifest } from "../../channels/whatsapp/manifest.ts"; import type { ChannelAgentPackageRuntimeLockSpec, ChannelManifest } from "../../manifest/types.ts"; +import { + selectActiveMessagingChannelIds, + selectEnabledMessagingAgentRender, + selectEnabledPostAgentInstallBuildFiles, +} from "../../post-agent-install-selection.ts"; type Env = Record; type JsonObject = Record; type MessagingAgentId = "openclaw" | "hermes"; type MessagingHookPhase = "agent-install" | "post-agent-install"; +type MessagingBuildCliPhase = MessagingBuildPhase | "managed-image-capability-union"; type MessagingRuntimeSetupKey = "nodePreloads" | "envAliases" | "secretScans"; type MessagingSerializableValue = | string @@ -341,19 +347,7 @@ export function applyMessagingAgentRenderToLocalFiles( export function activeChannels(plan: MessagingBuildPlan | null): string[] { if (!plan) return []; - const seen = new Set(); - const channels: string[] = []; - for (const item of plan.channels) { - const channel = String(item.channelId || "") - .trim() - .toLowerCase(); - if (!channel || seen.has(channel)) continue; - if (item.active === true && item.disabled !== true) { - seen.add(channel); - channels.push(channel); - } - } - return channels; + return selectActiveMessagingChannelIds(plan); } export function messagingRuntimePlanPath(env: Env = process.env): string { @@ -525,6 +519,24 @@ export function collectHermesMessagingUvPackages(plan: MessagingBuildPlan | null return collectHermesMessagingUvPackageInstalls(plan).map((install) => install.spec); } +/** + * Return the complete reviewed OpenClaw package set baked into a managed image. + * + * This deliberately reads committed built-in manifests directly. A serialized + * messaging plan is deployment input, not authority to choose packages that + * execute during the trusted image build. + */ +export function collectManagedImageOpenClawPluginInstallSpecs(env: Env): string[] { + return collectManagedImageOpenClawPluginInstalls(env).map((install) => install.spec); +} + +/** Return every pinned Hermes package required by a supported managed-image channel. */ +export function collectManagedImageHermesUvPackages(): string[] { + return collectTrustedHermesUvPackageInstalls(TRUSTED_CHANNEL_MANIFESTS).map( + (install) => install.spec, + ); +} + function collectOpenClawMessagingPluginInstalls( plan: MessagingBuildPlan | null, env: Env, @@ -574,6 +586,51 @@ function collectOpenClawMessagingPluginInstalls( return installs; } +function collectManagedImageOpenClawPluginInstalls(env: Env): OpenClawPluginInstall[] { + const reviewedIntegrity = reviewedOpenClawPluginIntegrityByPackageSpec( + env, + TRUSTED_CHANNEL_MANIFESTS, + ); + const reviewedTarballUrls = reviewedOpenClawPluginTarballUrlByPackageSpec( + env, + TRUSTED_CHANNEL_MANIFESTS, + ); + const runtimeLocks = reviewedOpenClawPluginRuntimeLocksByPackageSpec( + env, + TRUSTED_CHANNEL_MANIFESTS, + ); + const installs: OpenClawPluginInstall[] = []; + const seen = new Set(); + + for (const manifest of TRUSTED_CHANNEL_MANIFESTS) { + for (const packageSpec of manifest.agentPackages ?? []) { + if (packageSpec.agent !== "openclaw" || packageSpec.manager !== "openclaw-plugin") continue; + const spec = resolveOpenClawPackageSpec(packageSpec.spec, env); + const npmPackage = requireExactNpmPackageSpec(spec, manifest.id); + const integrity = reviewedIntegrity[npmPackage.packageSpec]; + const tarballUrl = reviewedTarballUrls[npmPackage.packageSpec]; + if (packageSpec.pin !== true || !integrity || !tarballUrl) { + throw new MessagingBuildApplierError( + `Managed-image OpenClaw package ${npmPackage.packageSpec} must have a committed integrity pin and tarball URL`, + ); + } + if (seen.has(npmPackage.packageSpec)) continue; + seen.add(npmPackage.packageSpec); + installs.push({ + spec, + npmPackageSpec: npmPackage.packageSpec, + integrity, + tarballUrl, + ...(runtimeLocks[npmPackage.packageSpec] + ? { runtimeLock: runtimeLocks[npmPackage.packageSpec] } + : {}), + pin: true, + }); + } + } + return installs; +} + /** * Security boundary: NEMOCLAW_MESSAGING_PLAN_B64 is a derived build artifact, * not authority to choose root-time OpenClaw plugins. Invalid state: a serialized @@ -643,9 +700,18 @@ function collectHermesMessagingUvPackageInstalls( */ function trustedHermesUvPackageSpecsForPlan(plan: MessagingBuildPlan | null): Set { const active = new Set(activeChannels(plan)); + return new Set( + collectTrustedHermesUvPackageInstalls( + TRUSTED_CHANNEL_MANIFESTS.filter((manifest) => active.has(manifest.id)), + ).map((install) => install.spec), + ); +} + +function collectTrustedHermesUvPackageInstalls( + manifests: readonly ChannelManifest[], +): HermesUvPackageInstall[] { const specs = new Set(); - for (const manifest of TRUSTED_CHANNEL_MANIFESTS) { - if (!active.has(manifest.id)) continue; + for (const manifest of manifests) { for (const packageSpec of manifest.agentPackages ?? []) { if (packageSpec.agent !== "hermes" || packageSpec.manager !== "hermes-uv-pip") continue; if (!isPinnedHermesUvPackageSpec(packageSpec.spec)) { @@ -656,7 +722,7 @@ function trustedHermesUvPackageSpecsForPlan(plan: MessagingBuildPlan | null): Se specs.add(packageSpec.spec); } } - return specs; + return [...specs].map((spec) => ({ spec })); } export function openClawDoctorEnvOverrides( @@ -689,7 +755,11 @@ export function openClawDoctorEnvOverrides( } export function installOpenClawMessagingPlugins(plan: MessagingBuildPlan | null, env: Env): void { - for (const install of collectOpenClawMessagingPluginInstalls(plan, env)) { + installOpenClawPluginPackages(collectOpenClawMessagingPluginInstalls(plan, env), env); +} + +function installOpenClawPluginPackages(installs: readonly OpenClawPluginInstall[], env: Env): void { + for (const install of installs) { const installCache = install.runtimeLock ? requireWritableRuntimeInstallCache(install.runtimeLock, env) : undefined; @@ -974,10 +1044,7 @@ function resolveAgentRenderTarget( } function enabledAgentRender(plan: MessagingBuildPlan): MessagingRenderEntry[] { - const active = new Set(activeChannels(plan)); - return plan.agentRender.filter( - (render) => render.agent === plan.agent && active.has(render.channelId), - ); + return selectEnabledMessagingAgentRender(plan); } function enabledBuildStepsForPhase( @@ -985,6 +1052,9 @@ function enabledBuildStepsForPhase( phase: MessagingHookPhase, ): MessagingBuildStep[] { if (!plan) return []; + if (phase === "post-agent-install") { + return selectEnabledPostAgentInstallBuildFiles(plan); + } return enabledBuildSteps(plan).filter((step) => buildStepMatchesPhase(plan, step, phase)); } @@ -1714,11 +1784,26 @@ function formatError(error: unknown): string { export type MessagingBuildPhase = "runtime-setup" | "agent-install" | "post-agent-install"; +export interface MessagingBuildPhaseOptions { + /** + * A managed image already contains the reviewed capability union. Keep its + * durable HOME writes to the explicit render/build-file plan instead of + * invoking OpenClaw's broad migration and repair command. + */ + readonly managedStartupRuntime?: boolean; +} + export function applyMessagingBuildPhase( plan: MessagingBuildPlan | null, phase: MessagingBuildPhase, env: Env = process.env, + options: MessagingBuildPhaseOptions = {}, ): readonly string[] { + if (options.managedStartupRuntime && phase !== "post-agent-install") { + throw new MessagingBuildApplierError( + "Managed startup runtime mode is only valid for post-agent-install", + ); + } if (phase === "runtime-setup") { const target = writeMessagingRuntimePlanArtifact(plan, messagingRuntimePlanPath(env)); return target ? [target] : []; @@ -1732,7 +1817,7 @@ export function applyMessagingBuildPhase( ...applyPostAgentInstallBuildFilesToLocalFiles(plan), ]; const appliedTargets = applyPostAgentInstallOutputs(); - if (plan?.agent === "openclaw") { + if (plan?.agent === "openclaw" && !options.managedStartupRuntime) { runOpenClawMessagingDoctor(plan, env); return uniqueStrings([...appliedTargets, ...applyPostAgentInstallOutputs()]); } @@ -1764,6 +1849,10 @@ function installHermesMessagingUvPackages(plan: MessagingBuildPlan | null, env: const selectedPackages = collectHermesMessagingUvPackageInstalls(plan).map( (install) => install.spec, ); + installHermesUvPackages(selectedPackages, env); +} + +function installHermesUvPackages(selectedPackages: readonly string[], env: Env): void { if (selectedPackages.length === 0) return; runCommand( [ @@ -1780,6 +1869,22 @@ function installHermesMessagingUvPackages(plan: MessagingBuildPlan | null, env: ); } +export function installManagedImageCapabilityUnion( + agent: MessagingAgentId, + env: Env = process.env, +): void { + if (env.NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION !== "1") { + throw new MessagingBuildApplierError( + "Managed-image capability union installation requires NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION=1", + ); + } + if (agent === "openclaw") { + installOpenClawPluginPackages(collectManagedImageOpenClawPluginInstalls(env), env); + return; + } + installHermesUvPackages(collectManagedImageHermesUvPackages(), env); +} + export function describeMessagingBuildPhase( plan: MessagingBuildPlan | null, phase: MessagingBuildPhase, @@ -1802,23 +1907,56 @@ export function describeMessagingBuildPhase( } export function main(argv: readonly string[] = process.argv.slice(2)): void { - const { agent, phase, dryRun } = parseMessagingBuildArgs(argv); + const { agent, phase, dryRun, managedStartupRuntime } = parseMessagingBuildArgs(argv); const plan = readMessagingBuildPlanFromEnv(process.env, agent); + if (phase === "managed-image-capability-union") { + if (plan) { + throw new MessagingBuildApplierError( + "Managed-image capability union must be built without a serialized messaging plan", + ); + } + if (dryRun) { + console.log( + JSON.stringify( + { + agent, + phase, + channels: [], + runtimePlanPath: "", + doctorEnv: {}, + installSpecs: + agent === "openclaw" + ? collectManagedImageOpenClawPluginInstallSpecs(process.env) + : [], + hermesUvPackages: agent === "hermes" ? collectManagedImageHermesUvPackages() : [], + openclawVersion: process.env.OPENCLAW_VERSION || "", + }, + null, + 2, + ), + ); + return; + } + installManagedImageCapabilityUnion(agent, process.env); + return; + } if (dryRun) { console.log(JSON.stringify(describeMessagingBuildPhase(plan, phase, process.env), null, 2)); return; } - applyMessagingBuildPhase(plan, phase, process.env); + applyMessagingBuildPhase(plan, phase, process.env, { managedStartupRuntime }); } function parseMessagingBuildArgs(argv: readonly string[]): { readonly agent: MessagingAgentId; - readonly phase: MessagingBuildPhase; + readonly phase: MessagingBuildCliPhase; readonly dryRun: boolean; + readonly managedStartupRuntime: boolean; } { let agent: MessagingAgentId | undefined; - let phase: MessagingBuildPhase | undefined; + let phase: MessagingBuildCliPhase | undefined; let dryRun = false; + let managedStartupRuntime = false; for (let index = 0; index < argv.length; index += 1) { const arg = argv[index]; @@ -1826,6 +1964,10 @@ function parseMessagingBuildArgs(argv: readonly string[]): { dryRun = true; continue; } + if (arg === "--managed-startup-runtime") { + managedStartupRuntime = true; + continue; + } if (arg === "--agent") { agent = readAgentArg(argv[index + 1]); index += 1; @@ -1851,10 +1993,17 @@ function parseMessagingBuildArgs(argv: readonly string[]): { throw new MessagingBuildApplierError(`Unknown messaging build applier argument: ${arg}`); } + const resolvedPhase = phase ?? "post-agent-install"; + if (managedStartupRuntime && resolvedPhase !== "post-agent-install") { + throw new MessagingBuildApplierError( + "--managed-startup-runtime requires --phase post-agent-install", + ); + } return { agent: agent ?? "openclaw", - phase: phase ?? "post-agent-install", + phase: resolvedPhase, dryRun, + managedStartupRuntime, }; } @@ -1865,12 +2014,17 @@ function readAgentArg(value: string | undefined): MessagingAgentId { throw new MessagingBuildApplierError("--agent must be 'openclaw' or 'hermes'"); } -function readPhaseArg(value: string | undefined): MessagingBuildPhase { - if (value === "runtime-setup" || value === "agent-install" || value === "post-agent-install") { +function readPhaseArg(value: string | undefined): MessagingBuildCliPhase { + if ( + value === "runtime-setup" || + value === "agent-install" || + value === "post-agent-install" || + value === "managed-image-capability-union" + ) { return value; } throw new MessagingBuildApplierError( - "--phase must be 'runtime-setup', 'agent-install', or 'post-agent-install'", + "--phase must be 'runtime-setup', 'agent-install', 'post-agent-install', or 'managed-image-capability-union'", ); } diff --git a/src/lib/messaging/clone-rebind.ts b/src/lib/messaging/clone-rebind.ts new file mode 100644 index 0000000000..8bd9db40ef --- /dev/null +++ b/src/lib/messaging/clone-rebind.ts @@ -0,0 +1,167 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createBuiltInChannelManifestRegistry } from "./channels/built-ins"; +import { hydrateDerivedSandboxMessagingPlanFields } from "./hydration"; +import type { MessagingAgentId, SandboxMessagingPlan } from "./manifest"; +import { compactSandboxMessagingPlanForPersistence } from "./persistence"; +import { parseSandboxMessagingPlan } from "./plan-validation"; + +export interface SandboxMessagingCloneRebindInput { + readonly sourceSandboxName: string; + readonly destinationSandboxName: string; + readonly agent: MessagingAgentId; + readonly sourcePlan: unknown; + /** + * Explicit non-secret inputs used by manifest renderers. Clone rebinding + * never consults process.env, so an unrelated host credential cannot change + * the destination plan or enter its fingerprints. + */ + readonly environment?: Readonly>; +} + +export class SandboxMessagingCloneRebindError extends Error { + constructor(message: string) { + super(`Cannot rebind managed messaging plan: ${message}`); + this.name = "SandboxMessagingCloneRebindError"; + } +} + +function fail(message: string): never { + throw new SandboxMessagingCloneRebindError(message); +} + +function requireSandboxName(value: string, label: string): string { + const normalized = value.trim(); + if (!normalized || normalized !== value) fail(`${label} sandbox name is invalid`); + return normalized; +} + +/** + * Recompile one secret-free managed messaging plan for a destination sandbox. + * + * Compact persistence data is the retained intent boundary. All target-bound + * provider names and executable derived fields are rebuilt from current + * built-in manifests with an explicit, credential-free environment. + */ +export function rebindSandboxMessagingPlanForClone( + input: SandboxMessagingCloneRebindInput, +): SandboxMessagingPlan { + const sourceSandboxName = requireSandboxName(input.sourceSandboxName, "source"); + const destinationSandboxName = requireSandboxName(input.destinationSandboxName, "destination"); + if (sourceSandboxName === destinationSandboxName) { + fail("source and destination sandbox names must differ"); + } + + const manifestRegistry = createBuiltInChannelManifestRegistry(); + const supportedChannelIds = manifestRegistry + .listAvailable({ agent: input.agent }) + .map((manifest) => manifest.id); + const environment = Object.freeze({ ...(input.environment ?? {}) }); + const sourcePlan = parseSandboxMessagingPlan(input.sourcePlan, { + sandboxName: sourceSandboxName, + agent: input.agent, + supportedChannelIds, + environment, + }); + if (!sourcePlan) fail("source plan is invalid or uses a non-built-in channel"); + const sourceChannelIds = new Set(sourcePlan.channels.map((channel) => channel.channelId)); + if (sourcePlan.disabledChannels.some((channelId) => !sourceChannelIds.has(channelId))) { + fail("source plan disables a channel that is not configured"); + } + for (const channel of sourcePlan.channels) { + const manifest = manifestRegistry.get(channel.channelId); + if (!manifest || !manifest.supportedAgents.includes(input.agent)) { + fail(`source channel ${channel.channelId} is not a supported built-in`); + } + const listedDisabled = sourcePlan.disabledChannels.includes(channel.channelId); + if (channel.disabled !== listedDisabled || (channel.active && !channel.configured)) { + fail(`source channel ${channel.channelId} has inconsistent lifecycle state`); + } + const seenInputIds = new Set(); + for (const planInput of channel.inputs) { + if (seenInputIds.has(planInput.inputId)) { + fail(`source channel ${channel.channelId} repeats input ${planInput.inputId}`); + } + seenInputIds.add(planInput.inputId); + const manifestInput = manifest.inputs.find((candidate) => candidate.id === planInput.inputId); + if (!manifestInput || manifestInput.kind !== planInput.kind) { + fail( + `source channel ${channel.channelId} input ${planInput.inputId} is not manifest-owned`, + ); + } + if (planInput.kind === "secret" && planInput.value !== undefined) { + fail("source plan contains a raw secret input value"); + } + } + if (channel.active && !channel.disabled) { + for (const requiredInput of manifest.inputs.filter((candidate) => candidate.required)) { + const retained = channel.inputs.find((candidate) => candidate.inputId === requiredInput.id); + const available = + requiredInput.kind === "secret" + ? retained?.credentialAvailable === true + : retained?.value !== undefined; + if (!available) { + fail( + `active source channel ${channel.channelId} is missing required input ${requiredInput.id}`, + ); + } + } + } + } + + const compact = compactSandboxMessagingPlanForPersistence(sourcePlan); + const credentialBindings = (compact.credentialBindings ?? []).map( + ({ credentialHash: _sourceCredentialHash, ...binding }) => binding, + ); + const targetIntent = { + ...compact, + // A source hash describes the source gateway credential, not the explicit + // credential that clone provisioning will write into the destination + // provider. Preserve "unknown-token" overlap semantics until that exact + // destination credential is durably proven; never copy a stale hash. + credentialBindings, + sandboxName: destinationSandboxName, + // These fields are executable output, not retained clone intent. + networkPolicy: { presets: [], entries: [] }, + agentRender: [], + buildSteps: [], + runtimeSetup: { nodePreloads: [], envAliases: [], secretScans: [] }, + stateUpdates: [], + healthChecks: [], + } as const; + const normalized = parseSandboxMessagingPlan(targetIntent, { + sandboxName: destinationSandboxName, + agent: input.agent, + supportedChannelIds, + environment, + }); + if (!normalized) fail("destination intent could not be normalized"); + const hydrated = hydrateDerivedSandboxMessagingPlanFields(normalized, { environment }); + const rebound = parseSandboxMessagingPlan(hydrated, { + sandboxName: destinationSandboxName, + agent: input.agent, + supportedChannelIds, + environment, + }); + if (!rebound) fail("destination plan could not be validated after manifest hydration"); + const secretProvenanceNeutralRebound = { + ...rebound, + credentialBindings: rebound.credentialBindings.map( + ({ credentialHash: _ambientCredentialHash, ...binding }) => binding, + ), + }; + for (const binding of secretProvenanceNeutralRebound.credentialBindings) { + const credential = manifestRegistry + .get(binding.channelId) + ?.credentials.find((candidate) => candidate.id === binding.credentialId); + const expectedProviderName = credential?.providerName.replaceAll( + "{sandboxName}", + destinationSandboxName, + ); + if (!expectedProviderName || binding.providerName !== expectedProviderName) { + fail(`destination provider identity for ${binding.channelId} could not be proven`); + } + } + return secretProvenanceNeutralRebound; +} diff --git a/src/lib/messaging/compiler/engines/credential-binding-engine.ts b/src/lib/messaging/compiler/engines/credential-binding-engine.ts index 1a87227e59..11030a0cb2 100644 --- a/src/lib/messaging/compiler/engines/credential-binding-engine.ts +++ b/src/lib/messaging/compiler/engines/credential-binding-engine.ts @@ -1,19 +1,20 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { hashCredential } from "../../../security/credential-hash"; import type { ChannelManifest, SandboxMessagingCredentialBindingPlan, SandboxMessagingInputReference, } from "../../manifest"; import type { ManifestCompilerContext } from "../types"; -import { hashCredential } from "../../../security/credential-hash"; import { resolveSandboxNameTemplate } from "./template"; export function planCredentialBindings( manifest: ChannelManifest, context: ManifestCompilerContext, inputs: readonly SandboxMessagingInputReference[], + environment: Readonly> = process.env, ): SandboxMessagingCredentialBindingPlan[] { return manifest.credentials.map((credential) => { const sourceInput = inputs.find((input) => input.inputId === credential.sourceInput); @@ -24,7 +25,7 @@ export function planCredentialBindings( const envKey = sourceInput?.sourceEnv ?? credential.providerEnvKey; const credentialHash = credentialAvailable - ? (hashCredential(process.env[envKey]) ?? undefined) + ? (hashCredential(environment[envKey]) ?? undefined) : undefined; return { diff --git a/src/lib/messaging/compiler/engines/host-forward-engine.ts b/src/lib/messaging/compiler/engines/host-forward-engine.ts index 91bc8c796f..3408c08bc6 100644 --- a/src/lib/messaging/compiler/engines/host-forward-engine.ts +++ b/src/lib/messaging/compiler/engines/host-forward-engine.ts @@ -17,10 +17,11 @@ export function planHostForward( inputs: readonly SandboxMessagingInputReference[], active: boolean, referenceResolver?: RenderTemplateReferenceResolver, + environment: Readonly> = process.env, ): SandboxMessagingHostForwardPlan | undefined { if (!active || !manifest.hostForward) return undefined; - const context = { inputs, env: process.env, referenceResolver }; + const context = { inputs, env: environment, referenceResolver }; if (!isTruthyRenderTemplate(manifest.hostForward.when, context)) return undefined; const portValue = resolveRenderTemplatesInValue(manifest.hostForward.port, context); diff --git a/src/lib/messaging/hydration.ts b/src/lib/messaging/hydration.ts index 605f1017f9..07d79f3dcf 100644 --- a/src/lib/messaging/hydration.ts +++ b/src/lib/messaging/hydration.ts @@ -42,12 +42,19 @@ import { normalizePersistedInputs, } from "./persistence"; +export interface SandboxMessagingHydrationOptions { + /** Explicit environment seam for deterministic rehydration without ambient credentials. */ + readonly environment?: Readonly>; +} + export function hydrateDerivedSandboxMessagingPlanFields( plan: SandboxMessagingPlan, + options: SandboxMessagingHydrationOptions = {}, ): SandboxMessagingPlan { + const environment = options.environment ?? process.env; const manifestRegistry = createBuiltInChannelManifestRegistry(); const channels = plan.channels.map((channel) => - hydrateChannelFromManifest(plan, channel, manifestRegistry.get(channel.channelId)), + hydrateChannelFromManifest(plan, channel, manifestRegistry.get(channel.channelId), environment), ); const hydratedPlan = { ...plan, channels }; const manifests = channels.flatMap((channel) => { @@ -63,7 +70,7 @@ export function hydrateDerivedSandboxMessagingPlanFields( agentRender: plan.agentRender.length > 0 ? plan.agentRender - : agentRenderFromManifests(hydratedPlan, manifestRegistry), + : agentRenderFromManifests(hydratedPlan, manifestRegistry, environment), buildSteps: plan.buildSteps.length > 0 ? plan.buildSteps @@ -82,6 +89,7 @@ function hydrateChannelFromManifest( plan: SandboxMessagingPlan, channel: SandboxMessagingChannelPlan, manifest: ChannelManifest | undefined, + environment: Readonly>, ): SandboxMessagingChannelPlan { const { hostForward: _oldHostForward, ...channelWithoutHostForward } = channel; const disabled = channel.disabled || plan.disabledChannels.includes(channel.channelId); @@ -91,7 +99,7 @@ function hydrateChannelFromManifest( const configured = channel.configured; const active = channel.active && !disabled; const hostForward = manifest - ? planHostForward(manifest, inputs, active, createBuiltInRenderTemplateResolver()) + ? planHostForward(manifest, inputs, active, createBuiltInRenderTemplateResolver(), environment) : undefined; return { ...channelWithoutHostForward, @@ -231,6 +239,7 @@ function runtimeSetupHasEntries(setup: SandboxMessagingRuntimeSetupPlan | undefi function agentRenderFromManifests( plan: SandboxMessagingPlan, manifestRegistry: ReturnType, + environment: Readonly>, ): SandboxMessagingAgentRenderPlan[] { const render: SandboxMessagingAgentRenderPlan[] = []; const referenceResolver = createBuiltInRenderTemplateResolver(); @@ -239,7 +248,7 @@ function agentRenderFromManifests( if (!manifest) continue; const context = { inputs: channel.inputs, - env: process.env, + env: environment, referenceResolver, }; diff --git a/src/lib/messaging/index.ts b/src/lib/messaging/index.ts index 3ad9587144..1da55c0c87 100644 --- a/src/lib/messaging/index.ts +++ b/src/lib/messaging/index.ts @@ -3,6 +3,7 @@ export * from "./applier"; export * from "./channels"; +export * from "./clone-rebind"; export * from "./compiler"; export * from "./diagnostics"; export * from "./hooks"; diff --git a/src/lib/messaging/persistence.ts b/src/lib/messaging/persistence.ts index 1d07305ca4..5812031c14 100644 --- a/src/lib/messaging/persistence.ts +++ b/src/lib/messaging/persistence.ts @@ -1,10 +1,8 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { - createBuiltInChannelManifestRegistry, - createBuiltInRenderTemplateResolver, -} from "./channels"; +import { createBuiltInChannelManifestRegistry } from "./channels/built-ins"; +import { createBuiltInRenderTemplateResolver } from "./channels/template-resolver"; import { planCredentialBindings } from "./compiler/engines/credential-binding-engine"; import { planHostForward } from "./compiler/engines/host-forward-engine"; import type { ManifestCompilerContext } from "./compiler/types"; @@ -131,6 +129,7 @@ export function compactSandboxMessagingPlanForPersistence( export function normalizePersistedSandboxMessagingPlanShape( plan: MaybeCompactMessagingPlan, + environment: Readonly> = process.env, ): SandboxMessagingPlan { const manifestRegistry = createBuiltInChannelManifestRegistry(); const disabledChannels = plan.disabledChannels.filter( @@ -138,9 +137,19 @@ export function normalizePersistedSandboxMessagingPlanShape( ); const disabledSet = new Set(disabledChannels); const channels = plan.channels.map((channel) => - normalizePersistedChannel(channel, disabledSet, manifestRegistry.get(channel.channelId)), + normalizePersistedChannel( + channel, + disabledSet, + manifestRegistry.get(channel.channelId), + environment, + ), + ); + const credentialBindings = normalizePersistedCredentialBindings( + plan, + channels, + manifestRegistry, + environment, ); - const credentialBindings = normalizePersistedCredentialBindings(plan, channels, manifestRegistry); const normalizedPlan: SandboxMessagingPlan = { ...plan, channels, @@ -184,6 +193,7 @@ function normalizePersistedChannel( channel: MaybeCompactMessagingChannelPlan, disabledSet: ReadonlySet, manifest: ChannelManifest | undefined, + environment: Readonly>, ): SandboxMessagingChannelPlan { const disabled = channel.disabled ?? disabledSet.has(channel.channelId); const configured = channel.configured ?? true; @@ -194,7 +204,13 @@ function normalizePersistedChannel( const active = channel.active ?? (configured && !disabled && requiredInputsAvailable(manifest, inputs)); const hostForward = manifest - ? planHostForward(manifest, inputs, active && !disabled, createBuiltInRenderTemplateResolver()) + ? planHostForward( + manifest, + inputs, + active && !disabled, + createBuiltInRenderTemplateResolver(), + environment, + ) : undefined; return { @@ -308,6 +324,7 @@ function normalizePersistedCredentialBindings( plan: MaybeCompactMessagingPlan, channels: readonly SandboxMessagingChannelPlan[], manifestRegistry: ReturnType, + environment: Readonly>, ): SandboxMessagingCredentialBindingPlan[] { const persisted = plan.credentialBindings ?? []; if ( @@ -337,6 +354,7 @@ function normalizePersistedCredentialBindings( planForBindings, manifests, new Map(channels.map((channel) => [channel.channelId, channel.inputs] as const)), + environment, ); return generated.map((binding) => overlayPersistedCredentialBinding(binding, persisted)); } @@ -345,12 +363,16 @@ function credentialBindingsFromManifests( plan: SandboxMessagingPlan, manifests: readonly ChannelManifest[], inputRegistry: ReadonlyMap, + environment: Readonly>, ): SandboxMessagingCredentialBindingPlan[] { const context = compilerContext(plan); return manifests.flatMap((manifest) => - planCredentialBindings(manifest, context, inputRegistry.get(manifest.id) ?? []).map((binding) => - overlayPersistedCredentialBinding(binding, plan.credentialBindings), - ), + planCredentialBindings( + manifest, + context, + inputRegistry.get(manifest.id) ?? [], + environment, + ).map((binding) => overlayPersistedCredentialBinding(binding, plan.credentialBindings)), ); } diff --git a/src/lib/messaging/plan-validation.ts b/src/lib/messaging/plan-validation.ts index 6b59cf31f8..853306e419 100644 --- a/src/lib/messaging/plan-validation.ts +++ b/src/lib/messaging/plan-validation.ts @@ -18,6 +18,8 @@ export interface SandboxMessagingPlanParseOptions { sandboxName?: string | null; agent?: MessagingAgentId | string | null; supportedChannelIds?: readonly MessagingChannelId[] | readonly string[] | null; + /** Explicit environment seam for deterministic rehydration without ambient credentials. */ + environment?: Readonly>; } export function parseSandboxMessagingPlan( @@ -80,7 +82,10 @@ export function parseSandboxMessagingPlan( if (!value.disabledChannels.every((channelId) => typeof channelId === "string")) return null; return cloneSandboxMessagingPlan( - normalizePersistedSandboxMessagingPlanShape(value as MaybeCompactMessagingPlan), + normalizePersistedSandboxMessagingPlanShape( + value as MaybeCompactMessagingPlan, + options.environment, + ), ); } diff --git a/src/lib/messaging/post-agent-install-selection.ts b/src/lib/messaging/post-agent-install-selection.ts new file mode 100644 index 0000000000..eabf11fe3a --- /dev/null +++ b/src/lib/messaging/post-agent-install-selection.ts @@ -0,0 +1,78 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +interface SelectionHook { + readonly id: string; + readonly phase: string; +} + +interface SelectionChannel { + readonly channelId: string; + readonly active?: boolean; + readonly disabled?: boolean; + readonly hooks?: readonly SelectionHook[]; +} + +interface SelectionPlanBase { + readonly channels: readonly SelectionChannel[]; +} + +/** + * Canonical active-channel selection shared by the image applier and the + * managed-startup transaction planner. Durable rollback receipts must cover + * exactly the outputs that the applier can mutate. + */ +export function selectActiveMessagingChannelIds(plan: SelectionPlanBase): string[] { + const seen = new Set(); + const channels: string[] = []; + for (const item of plan.channels) { + const channel = String(item.channelId || "") + .trim() + .toLowerCase(); + if (!channel || seen.has(channel)) continue; + if (item.active === true && item.disabled !== true) { + seen.add(channel); + channels.push(channel); + } + } + return channels; +} + +export function selectEnabledMessagingAgentRender< + Render extends { + readonly agent: string; + readonly channelId: string; + }, +>( + plan: SelectionPlanBase & { + readonly agent: string; + readonly agentRender: readonly Render[]; + }, +): Render[] { + const active = new Set(selectActiveMessagingChannelIds(plan)); + return plan.agentRender.filter( + (render) => render.agent === plan.agent && active.has(render.channelId), + ); +} + +export function selectEnabledPostAgentInstallBuildFiles< + Step extends { + readonly channelId: string; + readonly kind: string; + readonly hookId?: string; + }, +>( + plan: SelectionPlanBase & { + readonly buildSteps: readonly Step[]; + }, +): Step[] { + const active = new Set(selectActiveMessagingChannelIds(plan)); + return plan.buildSteps.filter((step) => { + if (!active.has(step.channelId) || step.kind !== "build-file") return false; + if (!step.hookId) return true; + const hookPhase = plan.channels + .find((channel) => channel.channelId === step.channelId) + ?.hooks?.find((hook) => hook.id === step.hookId)?.phase; + return hookPhase === undefined || hookPhase === "post-agent-install"; + }); +} diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 1b70da187f..f5c341a157 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -49,7 +49,8 @@ const sandboxBuildPatchConfig: typeof import("./onboard/sandbox-build-patch-conf const baseImageResolutionFlow: typeof import("./onboard/base-image-resolution-flow") = require("./onboard/base-image-resolution-flow"); const sandboxCreateIntentResolution: typeof import("./onboard/sandbox-create-intent-resolution") = require("./onboard/sandbox-create-intent-resolution"); const sandboxCreatePlanMaterialization: typeof import("./onboard/sandbox-create-plan-materialization") = require("./onboard/sandbox-create-plan-materialization"); -const sandboxCreateLaunch: typeof import("./onboard/sandbox-create-launch") = require("./onboard/sandbox-create-launch"); +const managedWorkloadOnboard: typeof import("./onboard/managed-workload/onboard-orchestration") = + require("./onboard/managed-workload/onboard-orchestration"); const onboardEntryOptions: typeof import("./onboard/entry-options") = require("./onboard/entry-options"); const onboardSessionBootstrap: typeof import("./onboard/session-bootstrap") = require("./onboard/session-bootstrap"); const channelState: typeof import("./onboard/channel-state") = require("./onboard/channel-state"); @@ -76,6 +77,7 @@ const dockerDriverGatewayLaunch: typeof import("./onboard/docker-driver-gateway- const dockerDriverGatewayRuntime: typeof import("./onboard/docker-driver-gateway-runtime") = require("./onboard/docker-driver-gateway-runtime"); const gatewayService: typeof import("./onboard/docker-driver-gateway-service") = require("./onboard/docker-driver-gateway-service"); const dockerDriverGatewayCutover: typeof import("./onboard/docker-driver-gateway-cutover") = require("./onboard/docker-driver-gateway-cutover"); +const computeRuntime: typeof import("./onboard/compute/runtime") = require("./onboard/compute/runtime"); const { reapHostGatewayBeforeLaunchOrFail, reapDuplicateHostGatewaysExceptOrFail } = require("./onboard/docker-driver-gateway-prelaunch") as typeof import("./onboard/docker-driver-gateway-prelaunch"); const { @@ -122,9 +124,7 @@ const { }: typeof import("./onboard/created-sandbox-finalization") = require("./onboard/created-sandbox-finalization"); const providerKeyBridge: typeof import("./onboard/provider-key-bridge") = require("./onboard/provider-key-bridge"); const compatibleEndpointGatewayRoute: typeof import("./onboard/inference-providers/compatible-endpoint-gateway-route") = require("./onboard/inference-providers/compatible-endpoint-gateway-route"); -const { - isLinuxDockerDriverGatewayEnabled, -}: typeof import("./onboard/docker-driver-platform") = require("./onboard/docker-driver-platform"); +const { isLinuxDockerDriverGatewayEnabled } = computeRuntime; const { reconcileGatewayGpuReuseForGpuIntent, }: typeof import("./onboard/gateway-gpu-passthrough") = require("./onboard/gateway-gpu-passthrough"); @@ -336,7 +336,7 @@ const { waitForGatewayHealth, }: typeof import("./onboard/gateway-health-wait") = require("./onboard/gateway-health-wait"); const { - waitForStandaloneDockerDriverGateway, + waitForStandaloneManagedDriverGateway, }: typeof import("./onboard/docker-driver-gateway-readiness") = require("./onboard/docker-driver-gateway-readiness"); const { resolveOpenshell } = require("./adapters/openshell/resolve"); const credentials: typeof import("./credentials/store") = require("./credentials/store"); @@ -533,7 +533,7 @@ const { isGatewayTcpReady: probeGatewayTcpReady } = require("./onboard/gateway-tcp-readiness") as typeof import("./onboard/gateway-tcp-readiness"); const { trackChildExit } = require("./onboard/child-exit-tracker") as typeof import("./onboard/child-exit-tracker"); -const { reportDockerDriverGatewayStartFailure } = +const { reportManagedDriverGatewayStartFailure } = require("./onboard/docker-driver-gateway-failure") as typeof import("./onboard/docker-driver-gateway-failure"); const { createFinalGatewayStartFailureHandler, @@ -643,12 +643,44 @@ const RESET = USE_COLOR ? "\x1b[0m" : ""; let OPENSHELL_BIN: string | null = null; let GATEWAY_PORT = DEFAULT_GATEWAY_PORT; let GATEWAY_NAME = gatewayBinding.resolveGatewayName(GATEWAY_PORT); +let ACTIVE_OPEN_SHELL_COMPUTE_PLAN = computeRuntime.resolveCurrentOpenShellComputePlan(); + +function getActiveManagedGatewayProfile(): + | import("./onboard/compute/managed-gateway-profile").ManagedGatewayDriverProfile + | null { + return computeRuntime.resolveManagedGatewayDriverProfile(ACTIVE_OPEN_SHELL_COMPUTE_PLAN); +} + +function isManagedDriverGatewayEnabled(): boolean { + return getActiveManagedGatewayProfile() !== null; +} + +function activeManagedGatewayHasCapability( + capability: keyof import("./onboard/compute/managed-gateway-profile").ManagedGatewayDriverCapabilities, +): boolean { + return getActiveManagedGatewayProfile()?.capabilities[capability] === true; +} + +function activeRuntimeSupportsHostLocalInference(): boolean { + return computeRuntime.resolveOpenShellComputeCapabilities(ACTIVE_OPEN_SHELL_COMPUTE_PLAN) + .hostLocalInference; +} + +function isActiveDockerComputeDriver(): boolean { + return computeRuntime.isDockerComputeDriver(ACTIVE_OPEN_SHELL_COMPUTE_PLAN); +} + +function isActiveDockerManagedGateway(): boolean { + return isActiveDockerComputeDriver() && isManagedDriverGatewayEnabled(); +} + const { clearDockerDriverGatewayRuntimeFiles, createGatewayServicePortOwnership, getDockerDriverGatewayEnv, getDockerDriverGatewayPid, getDockerDriverGatewayPortListenerPid, + getOpenShellSupervisorImage, getDockerDriverGatewayReuseDrift: getGatewayReuseDrift, getDockerDriverGatewayRuntimeDrift, getDockerDriverGatewayRuntimeDriftFromSnapshot, @@ -771,7 +803,7 @@ const { isSandboxReady, parseSandboxStatus, getSandboxStateFromOutputs } = gatew const waitForSandboxReady = sandboxReadinessTracing.createSandboxReadyWaiter({ runCaptureOpenshell, isSandboxReady, - isLinuxDockerDriverGatewayEnabled, + isLinuxDockerDriverGatewayEnabled: isManagedDriverGatewayEnabled, sleep: sleepSeconds, }); const { hasStaleGateway, isSelectedGateway, isGatewayHealthy, getGatewayReuseState } = @@ -1125,14 +1157,33 @@ function areRequiredDockerDriverBinariesPresent( function ensureOpenshellForOnboard( exitProcess: (code: number) => never = (code) => process.exit(code), + computePlan: import("./onboard/compute/plan").OpenShellComputePlan = ACTIVE_OPEN_SHELL_COMPUTE_PLAN, ): OpenShellInstallResult { - return openshellInstallFlow.ensureOpenshellForOnboard(getOpenShellInstallDeps(exitProcess)); + return openshellInstallFlow.ensureOpenshellForOnboard( + getOpenShellInstallDeps(exitProcess, computePlan), + ); } function getOpenShellInstallDeps( exitProcess: (code: number) => never = (code) => process.exit(code), + computePlan: import("./onboard/compute/plan").OpenShellComputePlan = ACTIVE_OPEN_SHELL_COMPUTE_PLAN, ): OpenShellInstallDeps { + const currentProfile = computeRuntime.resolveManagedGatewayDriverProfile(computePlan); return { + getManagedGatewayBinaryRequirements: (platform = process.platform, arch = process.arch) => { + const plan = + platform === process.platform && arch === process.arch + ? computePlan + : computeRuntime.resolveCurrentOpenShellComputePlan(platform, arch); + const profile = computeRuntime.resolveManagedGatewayDriverProfile(plan); + return profile + ? { + driverLabel: profile.displayName, + gateway: true, + sandbox: profile.capabilities.localSupervisorBinary && platform === "linux", + } + : null; + }, isLinuxDockerDriverGatewayEnabled, resolveOpenShellGatewayBinary, resolveOpenShellSandboxBinary, @@ -1147,7 +1198,7 @@ function getOpenShellInstallDeps( versionGte, hasRequiredOpenshellMessagingFeatures: () => // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. - (require("./onboard/openshell-feature-gate") as typeof import("./onboard/openshell-feature-gate")).hasRequiredOpenshellMessagingFeatures({ openshellBin: resolveOpenshell(), gatewayBin: resolveOpenShellGatewayBinary(), sandboxBin: resolveOpenShellSandboxBinary(), allowExternalGatewayBin: Boolean(process.env.NEMOCLAW_OPENSHELL_GATEWAY_BIN?.trim()), allowExternalSandboxBin: Boolean(process.env.NEMOCLAW_OPENSHELL_SANDBOX_BIN?.trim()), requireSandboxBin: process.platform !== "darwin" || Boolean(process.env.NEMOCLAW_OPENSHELL_SANDBOX_BIN?.trim()) }), + (require("./onboard/openshell-feature-gate") as typeof import("./onboard/openshell-feature-gate")).hasRequiredOpenshellMessagingFeatures({ openshellBin: resolveOpenshell(), gatewayBin: resolveOpenShellGatewayBinary(), sandboxBin: resolveOpenShellSandboxBinary(), allowExternalGatewayBin: Boolean(process.env.NEMOCLAW_OPENSHELL_GATEWAY_BIN?.trim()), allowExternalSandboxBin: Boolean(process.env.NEMOCLAW_OPENSHELL_SANDBOX_BIN?.trim()), requireSandboxBin: computeRuntime.requiresHostSandboxBinaryForInstall(currentProfile, { explicitSandboxBinary: Boolean(process.env.NEMOCLAW_OPENSHELL_SANDBOX_BIN?.trim()), platform: process.platform }) }), shouldAllowOpenshellAboveBlueprintMax, cliDisplayName, log: console.log, @@ -1243,40 +1294,57 @@ function retireLegacyGatewayForDockerDriverUpgrade(): void { } } -function logDockerDriverGatewayRestart(reason: string): void { - console.log(` Existing OpenShell Docker-driver gateway is stale (${reason}); restarting...`); +function logManagedDriverGatewayRestart(driverLabel: string, reason: string): void { + console.log( + ` Existing OpenShell ${driverLabel}-driver gateway is stale (${reason}); restarting...`, + ); } async function refreshDockerDriverGatewayReuseState( gatewayReuseState: GatewayReuseState, ): Promise { - if (!isLinuxDockerDriverGatewayEnabled() || gatewayReuseState !== "healthy") { + const profile = getActiveManagedGatewayProfile(); + if (!profile || gatewayReuseState !== "healthy") { return gatewayReuseState; } const gatewayBin = resolveOpenShellGatewayBinary(); - const baseDesiredEnv = getDockerDriverGatewayEnv( - runCaptureOpenshell(["--version"], { ignoreError: true }), + if (!gatewayBin) { + console.log( + ` Existing OpenShell ${profile.displayName}-driver gateway cannot be verified without its gateway binary; it will be recreated.`, + ); + return "stale"; + } + let runtime: ResolvedManagedGatewayRuntime; + try { + const adapter = computeRuntime.resolveManagedGatewayRuntimeAdapter( + profile, + MANAGED_GATEWAY_RUNTIME_ADAPTERS, + ); + runtime = adapter.build({ + gatewayBin, + openshellVersionOutput: runCaptureOpenshell(["--version"], { ignoreError: true }), + profile, + stateDir: getDockerDriverGatewayStateDir(), + }); + } catch (error) { + console.log( + ` Existing OpenShell ${profile.displayName}-driver gateway runtime cannot be verified (${compactText(String(error))}); it will be recreated.`, + ); + return "stale"; + } + const desiredEnv = runtime.runtimeIdentity.desiredEnv; + const driftBin = dockerDriverGatewayLaunch.resolveDriftGatewayBin( + runtime.runtimeIdentity, + gatewayBin, ); - const runtimeIdentity = gatewayBin - ? dockerDriverGatewayLaunch.buildDockerDriverGatewayRuntimeIdentity({ - gatewayBin, - gatewayEnv: baseDesiredEnv, - stateDir: getDockerDriverGatewayStateDir(), - sandboxBin: resolveOpenShellSandboxBinary(), - gatewayName: GATEWAY_NAME, - compatContainerName: gatewayBinding.resolveGatewayCompatContainerName(GATEWAY_PORT), - }) - : null; - const desiredEnv = runtimeIdentity?.desiredEnv ?? baseDesiredEnv; - const driftBin = dockerDriverGatewayLaunch.resolveDriftGatewayBin(runtimeIdentity, gatewayBin); - const identityBin = runtimeIdentity?.identityGatewayBin ?? gatewayBin; + const identityBin = runtime.runtimeIdentity.identityGatewayBin ?? gatewayBin; const managedServicePid = gatewayService.getTrustedActiveOpenShellGatewayUserServicePid(); const pid = getDockerDriverGatewayPid(); if (pid !== null && isDockerDriverGatewayProcessAlive()) { const drift = getGatewayReuseDrift(pid, desiredEnv, driftBin, managedServicePid); if (drift) { console.log( - ` Existing OpenShell Docker-driver gateway is stale (${drift.reason}); it will be recreated.`, + ` Existing OpenShell ${profile.displayName}-driver gateway is stale (${drift.reason}); it will be recreated.`, ); return "stale"; } @@ -1292,7 +1360,7 @@ async function refreshDockerDriverGatewayReuseState( if (dockerGatewayPid !== managedServicePid) rememberDockerDriverGatewayPid(dockerGatewayPid); if (drift) { console.log( - ` Existing OpenShell Docker-driver gateway is stale (${drift.reason}); it will be recreated.`, + ` Existing OpenShell ${profile.displayName}-driver gateway is stale (${drift.reason}); it will be recreated.`, ); return "stale"; } @@ -1309,17 +1377,22 @@ async function refreshDockerDriverGatewayReuseState( function destroyGateway( clearRegistry: () => void = registry.clearAll, - isDockerDriverGatewayEnabledForDestroy: () => boolean = isLinuxDockerDriverGatewayEnabled, + isManagedDriverGatewayEnabledForDestroy: () => boolean = isManagedDriverGatewayEnabled, ): boolean { return destroyGatewayWithVolumeCleanup({ clearRegistry, dockerRemoveVolumesByPrefix, gatewayName: GATEWAY_NAME, hasLifecycleCommands: () => gatewayCliSupportsLifecycleCommands(runCaptureOpenshell), - isDockerDriverGatewayEnabled: isDockerDriverGatewayEnabledForDestroy, + isDockerDriverGatewayEnabled: () => + isManagedDriverGatewayEnabledForDestroy() && isActiveDockerComputeDriver(), + isManagedDriverGatewayEnabled: isManagedDriverGatewayEnabledForDestroy, removeDockerDriverGatewayRegistration, runOpenshell, stopDockerDriverGatewayProcess, + shouldCleanupLegacyDockerVolumes: () => + isManagedDriverGatewayEnabledForDestroy() && + activeManagedGatewayHasCapability("legacyDockerVolumeCleanup"), }); } @@ -1444,7 +1517,7 @@ function attachGatewayMetadataIfNeeded({ // flow explicitly forces a refresh after recreating bootstrap secrets. if (!forceRefresh && hasStaleGateway(gwInfo)) return true; - if (isLinuxDockerDriverGatewayEnabled()) { + if (isManagedDriverGatewayEnabled()) { return registerDockerDriverGatewayEndpoint(); } @@ -1475,14 +1548,21 @@ async function preflight( preflightOpts, { nonInteractive: isNonInteractive(), + computePlan: ACTIVE_OPEN_SHELL_COMPUTE_PLAN, + env: process.env, }, ); - await preflightUtils.checkContainerRuntimeResources(host, { - ignored: process.env.NEMOCLAW_IGNORE_RUNTIME_RESOURCES === "1", - nonInteractive: isNonInteractive(), - confirm: () => promptYesNoOrDefault(" Continue with onboarding?", null, false), - }); + const runtimePreflightBehavior = fatalRuntimePreflight.resolveFatalRuntimePreflightDriverBehavior( + ACTIVE_OPEN_SHELL_COMPUTE_PLAN, + ); + if (runtimePreflightBehavior.checkContainerRuntimeResources) { + await preflightUtils.checkContainerRuntimeResources(host, { + ignored: process.env.NEMOCLAW_IGNORE_RUNTIME_RESOURCES === "1", + nonInteractive: isNonInteractive(), + confirm: () => promptYesNoOrDefault(" Continue with onboarding?", null, false), + }); + } ensureOpenshellForOnboard(); // Bind the one lifecycle authority before applying any legacy listener-name @@ -1513,8 +1593,9 @@ async function preflight( gatewayReuseState = await runPreflightGatewaySequence({ gatewayReuseState, externallySupervised: gatewayExternallySupervised, - supportsLifecycleCommands: gatewayCliSupportsLifecycleCommands(runCaptureOpenshell), - isDockerDriverGatewayEnabled: isLinuxDockerDriverGatewayEnabled(), + supportsLifecycleCommands: + !isManagedDriverGatewayEnabled() && gatewayCliSupportsLifecycleCommands(runCaptureOpenshell), + isDockerDriverGatewayEnabled: isManagedDriverGatewayEnabled(), gatewayName: GATEWAY_NAME, cliDisplayName: cliDisplayName(), dashboardPort: getOnboardDashboardPort(), @@ -1585,11 +1666,12 @@ async function preflight( if (portCheck.ok) continue; } if (kind === "gateway") { - const dockerGatewayPid = getDockerDriverGatewayPortListenerPid(portCheck); - if (dockerGatewayPid !== null) { - rememberDockerDriverGatewayPid(dockerGatewayPid); + const managedGatewayPid = getDockerDriverGatewayPortListenerPid(portCheck); + if (managedGatewayPid !== null) { + rememberDockerDriverGatewayPid(managedGatewayPid); + const driverLabel = getActiveManagedGatewayProfile()?.displayName ?? "managed"; console.log( - ` ✓ Port ${port} already owned by NemoClaw OpenShell Docker gateway (${label})`, + ` ✓ Port ${port} already owned by NemoClaw OpenShell ${driverLabel} gateway (${label})`, ); continue; } @@ -1712,23 +1794,27 @@ async function startGatewayWithOptions( assertGatewayStartAllowed(exitOnFailure); step(2, 8, "Starting OpenShell gateway"); - if (isLinuxDockerDriverGatewayEnabled()) { - const selectedGpuRoute = dockerGpuRoute.initialDockerGpuRoute( - dockerGpuRoute.resolveDockerGpuRoutePlan( - { sandboxGpuEnabled: gpuPassthrough, hostGpuPlatform: _gpu?.platform }, - { - dockerDriverGateway: true, - dockerDesktopWsl: dockerGpuSandboxCreate.isDockerDesktopWslRuntime(), - }, - ), - ); - return startDockerDriverGateway({ + if (isManagedDriverGatewayEnabled()) { + const selectedGpuRoute = isActiveDockerComputeDriver() + ? dockerGpuRoute.initialDockerGpuRoute( + dockerGpuRoute.resolveDockerGpuRoutePlan( + { sandboxGpuEnabled: gpuPassthrough, hostGpuPlatform: _gpu?.platform }, + { + dockerDriverGateway: true, + dockerDesktopWsl: dockerGpuSandboxCreate.isDockerDesktopWslRuntime(), + }, + ), + ) + : null; + return startManagedDriverGateway({ exitOnFailure, - skipSandboxBridgeReachability: dockerGpuLocalInference.shouldSkipGpuBridgeProbe( - gpuPassthrough, - _gpu?.platform, - selectedGpuRoute, - ), + skipSandboxBridgeReachability: + selectedGpuRoute !== null && + dockerGpuLocalInference.shouldSkipGpuBridgeProbe( + gpuPassthrough, + _gpu?.platform, + selectedGpuRoute, + ), }); } @@ -1897,46 +1983,278 @@ async function startGatewayWithOptions( process.env.OPENSHELL_GATEWAY = GATEWAY_NAME; } +type ManagedGatewayRuntimeAdapterContext = { + gatewayBin: string; + openshellVersionOutput: string; + profile: import("./onboard/compute/managed-gateway-profile").ManagedGatewayDriverProfile; + stateDir: string; +}; + +type ResolvedManagedGatewayRuntime = { + gatewayEnv: Record; + runtimeDiagnostics: readonly string[]; + runtimeIdentity: import("./onboard/docker-driver-gateway-launch").ManagedDriverGatewayRuntimeIdentity; + verifySandboxReachability(exitOnFailure: boolean, options?: { skip?: boolean }): Promise; + writeRuntimeMarker?(pid: number): void; +}; + +type ManagedGatewayRuntimeAdapter = + import("./onboard/compute/managed-gateway-profile").ManagedGatewayRuntimeAdapter & { + build(context: ManagedGatewayRuntimeAdapterContext): ResolvedManagedGatewayRuntime; + }; + +const MANAGED_GATEWAY_RUNTIME_ADAPTERS = { + docker: { + driverName: "docker", + launchPolicy: "docker-compat", + runtimeMarkerPolicy: "docker-compat-v1", + sandboxReachability: "docker-bridge", + build({ + gatewayBin, + openshellVersionOutput, + profile, + stateDir, + }: ManagedGatewayRuntimeAdapterContext): ResolvedManagedGatewayRuntime { + if (profile.launchPolicy !== "docker-compat") { + throw new Error("Docker managed-gateway profile must use docker-compat launch policy"); + } + const gatewayEnv = getDockerDriverGatewayEnv(openshellVersionOutput); + const runtimeIdentity = dockerDriverGatewayLaunch.buildDockerDriverGatewayRuntimeIdentity({ + gatewayBin, + gatewayEnv, + stateDir, + sandboxBin: profile.capabilities.localSupervisorBinary + ? resolveOpenShellSandboxBinary() + : null, + gatewayName: GATEWAY_NAME, + compatContainerName: gatewayBinding.resolveGatewayCompatContainerName(GATEWAY_PORT), + ensureLocalTlsBundle: true, + removeEnvironmentKeys: profile.incompatibleRuntimeEnvironmentKeys, + }); + const { verifySandboxBridgeGatewayReachableOrExit } = + require("./onboard/gateway-sandbox-reachability") as typeof import("./onboard/gateway-sandbox-reachability"); + return { + gatewayEnv, + runtimeDiagnostics: ["docker info --format '{{json .CDISpecDirs}}'"], + runtimeIdentity, + verifySandboxReachability: (fail, options) => + verifySandboxBridgeGatewayReachableOrExit(fail, { + skip: options?.skip, + port: GATEWAY_PORT, + }), + writeRuntimeMarker: (pid) => + dockerDriverGatewayRuntimeMarker.writeDockerDriverGatewayRuntimeMarkerForStateDir( + stateDir, + { + pid, + desiredEnv: runtimeIdentity.desiredEnv, + endpoint: getDockerDriverGatewayEndpoint(), + gatewayBin: runtimeIdentity.driftGatewayBin, + openshellVersion: getInstalledOpenshellVersion(openshellVersionOutput), + dockerHost: process.env.DOCKER_HOST || null, + }, + ), + }; + }, + }, + podman: { + driverName: "podman", + launchPolicy: "host-only", + runtimeMarkerPolicy: "process-env", + sandboxReachability: "podman-host", + build({ + gatewayBin, + openshellVersionOutput, + profile, + stateDir, + }: ManagedGatewayRuntimeAdapterContext): ResolvedManagedGatewayRuntime { + if (profile.launchPolicy !== "host-only") { + throw new Error("Podman managed-gateway profile must remain host-only"); + } + const podmanSocketPath = process.env.OPENSHELL_PODMAN_SOCKET?.trim(); + if (!podmanSocketPath) { + throw new Error("Qualified Podman runtime did not provide OPENSHELL_PODMAN_SOCKET"); + } + const qualifiedPodman = computeRuntime.assessNativePodman({ env: process.env }); + if (qualifiedPodman.socketPath !== podmanSocketPath) { + throw new Error("Qualified Podman socket does not match the managed gateway runtime."); + } + computeRuntime.ensureManagedGatewayLocalTlsBundle({ + additionalServerDnsSans: ["host.containers.internal"], + gatewayBin, + stateDir, + }); + const gatewayEnv = computeRuntime.buildPodmanDriverGatewayEnv({ + gatewayPort: GATEWAY_PORT, + stateDir, + podmanSocketPath, + podmanNetworkName: process.env.OPENSHELL_PODMAN_NETWORK_NAME || "openshell", + supervisorImage: getOpenShellSupervisorImage(openshellVersionOutput), + }); + const runtimeIdentity = dockerDriverGatewayLaunch.buildHostManagedGatewayRuntimeIdentity({ + gatewayBin, + gatewayEnv, + gatewayName: GATEWAY_NAME, + removeEnvironmentKeys: profile.incompatibleRuntimeEnvironmentKeys, + runtimeEnvironmentKeys: profile.runtimeEnvironmentKeys, + }); + return { + gatewayEnv, + runtimeDiagnostics: [ + `podman --url unix://${podmanSocketPath} info`, + `podman --url unix://${podmanSocketPath} network inspect ${ + process.env.OPENSHELL_PODMAN_NETWORK_NAME || "openshell" + }`, + ], + runtimeIdentity, + verifySandboxReachability: (fail, options) => + computeRuntime.verifyPodmanSandboxGatewayReachableOrExit(fail, { + skip: options?.skip, + networkName: process.env.OPENSHELL_PODMAN_NETWORK_NAME || "openshell", + podmanSocketPath, + port: GATEWAY_PORT, + redact, + socketAuthority: qualifiedPodman.socketAuthority, + }), + }; + }, + }, +} as const satisfies Record; + +function createActivePodmanWatcherController(): import("./onboard/compute/podman/sandbox-create-authority").PodmanSandboxCreateRuntimeAuthority { + if (ACTIVE_OPEN_SHELL_COMPUTE_PLAN.driverName !== "podman") { + throw new Error("Podman watcher controller requires the active Podman compute driver."); + } + const profile = getActiveManagedGatewayProfile(); + if (!profile || profile.driverName !== "podman") { + throw new Error("Podman watcher controller requires a NemoClaw-managed Podman gateway."); + } + const gatewayBin = resolveOpenShellGatewayBinary(); + if (!gatewayBin) + throw new Error("Podman watcher controller could not resolve the gateway binary."); + const stateDir = getDockerDriverGatewayStateDir(); + const adapter = computeRuntime.resolveManagedGatewayRuntimeAdapter( + profile, + MANAGED_GATEWAY_RUNTIME_ADAPTERS, + ); + const qualifiedPodman = computeRuntime.assessNativePodman({ env: process.env }); + const configuredSocketPath = process.env.OPENSHELL_PODMAN_SOCKET?.trim(); + if (!configuredSocketPath || qualifiedPodman.socketPath !== configuredSocketPath) { + throw new Error("Qualified Podman socket does not match the active compute runtime."); + } + const runtime = adapter.build({ + gatewayBin, + openshellVersionOutput: runCaptureOpenshell(["--version"], { ignoreError: true }), + profile, + stateDir, + }); + const socketPath = runtime.gatewayEnv.OPENSHELL_PODMAN_SOCKET?.trim(); + if (!socketPath) { + throw new Error( + "Podman watcher controller requires the exact host launch and socket identity.", + ); + } + return computeRuntime.createPodmanSandboxCreateRuntimeAuthority({ + driverLabel: profile.displayName, + gatewayBin, + gatewayName: GATEWAY_NAME, + gatewayPort: GATEWAY_PORT, + getRememberedGatewayPid: getDockerDriverGatewayPid, + getRuntimeDrift: (pid, desiredEnv, driftGatewayBin, trustedServicePid) => + getGatewayReuseDrift(pid, { ...desiredEnv }, driftGatewayBin, trustedServicePid), + isGatewayHealthy: () => + isGatewayHealthy( + runCaptureOpenshell(["status"], { ignoreError: true }), + runCaptureOpenshell(["gateway", "info", "-g", GATEWAY_NAME], { + ignoreError: true, + }), + runCaptureOpenshell(["gateway", "info"], { ignoreError: true }), + ), + isPidAlive, + rememberGatewayPid: rememberDockerDriverGatewayPid, + runtimeIdentity: runtime.runtimeIdentity, + socketAuthority: qualifiedPodman.socketAuthority, + socketPath, + stateDir, + }); +} + +const ACTIVE_SANDBOX_RUNTIME_AUTHORITY_ADAPTERS = { + docker: { driverName: "docker", resolve: () => null }, + kubernetes: { driverName: "kubernetes", resolve: () => null }, + podman: { + driverName: "podman", + resolve: createActivePodmanWatcherController, + revalidate: (authority: unknown) => { + const podmanAuthority = authority as Partial< + import("./onboard/compute/podman/sandbox-create-authority").PodmanSandboxCreateRuntimeAuthority + > | null; + if ( + !podmanAuthority || + typeof podmanAuthority.socketPath !== "string" || + !podmanAuthority.socketAuthority || + podmanAuthority.socketAuthority.socketPath !== podmanAuthority.socketPath + ) { + throw new Error("Podman sandbox mutation requires its exact socket authority."); + } + computeRuntime.assertPodmanSocketAuthority(podmanAuthority.socketAuthority); + }, + }, +} as const satisfies import("./onboard/compute/runtime-authority").SandboxRuntimeAuthorityAdapterRegistry; + /** - * Reconcile or create the host Docker-driver gateway. The public onboard() + * Reconcile or create a NemoClaw-managed host gateway. The public onboard() * entrypoint holds acquireOnboardLock()'s atomic cross-process filesystem lock - * (created with openSync("wx")) across this whole call, so separate concurrent - * `nemoclaw onboard` CLI processes cannot race creation. - * The strict post-reap bind check below remains a second boundary against - * recovery commands or external processes that do not participate in that - * lock; the OS then permits only one child to bind the port. + * across this whole call. The strict post-reap bind check remains a second + * boundary for recovery commands or external processes outside that lock. */ -async function startDockerDriverGateway({ +async function startManagedDriverGateway({ exitOnFailure = true, skipSandboxBridgeReachability = false, }: { exitOnFailure?: boolean; skipSandboxBridgeReachability?: boolean; } = {}): Promise { + const profile = getActiveManagedGatewayProfile(); + if (!profile) { + throw new Error( + `OpenShell compute driver '${ACTIVE_OPEN_SHELL_COMPUTE_PLAN.driverName}' is not NemoClaw-managed.`, + ); + } + const adapter = computeRuntime.resolveManagedGatewayRuntimeAdapter( + profile, + MANAGED_GATEWAY_RUNTIME_ADAPTERS, + ); const gatewayBin = resolveOpenShellGatewayBinary(); + if (!gatewayBin) { + console.error(` OpenShell ${profile.displayName}-driver gateway binary not found.`); + console.error( + ` Install OpenShell v${SUPPORTED_OPENSHELL_FALLBACK_VERSION}, or set NEMOCLAW_OPENSHELL_GATEWAY_BIN.`, + ); + if (exitOnFailure) process.exit(1); + throw new Error("OpenShell gateway binary not found"); + } const openshellVersionOutput = runCaptureOpenshell(["--version"], { ignoreError: true }); - const gatewayEnv = getDockerDriverGatewayEnv(openshellVersionOutput); const stateDir = getDockerDriverGatewayStateDir(); - const runtimeIdentity = gatewayBin - ? dockerDriverGatewayLaunch.buildDockerDriverGatewayRuntimeIdentity({ - gatewayBin, - gatewayEnv, - stateDir, - sandboxBin: resolveOpenShellSandboxBinary(), - gatewayName: GATEWAY_NAME, - compatContainerName: gatewayBinding.resolveGatewayCompatContainerName(GATEWAY_PORT), - ensureLocalTlsBundle: true, - }) - : null; - const gatewayLaunch = runtimeIdentity?.launch ?? null; + const runtime = adapter.build({ + gatewayBin, + openshellVersionOutput, + profile, + stateDir, + }); + const gatewayEnv = runtime.gatewayEnv; + if (gatewayEnv.OPENSHELL_LOCAL_TLS_DIR) { + process.env.OPENSHELL_LOCAL_TLS_DIR = gatewayEnv.OPENSHELL_LOCAL_TLS_DIR; + } + const runtimeIdentity = runtime.runtimeIdentity; + const gatewayLaunch = runtimeIdentity.launch; const driftGatewayBin = dockerDriverGatewayLaunch.resolveDriftGatewayBin( runtimeIdentity, gatewayBin, ); - const driftGatewayEnv = runtimeIdentity?.desiredEnv ?? gatewayEnv; - const identityGatewayBin = runtimeIdentity?.identityGatewayBin ?? gatewayBin; - const { verifySandboxBridgeGatewayReachableOrExit } = - require("./onboard/gateway-sandbox-reachability") as typeof import("./onboard/gateway-sandbox-reachability"); + const driftGatewayEnv = runtimeIdentity.desiredEnv; + const identityGatewayBin = runtimeIdentity.identityGatewayBin ?? gatewayBin; + const verifySandboxReachability = runtime.verifySandboxReachability; const initialPortCheck = await checkGatewayPortAvailable(); const servicePortOwnership = createGatewayServicePortOwnership(initialPortCheck, { exitOnFailure, @@ -1950,8 +2268,12 @@ async function startDockerDriverGateway({ }), }); if ( - await dockerDriverGatewayEnv.startPackageManagedDockerDriverGatewayWithEnvOverride({ + profile.capabilities.packageManagedService && + (await dockerDriverGatewayEnv.startPackageManagedDriverGatewayWithEnvOverride({ + allowWildcardBind: profile.allowWildcardBind, clearDockerDriverGatewayRuntimeFiles, + driverLabel: profile.displayName, + driverName: profile.driverName, exitOnFailure, gatewayEnv: driftGatewayEnv, gatewayName: GATEWAY_NAME, @@ -1962,20 +2284,17 @@ async function startDockerDriverGateway({ runCaptureOpenshell, skipSandboxBridgeReachability, validatePortOwnerForOpenShellGatewayUserServiceStart: servicePortOwnership.validatePortOwner, - verifySandboxBridgeGatewayReachableOrExit: (fail, options) => - verifySandboxBridgeGatewayReachableOrExit(fail, { - ...options, - port: GATEWAY_PORT, - }), - }) + verifySandboxBridgeGatewayReachableOrExit: verifySandboxReachability, + })) ) return; const initialHealth = dockerDriverGatewayCutover.readDockerDriverGatewayHealth( runCaptureOpenshell, GATEWAY_NAME, ); - const cutover = await dockerDriverGatewayCutover.runDockerDriverGatewayCutover( + const cutover = await dockerDriverGatewayCutover.runManagedDriverGatewayCutover( { + driverLabel: profile.displayName, gatewayBin, identityGatewayBin, driftGatewayBin, @@ -1991,15 +2310,12 @@ async function startDockerDriverGateway({ isDockerDriverGatewayProcessAlive, isGatewayHealthy, getDockerDriverGatewayRuntimeDrift, - logDockerDriverGatewayRestart, + logDockerDriverGatewayRestart: (reason) => + logManagedDriverGatewayRestart(profile.displayName, reason), registerDockerDriverGatewayEndpoint, isDockerDriverGatewayHttpReady: () => isDockerDriverGatewayHttpReady(undefined, undefined, driftGatewayEnv), - verifySandboxBridgeGatewayReachableOrExit: (fail, options) => - verifySandboxBridgeGatewayReachableOrExit(fail, { - ...options, - port: GATEWAY_PORT, - }), + verifySandboxBridgeGatewayReachableOrExit: verifySandboxReachability, readGatewayHealth: () => ({ status: runCaptureOpenshell(["status"], { ignoreError: true }), namedInfo: runCaptureOpenshell(["gateway", "info", "-g", GATEWAY_NAME], { @@ -2016,25 +2332,23 @@ async function startDockerDriverGateway({ }, reportUntrustedGatewayPort: servicePortOwnership.reportUntrustedGatewayPort, reportMissingGatewayBinary: () => { - console.error(" OpenShell Docker-driver gateway binary not found."); - console.error( - ` Install OpenShell v${SUPPORTED_OPENSHELL_FALLBACK_VERSION}, or set NEMOCLAW_OPENSHELL_GATEWAY_BIN.`, - ); - if (exitOnFailure) process.exit(1); - throw new Error("OpenShell gateway binary not found"); + throw new Error("OpenShell gateway binary disappeared during managed cutover"); }, log: (message) => console.log(message), }, ); if (cutover === "reused") return; - if (!gatewayBin || !gatewayLaunch) { + if (!gatewayLaunch) { throw new Error("OpenShell gateway launch missing after cutover"); } fs.mkdirSync(stateDir, { recursive: true, mode: 0o700 }); const logPath = path.join(stateDir, "openshell-gateway.log"); - const logFd = dockerDriverGatewayLaunch.openDockerDriverGatewayLog(logPath, { exitOnFailure }); - console.log(" Starting OpenShell Docker-driver gateway..."); + const logFd = dockerDriverGatewayLaunch.openManagedDriverGatewayLog(logPath, { + driverLabel: profile.displayName, + exitOnFailure, + }); + console.log(` Starting OpenShell ${profile.displayName}-driver gateway...`); console.log(` Gateway log: ${logPath}`); const launch = gatewayLaunch; dockerDriverGatewayLaunch.prepareAndLogDockerDriverGatewayLaunch(launch); @@ -2046,21 +2360,11 @@ async function startDockerDriverGateway({ throw new Error("OpenShell gateway process did not return a pid"); } rememberDockerDriverGatewayPid(childPid); - dockerDriverGatewayRuntimeMarker.writeDockerDriverGatewayRuntimeMarkerForStateDir( - getDockerDriverGatewayStateDir(), - { - pid: childPid, - desiredEnv: driftGatewayEnv, - endpoint: getDockerDriverGatewayEndpoint(), - gatewayBin: driftGatewayBin, - openshellVersion: getInstalledOpenshellVersion(openshellVersionOutput), - dockerHost: process.env.DOCKER_HOST || null, - }, - ); + runtime.writeRuntimeMarker?.(childPid); const pollCount = envInt("NEMOCLAW_HEALTH_POLL_COUNT", 30); const pollInterval = envInt("NEMOCLAW_HEALTH_POLL_INTERVAL", 2); - const gatewayStartup = await waitForStandaloneDockerDriverGateway({ + const gatewayStartup = await waitForStandaloneManagedDriverGateway({ childExited: () => childExit.exited, childPid, gatewayName: GATEWAY_NAME, @@ -2070,9 +2374,8 @@ async function startDockerDriverGateway({ isGatewayTcpReady, isPidAlive, onHealthy: async () => { - await verifySandboxBridgeGatewayReachableOrExit(exitOnFailure, { + await verifySandboxReachability(exitOnFailure, { skip: skipSandboxBridgeReachability, - port: GATEWAY_PORT, }); }, registerGatewayEndpoint: registerDockerDriverGatewayEndpoint, @@ -2080,16 +2383,22 @@ async function startDockerDriverGateway({ sleepSeconds, }); if (gatewayStartup === "healthy") { - console.log(" ✓ Docker-driver gateway is healthy"); + console.log(` ✓ ${profile.displayName}-driver gateway is healthy`); return; } - reportDockerDriverGatewayStartFailure(logPath, childExit, { exitOnFailure }); + reportManagedDriverGatewayStartFailure(logPath, childExit, { + driverLabel: profile.displayName, + exitOnFailure, + runtimeDiagnostics: runtime.runtimeDiagnostics, + }); if (gatewayStartup === "exited") { - throw new Error("Docker-driver gateway failed to start because the process exited"); + throw new Error( + `${profile.displayName}-driver gateway failed to start because the process exited`, + ); } const waitLimit = formatGatewayHealthWaitLimit(pollCount, pollInterval); - throw new Error(`Docker-driver gateway failed to start within ${waitLimit}`); + throw new Error(`${profile.displayName}-driver gateway failed to start within ${waitLimit}`); } async function startGateway( @@ -2099,15 +2408,64 @@ async function startGateway( return startGatewayWithOptions(_gpu, { exitOnFailure: true, gpuPassthrough }); } -async function startGatewayForRecovery(options = {}): Promise { - return require("./onboard/gateway-recovery").startGatewayForRecovery(options, { - assertGatewayStartAllowed, - getGatewayStartEnv, - runCaptureOpenshell, - runOpenshell, - startGatewayWithOptions, - isLinuxDockerDriverGatewayEnabled, - }); +async function startGatewayForRecovery( + options: import("./onboard/gateway-recovery").StartGatewayForRecoveryOptions = {}, +): Promise { + const previousComputePlan = ACTIVE_OPEN_SHELL_COMPUTE_PLAN; + const previousRecoveryRuntimeEnvironment = new Map(); + try { + if (options.computeDriver) { + const requestedDriver = options.computeDriver === "vm" ? "docker" : options.computeDriver; + ACTIVE_OPEN_SHELL_COMPUTE_PLAN = computeRuntime.resolveOpenShellComputeSelection({ + requestedDriver, + autoPlan: computeRuntime.resolveCurrentOpenShellComputePlan(), + }); + if (computeRuntime.supportsManagedGatewayRecoveryRuntime(requestedDriver)) { + const recoveryGatewayName = + options.gatewayName ?? + gatewayBinding.resolveGatewayName(options.gatewayPort ?? GATEWAY_PORT); + const recoveryRuntime = computeRuntime.resolveManagedGatewayRecoveryRuntime({ + driverName: requestedDriver, + environment: process.env, + stateDir: gatewayBinding.resolveManagedGatewayStateDirectory(recoveryGatewayName), + }); + computeRuntime.qualifyManagedGatewayRecoveryRuntime(recoveryRuntime); + for (const key of Object.keys(recoveryRuntime.environment)) { + previousRecoveryRuntimeEnvironment.set(key, process.env[key]); + } + Object.assign(process.env, recoveryRuntime.environment); + } + } + const gatewayRecovery: typeof import("./onboard/gateway-recovery") = + require("./onboard/gateway-recovery"); + return await gatewayRecovery.startGatewayForRecovery(options, { + assertGatewayStartAllowed, + getGatewayStartEnv, + runCaptureOpenshell, + runOpenshell, + startGatewayWithOptions: async (gpu, recoveryTarget) => { + const previousGatewayName = GATEWAY_NAME; + const previousGatewayPort = GATEWAY_PORT; + try { + GATEWAY_NAME = recoveryTarget.gatewayName; + GATEWAY_PORT = recoveryTarget.gatewayPort; + return await startGatewayWithOptions(gpu, { + exitOnFailure: recoveryTarget.exitOnFailure, + }); + } finally { + GATEWAY_NAME = previousGatewayName; + GATEWAY_PORT = previousGatewayPort; + } + }, + isManagedDriverGatewayEnabled, + }); + } finally { + ACTIVE_OPEN_SHELL_COMPUTE_PLAN = previousComputePlan; + for (const [key, previous] of previousRecoveryRuntimeEnvironment) { + if (previous === undefined) delete process.env[key]; + else process.env[key] = previous; + } + } } const applyOverlayfsAutoFix = overlayfsAutoFix.createOverlayfsAutoFix({ @@ -2139,9 +2497,9 @@ const { async function recoverGatewayRuntime() { assertGatewayStartAllowed(false); - if (isLinuxDockerDriverGatewayEnabled()) { + if (isManagedDriverGatewayEnabled()) { try { - await startDockerDriverGateway({ exitOnFailure: false }); + await startManagedDriverGateway({ exitOnFailure: false }); return true; } catch { return false; @@ -2207,7 +2565,7 @@ async function recoverGatewayRuntime() { const { getSandboxRuntimeRegistryFields, hasSandboxGpuDrift, updateReusedSandboxMetadata } = sandboxRegistryMetadata.createSandboxRegistryMetadataHelpers({ - isLinuxDockerDriverGatewayEnabled, + getOpenShellComputeDriverName: () => ACTIVE_OPEN_SHELL_COMPUTE_PLAN.driverName, getInstalledOpenshellVersion, runCaptureOpenshell, }); @@ -2216,6 +2574,8 @@ const { getSandboxRuntimeRegistryFields, hasSandboxGpuDrift, updateReusedSandbox async function createSandboxWithBaseImageResolution( baseImageResolutionContext: import("./onboard/base-image-resolution-flow").BaseImageResolutionContext, + computePlan: import("./onboard/compute/plan").OpenShellComputePlan, + managedWorkloadRebuild: import("./onboard/workload/rebuild").ManagedWorkloadRebuildHandoff | null, gpu: ReturnType, model: string, provider: string, @@ -2239,6 +2599,12 @@ async function createSandboxWithBaseImageResolution( "sandbox name", ); preparedDcodeRebuild.assertPreparedDcodeTarget(preparedBuildContext, agent, fromDockerfile); + const effectiveAgent = sandboxAgent.getEffectiveSandboxAgent(agent); + const requestedAgentName = getRequestedSandboxAgentName(effectiveAgent); + const legacyDockerfilePath = + effectiveAgent.dockerfilePath ?? + effectiveAgent.legacyPaths?.dockerfile ?? + path.join(ROOT, "Dockerfile"); enabledChannels = filterEnabledChannelsByAgent(enabledChannels, agent); const effectiveSandboxGpuConfig = sandboxGpuConfig ?? resolveSandboxGpuConfig(gpu, { flag: null, device: null }); @@ -2286,6 +2652,19 @@ async function createSandboxWithBaseImageResolution( // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. const { existingEntry, preservedMcpState, liveExists, effectiveToolDisclosure, toolDisclosureMigrationNeeded, toolDisclosureMigrationNote } = toolDisclosureFlow.prepareSandboxToolDisclosure(sandboxName, preparedBuildContext?.rebuildTarget?.fromDockerfile ? preparedBuildContext.stagedDockerfile : fromDockerfile, isRecreateSandbox(createIntent?.recreate), inspectSandboxForCreate, createIntent?.toolDisclosure ?? null); + if (existingEntry) { + // The name may be selected interactively after the early session/registry + // binding pass. Re-check the surviving registry row before reuse or + // destructive recreation so "auto" can never migrate a sandbox between + // runtime drivers implicitly. + computeRuntime.resolveOpenShellComputeSelection({ + requestedDriver: computePlan.driverName, + persistedDriver: + existingEntry.openshellDriver ?? + computeRuntime.resolveCurrentOpenShellComputePlan().driverName, + autoPlan: computePlan, + }); + } // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. const recreateRuntime = sandboxRecreateTransaction.createSandboxRecreateRuntime(onboardSession, createIntent?.recreateTransaction, sandboxName, GATEWAY_NAME, existingEntry, getSandboxRecreateObservation, note); const restoreReusedSandboxDashboard = (selectionVerified: boolean): void => { @@ -2314,6 +2693,20 @@ async function createSandboxWithBaseImageResolution( const observabilityDrift = observabilityPolicy.hasRegisteredDcodeObservabilityDrift(liveExists, isManagedDcodeAgent, existingEntry, createIntent?.observabilityEnabled); // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. const dcodeAutoApprovalPlan = dcodeAutoApprovalFlow.prepareDcodeAutoApprovalCreatePlan({ sandboxName, liveExists, managedDcodeAgent: isManagedDcodeAgent, registryEntry: existingEntry, requestedMode: createIntent?.dcodeAutoApprovalMode }, { error: console.error, exitProcess: (code) => process.exit(code) }); + const envMessagingState = MessagingHostStateApplier.readPlanStateFromEnv(); + const plannedMessagingState = + envMessagingState?.plan.sandboxName === sandboxName ? envMessagingState : undefined; + // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. + const managedWorkloadRuntime = managedWorkloadOnboard.createManagedWorkloadOnboardRuntime({ + computePlan, managedWorkloadRebuild, agentName: requestedAgentName, legacyDockerfilePath, customDockerfilePath: fromDockerfile ?? (preparedBuildContext ? preparedBuildContext.stagedDockerfile : null), + rootDir: ROOT, model, provider, preferredInferenceApi, + endpointUrl: createIntent?.endpointUrl ?? null, + startupProfile: { chatUiUrl, effectiveDashboardPort: effectivePort, manageDashboard, dashboardBindAddress: process.env.NEMOCLAW_DASHBOARD_BIND, wslExposure: requestedAgentName === "openclaw" && isWsl(), hermesDashboardState, webSearch: webSearchConfig, toolDisclosure: effectiveToolDisclosure, hermesToolGateways, messagingPlan: plannedMessagingState?.plan ?? null, dcodeAutoApprovalMode: dcodeAutoApprovalPlan.mode, observabilityEnabled: createIntent?.observabilityEnabled === true, environment: process.env }, + note, + fallbackBuildEstimate: () => process.env.NEMOCLAW_IGNORE_RUNTIME_RESOURCES === "1" ? null : formatSandboxBuildEstimateNote(assessHost(), "trusted-dockerfile-fallback"), + }, { + resolveAgentInferenceApi: inferenceConfig.resolveAgentInferenceApi, getSandboxInferenceConfig, + }); // #4614: capture default AFTER prune so a stale registry row isn't read as a live sandbox. const sandboxWasLiveDefault = liveExists && wasSandboxDefault(registry.getDefault(), sandboxName); @@ -2328,10 +2721,10 @@ async function createSandboxWithBaseImageResolution( note, }); let pendingStateRestoreBackupPath = recreateProtection.selectPreUpgradeBackup(liveExists); + let sandboxRuntimeAuthority: unknown = null; if (liveExists) { const existingSandboxState = getSandboxReuseState(sandboxName); - const requestedAgentName = getRequestedSandboxAgentName(agent); const agentDrift = getSandboxAgentDrift(sandboxName, requestedAgentName); let recreateForAgentDrift = agentDrift.changed && isRecreateSandbox(createIntent?.recreate); @@ -2564,163 +2957,117 @@ async function createSandboxWithBaseImageResolution( baseImageResolutionFlow.captureBaseResolution(baseImageResolutionContext, previousEntry?.imageTag); policyPresetCarry.applyRecreatePolicyCarryForward(sandboxName, isNonInteractive(), note); - const noRestorePending = pendingStateRestore === null && pendingStateRestoreBackupPath === null; - // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. - if (noRestorePending && !notReadyRecreateInProgress && !shouldSkipPreRecreateBackup(process.env)) { - note(" Backing up workspace state before recreating sandbox..."); - const result = recreateProtection.backup(); - if (!result.ok) { - console.error( - " Set NEMOCLAW_RECREATE_WITHOUT_BACKUP=1 to recreate without preserving state.", - ); - process.exit(1); - } - pendingStateRestore = result.backup; - } + // Resolve and validate the immutable workload before deleting the live + // sandbox. Catalog or contract failures must leave the existing workload + // untouched. + const replacementWorkload = await managedWorkloadRuntime.ensurePreparedWorkload(); + managedWorkloadRuntime.ensurePreparedProfile(replacementWorkload); - note(` Deleting and recreating sandbox '${sandboxName}'...`); + sandboxRuntimeAuthority = computeRuntime.runAuthorizedSandboxRecreateDeletion( + computePlan.driverName, + undefined, + ACTIVE_SANDBOX_RUNTIME_AUTHORITY_ADAPTERS, + { + beforeDelete() { + const noRestorePending = + pendingStateRestore === null && pendingStateRestoreBackupPath === null; + // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. + if (noRestorePending && !notReadyRecreateInProgress && !shouldSkipPreRecreateBackup(process.env)) { + note(" Backing up workspace state before recreating sandbox..."); + const result = recreateProtection.backup(); + if (!result.ok) { + console.error( + " Set NEMOCLAW_RECREATE_WITHOUT_BACKUP=1 to recreate without preserving state.", + ); + process.exit(1); + } + pendingStateRestore = result.backup; + } - recreateRuntime.advance("deleting"); - runSandboxProviderPreDeleteCleanup(sandboxName, { runOpenshell, redact }); - runOpenshell(["sandbox", "delete", sandboxName], { ignoreError: true }); - recreateRuntime.confirmDeleted(); - if (previousEntry?.imageTag) { - // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. - const rmiResult = dockerRmi(previousEntry.imageTag, { ignoreError: true, suppressOutput: true }); - if (rmiResult.status !== 0) { - console.warn(` Warning: failed to remove old sandbox image '${previousEntry.imageTag}'.`); - } - } - sandboxLifecycle.removeSandboxUnlessSessionReservation(previousEntry, sandboxName); + note(` Deleting and recreating sandbox '${sandboxName}'...`); + recreateRuntime.advance("deleting"); + runSandboxProviderPreDeleteCleanup(sandboxName, { runOpenshell, redact }); + }, + deleteSandbox() { + runOpenshell(["sandbox", "delete", sandboxName], { ignoreError: true }); + }, + afterDelete() { + recreateRuntime.confirmDeleted(); + const replacementReusesPreviousImage = + replacementWorkload.source.kind === "managed-image" && + previousEntry?.imageTag === replacementWorkload.source.reference; + if ( + previousEntry?.imageTag && + previousEntry.workload?.shared !== true && + !replacementReusesPreviousImage + ) { + // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. + const rmiResult = dockerRmi(previousEntry.imageTag, { ignoreError: true, suppressOutput: true }); + if (rmiResult.status !== 0) { + console.warn( + ` Warning: failed to remove old sandbox image '${previousEntry.imageTag}'.`, + ); + } + } + sandboxLifecycle.removeSandboxUnlessSessionReservation(previousEntry, sandboxName); + }, + }, + ); } + const preparedSandboxWorkload = await managedWorkloadRuntime.ensurePreparedWorkload(); + managedWorkloadRuntime.ensurePreparedProfile(preparedSandboxWorkload); + applyExtraProviderReconciliation({ extraProviders: resolvedCreateIntent.extraProviders, staleExtraProviders: resolvedCreateIntent.staleExtraProviders ?? [], }); - // Stage build context — use the custom Dockerfile path when provided, - // otherwise use the optimised default that only sends what the build needs. - // The build context contains source code, scripts, and potentially API keys - // in env args, so it must not persist in /tmp after a failed sandbox create. - // run() calls process.exit() on failure (bypassing normal control flow), so - // we register a process 'exit' handler to guarantee cleanup in all cases. - const { buildCtx, stagedDockerfile, origin, cleanupBuildCtx } = - preparedDcodeRebuild.resolveSandboxBuildContext( - { - preparedBuildContext, - agent, - fromDockerfile, - }, - { - createAgentSandbox: (selectedAgent) => - baseImageResolutionFlow.createAgentSandboxWithResolution( - baseImageResolutionContext, - selectedAgent, - agentOnboard.createAgentSandbox, - ), - }, + const dockerDriverGateway = isActiveDockerManagedGateway(); + if (!liveExists) { + sandboxRuntimeAuthority = computeRuntime.resolveSandboxRuntimeAuthority( + computePlan.driverName, + undefined, + ACTIVE_SANDBOX_RUNTIME_AUTHORITY_ADAPTERS, ); - // Returns true if the build context was fully removed, false otherwise. - // The caller uses this to decide whether the process 'exit' safety net - // can be deregistered — if inline cleanup fails, we leave the handler - // armed so the temp dir is still removed on process exit. - const dockerDriverGateway = isLinuxDockerDriverGatewayEnabled(); - const { gpuRoutePlan, sandboxGpuLogMessage } = resolvedCreateIntent; - const materializationCapabilities = await sandboxCreateIntentResolver.rebind( - { - sandboxName, - enabledChannels, - webSearchConfig, - agent, - ...(createIntent?.reuseRegisteredCredentials ? { reuseRegisteredCredentials: true } : {}), + } + // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. + const { initialSandboxPolicy, policyTier: resolvedCreatePolicyTier, messagingProviders, gpuRoutePlan, compatibilityPolicyPath, initialGpuRoute, sandboxReadyTimeoutSecs, buildId, dashboardRemoteBindPrepared, legacyBuildContext, launch: { createArgv, effectiveDashboardPort, managedStartupRootApplyRequest, prebuild, sandboxEnv, sandboxStartupCommand } } = await managedWorkloadOnboard.prepareOnboardSandboxWorkloadLaunch({ + runtime: managedWorkloadRuntime, workload: preparedSandboxWorkload, + legacy: { + preparedBuildContext, agent, fromDockerfile, + createAgentSandbox: (selectedAgent) => + baseImageResolutionFlow.createAgentSandboxWithResolution(baseImageResolutionContext, selectedAgent, agentOnboard.createAgentSandbox), + patchInput: { preparedBuildContext, agent, fromDockerfile, model, chatUiUrl, provider, endpointUrl: createIntent?.endpointUrl ?? null, preferredInferenceApi, webSearchConfig, toolDisclosure: effectiveToolDisclosure, ...(isManagedDcodeAgent ? { dcodeAutoApprovalMode: dcodeAutoApprovalPlan.mode } : {}), hermesToolGateways, sandboxGpuConfig: effectiveSandboxGpuConfig, ...baseImageResolutionFlow.getBaseImageResolutionPatchOptions(baseImageResolutionContext), gatewayPort: GATEWAY_PORT }, }, - resolvedCreateIntent, - ); - const { - activeMessagingChannels, - initialSandboxPolicy, - policyTier: resolvedCreatePolicyTier, - createArgs, - messagingProviders, - compatibilityPolicyPath, - } = sandboxCreatePlanMaterialization.materializeSandboxCreatePlan({ - intent: resolvedCreateIntent, - buildCtx, - messagingTokenDefs: materializationCapabilities.messagingTokenDefs, - runProviderPreDeleteCleanup: () => - runSandboxProviderPreDeleteCleanup(sandboxName, { - runOpenshell, - redact, - tolerateMissingSandbox: true, - }), - upsertMessagingProviders, - getHermesToolGatewayProviderName: (targetSandbox) => - getHermesToolGatewayBroker().getHermesToolGatewayProviderName(targetSandbox), - discloseInitialSandboxPolicy, + plan: { + intent: resolvedCreateIntent, + rebindMessagingTokenDefs: async () => + (await sandboxCreateIntentResolver.rebind({ sandboxName, enabledChannels, webSearchConfig, agent, ...(createIntent?.reuseRegisteredCredentials ? { reuseRegisteredCredentials: true } : {}) }, resolvedCreateIntent)).messagingTokenDefs, + runProviderPreDeleteCleanup: () => + runSandboxProviderPreDeleteCleanup(sandboxName, { runOpenshell, redact, tolerateMissingSandbox: true }), + upsertMessagingProviders, + getHermesToolGatewayProviderName: (targetSandbox) => + getHermesToolGatewayBroker().getHermesToolGatewayProviderName(targetSandbox), + discloseInitialSandboxPolicy, + }, + launchInput: { agent, observabilityEnabled: createIntent?.observabilityEnabled === true, chatUiUrl, sandboxName, env: process.env, extraPlaceholderKeys: resolvedCreateIntent.extraPlaceholderKeys, getDashboardForwardPort, hermesDashboardState, manageDashboard, openshellShellCommand, openshellArgv }, + plannedMessagingPlan: plannedMessagingState?.plan ?? null, + gpu: { provider, config: effectiveSandboxGpuConfig, dockerDriverGateway, gatewayPort: GATEWAY_PORT }, + dependencies: { materializeSandboxCreatePlan: sandboxCreatePlanMaterialization.materializeSandboxCreatePlan, prepareSandboxBuildPatchConfig: sandboxBuildPatchConfig.prepareSandboxBuildPatchConfig }, }); - if (initialSandboxPolicy.cleanup) { - process.on("exit", initialSandboxPolicy.cleanup); - } - if (sandboxGpuLogMessage) console.log(sandboxGpuLogMessage); - console.log(` Creating sandbox '${sandboxName}' (this takes a few minutes on first run)...`); - const envMessagingState = MessagingHostStateApplier.readPlanStateFromEnv(); - const plannedMessagingState = - envMessagingState?.plan.sandboxName === sandboxName ? envMessagingState : undefined; - const configuredMessagingChannels = - getChannelsFromPlan(plannedMessagingState?.plan) ?? activeMessagingChannels; - sandboxBuildPatchConfig.prepareSandboxBuildPatchConfig({ configuredMessagingChannels }); - const initialGpuRoute = dockerGpuRoute.initialDockerGpuRoute(gpuRoutePlan); - const { buildId, dashboardRemoteBindPrepared } = - await preparedDcodeRebuild.resolveSandboxBuildPatch({ - preparedBuildContext, - agent, - fromDockerfile, - stagedDockerfile, - model, - chatUiUrl, - provider, - endpointUrl: createIntent?.endpointUrl ?? null, - preferredInferenceApi, - webSearchConfig, - toolDisclosure: effectiveToolDisclosure, - ...(isManagedDcodeAgent ? { dcodeAutoApprovalMode: dcodeAutoApprovalPlan.mode } : {}), - hermesToolGateways, - sandboxGpuConfig: effectiveSandboxGpuConfig, - selectedGpuRoute: initialGpuRoute, - ...baseImageResolutionFlow.getBaseImageResolutionPatchOptions(baseImageResolutionContext), - gatewayPort: GATEWAY_PORT, - }); - const sandboxReadyTimeoutSecs = getSandboxReadyTimeoutSecs(effectiveSandboxGpuConfig); - const { createArgv, effectiveDashboardPort, prebuild, sandboxEnv, sandboxStartupCommand } = - await sandboxCreateLaunch.prepareSandboxCreateLaunchWithPrebuild({ - agent, - observabilityEnabled: createIntent?.observabilityEnabled === true, - chatUiUrl, - createArgs: dockerGpuRoute.renderSandboxCreateArgsForGpuRoute(createArgs, initialGpuRoute, { - compatibilityPolicyPath, - }), - sandboxName, - env: process.env, - extraPlaceholderKeys: resolvedCreateIntent.extraPlaceholderKeys, - getDashboardForwardPort, - hermesDashboardState, - manageDashboard, - openshellShellCommand, - openshellArgv, - prebuild: { buildCtx, buildId, dockerDriverGateway, origin }, - }); const restoreBackupPath = pendingStateRestore?.manifest?.backupPath ?? pendingStateRestoreBackupPath; recreateRuntime.advance("creating"); const { createResult, - dockerGpuCreatePatch, + gpuHooks, route: selectedGpuRoute, firstCreateOutput, registryImageRef, } = await sandboxGpuCreateFlow.runSandboxGpuCreateFlow( { + computeDriver: computePlan.driverName, sandboxName, provider, sandboxGpuConfig: effectiveSandboxGpuConfig, @@ -2736,7 +3083,11 @@ async function createSandboxWithBaseImageResolution( prebuild, restoreBackupPath, terminalAgent: agentDefs.isTerminalAgent(agent), - ...sandboxGpuCreateFlow.resolveDockerStartupCommandPatch(agent, dockerDriverGateway), + managedStartupRootApplyRequest, + runtimeAuthority: sandboxRuntimeAuthority, + ...computeRuntime.resolveManagedStartupRuntimeRequirements(agent, computePlan.driverName, { + managedGatewayOwned: isManagedDriverGatewayEnabled(), + }), }, { runOpenshell, @@ -2756,8 +3107,8 @@ async function createSandboxWithBaseImageResolution( // Only deregister the 'exit' safety net when inline cleanup succeeded; // otherwise leave it armed so a later process.exit() still removes the // temp dir (which may hold source and env-arg API keys). - if (cleanupBuildCtx()) { - process.removeListener("exit", cleanupBuildCtx); + if (legacyBuildContext?.cleanupBuildCtx()) { + process.removeListener("exit", legacyBuildContext.cleanupBuildCtx); } if (manageDashboard) { @@ -2779,8 +3130,8 @@ async function createSandboxWithBaseImageResolution( dockerDriverGateway, selectedRoute: selectedGpuRoute, verifyDirectSandboxGpu, - verifyGpuOrExit: dockerGpuCreatePatch.verifyGpuOrExit, - selectedMode: dockerGpuCreatePatch.selectedMode, + verifyGpuOrExit: gpuHooks.verifyGpuOrExit, + selectedMode: gpuHooks.selectedMode, runCaptureOpenshell, log: console.log, }, @@ -2801,12 +3152,18 @@ async function createSandboxWithBaseImageResolution( hermesDashboardForwarding.ensureForState(finalHermesDashboardState, sandboxName, true); } - // openshell tags images with seconds; buildId is ms. Parse actual tag from output. Fixes #2672. - const resolvedImageTag = - registryImageRef ?? - prebuild.imageRef ?? - buildContext.extractBuiltImageRef(`${firstCreateOutput}\n${createResult.output}`) ?? - resolveSandboxImageTagFromCreateOutput(`${firstCreateOutput}\n${createResult.output}`, buildId); + const { resolvedImageTag, workloadReceipt } = + managedWorkloadOnboard.resolveOnboardSandboxWorkloadReceipt({ + runtime: managedWorkloadRuntime, + workload: preparedSandboxWorkload, + registryImageRef, + prebuildImageRef: prebuild.imageRef, + firstCreateOutput, + createOutput: createResult.output, + buildId, + extractBuiltImageRef: buildContext.extractBuiltImageRef, + resolveSandboxImageTagFromCreateOutput, + }); const sandboxRuntimeFields = getSandboxRuntimeRegistryFields(effectiveSandboxGpuConfig); recreateRuntime.recordCreated(); finalizeCreatedSandbox( @@ -2842,6 +3199,7 @@ async function createSandboxWithBaseImageResolution( agent, agentVersionKnown: !fromDockerfile, imageTag: resolvedImageTag, + workload: workloadReceipt, openclawImagePluginInstalls, appliedPolicies: initialSandboxPolicy.appliedPresets, toolDisclosure: effectiveToolDisclosure, @@ -2895,13 +3253,20 @@ async function createSandboxWithBaseImageResolution( } type CreateSandboxArgs = - Parameters extends [unknown, ...infer Args] + Parameters extends [ + unknown, + unknown, + unknown, + ...infer Args, + ] ? Args : never; async function createSandbox(...args: CreateSandboxArgs): Promise { return createSandboxWithBaseImageResolution( baseImageResolutionFlow.createBaseImageResolutionContext({ fresh: false }), + ACTIVE_OPEN_SHELL_COMPUTE_PLAN, + null, ...args, ); } @@ -3138,8 +3503,9 @@ async function handleNimLocalSelection( let ngcApiKey: string | null = null; if (!nim.isNgcLoggedIn()) { if (isNonInteractive()) { + const engine = docker.hostContainerEngineDisplayName(); console.error( - " Docker is not logged in to nvcr.io. In non-interactive mode, run `docker login nvcr.io` first and retry.", + ` ${engine} is not logged in to nvcr.io. In non-interactive mode, run \`${docker.hostContainerEngineExecutable()} login nvcr.io\` first and retry.`, ); process.exit(1); } @@ -3170,7 +3536,9 @@ async function handleNimLocalSelection( if (!ngcApiKey && !isNonInteractive()) { console.log(""); console.log(" NGC API Key required to download NIM model weights at runtime."); - console.log(" (Docker is logged in to nvcr.io, but the key was not saved.)"); + console.log( + ` (${docker.hostContainerEngineDisplayName()} is logged in to nvcr.io, but the key was not saved.)`, + ); const ngcKey = await credentialPrompt.readValue(" NGC API Key: "); if (credentialPrompt.returningToProviderSelection(ngcKey)) return "retry-selection"; ngcApiKey = ngcKey || null; @@ -3544,8 +3912,16 @@ function getSetupNimDeps(): SetupNimDeps { getNonInteractiveProvider, getNonInteractiveModel, createNvidiaFeaturedModelSession, - detectInferenceProviderHostState, + detectInferenceProviderHostState: (input) => + detectInferenceProviderHostState({ + ...input, + localInferenceEnabled: activeRuntimeSupportsHostLocalInference(), + }), getAgentInferenceProviderOptions, + filterProviderOptions: (options) => + activeRuntimeSupportsHostLocalInference() + ? [...options] + : options.filter(({ key }) => Object.hasOwn(REMOTE_PROVIDER_CONFIG, key)), loadRoutedProfile: () => loadBlueprintProfile("routed"), readRecordedProvider, readRecordedNimContainer, @@ -3668,7 +4044,7 @@ const sandboxCreateIntentResolver = sandboxCreateIntentResolution.createSandboxC getAgentPolicyPath: (agent) => (agent ? agentOnboard.getAgentPolicyPath(agent) : null), resolveGpuPlan: (config) => dockerGpuSandboxCreate.resolveDockerGpuSandboxCreatePlan(config, { - dockerDriverGateway: isLinuxDockerDriverGatewayEnabled(), + dockerDriverGateway: isActiveDockerManagedGateway(), }), appendResourceCreateArgs: (args, resourceProfile) => appendResourceFlagsForProfile(args, resourceProfile, getOpenshellBinary(), { @@ -3890,6 +4266,10 @@ async function preflightAuthoritativeRebuildTarget( const fail = (message: string): never => { throw new Error(message); }; + const computePlan = computeRuntime.resolveOpenShellComputeSelection({ + requestedDriver: opts.computeDriver ?? "auto", + autoPlan: computeRuntime.resolveCurrentOpenShellComputePlan(), + }); try { await authoritativeRebuildTarget.preflightAuthoritativeRebuildTarget( { ...opts, controlUiPort: opts.controlUiPort ?? null }, @@ -3904,13 +4284,16 @@ async function preflightAuthoritativeRebuildTarget( }, { nonInteractive: true, + computePlan, + env: process.env, exitProcess: (code) => fail(`onboard runtime preflight exited with code ${String(code)}`), }, ), ensureOpenshell: () => - ensureOpenshellForOnboard((code) => - fail(`OpenShell component preflight exited with code ${String(code)}`), + ensureOpenshellForOnboard( + (code) => fail(`OpenShell component preflight exited with code ${String(code)}`), + computePlan, ), inferenceRouteReady: (p, m) => isInferenceRouteReady(authoritativeGateway.name, p, m), captureForwardList: () => runCaptureOpenshell(["forward", "list"], { ignoreError: true }), @@ -3934,8 +4317,10 @@ async function runOnboard(opts: OnboardOptions = {}): Promise { const authoritativeGateway = authoritativeRebuildTarget.resolveAuthoritativeOnboardGatewayBinding(opts); const previousGatewayBinding = { name: GATEWAY_NAME, port: GATEWAY_PORT }; + const previousComputePlan = ACTIVE_OPEN_SHELL_COMPUTE_PLAN; const previousOpenshellGateway = process.env.OPENSHELL_GATEWAY; const previousOpenshellLocalTlsDir = process.env.OPENSHELL_LOCAL_TLS_DIR; + const previousOpenshellPodmanSocket = process.env.OPENSHELL_PODMAN_SOCKET; const preparedDcodeRuntime = preparedDcodeRebuild.createPreparedDcodeRebuildRuntime( opts, authoritativeGateway?.name ?? GATEWAY_NAME, @@ -3949,6 +4334,7 @@ async function runOnboard(opts: OnboardOptions = {}): Promise { onboardRuntimeBoundary.reset(); if (!authoritativeGateway) delete process.env.OPENSHELL_GATEWAY; preparedDcodeRuntime.applyGatewayEnv(process.env); + const persistedSessionAtEntry = onboardSession.loadSession(); const { resume, fresh, requestedFromDockerfile, requestedSandboxName, cannotPrompt } = onboardEntryOptions.resolveOnboardEntryOptions( { @@ -3956,7 +4342,7 @@ async function runOnboard(opts: OnboardOptions = {}): Promise { env: process.env, stdinIsTty: Boolean(process.stdin && process.stdin.isTTY), stdoutIsTty: Boolean(process.stdout && process.stdout.isTTY), - persistedSessionStatus: onboardSession.loadSession()?.status ?? null, + persistedSessionStatus: persistedSessionAtEntry?.status ?? null, }, { isNonInteractive, @@ -3973,6 +4359,31 @@ async function runOnboard(opts: OnboardOptions = {}): Promise { initialHint: opts.baseImageResolutionHint, initialPreResolvedMetadata: opts.preResolvedBaseImageMetadata, }); + const autoComputePlan = computeRuntime.resolveCurrentOpenShellComputePlan(); + const durableSandboxName = + requestedSandboxName ?? (resume ? (persistedSessionAtEntry?.sandboxName ?? null) : null); + const durableSandboxEntry = durableSandboxName ? registry.getSandbox(durableSandboxName) : null; + const persistedComputeDriver = computeRuntime.resolvePersistedOpenShellComputeDriver([ + { + source: "onboarding session", + driverName: resume ? persistedSessionAtEntry?.metadata.openshellDriver : null, + }, + { + source: durableSandboxName ? `sandbox registry '${durableSandboxName}'` : "sandbox registry", + // Legacy rows predate explicit driver identity. Preserve their prior + // platform-derived behavior instead of allowing cross-driver adoption. + driverName: durableSandboxEntry + ? (durableSandboxEntry.openshellDriver ?? autoComputePlan.driverName) + : null, + }, + ]); + const requestedComputeDriver = + opts.computeDriver ?? process.env.NEMOCLAW_COMPUTE_DRIVER?.trim() ?? "auto"; + let onboardingComputePlan = computeRuntime.resolveOpenShellComputeSelection({ + requestedDriver: requestedComputeDriver, + persistedDriver: persistedComputeDriver, + autoPlan: autoComputePlan, + }); if (isNonInteractive()) policyTierEnv.validatePolicyTierEnvEarly(); const noticeAccepted = await ensureUsageNoticeConsent({ nonInteractive: isNonInteractive(), @@ -4003,6 +4414,58 @@ async function runOnboard(opts: OnboardOptions = {}): Promise { console.error(` rm -f "${lockResult.lockFile}"`); process.exit(1); } + let lockReleased = false; + const releaseOnboardLock = () => { + if (lockReleased || !ownsOnboardLock) return; + lockReleased = true; + onboardSession.releaseOnboardLock(); + }; + if (ownsOnboardLock) process.once("exit", releaseOnboardLock); + try { + const driverBindingGateway = gatewayBinding.resolveCoreOnboardGatewayBinding({ + authoritativeGateway, + currentGateway: { name: GATEWAY_NAME, port: GATEWAY_PORT }, + resume, + sandbox: durableSandboxEntry, + }); + const sharedGatewayDriver = gatewayBinding.resolveGatewayComputeDriverBinding({ + gatewayName: driverBindingGateway.name, + sandboxes: registry.listSandboxes().sandboxes, + legacyDriverName: autoComputePlan.driverName, + }); + const runtimeBinding = computeRuntime.readManagedGatewayRuntimeBinding( + gatewayBinding.resolveManagedGatewayStateDirectory(driverBindingGateway.name), + ); + const lockedPersistedComputeDriver = computeRuntime.resolvePersistedOpenShellComputeDriver([ + { + source: "requested sandbox or onboarding session", + driverName: persistedComputeDriver, + }, + { + source: `gateway '${driverBindingGateway.name}'`, + driverName: sharedGatewayDriver, + }, + { + source: `gateway runtime '${driverBindingGateway.name}'`, + driverName: runtimeBinding?.driverName, + }, + ]); + onboardingComputePlan = computeRuntime.resolveOpenShellComputeSelection({ + requestedDriver: requestedComputeDriver, + persistedDriver: lockedPersistedComputeDriver, + autoPlan: autoComputePlan, + }); + const sandboxGpuUnsupportedMessage = + fatalRuntimePreflight.resolveFatalRuntimePreflightDriverBehavior( + onboardingComputePlan, + ).sandboxGpuUnsupportedMessage; + if (sandboxGpuUnsupportedMessage && (opts.gpu === true || opts.sandboxGpu === "enable")) { + throw new computeRuntime.OpenShellComputeSelectionError(sandboxGpuUnsupportedMessage); + } + } catch (error) { + releaseOnboardLock(); + throw error; + } // Stage any pre-fix plaintext credentials.json into process.env so the // provider upserts later in this run can pick the values up. The file is // NOT removed here — the secure unlink runs only after onboarding @@ -4044,14 +4507,6 @@ async function runOnboard(opts: OnboardOptions = {}): Promise { ); } - let lockReleased = false; - const releaseOnboardLock = () => { - if (lockReleased || !ownsOnboardLock) return; - lockReleased = true; - onboardSession.releaseOnboardLock(); - }; - if (ownsOnboardLock) process.once("exit", releaseOnboardLock); - if (authoritativeGateway) { GATEWAY_NAME = authoritativeGateway.name; GATEWAY_PORT = authoritativeGateway.port; @@ -4063,6 +4518,8 @@ async function runOnboard(opts: OnboardOptions = {}): Promise { span: null, }; let traceCompleted = false; + let restoreHostLocalInferenceRuntime: () => void = () => undefined; + ACTIVE_OPEN_SHELL_COMPUTE_PLAN = onboardingComputePlan; try { onboardTrace = onboardTracing.startOnboardTrace(opts, process.env); let selectedMessagingChannels: string[] = []; @@ -4077,6 +4534,7 @@ async function runOnboard(opts: OnboardOptions = {}): Promise { authoritativeResumeConfig: opts.authoritativeResumeConfig === true, agentFlag: opts.agent || null, envAgent: process.env.NEMOCLAW_AGENT || null, + openshellDriver: onboardingComputePlan.driverName, ...stationSessionInput, }, { @@ -4095,6 +4553,20 @@ async function runOnboard(opts: OnboardOptions = {}): Promise { exitProcess: (code) => process.exit(code), }, ); + if (session) { + const recordedDriver = session.metadata.openshellDriver?.trim() || null; + if (recordedDriver && recordedDriver !== onboardingComputePlan.driverName) { + throw new computeRuntime.OpenShellComputeSelectionError( + `Onboarding session driver '${recordedDriver}' does not match selected driver '${onboardingComputePlan.driverName}'.`, + ); + } + if (!recordedDriver) { + session = onboardSession.updateSession((current) => { + current.metadata.openshellDriver = onboardingComputePlan.driverName; + return current; + }); + } + } await onboardRuntimeBoundary.recordOnboardStarted(resume); // Resume backstop: a session may exist without a sandboxName if sandbox // creation failed before that step. Non-interactive --from cannot infer a @@ -4197,6 +4669,8 @@ async function runOnboard(opts: OnboardOptions = {}): Promise { requestedGpuPassthrough: opts.gpu === true, }; + const runtimePreflightBehavior = + fatalRuntimePreflight.resolveFatalRuntimePreflightDriverBehavior(onboardingComputePlan); const [preflightPhase, gatewayPhase]: readonly [ import("./onboard/machine/sequence-runner").OnboardSequencePhase, import("./onboard/machine/sequence-runner").OnboardSequencePhase, @@ -4215,12 +4689,30 @@ async function runOnboard(opts: OnboardOptions = {}): Promise { getResumeSandboxGpuOverrides, detectGpu: nim.detectGpu, runPreflight: (preflightOptions) => preflight({ ...opts, ...preflightOptions }), - assessHost, + assessHost: () => assessHost({ skipDockerProbe: runtimePreflightBehavior.skipDockerProbe }), assertCdiNvidiaGpuSpecPresent: preflightUtils.assertCdiNvidiaGpuSpecPresent, - rejectUnsupportedContainerRuntime: fatalRuntimePreflight.rejectUnsupportedContainerRuntime, - assertDockerBridgeAndContainerDnsHealthy, - resolveSandboxGpuConfig, - validateSandboxGpuPreflight, + rejectUnsupportedContainerRuntime: (host) => { + fatalRuntimePreflight.assertSelectedContainerRuntimeReady(host, onboardingComputePlan, { + env: process.env, + }); + }, + assertDockerBridgeAndContainerDnsHealthy: (host) => { + if (runtimePreflightBehavior.checkDockerBridgeDns) { + assertDockerBridgeAndContainerDnsHealthy(host); + } + }, + resolveSandboxGpuConfig: (gpu, configOptions) => + resolveSandboxGpuConfig(gpu, { + ...configOptions, + flag: configOptions.flag ?? runtimePreflightBehavior.defaultSandboxGpuFlag, + }), + validateSandboxGpuPreflight: (config) => { + if (runtimePreflightBehavior.sandboxGpuUnsupportedMessage && config.sandboxGpuEnabled) { + console.error(` ${runtimePreflightBehavior.sandboxGpuUnsupportedMessage}`); + process.exit(1); + } + validateSandboxGpuPreflight(config); + }, skippedStepMessage, recordStateSkipped, startRecordedStep, @@ -4235,6 +4727,7 @@ async function runOnboard(opts: OnboardOptions = {}): Promise { ...machineGatewayOwnerDeps, refreshDockerDriverGatewayReuseState, gatewayCliSupportsLifecycleCommands: () => + !isManagedDriverGatewayEnabled() && gatewayCliSupportsLifecycleCommands(runCaptureOpenshell), verifyGatewayContainerRunning, waitForGatewayHttpReady, @@ -4246,7 +4739,7 @@ async function runOnboard(opts: OnboardOptions = {}): Promise { getGatewayClusterImageDrift, stopAllDashboardForwards, reconcileGatewayGpuReuseForGpuIntent, - isLinuxDockerDriverGatewayEnabled, + isLinuxDockerDriverGatewayEnabled: isActiveDockerManagedGateway, retireLegacyGatewayForDockerDriverUpgrade, destroyGatewayRuntimeForGpuReuse: () => destroyGateway( @@ -4276,6 +4769,10 @@ async function runOnboard(opts: OnboardOptions = {}): Promise { if (!initialContext.sandboxGpuConfig) { throw new Error("Preflight did not produce a sandbox GPU configuration."); } + restoreHostLocalInferenceRuntime = + computeRuntime.activateHostLocalInferenceRuntime(onboardingComputePlan, { + environment: process.env, + }); session = initialFlowResult.session; const sandboxGpuConfig = initialContext.sandboxGpuConfig; const { gpuPassthrough } = initialContext; @@ -4435,7 +4932,12 @@ async function runOnboard(opts: OnboardOptions = {}): Promise { planRegisteredExtraProviders(gatewayName, { runOpenshell }), resolveSandboxCreateIntent: sandboxCreateIntentResolver.resolve, createSandbox: preparedDcodeRuntime.bindCreateSandbox( - createSandboxWithBaseImageResolution.bind(null, baseImageResolutionContext), + createSandboxWithBaseImageResolution.bind( + null, + baseImageResolutionContext, + onboardingComputePlan, + opts.managedWorkloadRebuild ?? null, + ), ), updateSandboxRegistry: (name, updates) => registry.updateSandbox(name, updates), getSandboxAgentRegistryFields, @@ -4620,15 +5122,19 @@ async function runOnboard(opts: OnboardOptions = {}): Promise { completed = true; traceCompleted = finalFlowResult.session.machine.state === "complete"; } finally { + restoreHostLocalInferenceRuntime(); releaseOnboardLock(); onboardRuntimeBoundary.clear(); onboardTracing.finishOnboardTrace(onboardTrace, traceCompleted); GATEWAY_NAME = previousGatewayBinding.name; GATEWAY_PORT = previousGatewayBinding.port; + ACTIVE_OPEN_SHELL_COMPUTE_PLAN = previousComputePlan; if (previousOpenshellGateway === undefined) delete process.env.OPENSHELL_GATEWAY; else process.env.OPENSHELL_GATEWAY = previousOpenshellGateway; if (previousOpenshellLocalTlsDir === undefined) delete process.env.OPENSHELL_LOCAL_TLS_DIR; else process.env.OPENSHELL_LOCAL_TLS_DIR = previousOpenshellLocalTlsDir; + if (previousOpenshellPodmanSocket === undefined) delete process.env.OPENSHELL_PODMAN_SOCKET; + else process.env.OPENSHELL_PODMAN_SOCKET = previousOpenshellPodmanSocket; resetGatewayOwnerBinding(); } } @@ -4719,7 +5225,9 @@ module.exports = { findAvailableDashboardPort, startGatewayForRecovery, openshellArgv, + runOpenshell, runCaptureOpenshell, + sleepSeconds, agentSupportsWebSearch, agentSupportsWebSearchProvider, createSetupInference, diff --git a/src/lib/onboard/__test-helpers__/sandbox-gpu-create-flow.ts b/src/lib/onboard/__test-helpers__/sandbox-gpu-create-flow.ts index f0224759b0..afa57c1dd1 100644 --- a/src/lib/onboard/__test-helpers__/sandbox-gpu-create-flow.ts +++ b/src/lib/onboard/__test-helpers__/sandbox-gpu-create-flow.ts @@ -20,6 +20,7 @@ export const GPU_IMAGE_ID = `sha256:${"a".repeat(64)}`; export function createGpuFlowInput(): SandboxGpuCreateFlowInput { return { + computeDriver: "docker", sandboxName: "alpha", provider: "nim", sandboxGpuConfig: { @@ -64,8 +65,10 @@ export function createGpuPatchFixture() { maybeApplyDuringCreate: vi.fn(), createFailureMessage: vi.fn(() => null), exitOnPatchError: vi.fn(), + rollbackManagedStartupAfterCreateFailure: vi.fn(), ensureApplied: vi.fn(), waitForSupervisorReconnectIfNeeded: vi.fn(), + commitAfterReady: vi.fn(), selectedMode: vi.fn(() => null), printReadinessFailureIfEnabled: vi.fn(), verifyGpuOrExit: vi.fn(() => VERIFIED_GPU_PROOF), diff --git a/src/lib/onboard/authoritative-rebuild-target.ts b/src/lib/onboard/authoritative-rebuild-target.ts index 558b4c40f1..c749b270bd 100644 --- a/src/lib/onboard/authoritative-rebuild-target.ts +++ b/src/lib/onboard/authoritative-rebuild-target.ts @@ -21,7 +21,7 @@ export type AuthoritativeGatewayOptions = Pick< export type AuthoritativeRebuildPreflightOptions = Pick< OnboardOptions, - "sandboxGpu" | "sandboxGpuDevice" | "noGpu" | "controlUiPort" + "sandboxGpu" | "sandboxGpuDevice" | "noGpu" | "controlUiPort" | "computeDriver" > & { authoritativeResumeConfig: true; /** Internal prepared-backup recovery defers route repair to authoritative onboard. */ diff --git a/src/lib/onboard/command-support.test.ts b/src/lib/onboard/command-support.test.ts index 0c46a7a2cb..4e5ede8e88 100644 --- a/src/lib/onboard/command-support.test.ts +++ b/src/lib/onboard/command-support.test.ts @@ -42,6 +42,17 @@ describe("buildOnboardFlags --observability help", () => { }); }); +describe("buildOnboardFlags --compute-driver", () => { + it("exposes only qualified public selections", () => { + const flags = buildOnboardFlags(); + + expect(flags["compute-driver"].options).toEqual(["auto", "docker", "podman"]); + expect(flags["compute-driver"].description).toContain( + "auto preserves a registered sandbox driver", + ); + }); +}); + describe("buildOnboardFlags --events help", () => { it("exposes the JSONL observer only on the canonical onboard command", () => { const onboardFlags = buildOnboardFlags({ includeEvents: true }); diff --git a/src/lib/onboard/command-support.ts b/src/lib/onboard/command-support.ts index 1f0ee3d418..50d0e89667 100644 --- a/src/lib/onboard/command-support.ts +++ b/src/lib/onboard/command-support.ts @@ -4,6 +4,10 @@ import { Flags } from "@oclif/core"; import { TOOL_DISCLOSURE_VALUES, type ToolDisclosure } from "../tool-disclosure"; import { describeAgentFlag } from "./agent-flag-help"; +import { + OPEN_SHELL_COMPUTE_DRIVER_REQUESTS, + type OpenShellComputeDriverRequest, +} from "./compute/plan"; import { NOTICE_ACCEPT_FLAG, NOTICE_ACCEPT_FLAG_NAME } from "./usage-notice"; type AgentRegistryReader = () => readonly string[]; @@ -46,7 +50,7 @@ function agentFlagDescription(): string { } export const onboardUsage = [ - `onboard [--non-interactive] [--resume | --fresh] [--recreate-sandbox] [--gpu | --no-gpu] [--from ] [--name ] [--sandbox-gpu | --no-sandbox-gpu] [--sandbox-gpu-device ] [--agent ] [--agents ] [--tool-disclosure ] [--observability | --no-observability] [--control-ui-port ] [--events=jsonl] [--yes | -y] [--no-ollama-autostart] [${NOTICE_ACCEPT_FLAG}]`, + `onboard [--non-interactive] [--resume | --fresh] [--recreate-sandbox] [--compute-driver ] [--gpu | --no-gpu] [--from ] [--name ] [--sandbox-gpu | --no-sandbox-gpu] [--sandbox-gpu-device ] [--agent ] [--agents ] [--tool-disclosure ] [--observability | --no-observability] [--control-ui-port ] [--events=jsonl] [--yes | -y] [--no-ollama-autostart] [${NOTICE_ACCEPT_FLAG}]`, ]; export const onboardExamples = [ @@ -54,6 +58,7 @@ export const onboardExamples = [ "<%= config.bin %> onboard --name alpha", "<%= config.bin %> onboard --resume", "<%= config.bin %> onboard --fresh", + "<%= config.bin %> onboard --compute-driver podman --name alpha", "<%= config.bin %> onboard --from ./Dockerfile --name alpha", "<%= config.bin %> onboard --agents ./agents.yaml", "<%= config.bin %> onboard --sandbox-gpu --sandbox-gpu-device nvidia.com/gpu=0", @@ -65,6 +70,7 @@ export type OnboardFlags = { resume?: boolean; fresh?: boolean; "recreate-sandbox"?: boolean; + "compute-driver"?: OpenShellComputeDriverRequest; gpu?: boolean; "no-gpu"?: boolean; from?: string; @@ -95,6 +101,11 @@ export function buildOnboardFlags(options: { includeEvents?: boolean } = {}): Re exclusive: ["resume"], }), "recreate-sandbox": Flags.boolean({ description: "Delete and recreate an existing sandbox" }), + "compute-driver": Flags.string({ + description: + "OpenShell compute driver; auto preserves a registered sandbox driver and otherwise uses the platform default", + options: [...OPEN_SHELL_COMPUTE_DRIVER_REQUESTS], + }), gpu: Flags.boolean({ description: "Require OpenShell GPU passthrough for the gateway and sandbox", exclusive: ["no-gpu", "no-sandbox-gpu"], diff --git a/src/lib/onboard/command.test.ts b/src/lib/onboard/command.test.ts index 82257f5520..7edf5735fb 100644 --- a/src/lib/onboard/command.test.ts +++ b/src/lib/onboard/command.test.ts @@ -39,6 +39,7 @@ describe("onboard command options", () => { "non-interactive": true, resume: true, "recreate-sandbox": true, + "compute-driver": "podman", from: dockerfilePath, name: "second-assistant", "sandbox-gpu": true, @@ -59,6 +60,7 @@ describe("onboard command options", () => { resume: true, fresh: false, recreateSandbox: true, + computeDriver: "podman", fromDockerfile: dockerfilePath, sandboxName: "second-assistant", sandboxGpu: "enable", @@ -82,6 +84,7 @@ describe("onboard command options", () => { resume: false, fresh: false, recreateSandbox: false, + computeDriver: "auto", fromDockerfile: null, sandboxName: null, sandboxGpu: null, @@ -131,6 +134,28 @@ describe("onboard command options", () => { expect(errors.join("\n")).toContain("must be one of: progressive, direct"); }); + it("uses the explicit compute driver before the environment and rejects unknown values", () => { + expect( + resolve({ "compute-driver": "podman" }, { env: { NEMOCLAW_COMPUTE_DRIVER: "docker" } }) + .computeDriver, + ).toBe("podman"); + expect(resolve({}, { env: { NEMOCLAW_COMPUTE_DRIVER: " PODMAN " } }).computeDriver).toBe( + "podman", + ); + + const errors: string[] = []; + expect(() => + resolve( + {}, + { + env: { NEMOCLAW_COMPUTE_DRIVER: "container" }, + error: (message = "") => errors.push(message), + }, + ), + ).toThrow("exit:1"); + expect(errors.join("\n")).toContain("must be one of: auto, docker, podman"); + }); + it("preserves the requested Dockerfile path after validating the resolved file", () => { const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-from-")); const dockerfilePath = path.join(tmpDir, "Custom.Dockerfile"); diff --git a/src/lib/onboard/command.ts b/src/lib/onboard/command.ts index cba531b557..d809bc11a4 100644 --- a/src/lib/onboard/command.ts +++ b/src/lib/onboard/command.ts @@ -12,6 +12,10 @@ import { } from "../tool-disclosure"; import { applyAgentsManifestEnv } from "./agents-manifest"; import type { OnboardFlags } from "./command-support"; +import { + type OpenShellComputeDriverRequest, + resolveOpenShellComputeDriverRequest, +} from "./compute/plan"; import { GatewayManagementDeclarationError } from "./gateway-management"; import { managedSandboxFeatureIssue } from "./managed-sandbox-feature"; import { DCODE_OBSERVABILITY_FEATURE } from "./observability-policy-presets"; @@ -23,6 +27,7 @@ export interface OnboardCommandOptions { resume: boolean; fresh: boolean; recreateSandbox: boolean; + computeDriver: OpenShellComputeDriverRequest; fromDockerfile: string | null; sandboxName: string | null; sandboxGpu: "enable" | "disable" | null; @@ -153,8 +158,10 @@ export function resolveOnboardOptions( const agent = resolveAgent(flags.agent, deps); validateObservabilityAgent(flags.observability, agent, deps); let toolDisclosure: ToolDisclosure | null; + let computeDriver: OpenShellComputeDriverRequest; try { toolDisclosure = resolveToolDisclosureRequest(flags["tool-disclosure"], deps.env); + computeDriver = resolveOpenShellComputeDriverRequest(flags["compute-driver"], deps.env); } catch (error) { fail(deps, ` ${error instanceof Error ? error.message : String(error)}`); } @@ -163,6 +170,7 @@ export function resolveOnboardOptions( resume: flags.resume === true, fresh: flags.fresh === true, recreateSandbox: flags["recreate-sandbox"] === true, + computeDriver, fromDockerfile: resolveFileOption("--from", flags.from, deps, true), sandboxName: flags.name ?? null, sandboxGpu: resolveSandboxGpu(flags), diff --git a/src/lib/onboard/compute/driver-pluggability-contract.test.ts b/src/lib/onboard/compute/driver-pluggability-contract.test.ts new file mode 100644 index 0000000000..d7d1bd9910 --- /dev/null +++ b/src/lib/onboard/compute/driver-pluggability-contract.test.ts @@ -0,0 +1,510 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import { + collectDoctorHostRuntimeCheck, + type DoctorHostRuntimeAdapterRegistry, +} from "../../actions/sandbox/doctor-host-command"; +import { + resolveSandboxLifecycleRuntimeAdapter, + type SandboxLifecycleRuntimeAdapter, + type SandboxLifecycleRuntimeAdapterRegistry, +} from "../../actions/sandbox/runtime/lifecycle-runtime"; +import { resolveManagedSnapshotRuntimeAuthority } from "../../actions/sandbox/snapshot/runtime-authority"; +import { + createManagedSnapshotRuntimePatch, + type ManagedSnapshotRuntimePatchContext, +} from "../../actions/sandbox/snapshot/runtime-patch"; +import type { SandboxEntry } from "../../state/registry"; +import type { ManagedGatewayRuntimeBinding } from "../docker-driver-gateway-config"; +import { + MANAGED_IMAGE_CAPABILITY_CONTRACT_VERSION, + MANAGED_IMAGE_CONTRACT_VERSION, + MANAGED_IMAGE_PLATFORM, + MANAGED_IMAGE_REPOSITORIES, + MANAGED_IMAGE_SOURCE_REPOSITORY, + MANAGED_IMAGE_STARTUP_PROFILE_CONTRACT_VERSION, + type ManagedImageContractV1, +} from "../managed-image/contract"; +import { + createSandboxCreateRuntimePatch, + type SandboxCreateRuntimePatchAdapterRegistry, + type SandboxCreateRuntimePatchRequest, +} from "../sandbox-create-runtime/registry"; +import type { SandboxCreateRuntimePatch } from "../sandbox-create-runtime/types"; +import { resolveSandboxWorkloadRuntimeCapabilities } from "../workload/runtime"; +import { resolveSandboxWorkloadSource, SandboxWorkloadSourceError } from "../workload/source"; +import { + type ManagedGatewayDriverProfile, + type ManagedGatewayDriverProfileRegistry, + type ManagedGatewayRuntimeAdapterRegistry, + resolveManagedGatewayDriverProfile, + resolveManagedGatewayRuntimeAdapter, +} from "./managed-gateway-profile"; +import { + type ManagedStartupRuntimeRequirementsAdapterRegistry, + resolveManagedStartupRuntimeRequirements, +} from "./managed-startup-runtime-requirements"; +import { + type OpenShellComputeCapabilitiesRegistry, + type OpenShellComputePlanRegistry, + resolveOpenShellComputeCapabilities, + resolveOpenShellComputeSelection, +} from "./plan"; +import { + type ManagedGatewayRecoveryAdapterRegistry, + qualifyManagedGatewayRecoveryRuntime, + resolveManagedGatewayRecoveryRuntime, + supportsManagedGatewayRecoveryRuntime, +} from "./recovery-runtime"; +import { + resolveSandboxRuntimeAuthority, + runAuthorizedSandboxRecreateDeletion, + type SandboxRuntimeAuthorityAdapterRegistry, +} from "./runtime-authority"; + +const MXC_DRIVER = "mxc"; +const MXC_ENDPOINT = "unix:///run/user/1000/mxc/control.sock"; + +const MANAGED_IMAGE_SUPPORT = { + exactDigestReferences: true, + platforms: [MANAGED_IMAGE_PLATFORM], + startupProfileContractVersions: [MANAGED_IMAGE_STARTUP_PROFILE_CONTRACT_VERSION], + capabilityContractVersions: [MANAGED_IMAGE_CAPABILITY_CONTRACT_VERSION], +} as const; + +function hermesContract(): ManagedImageContractV1 { + const image = MANAGED_IMAGE_REPOSITORIES.hermes; + const digest = `sha256:${"a".repeat(64)}` as const; + return { + contractVersion: MANAGED_IMAGE_CONTRACT_VERSION, + agent: "hermes", + platform: MANAGED_IMAGE_PLATFORM, + image, + digest, + reference: `${image}@${digest}`, + source: { + repository: MANAGED_IMAGE_SOURCE_REPOSITORY, + revision: "b".repeat(40), + release: "v0.0.98", + cohort: "ghrun-7744-3", + }, + startupProfileContractVersion: MANAGED_IMAGE_STARTUP_PROFILE_CONTRACT_VERSION, + capabilityContractVersion: MANAGED_IMAGE_CAPABILITY_CONTRACT_VERSION, + }; +} + +describe("OpenShell compute-driver pluggability contract", () => { + it("routes an injected MXC driver through every runtime seam without inherited Docker or Podman behavior (#7744)", () => { + const plans: OpenShellComputePlanRegistry = { + mxc: { driverName: MXC_DRIVER, gatewayLauncher: "nemoclaw" }, + }; + const capabilities: OpenShellComputeCapabilitiesRegistry = { + mxc: { hostLocalInference: false }, + }; + const plan = resolveOpenShellComputeSelection( + { + requestedDriver: MXC_DRIVER, + autoPlan: { driverName: "unused-auto", gatewayLauncher: "openshell" }, + }, + plans, + ); + expect(plan).toEqual({ driverName: MXC_DRIVER, gatewayLauncher: "nemoclaw" }); + expect(resolveOpenShellComputeCapabilities(plan, capabilities)).toEqual({ + hostLocalInference: false, + }); + + const mxcProfile = { + allowWildcardBind: false, + driverName: MXC_DRIVER, + displayName: "MXC", + incompatibleRuntimeEnvironmentKeys: [], + launchPolicy: "host-only", + runtimeMarkerPolicy: "process-env", + runtimeEnvironmentKeys: ["OPENSHELL_MXC_ENDPOINT"], + sandboxReachability: "driver-native", + capabilities: { + containerizedGatewayCompat: false, + legacyDockerGatewayCleanup: false, + legacyDockerVolumeCleanup: false, + localSupervisorBinary: false, + packageManagedService: false, + }, + } as const satisfies ManagedGatewayDriverProfile; + const profiles: ManagedGatewayDriverProfileRegistry = { mxc: mxcProfile }; + const resolvedProfile = resolveManagedGatewayDriverProfile(plan, profiles); + expect(resolvedProfile).toBe(mxcProfile); + + const lifecycleAdapter = { + driverName: MXC_DRIVER, + launchPolicy: "host-only", + runtimeMarkerPolicy: "process-env", + sandboxReachability: "driver-native", + marker: "mxc-only", + } as const; + const lifecycleAdapters: ManagedGatewayRuntimeAdapterRegistry = { + mxc: lifecycleAdapter, + }; + expect( + resolveManagedGatewayRuntimeAdapter( + resolvedProfile as ManagedGatewayDriverProfile, + lifecycleAdapters, + ), + ).toBe(lifecycleAdapter); + + const sandboxLifecycleAdapter = { + channelStopTransport: "openshell", + displayName: "MXC", + driverName: MXC_DRIVER, + preflight: vi.fn(() => null), + start: vi.fn(() => ({ exitCode: 0 })), + stop: vi.fn((_input, _deps, hooks) => { + hooks.beforeStop(); + return { exitCode: 0, state: "stopped" as const }; + }), + } satisfies SandboxLifecycleRuntimeAdapter; + const sandboxLifecycleAdapters: SandboxLifecycleRuntimeAdapterRegistry = { + mxc: sandboxLifecycleAdapter, + }; + expect(resolveSandboxLifecycleRuntimeAdapter(MXC_DRIVER, sandboxLifecycleAdapters)).toBe( + sandboxLifecycleAdapter, + ); + + const resolveRecoveryEnvironment = vi.fn(() => ({ + OPENSHELL_MXC_ENDPOINT: MXC_ENDPOINT, + })); + const qualifyRecoveryEnvironment = vi.fn(() => ({ endpoint: MXC_ENDPOINT })); + const recoveryAdapters: ManagedGatewayRecoveryAdapterRegistry = { + mxc: { + driverName: MXC_DRIVER, + qualifyEnvironment: qualifyRecoveryEnvironment, + resolveEnvironment: resolveRecoveryEnvironment, + }, + }; + const runtimeBinding: ManagedGatewayRuntimeBinding = { + version: 1, + driverName: MXC_DRIVER, + configSha256: "c".repeat(64), + values: { endpoint: MXC_ENDPOINT }, + }; + const recoveryRuntime = resolveManagedGatewayRecoveryRuntime( + { + driverName: MXC_DRIVER, + environment: {}, + stateDir: "/state/mxc", + }, + recoveryAdapters, + () => runtimeBinding, + ); + expect(recoveryRuntime).toEqual({ + driverName: MXC_DRIVER, + environment: { OPENSHELL_MXC_ENDPOINT: MXC_ENDPOINT }, + }); + expect(supportsManagedGatewayRecoveryRuntime(MXC_DRIVER, recoveryAdapters)).toBe(true); + expect(qualifyManagedGatewayRecoveryRuntime(recoveryRuntime, recoveryAdapters)).toEqual({ + endpoint: MXC_ENDPOINT, + }); + expect(resolveRecoveryEnvironment).toHaveBeenCalledExactlyOnceWith(runtimeBinding); + expect(qualifyRecoveryEnvironment).toHaveBeenCalledExactlyOnceWith(recoveryRuntime.environment); + + const inspectDoctorRuntime = vi.fn(() => ({ + group: "Host" as const, + label: "MXC runtime", + status: "ok" as const, + detail: `connected via ${MXC_ENDPOINT}`, + })); + const doctorAdapters: DoctorHostRuntimeAdapterRegistry = { + mxc: { + driverName: MXC_DRIVER, + inspect: inspectDoctorRuntime, + }, + }; + expect( + collectDoctorHostRuntimeCheck( + { + driverName: MXC_DRIVER, + managedGatewayStateDirectory: "/state/mxc", + }, + doctorAdapters, + ), + ).toEqual({ + group: "Host", + label: "MXC runtime", + status: "ok", + detail: `connected via ${MXC_ENDPOINT}`, + }); + expect(inspectDoctorRuntime).toHaveBeenCalledExactlyOnceWith( + { + driverName: MXC_DRIVER, + managedGatewayStateDirectory: "/state/mxc", + }, + { + captureHostCommand: expect.any(Function), + }, + ); + + const workloadRuntime = resolveSandboxWorkloadRuntimeCapabilities( + plan, + { + mxc: { + support: MANAGED_IMAGE_SUPPORT, + hostArchitectures: ["amd64"], + managedImageSelectionPolicy: "require-managed", + legacyDockerfileBuilds: false, + }, + }, + "x64", + ); + expect(workloadRuntime).toEqual({ + driverName: MXC_DRIVER, + managedImageSelectionPolicy: "require-managed", + legacyDockerfileBuilds: false, + managedImages: MANAGED_IMAGE_SUPPORT, + }); + + const contract = hermesContract(); + expect( + resolveSandboxWorkloadSource({ + agentName: "hermes", + legacyDockerfilePath: "agents/hermes/Dockerfile", + runtime: workloadRuntime, + catalog: { hermes: contract }, + }), + ).toEqual({ + kind: "managed-image", + reference: contract.reference, + contract, + }); + expect(() => + resolveSandboxWorkloadSource({ + agentName: "hermes", + legacyDockerfilePath: "agents/hermes/Dockerfile", + customDockerfilePath: "agents/hermes/CustomDockerfile", + runtime: workloadRuntime, + catalog: { hermes: contract }, + }), + ).toThrow(SandboxWorkloadSourceError); + }); + + it("injects MXC authority and mutation adapters without coordinator-owned runtime branches (#7744)", () => { + const runtimeAuthority = { + driverName: MXC_DRIVER, + endpoint: MXC_ENDPOINT, + owner: "mxc-user-service", + } as const; + const authorityContext = { + destinationSandboxName: "beta", + sourceSandboxName: "alpha", + }; + const resolveAuthority = vi.fn(() => runtimeAuthority); + const authorityAdapters: SandboxRuntimeAuthorityAdapterRegistry = { + mxc: { + driverName: MXC_DRIVER, + resolve: resolveAuthority, + }, + }; + + expect(resolveSandboxRuntimeAuthority(MXC_DRIVER, authorityContext, authorityAdapters)).toBe( + runtimeAuthority, + ); + expect(resolveAuthority).toHaveBeenCalledExactlyOnceWith(authorityContext); + + const selectedPatch = { + commitAfterReady: vi.fn(), + createFailureMessage: vi.fn(() => null), + ensureApplied: vi.fn(), + exitOnPatchError: vi.fn(), + maybeApplyDuringCreate: vi.fn(), + revalidateBeforeMutation: vi.fn(), + rollbackManagedStartupAfterCreateFailure: vi.fn(), + waitForSupervisorReconnectIfNeeded: vi.fn(), + } satisfies SandboxCreateRuntimePatch; + const createPatch = vi.fn(() => selectedPatch); + const patchAdapters: SandboxCreateRuntimePatchAdapterRegistry = { + mxc: { + driverName: MXC_DRIVER, + create: createPatch, + }, + }; + const patchInput: SandboxCreateRuntimePatchRequest = { + driverName: MXC_DRIVER, + lifecycle: { + deps: { + runCaptureOpenshell: vi.fn(() => ""), + runOpenshell: vi.fn(() => ({ status: 0 })), + sleep: vi.fn(), + }, + openshellSandboxCommand: ["nemoclaw-start"], + persistStartupCommand: false, + requiredUlimits: null, + sandboxGpuEnabled: false, + sandboxName: "beta", + timeoutSecs: 30, + }, + runtimeAuthority, + }; + + expect(createSandboxCreateRuntimePatch(patchInput, patchAdapters)).toBe(selectedPatch); + expect(createPatch).toHaveBeenCalledExactlyOnceWith(patchInput); + expect(patchInput).not.toHaveProperty("docker"); + + const resolveRequirements = vi.fn(() => ({ + persistStartupCommand: true, + requiredUlimits: [{ name: "nofile", soft: 8192, hard: 8192 }], + })); + const requirementAdapters: ManagedStartupRuntimeRequirementsAdapterRegistry = { + mxc: { + driverName: MXC_DRIVER, + resolve: resolveRequirements, + }, + }; + const managedAgent = { + name: "hermes", + } as Parameters[0]; + const managedGatewayContext = { managedGatewayOwned: true } as const; + + expect( + resolveManagedStartupRuntimeRequirements( + managedAgent, + MXC_DRIVER, + managedGatewayContext, + requirementAdapters, + ), + ).toEqual({ + persistStartupCommand: true, + requiredUlimits: [{ name: "nofile", soft: 8192, hard: 8192 }], + }); + expect(resolveRequirements).toHaveBeenCalledExactlyOnceWith("hermes", managedGatewayContext); + + const sourceEntry = { + agent: "hermes", + name: "alpha", + openshellDriver: MXC_DRIVER, + } as SandboxEntry; + const snapshotContext: ManagedSnapshotRuntimePatchContext = { + destinationSandboxName: "beta", + sourceEntry, + }; + const snapshotAuthorityAdapters: SandboxRuntimeAuthorityAdapterRegistry = + { + mxc: { + driverName: MXC_DRIVER, + resolve: vi.fn(() => runtimeAuthority), + }, + }; + const resolveSnapshotAuthority = vi.fn( + (driverName: string, context: ManagedSnapshotRuntimePatchContext) => + resolveManagedSnapshotRuntimeAuthority(driverName, context, snapshotAuthorityAdapters), + ); + const resolveSnapshotRequirements = vi.fn( + ( + driverName: string, + context: ManagedSnapshotRuntimePatchContext, + requirementsContext: { readonly managedGatewayOwned: boolean }, + ) => + resolveManagedStartupRuntimeRequirements( + context.sourceEntry.agent ? { name: context.sourceEntry.agent } : null, + driverName, + requirementsContext, + requirementAdapters, + ), + ); + const createSnapshotPatch = vi.fn((input: SandboxCreateRuntimePatchRequest) => + createSandboxCreateRuntimePatch(input, patchAdapters), + ); + + expect( + createManagedSnapshotRuntimePatch( + { + ...snapshotContext, + lifecycle: { + deps: patchInput.lifecycle.deps, + openshellSandboxCommand: patchInput.lifecycle.openshellSandboxCommand, + sandboxName: patchInput.lifecycle.sandboxName, + timeoutSecs: patchInput.lifecycle.timeoutSecs, + }, + }, + { + createRuntimePatch: createSnapshotPatch, + resolveRuntimeAuthority: resolveSnapshotAuthority, + resolveRuntimeRequirements: resolveSnapshotRequirements, + }, + ), + ).toBe(selectedPatch); + expect(resolveSnapshotAuthority).toHaveBeenCalledExactlyOnceWith(MXC_DRIVER, snapshotContext); + expect(resolveSnapshotRequirements).toHaveBeenCalledExactlyOnceWith( + MXC_DRIVER, + snapshotContext, + { managedGatewayOwned: true }, + ); + expect(createSnapshotPatch).toHaveBeenCalledExactlyOnceWith({ + driverName: MXC_DRIVER, + lifecycle: { + deps: patchInput.lifecycle.deps, + openshellSandboxCommand: patchInput.lifecycle.openshellSandboxCommand, + persistStartupCommand: true, + requiredUlimits: [{ name: "nofile", soft: 8192, hard: 8192 }], + sandboxGpuEnabled: false, + sandboxName: patchInput.lifecycle.sandboxName, + timeoutSecs: patchInput.lifecycle.timeoutSecs, + }, + runtimeAuthority, + }); + }); + + it("fails runtime qualification before delete or driver-native mutation can run", () => { + const beforeDelete = vi.fn(); + const deleteSandbox = vi.fn(); + const mutatePodman = vi.fn(); + const authorityAdapters: SandboxRuntimeAuthorityAdapterRegistry = { + podman: { + driverName: "podman", + resolve: vi.fn(() => { + throw new Error("rootless Podman API is unavailable"); + }), + }, + }; + + expect(() => + runAuthorizedSandboxRecreateDeletion("podman", undefined, authorityAdapters, { + beforeDelete, + deleteSandbox, + afterDelete: mutatePodman, + }), + ).toThrow("rootless Podman API is unavailable"); + expect(beforeDelete).not.toHaveBeenCalled(); + expect(deleteSandbox).not.toHaveBeenCalled(); + expect(mutatePodman).not.toHaveBeenCalled(); + }); + + it("revalidates runtime authority after preparation and immediately before delete", () => { + const authority = { socketPath: "/run/user/1000/podman/podman.sock" }; + const beforeDelete = vi.fn(); + const deleteSandbox = vi.fn(); + const afterDelete = vi.fn(); + const revalidate = vi.fn(() => { + throw new Error("Podman socket authority changed during recreate preparation"); + }); + const authorityAdapters: SandboxRuntimeAuthorityAdapterRegistry = { + podman: { + driverName: "podman", + resolve: vi.fn(() => authority), + revalidate, + }, + }; + + expect(() => + runAuthorizedSandboxRecreateDeletion("podman", undefined, authorityAdapters, { + beforeDelete, + deleteSandbox, + afterDelete, + }), + ).toThrow("socket authority changed"); + expect(beforeDelete).toHaveBeenCalledOnce(); + expect(revalidate).toHaveBeenCalledExactlyOnceWith(authority, undefined); + expect(deleteSandbox).not.toHaveBeenCalled(); + expect(afterDelete).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/onboard/compute/host-local-inference-runtime.test.ts b/src/lib/onboard/compute/host-local-inference-runtime.test.ts new file mode 100644 index 0000000000..3c9a274a25 --- /dev/null +++ b/src/lib/onboard/compute/host-local-inference-runtime.test.ts @@ -0,0 +1,348 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { + type HostContainerEngineCommand, + hostContainerEngineArgv, + resetHostContainerEngineForTests, +} from "../../adapters/container-engine"; +import { + activateHostLocalInferenceRuntime, + type HostLocalInferenceRuntimeAdapterRegistry, + translateLocalInferenceArgsForPodman, +} from "./host-local-inference-runtime"; +import { + capturePodmanSocketAuthority, + type PodmanSocketAuthorityDeps, +} from "./podman/socket-authority"; +import type { NativePodmanPreflightReceipt } from "./podman-preflight"; + +const SOCKET_PATH = "/run/user/1000/podman/podman.sock"; + +function socketAuthorityDeps(): PodmanSocketAuthorityDeps { + const paths = [SOCKET_PATH, "/run/user/1000/podman", "/run/user/1000", "/run/user", "/run", "/"]; + const stats = new Map( + paths.map((filePath, index) => [ + filePath, + { + dev: 8n, + ino: BigInt(100 + index), + mode: filePath === SOCKET_PATH ? 0o140600n : 0o40700n, + uid: filePath === "/run/user" || filePath === "/run" || filePath === "/" ? 0n : 1000n, + isDirectory: () => filePath !== SOCKET_PATH, + isSocket: () => filePath === SOCKET_PATH, + }, + ]), + ); + return { + uid: 1000, + lstat: (filePath: string) => { + const stat = stats.get(filePath); + if (!stat) throw new Error(`unexpected lstat ${filePath}`); + return stat; + }, + }; +} + +function qualifiedPodmanRuntime( + deps: PodmanSocketAuthorityDeps, + architecture: "amd64" | "arm64" = "amd64", +): NativePodmanPreflightReceipt { + return { + driverName: "podman", + version: "5.6.2", + socketPath: SOCKET_PATH, + socketAuthority: capturePodmanSocketAuthority(SOCKET_PATH, deps), + rootless: true, + cgroupVersion: "v2", + os: "linux", + architecture, + networkBackend: "netavark", + cdiDevices: [ + "nvidia.com/gpu=all", + "nvidia.com/gpu=0", + "nvidia.com/gpu=1:0", + "nvidia.com/gpu=2", + "nvidia.com/gpu=GPU-aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", + "nvidia.com/gpu=MIG-aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", + "nvidia.com/gpu=MIG-GPU-aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/1/0", + ], + }; +} + +describe("host-local inference compute runtime", () => { + afterEach(() => { + resetHostContainerEngineForTests(); + }); + + it("activates Podman through the exact qualified rootless socket", () => { + const authorityDeps = socketAuthorityDeps(); + let command: HostContainerEngineCommand | null = null; + const restore = vi.fn(); + const configure = vi.fn((next: HostContainerEngineCommand) => { + command = next; + return restore; + }); + + const result = activateHostLocalInferenceRuntime( + { driverName: "podman" }, + { + environment: { + OPENSHELL_PODMAN_SOCKET: SOCKET_PATH, + NEMOCLAW_PODMAN_BIN: "/usr/bin/podman", + }, + qualifiedPodmanRuntime: qualifiedPodmanRuntime(authorityDeps), + socketAuthorityDeps: authorityDeps, + configureContainerEngine: configure, + }, + ); + + expect(result).toBe(restore); + expect(configure).toHaveBeenCalledOnce(); + expect(command).toMatchObject({ + driverName: "podman", + executable: "/usr/bin/podman", + prefixArgs: ["--url", `unix://${SOCKET_PATH}`], + runtimeArchitecture: "amd64", + sandboxNetworkName: "openshell", + hostGatewayTarget: "host-gateway", + }); + expect(() => command?.assertAuthority?.()).not.toThrow(); + }); + + it.each([ + ["all", ["nvidia.com/gpu=all"]], + ["device=0", ["nvidia.com/gpu=0"]], + ["device=1:0", ["nvidia.com/gpu=1:0"]], + [ + "device=GPU-aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", + ["nvidia.com/gpu=GPU-aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee"], + ], + [ + "device=MIG-aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", + ["nvidia.com/gpu=MIG-aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee"], + ], + [ + "device=MIG-GPU-aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/1/0", + ["nvidia.com/gpu=MIG-GPU-aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/1/0"], + ], + ['"device=0,2"', ["nvidia.com/gpu=0", "nvidia.com/gpu=2"]], + ])("preserves Docker GPU selector %s as exact Podman CDI devices", (selector, devices) => { + const availableCdiDevices = qualifiedPodmanRuntime(socketAuthorityDeps()).cdiDevices; + const translated = translateLocalInferenceArgsForPodman(["run", "--gpus", selector, "image"], { + availableCdiDevices, + }); + expect(translated.filter((value) => value.startsWith("nvidia.com/gpu="))).toEqual(devices); + expect(translated.filter((value) => value === "--device")).toHaveLength(devices.length); + expect(translated).not.toContain("--gpus"); + }); + + it("translates managed NIM and vLLM Docker-compatible arguments without name leakage", () => { + expect( + translateLocalInferenceArgsForPodman([ + "run", + "--gpus", + "all", + "--filter", + "name=^/nemoclaw-vllm$", + ]), + ).toEqual(["run", "--device", "nvidia.com/gpu=all", "--filter", "name=^nemoclaw-vllm$"]); + expect(translateLocalInferenceArgsForPodman(["run", "--gpus=device=0", "image"])).toEqual([ + "run", + "--device", + "nvidia.com/gpu=0", + "image", + ]); + }); + + it("routes full managed NIM and vLLM argv through Podman with GPU attachment intact", () => { + const authorityDeps = socketAuthorityDeps(); + const restore = activateHostLocalInferenceRuntime( + { driverName: "podman" }, + { + environment: { OPENSHELL_PODMAN_SOCKET: SOCKET_PATH }, + qualifiedPodmanRuntime: qualifiedPodmanRuntime(authorityDeps), + socketAuthorityDeps: authorityDeps, + }, + ); + try { + const prefix = ["podman", "--url", `unix://${SOCKET_PATH}`]; + expect( + hostContainerEngineArgv([ + "run", + "-d", + "--gpus", + "all", + "-p", + "8000:8000", + "--name", + "nemoclaw-nim-alpha", + "--shm-size", + "16g", + "-e", + "NGC_API_KEY", + "nvcr.io/nim/nvidia/model:latest", + ]), + ).toEqual([ + ...prefix, + "run", + "-d", + "--device", + "nvidia.com/gpu=all", + "-p", + "8000:8000", + "--name", + "nemoclaw-nim-alpha", + "--shm-size", + "16g", + "-e", + "NGC_API_KEY", + "nvcr.io/nim/nvidia/model:latest", + ]); + expect( + hostContainerEngineArgv([ + "run", + "-d", + "--pull=never", + "--gpus", + '"device=0,2"', + "--ipc=host", + "-v", + "/home/test/.cache/huggingface:/root/.cache/huggingface", + "--label", + "com.nvidia.nemoclaw.managed-vllm=true", + "--name", + "nemoclaw-vllm", + "nvcr.io/nvidia/vllm@sha256:deadbeef", + ]), + ).toEqual([ + ...prefix, + "run", + "-d", + "--pull=never", + "--device", + "nvidia.com/gpu=0", + "--device", + "nvidia.com/gpu=2", + "--ipc=host", + "-v", + "/home/test/.cache/huggingface:/root/.cache/huggingface", + "--label", + "com.nvidia.nemoclaw.managed-vllm=true", + "--name", + "nemoclaw-vllm", + "nvcr.io/nvidia/vllm@sha256:deadbeef", + ]); + } finally { + restore(); + } + }); + + it("fails closed instead of dropping or leaking unsupported Docker GPU modes", () => { + expect(() => + translateLocalInferenceArgsForPodman(["run", "--gpus", "capabilities=compute", "image"]), + ).toThrow("cannot translate Docker GPU selector"); + expect(() => + translateLocalInferenceArgsForPodman(["run", "--gpus", "device=0,0", "image"]), + ).toThrow("duplicate device"); + expect(() => + translateLocalInferenceArgsForPodman(["run", "--runtime", "nvidia", "image"]), + ).toThrow("refuses Docker's NVIDIA runtime mode"); + expect(() => + translateLocalInferenceArgsForPodman(["run", "--device", "/dev/nvidia0", "image"]), + ).toThrow("refuses raw NVIDIA device paths"); + expect(() => + translateLocalInferenceArgsForPodman(["run", "--gpus", "device=9", "image"], { + availableCdiDevices: ["nvidia.com/gpu=all", "nvidia.com/gpu=0"], + }), + ).toThrow("does not advertise"); + }); + + it("keeps Linux arm64 on the same exact Podman runtime authority", () => { + const authorityDeps = socketAuthorityDeps(); + let command: HostContainerEngineCommand | null = null; + + activateHostLocalInferenceRuntime( + { driverName: "podman" }, + { + environment: { + OPENSHELL_PODMAN_NETWORK_NAME: "openshell-arm64", + OPENSHELL_PODMAN_SOCKET: SOCKET_PATH, + }, + qualifiedPodmanRuntime: qualifiedPodmanRuntime(authorityDeps, "arm64"), + socketAuthorityDeps: authorityDeps, + configureContainerEngine: (next) => { + command = next; + return () => undefined; + }, + }, + ); + + expect(command).toMatchObject({ + driverName: "podman", + runtimeArchitecture: "arm64", + sandboxNetworkName: "openshell-arm64", + }); + expect(() => command?.assertAuthority?.()).not.toThrow(); + }); + + it("rejects socket/runtime mismatch and later socket replacement", () => { + const authorityDeps = socketAuthorityDeps(); + const qualified = qualifiedPodmanRuntime(authorityDeps); + expect(() => + activateHostLocalInferenceRuntime( + { driverName: "podman" }, + { + environment: { OPENSHELL_PODMAN_SOCKET: "/run/user/1000/podman/other.sock" }, + qualifiedPodmanRuntime: qualified, + socketAuthorityDeps: authorityDeps, + }, + ), + ).toThrow("does not match the qualified Podman socket"); + + let socketInode = 100n; + const changingDeps = socketAuthorityDeps(); + const originalLstat = changingDeps.lstat!; + const mutableDeps: PodmanSocketAuthorityDeps = { + ...changingDeps, + lstat: (filePath) => { + const stat = originalLstat(filePath); + return filePath === SOCKET_PATH ? { ...stat, ino: socketInode } : stat; + }, + }; + let command: HostContainerEngineCommand | null = null; + activateHostLocalInferenceRuntime( + { driverName: "podman" }, + { + environment: { OPENSHELL_PODMAN_SOCKET: SOCKET_PATH }, + qualifiedPodmanRuntime: qualifiedPodmanRuntime(mutableDeps), + socketAuthorityDeps: mutableDeps, + configureContainerEngine: (next) => { + command = next; + return () => undefined; + }, + }, + ); + socketInode = 101n; + expect(() => command?.assertAuthority?.()).toThrow("changed after it was qualified"); + }); + + it("routes an injected MXC adapter without inheriting Podman", () => { + const activate = vi.fn(() => vi.fn()); + const adapters: HostLocalInferenceRuntimeAdapterRegistry = { + mxc: { driverName: "mxc", activate }, + }; + + activateHostLocalInferenceRuntime({ driverName: "mxc" }, {}, adapters); + + expect(activate).toHaveBeenCalledOnce(); + }); + + it("fails closed when a driver has no local-inference runtime adapter", () => { + expect(() => activateHostLocalInferenceRuntime({ driverName: "mxc" }, {}, {})).toThrow( + "has no registered host-local inference runtime adapter", + ); + }); +}); diff --git a/src/lib/onboard/compute/host-local-inference-runtime.ts b/src/lib/onboard/compute/host-local-inference-runtime.ts new file mode 100644 index 0000000000..39aa93b4ee --- /dev/null +++ b/src/lib/onboard/compute/host-local-inference-runtime.ts @@ -0,0 +1,257 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + configureHostContainerEngine, + type HostContainerEngineCommand, +} from "../../adapters/container-engine"; +import type { OpenShellComputePlan } from "./plan"; +import { + assertPodmanGpuAttachmentQualified, + normalizeNvidiaCdiDevice, +} from "./podman/gpu-attachment"; +import { + assertPodmanSocketAuthority, + type PodmanSocketAuthority, + type PodmanSocketAuthorityDeps, +} from "./podman/socket-authority"; +import { + assessNativePodman, + type NativePodmanPreflightDeps, + type NativePodmanPreflightReceipt, +} from "./podman-preflight"; + +export interface HostLocalInferenceRuntimeAdapter { + readonly driverName: string; + activate(input: HostLocalInferenceRuntimeActivationInput): () => void; +} + +export type HostLocalInferenceRuntimeAdapterRegistry = Readonly< + Record +>; + +export interface HostLocalInferenceRuntimeActivationInput { + readonly environment?: NodeJS.ProcessEnv; + readonly nativePodmanDeps?: NativePodmanPreflightDeps; + readonly podmanBin?: string; + readonly qualifiedPodmanRuntime?: NativePodmanPreflightReceipt; + readonly socketAuthorityDeps?: PodmanSocketAuthorityDeps; + readonly configureContainerEngine?: typeof configureHostContainerEngine; +} + +function dockerRuntimeAdapter(driverName: string): HostLocalInferenceRuntimeAdapter { + return { + driverName, + activate(input) { + const configure = input.configureContainerEngine ?? configureHostContainerEngine; + return configure({ + driverName: "docker", + executable: "docker", + }); + }, + }; +} + +export interface PodmanLocalInferenceTranslationOptions { + readonly availableCdiDevices?: readonly string[]; +} + +function stripExactDoubleQuotes(raw: string): string { + const trimmed = raw.trim(); + const startsQuoted = trimmed.startsWith('"'); + const endsQuoted = trimmed.endsWith('"'); + if (startsQuoted !== endsQuoted) { + throw new Error(`Podman local inference cannot translate Docker GPU selector '${raw}' to CDI.`); + } + return startsQuoted ? trimmed.slice(1, -1) : trimmed; +} + +function normalizePodmanGpuSelector( + raw: string, + options: PodmanLocalInferenceTranslationOptions, +): string[] { + const selector = stripExactDoubleQuotes(raw); + const names = + selector === "all" + ? ["all"] + : selector.startsWith("device=") + ? selector.slice("device=".length).split(",") + : []; + if (names.length === 0 || names.some((name) => !name.trim())) { + throw new Error(`Podman local inference cannot translate Docker GPU selector '${raw}' to CDI.`); + } + + const devices = names.map((name) => normalizeNvidiaCdiDevice(name.trim())); + if (new Set(devices).size !== devices.length) { + throw new Error(`Podman local inference GPU selector '${raw}' contains a duplicate device.`); + } + if (options.availableCdiDevices) { + for (const device of devices) { + assertPodmanGpuAttachmentQualified(options.availableCdiDevices, { + kind: "cdi", + device, + }); + } + } + return devices; +} + +/** + * Translate the small Docker-compatible argv subset used by managed NIM and + * single-host vLLM. GPU selection is converted to Podman's CDI-native form; + * a Docker-only selector fails closed instead of silently launching on CPU. + */ +export function translateLocalInferenceArgsForPodman( + args: readonly string[], + options: PodmanLocalInferenceTranslationOptions = {}, +): string[] { + const translated: string[] = []; + for (let index = 0; index < args.length; index += 1) { + const value = args[index] ?? ""; + if (value === "--gpus") { + const selector = args[index + 1]; + if (!selector) throw new Error("Podman local inference requires a GPU selector value."); + for (const device of normalizePodmanGpuSelector(selector, options)) { + translated.push("--device", device); + } + index += 1; + continue; + } + if (value.startsWith("--gpus=")) { + const selector = value.slice("--gpus=".length); + for (const device of normalizePodmanGpuSelector(selector, options)) { + translated.push("--device", device); + } + continue; + } + if (value === "--runtime" && String(args[index + 1] ?? "").toLowerCase() === "nvidia") { + throw new Error( + "Podman local inference refuses Docker's NVIDIA runtime mode; an exact CDI device is required.", + ); + } + if (value.toLowerCase() === "--runtime=nvidia") { + throw new Error( + "Podman local inference refuses Docker's NVIDIA runtime mode; an exact CDI device is required.", + ); + } + if (value === "--device") { + const device = args[index + 1]; + if (!device) throw new Error("Podman local inference requires a --device value."); + if (device.startsWith("nvidia.com/gpu=")) { + const normalized = normalizeNvidiaCdiDevice(device); + if (options.availableCdiDevices) { + assertPodmanGpuAttachmentQualified(options.availableCdiDevices, { + kind: "cdi", + device: normalized, + }); + } + translated.push("--device", normalized); + index += 1; + continue; + } + if (/^\/dev\/nvidia/u.test(device)) { + throw new Error( + "Podman local inference refuses raw NVIDIA device paths; an exact CDI device is required.", + ); + } + } + if (value.startsWith("name=^/") && value.endsWith("$")) { + translated.push(`name=^${value.slice("name=^/".length)}`); + continue; + } + translated.push(value); + } + return translated; +} + +function podmanCommand(input: { + readonly assertAuthority: (authority: PodmanSocketAuthority) => void; + readonly authority: PodmanSocketAuthority; + readonly availableCdiDevices: readonly string[]; + readonly architecture: "amd64" | "arm64"; + readonly networkName: string; + readonly podmanBin: string; +}): HostContainerEngineCommand { + return { + driverName: "podman", + executable: input.podmanBin, + prefixArgs: ["--url", `unix://${input.authority.socketPath}`], + runtimeArchitecture: input.architecture, + sandboxNetworkName: input.networkName, + hostGatewayTarget: "host-gateway", + assertAuthority: () => input.assertAuthority(input.authority), + translateArgs: (args) => + translateLocalInferenceArgsForPodman(args, { + availableCdiDevices: input.availableCdiDevices, + }), + }; +} + +const podmanRuntimeAdapter: HostLocalInferenceRuntimeAdapter = { + driverName: "podman", + activate(input) { + const environment = input.environment ?? process.env; + const socketPath = environment.OPENSHELL_PODMAN_SOCKET?.trim(); + if (!socketPath) { + throw new Error( + "Native Podman host-local inference requires the qualified OPENSHELL_PODMAN_SOCKET.", + ); + } + const qualified = + input.qualifiedPodmanRuntime ?? + assessNativePodman({ + ...input.nativePodmanDeps, + env: environment, + }); + if (qualified.driverName !== "podman" || qualified.socketPath !== socketPath) { + throw new Error( + "Native Podman host-local inference runtime does not match the qualified Podman socket.", + ); + } + const authority = qualified.socketAuthority; + if (authority.socketPath !== socketPath) { + throw new Error( + "Native Podman host-local inference socket authority does not match the qualified runtime.", + ); + } + const assertAuthority = (expected: PodmanSocketAuthority) => + assertPodmanSocketAuthority(expected, input.socketAuthorityDeps); + assertAuthority(authority); + const configure = input.configureContainerEngine ?? configureHostContainerEngine; + return configure( + podmanCommand({ + assertAuthority, + authority, + availableCdiDevices: qualified.cdiDevices, + architecture: qualified.architecture, + networkName: environment.OPENSHELL_PODMAN_NETWORK_NAME?.trim() || "openshell", + podmanBin: input.podmanBin ?? environment.NEMOCLAW_PODMAN_BIN?.trim() ?? "podman", + }), + ); + }, +}; + +export const CURRENT_HOST_LOCAL_INFERENCE_RUNTIME_ADAPTERS = { + docker: dockerRuntimeAdapter("docker"), + kubernetes: dockerRuntimeAdapter("kubernetes"), + podman: podmanRuntimeAdapter, +} as const satisfies HostLocalInferenceRuntimeAdapterRegistry; + +/** + * Activate the host-container lifecycle used by Ollama's optional probe + * containers and by managed NIM/vLLM. The registry is keyed by compute driver, + * so MXC can supply its own engine/endpoint without inheriting Podman. + */ +export function activateHostLocalInferenceRuntime( + plan: Pick, + input: HostLocalInferenceRuntimeActivationInput = {}, + adapters: HostLocalInferenceRuntimeAdapterRegistry = CURRENT_HOST_LOCAL_INFERENCE_RUNTIME_ADAPTERS, +): () => void { + const adapter = Object.hasOwn(adapters, plan.driverName) ? adapters[plan.driverName] : undefined; + if (!adapter || adapter.driverName !== plan.driverName) { + throw new Error( + `OpenShell compute driver '${plan.driverName}' has no registered host-local inference runtime adapter.`, + ); + } + return adapter.activate(input); +} diff --git a/src/lib/onboard/compute/host-process-identity.test.ts b/src/lib/onboard/compute/host-process-identity.test.ts new file mode 100644 index 0000000000..340543afc5 --- /dev/null +++ b/src/lib/onboard/compute/host-process-identity.test.ts @@ -0,0 +1,48 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import { captureLinuxHostProcessIdentity } from "./host-process-identity"; + +function linuxStat(pid: number, name: string, startTicks: string): string { + return `${String(pid)} (${name}) ${["S", ...Array(18).fill("0"), startTicks].join(" ")}`; +} + +describe("host process identity", () => { + it("binds argv to matching process-start samples across the read boundary", () => { + const readFile = vi + .fn() + .mockReturnValueOnce(linuxStat(41, "openshell gateway) worker", "9001")) + .mockReturnValueOnce(Buffer.from("/usr/bin/openshell-gateway\0serve\0")) + .mockReturnValueOnce(linuxStat(41, "openshell gateway) worker", "9001")); + + expect(captureLinuxHostProcessIdentity(41, { readFile })).toEqual({ + argv: ["/usr/bin/openshell-gateway", "serve"], + startIdentity: "linux-proc-start:9001", + }); + expect(readFile.mock.calls.map(([filePath]) => filePath)).toEqual([ + "/proc/41/stat", + "/proc/41/cmdline", + "/proc/41/stat", + ]); + }); + + it("rejects argv captured across PID reuse", () => { + const readFile = vi + .fn() + .mockReturnValueOnce(linuxStat(41, "old", "9001")) + .mockReturnValueOnce(Buffer.from("/usr/bin/foreign\0")) + .mockReturnValueOnce(linuxStat(41, "new", "9002")); + + expect(captureLinuxHostProcessIdentity(41, { readFile })).toBeNull(); + }); + + it("treats a process that disappears before capture as inactive", () => { + const readFile = vi.fn(() => { + throw Object.assign(new Error("gone"), { code: "ENOENT" }); + }); + + expect(captureLinuxHostProcessIdentity(41, { readFile })).toBeNull(); + }); +}); diff --git a/src/lib/onboard/compute/host-process-identity.ts b/src/lib/onboard/compute/host-process-identity.ts new file mode 100644 index 0000000000..b284a8a9ad --- /dev/null +++ b/src/lib/onboard/compute/host-process-identity.ts @@ -0,0 +1,144 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { type SpawnSyncOptions, spawnSync } from "node:child_process"; +import fs from "node:fs"; + +const LINUX_START_TICKS = /^\d+$/u; + +export interface HostProcessIdentity { + readonly argv: readonly string[]; + readonly startIdentity: string; +} + +export interface HostProcessIdentityCommandResult { + readonly error?: Error; + readonly status: number | null; + readonly stderr?: Buffer | string | null; + readonly stdout?: Buffer | string | null; +} + +export type RunHostProcessIdentityCommand = ( + command: string, + args: string[], + options?: SpawnSyncOptions, +) => HostProcessIdentityCommandResult; + +export type ReadHostProcessFile = (filePath: string, encoding?: BufferEncoding) => Buffer | string; + +export interface LinuxHostProcessIdentityOptions { + readonly readFile?: ReadHostProcessFile; +} + +function output(value: Buffer | string | null | undefined): string { + return typeof value === "string" ? value : Buffer.isBuffer(value) ? value.toString("utf8") : ""; +} + +function errnoCode(error: unknown): string { + return error !== null && typeof error === "object" && "code" in error + ? String((error as NodeJS.ErrnoException).code) + : ""; +} + +/** + * Read argv plus Linux `/proc//stat` field 22 as one canonical process + * generation identity. Reading stat twice excludes PID reuse across capture. + */ +export function captureLinuxHostProcessIdentity( + pid: number, + options: LinuxHostProcessIdentityOptions = {}, +): HostProcessIdentity | null { + if (!Number.isSafeInteger(pid) || pid <= 0) return null; + const readProcessFile: ReadHostProcessFile = + options.readFile ?? + ((filePath, encoding) => + encoding ? fs.readFileSync(filePath, encoding) : fs.readFileSync(filePath)); + const readStat = (): string | null => { + let stat: string; + try { + stat = String(readProcessFile(`/proc/${String(pid)}/stat`, "utf8")); + } catch (error) { + const code = errnoCode(error); + if (code === "ENOENT" || code === "ESRCH") return null; + throw new Error(`Failed to read host process stat identity: ${String(error)}`); + } + const closeParen = stat.lastIndexOf(")"); + if (closeParen < 0) return null; + const fields = stat + .slice(closeParen + 1) + .trim() + .split(/\s+/u); + const startTicks = fields[19]; + return startTicks && LINUX_START_TICKS.test(startTicks) + ? `linux-proc-start:${startTicks}` + : null; + }; + const firstStart = readStat(); + if (firstStart === null) return null; + + let argvBuffer: Buffer; + try { + const capturedArgv = readProcessFile(`/proc/${String(pid)}/cmdline`); + argvBuffer = Buffer.isBuffer(capturedArgv) ? capturedArgv : Buffer.from(capturedArgv, "utf8"); + } catch (error) { + const code = errnoCode(error); + if (code === "ENOENT" || code === "ESRCH") return null; + throw new Error(`Failed to read host process argv identity: ${String(error)}`); + } + const argv = argvBuffer.toString("utf8").split("\0"); + if (argv.at(-1) === "") argv.pop(); + if (argv.length === 0 || argv.some((value) => !value)) { + throw new Error("Host process returned incomplete Linux identity"); + } + const secondStart = readStat(); + if (secondStart !== firstStart) return null; + return { argv, startIdentity: firstStart }; +} + +export function captureDarwinHostProcessIdentity( + pid: number, + options: { + readonly env?: NodeJS.ProcessEnv; + readonly run?: RunHostProcessIdentityCommand; + } = {}, +): HostProcessIdentity | null { + if (!Number.isSafeInteger(pid) || pid <= 0) return null; + const run = options.run ?? spawnSync; + const result = run("/bin/ps", ["-ww", "-p", String(pid), "-o", "lstart=", "-o", "command="], { + encoding: "utf8", + env: options.env ?? process.env, + stdio: ["ignore", "pipe", "pipe"], + }); + const stdout = output(result.stdout).trim(); + const stderr = output(result.stderr).trim(); + if (!result.error && result.status === 1 && !stdout && !stderr) return null; + if (result.error || result.status !== 0) { + const reason = result.error?.message || stderr || `exit ${String(result.status)}`; + throw new Error(`Failed to read host process identity: ${reason}`); + } + const match = /^(\S{3}\s+\S{3}\s+\d{1,2}\s+\d{2}:\d{2}:\d{2}\s+\d{4})\s+(.+)$/u.exec(stdout); + if (!match?.[1] || !match[2]) { + throw new Error("Host process returned incomplete macOS identity"); + } + // macOS ps exposes the complete serialized argv rather than NUL-delimited entries. + return { argv: [match[2]], startIdentity: `darwin-lstart:${match[1]}` }; +} + +export function captureHostProcessIdentity( + pid: number, + options: { + readonly env?: NodeJS.ProcessEnv; + readonly platform?: NodeJS.Platform; + readonly run?: RunHostProcessIdentityCommand; + } = {}, +): HostProcessIdentity | null { + const platform = options.platform ?? process.platform; + if (platform === "linux") return captureLinuxHostProcessIdentity(pid); + if (platform === "darwin") { + return captureDarwinHostProcessIdentity(pid, { + env: options.env, + run: options.run, + }); + } + return null; +} diff --git a/src/lib/onboard/compute/managed-gateway-profile.test.ts b/src/lib/onboard/compute/managed-gateway-profile.test.ts new file mode 100644 index 0000000000..dfeef61d55 --- /dev/null +++ b/src/lib/onboard/compute/managed-gateway-profile.test.ts @@ -0,0 +1,156 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; +import { + CURRENT_MANAGED_GATEWAY_DRIVER_PROFILES, + requiresHostSandboxBinaryForInstall, + resolveManagedGatewayDriverProfile, + resolveManagedGatewayRuntimeAdapter, +} from "./managed-gateway-profile"; + +describe("managed gateway driver profiles", () => { + it("keeps Podman host-only and free of Docker cleanup capabilities", () => { + const profile = resolveManagedGatewayDriverProfile({ + driverName: "podman", + gatewayLauncher: "nemoclaw", + }); + expect(profile).toMatchObject({ + driverName: "podman", + launchPolicy: "host-only", + capabilities: { + containerizedGatewayCompat: false, + legacyDockerGatewayCleanup: false, + legacyDockerVolumeCleanup: false, + localSupervisorBinary: false, + }, + }); + }); + + it("does not impose a managed host lifecycle on an OpenShell-owned MXC plan", () => { + expect( + resolveManagedGatewayDriverProfile({ + driverName: "mxc", + gatewayLauncher: "openshell", + }), + ).toBeNull(); + }); + + it("derives the host sandbox binary requirement from the active runtime profile", () => { + expect( + requiresHostSandboxBinaryForInstall(CURRENT_MANAGED_GATEWAY_DRIVER_PROFILES.docker, { + explicitSandboxBinary: false, + platform: "linux", + }), + ).toBe(true); + expect( + requiresHostSandboxBinaryForInstall(CURRENT_MANAGED_GATEWAY_DRIVER_PROFILES.podman, { + explicitSandboxBinary: false, + platform: "linux", + }), + ).toBe(false); + expect( + requiresHostSandboxBinaryForInstall(CURRENT_MANAGED_GATEWAY_DRIVER_PROFILES.podman, { + explicitSandboxBinary: true, + platform: "linux", + }), + ).toBe(true); + expect( + requiresHostSandboxBinaryForInstall(null, { + explicitSandboxBinary: false, + platform: "linux", + }), + ).toBe(true); + }); + + it("accepts an independently registered NemoClaw-managed MXC profile", () => { + expect( + resolveManagedGatewayDriverProfile( + { driverName: "mxc", gatewayLauncher: "nemoclaw" }, + { + ...CURRENT_MANAGED_GATEWAY_DRIVER_PROFILES, + mxc: { + allowWildcardBind: false, + driverName: "mxc", + displayName: "MXC", + incompatibleRuntimeEnvironmentKeys: [], + launchPolicy: "host-only", + runtimeMarkerPolicy: "process-env", + runtimeEnvironmentKeys: ["MXC_ENDPOINT"], + sandboxReachability: "driver-native", + capabilities: { + containerizedGatewayCompat: false, + legacyDockerGatewayCleanup: false, + legacyDockerVolumeCleanup: false, + localSupervisorBinary: false, + packageManagedService: false, + }, + }, + }, + ), + ).toMatchObject({ driverName: "mxc" }); + }); + + it("fails a NemoClaw-owned driver without a matching profile", () => { + expect(() => + resolveManagedGatewayDriverProfile({ + driverName: "mxc", + gatewayLauncher: "nemoclaw", + }), + ).toThrow("requires a registered NemoClaw managed-gateway profile"); + }); + + it("resolves an independently injected runtime adapter by driver identity", () => { + const profile = { + allowWildcardBind: false, + driverName: "mxc", + displayName: "MXC", + incompatibleRuntimeEnvironmentKeys: [], + launchPolicy: "host-only", + runtimeMarkerPolicy: "process-env", + runtimeEnvironmentKeys: ["MXC_ENDPOINT"], + sandboxReachability: "driver-native", + capabilities: { + containerizedGatewayCompat: false, + legacyDockerGatewayCleanup: false, + legacyDockerVolumeCleanup: false, + localSupervisorBinary: false, + packageManagedService: false, + }, + } as const; + const adapter = { + driverName: "mxc", + launchPolicy: "host-only", + runtimeMarkerPolicy: "process-env", + sandboxReachability: "driver-native", + build: () => "mxc-runtime", + } as const; + expect(resolveManagedGatewayRuntimeAdapter(profile, { mxc: adapter })).toBe(adapter); + }); + + it("fails before launch when a managed profile has no matching runtime adapter", () => { + expect(() => + resolveManagedGatewayRuntimeAdapter(CURRENT_MANAGED_GATEWAY_DRIVER_PROFILES.podman, { + docker: { + driverName: "docker", + launchPolicy: "docker-compat", + runtimeMarkerPolicy: "docker-compat-v1", + sandboxReachability: "docker-bridge", + }, + }), + ).toThrow("requires a matching runtime adapter"); + }); + + it("rejects an adapter that would inherit another runtime's lifecycle", () => { + expect(() => + resolveManagedGatewayRuntimeAdapter(CURRENT_MANAGED_GATEWAY_DRIVER_PROFILES.podman, { + podman: { + driverName: "podman", + launchPolicy: "docker-compat", + runtimeMarkerPolicy: "docker-compat-v1", + sandboxReachability: "docker-bridge", + }, + }), + ).toThrow("does not match its registered lifecycle profile"); + }); +}); diff --git a/src/lib/onboard/compute/managed-gateway-profile.ts b/src/lib/onboard/compute/managed-gateway-profile.ts new file mode 100644 index 0000000000..9f27c45667 --- /dev/null +++ b/src/lib/onboard/compute/managed-gateway-profile.ts @@ -0,0 +1,160 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { OpenShellComputePlan } from "./plan"; + +export type ManagedGatewayLaunchPolicy = "docker-compat" | "host-only"; +export type ManagedGatewaySandboxReachability = "docker-bridge" | "podman-host" | "driver-native"; +export type ManagedGatewayRuntimeMarkerPolicy = "docker-compat-v1" | "process-env"; + +export interface ManagedGatewayDriverCapabilities { + readonly containerizedGatewayCompat: boolean; + readonly legacyDockerGatewayCleanup: boolean; + readonly legacyDockerVolumeCleanup: boolean; + readonly localSupervisorBinary: boolean; + readonly packageManagedService: boolean; +} + +export interface ManagedGatewayDriverProfile { + readonly allowWildcardBind: boolean; + readonly driverName: string; + readonly displayName: string; + readonly incompatibleRuntimeEnvironmentKeys: readonly string[]; + readonly launchPolicy: ManagedGatewayLaunchPolicy; + readonly runtimeMarkerPolicy: ManagedGatewayRuntimeMarkerPolicy; + readonly runtimeEnvironmentKeys: readonly string[]; + readonly sandboxReachability: ManagedGatewaySandboxReachability; + readonly capabilities: ManagedGatewayDriverCapabilities; +} + +export type ManagedGatewayDriverProfileRegistry = Readonly< + Record +>; + +export interface ManagedGatewayRuntimeAdapter { + readonly driverName: string; + readonly launchPolicy: ManagedGatewayLaunchPolicy; + readonly runtimeMarkerPolicy: ManagedGatewayRuntimeMarkerPolicy; + readonly sandboxReachability: ManagedGatewaySandboxReachability; +} + +export type ManagedGatewayRuntimeAdapterRegistry< + TAdapter extends ManagedGatewayRuntimeAdapter = ManagedGatewayRuntimeAdapter, +> = Readonly>; + +export const CURRENT_MANAGED_GATEWAY_DRIVER_PROFILES = { + docker: { + allowWildcardBind: false, + driverName: "docker", + displayName: "Docker", + incompatibleRuntimeEnvironmentKeys: ["OPENSHELL_PODMAN_SOCKET", "OPENSHELL_SUPERVISOR_IMAGE"], + launchPolicy: "docker-compat", + runtimeMarkerPolicy: "docker-compat-v1", + runtimeEnvironmentKeys: ["DOCKER_HOST"], + sandboxReachability: "docker-bridge", + capabilities: { + containerizedGatewayCompat: true, + legacyDockerGatewayCleanup: true, + legacyDockerVolumeCleanup: true, + localSupervisorBinary: true, + packageManagedService: true, + }, + }, + podman: { + allowWildcardBind: true, + driverName: "podman", + displayName: "Podman", + incompatibleRuntimeEnvironmentKeys: [ + "DOCKER_HOST", + "OPENSHELL_DOCKER_NETWORK_NAME", + "OPENSHELL_DOCKER_SUPERVISOR_IMAGE", + "OPENSHELL_DOCKER_SUPERVISOR_BIN", + "OPENSHELL_GRPC_ENDPOINT", + "OPENSHELL_DISABLE_GATEWAY_AUTH", + "OPENSHELL_DISABLE_TLS", + ], + launchPolicy: "host-only", + runtimeMarkerPolicy: "process-env", + runtimeEnvironmentKeys: ["OPENSHELL_PODMAN_SOCKET"], + sandboxReachability: "podman-host", + capabilities: { + containerizedGatewayCompat: false, + legacyDockerGatewayCleanup: false, + legacyDockerVolumeCleanup: false, + localSupervisorBinary: false, + packageManagedService: true, + }, + }, +} as const satisfies ManagedGatewayDriverProfileRegistry; + +export class ManagedGatewayDriverProfileError extends Error { + constructor(message: string) { + super(message); + this.name = "ManagedGatewayDriverProfileError"; + } +} + +/** + * Resolve the NemoClaw-owned gateway profile independently from compute + * identity. OpenShell-owned and future extension launchers do not inherit a + * Docker/Podman host lifecycle simply because their driver name is known. + */ +export function resolveManagedGatewayDriverProfile( + plan: OpenShellComputePlan, + profiles: ManagedGatewayDriverProfileRegistry = CURRENT_MANAGED_GATEWAY_DRIVER_PROFILES, +): ManagedGatewayDriverProfile | null { + if (plan.gatewayLauncher !== "nemoclaw") return null; + const profile = Object.hasOwn(profiles, plan.driverName) ? profiles[plan.driverName] : undefined; + if (!profile || profile.driverName !== plan.driverName) { + throw new ManagedGatewayDriverProfileError( + `OpenShell compute driver '${plan.driverName}' requires a registered NemoClaw managed-gateway profile.`, + ); + } + return profile; +} + +export function isDockerComputeDriver(plan: OpenShellComputePlan): boolean { + return plan.driverName === "docker"; +} + +/** + * Decide whether install-integrity validation must include a host-visible + * `openshell-sandbox` binary. Managed runtimes own this requirement through + * their profile; externally launched runtimes retain the existing platform + * fallback. An explicitly configured binary is always validated. + */ +export function requiresHostSandboxBinaryForInstall( + profile: ManagedGatewayDriverProfile | null, + options: { + readonly explicitSandboxBinary: boolean; + readonly platform?: NodeJS.Platform; + }, +): boolean { + if (options.explicitSandboxBinary) return true; + if (profile) return profile.capabilities.localSupervisorBinary; + return (options.platform ?? process.platform) !== "darwin"; +} + +export function resolveManagedGatewayRuntimeAdapter( + profile: ManagedGatewayDriverProfile, + adapters: ManagedGatewayRuntimeAdapterRegistry, +): TAdapter { + const adapter = Object.hasOwn(adapters, profile.driverName) + ? adapters[profile.driverName] + : undefined; + if (!adapter || adapter.driverName !== profile.driverName) { + throw new ManagedGatewayDriverProfileError( + `NemoClaw managed-gateway profile '${profile.driverName}' requires a matching runtime adapter.`, + ); + } + if ( + adapter.launchPolicy !== profile.launchPolicy || + adapter.runtimeMarkerPolicy !== profile.runtimeMarkerPolicy || + adapter.sandboxReachability !== profile.sandboxReachability + ) { + throw new ManagedGatewayDriverProfileError( + `NemoClaw managed-gateway runtime adapter '${profile.driverName}' does not match its registered lifecycle profile.`, + ); + } + return adapter; +} diff --git a/src/lib/onboard/compute/managed-startup-runtime-requirements.test.ts b/src/lib/onboard/compute/managed-startup-runtime-requirements.test.ts new file mode 100644 index 0000000000..8d244e1dab --- /dev/null +++ b/src/lib/onboard/compute/managed-startup-runtime-requirements.test.ts @@ -0,0 +1,109 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; +import { + DCODE_MANAGED_RUNTIME_ULIMITS, + resolveManagedStartupRuntimeRequirements, +} from "./managed-startup-runtime-requirements"; + +const agent = (name: string) => + ({ name }) as Parameters[0]; +const managedGateway = { managedGatewayOwned: true } as const; +const externalGateway = { managedGatewayOwned: false } as const; + +describe("driver-neutral managed-startup runtime requirements", () => { + it("preserves the existing Docker command and DCode limit behavior", () => { + expect( + resolveManagedStartupRuntimeRequirements(agent("openclaw"), "docker", managedGateway), + ).toEqual({ + persistStartupCommand: false, + requiredUlimits: null, + }); + expect( + resolveManagedStartupRuntimeRequirements(agent("hermes"), "docker", managedGateway), + ).toEqual({ + persistStartupCommand: true, + requiredUlimits: null, + }); + expect( + resolveManagedStartupRuntimeRequirements( + agent("langchain-deepagents-code"), + "docker", + managedGateway, + ), + ).toEqual({ + persistStartupCommand: true, + requiredUlimits: DCODE_MANAGED_RUNTIME_ULIMITS, + }); + }); + + it.each([ + "openclaw", + "hermes", + "langchain-deepagents-code", + ])("does not mutate an externally managed Docker gateway for %s", (name) => { + expect( + resolveManagedStartupRuntimeRequirements(agent(name), "docker", externalGateway), + ).toEqual({ + persistStartupCommand: false, + requiredUlimits: null, + }); + }); + + it.each([ + "openclaw", + "hermes", + "langchain-deepagents-code", + ])("persists the image-owned hold for %s on Podman", (name) => { + expect( + resolveManagedStartupRuntimeRequirements(agent(name), "podman", managedGateway) + .persistStartupCommand, + ).toBe(true); + }); + + it("applies DCode's exact limits on Podman without assigning them to other agents", () => { + expect( + resolveManagedStartupRuntimeRequirements( + agent("langchain-deepagents-code"), + "podman", + managedGateway, + ).requiredUlimits, + ).toEqual(DCODE_MANAGED_RUNTIME_ULIMITS); + expect( + resolveManagedStartupRuntimeRequirements(agent("openclaw"), "podman", managedGateway) + .requiredUlimits, + ).toBeNull(); + }); + + it("lets an MXC-shaped runtime inject its own requirements without Docker inheritance", () => { + expect( + resolveManagedStartupRuntimeRequirements(agent("openclaw"), "mxc", managedGateway, { + mxc: { + driverName: "mxc", + resolve: () => ({ + persistStartupCommand: true, + requiredUlimits: [{ name: "nofile", soft: 4096, hard: 4096 }], + }), + }, + }), + ).toEqual({ + persistStartupCommand: true, + requiredUlimits: [{ name: "nofile", soft: 4096, hard: 4096 }], + }); + }); + + it("fails closed for an unregistered or identity-mismatched runtime", () => { + expect(() => + resolveManagedStartupRuntimeRequirements(agent("openclaw"), "mxc", managedGateway), + ).toThrow("has no managed-startup requirements adapter"); + expect(() => + resolveManagedStartupRuntimeRequirements(agent("openclaw"), "mxc", managedGateway, { + mxc: { + driverName: "docker", + resolve: () => ({ persistStartupCommand: false, requiredUlimits: null }), + }, + }), + ).toThrow("has no managed-startup requirements adapter"); + }); +}); diff --git a/src/lib/onboard/compute/managed-startup-runtime-requirements.ts b/src/lib/onboard/compute/managed-startup-runtime-requirements.ts new file mode 100644 index 0000000000..acd47fb01d --- /dev/null +++ b/src/lib/onboard/compute/managed-startup-runtime-requirements.ts @@ -0,0 +1,110 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { AgentDefinition } from "../../agent/defs"; +import { MANAGED_STARTUP_AGENTS } from "../managed-startup/profile"; + +const DCODE_AGENT = "langchain-deepagents-code"; +const MANAGED_IMAGE_AGENTS = new Set(MANAGED_STARTUP_AGENTS); + +export interface SandboxRuntimeUlimit { + readonly name: string; + readonly soft: number; + readonly hard: number; +} + +export interface ManagedStartupRuntimeRequirements { + readonly persistStartupCommand: boolean; + readonly requiredUlimits: readonly SandboxRuntimeUlimit[] | null; +} + +export interface ManagedStartupRuntimeRequirementsContext { + /** + * Whether NemoClaw owns the selected gateway lifecycle and may therefore + * recreate its runtime container to persist command/resource requirements. + */ + readonly managedGatewayOwned: boolean; +} + +export interface ManagedStartupRuntimeRequirementsAdapter { + readonly driverName: string; + resolve( + agentName: string | null, + context: ManagedStartupRuntimeRequirementsContext, + ): ManagedStartupRuntimeRequirements; +} + +export type ManagedStartupRuntimeRequirementsAdapterRegistry = Readonly< + Record +>; + +// DCode's image-owned startup contract fails closed unless the supervisor and +// every child inherit these exact limits. Keep the requirement independent of +// the container engine that materializes it. +export const DCODE_MANAGED_RUNTIME_ULIMITS: readonly SandboxRuntimeUlimit[] = [ + { name: "nproc", soft: 512, hard: 512 }, + { name: "nofile", soft: 65_536, hard: 65_536 }, +]; + +const NONE: ManagedStartupRuntimeRequirements = { + persistStartupCommand: false, + requiredUlimits: null, +}; + +export const CURRENT_MANAGED_STARTUP_RUNTIME_REQUIREMENTS_ADAPTERS = { + docker: { + driverName: "docker", + resolve(agentName, context) { + // Preserve the established ownership boundary: OpenShell-managed Docker + // gateways are not ours to recreate solely for startup persistence or + // DCode resource limits. + if (!context.managedGatewayOwned) return NONE; + return { + // The existing Docker driver preserves OpenClaw's command, while + // Hermes and DCode require the established resource-only recreation. + persistStartupCommand: agentName === "hermes" || agentName === DCODE_AGENT, + requiredUlimits: agentName === DCODE_AGENT ? DCODE_MANAGED_RUNTIME_ULIMITS : null, + }; + }, + }, + kubernetes: { + driverName: "kubernetes", + resolve() { + return NONE; + }, + }, + podman: { + driverName: "podman", + resolve(agentName) { + if (!agentName || !MANAGED_IMAGE_AGENTS.has(agentName)) return NONE; + // The native Podman driver intentionally starts `sleep infinity`; every + // managed image therefore needs the image-owned startup hold persisted + // into the final container configuration. + return { + persistStartupCommand: true, + requiredUlimits: agentName === DCODE_AGENT ? DCODE_MANAGED_RUNTIME_ULIMITS : null, + }; + }, + }, +} as const satisfies ManagedStartupRuntimeRequirementsAdapterRegistry; + +export function resolveManagedStartupRuntimeRequirements( + agent: Pick | null | undefined, + driverName: string, + context: ManagedStartupRuntimeRequirementsContext, + adapters: ManagedStartupRuntimeRequirementsAdapterRegistry = CURRENT_MANAGED_STARTUP_RUNTIME_REQUIREMENTS_ADAPTERS, +): ManagedStartupRuntimeRequirements { + const adapter = Object.hasOwn(adapters, driverName) ? adapters[driverName] : undefined; + if (!adapter || adapter.driverName !== driverName) { + throw new Error( + `OpenShell compute driver '${driverName}' has no managed-startup requirements adapter.`, + ); + } + const resolved = adapter.resolve(agent?.name ?? null, context); + return { + persistStartupCommand: resolved.persistStartupCommand, + requiredUlimits: resolved.requiredUlimits + ? resolved.requiredUlimits.map((limit) => ({ ...limit })) + : null, + }; +} diff --git a/src/lib/onboard/compute/plan.test.ts b/src/lib/onboard/compute/plan.test.ts new file mode 100644 index 0000000000..02f57b8abf --- /dev/null +++ b/src/lib/onboard/compute/plan.test.ts @@ -0,0 +1,207 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; +import { isLinuxDockerDriverGatewayEnabled } from "../docker-driver-platform"; +import { + CURRENT_OPEN_SHELL_COMPUTE_PLANS, + type OpenShellComputeCapabilitiesRegistry, + type OpenShellComputePlan, + type OpenShellComputePlanRegistry, + OpenShellComputeSelectionError, + resolveCurrentOpenShellComputePlan, + resolveOpenShellComputeCapabilities, + resolveOpenShellComputeSelection, + resolvePersistedOpenShellComputeDriver, + usesManagedDockerGateway, +} from "./plan"; + +const AUTO_DOCKER_PLAN: OpenShellComputePlan = { + driverName: "docker", + gatewayLauncher: "nemoclaw", +}; + +describe("current OpenShell compute plan", () => { + it.each([ + { + label: "Linux x64", + platform: "linux" as const, + arch: "x64" as const, + driverName: "docker", + gatewayLauncher: "nemoclaw", + }, + { + label: "Linux arm64", + platform: "linux" as const, + arch: "arm64" as const, + driverName: "docker", + gatewayLauncher: "nemoclaw", + }, + { + label: "Apple Silicon macOS", + platform: "darwin" as const, + arch: "arm64" as const, + driverName: "docker", + gatewayLauncher: "nemoclaw", + }, + { + label: "Intel macOS", + platform: "darwin" as const, + arch: "x64" as const, + driverName: "kubernetes", + gatewayLauncher: "openshell", + }, + { + label: "Windows x64", + platform: "win32" as const, + arch: "x64" as const, + driverName: "kubernetes", + gatewayLauncher: "openshell", + }, + ])("preserves the existing driver and gateway-launch behavior on $label (#7744)", ({ + platform, + arch, + driverName, + gatewayLauncher, + }) => { + expect(resolveCurrentOpenShellComputePlan(platform, arch)).toEqual({ + driverName, + gatewayLauncher, + }); + expect(isLinuxDockerDriverGatewayEnabled(platform, arch)).toBe(driverName === "docker"); + }); + + it.each([ + { driverName: "docker", gatewayLauncher: "nemoclaw", expected: true }, + { driverName: "docker", gatewayLauncher: "openshell", expected: false }, + { driverName: "podman", gatewayLauncher: "nemoclaw", expected: false }, + { driverName: "mxc", gatewayLauncher: "nemoclaw", expected: false }, + ] as const)("reports Docker lifecycle ownership as $expected for $driverName with the $gatewayLauncher launcher (#7744)", ({ + driverName, + gatewayLauncher, + expected, + }) => { + expect(usesManagedDockerGateway({ driverName, gatewayLauncher })).toBe(expected); + }); + + it("resolves an internal Podman request without exposing public wiring (#7744)", () => { + expect( + resolveOpenShellComputeSelection({ + requestedDriver: "podman", + autoPlan: AUTO_DOCKER_PLAN, + }), + ).toEqual({ + driverName: "podman", + gatewayLauncher: "nemoclaw", + }); + }); + + it("keeps auto deterministic and preserves a persisted driver (#7744)", () => { + expect( + resolveOpenShellComputeSelection({ + requestedDriver: "auto", + persistedDriver: "podman", + autoPlan: AUTO_DOCKER_PLAN, + }), + ).toEqual({ + driverName: "podman", + gatewayLauncher: "nemoclaw", + }); + expect( + resolveOpenShellComputeSelection({ + requestedDriver: "auto", + autoPlan: AUTO_DOCKER_PLAN, + }), + ).toEqual(AUTO_DOCKER_PLAN); + }); + + it("collapses matching durable driver evidence and rejects drift (#7744)", () => { + expect( + resolvePersistedOpenShellComputeDriver([ + { source: "onboarding session", driverName: " podman " }, + { source: "sandbox registry", driverName: "podman" }, + { source: "empty legacy record", driverName: null }, + ]), + ).toBe("podman"); + + expect(() => + resolvePersistedOpenShellComputeDriver([ + { source: "onboarding session", driverName: "docker" }, + { source: "sandbox registry", driverName: "podman" }, + ]), + ).toThrow( + "Conflicting persisted OpenShell compute drivers: onboarding session='docker', sandbox registry='podman'.", + ); + }); + + it("rejects an explicit driver that differs from persisted identity (#7744)", () => { + expect(() => + resolveOpenShellComputeSelection({ + requestedDriver: "podman", + persistedDriver: "docker", + autoPlan: AUTO_DOCKER_PLAN, + }), + ).toThrow( + "Requested OpenShell compute driver 'podman' does not match existing sandbox driver 'docker'.", + ); + }); + + it("fails an unknown driver closed (#7744)", () => { + expect(() => + resolveOpenShellComputeSelection({ + requestedDriver: "future-runtime", + autoPlan: AUTO_DOCKER_PLAN, + }), + ).toThrow(OpenShellComputeSelectionError); + }); + + it("accepts an injected MXC-shaped plan without inheriting Podman lifecycle (#7744)", () => { + const plans: OpenShellComputePlanRegistry = { + ...CURRENT_OPEN_SHELL_COMPUTE_PLANS, + mxc: { + driverName: "mxc", + gatewayLauncher: "openshell", + }, + }; + + expect( + resolveOpenShellComputeSelection( + { + requestedDriver: "mxc", + autoPlan: AUTO_DOCKER_PLAN, + }, + plans, + ), + ).toEqual({ + driverName: "mxc", + gatewayLauncher: "openshell", + }); + + const capabilities: OpenShellComputeCapabilitiesRegistry = { + mxc: { hostLocalInference: false }, + }; + expect(resolveOpenShellComputeCapabilities({ driverName: "mxc" }, capabilities)).toEqual({ + hostLocalInference: false, + }); + }); + + it("keeps compute capabilities independent from gateway ownership", () => { + expect(resolveOpenShellComputeCapabilities(CURRENT_OPEN_SHELL_COMPUTE_PLANS.docker)).toEqual({ + hostLocalInference: true, + }); + expect(resolveOpenShellComputeCapabilities(CURRENT_OPEN_SHELL_COMPUTE_PLANS.podman)).toEqual({ + hostLocalInference: true, + }); + expect( + resolveOpenShellComputeCapabilities(CURRENT_OPEN_SHELL_COMPUTE_PLANS.kubernetes), + ).toEqual({ + hostLocalInference: true, + }); + }); + + it("fails an unregistered compute capability profile closed", () => { + expect(() => resolveOpenShellComputeCapabilities({ driverName: "mxc" })).toThrow( + "has no registered capability profile", + ); + }); +}); diff --git a/src/lib/onboard/compute/plan.ts b/src/lib/onboard/compute/plan.ts new file mode 100644 index 0000000000..c7c150a5a4 --- /dev/null +++ b/src/lib/onboard/compute/plan.ts @@ -0,0 +1,176 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export type OpenShellGatewayLauncher = "nemoclaw" | "openshell"; +export const OPEN_SHELL_COMPUTE_DRIVER_ENV = "NEMOCLAW_COMPUTE_DRIVER"; +export const OPEN_SHELL_COMPUTE_DRIVER_REQUESTS = ["auto", "docker", "podman"] as const; +export type OpenShellComputeDriverRequest = (typeof OPEN_SHELL_COMPUTE_DRIVER_REQUESTS)[number]; + +/** + * Keeps OpenShell driver identity separate from the component that launches + * its gateway. A future driver does not inherit Docker lifecycle behavior + * because NemoClaw launches its gateway. + */ +export interface OpenShellComputePlan { + readonly driverName: string; + readonly gatewayLauncher: OpenShellGatewayLauncher; +} + +export type OpenShellComputePlanRegistry = Readonly>; + +export interface OpenShellComputeCapabilities { + readonly hostLocalInference: boolean; +} + +export type OpenShellComputeCapabilitiesRegistry = Readonly< + Record +>; + +/** + * Driver plans known to the internal selection seam. Registering a plan does + * not expose a CLI selection; public wiring follows runtime qualification. + */ +export const CURRENT_OPEN_SHELL_COMPUTE_PLANS = { + docker: { + driverName: "docker", + gatewayLauncher: "nemoclaw", + }, + kubernetes: { + driverName: "kubernetes", + gatewayLauncher: "openshell", + }, + podman: { + driverName: "podman", + gatewayLauncher: "nemoclaw", + }, +} as const satisfies OpenShellComputePlanRegistry; + +export const CURRENT_OPEN_SHELL_COMPUTE_CAPABILITIES = { + docker: { hostLocalInference: true }, + kubernetes: { hostLocalInference: true }, + podman: { hostLocalInference: true }, +} as const satisfies OpenShellComputeCapabilitiesRegistry; + +export interface ResolveOpenShellComputeSelectionInput { + readonly requestedDriver?: string | null; + readonly persistedDriver?: string | null; + readonly autoPlan: OpenShellComputePlan; +} + +export interface PersistedOpenShellComputeDriverEvidence { + readonly source: string; + readonly driverName?: string | null; +} + +export class OpenShellComputeSelectionError extends Error { + constructor(message: string) { + super(message); + this.name = "OpenShellComputeSelectionError"; + } +} + +export function resolveOpenShellComputeDriverRequest( + requested: string | null | undefined, + environment: NodeJS.ProcessEnv = process.env, +): OpenShellComputeDriverRequest { + const fromFlag = requested?.trim(); + const fromEnvironment = environment[OPEN_SHELL_COMPUTE_DRIVER_ENV]?.trim(); + const candidate = (fromFlag || fromEnvironment || "auto").toLowerCase(); + if (!(OPEN_SHELL_COMPUTE_DRIVER_REQUESTS as readonly string[]).includes(candidate)) { + throw new OpenShellComputeSelectionError( + `${OPEN_SHELL_COMPUTE_DRIVER_ENV} and --compute-driver must be one of: ${OPEN_SHELL_COMPUTE_DRIVER_REQUESTS.join(", ")}.`, + ); + } + return candidate as OpenShellComputeDriverRequest; +} + +/** + * Collapse durable driver evidence before any runtime mutation. Multiple + * durable records may describe the same sandbox while onboarding is being + * resumed or rebuilt; disagreement is corruption, not a migration request. + */ +export function resolvePersistedOpenShellComputeDriver( + evidence: readonly PersistedOpenShellComputeDriverEvidence[], +): string | null { + const observed = evidence.flatMap(({ source, driverName }) => { + const normalized = driverName?.trim(); + return normalized ? [{ source, driverName: normalized }] : []; + }); + const distinctDrivers = new Set(observed.map(({ driverName }) => driverName)); + if (distinctDrivers.size <= 1) return observed[0]?.driverName ?? null; + + throw new OpenShellComputeSelectionError( + `Conflicting persisted OpenShell compute drivers: ${observed + .map(({ source, driverName }) => `${source}='${driverName}'`) + .join(", ")}.`, + ); +} + +/** + * Resolve an internal driver request without changing an existing sandbox's + * persisted driver. The plan registry is injectable so future drivers such as + * MXC do not need Docker or Podman lifecycle behavior. + */ +export function resolveOpenShellComputeSelection( + input: ResolveOpenShellComputeSelectionInput, + plans: OpenShellComputePlanRegistry = CURRENT_OPEN_SHELL_COMPUTE_PLANS, +): OpenShellComputePlan { + const requestedDriver = input.requestedDriver ?? "auto"; + const persistedDriver = input.persistedDriver ?? null; + const driverName = + requestedDriver === "auto" ? (persistedDriver ?? input.autoPlan.driverName) : requestedDriver; + + if (persistedDriver !== null && driverName !== persistedDriver) { + throw new OpenShellComputeSelectionError( + `Requested OpenShell compute driver '${driverName}' does not match existing sandbox driver '${persistedDriver}'.`, + ); + } + + const plan = Object.hasOwn(plans, driverName) ? plans[driverName] : undefined; + if (plan === undefined || plan.driverName !== driverName) { + throw new OpenShellComputeSelectionError( + `OpenShell compute driver '${driverName}' is not registered.`, + ); + } + return { ...plan }; +} + +/** + * Compute/workload capabilities are independent from gateway lifecycle + * ownership. Future drivers such as MXC must register their runtime behavior + * explicitly instead of inheriting Docker behavior from a launcher choice. + */ +export function resolveOpenShellComputeCapabilities( + plan: Pick, + capabilities: OpenShellComputeCapabilitiesRegistry = CURRENT_OPEN_SHELL_COMPUTE_CAPABILITIES, +): OpenShellComputeCapabilities { + const resolved = Object.hasOwn(capabilities, plan.driverName) + ? capabilities[plan.driverName] + : undefined; + if (!resolved) { + throw new OpenShellComputeSelectionError( + `OpenShell compute driver '${plan.driverName}' has no registered capability profile.`, + ); + } + return { ...resolved }; +} + +/** + * Describes the behavior NemoClaw uses today. Driver selection will move behind + * this seam without changing the existing Docker and Kubernetes paths first. + */ +export function resolveCurrentOpenShellComputePlan( + platform: NodeJS.Platform = process.platform, + arch: NodeJS.Architecture = process.arch, +): OpenShellComputePlan { + const managedDockerGateway = platform === "linux" || (platform === "darwin" && arch === "arm64"); + const driverName = managedDockerGateway ? "docker" : "kubernetes"; + + return { ...CURRENT_OPEN_SHELL_COMPUTE_PLANS[driverName] }; +} + +export function usesManagedDockerGateway( + plan: Pick, +): boolean { + return plan.driverName === "docker" && plan.gatewayLauncher === "nemoclaw"; +} diff --git a/src/lib/onboard/compute/podman-preflight.test.ts b/src/lib/onboard/compute/podman-preflight.test.ts new file mode 100644 index 0000000000..d5d1350048 --- /dev/null +++ b/src/lib/onboard/compute/podman-preflight.test.ts @@ -0,0 +1,245 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; +import { + assessNativePodman, + isPodmanVersionSupported, + nativePodmanSocketCandidates, +} from "./podman-preflight"; + +const INFO = JSON.stringify({ + host: { + arch: "amd64", + os: "linux", + cgroupVersion: "v2", + networkBackend: "netavark", + security: { rootless: true }, + cdi: { + devices: ["nvidia.com/gpu=all", "nvidia.com/gpu=GPU-deadbeef"], + }, + }, +}); +const SOCKET_AUTHORITY = { + directoryChain: [ + { device: "8", inode: "7000", mode: "448", ownerUid: "1000", path: "/runtime" }, + { device: "8", inode: "1", mode: "493", ownerUid: "0", path: "/" }, + ], + device: "8", + inode: "9001", + ownerUid: "1000", + socketPath: "/runtime/podman.sock", +} as const; + +function socketStat(filePath: string) { + const socket = filePath === "/runtime/podman.sock"; + return { + dev: 8, + ino: socket ? 9001 : filePath === "/runtime" ? 7000 : 1, + mode: socket ? 0o600 : filePath === "/runtime" ? 0o700 : 0o755, + uid: socket || filePath === "/runtime" ? 1000 : 0, + isDirectory: () => !socket, + isSocket: () => socket, + }; +} + +function successfulRun(command: string, args: readonly string[]) { + switch (`${command}:${args[0] ?? ""}`) { + case "lsof:-v": + return { status: 0, stdout: "", stderr: "" }; + case "podman:--version": + return { status: 0, stdout: "podman version 5.6.2\n", stderr: "" }; + case "podman:unshare": + return { status: 0, stdout: "0 1000 1\n1 100000 65536\n", stderr: "" }; + case "podman:--url": + return { status: 0, stdout: INFO, stderr: "" }; + case "nvidia-ctk:cdi": + return { + status: 0, + stdout: [ + "nvidia.com/gpu=all", + "nvidia.com/gpu=0", + "nvidia.com/gpu=GPU-deadbeef", + "nvidia.com/gpu=MIG-GPU-deadbeef/1/0", + "", + ].join("\n"), + stderr: "", + }; + default: + return { status: 1, stdout: "", stderr: "unexpected command" }; + } +} + +describe("native Podman preflight", () => { + it.each([ + ["4.9.9", false], + ["5.0.0", true], + ["5.6.2", true], + ["6.0.0-dev", true], + ["unknown", false], + ])("checks supported version %s", (version, expected) => { + expect(isPodmanVersionSupported(version)).toBe(expected); + }); + + it("uses explicit socket authority without falling through to ambient candidates", () => { + expect( + nativePodmanSocketCandidates({ + env: { + HOME: "/home/tester", + XDG_RUNTIME_DIR: "/run/user/1000", + OPENSHELL_PODMAN_SOCKET: "/runtime/podman.sock", + }, + uid: 1000, + }), + ).toEqual(["/runtime/podman.sock"]); + }); + + it("qualifies Linux amd64 rootless Podman through the exact API socket", () => { + const receipt = assessNativePodman({ + platform: "linux", + architecture: "x64", + env: { OPENSHELL_PODMAN_SOCKET: "/runtime/podman.sock" }, + uid: 1000, + lstatSync: socketStat as never, + run: successfulRun, + }); + + expect(receipt).toEqual({ + driverName: "podman", + version: "5.6.2", + socketPath: "/runtime/podman.sock", + socketAuthority: SOCKET_AUTHORITY, + rootless: true, + cgroupVersion: "v2", + os: "linux", + architecture: "amd64", + networkBackend: "netavark", + cdiDevices: [ + "nvidia.com/gpu=0", + "nvidia.com/gpu=GPU-deadbeef", + "nvidia.com/gpu=MIG-GPU-deadbeef/1/0", + "nvidia.com/gpu=all", + ], + }); + }); + + it("qualifies Linux arm64 and preserves the Podman CDI device identities", () => { + const receipt = assessNativePodman({ + platform: "linux", + architecture: "arm64", + env: { OPENSHELL_PODMAN_SOCKET: "/runtime/podman.sock" }, + uid: 1000, + lstatSync: socketStat as never, + run: (command, args) => + args[0] === "--url" + ? { + status: 0, + stdout: JSON.stringify({ + ...JSON.parse(INFO), + host: { ...JSON.parse(INFO).host, arch: "aarch64" }, + }), + stderr: "", + } + : successfulRun(command, args), + }); + + expect(receipt.architecture).toBe("arm64"); + expect(receipt.cdiDevices).toContain("nvidia.com/gpu=all"); + expect(receipt.cdiDevices).toContain("nvidia.com/gpu=MIG-GPU-deadbeef/1/0"); + }); + + it("rejects a socket replacement during the Podman info probe", () => { + const assertSocketAuthority = vi + .fn() + .mockImplementationOnce(() => {}) + .mockImplementationOnce(() => { + throw new Error("Podman socket authority changed after it was qualified."); + }); + + expect(() => + assessNativePodman({ + platform: "linux", + architecture: "x64", + env: { OPENSHELL_PODMAN_SOCKET: "/runtime/podman.sock" }, + uid: 1000, + lstatSync: socketStat as never, + assertSocketAuthority, + run: successfulRun, + }), + ).toThrow("socket authority changed"); + expect(assertSocketAuthority).toHaveBeenCalledTimes(2); + }); + + it("rejects nonroot, cgroup-v1, unsupported architectures, and missing subordinate mappings", () => { + const cases = [ + { + info: { + ...JSON.parse(INFO), + host: { ...JSON.parse(INFO).host, security: { rootless: false } }, + }, + message: "requires a rootless Podman", + }, + { + info: { ...JSON.parse(INFO), host: { ...JSON.parse(INFO).host, cgroupVersion: "v1" } }, + message: "requires cgroups v2", + }, + { + info: { ...JSON.parse(INFO), host: { ...JSON.parse(INFO).host, arch: "ppc64le" } }, + message: "requires amd64 or arm64", + }, + ]; + for (const testCase of cases) { + expect(() => + assessNativePodman({ + platform: "linux", + architecture: "x64", + env: { OPENSHELL_PODMAN_SOCKET: "/runtime/podman.sock" }, + uid: 1000, + lstatSync: socketStat as never, + run: (command, args) => + args[0] === "--version" + ? successfulRun(command, args) + : args[0] === "unshare" + ? successfulRun(command, args) + : { status: 0, stdout: JSON.stringify(testCase.info), stderr: "" }, + }), + ).toThrow(testCase.message); + } + + expect(() => + assessNativePodman({ + platform: "linux", + architecture: "x64", + env: { OPENSHELL_PODMAN_SOCKET: "/runtime/podman.sock" }, + uid: 1000, + lstatSync: socketStat as never, + run: (command, args) => + args[0] === "unshare" + ? { status: 0, stdout: "0 1000 1\n", stderr: "" } + : successfulRun(command, args), + }), + ).toThrow("requires a subordinate UID range"); + }); + + it("fails before probing for unsupported host platforms", () => { + expect(() => + assessNativePodman({ + platform: "darwin", + architecture: "arm64", + run: () => { + throw new Error("must not run"); + }, + }), + ).toThrow("requires Linux amd64 or arm64"); + }); + + it("fails with remediation when complete listener inspection is unavailable", () => { + expect(() => + assessNativePodman({ + platform: "linux", + architecture: "x64", + run: () => ({ status: 1, stdout: "", stderr: "missing" }), + }), + ).toThrow("requires lsof"); + }); +}); diff --git a/src/lib/onboard/compute/podman-preflight.ts b/src/lib/onboard/compute/podman-preflight.ts new file mode 100644 index 0000000000..9e0ba451b2 --- /dev/null +++ b/src/lib/onboard/compute/podman-preflight.ts @@ -0,0 +1,350 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { normalizeNvidiaCdiDevice } from "./podman/gpu-attachment"; +import { + assertPodmanSocketAuthority, + capturePodmanSocketAuthority, + type PodmanSocketAuthority, +} from "./podman/socket-authority"; + +export const MINIMUM_NATIVE_PODMAN_VERSION = "5.0.0"; + +export interface NativePodmanPreflightReceipt { + readonly driverName: "podman"; + readonly version: string; + readonly socketPath: string; + readonly socketAuthority: PodmanSocketAuthority; + readonly rootless: true; + readonly cgroupVersion: "v2"; + readonly os: "linux"; + readonly architecture: "amd64" | "arm64"; + readonly networkBackend: string; + readonly cdiDevices: readonly string[]; +} + +interface CommandResult { + readonly status: number | null; + readonly stdout: string; + readonly stderr: string; +} + +export interface NativePodmanPreflightDeps { + readonly platform?: NodeJS.Platform; + readonly architecture?: NodeJS.Architecture; + readonly env?: NodeJS.ProcessEnv; + readonly uid?: number; + readonly home?: string; + readonly lstatSync?: typeof fs.lstatSync; + readonly assertSocketAuthority?: (expected: PodmanSocketAuthority) => void; + readonly run?: (command: string, args: readonly string[]) => CommandResult; +} + +export class NativePodmanPreflightError extends Error { + constructor(message: string) { + super(message); + this.name = "NativePodmanPreflightError"; + } +} + +function defaultRun(command: string, args: readonly string[]): CommandResult { + const result = spawnSync(command, [...args], { + encoding: "utf-8", + stdio: ["ignore", "pipe", "pipe"], + timeout: 15_000, + }); + return { + status: result.status, + stdout: result.stdout || "", + stderr: result.stderr || result.error?.message || "", + }; +} + +function dottedVersion(value: string): readonly number[] | null { + const match = value.match(/(\d+)\.(\d+)\.(\d+)/); + return match ? match.slice(1, 4).map(Number) : null; +} + +export function isPodmanVersionSupported( + actual: string, + minimum = MINIMUM_NATIVE_PODMAN_VERSION, +): boolean { + const actualParts = dottedVersion(actual); + const minimumParts = dottedVersion(minimum); + if (!actualParts || !minimumParts) return false; + for (let index = 0; index < minimumParts.length; index += 1) { + const delta = (actualParts[index] ?? 0) - (minimumParts[index] ?? 0); + if (delta !== 0) return delta > 0; + } + return true; +} + +function record(value: unknown): Record | null { + return typeof value === "object" && value !== null && !Array.isArray(value) + ? (value as Record) + : null; +} + +function field(value: unknown, ...names: string[]): unknown { + const object = record(value); + if (!object) return undefined; + for (const name of names) { + if (Object.hasOwn(object, name)) return object[name]; + } + return undefined; +} + +function stringField(value: unknown, ...names: string[]): string { + const candidate = field(value, ...names); + return typeof candidate === "string" ? candidate.trim() : ""; +} + +function booleanField(value: unknown, ...names: string[]): boolean | null { + const candidate = field(value, ...names); + return typeof candidate === "boolean" ? candidate : null; +} + +function normalizePodmanArchitecture(value: string): "amd64" | "arm64" | null { + if (value === "amd64" || value === "x86_64") return "amd64"; + if (value === "arm64" || value === "aarch64") return "arm64"; + return null; +} + +function collectCdiDevices(value: unknown, devices: Set): void { + if (typeof value === "string") { + if (value.startsWith("nvidia.com/gpu=")) devices.add(value); + return; + } + if (Array.isArray(value)) { + for (const entry of value) collectCdiDevices(entry, devices); + return; + } + const object = record(value); + if (!object) return; + for (const [key, entry] of Object.entries(object)) { + if (key.startsWith("nvidia.com/gpu=")) devices.add(key); + collectCdiDevices(entry, devices); + } +} + +function parseNvidiaCdiDeviceList(output: string): string[] { + const devices = new Set(); + for (const line of output.split(/\r?\n/u)) { + const candidate = line.trim(); + if (!candidate.startsWith("nvidia.com/gpu=")) continue; + try { + devices.add(normalizeNvidiaCdiDevice(candidate)); + } catch { + // Ignore diagnostics and malformed provider output. Requested devices + // are still checked exactly against the resulting qualified inventory. + } + } + return [...devices].sort(); +} + +function listNvidiaCdiDevices( + run: NonNullable, +): readonly string[] { + const result = run("nvidia-ctk", ["cdi", "list"]); + return result.status === 0 ? parseNvidiaCdiDeviceList(result.stdout) : []; +} + +function unique(values: readonly string[]): string[] { + return [...new Set(values)]; +} + +export function nativePodmanSocketCandidates(input: { + readonly env?: NodeJS.ProcessEnv; + readonly uid?: number; + readonly home?: string; +}): string[] { + const env = input.env ?? process.env; + const uid = input.uid ?? (typeof process.getuid === "function" ? process.getuid() : -1); + const home = input.home ?? env.HOME ?? os.homedir(); + const explicit = env.OPENSHELL_PODMAN_SOCKET?.trim(); + if (explicit) return [explicit]; + + return unique( + [ + env.XDG_RUNTIME_DIR?.trim() + ? path.join(env.XDG_RUNTIME_DIR.trim(), "podman", "podman.sock") + : "", + uid >= 0 ? `/run/user/${String(uid)}/podman/podman.sock` : "", + home ? path.join(home, ".local", "share", "containers", "podman", "podman.sock") : "", + ].filter(Boolean), + ); +} + +function assertAbsoluteSocketPath(socketPath: string, explicit: boolean): void { + if (!path.isAbsolute(socketPath)) { + throw new NativePodmanPreflightError( + `${explicit ? "OPENSHELL_PODMAN_SOCKET" : "Podman socket"} must be an absolute path.`, + ); + } +} + +function parsePodmanInfo( + output: string, + socketAuthority: PodmanSocketAuthority, +): NativePodmanPreflightReceipt { + const socketPath = socketAuthority.socketPath; + let parsed: unknown; + try { + parsed = JSON.parse(output); + } catch { + throw new NativePodmanPreflightError( + `Podman API socket '${socketPath}' returned unreadable system information.`, + ); + } + const host = field(parsed, "host", "Host"); + const security = field(host, "security", "Security"); + const rootless = booleanField(security, "rootless", "Rootless"); + const cgroupVersion = stringField( + host, + "cgroupVersion", + "cgroupsVersion", + "CgroupVersion", + "CgroupsVersion", + ).toLowerCase(); + const hostOs = stringField(host, "os", "OS").toLowerCase(); + const architecture = normalizePodmanArchitecture(stringField(host, "arch", "Arch").toLowerCase()); + const networkBackend = stringField(host, "networkBackend", "NetworkBackend") || "unknown"; + const cdiDevices = new Set(); + collectCdiDevices(host, cdiDevices); + + if (rootless !== true) { + throw new NativePodmanPreflightError( + "Native Podman support requires a rootless Podman API service.", + ); + } + if (cgroupVersion !== "v2") { + throw new NativePodmanPreflightError( + `Native Podman support requires cgroups v2; detected '${cgroupVersion || "unknown"}'.`, + ); + } + if (hostOs !== "linux") { + throw new NativePodmanPreflightError( + `Native Podman support currently requires Linux; the Podman service reports '${hostOs || "unknown"}'.`, + ); + } + if (architecture === null) { + throw new NativePodmanPreflightError( + `Native Podman support requires amd64 or arm64; the Podman service reports '${stringField(host, "arch", "Arch") || "unknown"}'.`, + ); + } + + return { + driverName: "podman", + version: "", + socketPath, + socketAuthority, + rootless: true, + cgroupVersion: "v2", + os: "linux", + architecture, + networkBackend, + cdiDevices: [...cdiDevices].sort(), + }; +} + +function hasSubordinateIdMapping(output: string): boolean { + return output + .trim() + .split(/\r?\n/) + .some((line) => { + const values = line.trim().split(/\s+/).map(Number); + return values.length === 3 && values.every(Number.isFinite) && (values[2] ?? 0) > 1; + }); +} + +function assertSubordinateIds(run: NonNullable): void { + for (const map of ["uid_map", "gid_map"] as const) { + const result = run("podman", ["unshare", "cat", `/proc/self/${map}`]); + if (result.status !== 0 || !hasSubordinateIdMapping(result.stdout)) { + throw new NativePodmanPreflightError( + `Rootless Podman requires a subordinate ${map === "uid_map" ? "UID" : "GID"} range for the current user.`, + ); + } + } +} + +export function assessNativePodman( + deps: NativePodmanPreflightDeps = {}, +): NativePodmanPreflightReceipt { + const platform = deps.platform ?? process.platform; + const architecture = deps.architecture ?? process.arch; + const env = deps.env ?? process.env; + const run = deps.run ?? defaultRun; + const lstatSync = deps.lstatSync ?? fs.lstatSync; + const proveSocketAuthority = + deps.assertSocketAuthority ?? + ((expected: PodmanSocketAuthority) => + assertPodmanSocketAuthority(expected, { + lstat: (filePath) => lstatSync(filePath), + uid: deps.uid, + })); + + if (platform !== "linux" || !["x64", "arm64"].includes(architecture)) { + throw new NativePodmanPreflightError( + `Native Podman support requires Linux amd64 or arm64; detected ${platform} ${architecture}.`, + ); + } + + const listenerInspector = run("lsof", ["-v"]); + if (listenerInspector.status !== 0) { + throw new NativePodmanPreflightError( + "Native Podman support requires lsof for complete gateway listener ownership proof. Install lsof and retry.", + ); + } + + const versionResult = run("podman", ["--version"]); + const version = dottedVersion(versionResult.stdout)?.join(".") ?? ""; + if (versionResult.status !== 0 || !isPodmanVersionSupported(version)) { + throw new NativePodmanPreflightError( + `Native Podman support requires Podman ${MINIMUM_NATIVE_PODMAN_VERSION} or newer; detected '${version || "unavailable"}'.`, + ); + } + + const explicitSocket = Boolean(env.OPENSHELL_PODMAN_SOCKET?.trim()); + const candidates = nativePodmanSocketCandidates({ + env, + uid: deps.uid, + home: deps.home, + }); + let lastDiagnostic = ""; + for (const socketPath of candidates) { + assertAbsoluteSocketPath(socketPath, explicitSocket); + let socketAuthority: PodmanSocketAuthority; + try { + socketAuthority = capturePodmanSocketAuthority(socketPath, { + lstat: (filePath) => lstatSync(filePath), + uid: deps.uid, + }); + } catch (error) { + lastDiagnostic = error instanceof Error ? error.message : String(error); + continue; + } + proveSocketAuthority(socketAuthority); + const info = run("podman", ["--url", `unix://${socketPath}`, "info", "--format", "json"]); + proveSocketAuthority(socketAuthority); + if (info.status !== 0) { + lastDiagnostic = info.stderr.trim() || `Podman API probe failed for ${socketPath}`; + continue; + } + const receipt = parsePodmanInfo(info.stdout, socketAuthority); + assertSubordinateIds(run); + return { + ...receipt, + version, + cdiDevices: unique([...receipt.cdiDevices, ...listNvidiaCdiDevices(run)]).sort(), + }; + } + + throw new NativePodmanPreflightError( + `No responsive rootless Podman API socket was found${lastDiagnostic ? ` (${lastDiagnostic})` : ""}. Enable podman.socket or set OPENSHELL_PODMAN_SOCKET.`, + ); +} diff --git a/src/lib/onboard/compute/podman/active-watcher.test.ts b/src/lib/onboard/compute/podman/active-watcher.test.ts new file mode 100644 index 0000000000..003c8ef08d --- /dev/null +++ b/src/lib/onboard/compute/podman/active-watcher.test.ts @@ -0,0 +1,172 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import type { TrustedActiveOpenShellGatewayUserServiceIdentity } from "../../docker-driver-gateway-service"; +import { createActivePodmanWatcherController } from "./active-watcher"; + +function serviceReceipt( + pid: number, + processStartIdentity: string, + invocationId: string, +): TrustedActiveOpenShellGatewayUserServiceIdentity { + return { + execStart: + "{ path=/usr/bin/openshell-gateway ; argv[]=/usr/bin/openshell-gateway --port 8080 ; }", + execStartPath: "/usr/bin/openshell-gateway", + invocationId, + manager: "systemd", + pid, + processArgv: ["/usr/bin/openshell-gateway", "--port", "8080"], + processStartIdentity, + serviceName: "openshell-gateway", + unitPath: "/usr/lib/systemd/user/openshell-gateway.service", + } as unknown as TrustedActiveOpenShellGatewayUserServiceIdentity; +} + +function baseInput() { + return { + desiredEnv: { OPENSHELL_PODMAN_SOCKET: "/run/user/1000/podman/podman.sock" }, + driftGatewayBin: "/usr/bin/openshell-gateway", + driverLabel: "Podman", + gatewayBin: "/usr/bin/openshell-gateway", + gatewayName: "nemoclaw", + gatewayPort: 8080, + getRememberedGatewayPid: vi.fn(() => null), + getRuntimeDrift: vi.fn(() => null), + isGatewayHealthy: vi.fn(() => true), + isPidAlive: vi.fn(() => false), + launch: { + args: [], + command: "/usr/bin/openshell-gateway", + env: {}, + mode: "host" as const, + processGatewayBin: "/usr/bin/openshell-gateway", + }, + rememberGatewayPid: vi.fn(), + stateDir: "/tmp/nemoclaw-podman-test", + }; +} + +describe("active Podman watcher composition", () => { + it("uses the exact active package service and never invokes standalone lifecycle", () => { + let active = true; + let receipt = serviceReceipt(41, "linux-proc-start:100", "1".repeat(32)); + let listeners = [41]; + const stopService = vi.fn(() => { + active = false; + listeners = []; + }); + const resumeService = vi.fn(() => { + active = true; + receipt = serviceReceipt(42, "linux-proc-start:200", "2".repeat(32)); + listeners = [42]; + return receipt; + }); + const stopHostGateways = vi.fn(() => ({ + failed: [], + skippedDeadPids: [], + skippedNonMatchingPids: [], + stopped: [], + sudoRemediationPids: [], + })); + const spawnGateway = vi.fn(() => ({ pid: 91, unref: vi.fn() })); + const input = baseInput(); + const controller = createActivePodmanWatcherController({ + ...input, + deps: { + assertServiceInactive: vi.fn(() => { + if (active) throw new Error("service is active"); + }), + captureService: vi.fn(() => (active ? receipt : null)), + hasService: vi.fn(() => true), + openGatewayLog: vi.fn(() => 9), + resumeService, + spawnGateway, + stopHostGateways, + stopService, + watcher: { + captureListenerPids: () => listeners, + captureProcessStartIdentity: (pid) => + active && pid === receipt.pid ? receipt.processStartIdentity : null, + }, + }, + readiness: { now: () => 0, sleep: vi.fn() }, + }); + + const lease = controller.quiesceAndProve(); + lease.assertStillStopped(); + lease.resumeAndProve(); + + expect(stopService).toHaveBeenCalledOnce(); + expect(resumeService).toHaveBeenCalledOnce(); + expect(stopHostGateways).not.toHaveBeenCalled(); + expect(spawnGateway).not.toHaveBeenCalled(); + expect(input.getRuntimeDrift).toHaveBeenCalledWith( + 41, + input.desiredEnv, + input.driftGatewayBin, + 41, + ); + }); + + it("uses the standalone launch when an installed service is proven inactive", () => { + let listener = 51; + let rememberedPid = 51; + let startIdentity: string | null = "linux-proc-start:300"; + const stopHostGateways = vi.fn(() => { + listener = 0; + startIdentity = null; + return { + failed: [], + skippedDeadPids: [], + skippedNonMatchingPids: [], + stopped: [51], + sudoRemediationPids: [], + }; + }); + const spawnGateway = vi.fn(() => { + listener = 52; + startIdentity = "linux-proc-start:400"; + return { pid: 52, unref: vi.fn() }; + }); + const input = { + ...baseInput(), + getRememberedGatewayPid: vi.fn(() => rememberedPid), + rememberGatewayPid: vi.fn((pid: number) => { + rememberedPid = pid; + }), + }; + const captureService = vi.fn(() => null); + const controller = createActivePodmanWatcherController({ + ...input, + deps: { + captureService, + hasService: vi.fn(() => true), + openGatewayLog: vi.fn(() => 9), + spawnGateway, + stopHostGateways, + watcher: { + captureListenerPids: () => (listener > 0 ? [listener] : []), + captureProcessStartIdentity: (pid) => (pid === listener ? startIdentity : null), + }, + }, + readiness: { now: () => 0, sleep: vi.fn() }, + }); + + const lease = controller.quiesceAndProve(); + lease.resumeAndProve(); + + expect(stopHostGateways).toHaveBeenCalledOnce(); + expect(captureService).toHaveBeenCalledOnce(); + expect(spawnGateway).toHaveBeenCalledOnce(); + expect(input.rememberGatewayPid).toHaveBeenCalledWith(52); + expect(input.getRuntimeDrift).toHaveBeenCalledWith( + 51, + input.desiredEnv, + input.driftGatewayBin, + null, + ); + }); +}); diff --git a/src/lib/onboard/compute/podman/active-watcher.ts b/src/lib/onboard/compute/podman/active-watcher.ts new file mode 100644 index 0000000000..8332852f15 --- /dev/null +++ b/src/lib/onboard/compute/podman/active-watcher.ts @@ -0,0 +1,229 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import crypto from "node:crypto"; +import path from "node:path"; + +import { + type ManagedDriverGatewayLaunch, + openManagedDriverGatewayLog, + spawnDockerDriverGateway, +} from "../../docker-driver-gateway-launch"; +import { + assertTrustedOpenShellGatewayUserServiceInactive, + captureTrustedOpenShellGatewayUserServiceIfActive, + hasOpenShellGatewayUserService, + type OpenShellGatewayUserServiceOptions, + resumeTrustedOpenShellGatewayUserServiceAndProveActive, + stopTrustedOpenShellGatewayUserServiceAndProveInactive, + type TrustedActiveOpenShellGatewayUserServiceIdentity, +} from "../../docker-driver-gateway-service"; +import { stopHostGatewayProcesses } from "../../host-gateway-process"; +import type { PodmanOpenShellWatcherController } from "./sandbox-recreate"; +import { + createPodmanProductionWatcherController, + type PodmanProductionWatcherControllerOptions, +} from "./watcher-runtime"; + +type RuntimeDrift = { readonly reason: string } | null; +type ServiceReceipt = TrustedActiveOpenShellGatewayUserServiceIdentity; + +interface SpawnedGateway { + readonly pid?: number; + unref(): void; +} + +export interface ActivePodmanWatcherInput { + readonly desiredEnv: Readonly>; + readonly driftGatewayBin: string | null; + readonly driverLabel: string; + readonly gatewayBin: string; + readonly gatewayName: string; + readonly gatewayPort: number; + readonly getRememberedGatewayPid: () => number | null; + readonly getRuntimeDrift: ( + pid: number, + desiredEnv: Readonly>, + driftGatewayBin: string | null, + trustedServicePid: number | null, + ) => RuntimeDrift; + readonly isGatewayHealthy: () => boolean; + readonly isPidAlive: (pid: number) => boolean; + readonly launch: ManagedDriverGatewayLaunch; + readonly rememberGatewayPid: (pid: number) => void; + readonly serviceOptions?: OpenShellGatewayUserServiceOptions; + readonly stateDir: string; + readonly deps?: { + readonly assertServiceInactive?: typeof assertTrustedOpenShellGatewayUserServiceInactive; + readonly captureService?: typeof captureTrustedOpenShellGatewayUserServiceIfActive; + readonly hasService?: typeof hasOpenShellGatewayUserService; + readonly openGatewayLog?: typeof openManagedDriverGatewayLog; + readonly resumeService?: typeof resumeTrustedOpenShellGatewayUserServiceAndProveActive; + readonly spawnGateway?: (launch: ManagedDriverGatewayLaunch, logFd: number) => SpawnedGateway; + readonly stopHostGateways?: typeof stopHostGatewayProcesses; + readonly stopService?: typeof stopTrustedOpenShellGatewayUserServiceAndProveInactive; + readonly watcher?: PodmanProductionWatcherControllerOptions["deps"]; + }; + readonly readiness?: PodmanProductionWatcherControllerOptions["readiness"]; +} + +function identityHash(value: unknown): string { + return crypto.createHash("sha256").update(JSON.stringify(value)).digest("hex"); +} + +function sortedEnvironment( + environment: Readonly>, +): Readonly> { + return Object.fromEntries( + Object.entries(environment).sort(([left], [right]) => left.localeCompare(right)), + ); +} + +function serviceOwnerIdentity(receipt: ServiceReceipt): string { + return receipt.manager === "systemd" + ? identityHash({ + manager: receipt.manager, + serviceName: receipt.serviceName, + unitPath: receipt.unitPath, + }) + : identityHash({ + formulaName: receipt.formulaName, + formulaTap: receipt.formulaTap, + manager: receipt.manager, + serviceIdentity: receipt.serviceIdentity, + serviceName: receipt.serviceName, + }); +} + +function serviceLaunchIdentity(receipt: ServiceReceipt): string { + return receipt.manager === "systemd" + ? identityHash({ + execStart: receipt.execStart, + execStartPath: receipt.execStartPath, + manager: receipt.manager, + processArgv: receipt.processArgv, + serviceName: receipt.serviceName, + unitPath: receipt.unitPath, + }) + : identityHash({ + formulaName: receipt.formulaName, + formulaTap: receipt.formulaTap, + manager: receipt.manager, + processArgv: receipt.processArgv, + serviceIdentity: receipt.serviceIdentity, + serviceName: receipt.serviceName, + }); +} + +/** + * Compose the exact gateway owner used by native Podman cutover. + * + * Package-managed gateways are stopped and resumed only through their captured + * service authority. Standalone gateways use the immutable host launch slot. + * PID and process-generation evidence never enter the stable authority hashes. + */ +export function createActivePodmanWatcherController( + input: ActivePodmanWatcherInput, +): PodmanOpenShellWatcherController { + if (input.launch.mode !== "host") { + throw new Error("Podman watcher requires an exact host gateway launch."); + } + const hasService = input.deps?.hasService ?? hasOpenShellGatewayUserService; + const captureService = + input.deps?.captureService ?? captureTrustedOpenShellGatewayUserServiceIfActive; + const assertServiceInactive = + input.deps?.assertServiceInactive ?? assertTrustedOpenShellGatewayUserServiceInactive; + const stopService = + input.deps?.stopService ?? stopTrustedOpenShellGatewayUserServiceAndProveInactive; + const resumeService = + input.deps?.resumeService ?? resumeTrustedOpenShellGatewayUserServiceAndProveActive; + const stopHostGateways = input.deps?.stopHostGateways ?? stopHostGatewayProcesses; + const openGatewayLog = input.deps?.openGatewayLog ?? openManagedDriverGatewayLog; + const spawnGateway = input.deps?.spawnGateway ?? spawnDockerDriverGateway; + const serviceOptions = input.serviceOptions ?? {}; + let trustedServicePid: number | null = null; + + const rememberServicePid = (receipt: ServiceReceipt): ServiceReceipt => { + trustedServicePid = receipt.pid; + return receipt; + }; + const service = { + captureActive: () => { + if (!hasService(serviceOptions)) return null; + const receipt = captureService(serviceOptions); + return receipt ? rememberServicePid(receipt) : null; + }, + assertInactive: (receipt: ServiceReceipt) => assertServiceInactive(receipt, serviceOptions), + stopAndProveInactive: (receipt: ServiceReceipt) => stopService(receipt, serviceOptions), + resumeAndProve: (receipt: ServiceReceipt) => + rememberServicePid(resumeService(receipt, serviceOptions)), + describe: (receipt: ServiceReceipt) => ({ + launchIdentity: serviceLaunchIdentity(receipt), + ownerIdentity: serviceOwnerIdentity(receipt), + pid: receipt.pid, + processStartIdentity: receipt.processStartIdentity, + }), + }; + const launchIdentity = identityHash({ + args: input.launch.args, + argv0: input.launch.argv0 ?? null, + command: input.launch.command, + desiredEnv: sortedEnvironment(input.desiredEnv), + }); + const ownerIdentity = identityHash({ + gatewayName: input.gatewayName, + gatewayPort: input.gatewayPort, + stateDir: input.stateDir, + }); + + return createPodmanProductionWatcherController({ + gatewayName: input.gatewayName, + gatewayPort: input.gatewayPort, + getRuntimeDrift: (pid) => + input.getRuntimeDrift(pid, input.desiredEnv, input.driftGatewayBin, trustedServicePid), + isGatewayHealthy: input.isGatewayHealthy, + service, + standalone: { + launchIdentity, + ownerIdentity, + readOwnedPid: input.getRememberedGatewayPid, + stop: (pid) => { + const result = stopHostGateways( + {}, + { + clearRuntimeFiles: false, + gatewayBin: input.gatewayBin, + openShellGatewayName: input.gatewayName, + openShellGatewayPort: input.gatewayPort, + pids: [pid], + stateDir: input.stateDir, + usePidFile: false, + usePgrepFallback: false, + }, + ); + if ( + result.failed.length > 0 || + result.skippedNonMatchingPids.length > 0 || + (result.stopped.length === 0 && input.isPidAlive(pid)) + ) { + throw new Error("Stopping the exact standalone Podman gateway watcher failed."); + } + }, + resume: () => { + const logFd = openGatewayLog(path.join(input.stateDir, "openshell-gateway.log"), { + driverLabel: input.driverLabel, + }); + const child = spawnGateway(input.launch, logFd); + child.unref(); + const pid = child.pid ?? 0; + if (pid <= 0) { + throw new Error("Resuming the standalone Podman gateway watcher returned no PID."); + } + input.rememberGatewayPid(pid); + return pid; + }, + }, + deps: input.deps?.watcher, + readiness: input.readiness, + }); +} diff --git a/src/lib/onboard/compute/podman/gateway-env.test.ts b/src/lib/onboard/compute/podman/gateway-env.test.ts new file mode 100644 index 0000000000..e9f9485c3b --- /dev/null +++ b/src/lib/onboard/compute/podman/gateway-env.test.ts @@ -0,0 +1,153 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { + buildPersistedPodmanDriverGatewayEnv, + buildPodmanDriverGatewayEnv, + PODMAN_DRIVER_GATEWAY_RUNTIME_ENV_KEYS, +} from "./gateway-env"; + +const dirs: string[] = []; + +afterEach(() => { + for (const dir of dirs.splice(0)) fs.rmSync(dir, { recursive: true, force: true }); +}); + +describe("Podman-driver gateway environment", () => { + it("renders an authenticated native Podman config without Docker authority", () => { + const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-podman-gateway-")); + dirs.push(stateDir); + const env = buildPodmanDriverGatewayEnv({ + gatewayPort: 8080, + stateDir, + podmanSocketPath: "/run/user/1000/podman/podman.sock", + supervisorImage: "ghcr.io/nvidia/openshell/supervisor@sha256:abc", + }); + const toml = fs.readFileSync(env.OPENSHELL_GATEWAY_CONFIG, "utf-8"); + + expect(env).toMatchObject({ + OPENSHELL_DRIVERS: "podman", + OPENSHELL_BIND_ADDRESS: "0.0.0.0", + OPENSHELL_PODMAN_SOCKET: "/run/user/1000/podman/podman.sock", + }); + expect(env).not.toHaveProperty("DOCKER_HOST"); + expect(env.NEMOCLAW_MANAGED_GATEWAY_CONFIG_SHA256).toMatch(/^[0-9a-f]{64}$/); + expect(toml).toContain('compute_drivers = ["podman"]'); + expect(toml).toContain("[openshell.drivers.podman]"); + expect(toml).toContain('socket_path = "/run/user/1000/podman/podman.sock"'); + expect(toml).toContain('network_name = "openshell"'); + expect(toml).toContain('supervisor_image = "ghcr.io/nvidia/openshell/supervisor@sha256:abc"'); + expect(toml).not.toContain("[openshell.drivers.docker]"); + expect(PODMAN_DRIVER_GATEWAY_RUNTIME_ENV_KEYS).not.toContain("DOCKER_HOST"); + expect(PODMAN_DRIVER_GATEWAY_RUNTIME_ENV_KEYS).toContain( + "NEMOCLAW_MANAGED_GATEWAY_CONFIG_SHA256", + ); + }); + + it("changes runtime identity when the configured Podman network changes", () => { + const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-podman-gateway-network-")); + dirs.push(stateDir); + const common = { + gatewayPort: 8080, + podmanSocketPath: "/run/user/1000/podman/podman.sock", + supervisorImage: "ghcr.io/nvidia/openshell/supervisor@sha256:abc", + }; + + const first = buildPodmanDriverGatewayEnv({ + ...common, + stateDir, + podmanNetworkName: "openshell-a", + }); + const firstFingerprint = first.NEMOCLAW_MANAGED_GATEWAY_CONFIG_SHA256; + const second = buildPodmanDriverGatewayEnv({ + ...common, + stateDir, + podmanNetworkName: "openshell-b", + }); + + expect(firstFingerprint).not.toBe(second.NEMOCLAW_MANAGED_GATEWAY_CONFIG_SHA256); + }); + + it("reconstructs persisted runtime identity without rewriting gateway config", () => { + const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-podman-recovery-")); + dirs.push(stateDir); + const configPath = path.join(stateDir, "openshell-gateway.toml"); + const initial = buildPodmanDriverGatewayEnv({ + gatewayPort: 8443, + podmanSocketPath: "/run/user/1001/podman/podman.sock", + stateDir, + supervisorImage: "ghcr.io/nvidia/openshell/supervisor@sha256:def", + }); + const initialConfig = fs.readFileSync(configPath); + const initialConfigStat = fs.statSync(configPath); + const env = buildPersistedPodmanDriverGatewayEnv({ + configSha256: initial.NEMOCLAW_MANAGED_GATEWAY_CONFIG_SHA256, + gatewayPort: 8443, + podmanSocketPath: "/run/user/1001/podman/podman.sock", + stateDir, + supervisorImage: "ghcr.io/nvidia/openshell/supervisor@sha256:def", + }); + + expect(env).toMatchObject({ + NEMOCLAW_MANAGED_GATEWAY_CONFIG_SHA256: initial.NEMOCLAW_MANAGED_GATEWAY_CONFIG_SHA256, + OPENSHELL_GATEWAY_CONFIG: configPath, + OPENSHELL_PODMAN_SOCKET: "/run/user/1001/podman/podman.sock", + OPENSHELL_SERVER_PORT: "8443", + OPENSHELL_SUPERVISOR_IMAGE: "ghcr.io/nvidia/openshell/supervisor@sha256:def", + }); + expect(fs.readFileSync(configPath)).toEqual(initialConfig); + expect(fs.statSync(configPath).mtimeMs).toBe(initialConfigStat.mtimeMs); + }); + + it("rejects persisted config whose content no longer matches the protected binding", () => { + const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-podman-tamper-")); + dirs.push(stateDir); + const initial = buildPodmanDriverGatewayEnv({ + gatewayPort: 8443, + podmanSocketPath: "/run/user/1001/podman/podman.sock", + stateDir, + supervisorImage: "ghcr.io/nvidia/openshell/supervisor@sha256:def", + }); + fs.appendFileSync(path.join(stateDir, "openshell-gateway.toml"), "\n# tampered\n"); + + expect(() => + buildPersistedPodmanDriverGatewayEnv({ + configSha256: initial.NEMOCLAW_MANAGED_GATEWAY_CONFIG_SHA256, + gatewayPort: 8443, + podmanSocketPath: "/run/user/1001/podman/podman.sock", + stateDir, + supervisorImage: "ghcr.io/nvidia/openshell/supervisor@sha256:def", + }), + ).toThrow("does not match its protected fingerprint"); + }); + + it("rejects a symlinked persisted gateway config before reading it", () => { + const sourceDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-podman-source-")); + const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-podman-symlink-")); + dirs.push(sourceDir, stateDir); + const initial = buildPodmanDriverGatewayEnv({ + gatewayPort: 8443, + podmanSocketPath: "/run/user/1001/podman/podman.sock", + stateDir: sourceDir, + supervisorImage: "ghcr.io/nvidia/openshell/supervisor@sha256:def", + }); + fs.symlinkSync( + path.join(sourceDir, "openshell-gateway.toml"), + path.join(stateDir, "openshell-gateway.toml"), + ); + + expect(() => + buildPersistedPodmanDriverGatewayEnv({ + configSha256: initial.NEMOCLAW_MANAGED_GATEWAY_CONFIG_SHA256, + gatewayPort: 8443, + podmanSocketPath: "/run/user/1001/podman/podman.sock", + stateDir, + supervisorImage: "ghcr.io/nvidia/openshell/supervisor@sha256:def", + }), + ).toThrow("unavailable or unsafe"); + }); +}); diff --git a/src/lib/onboard/compute/podman/gateway-env.ts b/src/lib/onboard/compute/podman/gateway-env.ts new file mode 100644 index 0000000000..54f403ac6d --- /dev/null +++ b/src/lib/onboard/compute/podman/gateway-env.ts @@ -0,0 +1,161 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import crypto from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; +import { WILDCARD_GATEWAY_BIND_ADDRESS } from "../../../core/gateway-address"; +import { + type ManagedGatewayDriverConfig, + writeManagedDriverGatewayConfig, +} from "../../docker-driver-gateway-config"; +import { assertManagedDriverGatewayAuthConfigSafe } from "../../docker-driver-gateway-env"; +import { + buildDockerDriverGatewayLocalTlsEnv, + getDockerDriverGatewayLocalTlsDir, +} from "../../docker-driver-gateway-local-tls"; + +export const PODMAN_DRIVER_GATEWAY_RUNTIME_ENV_KEYS = [ + "OPENSHELL_DRIVERS", + "OPENSHELL_BIND_ADDRESS", + "OPENSHELL_SERVER_PORT", + "OPENSHELL_DISABLE_TLS", + "OPENSHELL_DISABLE_GATEWAY_AUTH", + "OPENSHELL_LOCAL_TLS_DIR", + "OPENSHELL_DB_URL", + "OPENSHELL_SSH_GATEWAY_HOST", + "OPENSHELL_SSH_GATEWAY_PORT", + "OPENSHELL_PODMAN_SOCKET", + "OPENSHELL_SUPERVISOR_IMAGE", + "OPENSHELL_GATEWAY_CONFIG", + "NEMOCLAW_MANAGED_GATEWAY_CONFIG_SHA256", +] as const; + +export interface BuildPodmanDriverGatewayEnvOptions { + readonly gatewayPort: number; + readonly stateDir: string; + readonly podmanSocketPath: string; + readonly podmanNetworkName?: string; + readonly supervisorImage: string; +} + +export interface BuildPersistedPodmanDriverGatewayEnvOptions { + readonly configSha256: string; + readonly gatewayPort: number; + readonly podmanSocketPath: string; + readonly stateDir: string; + readonly supervisorImage: string; +} + +function podmanDriverGatewayBaseEnv({ + gatewayPort, + stateDir, + podmanSocketPath, + supervisorImage, +}: Omit): Record { + if (!path.isAbsolute(podmanSocketPath)) { + throw new Error("OpenShell Podman-driver gateway requires an absolute Podman socket path"); + } + return { + OPENSHELL_DRIVERS: "podman", + OPENSHELL_BIND_ADDRESS: WILDCARD_GATEWAY_BIND_ADDRESS, + OPENSHELL_SERVER_PORT: String(gatewayPort), + OPENSHELL_SSH_GATEWAY_HOST: "host.openshell.internal", + OPENSHELL_SSH_GATEWAY_PORT: String(gatewayPort), + ...buildDockerDriverGatewayLocalTlsEnv(stateDir), + OPENSHELL_DB_URL: `sqlite:${path.join(stateDir, "openshell.db")}`, + OPENSHELL_PODMAN_SOCKET: podmanSocketPath, + OPENSHELL_SUPERVISOR_IMAGE: supervisorImage, + }; +} + +function podmanDriverConfig( + env: Record, + stateDir: string, + networkName: string, +): ManagedGatewayDriverConfig { + const tlsDir = getDockerDriverGatewayLocalTlsDir(stateDir); + return { + driverName: "podman", + persistedRuntimeKeys: ["socket_path", "network_name", "supervisor_image"], + entries: [ + ["socket_path", env.OPENSHELL_PODMAN_SOCKET], + ["image_pull_policy", "missing"], + ["network_name", networkName], + ["stop_timeout_secs", 10], + ["supervisor_image", env.OPENSHELL_SUPERVISOR_IMAGE], + ["guest_tls_ca", path.join(tlsDir, "ca.crt")], + ["guest_tls_cert", path.join(tlsDir, "client", "tls.crt")], + ["guest_tls_key", path.join(tlsDir, "client", "tls.key")], + ], + }; +} + +export function buildPodmanDriverGatewayEnv({ + gatewayPort, + stateDir, + podmanSocketPath, + podmanNetworkName = "openshell", + supervisorImage, +}: BuildPodmanDriverGatewayEnvOptions): Record { + const env = podmanDriverGatewayBaseEnv({ + gatewayPort, + stateDir, + podmanSocketPath, + supervisorImage, + }); + env.OPENSHELL_GATEWAY_CONFIG = writeManagedDriverGatewayConfig( + stateDir, + env, + podmanDriverConfig(env, stateDir, podmanNetworkName), + ); + env.NEMOCLAW_MANAGED_GATEWAY_CONFIG_SHA256 = crypto + .createHash("sha256") + .update(fs.readFileSync(env.OPENSHELL_GATEWAY_CONFIG)) + .digest("hex"); + assertManagedDriverGatewayAuthConfigSafe(env, { + allowWildcardBind: true, + driverName: "Podman", + }); + return env; +} + +/** + * Reconstruct the exact non-secret Podman process environment from a protected + * managed-runtime binding without rewriting gateway config during recovery or + * snapshot preflight. + */ +export function buildPersistedPodmanDriverGatewayEnv( + options: BuildPersistedPodmanDriverGatewayEnvOptions, +): Record { + if (!/^[0-9a-f]{64}$/u.test(options.configSha256)) { + throw new Error("Managed Podman gateway config fingerprint is invalid"); + } + const env = podmanDriverGatewayBaseEnv(options); + const configPath = path.join(options.stateDir, "openshell-gateway.toml"); + let config: Buffer; + try { + const stat = fs.lstatSync(configPath); + if (!stat.isFile() || stat.isSymbolicLink()) { + throw new Error("not a regular file"); + } + config = fs.readFileSync(configPath); + } catch (error) { + throw new Error( + `Managed Podman gateway config is unavailable or unsafe: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + } + const actualConfigSha256 = crypto.createHash("sha256").update(config).digest("hex"); + if (actualConfigSha256 !== options.configSha256) { + throw new Error("Managed Podman gateway config does not match its protected fingerprint"); + } + env.OPENSHELL_GATEWAY_CONFIG = configPath; + env.NEMOCLAW_MANAGED_GATEWAY_CONFIG_SHA256 = options.configSha256; + assertManagedDriverGatewayAuthConfigSafe(env, { + allowWildcardBind: true, + driverName: "Podman", + }); + return env; +} diff --git a/src/lib/onboard/compute/podman/gateway-reachability.test.ts b/src/lib/onboard/compute/podman/gateway-reachability.test.ts new file mode 100644 index 0000000000..9f24e8cae3 --- /dev/null +++ b/src/lib/onboard/compute/podman/gateway-reachability.test.ts @@ -0,0 +1,161 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; +import { + buildPodmanGatewayProbeArgs, + verifyPodmanSandboxGatewayReachableOrExit, +} from "./gateway-reachability"; + +const SOCKET_AUTHORITY = { + directoryChain: [ + { + device: "8", + inode: "7000", + mode: "448", + ownerUid: "1000", + path: "/run/user/1000/podman", + }, + ], + device: "8", + inode: "9001", + ownerUid: "1000", + socketPath: "/run/user/1000/podman/podman.sock", +} as const; + +const REQUIRED_OPTIONS = { + assertSocketAuthority: () => {}, + port: 8080, + redact: (value: string) => value, + socketAuthority: SOCKET_AUTHORITY, +} as const; + +describe("Podman driver gateway reachability", () => { + it("probes the selected native Podman socket and Podman host route", () => { + expect( + buildPodmanGatewayProbeArgs({ + networkName: "openshell", + podmanSocketPath: "/run/user/1000/podman/podman.sock", + port: 8080, + probeImage: "probe@sha256:test", + probeName: "probe-name", + timeoutSeconds: 5, + }), + ).toEqual([ + "--url", + "unix:///run/user/1000/podman/podman.sock", + "run", + "--rm", + "--name", + "probe-name", + "--pull=missing", + "--network", + "openshell", + "probe@sha256:test", + "sh", + "-c", + "nc -zw5 host.containers.internal 8080", + ]); + }); + + it("accepts a successful native Podman probe without Docker compatibility", async () => { + const spawnSyncImpl = vi.fn((_command: string, _args: string[]) => ({ status: 0 })); + await verifyPodmanSandboxGatewayReachableOrExit(false, { + ...REQUIRED_OPTIONS, + podmanSocketPath: "/run/user/1000/podman/podman.sock", + probeName: "probe-name", + spawnSyncImpl, + }); + expect(spawnSyncImpl).toHaveBeenCalledOnce(); + expect(spawnSyncImpl.mock.calls[0]?.[0]).toBe("podman"); + expect(spawnSyncImpl.mock.calls[0]?.[1]).not.toContain("docker"); + expect(spawnSyncImpl.mock.calls[0]?.[1]).toContainEqual( + expect.stringMatching(/^docker\.io\/library\/busybox@sha256:/), + ); + }); + + it("does not run or clean up a probe after socket authority replacement", async () => { + const spawnSyncImpl = vi.fn(); + await expect( + verifyPodmanSandboxGatewayReachableOrExit(false, { + ...REQUIRED_OPTIONS, + assertSocketAuthority: () => { + throw new Error("Podman socket authority changed after it was qualified."); + }, + probeName: "probe-name", + spawnSyncImpl, + }), + ).rejects.toThrow("socket authority changed"); + expect(spawnSyncImpl).not.toHaveBeenCalled(); + }); + + it("does not clean up through a replaced socket after a failed probe", async () => { + const assertSocketAuthority = vi + .fn() + .mockImplementationOnce(() => {}) + .mockImplementationOnce(() => { + throw new Error("Podman socket authority changed after it was qualified."); + }); + const spawnSyncImpl = vi.fn().mockReturnValueOnce({ status: 1, stderr: "nc: timed out" }); + await expect( + verifyPodmanSandboxGatewayReachableOrExit(false, { + ...REQUIRED_OPTIONS, + assertSocketAuthority, + probeName: "probe-name", + spawnSyncImpl, + }), + ).rejects.toThrow("socket authority changed"); + expect(spawnSyncImpl).toHaveBeenCalledOnce(); + }); + + it("removes only its named probe and fails a proved TCP-path error", async () => { + const spawnSyncImpl = vi + .fn() + .mockReturnValueOnce({ status: 1, stderr: "nc: timed out" }) + .mockReturnValueOnce({ status: 0 }); + await expect( + verifyPodmanSandboxGatewayReachableOrExit(false, { + ...REQUIRED_OPTIONS, + podmanSocketPath: "/run/user/1000/podman/podman.sock", + probeName: "probe-name", + spawnSyncImpl, + }), + ).rejects.toThrow("Podman sandbox containers cannot reach"); + expect(spawnSyncImpl.mock.calls[1]?.[1]).toEqual([ + "--url", + "unix:///run/user/1000/podman/podman.sock", + "rm", + "--force", + "probe-name", + ]); + }); + + it("warns only when the pinned probe image cannot be established", async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + await verifyPodmanSandboxGatewayReachableOrExit(false, { + ...REQUIRED_OPTIONS, + podmanSocketPath: "/run/user/1000/podman/podman.sock", + probeName: "probe-name", + spawnSyncImpl: vi + .fn() + .mockReturnValueOnce({ status: 125, stderr: "pulling image: registry unavailable" }) + .mockReturnValueOnce({ status: 0 }), + }); + expect(warn).toHaveBeenCalledWith(expect.stringContaining("probe helper was unavailable")); + warn.mockRestore(); + }); + + it("fails closed when the selected Podman network is missing", async () => { + await expect( + verifyPodmanSandboxGatewayReachableOrExit(false, { + ...REQUIRED_OPTIONS, + podmanSocketPath: "/run/user/1000/podman/podman.sock", + probeName: "probe-name", + spawnSyncImpl: vi + .fn() + .mockReturnValueOnce({ status: 125, stderr: "network openshell not found" }) + .mockReturnValueOnce({ status: 0 }), + }), + ).rejects.toThrow("Podman sandbox containers cannot reach"); + }); +}); diff --git a/src/lib/onboard/compute/podman/gateway-reachability.ts b/src/lib/onboard/compute/podman/gateway-reachability.ts new file mode 100644 index 0000000000..bd1ffcb3fd --- /dev/null +++ b/src/lib/onboard/compute/podman/gateway-reachability.ts @@ -0,0 +1,172 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { type SpawnSyncOptions, spawnSync } from "node:child_process"; +import path from "node:path"; +import { assertPodmanSocketAuthority, type PodmanSocketAuthority } from "./socket-authority"; + +const DEFAULT_PROBE_IMAGE = + "docker.io/library/busybox@sha256:73aaf090f3d85aa34ee199857f03fa3a95c8ede2ffd4cc2cdb5b94e566b11662"; +const DEFAULT_NETWORK_NAME = "openshell"; +const PODMAN_HOST_INTERNAL_NAME = "host.containers.internal"; +const DEFAULT_TIMEOUT_SECONDS = 5; +const PROBE_OVERHEAD_MS = 10_000; + +interface PodmanProbeResult { + readonly error?: Error; + readonly status: number | null; + readonly stderr?: Buffer | string | null; + readonly stdout?: Buffer | string | null; +} + +type SpawnSyncLike = ( + command: string, + args: string[], + options: SpawnSyncOptions, +) => PodmanProbeResult; + +export interface PodmanGatewayReachabilityOptions { + readonly assertSocketAuthority?: (expected: PodmanSocketAuthority) => void; + readonly exitProcess?: (code: number) => never; + readonly networkName?: string; + readonly podmanBin?: string; + readonly socketAuthority: PodmanSocketAuthority; + readonly podmanSocketPath?: string; + readonly port: number; + readonly probeImage?: string; + readonly probeName?: string; + readonly redact: (value: string) => string; + readonly skip?: boolean; + readonly spawnSyncImpl?: SpawnSyncLike; + readonly timeoutSeconds?: number; +} + +function output(value: Buffer | string | null | undefined): string { + if (typeof value === "string") return value; + return Buffer.isBuffer(value) ? value.toString("utf-8") : ""; +} + +function boundedDetail( + result: PodmanProbeResult, + redact: PodmanGatewayReachabilityOptions["redact"], +): string { + return redact( + [result.error?.message, output(result.stderr), output(result.stdout)] + .filter((value): value is string => Boolean(value?.trim())) + .join(" | ") + .replace(/\s+/g, " ") + .trim() + .slice(-500), + ); +} + +function podmanSocketUrl(socketPath: string): string { + const normalized = socketPath.trim(); + if (!path.isAbsolute(normalized) || /[\0\r\n]/.test(normalized)) { + throw new Error("Podman gateway reachability requires a safe absolute Podman socket path"); + } + return `unix://${normalized}`; +} + +export function buildPodmanGatewayProbeArgs(options: { + readonly networkName: string; + readonly podmanSocketPath: string; + readonly port: number; + readonly probeImage: string; + readonly probeName: string; + readonly timeoutSeconds: number; +}): string[] { + return [ + "--url", + podmanSocketUrl(options.podmanSocketPath), + "run", + "--rm", + "--name", + options.probeName, + "--pull=missing", + "--network", + options.networkName, + options.probeImage, + "sh", + "-c", + `nc -zw${options.timeoutSeconds} ${PODMAN_HOST_INTERNAL_NAME} ${options.port}`, + ]; +} + +function helperUnavailable( + result: PodmanProbeResult, + redact: PodmanGatewayReachabilityOptions["redact"], +): boolean { + const detail = boundedDetail(result, redact); + return ( + result.status === 125 && + /image.*(pull|not known|not found)|manifest unknown|pull access denied|initializing source docker|pinging container registry|registry.*(?:timeout|unavailable)|tls handshake timeout/i.test( + detail, + ) + ); +} + +export async function verifyPodmanSandboxGatewayReachableOrExit( + exitOnFailure: boolean, + options: PodmanGatewayReachabilityOptions, +): Promise { + if (options.skip) { + console.log(" Skipping Podman sandbox-to-gateway reachability probe."); + return; + } + const podmanSocketPath = + options.podmanSocketPath ?? + process.env.OPENSHELL_PODMAN_SOCKET?.trim() ?? + options.socketAuthority.socketPath; + if (options.socketAuthority.socketPath !== podmanSocketPath) { + throw new Error( + "Podman gateway reachability socket authority does not match the requested socket.", + ); + } + const proveSocketAuthority = options.assertSocketAuthority ?? assertPodmanSocketAuthority; + const port = options.port; + const timeoutSeconds = Math.max(1, options.timeoutSeconds ?? DEFAULT_TIMEOUT_SECONDS); + const probeName = + options.probeName ?? `nemoclaw-gateway-probe-${process.pid}-${Date.now().toString(36)}`; + const podmanBin = options.podmanBin ?? process.env.NEMOCLAW_PODMAN_BIN?.trim() ?? "podman"; + const spawn = options.spawnSyncImpl ?? spawnSync; + const socketUrl = podmanSocketUrl(podmanSocketPath); + proveSocketAuthority(options.socketAuthority); + const result = spawn( + podmanBin, + buildPodmanGatewayProbeArgs({ + networkName: options.networkName ?? DEFAULT_NETWORK_NAME, + podmanSocketPath, + port, + probeImage: options.probeImage ?? DEFAULT_PROBE_IMAGE, + probeName, + timeoutSeconds, + }), + { + encoding: "utf-8", + stdio: ["ignore", "pipe", "pipe"], + timeout: timeoutSeconds * 1000 + PROBE_OVERHEAD_MS, + }, + ); + proveSocketAuthority(options.socketAuthority); + if (result.status === 0) return; + + proveSocketAuthority(options.socketAuthority); + spawn(podmanBin, ["--url", socketUrl, "rm", "--force", probeName], { + encoding: "utf-8", + stdio: "ignore", + timeout: PROBE_OVERHEAD_MS, + }); + const detail = boundedDetail(result, options.redact); + if (helperUnavailable(result, options.redact)) { + console.warn( + ` Could not verify the Podman sandbox route to ${PODMAN_HOST_INTERNAL_NAME}:${port}; continuing because the probe helper was unavailable${detail ? ` (${detail})` : ""}.`, + ); + return; + } + + const message = `Podman sandbox containers cannot reach the OpenShell gateway at ${PODMAN_HOST_INTERNAL_NAME}:${port}${detail ? ` (${detail})` : ""}.`; + console.error(` ${message}`); + if (exitOnFailure) (options.exitProcess ?? process.exit)(1); + throw new Error(message); +} diff --git a/src/lib/onboard/compute/podman/gpu-attachment.test.ts b/src/lib/onboard/compute/podman/gpu-attachment.test.ts new file mode 100644 index 0000000000..a8dafdd362 --- /dev/null +++ b/src/lib/onboard/compute/podman/gpu-attachment.test.ts @@ -0,0 +1,41 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; +import { assertPodmanGpuAttachmentQualified, resolvePodmanGpuAttachment } from "./gpu-attachment"; + +describe("Podman GPU attachment", () => { + it.each([ + [null, "nvidia.com/gpu=all"], + ["all", "nvidia.com/gpu=all"], + ["0", "nvidia.com/gpu=0"], + ["1:0", "nvidia.com/gpu=1:0"], + ["GPU-deadbeef", "nvidia.com/gpu=GPU-deadbeef"], + ["nvidia.com/gpu=MIG-deadbeef", "nvidia.com/gpu=MIG-deadbeef"], + [ + "MIG-GPU-aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/1/0", + "nvidia.com/gpu=MIG-GPU-aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/1/0", + ], + ])("normalizes %s to a qualified CDI identity", (requested, expected) => { + expect(resolvePodmanGpuAttachment(true, requested)).toEqual({ + kind: "cdi", + device: expected, + }); + }); + + it("returns no attachment for a CPU sandbox", () => { + expect(resolvePodmanGpuAttachment(false, "nvidia.com/gpu=all")).toBeNull(); + }); + + it("rejects unsafe names and CDI devices absent from exact-socket qualification", () => { + expect(() => resolvePodmanGpuAttachment(true, "../../dev/nvidia0")).toThrow( + "safe NVIDIA CDI name", + ); + expect(() => + assertPodmanGpuAttachmentQualified( + ["nvidia.com/gpu=0"], + resolvePodmanGpuAttachment(true, "all")!, + ), + ).toThrow("does not advertise"); + }); +}); diff --git a/src/lib/onboard/compute/podman/gpu-attachment.ts b/src/lib/onboard/compute/podman/gpu-attachment.ts new file mode 100644 index 0000000000..0429c62523 --- /dev/null +++ b/src/lib/onboard/compute/podman/gpu-attachment.ts @@ -0,0 +1,53 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +const NVIDIA_CDI_PREFIX = "nvidia.com/gpu="; +const CDI_DEVICE_NAME = /^[A-Za-z0-9][A-Za-z0-9_.:-]*$/u; +const LEGACY_MIG_DEVICE_NAME = /^MIG-GPU-[A-Za-z0-9-]+\/[0-9]+\/[0-9]+$/u; + +export interface PodmanGpuAttachment { + readonly kind: "cdi"; + readonly device: string; +} + +/** + * Return one canonical NVIDIA CDI device identity. The accepted selector + * surface covers the identities emitted by nvidia-ctk: `all`, numeric GPU and + * MIG indices, GPU/MIG UUIDs, and the legacy MIG-GPU-...// spelling. + * Availability is proved separately against the qualified Podman runtime. + */ +export function normalizeNvidiaCdiDevice(requestedDevice: string): string { + const requested = requestedDevice.trim(); + const device = requested.startsWith(NVIDIA_CDI_PREFIX) + ? requested + : `${NVIDIA_CDI_PREFIX}${requested}`; + const name = device.slice(NVIDIA_CDI_PREFIX.length); + if (!CDI_DEVICE_NAME.test(name) && !LEGACY_MIG_DEVICE_NAME.test(name)) { + throw new Error( + "Podman GPU device must be a safe NVIDIA CDI name such as 'all', '0', '1:0', 'GPU-...', or 'MIG-...'.", + ); + } + return device; +} + +export function resolvePodmanGpuAttachment( + enabled: boolean, + requestedDevice: string | null | undefined, +): PodmanGpuAttachment | null { + if (!enabled) return null; + return { + kind: "cdi", + device: normalizeNvidiaCdiDevice(requestedDevice?.trim() || "all"), + }; +} + +export function assertPodmanGpuAttachmentQualified( + availableDevices: readonly string[], + attachment: PodmanGpuAttachment, +): void { + if (!availableDevices.includes(attachment.device)) { + throw new Error( + `Rootless Podman does not advertise the requested CDI device '${attachment.device}'. Refresh the NVIDIA CDI specification and retry.`, + ); + } +} diff --git a/src/lib/onboard/compute/podman/sandbox-create-authority.ts b/src/lib/onboard/compute/podman/sandbox-create-authority.ts new file mode 100644 index 0000000000..464140a18a --- /dev/null +++ b/src/lib/onboard/compute/podman/sandbox-create-authority.ts @@ -0,0 +1,87 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { ManagedDriverGatewayRuntimeIdentity } from "../../docker-driver-gateway-launch"; +import type { OpenShellGatewayUserServiceOptions } from "../../docker-driver-gateway-service"; +import { createActivePodmanWatcherController } from "./active-watcher"; +import type { PodmanOpenShellWatcherController } from "./sandbox-recreate"; +import { + assertPodmanSocketAuthority, + capturePodmanSocketAuthority, + type PodmanSocketAuthority, +} from "./socket-authority"; + +export interface PodmanSandboxCreateRuntimeAuthority { + readonly socketAuthority: PodmanSocketAuthority; + readonly socketPath: string; + readonly watcherController: PodmanOpenShellWatcherController; +} + +export interface PodmanSandboxCreateRuntimeAuthorityInput { + readonly driverLabel: string; + readonly gatewayBin: string; + readonly gatewayName: string; + readonly gatewayPort: number; + readonly getRememberedGatewayPid: () => number | null; + readonly getRuntimeDrift: ( + pid: number, + desiredEnv: Readonly>, + driftGatewayBin: string | null, + trustedServicePid: number | null, + ) => { readonly reason: string } | null; + readonly isGatewayHealthy: () => boolean; + readonly isPidAlive: (pid: number) => boolean; + readonly rememberGatewayPid: (pid: number) => void; + readonly runtimeIdentity: ManagedDriverGatewayRuntimeIdentity; + readonly serviceOptions?: OpenShellGatewayUserServiceOptions; + readonly socketAuthority?: PodmanSocketAuthority; + readonly socketPath: string; + readonly stateDir: string; +} + +export interface PodmanSandboxCreateRuntimeAuthorityDependencies { + readonly assertSocketAuthority?: typeof assertPodmanSocketAuthority; + readonly captureSocketAuthority?: typeof capturePodmanSocketAuthority; + readonly createWatcherController?: typeof createActivePodmanWatcherController; +} + +export function createPodmanSandboxCreateRuntimeAuthority( + input: PodmanSandboxCreateRuntimeAuthorityInput, + dependencies: PodmanSandboxCreateRuntimeAuthorityDependencies = {}, +): PodmanSandboxCreateRuntimeAuthority { + const launch = input.runtimeIdentity.launch; + if (!launch || launch.mode !== "host" || !input.socketPath.trim()) { + throw new Error( + "Podman sandbox-create authority requires the exact host launch and socket identity.", + ); + } + const socketAuthority = + input.socketAuthority ?? + (dependencies.captureSocketAuthority ?? capturePodmanSocketAuthority)(input.socketPath); + if (socketAuthority.socketPath !== input.socketPath) { + throw new Error("Qualified Podman socket does not match sandbox-create runtime authority."); + } + (dependencies.assertSocketAuthority ?? assertPodmanSocketAuthority)(socketAuthority); + return { + socketAuthority, + socketPath: input.socketPath, + watcherController: ( + dependencies.createWatcherController ?? createActivePodmanWatcherController + )({ + desiredEnv: input.runtimeIdentity.desiredEnv, + driftGatewayBin: input.runtimeIdentity.driftGatewayBin, + driverLabel: input.driverLabel, + gatewayBin: input.gatewayBin, + gatewayName: input.gatewayName, + gatewayPort: input.gatewayPort, + getRememberedGatewayPid: input.getRememberedGatewayPid, + getRuntimeDrift: input.getRuntimeDrift, + isGatewayHealthy: input.isGatewayHealthy, + isPidAlive: input.isPidAlive, + launch, + rememberGatewayPid: input.rememberGatewayPid, + serviceOptions: input.serviceOptions, + stateDir: input.stateDir, + }), + }; +} diff --git a/src/lib/onboard/compute/podman/sandbox-recreate-spec.ts b/src/lib/onboard/compute/podman/sandbox-recreate-spec.ts new file mode 100644 index 0000000000..35d7c1781f --- /dev/null +++ b/src/lib/onboard/compute/podman/sandbox-recreate-spec.ts @@ -0,0 +1,946 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { openshellSandboxCommandEnvValue } from "../../docker-startup-command-env"; + +// Exact OpenShell v0.0.85 Podman identity contract. Later driver schemas must +// register a new adapter; discovery deliberately fails closed on label or name +// drift instead of guessing which container NemoClaw is allowed to replace. +export const PODMAN_MANAGED_LABEL = "openshell.managed"; +export const PODMAN_SANDBOX_ID_LABEL = "openshell.sandbox-id"; +export const PODMAN_SANDBOX_NAME_LABEL = "openshell.sandbox-name"; +export const PODMAN_SANDBOX_NAMESPACE_LABEL = "openshell.sandbox-namespace"; +export const PODMAN_SANDBOX_CONTAINER_PREFIX = "openshell-sandbox-"; + +const COMMAND_ENV = "OPENSHELL_SANDBOX_COMMAND"; +const TOKEN_FILE_ENV = "OPENSHELL_SANDBOX_TOKEN_FILE"; +const TLS_ENV_KEYS = ["OPENSHELL_TLS_CA", "OPENSHELL_TLS_CERT", "OPENSHELL_TLS_KEY"] as const; +const WORKSPACE_DESTINATION = "/sandbox"; +const SUPERVISOR_DESTINATION = "/opt/openshell/bin"; +const NETNS_TMPFS_DESTINATION = "/run/netns"; +const FULL_ID_PATTERN = /^(?:sha256:)?([0-9a-f]{64})$/iu; +const SAFE_CAPABILITY_PATTERN = /^(?:CAP_)?[A-Z][A-Z0-9_]*$/u; +const SAFE_ULIMIT_PATTERN = /^(?:RLIMIT_)?[A-Za-z][A-Za-z0-9_]*$/u; + +type JsonRecord = Record; + +export interface PodmanUlimit { + readonly hard: number; + readonly name: string; + readonly soft: number; +} + +export interface PodmanManagedSandboxInspect { + readonly raw: JsonRecord; + readonly containerId: string; + readonly immutableImage: string; + readonly labels: Readonly>; + readonly name: string; + readonly running: boolean; + readonly sandboxId: string; +} + +export interface PodmanManagedSandboxCreatePlan { + readonly args: readonly string[]; + readonly environmentInput: string; + readonly immutableImage: string; +} + +export type PodmanSandboxIdentityMode = "managed" | "watcher-invisible-backup"; + +function record(value: unknown, label: string): JsonRecord { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + throw new Error(`${label} must be an object.`); + } + return value as JsonRecord; +} + +function optionalRecord(value: unknown): JsonRecord | null { + return typeof value === "object" && value !== null && !Array.isArray(value) + ? (value as JsonRecord) + : null; +} + +function array(value: unknown, label: string): unknown[] { + if (!Array.isArray(value)) throw new Error(`${label} must be an array.`); + return value; +} + +function string(value: unknown, label: string, allowEmpty = false): string { + if ( + typeof value !== "string" || + (!allowEmpty && value.length === 0) || + value !== value.trim() || + value.includes("\0") + ) { + throw new Error(`${label} must be a safe${allowEmpty ? "" : " non-empty"} string.`); + } + return value; +} + +function optionalString(value: unknown, label: string): string { + if (value === undefined || value === null || value === "") return ""; + return string(value, label); +} + +function stringArray(value: unknown, label: string): string[] { + if (value === undefined || value === null) return []; + return array(value, label).map((entry, index) => string(entry, `${label}[${index}]`, true)); +} + +function boolean(value: unknown, label: string, fallback = false): boolean { + if (value === undefined || value === null) return fallback; + if (typeof value !== "boolean") throw new Error(`${label} must be a boolean.`); + return value; +} + +function safeInteger( + value: unknown, + label: string, + options: { fallback?: number; minimum?: number; maximum?: number } = {}, +): number { + if (value === undefined || value === null) return options.fallback ?? 0; + if ( + !Number.isSafeInteger(value) || + (options.minimum !== undefined && (value as number) < options.minimum) || + (options.maximum !== undefined && (value as number) > options.maximum) + ) { + throw new Error(`${label} must be a safe integer.`); + } + return value as number; +} + +function hasEntries(value: unknown): boolean { + if (value === undefined || value === null || value === "" || value === false || value === 0) { + return false; + } + if (Array.isArray(value)) return value.length > 0; + if (typeof value === "object") return Object.keys(value as JsonRecord).length > 0; + return true; +} + +function assertEmpty(value: unknown, label: string): void { + if (hasEntries(value)) { + throw new Error(`${label} cannot be reproduced faithfully by native Podman recreation.`); + } +} + +function fullId(value: unknown, label: string): string { + const candidate = string(value, label); + const match = candidate.match(FULL_ID_PATTERN); + if (!match?.[1]) throw new Error(`${label} must be a full immutable SHA-256 identifier.`); + return match[1].toLowerCase(); +} + +function immutableImage(value: unknown, label: string): string { + return `sha256:${fullId(value, label)}`; +} + +function safeDelimitedValue(value: unknown, label: string): string { + const candidate = string(value, label); + if (/[\r\n,]/u.test(candidate)) { + throw new Error(`${label} contains a delimiter Podman recreation cannot preserve safely.`); + } + return candidate; +} + +function safeLabelValue(value: unknown, label: string): string { + const candidate = string(value, label, true); + if (/[\r\n,]/u.test(candidate)) { + throw new Error(`${label} contains a delimiter Podman recreation cannot preserve safely.`); + } + return candidate; +} + +function absoluteContainerPath(value: unknown, label: string): string { + const candidate = safeDelimitedValue(value, label); + if (!candidate.startsWith("/") || candidate.includes("/../") || candidate.endsWith("/..")) { + throw new Error(`${label} must be an absolute normalized container path.`); + } + return candidate; +} + +function stringMap(value: unknown, label: string): Record { + const source = record(value, label); + const result: Record = {}; + for (const [key, entry] of Object.entries(source)) { + result[string(key, `${label} key`)] = string(entry, `${label}.${key}`, true); + } + return result; +} + +const PODMAN_OPEN_SHELL_IDENTITY_LABELS = new Set([ + PODMAN_MANAGED_LABEL, + PODMAN_SANDBOX_ID_LABEL, + PODMAN_SANDBOX_NAME_LABEL, + PODMAN_SANDBOX_NAMESPACE_LABEL, +]); + +function sandboxLabels( + config: JsonRecord, + expected: { + readonly identityMode?: PodmanSandboxIdentityMode; + readonly sandboxId?: string; + readonly sandboxName: string; + }, +): { labels: Record; sandboxId: string } { + const labels = stringMap(config.Labels, "Podman inspect Config.Labels"); + if (expected.identityMode === "watcher-invisible-backup") { + const leakedIdentity = [...PODMAN_OPEN_SHELL_IDENTITY_LABELS].filter((label) => + Object.hasOwn(labels, label), + ); + if (leakedIdentity.length > 0) { + throw new Error( + `Podman rollback backup remains visible to the OpenShell watcher through label(s): ${leakedIdentity.join(", ")}.`, + ); + } + const sandboxId = string(expected.sandboxId, "Expected Podman rollback backup sandbox ID"); + return { labels, sandboxId }; + } + if (labels[PODMAN_MANAGED_LABEL] !== "true") { + throw new Error(`Podman sandbox is missing exact label ${PODMAN_MANAGED_LABEL}=true.`); + } + if (labels[PODMAN_SANDBOX_NAME_LABEL] !== expected.sandboxName) { + throw new Error( + `Podman sandbox is missing exact label ${PODMAN_SANDBOX_NAME_LABEL}=${expected.sandboxName}.`, + ); + } + if (!Object.hasOwn(labels, PODMAN_SANDBOX_ID_LABEL)) { + throw new Error(`Podman sandbox is missing exact label ${PODMAN_SANDBOX_ID_LABEL}.`); + } + string(labels[PODMAN_SANDBOX_ID_LABEL], `Podman label ${PODMAN_SANDBOX_ID_LABEL}`); + if (!Object.hasOwn(labels, PODMAN_SANDBOX_NAMESPACE_LABEL)) { + throw new Error(`Podman sandbox is missing exact label ${PODMAN_SANDBOX_NAMESPACE_LABEL}.`); + } + return { + labels, + sandboxId: labels[PODMAN_SANDBOX_ID_LABEL] as string, + }; +} + +export function parsePodmanManagedSandboxInspect( + output: string, + expected: { + readonly containerId: string; + readonly identityMode?: PodmanSandboxIdentityMode; + readonly name: string; + readonly requireRunning?: boolean; + readonly sandboxId?: string; + readonly sandboxName: string; + }, +): PodmanManagedSandboxInspect { + let parsed: unknown; + try { + parsed = JSON.parse(output); + } catch { + throw new Error("Podman container inspect returned unreadable JSON."); + } + const entries = array(parsed, "Podman container inspect response"); + if (entries.length !== 1) { + throw new Error("Podman container inspect must identify exactly one container."); + } + const raw = record(entries[0], "Podman container inspect entry"); + const containerId = fullId(raw.Id, "Podman inspect Id"); + if (containerId !== fullId(expected.containerId, "Expected Podman container ID")) { + throw new Error("Podman managed sandbox identity changed after it was pinned."); + } + const name = string(raw.Name, "Podman inspect Name"); + if (name !== expected.name) { + throw new Error(`Podman managed sandbox name changed from '${expected.name}' to '${name}'.`); + } + const config = record(raw.Config, "Podman inspect Config"); + const { labels, sandboxId } = sandboxLabels(config, expected); + const state = record(raw.State, "Podman inspect State"); + const running = boolean(state.Running, "Podman inspect State.Running"); + if (expected.requireRunning && !running) { + throw new Error("Podman managed sandbox must be running before recreation."); + } + return { + raw, + containerId, + immutableImage: immutableImage(raw.Image, "Podman inspect Image"), + labels, + name, + running, + sandboxId, + }; +} + +function environment( + config: JsonRecord, + command: readonly string[] | null, + requireCommandEnvironment: boolean, +): string[] { + const commandValue = command === null ? null : openshellSandboxCommandEnvValue(command); + if (command !== null && !commandValue) { + throw new Error("Podman sandbox startup command must not be empty."); + } + const values = stringArray(config.Env, "Podman inspect Config.Env"); + const byKey = new Map(); + for (const entry of values) { + const separator = entry.indexOf("="); + if (separator <= 0) throw new Error("Podman inspect environment contains a malformed entry."); + const key = entry.slice(0, separator); + if (!/^[A-Za-z_][A-Za-z0-9_]*$/u.test(key) || byKey.has(key)) { + throw new Error(`Podman inspect environment key '${key}' is invalid or duplicated.`); + } + if (entry.endsWith("=*******")) { + throw new Error("Podman env-type secrets cannot be reproduced from redacted inspect data."); + } + byKey.set(key, entry.slice(separator + 1)); + } + if (command === null) { + const preserved = byKey.get(COMMAND_ENV); + if (!preserved || preserved !== openshellSandboxCommandEnvValue(preserved.split(" "))) { + throw new Error( + `Podman rollback backup cannot preserve a non-canonical ${COMMAND_ENV} value.`, + ); + } + } else if (requireCommandEnvironment && byKey.get(COMMAND_ENV) !== commandValue) { + throw new Error( + `Podman replacement environment does not preserve the requested ${COMMAND_ENV} value.`, + ); + } + if (commandValue !== null) byKey.set(COMMAND_ENV, commandValue); + return [...byKey.entries()] + .sort(([left], [right]) => left.localeCompare(right)) + .map(([key, value]) => `${key}=${value}`); +} + +function environmentFileInput(env: readonly string[]): string { + for (const entry of env) { + if (/[\r\n]/u.test(entry)) { + throw new Error( + "Podman inspect environment contains a multiline value that cannot be passed without leaking it in process arguments.", + ); + } + } + return `${env.join("\n")}\n`; +} + +function envValue(env: readonly string[], key: string): string | null { + const prefix = `${key}=`; + const entry = env.find((candidate) => candidate.startsWith(prefix)); + return entry ? entry.slice(prefix.length) : null; +} + +function mountOptionList(mount: JsonRecord, label: string): string[] { + return stringArray(mount.Options, `${label}.Options`); +} + +function mountValue( + mount: JsonRecord, + imagePins: Readonly>, + index: number, +): string { + const label = `Podman inspect Mounts[${index}]`; + const type = string(mount.Type, `${label}.Type`).toLowerCase(); + const destination = absoluteContainerPath(mount.Destination, `${label}.Destination`); + const readWrite = boolean(mount.RW, `${label}.RW`); + const subpath = optionalString(mount.SubPath, `${label}.SubPath`); + const options = mountOptionList(mount, label); + const result = [`type=${type}`]; + + if (type === "volume") { + const name = safeDelimitedValue(mount.Name, `${label}.Name`); + const driver = optionalString(mount.Driver, `${label}.Driver`); + if (driver && driver !== "local") { + throw new Error(`${label} uses unsupported volume driver '${driver}'.`); + } + if (options.length > 0) { + throw new Error(`${label} has volume options that cannot be reproduced faithfully.`); + } + result.push(`source=${name}`, `destination=${destination}`, `ro=${String(!readWrite)}`); + } else if (type === "bind") { + const source = safeDelimitedValue(mount.Source, `${label}.Source`); + if (!source.startsWith("/")) throw new Error(`${label}.Source must be an absolute host path.`); + const unsupported = options.filter((option) => option !== "rbind"); + if (unsupported.length > 0) { + throw new Error(`${label} has unsupported bind options: ${unsupported.join(", ")}.`); + } + result.push(`source=${source}`, `destination=${destination}`, `ro=${String(!readWrite)}`); + const propagation = optionalString(mount.Propagation, `${label}.Propagation`); + if (propagation) result.push(`bind-propagation=${propagation}`); + const mode = optionalString(mount.Mode, `${label}.Mode`); + if (mode === "z") result.push("relabel=shared"); + else if (mode === "Z") result.push("relabel=private"); + else if (mode) throw new Error(`${label} has unsupported SELinux mode '${mode}'.`); + } else if (type === "image") { + const source = safeDelimitedValue(mount.Source, `${label}.Source`); + const pinned = imagePins[source]; + if (!pinned) + throw new Error(`${label} image source '${source}' was not pinned before mutation.`); + if (options.length > 0) { + throw new Error(`${label} has image options that cannot be reproduced faithfully.`); + } + result.push( + `source=${immutableImage(pinned, `${label} pinned image`)}`, + `destination=${destination}`, + `rw=${String(readWrite)}`, + ); + } else { + throw new Error(`${label} uses unsupported mount type '${type}'.`); + } + + if (subpath) result.push(`subpath=${safeDelimitedValue(subpath, `${label}.SubPath`)}`); + return result.join(","); +} + +function assertRequiredMounts( + mounts: readonly JsonRecord[], + env: readonly string[], + sandboxId: string, +): void { + const workspace = mounts.filter( + (mount) => + String(mount.Type).toLowerCase() === "volume" && + mount.Name === `${PODMAN_SANDBOX_CONTAINER_PREFIX}${sandboxId}-workspace` && + mount.Destination === WORKSPACE_DESTINATION && + mount.RW === true, + ); + if (workspace.length !== 1) { + throw new Error("Podman recreation requires exactly one writable /sandbox workspace volume."); + } + const supervisor = mounts.filter( + (mount) => + String(mount.Type).toLowerCase() === "image" && + mount.Destination === SUPERVISOR_DESTINATION && + mount.RW === false, + ); + if (supervisor.length !== 1) { + throw new Error( + "Podman recreation requires exactly one read-only supervisor image mount at /opt/openshell/bin.", + ); + } + + const tlsPaths = TLS_ENV_KEYS.map((key) => envValue(env, key)); + if (tlsPaths.some(Boolean) && !tlsPaths.every(Boolean)) { + throw new Error("Podman recreation requires all three OpenShell TLS path variables together."); + } + for (const [index, target] of tlsPaths.entries()) { + if (!target) continue; + const normalized = absoluteContainerPath(target, TLS_ENV_KEYS[index] ?? "OpenShell TLS path"); + const matches = mounts.filter( + (mount) => + String(mount.Type).toLowerCase() === "bind" && + mount.Destination === normalized && + mount.RW === false, + ); + if (matches.length !== 1) { + throw new Error( + `Podman recreation cannot prove the read-only TLS mount for '${normalized}'.`, + ); + } + } +} + +export function podmanImageMountSources(inspect: PodmanManagedSandboxInspect): string[] { + const mounts = array(inspect.raw.Mounts, "Podman inspect Mounts").map((entry, index) => + record(entry, `Podman inspect Mounts[${index}]`), + ); + return [ + ...new Set( + mounts + .filter((mount) => String(mount.Type).toLowerCase() === "image") + .map((mount, index) => + safeDelimitedValue(mount.Source, `Podman inspect image Mounts[${index}].Source`), + ), + ), + ].sort(); +} + +function secretArguments(config: JsonRecord, env: readonly string[], sandboxId: string): string[] { + const secrets = ( + config.Secrets === undefined || config.Secrets === null + ? [] + : array(config.Secrets, "Podman inspect Config.Secrets") + ).map((entry, index) => record(entry, `Podman inspect Config.Secrets[${index}]`)); + const cmd = stringArray(config.Cmd, "Podman inspect Config.Cmd"); + const tokenTarget = envValue(env, TOKEN_FILE_ENV); + const argumentTarget = (flag: string): string | null => { + const index = cmd.indexOf(flag); + if (index === -1) return null; + if (cmd.indexOf(flag, index + 1) !== -1) { + throw new Error(`Podman inspect command repeats reserved supervisor flag '${flag}'.`); + } + return cmd[index + 1] ?? null; + }; + const targets = new Map([ + ["openshell-token-", tokenTarget], + ["openshell-proxy-auth-", argumentTarget("--upstream-proxy-auth-file")], + ["openshell-trusted-init-", argumentTarget("--trusted-init-file")], + ]); + const args: string[] = []; + const seenTargets = new Set(); + for (const [index, secret] of secrets.entries()) { + const label = `Podman inspect Config.Secrets[${index}]`; + const name = safeDelimitedValue(secret.Name, `${label}.Name`); + const secretId = fullId(secret.ID, `${label}.ID`); + const matchingPrefix = [...targets.keys()].find((prefix) => name.startsWith(prefix)); + if (!matchingPrefix) { + throw new Error(`${label} '${name}' has no deterministic OpenShell target mapping.`); + } + if (matchingPrefix === "openshell-token-" && name !== `openshell-token-${sandboxId}`) { + throw new Error(`${label} '${name}' does not match the exact OpenShell sandbox ID.`); + } + const targetValue = targets.get(matchingPrefix); + if (!targetValue) { + throw new Error(`${label} '${name}' is not paired with its required path evidence.`); + } + const target = absoluteContainerPath(targetValue, `${label} target`); + if (seenTargets.has(target)) throw new Error(`Podman secrets duplicate target '${target}'.`); + seenTargets.add(target); + const uid = safeInteger(secret.UID, `${label}.UID`, { minimum: 0 }); + const gid = safeInteger(secret.GID, `${label}.GID`, { minimum: 0 }); + const mode = safeInteger(secret.Mode, `${label}.Mode`, { minimum: 0, maximum: 0o7777 }); + args.push( + "--secret", + `${secretId},type=mount,target=${target},uid=${uid},gid=${gid},mode=${mode + .toString(8) + .padStart(4, "0")}`, + ); + } + if ( + tokenTarget && + !secrets.some((secret) => String(secret.Name).startsWith("openshell-token-")) + ) { + throw new Error("Podman token-file environment is missing its OpenShell token secret."); + } + return args; +} + +function tmpfsArguments(hostConfig: JsonRecord): string[] { + const tmpfs = record(hostConfig.Tmpfs, "Podman inspect HostConfig.Tmpfs"); + if (!Object.hasOwn(tmpfs, NETNS_TMPFS_DESTINATION)) { + throw new Error("Podman recreation requires the OpenShell /run/netns tmpfs mount."); + } + const args: string[] = []; + for (const [targetValue, optionValue] of Object.entries(tmpfs).sort(([a], [b]) => + a.localeCompare(b), + )) { + const target = absoluteContainerPath(targetValue, "Podman tmpfs target"); + const options = optionalString(optionValue, `Podman tmpfs options for ${target}`); + if ( + options && + options + .split(",") + .some( + (option) => + !/^(?:rw|ro|nosuid|nodev|noexec|exec|size=\d+|mode=[0-7]{3,4}|noatime|notmpcopyup)$/u.test( + option, + ), + ) + ) { + throw new Error(`Podman tmpfs '${target}' has options that cannot be reproduced faithfully.`); + } + args.push("--tmpfs", options ? `${target}:${options}` : target); + } + return args; +} + +function healthArguments(config: JsonRecord): string[] { + const health = record(config.Healthcheck, "Podman inspect Config.Healthcheck"); + const test = stringArray(health.Test, "Podman inspect Config.Healthcheck.Test"); + if (test.length < 2 || !["CMD", "CMD-SHELL"].includes(test[0] ?? "")) { + throw new Error("Podman recreation requires an explicit exec or shell healthcheck."); + } + if (config.StartupHealthCheck !== undefined && config.StartupHealthCheck !== null) { + throw new Error("Podman startup healthchecks are not yet reproducible by this primitive."); + } + const args = ["--health-cmd", JSON.stringify(test)]; + const durationFlags = [ + ["Interval", "--health-interval"], + ["Timeout", "--health-timeout"], + ["StartPeriod", "--health-start-period"], + ] as const; + for (const [field, flag] of durationFlags) { + const duration = safeInteger(health[field], `Podman healthcheck ${field}`, { minimum: 0 }); + args.push(flag, `${duration}ns`); + } + args.push( + "--health-retries", + String(safeInteger(health.Retries, "Podman healthcheck Retries", { minimum: 0 })), + ); + const action = optionalString(config.HealthcheckOnFailureAction, "HealthcheckOnFailureAction"); + if (action && action !== "none") args.push("--health-on-failure", action); + const destination = optionalString(config.HealthLogDestination, "HealthLogDestination"); + if (destination) args.push("--health-log-destination", destination); + const maxCount = safeInteger(config.HealthcheckMaxLogCount, "HealthcheckMaxLogCount", { + minimum: 0, + }); + if (maxCount > 0) args.push("--health-max-log-count", String(maxCount)); + const maxSize = safeInteger(config.HealthcheckMaxLogSize, "HealthcheckMaxLogSize", { + minimum: 0, + }); + if (maxSize > 0) args.push("--health-max-log-size", String(maxSize)); + return args; +} + +function normalizedUlimit(value: PodmanUlimit, label: string): PodmanUlimit { + let name = string(value.name, `${label}.name`); + if (!SAFE_ULIMIT_PATTERN.test(name)) throw new Error(`${label}.name '${name}' is invalid.`); + name = name.replace(/^RLIMIT_/iu, "").toLowerCase(); + const soft = safeInteger(value.soft, `${label}.soft`, { minimum: -1 }); + const hard = safeInteger(value.hard, `${label}.hard`, { minimum: -1 }); + if (hard !== -1 && (soft === -1 || soft > hard)) { + throw new Error(`${label} has a soft limit greater than its hard limit.`); + } + return { name, soft, hard }; +} + +function ulimitArguments(hostConfig: JsonRecord, required: readonly PodmanUlimit[]): string[] { + const merged = new Map(); + const existing = hostConfig.Ulimits ?? []; + for (const [index, entry] of array(existing, "Podman inspect HostConfig.Ulimits").entries()) { + const source = record(entry, `Podman inspect HostConfig.Ulimits[${index}]`); + const normalized = normalizedUlimit( + { + name: string(source.Name, `HostConfig.Ulimits[${index}].Name`), + soft: safeInteger(source.Soft, `HostConfig.Ulimits[${index}].Soft`, { minimum: -1 }), + hard: safeInteger(source.Hard, `HostConfig.Ulimits[${index}].Hard`, { minimum: -1 }), + }, + `HostConfig.Ulimits[${index}]`, + ); + if (merged.has(normalized.name)) throw new Error(`Podman ulimit '${normalized.name}' repeats.`); + merged.set(normalized.name, normalized); + } + for (const [index, entry] of required.entries()) { + const normalized = normalizedUlimit(entry, `Required Podman ulimit[${index}]`); + merged.set(normalized.name, normalized); + } + return [...merged.values()] + .sort((left, right) => left.name.localeCompare(right.name)) + .flatMap((limit) => ["--ulimit", `${limit.name}=${String(limit.soft)}:${String(limit.hard)}`]); +} + +function resourceArguments(hostConfig: JsonRecord): string[] { + const args: string[] = []; + const numericFlags = [ + ["CpuShares", "--cpu-shares", 0], + ["CpuPeriod", "--cpu-period", 0], + ["CpuQuota", "--cpu-quota", -1], + ["Memory", "--memory", 0], + ["MemoryReservation", "--memory-reservation", 0], + ["MemorySwap", "--memory-swap", -1], + ["PidsLimit", "--pids-limit", -1], + ["ShmSize", "--shm-size", 0], + ] as const; + for (const [field, flag, minimum] of numericFlags) { + const value = safeInteger(hostConfig[field], `Podman HostConfig.${field}`, { minimum }); + if (value !== 0) args.push(flag, String(value)); + } + const cpuQuota = safeInteger(hostConfig.CpuQuota, "Podman HostConfig.CpuQuota", { minimum: -1 }); + const cpuPeriod = safeInteger(hostConfig.CpuPeriod, "Podman HostConfig.CpuPeriod", { + minimum: 0, + }); + const nanoCpus = safeInteger(hostConfig.NanoCpus, "Podman HostConfig.NanoCpus", { + minimum: 0, + }); + if (nanoCpus > 0 && !(cpuQuota > 0 && cpuPeriod > 0)) { + args.push("--cpus", String(nanoCpus / 1_000_000_000)); + } + for (const [field, flag] of [ + ["CpusetCpus", "--cpuset-cpus"], + ["CpusetMems", "--cpuset-mems"], + ] as const) { + const value = optionalString(hostConfig[field], `Podman HostConfig.${field}`); + if (value) { + if (!/^\d+(?:-\d+)?(?:,\d+(?:-\d+)?)*$/u.test(value)) { + throw new Error(`Podman HostConfig.${field} is malformed.`); + } + args.push(flag, value); + } + } + if (hostConfig.MemorySwappiness !== undefined && hostConfig.MemorySwappiness !== null) { + const swappiness = safeInteger( + hostConfig.MemorySwappiness, + "Podman HostConfig.MemorySwappiness", + { + minimum: -1, + maximum: 100, + }, + ); + if (swappiness !== -1 && swappiness !== 0) { + throw new Error( + "Podman non-neutral memory swappiness cannot be preserved on the required rootless v2 path.", + ); + } + } + if (boolean(hostConfig.OomKillDisable, "Podman HostConfig.OomKillDisable")) { + args.push("--oom-kill-disable"); + } + const oomScore = safeInteger(hostConfig.OomScoreAdj, "Podman HostConfig.OomScoreAdj", { + minimum: -1000, + maximum: 1000, + }); + if (oomScore !== 0) args.push("--oom-score-adj", String(oomScore)); + return args; +} + +function capabilityAndSecurityArguments(hostConfig: JsonRecord): string[] { + if (boolean(hostConfig.Privileged, "Podman HostConfig.Privileged")) { + throw new Error("Privileged OpenShell containers are not eligible for managed recreation."); + } + const args: string[] = []; + for (const [field, flag] of [ + ["CapDrop", "--cap-drop"], + ["CapAdd", "--cap-add"], + ] as const) { + const seen = new Set(); + for (const capability of stringArray(hostConfig[field], `Podman HostConfig.${field}`)) { + if (!SAFE_CAPABILITY_PATTERN.test(capability) || seen.has(capability)) { + throw new Error(`Podman HostConfig.${field} contains an invalid or duplicate capability.`); + } + seen.add(capability); + args.push(flag, capability); + } + } + for (const option of stringArray(hostConfig.SecurityOpt, "Podman HostConfig.SecurityOpt")) { + args.push("--security-opt", option); + } + if (boolean(hostConfig.ReadonlyRootfs, "Podman HostConfig.ReadonlyRootfs")) { + args.push("--read-only"); + } + return args; +} + +function networkAndPortArguments(raw: JsonRecord, hostConfig: JsonRecord): string[] { + const networkSettings = record(raw.NetworkSettings, "Podman inspect NetworkSettings"); + const networks = record(networkSettings.Networks, "Podman inspect NetworkSettings.Networks"); + const names = Object.keys(networks); + if (names.length !== 1) { + throw new Error("Podman recreation requires exactly one attached named network."); + } + const networkName = safeDelimitedValue(names[0], "Podman network name"); + const mode = optionalString(hostConfig.NetworkMode, "Podman HostConfig.NetworkMode"); + if (mode && !["bridge", "default", networkName].includes(mode)) { + throw new Error(`Podman network mode '${mode}' cannot be reproduced faithfully.`); + } + const network = record(networks[networkName], `Podman network '${networkName}'`); + const networkId = fullId(network.NetworkID, `Podman network '${networkName}' ID`); + assertEmpty(network.DriverOpts, `Podman network '${networkName}' driver options`); + assertEmpty(network.IPAMConfig, `Podman network '${networkName}' IPAM options`); + assertEmpty(network.Links, `Podman network '${networkName}' links`); + const args = ["--network", networkId]; + + const originalId = fullId(raw.Id, "Podman inspect Id"); + const originalName = string(raw.Name, "Podman inspect Name"); + const automaticAliases = new Set([originalId, originalId.slice(0, 12), originalName]); + for (const alias of stringArray(network.Aliases, `Podman network '${networkName}' aliases`)) { + if (!automaticAliases.has(alias)) args.push("--network-alias", alias); + } + + const configured = optionalRecord(hostConfig.PortBindings) ?? {}; + const actual = optionalRecord(networkSettings.Ports) ?? {}; + for (const [portKey, bindingsValue] of Object.entries(configured).sort(([a], [b]) => + a.localeCompare(b), + )) { + const match = portKey.match(/^([1-9]\d{0,4})\/(tcp|udp|sctp)$/u); + const containerPort = Number(match?.[1] ?? 0); + if (!match || containerPort > 65_535) { + throw new Error(`Podman port binding '${portKey}' is malformed.`); + } + const configuredBindings = array(bindingsValue, `Podman HostConfig.PortBindings.${portKey}`); + const actualBindings = array(actual[portKey], `Podman NetworkSettings.Ports.${portKey}`); + if (configuredBindings.length !== actualBindings.length || actualBindings.length === 0) { + throw new Error(`Podman port binding '${portKey}' lacks exact running-port evidence.`); + } + for (const [index, bindingValue] of actualBindings.entries()) { + const binding = record(bindingValue, `Podman actual port ${portKey}[${index}]`); + const hostPort = safeInteger( + Number(string(binding.HostPort, `Podman actual port ${portKey}[${index}].HostPort`)), + `Podman actual port ${portKey}[${index}].HostPort`, + { minimum: 1, maximum: 65_535 }, + ); + const hostIp = optionalString(binding.HostIp ?? binding.HostIP, "Podman actual host IP"); + if (hostIp && /[\r\n,]/u.test(hostIp)) throw new Error("Podman host IP is malformed."); + args.push( + "--publish", + `${hostIp ? `${hostIp}:` : ""}${hostPort}:${containerPort}/${match[2]}`, + ); + } + } + if (Object.keys(actual).some((key) => !Object.hasOwn(configured, key))) { + throw new Error("Podman running port data contains an unconfigured published port."); + } + return args; +} + +function hostArguments(config: JsonRecord, hostConfig: JsonRecord): string[] { + const args: string[] = []; + const hostname = optionalString(config.Hostname, "Podman Config.Hostname"); + if (hostname) args.push("--hostname", hostname); + const user = optionalString(config.User, "Podman Config.User"); + if (user) args.push("--user", user); + const workdir = optionalString(config.WorkingDir, "Podman Config.WorkingDir"); + if (workdir) args.push("--workdir", absoluteContainerPath(workdir, "Podman working directory")); + if (boolean(config.Tty, "Podman Config.Tty")) args.push("--tty"); + if (boolean(config.OpenStdin, "Podman Config.OpenStdin")) args.push("--interactive"); + const stopSignal = optionalString(config.StopSignal, "Podman Config.StopSignal"); + if (stopSignal) args.push("--stop-signal", stopSignal); + const stopTimeout = safeInteger(config.StopTimeout, "Podman Config.StopTimeout", { minimum: 0 }); + args.push("--stop-timeout", String(stopTimeout)); + const umask = optionalString(config.Umask, "Podman Config.Umask"); + if (umask) { + if (!/^[0-7]{4}$/u.test(umask)) throw new Error("Podman Config.Umask is malformed."); + args.push("--umask", umask); + } + for (const host of stringArray(hostConfig.ExtraHosts, "Podman HostConfig.ExtraHosts")) { + args.push("--add-host", host); + } + for (const group of stringArray(hostConfig.GroupAdd, "Podman HostConfig.GroupAdd")) { + args.push("--group-add", group); + } + for (const dns of stringArray(hostConfig.Dns, "Podman HostConfig.Dns")) { + args.push("--dns", dns); + } + for (const option of stringArray(hostConfig.DnsOptions, "Podman HostConfig.DnsOptions")) { + args.push("--dns-option", option); + } + for (const search of stringArray(hostConfig.DnsSearch, "Podman HostConfig.DnsSearch")) { + args.push("--dns-search", search); + } + const restart = optionalRecord(hostConfig.RestartPolicy); + if (restart) { + const name = optionalString(restart.Name, "Podman RestartPolicy.Name") || "no"; + const retries = safeInteger(restart.MaximumRetryCount, "Podman restart retries", { + minimum: 0, + }); + const value = name === "on-failure" && retries > 0 ? `${name}:${retries}` : name; + if (!["no", "always", "unless-stopped", "on-failure"].includes(name)) { + throw new Error(`Podman restart policy '${name}' is unsupported.`); + } + args.push("--restart", value); + } + if (boolean(hostConfig.Init, "Podman HostConfig.Init")) args.push("--init"); + return args; +} + +function assertSupportedShape(raw: JsonRecord, config: JsonRecord, hostConfig: JsonRecord): void { + assertEmpty(raw.Pod, "Podman pod membership"); + assertEmpty(raw.Dependencies, "Podman container dependencies"); + if (boolean(raw.IsInfra, "Podman IsInfra") || boolean(raw.IsService, "Podman IsService")) { + throw new Error("Podman infra and service containers are not eligible for recreation."); + } + for (const [field, label] of [ + ["Devices", "devices"], + ["VolumesFrom", "volumes-from"], + ["CgroupConf", "cgroup v2 settings"], + ["BlkioWeight", "block IO weight"], + ["BlkioWeightDevice", "block IO weight devices"], + ["BlkioDeviceReadBps", "block IO read limits"], + ["BlkioDeviceWriteBps", "block IO write limits"], + ["BlkioDeviceReadIOps", "block IO read IOPS"], + ["BlkioDeviceWriteIOps", "block IO write IOPS"], + ["CpuRealtimePeriod", "realtime CPU period"], + ["CpuRealtimeRuntime", "realtime CPU runtime"], + ["KernelMemory", "kernel memory"], + ] as const) { + assertEmpty(hostConfig[field], `Podman ${label}`); + } + if ( + boolean(hostConfig.AutoRemove, "Podman AutoRemove") || + boolean(hostConfig.AutoRemoveImage, "Podman AutoRemoveImage") || + boolean(hostConfig.PublishAllPorts, "Podman PublishAllPorts") + ) { + throw new Error( + "Podman auto-removal and publish-all settings are not eligible for recreation.", + ); + } + const cgroups = optionalString(hostConfig.Cgroups, "Podman HostConfig.Cgroups"); + if (cgroups && cgroups !== "default") { + throw new Error(`Podman cgroups mode '${cgroups}' cannot be reproduced faithfully.`); + } + for (const [field, allowed] of [ + ["IpcMode", ["", "shareable", "private"]], + ["PidMode", ["", "private"]], + ["UTSMode", ["", "private"]], + ["UsernsMode", ["", "host"]], + ["CgroupMode", ["", "private"]], + ] as const) { + const value = optionalString(hostConfig[field], `Podman HostConfig.${field}`); + if (!(allowed as readonly string[]).includes(value)) { + throw new Error(`Podman HostConfig.${field} '${value}' cannot be reproduced faithfully.`); + } + } + assertEmpty(config.ChrootDirs, "Podman chroot directories"); +} + +export function buildPodmanManagedSandboxCreatePlan(options: { + /** Null preserves the canonical command already present in inspect. */ + readonly command: readonly string[] | null; + readonly imagePins: Readonly>; + readonly inspect: PodmanManagedSandboxInspect; + readonly labels?: Readonly>; + readonly name?: string; + readonly requireCommandEnvironment?: boolean; + readonly requiredUlimits?: readonly PodmanUlimit[]; +}): PodmanManagedSandboxCreatePlan { + const raw = options.inspect.raw; + const config = record(raw.Config, "Podman inspect Config"); + const hostConfig = record(raw.HostConfig, "Podman inspect HostConfig"); + assertSupportedShape(raw, config, hostConfig); + const env = environment(config, options.command, options.requireCommandEnvironment === true); + const mounts = array(raw.Mounts, "Podman inspect Mounts").map((entry, index) => + record(entry, `Podman inspect Mounts[${index}]`), + ); + assertRequiredMounts(mounts, env, options.inspect.sandboxId); + const createName = string(options.name ?? options.inspect.name, "Podman recreate container name"); + if (!/^[A-Za-z0-9][A-Za-z0-9_.-]*$/u.test(createName)) { + throw new Error("Podman recreate container name is malformed."); + } + const createLabels = options.labels ?? options.inspect.labels; + + const args: string[] = [ + "create", + "--pull=never", + "--http-proxy=false", + "--name", + createName, + "--unsetenv-all", + "--env-file", + "/dev/stdin", + ]; + for (const [key, value] of Object.entries(createLabels).sort(([a], [b]) => a.localeCompare(b))) { + args.push( + "--label", + `${safeDelimitedValue(key, "Podman recreate label name")}=${safeLabelValue( + value, + `Podman recreate label ${key}`, + )}`, + ); + } + args.push( + ...hostArguments(config, hostConfig), + ...capabilityAndSecurityArguments(hostConfig), + ...resourceArguments(hostConfig), + ...ulimitArguments(hostConfig, options.requiredUlimits ?? []), + ...networkAndPortArguments(raw, hostConfig), + ...healthArguments(config), + ...tmpfsArguments(hostConfig), + ...secretArguments(config, env, options.inspect.sandboxId), + ); + for (const [index, mount] of mounts.entries()) { + args.push("--mount", mountValue(mount, options.imagePins, index)); + } + const entrypoint = stringArray(config.Entrypoint, "Podman inspect Config.Entrypoint"); + if (entrypoint.length > 0) args.push("--entrypoint", JSON.stringify(entrypoint)); + args.push(options.inspect.immutableImage); + args.push(...stringArray(config.Cmd, "Podman inspect Config.Cmd")); + return { + args, + environmentInput: environmentFileInput(env), + immutableImage: options.inspect.immutableImage, + }; +} + +/** Remove every v0.0.85 OpenShell identity label so watcher events cannot match the backup. */ +export function podmanWatcherInvisibleBackupLabels( + inspect: Pick, +): Readonly> { + return Object.fromEntries( + Object.entries(inspect.labels).filter( + ([label]) => !PODMAN_OPEN_SHELL_IDENTITY_LABELS.has(label), + ), + ); +} diff --git a/src/lib/onboard/compute/podman/sandbox-recreate-test-fixture.ts b/src/lib/onboard/compute/podman/sandbox-recreate-test-fixture.ts new file mode 100644 index 0000000000..14a3a31df8 --- /dev/null +++ b/src/lib/onboard/compute/podman/sandbox-recreate-test-fixture.ts @@ -0,0 +1,216 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { parsePodmanManagedSandboxInspect } from "./sandbox-recreate-spec"; + +export const OLD_ID = "a".repeat(64); +export const NEW_ID = "b".repeat(64); +export const BACKUP_ID = "6".repeat(64); +export const RESTORED_ID = "7".repeat(64); +export const ROOT_IMAGE_ID = "c".repeat(64); +export const SUPERVISOR_IMAGE_ID = "d".repeat(64); +export const SECRET_ID = "e".repeat(64); +export const NETWORK_ID = "f".repeat(64); +export const SOCKET_PATH = "/run/user/1000/podman/podman.sock"; +export const SOCKET_AUTHORITY = { + directoryChain: [ + { + device: "8", + inode: "7000", + mode: "448", + ownerUid: "1000", + path: "/run/user/1000/podman", + }, + ], + device: "8", + inode: "9001", + ownerUid: "1000", + socketPath: SOCKET_PATH, +} as const; +export const SANDBOX_NAME = "alpha"; +export const CONTAINER_NAME = "openshell-sandbox-alpha"; +export const SUPERVISOR_IMAGE = "ghcr.io/nvidia/openshell/supervisor:0.0.85"; + +export function validInspect(overrides: Record = {}): Record { + const value: Record = { + Id: OLD_ID, + Image: ROOT_IMAGE_ID, + Name: CONTAINER_NAME, + State: { Running: true }, + Pod: "", + Dependencies: [], + IsInfra: false, + IsService: false, + Config: { + Hostname: "sandbox-alpha", + User: "0:0", + Tty: false, + OpenStdin: false, + Env: [ + "OPENSHELL_SANDBOX=alpha", + "OPENSHELL_SANDBOX_COMMAND=sleep infinity", + "OPENSHELL_SANDBOX_TOKEN_FILE=/etc/openshell/auth/sandbox.jwt", + "OPENSHELL_TLS_CA=/etc/openshell/tls/client/ca.crt", + "OPENSHELL_TLS_CERT=/etc/openshell/tls/client/tls.crt", + "OPENSHELL_TLS_KEY=/etc/openshell/tls/client/tls.key", + "USER_VALUE=preserved", + ], + Cmd: ["--operator-mode"], + Image: "sandbox-image:mutable", + WorkingDir: "/sandbox", + Entrypoint: ["/opt/openshell/bin/openshell-sandbox"], + Labels: { + "custom.label": "preserved", + "openshell.managed": "true", + "openshell.sandbox-id": "sandbox-id", + "openshell.sandbox-name": SANDBOX_NAME, + "openshell.sandbox-namespace": "", + }, + StopSignal: "SIGTERM", + StartupHealthCheck: null, + Healthcheck: { + Test: ["CMD", "/opt/openshell/bin/openshell-sandbox", "__healthcheck"], + Interval: 5_000_000_000, + Timeout: 2_000_000_000, + Retries: 10, + StartPeriod: 5_000_000_000, + }, + HealthcheckOnFailureAction: "none", + HealthLogDestination: "local", + HealthcheckMaxLogCount: 5, + HealthcheckMaxLogSize: 500, + Secrets: [ + { + Name: "openshell-token-sandbox-id", + ID: SECRET_ID, + UID: 0, + GID: 0, + Mode: 0o400, + }, + ], + StopTimeout: 10, + Umask: "0022", + ChrootDirs: [], + }, + HostConfig: { + Binds: [], + NetworkMode: "bridge", + PortBindings: { + "22/tcp": [{ HostIp: "", HostPort: "0" }], + }, + RestartPolicy: { Name: "no", MaximumRetryCount: 0 }, + AutoRemove: false, + AutoRemoveImage: false, + PublishAllPorts: false, + VolumesFrom: [], + CapAdd: ["CAP_SYS_ADMIN", "CAP_NET_ADMIN"], + CapDrop: ["CAP_KILL", "CAP_NET_RAW"], + Dns: ["10.0.0.2"], + DnsOptions: ["ndots:1"], + DnsSearch: ["example.test"], + ExtraHosts: ["host.containers.internal:host-gateway", "host.openshell.internal:host-gateway"], + GroupAdd: ["44"], + IpcMode: "shareable", + CgroupMode: "private", + Cgroups: "default", + OomScoreAdj: 0, + PidMode: "private", + Privileged: false, + ReadonlyRootfs: false, + SecurityOpt: ["no-new-privileges", "seccomp=unconfined"], + Tmpfs: { "/run/netns": "rw,nosuid,nodev" }, + UTSMode: "private", + UsernsMode: "host", + ShmSize: 67_108_864, + CpuShares: 0, + Memory: 4_294_967_296, + NanoCpus: 2_000_000_000, + CpuPeriod: 100_000, + CpuQuota: 200_000, + CpuRealtimePeriod: 0, + CpuRealtimeRuntime: 0, + CpusetCpus: "", + CpusetMems: "", + Devices: [], + KernelMemory: 0, + MemoryReservation: 0, + MemorySwap: 0, + MemorySwappiness: 0, + OomKillDisable: false, + Init: false, + PidsLimit: 256, + Ulimits: [{ Name: "RLIMIT_NOFILE", Soft: 1024, Hard: 4096 }], + CgroupConf: {}, + BlkioWeight: 0, + BlkioWeightDevice: [], + BlkioDeviceReadBps: [], + BlkioDeviceWriteBps: [], + BlkioDeviceReadIOps: [], + BlkioDeviceWriteIOps: [], + }, + Mounts: [ + { + Type: "volume", + Name: "openshell-sandbox-sandbox-id-workspace", + Source: "/home/user/.local/share/containers/storage/volumes/workspace/_data", + Destination: "/sandbox", + Driver: "local", + Mode: "", + Options: [], + RW: true, + Propagation: "", + }, + { + Type: "image", + Source: SUPERVISOR_IMAGE, + Destination: "/opt/openshell/bin", + Driver: "", + Mode: "", + Options: [], + RW: false, + Propagation: "", + }, + ...[ + ["ca.crt", "/etc/openshell/tls/client/ca.crt"], + ["tls.crt", "/etc/openshell/tls/client/tls.crt"], + ["tls.key", "/etc/openshell/tls/client/tls.key"], + ].map(([file, destination]) => ({ + Type: "bind", + Name: "", + Source: `/run/openshell/tls/${file}`, + Destination: destination, + Driver: "", + Mode: "", + Options: ["rbind"], + RW: false, + Propagation: "rprivate", + })), + ], + NetworkSettings: { + Ports: { + "22/tcp": [{ HostIp: "127.0.0.1", HostPort: "41022" }], + }, + Networks: { + openshell: { + Aliases: [OLD_ID, OLD_ID.slice(0, 12), CONTAINER_NAME], + DriverOpts: {}, + IPAMConfig: {}, + Links: [], + NetworkID: NETWORK_ID, + IPAddress: "10.89.0.5", + MacAddress: "02:42:ac:11:00:02", + }, + }, + }, + }; + return { ...value, ...overrides }; +} + +export function parsedInspect(raw = validInspect()) { + return parsePodmanManagedSandboxInspect(JSON.stringify([raw]), { + containerId: String(raw.Id), + name: String(raw.Name), + requireRunning: true, + sandboxName: SANDBOX_NAME, + }); +} diff --git a/src/lib/onboard/compute/podman/sandbox-recreate.test.ts b/src/lib/onboard/compute/podman/sandbox-recreate.test.ts new file mode 100644 index 0000000000..c130390c81 --- /dev/null +++ b/src/lib/onboard/compute/podman/sandbox-recreate.test.ts @@ -0,0 +1,1372 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { writeFileSync } from "node:fs"; +import { describe, expect, it, vi } from "vitest"; +import { + createPodmanOpenShellWatcherController, + finalizePodmanManagedSandbox, + findPodmanManagedSandboxContainerIds, + type RunQualifiedPodmanCommand, + recreatePodmanManagedSandbox, + rollbackPodmanManagedSandbox, +} from "./sandbox-recreate"; +import { + buildPodmanManagedSandboxCreatePlan, + parsePodmanManagedSandboxInspect, + podmanImageMountSources, +} from "./sandbox-recreate-spec"; +import { + BACKUP_ID, + CONTAINER_NAME, + NETWORK_ID, + NEW_ID, + OLD_ID, + parsedInspect, + RESTORED_ID, + ROOT_IMAGE_ID, + SANDBOX_NAME, + SECRET_ID, + SOCKET_AUTHORITY, + SOCKET_PATH, + SUPERVISOR_IMAGE, + SUPERVISOR_IMAGE_ID, + validInspect, +} from "./sandbox-recreate-test-fixture"; + +function flagValues(args: readonly string[], flag: string): string[] { + const values: string[] = []; + for (let index = 0; index < args.length; index += 1) { + if (args[index] === flag && args[index + 1] !== undefined) + values.push(args[index + 1] as string); + } + return values; +} + +describe("Podman managed sandbox recreation specification", () => { + it("pins the full root and supervisor images while preserving the managed container shape", () => { + const inspect = parsedInspect(); + expect(podmanImageMountSources(inspect)).toEqual([SUPERVISOR_IMAGE]); + const plan = buildPodmanManagedSandboxCreatePlan({ + command: ["node", "agent.js", "--serve"], + imagePins: { [SUPERVISOR_IMAGE]: SUPERVISOR_IMAGE_ID }, + inspect, + requiredUlimits: [ + { name: "nofile", soft: 65_536, hard: 65_536 }, + { name: "memlock", soft: -1, hard: -1 }, + ], + }); + + expect(plan.immutableImage).toBe(`sha256:${ROOT_IMAGE_ID}`); + expect(plan.args.at(-2)).toBe(`sha256:${ROOT_IMAGE_ID}`); + expect(plan.args.at(-1)).toBe("--operator-mode"); + expect(flagValues(plan.args, "--env")).toEqual([]); + expect(flagValues(plan.args, "--env-file")).toEqual(["/dev/stdin"]); + expect(plan.environmentInput.trimEnd().split("\n")).toEqual([ + "OPENSHELL_SANDBOX=alpha", + "OPENSHELL_SANDBOX_COMMAND=node agent.js --serve", + "OPENSHELL_SANDBOX_TOKEN_FILE=/etc/openshell/auth/sandbox.jwt", + "OPENSHELL_TLS_CA=/etc/openshell/tls/client/ca.crt", + "OPENSHELL_TLS_CERT=/etc/openshell/tls/client/tls.crt", + "OPENSHELL_TLS_KEY=/etc/openshell/tls/client/tls.key", + "USER_VALUE=preserved", + ]); + expect(plan.args.some((argument) => argument.includes("USER_VALUE"))).toBe(false); + expect(plan.args).toContain("--http-proxy=false"); + expect(flagValues(plan.args, "--label")).toEqual([ + "custom.label=preserved", + "openshell.managed=true", + "openshell.sandbox-id=sandbox-id", + "openshell.sandbox-name=alpha", + "openshell.sandbox-namespace=", + ]); + expect(flagValues(plan.args, "--mount")).toEqual( + expect.arrayContaining([ + "type=volume,source=openshell-sandbox-sandbox-id-workspace,destination=/sandbox,ro=false", + `type=image,source=sha256:${SUPERVISOR_IMAGE_ID},destination=/opt/openshell/bin,rw=false`, + "type=bind,source=/run/openshell/tls/tls.key,destination=/etc/openshell/tls/client/tls.key,ro=true,bind-propagation=rprivate", + ]), + ); + expect(flagValues(plan.args, "--secret")).toEqual([ + `${SECRET_ID},type=mount,target=/etc/openshell/auth/sandbox.jwt,uid=0,gid=0,mode=0400`, + ]); + expect(flagValues(plan.args, "--ulimit")).toEqual(["memlock=-1:-1", "nofile=65536:65536"]); + expect(flagValues(plan.args, "--network")).toEqual([NETWORK_ID]); + expect(flagValues(plan.args, "--publish")).toEqual(["127.0.0.1:41022:22/tcp"]); + expect(flagValues(plan.args, "--health-cmd")).toEqual([ + '["CMD","/opt/openshell/bin/openshell-sandbox","__healthcheck"]', + ]); + expect(flagValues(plan.args, "--tmpfs")).toEqual(["/run/netns:rw,nosuid,nodev"]); + expect(flagValues(plan.args, "--cap-add")).toEqual(["CAP_SYS_ADMIN", "CAP_NET_ADMIN"]); + expect(flagValues(plan.args, "--security-opt")).toEqual([ + "no-new-privileges", + "seccomp=unconfined", + ]); + expect(flagValues(plan.args, "--cpu-period")).toEqual(["100000"]); + expect(flagValues(plan.args, "--cpu-quota")).toEqual(["200000"]); + expect(flagValues(plan.args, "--memory")).toEqual(["4294967296"]); + expect(flagValues(plan.args, "--pids-limit")).toEqual(["256"]); + expect(flagValues(plan.args, "--add-host")).toEqual([ + "host.containers.internal:host-gateway", + "host.openshell.internal:host-gateway", + ]); + }); + + it.each([ + { + label: "short container IDs", + mutate: (value: Record) => { + value.Id = "abc123"; + }, + message: "full immutable SHA-256", + }, + { + label: "missing exact managed label", + mutate: (value: Record) => { + (value.Config as Record).Labels = { + "openshell.managed": "true", + "openshell.sandbox-name": "other", + }; + }, + message: "missing exact label", + }, + { + label: "multiple inspect records", + mutate: (value: Record) => { + value.__multiple = true; + }, + message: "exactly one", + }, + ])("rejects $label before planning", ({ mutate, message }) => { + const value = validInspect(); + mutate(value); + const payload = value.__multiple ? [value, value] : [value]; + expect(() => + parsePodmanManagedSandboxInspect(JSON.stringify(payload), { + containerId: String(value.Id), + name: CONTAINER_NAME, + sandboxName: SANDBOX_NAME, + }), + ).toThrow(message); + }); + + it.each([ + "openshell.sandbox-id", + "openshell.sandbox-namespace", + ])("requires the exact v0.0.85 %s identity label", (label) => { + const value = validInspect(); + delete ((value.Config as Record).Labels as Record)[label]; + expect(() => parsedInspect(value)).toThrow(`missing exact label ${label}`); + }); + + it.each([ + { + label: "workspace mount", + mutate: (value: Record) => { + value.Mounts = (value.Mounts as unknown[]).slice(1); + }, + message: "workspace", + }, + { + label: "workspace identity", + mutate: (value: Record) => { + const workspace = (value.Mounts as Array>).find( + (mount) => mount.Destination === "/sandbox", + ); + if (workspace) workspace.Name = "openshell-sandbox-other-workspace"; + }, + message: "workspace", + }, + { + label: "supervisor image mount", + mutate: (value: Record) => { + value.Mounts = (value.Mounts as Array>).filter( + (mount) => mount.Type !== "image", + ); + }, + message: "supervisor image", + }, + { + label: "TLS bind evidence", + mutate: (value: Record) => { + value.Mounts = (value.Mounts as Array>).filter( + (mount) => mount.Destination !== "/etc/openshell/tls/client/tls.key", + ); + }, + message: "TLS mount", + }, + { + label: "token secret evidence", + mutate: (value: Record) => { + (value.Config as Record).Secrets = []; + }, + message: "token secret", + }, + { + label: "token secret sandbox identity", + mutate: (value: Record) => { + const secrets = (value.Config as Record).Secrets as Array< + Record + >; + if (secrets[0]) secrets[0].Name = "openshell-token-other-id"; + }, + message: "exact OpenShell sandbox ID", + }, + { + label: "deterministic secret target", + mutate: (value: Record) => { + (value.Config as Record).Secrets = [ + { Name: "arbitrary-secret", ID: SECRET_ID, UID: 0, GID: 0, Mode: 0o400 }, + ]; + }, + message: "no deterministic OpenShell target", + }, + { + label: "single network", + mutate: (value: Record) => { + const settings = value.NetworkSettings as Record; + settings.Networks = { openshell: {}, second: {} }; + }, + message: "exactly one attached", + }, + { + label: "immutable network identity", + mutate: (value: Record) => { + const settings = value.NetworkSettings as Record; + const networks = settings.Networks as Record>; + delete networks.openshell?.NetworkID; + }, + message: "network 'openshell' ID", + }, + { + label: "unsupported device inference", + mutate: (value: Record) => { + (value.HostConfig as Record).Devices = [{ PathOnHost: "/dev/nvidia0" }]; + }, + message: "devices cannot be reproduced", + }, + { + label: "redacted env secrets", + mutate: (value: Record) => { + (value.Config as Record).Env = ["SECRET=*******"]; + }, + message: "env-type secrets", + }, + { + label: "non-neutral memory swappiness", + mutate: (value: Record) => { + (value.HostConfig as Record).MemorySwappiness = 1; + }, + message: "non-neutral memory swappiness", + }, + ])("fails closed without faithful $label", ({ mutate, message }) => { + const value = validInspect(); + mutate(value); + const inspect = parsedInspect(value); + expect(() => + buildPodmanManagedSandboxCreatePlan({ + command: ["node", "agent.js"], + imagePins: { [SUPERVISOR_IMAGE]: SUPERVISOR_IMAGE_ID }, + inspect, + }), + ).toThrow(message); + }); + + it("requires every image mount to be resolved to a full immutable image ID", () => { + expect(() => + buildPodmanManagedSandboxCreatePlan({ + command: ["node", "agent.js"], + imagePins: {}, + inspect: parsedInspect(), + }), + ).toThrow("was not pinned"); + }); + + it.each([-1, 0])("accepts Podman v5 rootless cgroup-v2 neutral swappiness %s", (value) => { + const raw = validInspect(); + (raw.HostConfig as Record).MemorySwappiness = value; + expect(() => + buildPodmanManagedSandboxCreatePlan({ + command: ["node", "agent.js"], + imagePins: { [SUPERVISOR_IMAGE]: SUPERVISOR_IMAGE_ID }, + inspect: parsedInspect(raw), + }), + ).not.toThrow(); + }); +}); + +type FakePodmanOptions = { + discovery?: "normal" | "multiple" | "none"; + fail?: { + action: string; + afterEffect?: boolean; + occurrence?: number; + status: number | null; + stderr?: string; + }; + dropReplacementUlimits?: boolean; + failedCreateLeavesUnowned?: boolean; + mutateOldUserOnSecondInspect?: boolean; + omitCidFile?: boolean; + oldRestartsWhenReplacementStarts?: boolean; + oldExitsAfterStart?: boolean; + replacementCommandValue?: string; + replacementUser?: string; + throwCreateMessage?: string; + throwCreateAfterEffectMessage?: string; +}; + +function fakePodman(options: FakePodmanOptions = {}) { + let watcherStopped = false; + let oldExists = true; + let oldName = CONTAINER_NAME; + let oldRunning = true; + let backupExists = false; + let backupRunning = false; + let backupName = ""; + let restoredExists = false; + let restoredRunning = false; + let newExists = false; + let newRunning = false; + let backupEnvironment: string[] | null = null; + let backupUlimits: Array<{ Hard: number; Name: string; Soft: number }> | null = null; + let backupUser = "0:0"; + let replacementEnvironment: string[] | null = null; + let replacementUlimits: Array<{ Hard: number; Name: string; Soft: number }> | null = null; + let replacementUser = options.replacementUser ?? "0:0"; + let oldInspectCount = 0; + const calls: Array<{ + args: readonly string[]; + command: string; + input?: string; + watcherStopped: boolean; + }> = []; + const failureCounts = new Map(); + const fail = (action: string) => { + const occurrence = (failureCounts.get(action) ?? 0) + 1; + failureCounts.set(action, occurrence); + return options.fail?.action === action && + (options.fail.occurrence === undefined || options.fail.occurrence === occurrence) + ? { status: options.fail.status, stderr: options.fail.stderr ?? `${action} failed` } + : null; + }; + const inspect = (id: string) => { + const old = id === OLD_ID; + const backup = id === BACKUP_ID; + const restored = id === RESTORED_ID; + const raw = validInspect({ + Id: id, + Name: old ? oldName : backup ? backupName : CONTAINER_NAME, + State: { + Running: old + ? oldRunning + : backup + ? backupRunning + : restored + ? restoredRunning + : newRunning, + }, + }); + if (old && options.mutateOldUserOnSecondInspect && oldInspectCount >= 2) { + (raw.Config as Record).User = "1000:1000"; + } else if (backup) { + const config = raw.Config as Record; + config.Labels = { "custom.label": "preserved" }; + config.User = backupUser; + if (backupEnvironment) config.Env = backupEnvironment; + if (backupUlimits) (raw.HostConfig as Record).Ulimits = backupUlimits; + } else if (!old) { + const config = raw.Config as Record; + config.User = restored ? "0:0" : replacementUser; + if (restored && backupEnvironment) { + config.Env = backupEnvironment; + } else if (replacementEnvironment) { + config.Env = replacementEnvironment.map((entry) => + options.replacementCommandValue !== undefined && + entry.startsWith("OPENSHELL_SANDBOX_COMMAND=") + ? `OPENSHELL_SANDBOX_COMMAND=${options.replacementCommandValue}` + : entry, + ); + } + if (restored && backupUlimits) { + (raw.HostConfig as Record).Ulimits = backupUlimits; + } else if (replacementUlimits && !options.dropReplacementUlimits) { + (raw.HostConfig as Record).Ulimits = replacementUlimits; + } + } + const settings = raw.NetworkSettings as Record; + const networks = settings.Networks as Record>; + networks.openshell = { + ...(networks.openshell ?? {}), + Aliases: [id, id.slice(0, 12), old ? oldName : backup ? backupName : CONTAINER_NAME], + }; + return JSON.stringify([raw]); + }; + const create = (podmanArgs: readonly string[], inputValue: unknown) => { + const name = flagValues(podmanArgs, "--name")[0] ?? ""; + const input = typeof inputValue === "string" ? inputValue.trimEnd().split("\n") : []; + const isBackup = name !== CONTAINER_NAME; + const isRestore = + !isBackup && input.some((entry) => entry === "OPENSHELL_SANDBOX_COMMAND=sleep infinity"); + if (isRestore && newExists) return { status: 125, stderr: "name is in use" }; + if (!isBackup && !isRestore && options.throwCreateMessage) { + throw new Error(options.throwCreateMessage); + } + const failed = isBackup || isRestore ? null : fail("create-new"); + if (failed && !options.fail?.afterEffect) { + if (options.failedCreateLeavesUnowned) { + newExists = true; + newRunning = true; + } + return failed; + } + const cidFile = flagValues(podmanArgs, "--cidfile")[0]; + if (!cidFile) throw new Error("Fake Podman create requires --cidfile."); + const ulimits = flagValues(podmanArgs, "--ulimit").map((entry) => { + const [name = "", limits = ""] = entry.split("=", 2); + const [soft = "", hard = ""] = limits.split(":", 2); + return { Hard: Number(hard), Name: name, Soft: Number(soft) }; + }); + const createdId = isBackup ? BACKUP_ID : isRestore ? RESTORED_ID : NEW_ID; + if (isBackup) { + backupExists = true; + backupName = name; + backupEnvironment = input; + backupUlimits = ulimits; + } else if (isRestore) { + restoredExists = true; + } else { + replacementEnvironment = input; + replacementUlimits = ulimits; + newExists = true; + } + if (!options.omitCidFile || isBackup || isRestore) { + writeFileSync(cidFile, `${createdId}\n`, "utf-8"); + } + if (!isBackup && !isRestore && options.throwCreateAfterEffectMessage) { + throw new Error(options.throwCreateAfterEffectMessage); + } + return failed + ? { ...failed, stdout: `${createdId}\n` } + : { status: 0, stdout: `${createdId}\n` }; + }; + const run: RunQualifiedPodmanCommand = vi.fn((command, args, spawnOptions) => { + calls.push({ + command, + args: [...args], + input: typeof spawnOptions.input === "string" ? spawnOptions.input : undefined, + watcherStopped, + }); + expect(command).toBe("podman"); + expect(args.slice(0, 2)).toEqual(["--url", `unix://${SOCKET_PATH}`]); + const podmanArgs = args.slice(2); + if (podmanArgs[0] === "ps") { + const failed = fail("discover"); + if (failed) return failed; + if (options.discovery === "none") return { status: 0, stdout: "" }; + const visible = newExists + ? `${NEW_ID}\t${CONTAINER_NAME}` + : restoredExists + ? `${RESTORED_ID}\t${CONTAINER_NAME}` + : oldExists && oldName === CONTAINER_NAME + ? `${OLD_ID}\t${oldName}` + : ""; + return { + status: 0, + stdout: + options.discovery === "multiple" + ? `${visible}\n${"e".repeat(64)}\t${CONTAINER_NAME}` + : `${visible}\n`, + }; + } + if (podmanArgs[0] === "info") { + const failed = fail("info"); + if (failed) return failed; + return { + status: 0, + stdout: JSON.stringify({ + host: { security: { rootless: true } }, + store: { + graphRoot: "/home/test/.local/share/containers/storage", + runRoot: "/run/user/1000/containers", + }, + }), + }; + } + if (podmanArgs[0] === "container" && podmanArgs[1] === "inspect") { + const id = String(podmanArgs[2]); + const failed = fail(id === NEW_ID ? "inspect-new" : id === OLD_ID ? "inspect-old" : ""); + if (failed) return failed; + if ( + (id === OLD_ID && !oldExists) || + (id === BACKUP_ID && !backupExists) || + (id === RESTORED_ID && !restoredExists) || + (id === NEW_ID && !newExists) + ) { + return { status: 125, stderr: "no such container" }; + } + if (![OLD_ID, BACKUP_ID, RESTORED_ID, NEW_ID].includes(id)) { + return { status: 125, stderr: "no such container" }; + } + if (id === OLD_ID) oldInspectCount += 1; + return { status: 0, stdout: inspect(id) }; + } + if (podmanArgs[0] === "container" && podmanArgs[1] === "exists") { + const id = String(podmanArgs[2]); + return { + status: + (id === OLD_ID && oldExists) || + (id === BACKUP_ID && backupExists) || + (id === RESTORED_ID && restoredExists) || + (id === NEW_ID && newExists) + ? 0 + : 1, + }; + } + if (podmanArgs[0] === "image" && podmanArgs[1] === "inspect") { + const failed = fail("pin-image"); + return failed ?? { status: 0, stdout: `sha256:${SUPERVISOR_IMAGE_ID}\n` }; + } + if (podmanArgs[0] === "stop") { + const id = String(podmanArgs[1]); + const failed = fail( + id === OLD_ID + ? "stop-old" + : id === BACKUP_ID + ? "stop-backup" + : id === NEW_ID + ? "stop-new" + : "stop-restored", + ); + if (failed && !options.fail?.afterEffect) return failed; + if (id === OLD_ID) oldRunning = false; + else if (id === BACKUP_ID) backupRunning = false; + else if (id === RESTORED_ID) restoredRunning = false; + else newRunning = false; + return failed ?? { status: 0 }; + } + if (podmanArgs[0] === "create") { + return create(podmanArgs, spawnOptions.input); + } + if (podmanArgs[0] === "start") { + const id = String(podmanArgs[1]); + const failed = fail( + id === NEW_ID ? "start-new" : id === RESTORED_ID ? "start-old" : "start-backup", + ); + if (failed && !options.fail?.afterEffect) return failed; + if (id === OLD_ID) oldRunning = !options.oldExitsAfterStart; + else if (id === RESTORED_ID) restoredRunning = !options.oldExitsAfterStart; + else if (id === BACKUP_ID) backupRunning = true; + else { + newRunning = true; + if (options.oldRestartsWhenReplacementStarts) backupRunning = true; + } + return failed ?? { status: 0 }; + } + if (podmanArgs[0] === "rm") { + const id = String(podmanArgs[1]); + const failed = fail( + id === BACKUP_ID ? "remove-backup" : id === NEW_ID ? "remove-new" : "remove-original", + ); + if (failed && !options.fail?.afterEffect) return failed; + if ( + (id === OLD_ID && oldRunning) || + (id === BACKUP_ID && backupRunning) || + (id === RESTORED_ID && restoredRunning) || + (id === NEW_ID && newRunning) + ) { + return { status: 125, stderr: "container is running" }; + } + if (id === NEW_ID) newExists = false; + else if (id === BACKUP_ID) backupExists = false; + else if (id === RESTORED_ID) restoredExists = false; + else oldExists = false; + return failed ?? { status: 0 }; + } + throw new Error(`Unexpected fake Podman call: ${podmanArgs.join(" ")}`); + }); + return { + calls, + run, + setReplacementUser: (value: string) => { + replacementUser = value; + }, + setBackupUser: (value: string) => { + backupUser = value; + }, + setOldRunning: (value: boolean) => { + if (backupExists) backupRunning = value; + else if (restoredExists) restoredRunning = value; + else oldRunning = value; + }, + setWatcherStopped: (value: boolean) => { + watcherStopped = value; + }, + removeReplacement: () => { + newExists = false; + newRunning = false; + }, + state: () => ({ + backupExists, + backupName, + backupRunning, + newExists, + newRunning, + oldExists, + oldName, + oldRunning, + restoredExists, + restoredRunning, + watcherStopped, + }), + }; +} + +function watcherController(fake: ReturnType) { + return createPodmanOpenShellWatcherController({ + stopAndProve: () => { + fake.setWatcherStopped(true); + return { stopped: true }; + }, + assertStopped: () => { + if (!fake.state().watcherStopped) throw new Error("watcher is not stopped"); + }, + resumeAndProve: () => { + fake.setWatcherStopped(false); + }, + }); +} + +function recreateWith(fake: ReturnType, assertSocketAuthority = vi.fn()) { + return recreatePodmanManagedSandbox( + { + command: ["node", "agent.js"], + requiredUlimits: [{ name: "nofile", soft: 65_536, hard: 65_536 }], + sandboxName: SANDBOX_NAME, + socketAuthority: SOCKET_AUTHORITY, + socketPath: SOCKET_PATH, + watcherController: watcherController(fake), + }, + { + assertSocketAuthority, + now: () => new Date(1_700_000_000_000), + run: fake.run, + }, + ); +} + +describe("Podman managed sandbox recreation lifecycle", () => { + it("rejects a mismatched socket authority before the first Podman discovery call", () => { + const run = vi.fn(); + expect(() => + findPodmanManagedSandboxContainerIds("/run/user/1000/podman/replaced.sock", SANDBOX_NAME, { + assertSocketAuthority: vi.fn(), + run, + socketAuthority: SOCKET_AUTHORITY, + }), + ).toThrow("socket authority does not match"); + expect(run).not.toHaveBeenCalled(); + }); + + it("requires a watcher controller before any Podman operation", () => { + const fake = fakePodman(); + expect(() => + recreatePodmanManagedSandbox( + { + command: ["node", "agent.js"], + sandboxName: SANDBOX_NAME, + socketAuthority: SOCKET_AUTHORITY, + socketPath: SOCKET_PATH, + watcherController: undefined as never, + }, + { assertSocketAuthority: vi.fn(), run: fake.run }, + ), + ).toThrow("watcher controller"); + expect(fake.calls).toHaveLength(0); + }); + + it("qualifies every operation to the exact socket and starts a pinned replacement", () => { + const fake = fakePodman(); + const assertSocketAuthority = vi.fn(); + const transaction = recreateWith(fake, assertSocketAuthority); + expect(transaction).toMatchObject({ + applied: true, + backupContainerId: BACKUP_ID, + backupContainerName: "openshell-sandbox-alpha-nemoclaw-backup-1700000000000", + backupSemanticDigest: expect.stringMatching(/^[0-9a-f]{64}$/u), + command: ["node", "agent.js"], + driverName: "podman", + immutableImage: `sha256:${ROOT_IMAGE_ID}`, + newContainerId: NEW_ID, + oldContainerId: OLD_ID, + originalLabels: { + "custom.label": "preserved", + "openshell.managed": "true", + "openshell.sandbox-id": "sandbox-id", + "openshell.sandbox-name": SANDBOX_NAME, + "openshell.sandbox-namespace": "", + }, + originalName: CONTAINER_NAME, + originalSemanticDigest: expect.stringMatching(/^[0-9a-f]{64}$/u), + requiredUlimits: [{ name: "nofile", soft: 65_536, hard: 65_536 }], + sandboxName: SANDBOX_NAME, + semanticDigest: expect.stringMatching(/^[0-9a-f]{64}$/u), + socketPath: SOCKET_PATH, + }); + expect(fake.state()).toMatchObject({ + backupExists: true, + backupName: transaction.backupContainerName, + backupRunning: false, + newExists: true, + newRunning: true, + oldExists: false, + oldRunning: false, + watcherStopped: false, + }); + expect(fake.calls.every((call) => call.args[0] === "--url")).toBe(true); + expect(fake.calls.every((call) => call.args[1] === `unix://${SOCKET_PATH}`)).toBe(true); + expect(assertSocketAuthority).toHaveBeenCalledTimes(fake.calls.length + 1); + expect(fake.calls.map((call) => call.command)).not.toContain("docker"); + expect(fake.calls.map((call) => call.args.slice(2, 4))).toContainEqual(["start", NEW_ID]); + const createCall = fake.calls.find( + (call) => + call.args[2] === "create" && flagValues(call.args.slice(2), "--name")[0] === CONTAINER_NAME, + ); + expect(createCall?.args.some((argument) => argument.includes("USER_VALUE=preserved"))).toBe( + false, + ); + expect(createCall?.input).toContain("USER_VALUE=preserved"); + const backupCreateCall = fake.calls.find( + (call) => + call.args[2] === "create" && + flagValues(call.args.slice(2), "--name")[0] === transaction.backupContainerName, + ); + expect(flagValues(backupCreateCall?.args.slice(2) ?? [], "--label")).toEqual([ + "custom.label=preserved", + ]); + expect(backupCreateCall?.watcherStopped).toBe(false); + expect( + flagValues(backupCreateCall?.args.slice(2) ?? [], "--label").some((label) => + label.startsWith("openshell."), + ), + ).toBe(false); + expect( + fake.calls.find((call) => call.args[2] === "rm" && call.args[3] === OLD_ID)?.watcherStopped, + ).toBe(true); + }); + + it("does not send the first destructive command after socket authority changes", () => { + const baseline = fakePodman(); + recreateWith(baseline); + const stopIndex = baseline.calls.findIndex( + (call) => call.args[2] === "stop" && call.args[3] === OLD_ID, + ); + expect(stopIndex).toBeGreaterThan(0); + + const fake = fakePodman(); + let validations = 0; + expect(() => + recreateWith( + fake, + vi.fn(() => { + validations += 1; + if (validations >= stopIndex + 2) { + throw new Error("socket path replaced"); + } + }), + ), + ).toThrow(); + + expect(fake.calls.some((call) => call.args[2] === "stop" && call.args[3] === OLD_ID)).toBe( + false, + ); + expect(fake.state()).toMatchObject({ + oldExists: true, + oldRunning: true, + watcherStopped: false, + }); + }); + + it("recovers the watcher and leaves the managed original untouched when stop proof fails", () => { + const fake = fakePodman(); + const controller = createPodmanOpenShellWatcherController({ + stopAndProve: () => { + fake.setWatcherStopped(true); + return { stopped: true }; + }, + assertStopped: () => { + throw new Error("stop receipt rejected"); + }, + resumeAndProve: () => { + fake.setWatcherStopped(false); + }, + }); + expect(() => + recreatePodmanManagedSandbox( + { + command: ["node", "agent.js"], + sandboxName: SANDBOX_NAME, + socketAuthority: SOCKET_AUTHORITY, + socketPath: SOCKET_PATH, + watcherController: controller, + }, + { + assertSocketAuthority: vi.fn(), + now: () => new Date(1_700_000_000_000), + run: fake.run, + }, + ), + ).toThrow("stop proof failed"); + expect(fake.calls.some((call) => call.args[2] === "rm" && call.args[3] === OLD_ID)).toBe(false); + expect(fake.state()).toMatchObject({ + backupExists: false, + oldExists: true, + oldRunning: true, + watcherStopped: false, + }); + }); + + it("recovers before mutation when the exact rootless Podman API proof fails", () => { + const fake = fakePodman({ fail: { action: "info", status: 125 } }); + let error: unknown; + try { + recreateWith(fake); + } catch (caught) { + error = caught; + } + expect(error).toMatchObject({ rolledBack: true }); + expect(String(error)).toContain("cutover preflight failed"); + expect(fake.calls.some((call) => call.args[2] === "rm" && call.args[3] === OLD_ID)).toBe(false); + expect(fake.state()).toMatchObject({ + backupExists: false, + oldExists: true, + oldRunning: true, + watcherStopped: false, + }); + }); + + it.each([ + { occurrence: 3, restoredContainer: "old" }, + { occurrence: 4, restoredContainer: "recreated" }, + ])("restores service when Podman proof $occurrence fails after mutation begins", ({ + occurrence, + restoredContainer, + }) => { + const fake = fakePodman({ + fail: { action: "info", occurrence, status: 125 }, + }); + let error: unknown; + try { + recreateWith(fake); + } catch (caught) { + error = caught; + } + expect(error).toMatchObject({ rolledBack: true }); + expect(fake.state()).toMatchObject({ + backupExists: false, + oldRunning: restoredContainer === "old", + restoredRunning: restoredContainer === "recreated", + watcherStopped: false, + }); + }); + + it("rolls back under the held lease when the first watcher restart attempt fails", () => { + const fake = fakePodman(); + let resumeAttempts = 0; + const controller = createPodmanOpenShellWatcherController({ + stopAndProve: () => { + fake.setWatcherStopped(true); + return { stopped: true }; + }, + assertStopped: () => { + if (!fake.state().watcherStopped) throw new Error("watcher is not stopped"); + }, + resumeAndProve: () => { + resumeAttempts += 1; + if (resumeAttempts === 1) throw new Error("watcher restart unavailable"); + fake.setWatcherStopped(false); + }, + }); + let error: unknown; + try { + recreatePodmanManagedSandbox( + { + command: ["node", "agent.js"], + sandboxName: SANDBOX_NAME, + socketAuthority: SOCKET_AUTHORITY, + socketPath: SOCKET_PATH, + watcherController: controller, + }, + { + assertSocketAuthority: vi.fn(), + now: () => new Date(1_700_000_000_000), + run: fake.run, + }, + ); + } catch (caught) { + error = caught; + } + expect(error).toMatchObject({ rolledBack: true }); + expect(resumeAttempts).toBe(2); + expect( + fake.calls.find((call) => call.args[2] === "rm" && call.args[3] === NEW_ID)?.watcherStopped, + ).toBe(true); + expect(fake.state()).toMatchObject({ + backupExists: false, + newExists: false, + restoredExists: true, + restoredRunning: true, + watcherStopped: false, + }); + }); + + it("retains rollback evidence and fails closed when watcher restart cannot be proven", () => { + const fake = fakePodman(); + const controller = createPodmanOpenShellWatcherController({ + stopAndProve: () => { + fake.setWatcherStopped(true); + return { stopped: true }; + }, + assertStopped: () => { + if (!fake.state().watcherStopped) throw new Error("watcher is not stopped"); + }, + resumeAndProve: () => { + throw new Error("watcher restart unavailable"); + }, + }); + let error: unknown; + try { + recreatePodmanManagedSandbox( + { + command: ["node", "agent.js"], + sandboxName: SANDBOX_NAME, + socketAuthority: SOCKET_AUTHORITY, + socketPath: SOCKET_PATH, + watcherController: controller, + }, + { + assertSocketAuthority: vi.fn(), + now: () => new Date(1_700_000_000_000), + run: fake.run, + }, + ); + } catch (caught) { + error = caught; + } + expect(error).toMatchObject({ rolledBack: false }); + expect(fake.state()).toMatchObject({ + backupExists: true, + newExists: false, + restoredExists: true, + restoredRunning: true, + watcherStopped: true, + }); + }); + + it.each([null, 1])("does not mutate when stop leaves the original running (%s)", (status) => { + const fake = fakePodman({ fail: { action: "stop-old", status } }); + expect(() => recreateWith(fake)).toThrow("remains running"); + expect(fake.state()).toMatchObject({ + backupExists: false, + oldExists: true, + oldName: CONTAINER_NAME, + oldRunning: true, + watcherStopped: false, + }); + expect(fake.calls.some((call) => call.args[2] === "rename")).toBe(false); + }); + + it.each([ + "stop-old", + "remove-original", + "start-new", + ])("reconciles an unavailable %s status from exact inspect state", (action) => { + const fake = fakePodman({ fail: { action, afterEffect: true, status: null } }); + expect(recreateWith(fake)).toMatchObject({ applied: true, newContainerId: NEW_ID }); + expect(fake.state()).toMatchObject({ + backupExists: true, + backupName: "openshell-sandbox-alpha-nemoclaw-backup-1700000000000", + newRunning: true, + oldExists: false, + oldRunning: false, + watcherStopped: false, + }); + }); + + it("restores the exact original when stop-state inspection is temporarily unavailable", () => { + const fake = fakePodman({ + fail: { action: "inspect-old", occurrence: 3, status: null }, + }); + let error: unknown; + try { + recreateWith(fake); + } catch (caught) { + error = caught; + } + expect(error).toMatchObject({ rolledBack: true }); + expect(fake.state()).toMatchObject({ + oldExists: true, + oldRunning: true, + watcherStopped: false, + }); + expect(fake.calls.some((call) => call.args[2] === "rename")).toBe(false); + }); + + it("rejects an in-place semantic change on the pinned original before mutation", () => { + const fake = fakePodman({ mutateOldUserOnSecondInspect: true }); + expect(() => recreateWith(fake)).toThrow("pinned recreation semantics"); + expect(fake.calls.some((call) => call.args[2] === "stop")).toBe(false); + expect(fake.state()).toMatchObject({ + oldName: CONTAINER_NAME, + oldRunning: true, + }); + }); + + it.each([ + { + label: "container user", + options: { replacementUser: "1000:1000" }, + }, + { + label: "startup command environment", + options: { replacementCommandValue: "wrong command" }, + }, + { + label: "required ulimit", + options: { dropReplacementUlimits: true }, + }, + ])("rolls back before start when Podman changes the replacement $label", ({ options }) => { + const fake = fakePodman(options); + let error: unknown; + try { + recreateWith(fake); + } catch (caught) { + error = caught; + } + expect(error).toMatchObject({ rolledBack: true }); + expect(fake.calls.map((call) => call.args.slice(2, 4))).not.toContainEqual(["start", NEW_ID]); + expect(fake.state()).toMatchObject({ + newExists: false, + restoredExists: true, + restoredRunning: true, + watcherStopped: false, + }); + }); + + it("removes a cidfile-owned replacement and redacts env values when create throws", () => { + const fake = fakePodman({ + throwCreateAfterEffectMessage: "create wrapper exposed USER_VALUE=preserved", + }); + let error: unknown; + try { + recreateWith(fake); + } catch (caught) { + error = caught; + } + expect(error).toMatchObject({ rolledBack: true }); + expect(String(error)).not.toContain("preserved"); + expect(String(error)).toContain("[REDACTED]"); + expect(fake.state()).toMatchObject({ + newExists: false, + restoredExists: true, + restoredRunning: true, + watcherStopped: false, + }); + }); + + it("uses unambiguous stdout ownership when create status is unavailable", () => { + const fake = fakePodman({ + fail: { action: "create-new", afterEffect: true, status: null }, + omitCidFile: true, + }); + let error: unknown; + try { + recreateWith(fake); + } catch (caught) { + error = caught; + } + expect(error).toMatchObject({ rolledBack: true }); + expect(fake.state()).toMatchObject({ + newExists: false, + restoredExists: true, + restoredRunning: true, + watcherStopped: false, + }); + }); + + it("never deletes a concurrent exact-name container without create ownership evidence", () => { + const fake = fakePodman({ + fail: { action: "create-new", status: 125 }, + failedCreateLeavesUnowned: true, + }); + let error: unknown; + try { + recreateWith(fake); + } catch (caught) { + error = caught; + } + expect(error).toMatchObject({ rolledBack: false }); + expect( + fake.calls.some( + (call) => ["stop", "rm"].includes(String(call.args[2])) && call.args[3] === NEW_ID, + ), + ).toBe(false); + expect(fake.state()).toMatchObject({ + backupExists: true, + backupName: "openshell-sandbox-alpha-nemoclaw-backup-1700000000000", + newExists: true, + newRunning: true, + oldExists: false, + restoredExists: false, + watcherStopped: false, + }); + }); + + it("does not report rollback complete when the restored original exits immediately", () => { + const fake = fakePodman({ + fail: { action: "create-new", status: 125 }, + oldExitsAfterStart: true, + }); + let error: unknown; + try { + recreateWith(fake); + } catch (caught) { + error = caught; + } + expect(error).toMatchObject({ rolledBack: false }); + expect(fake.state()).toMatchObject({ + restoredExists: true, + restoredRunning: false, + watcherStopped: false, + }); + }); + + it("rolls back if the pinned backup restarts while the replacement is verified", () => { + const fake = fakePodman({ oldRestartsWhenReplacementStarts: true }); + let error: unknown; + try { + recreateWith(fake); + } catch (caught) { + error = caught; + } + expect(error).toMatchObject({ rolledBack: true }); + expect(fake.state()).toMatchObject({ + newExists: false, + restoredExists: true, + restoredRunning: true, + watcherStopped: false, + }); + }); + + it("restores the original when replacement startup fails", () => { + const fake = fakePodman({ fail: { action: "start-new", status: 125 } }); + let error: unknown; + try { + recreateWith(fake); + } catch (caught) { + error = caught; + } + expect(error).toMatchObject({ rolledBack: true }); + expect(fake.state()).toMatchObject({ + backupExists: false, + newExists: false, + newRunning: false, + oldExists: false, + restoredExists: true, + restoredRunning: true, + watcherStopped: false, + }); + }); + + it("reports a rollback failure at the first non-zero cleanup gate", () => { + const fake = fakePodman({ fail: { action: "stop-new", status: null } }); + const transaction = recreateWith(fake); + expect( + rollbackPodmanManagedSandbox( + { transaction, watcherController: watcherController(fake) }, + { assertSocketAuthority: vi.fn(), run: fake.run }, + ), + ).toEqual({ + originalRecreated: false, + originalStarted: false, + replacementRemoved: false, + rolledBack: false, + }); + expect(fake.state()).toMatchObject({ + backupExists: true, + newExists: true, + watcherStopped: false, + }); + }); + + it("restores the backup when the pinned replacement was already removed", () => { + const fake = fakePodman(); + const transaction = recreateWith(fake); + fake.removeReplacement(); + expect( + rollbackPodmanManagedSandbox( + { transaction, watcherController: watcherController(fake) }, + { assertSocketAuthority: vi.fn(), run: fake.run }, + ), + ).toEqual({ + originalRecreated: true, + originalStarted: true, + replacementRemoved: true, + rolledBack: true, + }); + expect(fake.state()).toMatchObject({ + backupExists: false, + newExists: false, + restoredExists: true, + restoredRunning: true, + watcherStopped: false, + }); + }); + + it("resumes the watcher without removing the replacement when backup proof fails", () => { + const fake = fakePodman(); + const transaction = recreateWith(fake); + fake.setBackupUser("1000:1000"); + let error: unknown; + try { + rollbackPodmanManagedSandbox( + { transaction, watcherController: watcherController(fake) }, + { assertSocketAuthority: vi.fn(), run: fake.run }, + ); + } catch (caught) { + error = caught; + } + expect(error).toMatchObject({ rolledBack: false }); + expect(String(error)).toContain("rollback failed"); + expect(fake.calls.map((call) => call.args.slice(2))).not.toContainEqual(["rm", NEW_ID]); + expect(fake.state()).toMatchObject({ + backupExists: true, + newExists: true, + newRunning: true, + restoredExists: false, + watcherStopped: false, + }); + }); + + it("refuses rollback without a proven watcher controller", () => { + const fake = fakePodman(); + const transaction = recreateWith(fake); + const callsBefore = fake.calls.length; + expect(() => + rollbackPodmanManagedSandbox( + { transaction, watcherController: undefined as never }, + { assertSocketAuthority: vi.fn(), run: fake.run }, + ), + ).toThrow("watcher controller"); + expect(fake.calls).toHaveLength(callsBefore); + }); + + it("rolls back instead of deleting the backup when the replacement is not ready", () => { + const fake = fakePodman(); + const transaction = recreateWith(fake); + expect( + finalizePodmanManagedSandbox( + { + replacementReady: false, + transaction, + watcherController: watcherController(fake), + }, + { assertSocketAuthority: vi.fn(), run: fake.run }, + ), + ).toEqual({ backupRemoved: false, rolledBack: true }); + expect(fake.state()).toMatchObject({ + backupExists: false, + newExists: false, + newRunning: false, + restoredExists: true, + restoredRunning: true, + watcherStopped: false, + }); + }); + + it("removes the pinned backup only after readiness and reports the exact rm status", () => { + const green = fakePodman(); + const greenTransaction = recreateWith(green); + expect( + finalizePodmanManagedSandbox( + { + replacementReady: true, + transaction: greenTransaction, + }, + { assertSocketAuthority: vi.fn(), run: green.run }, + ), + ).toEqual({ backupRemoved: true, rolledBack: false }); + expect(green.calls.map((call) => call.args.slice(2))).toContainEqual(["rm", BACKUP_ID]); + expect( + green.calls.find((call) => call.args[2] === "rm" && call.args[3] === BACKUP_ID) + ?.watcherStopped, + ).toBe(false); + expect(green.calls.at(-1)?.args.slice(2)).toEqual(["container", "exists", BACKUP_ID]); + + const leaking = fakePodman({ fail: { action: "remove-backup", status: 1 } }); + const leakingTransaction = recreateWith(leaking); + expect( + finalizePodmanManagedSandbox( + { + replacementReady: true, + transaction: leakingTransaction, + }, + { assertSocketAuthority: vi.fn(), run: leaking.run }, + ), + ).toEqual({ backupRemoved: false, rolledBack: false }); + + const uncertain = fakePodman({ + fail: { action: "remove-backup", afterEffect: true, status: null }, + }); + const uncertainTransaction = recreateWith(uncertain); + expect( + finalizePodmanManagedSandbox( + { + replacementReady: true, + transaction: uncertainTransaction, + }, + { assertSocketAuthority: vi.fn(), run: uncertain.run }, + ), + ).toEqual({ backupRemoved: true, rolledBack: false }); + }); + + it("retains the backup when replacement semantics drift before finalize", () => { + const fake = fakePodman(); + const transaction = recreateWith(fake); + fake.setReplacementUser("1000:1000"); + expect(() => + finalizePodmanManagedSandbox( + { replacementReady: true, transaction }, + { assertSocketAuthority: vi.fn(), run: fake.run }, + ), + ).toThrow("pinned recreation semantics"); + expect(fake.calls.map((call) => call.args.slice(2))).not.toContainEqual(["rm", BACKUP_ID]); + expect(fake.state()).toMatchObject({ + backupExists: true, + backupName: transaction.backupContainerName, + }); + }); + + it("retains a backup that is running when finalize is requested", () => { + const fake = fakePodman(); + const transaction = recreateWith(fake); + fake.setOldRunning(true); + expect(() => + finalizePodmanManagedSandbox( + { replacementReady: true, transaction }, + { assertSocketAuthority: vi.fn(), run: fake.run }, + ), + ).toThrow("backup is running"); + expect(fake.calls.map((call) => call.args.slice(2))).not.toContainEqual(["rm", BACKUP_ID]); + expect(fake.state().backupExists).toBe(true); + }); + + it.each([ + "none", + "multiple", + ] as const)("rejects %s exact-name discovery before inspect or mutation", (discovery) => { + const fake = fakePodman({ discovery }); + expect(() => recreateWith(fake)).toThrow("exactly one"); + expect(fake.calls).toHaveLength(1); + }); +}); diff --git a/src/lib/onboard/compute/podman/sandbox-recreate.ts b/src/lib/onboard/compute/podman/sandbox-recreate.ts new file mode 100644 index 0000000000..76dbbdb024 --- /dev/null +++ b/src/lib/onboard/compute/podman/sandbox-recreate.ts @@ -0,0 +1,1464 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { type SpawnSyncOptions, spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { NAME_VALID_PATTERN } from "../../../name-validation"; +import { + buildPodmanManagedSandboxCreatePlan, + PODMAN_MANAGED_LABEL, + PODMAN_SANDBOX_CONTAINER_PREFIX, + PODMAN_SANDBOX_ID_LABEL, + PODMAN_SANDBOX_NAME_LABEL, + type PodmanManagedSandboxCreatePlan, + type PodmanManagedSandboxInspect, + type PodmanUlimit, + parsePodmanManagedSandboxInspect, + podmanImageMountSources, + podmanWatcherInvisibleBackupLabels, +} from "./sandbox-recreate-spec"; +import { assertPodmanSocketAuthority, type PodmanSocketAuthority } from "./socket-authority"; + +const COMMAND_TIMEOUT_MS = 30_000; +const MAX_CONTAINER_NAME_LENGTH = 253; +const FULL_CONTAINER_ID = /^[0-9a-f]{64}$/u; +const FULL_IMAGE_ID = /^(?:sha256:)?[0-9a-f]{64}$/iu; + +export interface PodmanCommandResult { + readonly error?: Error; + readonly status: number | null; + readonly stderr?: Buffer | string | null; + readonly stdout?: Buffer | string | null; +} + +export type RunQualifiedPodmanCommand = ( + command: "podman", + args: readonly string[], + options: SpawnSyncOptions, +) => PodmanCommandResult; + +export interface PodmanManagedSandboxRecreateDeps { + readonly assertSocketAuthority?: (expected: PodmanSocketAuthority) => void; + readonly now?: () => Date; + readonly run?: RunQualifiedPodmanCommand; + readonly socketAuthority?: PodmanSocketAuthority; +} + +export interface PodmanManagedSandboxRecreateTransaction { + readonly applied: true; + readonly backupContainerId: string; + readonly backupContainerName: string; + readonly backupSemanticDigest: string; + readonly command: readonly string[]; + readonly driverName: "podman"; + readonly immutableImage: string; + readonly newContainerId: string; + readonly oldContainerId: string; + readonly originalLabels: Readonly>; + readonly originalName: string; + readonly originalSemanticDigest: string; + readonly requiredUlimits: readonly PodmanUlimit[]; + readonly sandboxName: string; + readonly semanticDigest: string; + readonly socketAuthority: PodmanSocketAuthority; + readonly socketPath: string; +} + +export interface PodmanManagedSandboxRollbackOutcome { + readonly originalRecreated: boolean; + readonly originalStarted: boolean; + readonly replacementRemoved: boolean; + readonly rolledBack: boolean; +} + +export interface PodmanManagedSandboxFinalizeOutcome { + readonly backupRemoved: boolean; + readonly rolledBack: boolean; +} + +export class PodmanManagedSandboxRecreateError extends Error { + readonly rolledBack: boolean | null; + + constructor(message: string, rolledBack: boolean | null = null) { + super(message); + this.name = "PodmanManagedSandboxRecreateError"; + this.rolledBack = rolledBack; + } +} + +const WATCHER_LEASE = Symbol("podman-openshell-watcher-stopped"); +const WATCHER_CONTROLLER = Symbol("podman-openshell-watcher-controller"); + +export interface PodmanOpenShellWatcherStoppedLease { + readonly [WATCHER_LEASE]: true; + assertStillStopped(): void; + resumeAndProve(): void; +} + +export interface PodmanOpenShellWatcherController { + readonly [WATCHER_CONTROLLER]: true; + quiesceAndProve(): PodmanOpenShellWatcherStoppedLease; +} + +/** + * Build the only supported watcher-stop lease. + * + * The integration owns exact gateway process/service identity. It must stop + * that process, return durable stop evidence, prove the same watcher remains + * absent on every assertion, and restart the same gateway before release. + * `stopAndProve` must restore the watcher itself before throwing because no + * receipt is then available to this layer; `resumeAndProve` must idempotently + * ensure that exact watcher is healthy rather than blindly launch a duplicate. + * The recreator additionally proves that the pinned rootless Podman API stays + * reachable while the lease is held. + */ +export function createPodmanOpenShellWatcherController(deps: { + readonly assertStopped: (receipt: TReceipt) => void; + readonly resumeAndProve: (receipt: TReceipt) => void; + readonly stopAndProve: () => TReceipt; +}): PodmanOpenShellWatcherController { + return { + [WATCHER_CONTROLLER]: true, + quiesceAndProve(): PodmanOpenShellWatcherStoppedLease { + const receipt = deps.stopAndProve(); + try { + deps.assertStopped(receipt); + } catch (error) { + let recoveryError: unknown = null; + try { + deps.resumeAndProve(receipt); + } catch (caught) { + recoveryError = caught; + } + const message = error instanceof Error ? error.message : String(error); + const recoveryMessage = + recoveryError === null + ? "The watcher was resumed." + : `Watcher recovery also failed: ${ + recoveryError instanceof Error ? recoveryError.message : String(recoveryError) + }.`; + throw new PodmanManagedSandboxRecreateError( + `OpenShell watcher stop proof failed: ${message}. ${recoveryMessage}`, + recoveryError === null, + ); + } + let active = true; + return { + [WATCHER_LEASE]: true, + assertStillStopped(): void { + if (!active) { + throw new PodmanManagedSandboxRecreateError( + "OpenShell watcher-stop lease has already been released.", + ); + } + deps.assertStopped(receipt); + }, + resumeAndProve(): void { + if (!active) { + throw new PodmanManagedSandboxRecreateError( + "OpenShell watcher-stop lease has already been released.", + ); + } + deps.assertStopped(receipt); + deps.resumeAndProve(receipt); + active = false; + }, + }; + }, + }; +} + +function requireWatcherLease(value: unknown): PodmanOpenShellWatcherStoppedLease { + if ( + typeof value !== "object" || + value === null || + (value as Partial)[WATCHER_LEASE] !== true + ) { + throw new PodmanManagedSandboxRecreateError( + "Podman managed sandbox cutover requires a proven OpenShell watcher-stop lease.", + ); + } + return value as PodmanOpenShellWatcherStoppedLease; +} + +function requireWatcherController(value: unknown): PodmanOpenShellWatcherController { + if ( + typeof value !== "object" || + value === null || + (value as Partial)[WATCHER_CONTROLLER] !== true || + typeof (value as Partial).quiesceAndProve !== "function" + ) { + throw new PodmanManagedSandboxRecreateError( + "Podman managed sandbox cutover requires an OpenShell watcher controller.", + ); + } + return value as PodmanOpenShellWatcherController; +} + +function defaultRun( + command: "podman", + args: readonly string[], + options: SpawnSyncOptions, +): PodmanCommandResult { + return spawnSync(command, [...args], options); +} + +function commandDeps(deps: PodmanManagedSandboxRecreateDeps): { + readonly now: () => Date; + readonly run: RunQualifiedPodmanCommand; +} { + return { + now: deps.now ?? (() => new Date()), + run: deps.run ?? defaultRun, + }; +} + +function output(value: Buffer | string | null | undefined): string { + if (typeof value === "string") return value; + return Buffer.isBuffer(value) ? value.toString("utf-8") : ""; +} + +function detail(result: PodmanCommandResult, sensitiveValues: readonly string[] = []): string { + let message = [result.error?.message, output(result.stderr), output(result.stdout)] + .filter((value): value is string => Boolean(value?.trim())) + .join(" | ") + .replace(/\s+/gu, " ") + .trim(); + const redactions = new Set(); + for (const entry of sensitiveValues) { + redactions.add(entry); + const separator = entry.indexOf("="); + const value = separator >= 0 ? entry.slice(separator + 1) : entry; + if (value.length >= 4) redactions.add(value); + } + for (const value of [...redactions].sort((left, right) => right.length - left.length)) { + message = message.split(value).join("[REDACTED]"); + } + return message.slice(-400); +} + +function socketUrl(socketPath: string): string { + const normalized = socketPath.trim(); + if (!path.isAbsolute(normalized) || /[\0\r\n]/u.test(normalized)) { + throw new PodmanManagedSandboxRecreateError( + "Podman managed sandbox recreation requires a safe absolute socket path.", + ); + } + return `unix://${normalized}`; +} + +function runPodman( + socketPath: string, + args: readonly string[], + deps: PodmanManagedSandboxRecreateDeps, + options: { readonly input?: string } = {}, +): PodmanCommandResult { + try { + if (deps.socketAuthority) { + if (deps.socketAuthority.socketPath !== socketPath) { + throw new PodmanManagedSandboxRecreateError( + "Podman socket authority does not match the requested managed-sandbox socket.", + ); + } + (deps.assertSocketAuthority ?? assertPodmanSocketAuthority)(deps.socketAuthority); + } + return commandDeps(deps).run("podman", ["--url", socketUrl(socketPath), ...args], { + encoding: "utf-8", + input: options.input, + stdio: [options.input === undefined ? "ignore" : "pipe", "pipe", "pipe"], + timeout: COMMAND_TIMEOUT_MS, + }); + } catch (error) { + return { + error: error instanceof Error ? error : new Error(String(error)), + status: null, + }; + } +} + +function bindSocketAuthority( + socketPath: string, + socketAuthority: PodmanSocketAuthority, + deps: PodmanManagedSandboxRecreateDeps, +): PodmanManagedSandboxRecreateDeps { + if (socketAuthority.socketPath !== socketPath) { + throw new PodmanManagedSandboxRecreateError( + "Podman socket authority does not match the requested managed-sandbox socket.", + ); + } + try { + (deps.assertSocketAuthority ?? assertPodmanSocketAuthority)(socketAuthority); + } catch (error) { + throw new PodmanManagedSandboxRecreateError( + `Podman socket authority could not be revalidated: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + } + return { ...deps, socketAuthority }; +} + +function requireZero(result: PodmanCommandResult, action: string): void { + if (result.status === 0) return; + const suffix = detail(result); + throw new PodmanManagedSandboxRecreateError( + `${action} failed with non-zero or unavailable Podman status${suffix ? `: ${suffix}` : "."}`, + ); +} + +function podmanRuntimeFingerprint( + socketPath: string, + deps: PodmanManagedSandboxRecreateDeps, +): string { + const result = runPodman(socketPath, ["info", "--format", "json"], deps); + requireZero(result, "Podman API health proof during OpenShell watcher cutover"); + let parsed: unknown; + try { + parsed = JSON.parse(output(result.stdout)); + } catch { + throw new PodmanManagedSandboxRecreateError( + "Podman API health proof returned unreadable JSON.", + ); + } + if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) { + throw new PodmanManagedSandboxRecreateError( + "Podman API health proof did not return an object.", + ); + } + const root = parsed as Record; + const hostValue = root.host ?? root.Host; + const storeValue = root.store ?? root.Store; + const host = + typeof hostValue === "object" && hostValue !== null && !Array.isArray(hostValue) + ? (hostValue as Record) + : null; + const securityValue = host?.security ?? host?.Security; + const security = + typeof securityValue === "object" && securityValue !== null && !Array.isArray(securityValue) + ? (securityValue as Record) + : null; + const store = + typeof storeValue === "object" && storeValue !== null && !Array.isArray(storeValue) + ? (storeValue as Record) + : null; + const rootless = security?.rootless ?? security?.Rootless; + const graphRoot = store?.graphRoot ?? store?.GraphRoot; + const runRoot = store?.runRoot ?? store?.RunRoot; + if ( + rootless !== true || + typeof graphRoot !== "string" || + !path.isAbsolute(graphRoot) || + typeof runRoot !== "string" || + !path.isAbsolute(runRoot) + ) { + throw new PodmanManagedSandboxRecreateError( + "OpenShell watcher cutover requires the same healthy rootless Podman API and absolute storage roots.", + ); + } + return createHash("sha256").update(`${graphRoot}\0${runRoot}`).digest("hex"); +} + +function proveWatcherStoppedWithPodman( + lease: PodmanOpenShellWatcherStoppedLease, + socketPath: string, + expectedFingerprint: string | null, + deps: PodmanManagedSandboxRecreateDeps, +): string { + lease.assertStillStopped(); + const fingerprint = podmanRuntimeFingerprint(socketPath, deps); + if (expectedFingerprint !== null && fingerprint !== expectedFingerprint) { + throw new PodmanManagedSandboxRecreateError( + "Podman runtime identity changed while the OpenShell watcher was stopped.", + ); + } + lease.assertStillStopped(); + return fingerprint; +} + +function sandboxContainerName(sandboxName: string): string { + if ( + typeof sandboxName !== "string" || + !NAME_VALID_PATTERN.test(sandboxName) || + sandboxName.length === 0 + ) { + throw new PodmanManagedSandboxRecreateError( + "Podman managed sandbox recreation requires a valid canonical sandbox name.", + ); + } + return `${PODMAN_SANDBOX_CONTAINER_PREFIX}${sandboxName}`; +} + +function parseDiscovery(outputValue: string, expectedName: string): string[] { + const lines = outputValue + .trim() + .split(/\r?\n/u) + .map((line) => line.trim()) + .filter(Boolean); + return lines.map((line) => { + const fields = line.split("\t"); + if ( + fields.length !== 2 || + fields[1] !== expectedName || + !FULL_CONTAINER_ID.test(fields[0] ?? "") + ) { + throw new PodmanManagedSandboxRecreateError( + "Podman discovery did not return an exact name and full immutable container ID.", + ); + } + return fields[0] as string; + }); +} + +export function findPodmanManagedSandboxContainerIds( + socketPath: string, + sandboxName: string, + deps: PodmanManagedSandboxRecreateDeps = {}, +): string[] { + const name = sandboxContainerName(sandboxName); + const result = runPodman( + socketPath, + [ + "ps", + "--all", + "--no-trunc", + "--filter", + `name=^${name}$`, + "--filter", + `label=${PODMAN_MANAGED_LABEL}=true`, + "--filter", + `label=${PODMAN_SANDBOX_NAME_LABEL}=${sandboxName}`, + "--format", + "{{.ID}}\t{{.Names}}", + ], + deps, + ); + requireZero(result, "Podman managed sandbox discovery"); + return parseDiscovery(output(result.stdout), name); +} + +function discoverManagedSandbox( + socketPath: string, + sandboxName: string, + deps: PodmanManagedSandboxRecreateDeps, +): string { + const ids = findPodmanManagedSandboxContainerIds(socketPath, sandboxName, deps); + if (ids.length !== 1) { + throw new PodmanManagedSandboxRecreateError( + `Podman discovery must identify exactly one '${sandboxContainerName( + sandboxName, + )}' managed container.`, + ); + } + return ids[0] as string; +} + +interface ExpectedManagedContainer { + readonly containerId: string; + readonly identityMode?: "managed" | "watcher-invisible-backup"; + readonly name: string; + readonly requireRunning?: boolean; + readonly sandboxId?: string; + readonly sandboxName: string; +} + +function inspectContainer( + socketPath: string, + expected: ExpectedManagedContainer, + deps: PodmanManagedSandboxRecreateDeps, +): PodmanManagedSandboxInspect { + const result = runPodman(socketPath, ["container", "inspect", expected.containerId], deps); + requireZero(result, `Podman inspect for '${expected.name}'`); + return parsePodmanManagedSandboxInspect(output(result.stdout), expected); +} + +function tryInspectContainer( + socketPath: string, + expected: ExpectedManagedContainer, + deps: PodmanManagedSandboxRecreateDeps, +): PodmanManagedSandboxInspect | null { + try { + return inspectContainer(socketPath, expected, deps); + } catch { + return null; + } +} + +function startAndVerifyContainer( + socketPath: string, + expected: Omit, + deps: PodmanManagedSandboxRecreateDeps, +): boolean { + runPodman(socketPath, ["start", expected.containerId], deps); + return ( + tryInspectContainer(socketPath, { ...expected, requireRunning: true }, deps)?.running === true + ); +} + +function containerExists( + socketPath: string, + containerId: string, + deps: PodmanManagedSandboxRecreateDeps, +): boolean | null { + const result = runPodman(socketPath, ["container", "exists", containerId], deps); + if (result.status === 0) return true; + if (result.status === 1) return false; + return null; +} + +function pinImageMounts( + socketPath: string, + inspect: PodmanManagedSandboxInspect, + deps: PodmanManagedSandboxRecreateDeps, +): Record { + const pins: Record = {}; + for (const source of podmanImageMountSources(inspect)) { + const result = runPodman(socketPath, ["image", "inspect", "--format", "{{.Id}}", source], deps); + requireZero(result, "Podman image-volume pinning"); + const imageId = output(result.stdout).trim(); + if (!FULL_IMAGE_ID.test(imageId)) { + throw new PodmanManagedSandboxRecreateError( + `Podman image-volume '${source}' did not resolve to a full immutable image ID.`, + ); + } + pins[source] = imageId; + } + return pins; +} + +function buildPinnedCreatePlan( + socketPath: string, + inspect: PodmanManagedSandboxInspect, + options: { + readonly command: readonly string[] | null; + readonly labels?: Readonly>; + readonly name?: string; + readonly requireCommandEnvironment?: boolean; + readonly requiredUlimits?: readonly PodmanUlimit[]; + }, + deps: PodmanManagedSandboxRecreateDeps, +): PodmanManagedSandboxCreatePlan { + return buildPodmanManagedSandboxCreatePlan({ + command: options.command, + imagePins: pinImageMounts(socketPath, inspect, deps), + inspect, + labels: options.labels, + name: options.name, + requireCommandEnvironment: options.requireCommandEnvironment, + requiredUlimits: options.requiredUlimits, + }); +} + +function createPlanSemanticDigest(plan: PodmanManagedSandboxCreatePlan): string { + return createHash("sha256") + .update( + JSON.stringify({ + args: plan.args, + environmentInput: plan.environmentInput, + version: 1, + }), + ) + .digest("hex"); +} + +function requireEquivalentCreatePlan( + expectedDigest: string, + actual: PodmanManagedSandboxCreatePlan, +): void { + if (createPlanSemanticDigest(actual) !== expectedDigest) { + throw new PodmanManagedSandboxRecreateError( + "Podman container inspect does not reproduce the pinned recreation semantics.", + ); + } +} + +interface PodmanCidFile { + readonly directory: string; + readonly file: string; +} + +function createCidFile(): PodmanCidFile { + const directory = mkdtempSync(path.join(tmpdir(), "nemoclaw-podman-recreate-")); + return { directory, file: path.join(directory, "replacement.cid") }; +} + +function cleanupCidFile(cidFile: PodmanCidFile): void { + try { + rmSync(cidFile.directory, { force: true, recursive: true }); + } catch { + // The cidfile contains only a container ID. Cleanup failure must not + // interrupt rollback or turn a verified recreation into an outage. + } +} + +function readCidFile(cidFile: PodmanCidFile): string | null { + let value: string; + try { + value = readFileSync(cidFile.file, "utf-8").trim(); + } catch (error) { + if (typeof error === "object" && error !== null && "code" in error && error.code === "ENOENT") { + return null; + } + throw error; + } + if (!FULL_CONTAINER_ID.test(value)) { + throw new PodmanManagedSandboxRecreateError( + "Podman create cidfile did not contain one full replacement container ID.", + ); + } + return value; +} + +function backupName(originalName: string, now: Date): string { + const suffix = `-nemoclaw-backup-${String(now.getTime())}`; + const prefixLength = MAX_CONTAINER_NAME_LENGTH - suffix.length; + if (prefixLength < 1) { + throw new PodmanManagedSandboxRecreateError("Podman backup container name is too long."); + } + return `${originalName.slice(0, prefixLength)}${suffix}`; +} + +function parseCreatedId(result: PodmanCommandResult): string { + const lines = output(result.stdout).trim().split(/\r?\n/u).filter(Boolean); + if (lines.length !== 1 || !FULL_CONTAINER_ID.test(lines[0] ?? "")) { + throw new PodmanManagedSandboxRecreateError( + "Podman create did not return one full replacement container ID.", + ); + } + return lines[0] as string; +} + +function ownedCreatedId(result: PodmanCommandResult, cidFileId: string | null): string | null { + let stdoutId: string | null = null; + try { + stdoutId = parseCreatedId(result); + } catch (error) { + if (result.status === 0 && !cidFileId) throw error; + } + if (cidFileId && stdoutId && cidFileId !== stdoutId) { + throw new PodmanManagedSandboxRecreateError( + "Podman create stdout and cidfile identified different replacement containers.", + ); + } + return cidFileId ?? stdoutId; +} + +interface PodmanCreateAttempt { + readonly containerId: string | null; + readonly detail: string; + readonly ok: boolean; +} + +function createContainerFromPlan( + socketPath: string, + plan: PodmanManagedSandboxCreatePlan, + deps: PodmanManagedSandboxRecreateDeps, +): PodmanCreateAttempt { + const cidFile = createCidFile(); + let result: PodmanCommandResult = { status: null }; + let callError: unknown = null; + let cidFileId: string | null = null; + let cidFileError: unknown = null; + try { + try { + result = runPodman( + socketPath, + ["create", "--cidfile", cidFile.file, ...plan.args.slice(1)], + deps, + { input: plan.environmentInput }, + ); + } catch (error) { + callError = error; + } + try { + cidFileId = readCidFile(cidFile); + } catch (error) { + cidFileError = error; + } + } finally { + cleanupCidFile(cidFile); + } + + let containerId: string | null = null; + let identityError = cidFileError; + try { + containerId = ownedCreatedId(result, cidFileId); + } catch (error) { + identityError ??= error; + } + const sensitiveEnvironment = plan.environmentInput.split(/\r?\n/u).filter(Boolean); + const error = callError ?? identityError; + const message = + error instanceof Error + ? detail({ error, status: result.status }, sensitiveEnvironment) + : result.status === 0 + ? "" + : detail(result, sensitiveEnvironment); + return { + containerId, + detail: message, + ok: callError === null && identityError === null && result.status === 0 && containerId !== null, + }; +} + +function removeOwnedContainer( + socketPath: string, + containerId: string, + deps: PodmanManagedSandboxRecreateDeps, +): boolean { + const exists = containerExists(socketPath, containerId, deps); + if (exists === false) return true; + if (exists === null) return false; + runPodman(socketPath, ["stop", containerId], deps); + runPodman(socketPath, ["rm", containerId], deps); + return containerExists(socketPath, containerId, deps) === false; +} + +function inspectWatcherInvisibleBackup( + transaction: Pick< + PodmanManagedSandboxRecreateTransaction, + "backupContainerId" | "backupContainerName" | "originalLabels" | "sandboxName" | "socketPath" + >, + deps: PodmanManagedSandboxRecreateDeps, +): PodmanManagedSandboxInspect { + return inspectContainer( + transaction.socketPath, + { + containerId: transaction.backupContainerId, + identityMode: "watcher-invisible-backup", + name: transaction.backupContainerName, + sandboxId: String(transaction.originalLabels[PODMAN_SANDBOX_ID_LABEL] ?? ""), + sandboxName: transaction.sandboxName, + }, + deps, + ); +} + +type PodmanRestoreIdentity = Pick< + PodmanManagedSandboxRecreateTransaction, + | "backupContainerId" + | "backupContainerName" + | "backupSemanticDigest" + | "originalLabels" + | "originalName" + | "originalSemanticDigest" + | "sandboxName" + | "socketAuthority" + | "socketPath" +>; + +function buildVerifiedRestorePlan( + transaction: PodmanRestoreIdentity, + deps: PodmanManagedSandboxRecreateDeps, +): PodmanManagedSandboxCreatePlan { + const backup = inspectWatcherInvisibleBackup(transaction, deps); + const backupPlan = buildPinnedCreatePlan( + transaction.socketPath, + backup, + { + command: null, + labels: podmanWatcherInvisibleBackupLabels(backup), + name: transaction.backupContainerName, + }, + deps, + ); + requireEquivalentCreatePlan(transaction.backupSemanticDigest, backupPlan); + const restorePlan = buildPinnedCreatePlan( + transaction.socketPath, + backup, + { + command: null, + labels: transaction.originalLabels, + name: transaction.originalName, + }, + deps, + ); + requireEquivalentCreatePlan(transaction.originalSemanticDigest, restorePlan); + return restorePlan; +} + +function restoreManagedFromBackup( + transaction: PodmanRestoreIdentity, + deps: PodmanManagedSandboxRecreateDeps, +): { readonly originalRecreated: boolean; readonly originalStarted: boolean } { + const restored = createContainerFromPlan( + transaction.socketPath, + buildVerifiedRestorePlan(transaction, deps), + deps, + ); + if (!restored.ok || !restored.containerId) { + if (restored.containerId) { + removeOwnedContainer(transaction.socketPath, restored.containerId, deps); + } + return { originalRecreated: false, originalStarted: false }; + } + const originalRecreated = + tryInspectContainer( + transaction.socketPath, + { + containerId: restored.containerId, + name: transaction.originalName, + sandboxName: transaction.sandboxName, + }, + deps, + ) !== null; + const originalStarted = + originalRecreated && + startAndVerifyContainer( + transaction.socketPath, + { + containerId: restored.containerId, + name: transaction.originalName, + sandboxName: transaction.sandboxName, + }, + deps, + ); + if (!originalStarted) return { originalRecreated, originalStarted: false }; + const runningOriginal = inspectContainer( + transaction.socketPath, + { + containerId: restored.containerId, + name: transaction.originalName, + requireRunning: true, + sandboxName: transaction.sandboxName, + }, + deps, + ); + requireEquivalentCreatePlan( + transaction.originalSemanticDigest, + buildPinnedCreatePlan(transaction.socketPath, runningOriginal, { command: null }, deps), + ); + return { originalRecreated: true, originalStarted: true }; +} + +function rollbackKnownTransaction( + transaction: PodmanManagedSandboxRecreateTransaction, + lease: PodmanOpenShellWatcherStoppedLease, + deps: PodmanManagedSandboxRecreateDeps, +): PodmanManagedSandboxRollbackOutcome { + const podmanFingerprint = proveWatcherStoppedWithPodman( + lease, + transaction.socketPath, + null, + deps, + ); + buildVerifiedRestorePlan(transaction, deps); + proveWatcherStoppedWithPodman(lease, transaction.socketPath, podmanFingerprint, deps); + const replacementExists = containerExists( + transaction.socketPath, + transaction.newContainerId, + deps, + ); + if ( + replacementExists !== false && + !tryInspectContainer( + transaction.socketPath, + { + containerId: transaction.newContainerId, + name: transaction.originalName, + sandboxName: transaction.sandboxName, + }, + deps, + ) + ) { + return { + originalRecreated: false, + originalStarted: false, + replacementRemoved: false, + rolledBack: false, + }; + } + proveWatcherStoppedWithPodman(lease, transaction.socketPath, podmanFingerprint, deps); + const replacementRemoved = + replacementExists === false || + removeOwnedContainer(transaction.socketPath, transaction.newContainerId, deps); + if (!replacementRemoved) { + return { + originalRecreated: false, + originalStarted: false, + replacementRemoved: false, + rolledBack: false, + }; + } + + proveWatcherStoppedWithPodman(lease, transaction.socketPath, podmanFingerprint, deps); + const { originalRecreated, originalStarted } = restoreManagedFromBackup(transaction, deps); + if (!originalStarted) { + return { + originalRecreated, + originalStarted: false, + replacementRemoved: true, + rolledBack: false, + }; + } + proveWatcherStoppedWithPodman(lease, transaction.socketPath, podmanFingerprint, deps); + return { + originalRecreated: true, + originalStarted: true, + replacementRemoved: true, + rolledBack: true, + }; +} + +interface PodmanRollbackAndWatcherResumeAttempt { + readonly outcome: PodmanManagedSandboxRollbackOutcome; + readonly resumeError: unknown; + readonly rollbackError: unknown; +} + +function failedRollbackOutcome(): PodmanManagedSandboxRollbackOutcome { + return { + originalRecreated: false, + originalStarted: false, + replacementRemoved: false, + rolledBack: false, + }; +} + +/** + * A rollback exception must never strand a watcher that this transaction + * successfully stopped. The outcome is complete only after both the original + * managed container and watcher restart have been proven. + */ +function rollbackAndResumeWatcher( + transaction: PodmanManagedSandboxRecreateTransaction, + lease: PodmanOpenShellWatcherStoppedLease, + deps: PodmanManagedSandboxRecreateDeps, +): PodmanRollbackAndWatcherResumeAttempt { + let outcome = failedRollbackOutcome(); + let rollbackError: unknown = null; + try { + outcome = rollbackKnownTransaction(transaction, lease, deps); + } catch (error) { + rollbackError = error; + } + + let resumeError: unknown = null; + try { + lease.resumeAndProve(); + } catch (error) { + resumeError = error; + } + if (resumeError !== null) outcome = { ...outcome, rolledBack: false }; + return { outcome, resumeError, rollbackError }; +} + +export function recreatePodmanManagedSandbox( + options: { + readonly command: readonly string[]; + readonly requiredUlimits?: readonly PodmanUlimit[]; + readonly sandboxName: string; + readonly socketAuthority: PodmanSocketAuthority; + readonly socketPath: string; + readonly watcherController: PodmanOpenShellWatcherController; + }, + deps: PodmanManagedSandboxRecreateDeps = {}, +): PodmanManagedSandboxRecreateTransaction { + const watcherController = requireWatcherController(options.watcherController); + const originalName = sandboxContainerName(options.sandboxName); + const socketPath = options.socketPath.trim(); + socketUrl(socketPath); + deps = bindSocketAuthority(socketPath, options.socketAuthority, deps); + const oldContainerId = discoverManagedSandbox(socketPath, options.sandboxName, deps); + const original = inspectContainer( + socketPath, + { + containerId: oldContainerId, + name: originalName, + requireRunning: true, + sandboxName: options.sandboxName, + }, + deps, + ); + const command = [...options.command]; + const requiredUlimits = (options.requiredUlimits ?? []).map((limit) => ({ ...limit })); + const plan = buildPinnedCreatePlan(socketPath, original, { command, requiredUlimits }, deps); + const semanticDigest = createPlanSemanticDigest(plan); + const originalPlan = buildPinnedCreatePlan(socketPath, original, { command: null }, deps); + const originalSemanticDigest = createPlanSemanticDigest(originalPlan); + const originalLabels = { ...original.labels }; + const pinnedAgain = discoverManagedSandbox(socketPath, options.sandboxName, deps); + if (pinnedAgain !== oldContainerId) { + throw new PodmanManagedSandboxRecreateError( + "Podman managed sandbox identity changed immediately before mutation.", + ); + } + const current = inspectContainer( + socketPath, + { + containerId: oldContainerId, + name: originalName, + requireRunning: true, + sandboxName: options.sandboxName, + }, + deps, + ); + requireEquivalentCreatePlan( + semanticDigest, + buildPinnedCreatePlan(socketPath, current, { command, requiredUlimits }, deps), + ); + requireEquivalentCreatePlan( + originalSemanticDigest, + buildPinnedCreatePlan(socketPath, current, { command: null }, deps), + ); + const backupContainerName = backupName(originalName, commandDeps(deps).now()); + const backupLabels = podmanWatcherInvisibleBackupLabels(original); + const backupPlan = buildPinnedCreatePlan( + socketPath, + original, + { + command: null, + labels: backupLabels, + name: backupContainerName, + }, + deps, + ); + const backupSemanticDigest = createPlanSemanticDigest(backupPlan); + const backupCreated = createContainerFromPlan(socketPath, backupPlan, deps); + if (!backupCreated.ok || !backupCreated.containerId) { + if (backupCreated.containerId) { + removeOwnedContainer(socketPath, backupCreated.containerId, deps); + } + throw new PodmanManagedSandboxRecreateError( + `Creating the watcher-invisible Podman rollback backup failed${ + backupCreated.detail ? `: ${backupCreated.detail}` : "." + } The original managed container remains running.`, + true, + ); + } + const backupContainerId = backupCreated.containerId; + const baseTransaction: PodmanRestoreIdentity & { + readonly command: readonly string[]; + readonly immutableImage: string; + readonly oldContainerId: string; + readonly requiredUlimits: readonly PodmanUlimit[]; + readonly semanticDigest: string; + } = { + backupContainerId, + backupContainerName, + backupSemanticDigest, + command, + immutableImage: plan.immutableImage, + oldContainerId, + originalLabels, + originalName, + originalSemanticDigest, + requiredUlimits, + sandboxName: options.sandboxName, + semanticDigest, + socketAuthority: options.socketAuthority, + socketPath, + }; + const backup = inspectWatcherInvisibleBackup(baseTransaction, deps); + if (backup.running) { + removeOwnedContainer(socketPath, backupContainerId, deps); + throw new PodmanManagedSandboxRecreateError( + "Watcher-invisible Podman rollback backup unexpectedly started.", + true, + ); + } + requireEquivalentCreatePlan( + backupSemanticDigest, + buildPinnedCreatePlan( + socketPath, + backup, + { command: null, labels: backupLabels, name: backupContainerName }, + deps, + ), + ); + let watcherLease: PodmanOpenShellWatcherStoppedLease; + try { + watcherLease = requireWatcherLease(watcherController.quiesceAndProve()); + } catch (error) { + removeOwnedContainer(socketPath, backupContainerId, deps); + throw error; + } + let podmanFingerprint: string; + try { + podmanFingerprint = proveWatcherStoppedWithPodman(watcherLease, socketPath, null, deps); + } catch (error) { + let watcherResumed = false; + try { + watcherLease.resumeAndProve(); + watcherResumed = true; + } catch { + // The original managed container remains running and the backup is + // watcher-invisible; no managed-container mutation is attempted. + } + removeOwnedContainer(socketPath, backupContainerId, deps); + const message = error instanceof Error ? error.message : String(error); + throw new PodmanManagedSandboxRecreateError( + `OpenShell watcher cutover preflight failed: ${message}. ${ + watcherResumed ? "The watcher was resumed." : "Watcher recovery could not be proven." + }`, + watcherResumed, + ); + } + const abortCutover = (message: string, replacementId: string | null = null): never => { + let restored = false; + try { + proveWatcherStoppedWithPodman(watcherLease, socketPath, podmanFingerprint, deps); + if (replacementId) removeOwnedContainer(socketPath, replacementId, deps); + const originalExists = containerExists(socketPath, oldContainerId, deps); + if (originalExists === true) { + const originalNow = tryInspectContainer( + socketPath, + { + containerId: oldContainerId, + name: originalName, + sandboxName: options.sandboxName, + }, + deps, + ); + restored = + originalNow !== null && + (originalNow.running || + startAndVerifyContainer( + socketPath, + { + containerId: oldContainerId, + name: originalName, + sandboxName: options.sandboxName, + }, + deps, + )); + } else if (originalExists === false) { + restored = restoreManagedFromBackup(baseTransaction, deps).originalStarted; + } + proveWatcherStoppedWithPodman(watcherLease, socketPath, podmanFingerprint, deps); + } catch { + restored = false; + } + let watcherResumed = false; + try { + watcherLease.resumeAndProve(); + watcherResumed = true; + } catch { + // Leave the invisible backup in place unless both recovery halves prove. + } + const rolledBack = restored && watcherResumed; + if (rolledBack) removeOwnedContainer(socketPath, backupContainerId, deps); + throw new PodmanManagedSandboxRecreateError( + `${message} ${ + rolledBack + ? "The original Podman sandbox was restored before the watcher resumed." + : "Podman sandbox rollback or watcher recovery did not complete." + }`, + rolledBack, + ); + }; + const proveStoppedOrAbort = (stage: string): void => { + try { + proveWatcherStoppedWithPodman(watcherLease, socketPath, podmanFingerprint, deps); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + abortCutover(`OpenShell watcher proof failed ${stage}: ${message}.`); + } + }; + + proveStoppedOrAbort("before stopping the original managed container"); + const stopped = runPodman(socketPath, ["stop", oldContainerId], deps); + const stoppedOriginal = tryInspectContainer( + socketPath, + { + containerId: oldContainerId, + name: originalName, + sandboxName: options.sandboxName, + }, + deps, + ); + if (!stoppedOriginal) { + const restored = startAndVerifyContainer( + socketPath, + { + containerId: oldContainerId, + name: originalName, + sandboxName: options.sandboxName, + }, + deps, + ); + abortCutover( + `Stopping the original Podman managed sandbox could not be reconciled${ + detail(stopped) ? `: ${detail(stopped)}` : "." + } ${restored ? "The exact original container was restored." : "The exact original container could not be restored."}`, + ); + } + if (stoppedOriginal?.running) { + abortCutover( + `Stopping the original Podman managed sandbox did not leave the pinned container stopped${ + detail(stopped) ? `: ${detail(stopped)}` : "." + } The original container remains running.`, + ); + } + proveStoppedOrAbort("before removing the original managed container"); + const removedOriginal = runPodman(socketPath, ["rm", oldContainerId], deps); + if (containerExists(socketPath, oldContainerId, deps) !== false) { + abortCutover( + `Removing the original managed container during the watcher-stopped cutover failed${ + detail(removedOriginal) ? `: ${detail(removedOriginal)}` : "." + }`, + ); + } + proveStoppedOrAbort("before creating the replacement managed container"); + const replacementCreated = createContainerFromPlan(socketPath, plan, deps); + if (!replacementCreated.ok || !replacementCreated.containerId) { + abortCutover( + `Creating the replacement Podman managed sandbox failed${ + replacementCreated.detail ? `: ${replacementCreated.detail}` : "." + }`, + replacementCreated.containerId, + ); + } + const newContainerId = replacementCreated.containerId as string; + if (newContainerId === oldContainerId || newContainerId === backupContainerId) { + abortCutover("Podman returned a previously pinned container ID for the replacement."); + } + const transaction: PodmanManagedSandboxRecreateTransaction = { + ...baseTransaction, + applied: true, + driverName: "podman", + newContainerId, + }; + + try { + const createdReplacement = inspectContainer( + socketPath, + { + containerId: newContainerId, + name: originalName, + sandboxName: options.sandboxName, + }, + deps, + ); + requireEquivalentCreatePlan( + semanticDigest, + buildPinnedCreatePlan( + socketPath, + createdReplacement, + { command, requireCommandEnvironment: true }, + deps, + ), + ); + proveWatcherStoppedWithPodman(watcherLease, socketPath, podmanFingerprint, deps); + if ( + !startAndVerifyContainer( + socketPath, + { + containerId: newContainerId, + name: originalName, + sandboxName: options.sandboxName, + }, + deps, + ) + ) { + throw new PodmanManagedSandboxRecreateError( + "Starting the replacement Podman managed sandbox could not be verified.", + ); + } + const replacement = inspectContainer( + socketPath, + { + containerId: newContainerId, + name: originalName, + requireRunning: true, + sandboxName: options.sandboxName, + }, + deps, + ); + requireEquivalentCreatePlan( + semanticDigest, + buildPinnedCreatePlan( + socketPath, + replacement, + { command, requireCommandEnvironment: true }, + deps, + ), + ); + const verifiedBackup = inspectWatcherInvisibleBackup(transaction, deps); + if (verifiedBackup.running) { + throw new PodmanManagedSandboxRecreateError( + "The pinned original Podman backup restarted while the replacement was being verified.", + ); + } + requireEquivalentCreatePlan( + backupSemanticDigest, + buildPinnedCreatePlan( + socketPath, + verifiedBackup, + { command: null, labels: backupLabels, name: backupContainerName }, + deps, + ), + ); + proveWatcherStoppedWithPodman(watcherLease, socketPath, podmanFingerprint, deps); + } catch (error) { + const attempt = rollbackAndResumeWatcher(transaction, watcherLease, deps); + if (attempt.outcome.rolledBack) removeOwnedContainer(socketPath, backupContainerId, deps); + const message = error instanceof Error ? error.message : String(error); + const rollbackMessage = + attempt.rollbackError === null + ? "" + : ` Rollback failed: ${ + attempt.rollbackError instanceof Error + ? attempt.rollbackError.message + : String(attempt.rollbackError) + }.`; + const resumeMessage = + attempt.resumeError === null + ? "" + : ` Watcher recovery failed: ${ + attempt.resumeError instanceof Error + ? attempt.resumeError.message + : String(attempt.resumeError) + }.`; + throw new PodmanManagedSandboxRecreateError( + attempt.outcome.rolledBack + ? `${message} The original Podman sandbox was restored before the watcher resumed.` + : `${message} Podman sandbox rollback did not complete.${rollbackMessage}${resumeMessage}`, + attempt.outcome.rolledBack, + ); + } + try { + watcherLease.resumeAndProve(); + } catch (error) { + let attempt: PodmanRollbackAndWatcherResumeAttempt | null = null; + try { + watcherLease.assertStillStopped(); + attempt = rollbackAndResumeWatcher(transaction, watcherLease, deps); + } catch { + // A partially resumed watcher is not safe for another managed removal. + } + if (attempt?.outcome.rolledBack) removeOwnedContainer(socketPath, backupContainerId, deps); + const message = error instanceof Error ? error.message : String(error); + throw new PodmanManagedSandboxRecreateError( + `Restarting the OpenShell watcher after Podman cutover failed: ${message}. ${ + attempt?.outcome.rolledBack + ? "The original sandbox was restored." + : "The watcher or sandbox rollback could not be proven." + }`, + attempt?.outcome.rolledBack ?? false, + ); + } + return transaction; +} + +export function rollbackPodmanManagedSandbox( + options: { + readonly transaction: PodmanManagedSandboxRecreateTransaction; + readonly watcherController: PodmanOpenShellWatcherController; + }, + deps: PodmanManagedSandboxRecreateDeps = {}, +): PodmanManagedSandboxRollbackOutcome { + const watcherController = requireWatcherController(options.watcherController); + deps = bindSocketAuthority( + options.transaction.socketPath, + options.transaction.socketAuthority, + deps, + ); + const lease = requireWatcherLease(watcherController.quiesceAndProve()); + const attempt = rollbackAndResumeWatcher(options.transaction, lease, deps); + if (attempt.rollbackError !== null) { + const message = + attempt.rollbackError instanceof Error + ? attempt.rollbackError.message + : String(attempt.rollbackError); + throw new PodmanManagedSandboxRecreateError( + `Podman sandbox rollback failed: ${message}. ${ + attempt.resumeError === null + ? "The watcher was resumed." + : "Watcher recovery could not be proven." + }`, + false, + ); + } + if (attempt.outcome.rolledBack) { + removeOwnedContainer( + options.transaction.socketPath, + options.transaction.backupContainerId, + deps, + ); + } + return attempt.outcome; +} + +export function finalizePodmanManagedSandbox( + options: + | { + readonly replacementReady: true; + readonly transaction: PodmanManagedSandboxRecreateTransaction; + } + | { + readonly replacementReady: false; + readonly transaction: PodmanManagedSandboxRecreateTransaction; + readonly watcherController: PodmanOpenShellWatcherController; + }, + deps: PodmanManagedSandboxRecreateDeps = {}, +): PodmanManagedSandboxFinalizeOutcome { + deps = bindSocketAuthority( + options.transaction.socketPath, + options.transaction.socketAuthority, + deps, + ); + if (!options.replacementReady) { + const rollback = rollbackPodmanManagedSandbox( + { + transaction: options.transaction, + watcherController: options.watcherController, + }, + deps, + ); + return { backupRemoved: false, rolledBack: rollback.rolledBack }; + } + const replacement = inspectContainer( + options.transaction.socketPath, + { + containerId: options.transaction.newContainerId, + name: options.transaction.originalName, + requireRunning: true, + sandboxName: options.transaction.sandboxName, + }, + deps, + ); + requireEquivalentCreatePlan( + options.transaction.semanticDigest, + buildPinnedCreatePlan( + options.transaction.socketPath, + replacement, + { + command: options.transaction.command, + requireCommandEnvironment: true, + }, + deps, + ), + ); + const backup = inspectWatcherInvisibleBackup(options.transaction, deps); + if (backup.running) { + throw new PodmanManagedSandboxRecreateError( + "The pinned original Podman backup is running; finalize will not delete it.", + ); + } + requireEquivalentCreatePlan( + options.transaction.backupSemanticDigest, + buildPinnedCreatePlan( + options.transaction.socketPath, + backup, + { + command: null, + labels: podmanWatcherInvisibleBackupLabels(backup), + name: options.transaction.backupContainerName, + }, + deps, + ), + ); + runPodman(options.transaction.socketPath, ["rm", options.transaction.backupContainerId], deps); + return { + backupRemoved: + containerExists( + options.transaction.socketPath, + options.transaction.backupContainerId, + deps, + ) === false, + rolledBack: false, + }; +} diff --git a/src/lib/onboard/compute/podman/socket-authority.test.ts b/src/lib/onboard/compute/podman/socket-authority.test.ts new file mode 100644 index 0000000000..3a062c6475 --- /dev/null +++ b/src/lib/onboard/compute/podman/socket-authority.test.ts @@ -0,0 +1,116 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import { assertPodmanSocketAuthority, capturePodmanSocketAuthority } from "./socket-authority"; + +const SOCKET_PATH = "/run/user/1000/podman/podman.sock"; + +function stat( + overrides: Partial<{ + dev: bigint; + directory: boolean; + ino: bigint; + mode: bigint; + socket: boolean; + uid: bigint; + }> = {}, +) { + return { + dev: overrides.dev ?? 8n, + ino: overrides.ino ?? 9001n, + mode: overrides.mode ?? 0o700n, + uid: overrides.uid ?? 1000n, + isDirectory: () => overrides.directory ?? false, + isSocket: () => overrides.socket ?? true, + }; +} + +function secureLstat( + socketOverrides: Parameters[0] = {}, + directoryOverrides: Readonly[0]>> = {}, +) { + const directoryInodes = new Map(); + return (filePath: string) => { + if (filePath === SOCKET_PATH) return stat(socketOverrides); + if (!directoryInodes.has(filePath)) { + directoryInodes.set(filePath, BigInt(7000 + directoryInodes.size)); + } + return stat({ + directory: true, + ino: directoryInodes.get(filePath), + uid: filePath.startsWith("/run/user/1000") ? 1000n : 0n, + ...(directoryOverrides[filePath] ?? {}), + }); + }; +} + +describe("Podman socket authority", () => { + it("captures exact current-user socket and secure path identity", () => { + const authority = capturePodmanSocketAuthority(SOCKET_PATH, { + lstat: vi.fn(secureLstat()), + uid: 1000, + }); + + expect(authority).toMatchObject({ + device: "8", + inode: "9001", + ownerUid: "1000", + socketPath: SOCKET_PATH, + }); + expect(authority.directoryChain.map(({ ownerUid, path }) => ({ ownerUid, path }))).toEqual([ + { ownerUid: "1000", path: "/run/user/1000/podman" }, + { ownerUid: "1000", path: "/run/user/1000" }, + { ownerUid: "0", path: "/run/user" }, + { ownerUid: "0", path: "/run" }, + { ownerUid: "0", path: "/" }, + ]); + }); + + it("rejects a foreign-owner socket before runtime probing", () => { + expect(() => + capturePodmanSocketAuthority(SOCKET_PATH, { + lstat: secureLstat({ uid: 2000n }), + uid: 1000, + }), + ).toThrow("owned by uid 2000"); + }); + + it("rejects an otherwise current-user socket beneath a foreign-writable directory", () => { + expect(() => + capturePodmanSocketAuthority(SOCKET_PATH, { + lstat: secureLstat({}, { "/run/user/1000/podman": { mode: 0o770n } }), + uid: 1000, + }), + ).toThrow("writable by another user or group"); + }); + + it("rejects path replacement after qualification", () => { + const expected = capturePodmanSocketAuthority(SOCKET_PATH, { + lstat: secureLstat(), + uid: 1000, + }); + + expect(() => + assertPodmanSocketAuthority(expected, { + lstat: secureLstat({ ino: 9002n }), + uid: 1000, + }), + ).toThrow("changed after it was qualified"); + }); + + it("rejects directory replacement after qualification", () => { + const expected = capturePodmanSocketAuthority(SOCKET_PATH, { + lstat: secureLstat(), + uid: 1000, + }); + + expect(() => + assertPodmanSocketAuthority(expected, { + lstat: secureLstat({}, { "/run/user/1000/podman": { ino: 8000n } }), + uid: 1000, + }), + ).toThrow("changed after it was qualified"); + }); +}); diff --git a/src/lib/onboard/compute/podman/socket-authority.ts b/src/lib/onboard/compute/podman/socket-authority.ts new file mode 100644 index 0000000000..f318afa53f --- /dev/null +++ b/src/lib/onboard/compute/podman/socket-authority.ts @@ -0,0 +1,170 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import path from "node:path"; + +interface PodmanSocketStat { + readonly dev: bigint | number; + readonly ino: bigint | number; + readonly mode: bigint | number; + readonly uid: bigint | number; + isDirectory(): boolean; + isSocket(): boolean; +} + +export interface PodmanSocketDirectoryAuthority { + readonly device: string; + readonly inode: string; + readonly mode: string; + readonly ownerUid: string; + readonly path: string; +} + +export interface PodmanSocketAuthority { + readonly directoryChain: readonly PodmanSocketDirectoryAuthority[]; + readonly device: string; + readonly inode: string; + readonly ownerUid: string; + readonly socketPath: string; +} + +export interface PodmanSocketAuthorityDeps { + readonly lstat?: (filePath: string) => PodmanSocketStat; + readonly uid?: number; +} + +function integerIdentity(value: bigint | number, label: string): string { + if (typeof value === "bigint") { + if (value >= 0n) return value.toString(10); + } else if (Number.isSafeInteger(value) && value >= 0) { + return String(value); + } + throw new Error(`Podman socket ${label} identity is invalid.`); +} + +function currentUid(configured: number | undefined): number { + const uid = configured ?? (typeof process.getuid === "function" ? process.getuid() : Number.NaN); + if (!Number.isSafeInteger(uid) || uid < 0) { + throw new Error("Podman socket authority requires the current Unix user ID."); + } + return uid; +} + +function integerValue(value: bigint | number, label: string): bigint { + if (typeof value === "bigint") { + if (value >= 0n) return value; + } else if (Number.isSafeInteger(value) && value >= 0) { + return BigInt(value); + } + throw new Error(`Podman socket ${label} identity is invalid.`); +} + +function normalizedSocketPath(socketPath: string): string { + const normalized = socketPath.trim(); + if ( + !normalized || + !path.isAbsolute(normalized) || + path.normalize(normalized) !== normalized || + /[\0\r\n]/u.test(normalized) + ) { + throw new Error("Podman socket authority requires a safe normalized absolute path."); + } + return normalized; +} + +function directoryChain( + socketPath: string, + uid: number, + lstat: (filePath: string) => PodmanSocketStat, +): readonly PodmanSocketDirectoryAuthority[] { + const immediateDirectory = path.dirname(socketPath); + const directories: string[] = []; + for (let directory = immediateDirectory; ; directory = path.dirname(directory)) { + directories.push(directory); + const parent = path.dirname(directory); + if (parent === directory) break; + } + + return Object.freeze( + directories.map((directory, index) => { + const stat = lstat(directory); + if (!stat.isDirectory()) { + throw new Error(`Podman socket path component '${directory}' is not a real directory.`); + } + const ownerUid = integerIdentity(stat.uid, "directory owner"); + if (index === 0 && ownerUid !== String(uid)) { + throw new Error( + `Podman socket directory is owned by uid ${ownerUid}; expected current uid ${String(uid)}.`, + ); + } + const mode = integerValue(stat.mode, "directory mode"); + if ((mode & 0o022n) !== 0n) { + throw new Error( + `Podman socket path component '${directory}' is writable by another user or group.`, + ); + } + return Object.freeze({ + device: integerIdentity(stat.dev, "directory device"), + inode: integerIdentity(stat.ino, "directory inode"), + mode: mode.toString(10), + ownerUid, + path: directory, + }); + }), + ); +} + +export function capturePodmanSocketAuthority( + socketPath: string, + deps: PodmanSocketAuthorityDeps = {}, +): PodmanSocketAuthority { + const normalized = normalizedSocketPath(socketPath); + const lstat = + deps.lstat ?? + ((filePath: string): PodmanSocketStat => fs.lstatSync(filePath, { bigint: true })); + const stat = lstat(normalized); + if (!stat.isSocket()) { + throw new Error("Podman socket authority path is not a Unix socket."); + } + const uid = currentUid(deps.uid); + const ownerUid = integerIdentity(stat.uid, "owner"); + if (ownerUid !== String(uid)) { + throw new Error( + `Podman socket authority is owned by uid ${ownerUid}; expected current uid ${String(uid)}.`, + ); + } + return Object.freeze({ + directoryChain: directoryChain(normalized, uid, lstat), + device: integerIdentity(stat.dev, "device"), + inode: integerIdentity(stat.ino, "inode"), + ownerUid, + socketPath: normalized, + }); +} + +export function assertPodmanSocketAuthority( + expected: PodmanSocketAuthority, + deps: PodmanSocketAuthorityDeps = {}, +): void { + const actual = capturePodmanSocketAuthority(expected.socketPath, deps); + if ( + actual.device !== expected.device || + actual.inode !== expected.inode || + actual.ownerUid !== expected.ownerUid || + actual.directoryChain.length !== expected.directoryChain.length || + actual.directoryChain.some((component, index) => { + const pinned = expected.directoryChain[index]; + return ( + !pinned || + component.device !== pinned.device || + component.inode !== pinned.inode || + component.mode !== pinned.mode || + component.ownerUid !== pinned.ownerUid || + component.path !== pinned.path + ); + }) + ) { + throw new Error("Podman socket authority changed after it was qualified."); + } +} diff --git a/src/lib/onboard/compute/podman/watcher-controller.test.ts b/src/lib/onboard/compute/podman/watcher-controller.test.ts new file mode 100644 index 0000000000..2922abe036 --- /dev/null +++ b/src/lib/onboard/compute/podman/watcher-controller.test.ts @@ -0,0 +1,214 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; +import { + createPodmanManagedGatewayWatcherController, + type PodmanGatewayWatcherSnapshot, + type PodmanManagedGatewayWatcherControllerDeps, +} from "./watcher-controller"; + +const ORIGINAL = Object.freeze({ + gatewayName: "nemoclaw", + gatewayPort: 8080, + launchIdentity: "launch-sha256", + ownerIdentity: "service-unit-sha256", + ownerKind: "managed-service", + pid: 4100, + processStartIdentity: "proc-start-100", +} as const satisfies PodmanGatewayWatcherSnapshot); + +const RESUMED = Object.freeze({ + ...ORIGINAL, + pid: 4200, + processStartIdentity: "proc-start-200", +} as const satisfies PodmanGatewayWatcherSnapshot); + +function harness(overrides: Partial = {}) { + let ownerStopped = false; + let watchers: PodmanGatewayWatcherSnapshot[] = [ORIGINAL]; + const alive = new Set(["4100:proc-start-100"]); + const healthy = new Set(["4100:proc-start-100"]); + const key = (entry: PodmanGatewayWatcherSnapshot) => + `${String(entry.pid)}:${entry.processStartIdentity}`; + const stopExactOwner = vi.fn((entry: PodmanGatewayWatcherSnapshot) => { + expect(entry).toEqual(ORIGINAL); + alive.delete(key(entry)); + healthy.delete(key(entry)); + watchers = []; + ownerStopped = true; + }); + const resumeSameOwner = vi.fn(() => { + ownerStopped = false; + watchers = [RESUMED]; + alive.add(key(RESUMED)); + healthy.add(key(RESUMED)); + }); + const deps: PodmanManagedGatewayWatcherControllerDeps = { + captureCurrent: () => ORIGINAL, + listTargetWatchers: () => watchers, + isProcessInstanceAlive: (entry) => alive.has(key(entry)), + isOwnerStopped: () => ownerStopped, + stopExactOwner, + resumeSameOwner, + isHealthy: (entry) => healthy.has(key(entry)), + now: (() => { + let now = 0; + return () => now; + })(), + resumePollIntervalMs: 0, + resumeTimeoutMs: 1_000, + sleep: () => {}, + ...overrides, + }; + return { + alive, + controller: createPodmanManagedGatewayWatcherController(deps), + deps, + healthy, + ownerStopped: () => ownerStopped, + resumeSameOwner, + setOwnerStopped: (value: boolean) => { + ownerStopped = value; + }, + setWatchers: (value: PodmanGatewayWatcherSnapshot[]) => { + watchers = value; + }, + stopExactOwner, + watchers: () => watchers, + }; +} + +describe("Podman managed host gateway watcher controller", () => { + it("holds an opaque stopped-owner lease and resumes the same launch identity healthy", () => { + const fake = harness(); + const lease = fake.controller.quiesceAndProve(); + + expect(fake.stopExactOwner).toHaveBeenCalledOnce(); + expect(fake.ownerStopped()).toBe(true); + expect(fake.watchers()).toEqual([]); + lease.assertStillStopped(); + lease.assertStillStopped(); + + lease.resumeAndProve(); + expect(fake.resumeSameOwner).toHaveBeenCalledOnce(); + expect(fake.watchers()).toEqual([RESUMED]); + expect(fake.ownerStopped()).toBe(false); + expect(() => lease.assertStillStopped()).toThrow("already been released"); + }); + + it("refuses an ambiguous target before asking any owner to stop", () => { + const fake = harness({ + listTargetWatchers: () => [ + ORIGINAL, + { ...ORIGINAL, pid: 4101, processStartIdentity: "proc-start-101" }, + ], + }); + + expect(() => fake.controller.quiesceAndProve()).toThrow("exactly one target-bound"); + expect(fake.stopExactOwner).not.toHaveBeenCalled(); + }); + + it("refuses a captured watcher that is not healthy before cutover", () => { + const fake = harness({ isHealthy: () => false }); + + expect(() => fake.controller.quiesceAndProve()).toThrow("not healthy before cutover"); + expect(fake.stopExactOwner).not.toHaveBeenCalled(); + }); + + it("detects a service respawn under the lease and does not launch a duplicate", () => { + const fake = harness(); + const lease = fake.controller.quiesceAndProve(); + fake.setOwnerStopped(false); + fake.setWatchers([RESUMED]); + fake.alive.add("4200:proc-start-200"); + fake.healthy.add("4200:proc-start-200"); + + expect(() => lease.assertStillStopped()).toThrow("lifecycle owner is not proven stopped"); + expect(fake.resumeSameOwner).not.toHaveBeenCalled(); + expect(fake.watchers()).toEqual([RESUMED]); + }); + + it("does not restart a stopped owner over an unexplained target watcher", () => { + const fake = harness(); + const lease = fake.controller.quiesceAndProve(); + fake.setWatchers([RESUMED]); + fake.alive.add("4200:proc-start-200"); + fake.healthy.add("4200:proc-start-200"); + + expect(() => lease.resumeAndProve()).toThrow("watcher appeared while the stop lease was held"); + expect(fake.resumeSameOwner).not.toHaveBeenCalled(); + }); + + it("fails closed when resume produces a different launch identity", () => { + const foreign = { ...RESUMED, launchIdentity: "different-launch" }; + const fake = harness({ + resumeSameOwner: () => { + fake.setOwnerStopped(false); + fake.setWatchers([foreign]); + fake.alive.add("4200:proc-start-200"); + fake.healthy.add("4200:proc-start-200"); + }, + }); + const lease = fake.controller.quiesceAndProve(); + + expect(() => lease.resumeAndProve()).toThrow("does not match the captured"); + }); + + it("fails closed when the same resumed owner never becomes healthy", () => { + const fake = harness({ + isHealthy: (entry) => entry.pid === ORIGINAL.pid, + resumeSameOwner: () => { + fake.setOwnerStopped(false); + fake.setWatchers([RESUMED]); + fake.alive.add("4200:proc-start-200"); + }, + }); + const lease = fake.controller.quiesceAndProve(); + + expect(() => lease.resumeAndProve()).toThrow("did not resume one exact healthy"); + }); + + it("accepts independent exact health proof when the resume command loses its reply", () => { + const fake = harness({ + resumeSameOwner: () => { + fake.setOwnerStopped(false); + fake.setWatchers([RESUMED]); + fake.alive.add("4200:proc-start-200"); + fake.healthy.add("4200:proc-start-200"); + throw new Error("service-manager reply lost"); + }, + }); + const lease = fake.controller.quiesceAndProve(); + + expect(() => lease.resumeAndProve()).not.toThrow(); + expect(fake.watchers()).toEqual([RESUMED]); + }); + + it("restores the exact watcher itself when the owner stop operation throws", () => { + const fake = harness({ + stopExactOwner: () => { + fake.alive.delete("4100:proc-start-100"); + fake.healthy.delete("4100:proc-start-100"); + fake.setWatchers([]); + fake.setOwnerStopped(true); + throw new Error("systemctl stop lost its reply"); + }, + }); + + expect(() => fake.controller.quiesceAndProve()).toThrow( + "The exact captured watcher was restored", + ); + expect(fake.resumeSameOwner).toHaveBeenCalledOnce(); + expect(fake.watchers()).toEqual([RESUMED]); + }); + + it("rejects target drift returned by watcher enumeration", () => { + const fake = harness({ + listTargetWatchers: () => [{ ...ORIGINAL, gatewayPort: 8081 }], + }); + + expect(() => fake.controller.quiesceAndProve()).toThrow("different gateway target"); + expect(fake.stopExactOwner).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/onboard/compute/podman/watcher-controller.ts b/src/lib/onboard/compute/podman/watcher-controller.ts new file mode 100644 index 0000000000..57944fe5e2 --- /dev/null +++ b/src/lib/onboard/compute/podman/watcher-controller.ts @@ -0,0 +1,411 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + createPodmanOpenShellWatcherController, + PodmanManagedSandboxRecreateError, + type PodmanOpenShellWatcherController, +} from "./sandbox-recreate"; + +const DEFAULT_RESUME_TIMEOUT_MS = 30_000; +const DEFAULT_RESUME_POLL_INTERVAL_MS = 250; +const MAX_OPAQUE_IDENTITY_LENGTH = 4_096; +const SAFE_GATEWAY_NAME = /^[A-Za-z0-9][A-Za-z0-9_.-]*$/u; +const CONTROL_CHARACTER = /[\u0000-\u001f\u007f-\u009f]/u; + +export type PodmanGatewayWatcherOwnerKind = "managed-service" | "standalone"; + +/** + * Immutable, non-secret evidence identifying one target-bound host gateway + * watcher and the lifecycle owner capable of stopping and resuming it. + * + * `ownerIdentity` and `launchIdentity` are deliberately opaque. The production + * caller derives them from existing trusted service identity or the exact + * standalone launch/runtime identity; this layer compares them but never logs + * them. `processStartIdentity` must distinguish PID reuse (Linux `/proc` start + * time, or an equivalent stable platform primitive). + */ +export interface PodmanGatewayWatcherSnapshot { + readonly gatewayName: string; + readonly gatewayPort: number; + readonly launchIdentity: string; + readonly ownerIdentity: string; + readonly ownerKind: PodmanGatewayWatcherOwnerKind; + readonly pid: number; + readonly processStartIdentity: string; +} + +export interface PodmanManagedGatewayWatcherControllerDeps { + /** + * Resolve the exact healthy watcher currently serving the target. Must throw + * rather than choose when PID, listener, runtime, or lifecycle-owner evidence + * is missing or ambiguous. + */ + readonly captureCurrent: () => PodmanGatewayWatcherSnapshot; + /** + * Return every host gateway watcher bound to this exact gateway target. + * Unknown or unclassifiable target listeners must make this callback throw, + * not disappear from the result. + */ + readonly listTargetWatchers: ( + target: Readonly>, + ) => readonly PodmanGatewayWatcherSnapshot[]; + /** Prove whether this exact PID/start-time process instance still exists. */ + readonly isProcessInstanceAlive: (snapshot: PodmanGatewayWatcherSnapshot) => boolean; + /** + * Prove the captured lifecycle owner is stopped. For a managed service this + * means the same trusted unit/formula is inactive, not merely that MainPID + * changed. For a standalone launch slot this means no process with the + * captured target, owner, and launch identity is live; it becomes false again + * when the exact resumed child appears. + */ + readonly isOwnerStopped: (snapshot: PodmanGatewayWatcherSnapshot) => boolean; + /** + * Stop only the captured owner. Managed services must be stopped through the + * exact service manager identity so automatic restart cannot escape the + * lease; standalone launches may stop only the captured process instance. + */ + readonly stopExactOwner: (snapshot: PodmanGatewayWatcherSnapshot) => void; + /** + * Idempotently resume the same captured service/standalone launch identity. + * It must never start a second watcher over an existing target listener. + */ + readonly resumeSameOwner: (snapshot: PodmanGatewayWatcherSnapshot) => void; + /** + * Prove the resumed target is operational using the production OpenShell + * gateway health path. Process liveness alone is not health. + */ + readonly isHealthy: (snapshot: PodmanGatewayWatcherSnapshot) => boolean; + readonly now?: () => number; + readonly resumePollIntervalMs?: number; + readonly resumeTimeoutMs?: number; + readonly sleep?: (milliseconds: number) => void; +} + +class WatcherProofError extends Error {} + +function sleepMs(milliseconds: number): void { + if (milliseconds <= 0 || !Number.isFinite(milliseconds)) return; + const buffer = new Int32Array(new SharedArrayBuffer(4)); + Atomics.wait(buffer, 0, 0, milliseconds); +} + +function safeOpaqueIdentity(value: string, field: string): string { + if ( + typeof value !== "string" || + value.length === 0 || + value.length > MAX_OPAQUE_IDENTITY_LENGTH || + value !== value.trim() || + CONTROL_CHARACTER.test(value) + ) { + throw new PodmanManagedSandboxRecreateError( + `Podman OpenShell watcher ${field} is missing or unsafe.`, + ); + } + return value; +} + +function snapshot( + value: PodmanGatewayWatcherSnapshot, + expectedTarget?: Readonly>, +): Readonly { + if (!value || typeof value !== "object") { + throw new PodmanManagedSandboxRecreateError( + "Podman OpenShell watcher identity proof did not return an object.", + ); + } + if (!SAFE_GATEWAY_NAME.test(value.gatewayName)) { + throw new PodmanManagedSandboxRecreateError( + "Podman OpenShell watcher identity has an invalid gateway name.", + ); + } + if ( + !Number.isSafeInteger(value.gatewayPort) || + value.gatewayPort < 1 || + value.gatewayPort > 65_535 + ) { + throw new PodmanManagedSandboxRecreateError( + "Podman OpenShell watcher identity has an invalid gateway port.", + ); + } + if (!Number.isSafeInteger(value.pid) || value.pid <= 0) { + throw new PodmanManagedSandboxRecreateError( + "Podman OpenShell watcher identity has an invalid process ID.", + ); + } + if (value.ownerKind !== "managed-service" && value.ownerKind !== "standalone") { + throw new PodmanManagedSandboxRecreateError( + "Podman OpenShell watcher identity has an unsupported lifecycle owner.", + ); + } + if ( + expectedTarget && + (value.gatewayName !== expectedTarget.gatewayName || + value.gatewayPort !== expectedTarget.gatewayPort) + ) { + throw new PodmanManagedSandboxRecreateError( + "Podman OpenShell watcher enumeration returned a different gateway target.", + ); + } + return Object.freeze({ + gatewayName: value.gatewayName, + gatewayPort: value.gatewayPort, + launchIdentity: safeOpaqueIdentity(value.launchIdentity, "launch identity"), + ownerIdentity: safeOpaqueIdentity(value.ownerIdentity, "owner identity"), + ownerKind: value.ownerKind, + pid: value.pid, + processStartIdentity: safeOpaqueIdentity(value.processStartIdentity, "process-start identity"), + }); +} + +function sameProcessInstance( + left: Readonly, + right: Readonly, +): boolean { + return ( + left.pid === right.pid && + left.processStartIdentity === right.processStartIdentity && + sameLaunchOwner(left, right) + ); +} + +function sameLaunchOwner( + left: Readonly, + right: Readonly, +): boolean { + return ( + left.gatewayName === right.gatewayName && + left.gatewayPort === right.gatewayPort && + left.ownerKind === right.ownerKind && + left.ownerIdentity === right.ownerIdentity && + left.launchIdentity === right.launchIdentity + ); +} + +function readTargetWatchers( + receipt: Readonly, + deps: PodmanManagedGatewayWatcherControllerDeps, +): readonly Readonly[] { + const target = { + gatewayName: receipt.gatewayName, + gatewayPort: receipt.gatewayPort, + } as const; + const watchers = deps.listTargetWatchers(target); + if (!Array.isArray(watchers)) { + throw new PodmanManagedSandboxRecreateError( + "Podman OpenShell watcher enumeration did not return an array.", + ); + } + return watchers.map((entry) => snapshot(entry, target)); +} + +function requireExclusiveCurrent( + receipt: Readonly, + deps: PodmanManagedGatewayWatcherControllerDeps, +): void { + const watchers = readTargetWatchers(receipt, deps); + if (watchers.length !== 1 || !sameProcessInstance(receipt, watchers[0] as typeof receipt)) { + throw new PodmanManagedSandboxRecreateError( + "Podman sandbox recreation requires exactly one target-bound OpenShell watcher matching the captured process and lifecycle owner.", + ); + } + if (!deps.isProcessInstanceAlive(receipt) || deps.isOwnerStopped(receipt)) { + throw new PodmanManagedSandboxRecreateError( + "The captured Podman OpenShell watcher or its lifecycle owner changed before cutover.", + ); + } + if (!deps.isHealthy(receipt)) { + throw new PodmanManagedSandboxRecreateError( + "The captured Podman OpenShell watcher is not healthy before cutover.", + ); + } +} + +function assertStopped( + receipt: Readonly, + deps: PodmanManagedGatewayWatcherControllerDeps, +): void { + if (deps.isProcessInstanceAlive(receipt)) { + throw new WatcherProofError("the captured OpenShell watcher process instance is still alive"); + } + if (!deps.isOwnerStopped(receipt)) { + throw new WatcherProofError( + "the captured OpenShell watcher lifecycle owner is not proven stopped", + ); + } + if (readTargetWatchers(receipt, deps).length !== 0) { + throw new WatcherProofError( + "a target-bound OpenShell watcher appeared while the stop lease was held", + ); + } +} + +function readinessWaitOptions(deps: PodmanManagedGatewayWatcherControllerDeps) { + const timeoutMs = deps.resumeTimeoutMs ?? DEFAULT_RESUME_TIMEOUT_MS; + const pollIntervalMs = deps.resumePollIntervalMs ?? DEFAULT_RESUME_POLL_INTERVAL_MS; + if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) { + throw new PodmanManagedSandboxRecreateError( + "Podman OpenShell watcher resume timeout must be positive.", + ); + } + if (!Number.isFinite(pollIntervalMs) || pollIntervalMs < 0) { + throw new PodmanManagedSandboxRecreateError( + "Podman OpenShell watcher resume poll interval must be non-negative.", + ); + } + const now = deps.now ?? Date.now; + const startedAt = now(); + if (!Number.isFinite(startedAt)) { + throw new PodmanManagedSandboxRecreateError( + "Podman OpenShell watcher resume clock is unavailable.", + ); + } + const maxAttempts = Math.max(1, Math.ceil(timeoutMs / Math.max(1, pollIntervalMs)) + 1); + return { + deadlineMs: startedAt + timeoutMs, + maxAttempts, + now, + pollIntervalMs, + sleep: deps.sleep ?? sleepMs, + }; +} + +function waitForExactHealthy( + condition: () => boolean, + options: ReturnType, +): boolean { + for (let attempt = 0; attempt < options.maxAttempts; attempt += 1) { + const now = options.now(); + if (!Number.isFinite(now) || now >= options.deadlineMs) return false; + if (condition()) return true; + if (attempt + 1 >= options.maxAttempts) return false; + const remainingMs = options.deadlineMs - options.now(); + if (!Number.isFinite(remainingMs) || remainingMs <= 0) return false; + options.sleep(Math.min(options.pollIntervalMs, remainingMs)); + } + return false; +} + +function exactHealthyReplacement( + receipt: Readonly, + deps: PodmanManagedGatewayWatcherControllerDeps, +): Readonly | null { + if (deps.isProcessInstanceAlive(receipt) || deps.isOwnerStopped(receipt)) return null; + const watchers = readTargetWatchers(receipt, deps); + if (watchers.length === 0) return null; + if (watchers.length !== 1) { + throw new WatcherProofError( + "multiple target-bound OpenShell watchers appeared while resuming the captured owner", + ); + } + const resumed = watchers[0] as Readonly; + if (!sameLaunchOwner(receipt, resumed)) { + throw new WatcherProofError( + "the resumed OpenShell watcher does not match the captured lifecycle owner and launch identity", + ); + } + if (!deps.isProcessInstanceAlive(resumed) || !deps.isHealthy(resumed)) return null; + return resumed; +} + +function resumeAndProve( + receipt: Readonly, + deps: PodmanManagedGatewayWatcherControllerDeps, +): void { + const existing = readTargetWatchers(receipt, deps); + if (existing.length > 1) { + throw new WatcherProofError( + "multiple target-bound OpenShell watchers exist; refusing to start another", + ); + } + if (existing.length === 1) { + const current = existing[0] as Readonly; + if (!sameLaunchOwner(receipt, current)) { + throw new WatcherProofError( + "a different target-bound OpenShell watcher exists; refusing to replace it", + ); + } + if ( + !deps.isOwnerStopped(receipt) && + deps.isProcessInstanceAlive(current) && + deps.isHealthy(current) + ) { + return; + } + throw new WatcherProofError( + "a target-bound OpenShell watcher exists without the exact captured owner and healthy process proof; refusing to start another", + ); + } + + if (!deps.isOwnerStopped(receipt)) { + throw new WatcherProofError( + "the captured lifecycle owner is neither stopped nor serving an exact healthy watcher", + ); + } + let resumeError: unknown = null; + try { + deps.resumeSameOwner(receipt); + } catch (error) { + // A service manager or spawn boundary can lose its reply after completing + // the requested start. Do not report failure until the same exact launch + // identity also fails the independent bounded process/health proof. + resumeError = error; + } + let resumed: Readonly | null = null; + const healthy = waitForExactHealthy(() => { + resumed = exactHealthyReplacement(receipt, deps); + return resumed !== null; + }, readinessWaitOptions(deps)); + if (!healthy || resumed === null) { + throw new WatcherProofError( + `the same OpenShell watcher lifecycle owner did not resume one exact healthy target-bound watcher within the bounded deadline${ + resumeError === null ? "" : " after its start operation failed" + }`, + ); + } +} + +/** + * Construct the production-safe host watcher lease used by native Podman + * sandbox recreation. + * + * The caller supplies platform/runtime probes, but cannot weaken the sequence: + * exclusive healthy identity -> exact owner stop -> repeated absence proof -> + * same owner/launch resume -> exact process and health proof. This deliberately + * excludes externally supervised gateways: only the lifecycle owner captured + * as NemoClaw-managed may be quiesced. + */ +export function createPodmanManagedGatewayWatcherController( + deps: PodmanManagedGatewayWatcherControllerDeps, +): PodmanOpenShellWatcherController { + return createPodmanOpenShellWatcherController({ + stopAndProve: () => { + const receipt = snapshot(deps.captureCurrent()); + requireExclusiveCurrent(receipt, deps); + try { + deps.stopExactOwner(receipt); + return receipt; + } catch (error) { + let recoveryError: unknown = null; + try { + resumeAndProve(receipt, deps); + } catch (caught) { + recoveryError = caught; + } + const message = error instanceof Error ? error.message : String(error); + const recoveryMessage = + recoveryError === null + ? "The exact captured watcher was restored." + : `Exact watcher recovery failed: ${ + recoveryError instanceof Error ? recoveryError.message : String(recoveryError) + }.`; + throw new PodmanManagedSandboxRecreateError( + `Stopping the exact Podman OpenShell watcher failed: ${message}. ${recoveryMessage}`, + recoveryError === null, + ); + } + }, + assertStopped: (receipt) => assertStopped(receipt, deps), + resumeAndProve: (receipt) => resumeAndProve(receipt, deps), + }); +} diff --git a/src/lib/onboard/compute/podman/watcher-runtime.test.ts b/src/lib/onboard/compute/podman/watcher-runtime.test.ts new file mode 100644 index 0000000000..bf09ab577e --- /dev/null +++ b/src/lib/onboard/compute/podman/watcher-runtime.test.ts @@ -0,0 +1,162 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import { + capturePodmanGatewayListenerPids, + createPodmanProductionWatcherController, +} from "./watcher-runtime"; + +describe("Podman production watcher runtime", () => { + it("leases and resumes the exact standalone launch slot", () => { + let listenerPids = [101]; + const starts = new Map([[101, "1001"]]); + const stop = vi.fn((pid: number) => { + listenerPids = []; + starts.delete(pid); + }); + const resume = vi.fn(() => { + listenerPids = [202]; + starts.set(202, "2002"); + return 202; + }); + const controller = createPodmanProductionWatcherController({ + gatewayName: "nemoclaw", + gatewayPort: 8080, + getRuntimeDrift: vi.fn(() => null), + isGatewayHealthy: vi.fn(() => true), + standalone: { + launchIdentity: "standalone-launch", + ownerIdentity: "standalone-owner", + readOwnedPid: () => listenerPids[0] ?? null, + resume, + stop, + }, + deps: { + captureListenerPids: () => listenerPids, + captureProcessStartIdentity: (pid) => starts.get(pid) ?? null, + }, + readiness: { now: () => 0, sleep: vi.fn() }, + }); + + const lease = controller.quiesceAndProve(); + lease.assertStillStopped(); + lease.resumeAndProve(); + + expect(stop).toHaveBeenCalledWith(101); + expect(resume).toHaveBeenCalledOnce(); + expect(listenerPids).toEqual([202]); + }); + + it("binds a package-managed service across a new process generation", () => { + type Receipt = { pid: number; start: string }; + let active = true; + let receipt: Receipt = { pid: 303, start: "3003" }; + let listenerPids = [303]; + const lifecycle = { + assertInactive: vi.fn(() => { + if (active) throw new Error("still active"); + }), + captureActive: vi.fn(() => receipt), + describe: vi.fn((value: Receipt) => ({ + launchIdentity: "service-launch", + ownerIdentity: "service-owner", + pid: value.pid, + processStartIdentity: value.start, + })), + resumeAndProve: vi.fn(() => { + active = true; + receipt = { pid: 404, start: "4004" }; + listenerPids = [404]; + return receipt; + }), + stopAndProveInactive: vi.fn(() => { + active = false; + listenerPids = []; + }), + }; + const controller = createPodmanProductionWatcherController({ + gatewayName: "nemoclaw", + gatewayPort: 8080, + getRuntimeDrift: vi.fn(() => null), + isGatewayHealthy: vi.fn(() => true), + service: lifecycle, + standalone: { + launchIdentity: "unused", + ownerIdentity: "unused", + readOwnedPid: () => null, + resume: vi.fn(() => 0), + stop: vi.fn(), + }, + deps: { + captureListenerPids: () => listenerPids, + captureProcessStartIdentity: (pid) => + pid === receipt.pid && active ? receipt.start : null, + }, + readiness: { now: () => 0, sleep: vi.fn() }, + }); + + const lease = controller.quiesceAndProve(); + lease.assertStillStopped(); + lease.resumeAndProve(); + + expect(lifecycle.stopAndProveInactive).toHaveBeenCalledWith({ + pid: 303, + start: "3003", + }); + expect(lifecycle.resumeAndProve).toHaveBeenCalledOnce(); + expect(listenerPids).toEqual([404]); + }); + + it("fails before stop when the exact listener has runtime drift", () => { + const stop = vi.fn(); + const controller = createPodmanProductionWatcherController({ + gatewayName: "nemoclaw", + gatewayPort: 8080, + getRuntimeDrift: vi.fn(() => ({ reason: "environment changed" })), + isGatewayHealthy: vi.fn(() => true), + standalone: { + launchIdentity: "standalone-launch", + ownerIdentity: "standalone-owner", + readOwnedPid: () => 101, + resume: vi.fn(() => 202), + stop, + }, + deps: { + captureListenerPids: () => [101], + captureProcessStartIdentity: () => "1001", + }, + }); + + expect(() => controller.quiesceAndProve()).toThrow("runtime identity drifted"); + expect(stop).not.toHaveBeenCalled(); + }); +}); + +describe("Podman gateway listener scan", () => { + it("returns every unique listener PID from a complete lsof scan", () => { + const run = vi.fn(() => ({ status: 0, stdout: "22\n11\n22\n" })); + expect(capturePodmanGatewayListenerPids(8080, { run })).toEqual([11, 22]); + expect(run).toHaveBeenCalledOnce(); + }); + + it("accepts an empty lsof scan only after an independent free-port proof", () => { + const run = vi + .fn() + .mockReturnValueOnce({ status: 1, stdout: "" }) + .mockReturnValueOnce({ status: 0, stdout: "" }); + expect(capturePodmanGatewayListenerPids(8080, { run })).toEqual([]); + expect(run).toHaveBeenCalledTimes(2); + }); + + it("rejects incomplete listener visibility", () => { + const run = vi + .fn() + .mockReturnValueOnce({ status: 1, stderr: "hidden" }) + .mockReturnValueOnce({ status: 1 }); + expect(() => capturePodmanGatewayListenerPids(8080, { run })).toThrow( + "could not completely enumerate", + ); + }); +}); diff --git a/src/lib/onboard/compute/podman/watcher-runtime.ts b/src/lib/onboard/compute/podman/watcher-runtime.ts new file mode 100644 index 0000000000..1029b63c74 --- /dev/null +++ b/src/lib/onboard/compute/podman/watcher-runtime.ts @@ -0,0 +1,404 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { type SpawnSyncOptions, spawnSync } from "node:child_process"; + +import { captureLinuxHostProcessIdentity } from "../host-process-identity"; +import { + createPodmanManagedGatewayWatcherController, + type PodmanGatewayWatcherSnapshot, + type PodmanManagedGatewayWatcherControllerDeps, +} from "./watcher-controller"; + +const PORT_SCAN_TIMEOUT_MS = 5_000; + +interface CommandResult { + readonly error?: Error; + readonly status: number | null; + readonly stderr?: Buffer | string | null; + readonly stdout?: Buffer | string | null; +} + +export type RunPodmanWatcherRuntimeCommand = ( + command: string, + args: readonly string[], + options: SpawnSyncOptions, +) => CommandResult; + +export interface PodmanManagedServiceWatcherDescriptor { + readonly launchIdentity: string; + readonly ownerIdentity: string; + readonly pid: number; + readonly processStartIdentity: string; +} + +export interface PodmanManagedServiceWatcherLifecycle { + assertInactive(receipt: TReceipt): void; + captureActive(): TReceipt | null; + describe(receipt: TReceipt): PodmanManagedServiceWatcherDescriptor; + resumeAndProve(receipt: TReceipt): TReceipt; + stopAndProveInactive(receipt: TReceipt): void; +} + +export interface PodmanStandaloneWatcherLifecycle { + readonly launchIdentity: string; + readonly ownerIdentity: string; + readOwnedPid(): number | null; + resume(): number; + stop(pid: number): void; +} + +export interface PodmanProductionWatcherControllerOptions { + readonly gatewayName: string; + readonly gatewayPort: number; + readonly getRuntimeDrift: (pid: number) => { readonly reason: string } | null; + readonly isGatewayHealthy: () => boolean; + readonly service?: PodmanManagedServiceWatcherLifecycle; + readonly standalone: PodmanStandaloneWatcherLifecycle; + readonly deps?: { + readonly captureListenerPids?: (port: number) => readonly number[]; + readonly captureProcessStartIdentity?: (pid: number) => string | null; + readonly run?: RunPodmanWatcherRuntimeCommand; + }; + readonly readiness?: Pick< + PodmanManagedGatewayWatcherControllerDeps, + "now" | "resumePollIntervalMs" | "resumeTimeoutMs" | "sleep" + >; +} + +function output(value: Buffer | string | null | undefined): string { + if (typeof value === "string") return value; + return Buffer.isBuffer(value) ? value.toString("utf8") : ""; +} + +function defaultRun( + command: string, + args: readonly string[], + options: SpawnSyncOptions, +): CommandResult { + return spawnSync(command, [...args], options); +} + +function parseListenerPids(text: string): number[] { + const pids = text + .split(/\r?\n/u) + .map((line) => Number.parseInt(line.trim(), 10)) + .filter((pid) => Number.isSafeInteger(pid) && pid > 0); + return [...new Set(pids)].sort((left, right) => left - right); +} + +function proveIpv4PortFree(port: number, run: RunPodmanWatcherRuntimeCommand): boolean { + const script = + "const net=require('node:net');const s=net.createServer();s.unref();" + + "s.once('error',()=>process.exit(1));" + + "s.listen({host:'0.0.0.0',port:Number(process.argv[1])}," + + "()=>s.close(()=>process.exit(0)));"; + const result = run(process.execPath, ["-e", script, String(port)], { + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + timeout: PORT_SCAN_TIMEOUT_MS, + }); + return result.status === 0; +} + +/** + * Complete synchronous listener proof for the create-stream polling boundary. + * An empty lsof result is accepted only when an independent wildcard bind also + * proves the target port is free. + */ +export function capturePodmanGatewayListenerPids( + port: number, + deps: { readonly run?: RunPodmanWatcherRuntimeCommand } = {}, +): readonly number[] { + if (!Number.isSafeInteger(port) || port < 1 || port > 65_535) { + throw new Error("Podman watcher listener scan requires a valid gateway port."); + } + const run = deps.run ?? defaultRun; + const result = run("lsof", ["-nP", "-tiTCP:" + String(port), "-sTCP:LISTEN"], { + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + timeout: PORT_SCAN_TIMEOUT_MS, + }); + if (result.status === 0) { + const pids = parseListenerPids(output(result.stdout)); + if (pids.length > 0) return pids; + throw new Error("Podman watcher listener scan returned success without a process ID."); + } + if (result.status === 1 && proveIpv4PortFree(port, run)) return []; + const detail = `${output(result.stderr)} ${result.error?.message ?? ""}`.trim(); + throw new Error( + `Podman watcher could not completely enumerate gateway port ${String(port)}${ + detail ? `: ${detail}` : "." + }`, + ); +} + +/** + * Capture Linux `/proc//stat` field 22 while handling spaces and closing + * parentheses in the process name. Two identical samples exclude a PID reuse + * race across the read boundary. + */ +export function captureLinuxProcessStartIdentity(pid: number): string | null { + return process.platform === "linux" + ? (captureLinuxHostProcessIdentity(pid)?.startIdentity ?? null) + : null; +} + +function exactStartIdentity(pid: number, capture: (pid: number) => string | null): string { + const identity = capture(pid); + if (!identity) { + throw new Error( + `Podman watcher could not prove the process-start identity for PID ${String(pid)}.`, + ); + } + return identity; +} + +function sameDescriptor( + left: Pick, + right: Pick, +): boolean { + return left.launchIdentity === right.launchIdentity && left.ownerIdentity === right.ownerIdentity; +} + +/** + * Bind the strict watcher lease to either the installed package service or the + * exact standalone launch slot NemoClaw owns. + */ +export function createPodmanProductionWatcherController( + options: PodmanProductionWatcherControllerOptions, +) { + const captureListeners = + options.deps?.captureListenerPids ?? + ((port: number) => capturePodmanGatewayListenerPids(port, options.deps)); + const captureStart = + options.deps?.captureProcessStartIdentity ?? captureLinuxProcessStartIdentity; + const target = { gatewayName: options.gatewayName, gatewayPort: options.gatewayPort }; + const serviceLifecycle = options.service ?? null; + let serviceReceipt: TServiceReceipt | null = null; + let serviceDescriptor: PodmanManagedServiceWatcherDescriptor | null = null; + let selectedOwner: "managed-service" | "standalone" | null = null; + let ownerStopped = false; + + const assertRuntime = (pid: number): void => { + const drift = options.getRuntimeDrift(pid); + if (drift) { + throw new Error(`Podman watcher runtime identity drifted: ${drift.reason}`); + } + }; + + const snapshot = ( + pid: number, + ownerKind: PodmanGatewayWatcherSnapshot["ownerKind"], + ownerIdentity: string, + launchIdentity: string, + expectedStartIdentity?: string, + ): PodmanGatewayWatcherSnapshot => { + assertRuntime(pid); + const processStartIdentity = exactStartIdentity(pid, captureStart); + if (expectedStartIdentity !== undefined && processStartIdentity !== expectedStartIdentity) { + throw new Error("Podman watcher process-start identity changed during capture."); + } + return { + ...target, + launchIdentity, + ownerIdentity, + ownerKind, + pid, + processStartIdentity, + }; + }; + + const requireOneListener = (): number => { + const pids = captureListeners(options.gatewayPort); + if (pids.length !== 1) { + throw new Error( + `Podman watcher requires exactly one gateway listener; found ${String(pids.length)}.`, + ); + } + return pids[0] as number; + }; + + const captureService = ( + receipt: TServiceReceipt, + listenerPid: number, + ): PodmanGatewayWatcherSnapshot => { + if (!serviceLifecycle) throw new Error("Podman watcher service lifecycle is unavailable."); + if (options.standalone.readOwnedPid() !== null) { + throw new Error( + "Podman watcher found both package-service and standalone lifecycle receipts.", + ); + } + const descriptor = serviceLifecycle.describe(receipt); + if (listenerPid !== descriptor.pid) { + throw new Error("Podman watcher package service does not own the exact gateway listener."); + } + const current = snapshot( + descriptor.pid, + "managed-service", + descriptor.ownerIdentity, + descriptor.launchIdentity, + descriptor.processStartIdentity, + ); + serviceReceipt = receipt; + serviceDescriptor = descriptor; + selectedOwner = "managed-service"; + return current; + }; + + const captureStandalone = (listenerPid: number): PodmanGatewayWatcherSnapshot => { + selectedOwner = "standalone"; + return snapshot( + listenerPid, + "standalone", + options.standalone.ownerIdentity, + options.standalone.launchIdentity, + ); + }; + + const captureCurrent = (): PodmanGatewayWatcherSnapshot => { + const listenerPid = requireOneListener(); + const receipt = serviceLifecycle?.captureActive() ?? null; + if (receipt) return captureService(receipt, listenerPid); + const standalonePid = options.standalone.readOwnedPid(); + if (standalonePid !== null) { + if (standalonePid !== listenerPid) { + throw new Error("Podman watcher standalone PID receipt does not own the exact listener."); + } + return captureStandalone(listenerPid); + } + throw new Error("Podman watcher could not prove a lifecycle owner for the exact listener."); + }; + + const listCurrent = (): readonly PodmanGatewayWatcherSnapshot[] => { + const pids = captureListeners(options.gatewayPort); + if (ownerStopped) { + if (selectedOwner === "managed-service" && serviceLifecycle && serviceReceipt) { + serviceLifecycle.assertInactive(serviceReceipt); + } + if (pids.length !== 0) { + throw new Error("A gateway listener appeared while the Podman watcher owner was stopped."); + } + return []; + } + if (pids.length === 0) return []; + if (pids.length !== 1) { + throw new Error("Podman watcher observed multiple target-bound gateway listeners."); + } + if (selectedOwner === null) { + throw new Error("Podman watcher lifecycle owner was not captured."); + } + if (selectedOwner === "standalone") { + if (options.standalone.readOwnedPid() !== pids[0]) { + throw new Error("Podman watcher standalone PID receipt drifted."); + } + return [ + snapshot( + pids[0] as number, + "standalone", + options.standalone.ownerIdentity, + options.standalone.launchIdentity, + ), + ]; + } + if (!serviceLifecycle) { + throw new Error("Podman watcher service lifecycle is unavailable."); + } + const receipt = serviceLifecycle.captureActive(); + if (!receipt) { + throw new Error("Podman watcher package service became inactive before cutover."); + } + const descriptor = serviceLifecycle.describe(receipt); + if ( + !serviceDescriptor || + !sameDescriptor(serviceDescriptor, descriptor) || + descriptor.pid !== pids[0] + ) { + throw new Error("Podman watcher package-service launch authority drifted."); + } + serviceReceipt = receipt; + return [ + snapshot( + descriptor.pid, + "managed-service", + descriptor.ownerIdentity, + descriptor.launchIdentity, + descriptor.processStartIdentity, + ), + ]; + }; + + return createPodmanManagedGatewayWatcherController({ + captureCurrent, + listTargetWatchers: () => listCurrent(), + isProcessInstanceAlive: (receipt) => { + const current = captureStart(receipt.pid); + return current !== null && current === receipt.processStartIdentity; + }, + isOwnerStopped: () => { + if (!ownerStopped) return false; + if (selectedOwner === "managed-service" && serviceLifecycle && serviceReceipt) { + serviceLifecycle.assertInactive(serviceReceipt); + } + return true; + }, + stopExactOwner: (receipt) => { + if (selectedOwner === "managed-service") { + if (!serviceLifecycle || !serviceReceipt) { + throw new Error("Podman watcher service receipt is missing."); + } + try { + serviceLifecycle.stopAndProveInactive(serviceReceipt); + ownerStopped = true; + } catch (error) { + try { + serviceLifecycle.assertInactive(serviceReceipt); + ownerStopped = true; + } catch { + ownerStopped = false; + } + throw error; + } + return; + } + options.standalone.stop(receipt.pid); + if ( + captureStart(receipt.pid) !== null || + captureListeners(options.gatewayPort).length !== 0 + ) { + throw new Error("Podman watcher standalone process stop could not be proven."); + } + ownerStopped = true; + }, + resumeSameOwner: () => { + if (!ownerStopped) { + throw new Error("Podman watcher lifecycle owner is not proven stopped."); + } + if (captureListeners(options.gatewayPort).length !== 0) { + throw new Error("Podman watcher target listener appeared before resume."); + } + if (selectedOwner === "managed-service") { + if (!serviceLifecycle || !serviceReceipt || !serviceDescriptor) { + throw new Error("Podman watcher service receipt is missing."); + } + const resumed = serviceLifecycle.resumeAndProve(serviceReceipt); + const resumedDescriptor = serviceLifecycle.describe(resumed); + if (!sameDescriptor(serviceDescriptor, resumedDescriptor)) { + throw new Error("Resumed Podman watcher service launch authority drifted."); + } + serviceReceipt = resumed; + } else { + const pid = options.standalone.resume(); + exactStartIdentity(pid, captureStart); + } + ownerStopped = false; + }, + isHealthy: (receipt) => { + const current = captureStart(receipt.pid); + if (current !== receipt.processStartIdentity) return false; + assertRuntime(receipt.pid); + return options.isGatewayHealthy(); + }, + ...options.readiness, + }); +} diff --git a/src/lib/onboard/compute/recovery-runtime.test.ts b/src/lib/onboard/compute/recovery-runtime.test.ts new file mode 100644 index 0000000000..23d89dd606 --- /dev/null +++ b/src/lib/onboard/compute/recovery-runtime.test.ts @@ -0,0 +1,167 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import type { ManagedGatewayRuntimeBinding } from "../docker-driver-gateway-config"; +import { + type ManagedGatewayRecoveryAdapterRegistry, + qualifyManagedGatewayRecoveryRuntime, + resolveManagedGatewayRecoveryRuntime, + supportsManagedGatewayRecoveryRuntime, +} from "./recovery-runtime"; + +function binding( + driverName = "podman", + values: ManagedGatewayRuntimeBinding["values"] = { + network_name: "openshell-custom", + socket_path: "/run/user/1000/podman/custom.sock", + supervisor_image: "ghcr.io/nvidia/openshell/supervisor@sha256:abc", + }, +): ManagedGatewayRuntimeBinding { + return { + version: 1, + driverName, + configSha256: "a".repeat(64), + values, + }; +} + +describe("managed gateway recovery runtime", () => { + it("recovers the exact persisted Podman socket, network, and supervisor image", () => { + expect( + resolveManagedGatewayRecoveryRuntime( + { + driverName: "podman", + environment: {}, + stateDir: "/state/podman", + }, + undefined, + () => binding(), + ), + ).toEqual({ + driverName: "podman", + environment: { + OPENSHELL_PODMAN_SOCKET: "/run/user/1000/podman/custom.sock", + OPENSHELL_PODMAN_NETWORK_NAME: "openshell-custom", + OPENSHELL_SUPERVISOR_IMAGE: "ghcr.io/nvidia/openshell/supervisor@sha256:abc", + }, + }); + }); + + it("rejects ambient runtime identity that conflicts with the protected binding", () => { + expect(() => + resolveManagedGatewayRecoveryRuntime( + { + driverName: "podman", + environment: { + OPENSHELL_PODMAN_SOCKET: "/run/user/1000/podman/other.sock", + }, + stateDir: "/state/podman", + }, + undefined, + () => binding(), + ), + ).toThrow("OPENSHELL_PODMAN_SOCKET does not match"); + }); + + it("fails closed on a missing or wrong-driver runtime binding", () => { + expect(() => + resolveManagedGatewayRecoveryRuntime( + { driverName: "podman", stateDir: "/state/missing" }, + undefined, + () => null, + ), + ).toThrow("Managed runtime binding is missing"); + expect(() => + resolveManagedGatewayRecoveryRuntime( + { driverName: "podman", stateDir: "/state/docker" }, + undefined, + () => binding("docker"), + ), + ).toThrow("does not match requested recovery driver"); + }); + + it("accepts an independently registered future runtime adapter", () => { + const qualifyEnvironment = vi.fn(() => ({ endpoint: "qualified" })); + const adapters: ManagedGatewayRecoveryAdapterRegistry = { + mxc: { + driverName: "mxc", + qualifyEnvironment, + resolveEnvironment: () => ({ OPENSHELL_MXC_ENDPOINT: "unix:///run/mxc.sock" }), + }, + }; + const runtime = resolveManagedGatewayRecoveryRuntime( + { driverName: "mxc", environment: {}, stateDir: "/state/mxc" }, + adapters, + () => binding("mxc", {}), + ); + expect(runtime).toEqual({ + driverName: "mxc", + environment: { OPENSHELL_MXC_ENDPOINT: "unix:///run/mxc.sock" }, + }); + expect(supportsManagedGatewayRecoveryRuntime("mxc", adapters)).toBe(true); + expect(qualifyManagedGatewayRecoveryRuntime(runtime, adapters)).toEqual({ + endpoint: "qualified", + }); + expect(qualifyEnvironment).toHaveBeenCalledExactlyOnceWith(runtime.environment); + }); + + it("rejects a recovery registry entry whose identity does not match its key", () => { + const adapters: ManagedGatewayRecoveryAdapterRegistry = { + mxc: { + driverName: "podman", + qualifyEnvironment: () => null, + resolveEnvironment: () => ({}), + }, + }; + + expect(() => supportsManagedGatewayRecoveryRuntime("mxc", adapters)).toThrow( + "mismatched identity", + ); + }); + + it("fails closed when an injected runtime cannot qualify its recovered environment", () => { + const qualificationFailure = new Error("MXC endpoint is not reachable"); + const qualifyEnvironment = vi.fn(() => { + throw qualificationFailure; + }); + const adapters: ManagedGatewayRecoveryAdapterRegistry = { + mxc: { + driverName: "mxc", + qualifyEnvironment, + resolveEnvironment: () => ({ OPENSHELL_MXC_ENDPOINT: "unix:///run/mxc.sock" }), + }, + }; + const runtime = resolveManagedGatewayRecoveryRuntime( + { driverName: "mxc", environment: {}, stateDir: "/state/mxc" }, + adapters, + () => binding("mxc", {}), + ); + + expect(() => qualifyManagedGatewayRecoveryRuntime(runtime, adapters)).toThrow( + qualificationFailure, + ); + expect(qualifyEnvironment).toHaveBeenCalledExactlyOnceWith(runtime.environment); + }); + + it("rejects a malformed runtime adapter without an environment qualifier", () => { + const adapters = { + mxc: { + driverName: "mxc", + resolveEnvironment: () => ({ OPENSHELL_MXC_ENDPOINT: "unix:///run/mxc.sock" }), + }, + } as unknown as ManagedGatewayRecoveryAdapterRegistry; + const runtime = { + driverName: "mxc", + environment: { OPENSHELL_MXC_ENDPOINT: "unix:///run/mxc.sock" }, + }; + + expect(() => supportsManagedGatewayRecoveryRuntime("mxc", adapters)).toThrow( + "does not implement environment qualification", + ); + expect(() => qualifyManagedGatewayRecoveryRuntime(runtime, adapters)).toThrow( + "does not implement environment qualification", + ); + }); +}); diff --git a/src/lib/onboard/compute/recovery-runtime.ts b/src/lib/onboard/compute/recovery-runtime.ts new file mode 100644 index 0000000000..8cb6bf11ad --- /dev/null +++ b/src/lib/onboard/compute/recovery-runtime.ts @@ -0,0 +1,143 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import path from "node:path"; + +import { + type ManagedGatewayRuntimeBinding, + readManagedGatewayRuntimeBinding, +} from "../docker-driver-gateway-config"; +import { assessNativePodman } from "./podman-preflight"; + +export interface ManagedGatewayRecoveryRuntime { + readonly driverName: string; + readonly environment: Readonly>; +} + +export interface ManagedGatewayRecoveryAdapter { + readonly driverName: string; + qualifyEnvironment(environment: Readonly>): unknown; + resolveEnvironment(binding: ManagedGatewayRuntimeBinding): Readonly>; +} + +export type ManagedGatewayRecoveryAdapterRegistry = Readonly< + Record +>; + +function requiredString( + binding: ManagedGatewayRuntimeBinding, + key: string, + description: string, +): string { + const value = binding.values[key]; + if (typeof value !== "string" || !value.trim() || /[\0\r\n]/u.test(value)) { + throw new Error(`Managed ${binding.driverName} runtime binding has no safe ${description}.`); + } + return value.trim(); +} + +const PODMAN_RECOVERY_ADAPTER: ManagedGatewayRecoveryAdapter = { + driverName: "podman", + qualifyEnvironment(environment) { + const receipt = assessNativePodman({ + env: { ...process.env, ...environment }, + }); + if (receipt.socketPath !== environment.OPENSHELL_PODMAN_SOCKET) { + throw new Error("Qualified Podman socket does not match the managed recovery runtime."); + } + return receipt; + }, + resolveEnvironment(binding) { + const socketPath = requiredString(binding, "socket_path", "socket_path"); + if (!path.isAbsolute(socketPath)) { + throw new Error("Managed Podman runtime binding socket_path must be absolute."); + } + const networkName = requiredString(binding, "network_name", "network_name"); + if (!/^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$/u.test(networkName)) { + throw new Error("Managed Podman runtime binding network_name is invalid."); + } + const supervisorImage = requiredString(binding, "supervisor_image", "supervisor_image"); + if (supervisorImage.length > 2048) { + throw new Error("Managed Podman runtime binding supervisor_image is too long."); + } + return { + OPENSHELL_PODMAN_SOCKET: socketPath, + OPENSHELL_PODMAN_NETWORK_NAME: networkName, + OPENSHELL_SUPERVISOR_IMAGE: supervisorImage, + }; + }, +}; + +export const CURRENT_MANAGED_GATEWAY_RECOVERY_ADAPTERS = { + podman: PODMAN_RECOVERY_ADAPTER, +} as const satisfies ManagedGatewayRecoveryAdapterRegistry; + +function resolveManagedGatewayRecoveryAdapter( + driverName: string, + adapters: ManagedGatewayRecoveryAdapterRegistry, +): ManagedGatewayRecoveryAdapter | null { + const adapter = Object.hasOwn(adapters, driverName) ? adapters[driverName] : undefined; + if (!adapter) return null; + if (adapter.driverName !== driverName) { + throw new Error(`Managed recovery runtime adapter '${driverName}' has mismatched identity.`); + } + if (typeof adapter.qualifyEnvironment !== "function") { + throw new Error( + `Managed recovery runtime adapter '${driverName}' does not implement environment qualification.`, + ); + } + return adapter; +} + +export function supportsManagedGatewayRecoveryRuntime( + driverName: string, + adapters: ManagedGatewayRecoveryAdapterRegistry = CURRENT_MANAGED_GATEWAY_RECOVERY_ADAPTERS, +): boolean { + return resolveManagedGatewayRecoveryAdapter(driverName, adapters) !== null; +} + +export function qualifyManagedGatewayRecoveryRuntime( + runtime: ManagedGatewayRecoveryRuntime, + adapters: ManagedGatewayRecoveryAdapterRegistry = CURRENT_MANAGED_GATEWAY_RECOVERY_ADAPTERS, +): unknown { + const adapter = resolveManagedGatewayRecoveryAdapter(runtime.driverName, adapters); + if (!adapter) { + throw new Error(`Managed recovery runtime adapter '${runtime.driverName}' is not registered.`); + } + return adapter.qualifyEnvironment(runtime.environment); +} + +export function resolveManagedGatewayRecoveryRuntime( + options: { + readonly driverName: string; + readonly environment?: NodeJS.ProcessEnv; + readonly stateDir: string; + }, + adapters: ManagedGatewayRecoveryAdapterRegistry = CURRENT_MANAGED_GATEWAY_RECOVERY_ADAPTERS, + readBinding: ( + stateDir: string, + ) => ManagedGatewayRuntimeBinding | null = readManagedGatewayRuntimeBinding, +): ManagedGatewayRecoveryRuntime { + const binding = readBinding(options.stateDir); + if (!binding) { + throw new Error(`Managed runtime binding is missing in '${options.stateDir}'.`); + } + if (binding.driverName !== options.driverName) { + throw new Error( + `Managed runtime binding driver '${binding.driverName}' does not match requested recovery driver '${options.driverName}'.`, + ); + } + const adapter = resolveManagedGatewayRecoveryAdapter(options.driverName, adapters); + if (!adapter) { + throw new Error(`Managed recovery runtime adapter '${options.driverName}' is not registered.`); + } + const environment = adapter.resolveEnvironment(binding); + const ambient = options.environment ?? process.env; + for (const [key, value] of Object.entries(environment)) { + const requested = ambient[key]?.trim(); + if (requested && requested !== value) { + throw new Error(`${key} does not match the managed ${options.driverName} runtime binding.`); + } + } + return { driverName: options.driverName, environment }; +} diff --git a/src/lib/onboard/compute/runtime-authority.ts b/src/lib/onboard/compute/runtime-authority.ts new file mode 100644 index 0000000000..1ac7427a14 --- /dev/null +++ b/src/lib/onboard/compute/runtime-authority.ts @@ -0,0 +1,76 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export interface SandboxRuntimeAuthorityAdapter { + readonly driverName: string; + resolve(context: TContext): unknown; + revalidate?(authority: unknown, context: TContext): void; +} + +export type SandboxRuntimeAuthorityAdapterRegistry = Readonly< + Record> +>; + +/** + * Resolve driver-owned sandbox-create authority without teaching the workflow + * coordinator about individual runtimes. A null result is a deliberate + * no-authority contract (for example Kubernetes direct creation), while an + * absent or mismatched adapter fails closed. + */ +export function resolveSandboxRuntimeAuthority( + driverName: string, + context: TContext, + adapters: SandboxRuntimeAuthorityAdapterRegistry, +): unknown { + const adapter = Object.hasOwn(adapters, driverName) ? adapters[driverName] : undefined; + if (!adapter || adapter.driverName !== driverName) { + throw new Error(`OpenShell compute driver '${driverName}' has no runtime-authority adapter.`); + } + return adapter.resolve(context); +} + +export function revalidateSandboxRuntimeAuthority( + driverName: string, + authority: unknown, + context: TContext, + adapters: SandboxRuntimeAuthorityAdapterRegistry, +): void { + const adapter = Object.hasOwn(adapters, driverName) ? adapters[driverName] : undefined; + if (!adapter || adapter.driverName !== driverName) { + throw new Error(`OpenShell compute driver '${driverName}' has no runtime-authority adapter.`); + } + if (adapter.revalidate) { + adapter.revalidate(authority, context); + return; + } + if (authority != null) { + throw new Error( + `OpenShell compute driver '${driverName}' has no runtime-authority revalidation hook.`, + ); + } +} + +export interface AuthorizedSandboxRecreateDeletionSteps { + beforeDelete(): void; + deleteSandbox(): void; + afterDelete(): void; +} + +/** + * Resolve the selected driver's exact authority before entering the ordinary + * recreate deletion boundary. A failed qualification therefore cannot run + * provider cleanup, sandbox deletion, or any post-delete runtime mutation. + */ +export function runAuthorizedSandboxRecreateDeletion( + driverName: string, + context: TContext, + adapters: SandboxRuntimeAuthorityAdapterRegistry, + steps: AuthorizedSandboxRecreateDeletionSteps, +): unknown { + const authority = resolveSandboxRuntimeAuthority(driverName, context, adapters); + steps.beforeDelete(); + revalidateSandboxRuntimeAuthority(driverName, authority, context, adapters); + steps.deleteSandbox(); + steps.afterDelete(); + return authority; +} diff --git a/src/lib/onboard/compute/runtime.ts b/src/lib/onboard/compute/runtime.ts new file mode 100644 index 0000000000..d84d019003 --- /dev/null +++ b/src/lib/onboard/compute/runtime.ts @@ -0,0 +1,26 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export { readManagedGatewayRuntimeBinding } from "../docker-driver-gateway-config"; +export { ensureDockerDriverGatewayLocalTlsBundle as ensureManagedGatewayLocalTlsBundle } from "../docker-driver-gateway-local-tls"; +export { isLinuxDockerDriverGatewayEnabled } from "../docker-driver-platform"; +export * from "./host-local-inference-runtime"; +/** + * Driver-neutral onboarding runtime facade. + * + * The large onboarding coordinator consumes one compute boundary while each + * registered driver keeps its own selection, lifecycle, preflight, and + * reachability implementation. New drivers such as MXC extend this boundary + * without adding another direct dependency to the coordinator. + */ +export * from "./managed-gateway-profile"; +export * from "./managed-startup-runtime-requirements"; +export * from "./plan"; +export * from "./podman/active-watcher"; +export * from "./podman/gateway-env"; +export * from "./podman/gateway-reachability"; +export * from "./podman/sandbox-create-authority"; +export * from "./podman/socket-authority"; +export * from "./podman-preflight"; +export * from "./recovery-runtime"; +export * from "./runtime-authority"; diff --git a/src/lib/onboard/docker-driver-gateway-config.ts b/src/lib/onboard/docker-driver-gateway-config.ts index 0c362a1987..4ba3b71c04 100644 --- a/src/lib/onboard/docker-driver-gateway-config.ts +++ b/src/lib/onboard/docker-driver-gateway-config.ts @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { randomBytes } from "node:crypto"; +import { createHash, randomBytes } from "node:crypto"; import fs from "node:fs"; import path from "node:path"; import { @@ -14,7 +14,19 @@ export { ensureDockerDriverGatewayJwtBundle } from "./docker-driver-gateway-jwt- // See docs/security/openshell-0.0.72-compatibility-review.mdx for the source-of-truth review. export const DOCKER_DRIVER_GATEWAY_CONFIG_NAME = "openshell-gateway.toml"; +export const MANAGED_GATEWAY_RUNTIME_BINDING_NAME = "managed-runtime.json"; export const DOCKER_DRIVER_GATEWAY_JWT_TTL_SECS = 0; +const MANAGED_GATEWAY_RUNTIME_BINDING_VERSION = 1; +const MANAGED_GATEWAY_RUNTIME_BINDING_MAX_BYTES = 64 * 1024; + +type ManagedGatewayRuntimeValue = string | number | boolean; + +export interface ManagedGatewayRuntimeBinding { + readonly version: typeof MANAGED_GATEWAY_RUNTIME_BINDING_VERSION; + readonly driverName: string; + readonly configSha256: string; + readonly values: Readonly>; +} function tomlString(value: string): string { return JSON.stringify(value); @@ -45,6 +57,157 @@ function writeRestrictedFileAtomic(filePath: string, value: string, mode = 0o600 } } +function sha256(value: string): string { + return createHash("sha256").update(value).digest("hex"); +} + +function managedGatewayRuntimeValues( + driver: ManagedGatewayDriverConfig, +): Record { + const persistedKeys = new Set(); + for (const key of driver.persistedRuntimeKeys) { + if (!/^[a-z][a-z0-9_]*$/u.test(key) || persistedKeys.has(key)) { + throw new Error(`Invalid or duplicate managed gateway persisted runtime key '${key}'.`); + } + persistedKeys.add(key); + } + const values: Record = {}; + for (const [key, value] of driver.entries) { + if (value === undefined || (typeof value === "string" && value.trim() === "")) continue; + if (!/^[a-z][a-z0-9_]*$/u.test(key) || Object.hasOwn(values, key)) { + throw new Error(`Invalid or duplicate managed gateway runtime key '${key}'.`); + } + if (persistedKeys.has(key)) values[key] = value; + persistedKeys.delete(key); + } + if (persistedKeys.size > 0) { + throw new Error( + `Managed gateway persisted runtime key(s) are absent from the driver config: ${[ + ...persistedKeys, + ].join(", ")}.`, + ); + } + return Object.fromEntries( + Object.entries(values).sort(([left], [right]) => left.localeCompare(right)), + ); +} + +function writeManagedGatewayRuntimeBinding( + stateDir: string, + driver: ManagedGatewayDriverConfig, + configText: string, +): void { + if (!/^[a-z0-9][a-z0-9._-]{0,63}$/u.test(driver.driverName)) { + throw new Error(`Invalid managed gateway driver name '${driver.driverName}'.`); + } + const binding: ManagedGatewayRuntimeBinding = { + version: MANAGED_GATEWAY_RUNTIME_BINDING_VERSION, + driverName: driver.driverName, + configSha256: sha256(configText), + values: managedGatewayRuntimeValues(driver), + }; + writeRestrictedFileAtomic( + path.join(stateDir, MANAGED_GATEWAY_RUNTIME_BINDING_NAME), + `${JSON.stringify(binding, null, 2)}\n`, + 0o600, + ); +} + +function readRestrictedRuntimeFile(filePath: string): string { + let descriptor: number; + try { + descriptor = fs.openSync(filePath, fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW ?? 0)); + } catch (error) { + if (error instanceof Error && "code" in error && error.code === "ELOOP") { + throw new Error( + `Managed gateway runtime file '${filePath}' failed ownership or mode checks.`, + ); + } + throw error; + } + try { + const metadata = fs.fstatSync(descriptor); + const currentUid = typeof process.getuid === "function" ? process.getuid() : metadata.uid; + if ( + !metadata.isFile() || + metadata.uid !== currentUid || + (metadata.mode & 0o077) !== 0 || + metadata.size > MANAGED_GATEWAY_RUNTIME_BINDING_MAX_BYTES + ) { + throw new Error( + `Managed gateway runtime file '${filePath}' failed ownership or mode checks.`, + ); + } + const value = fs.readFileSync(descriptor, "utf-8"); + if (Buffer.byteLength(value, "utf-8") !== metadata.size) { + throw new Error(`Managed gateway runtime file '${filePath}' changed while it was read.`); + } + return value; + } finally { + fs.closeSync(descriptor); + } +} + +export function readManagedGatewayRuntimeBinding( + stateDir: string, +): ManagedGatewayRuntimeBinding | null { + const bindingPath = path.join(stateDir, MANAGED_GATEWAY_RUNTIME_BINDING_NAME); + let raw: string; + try { + raw = readRestrictedRuntimeFile(bindingPath); + } catch (error) { + if (error instanceof Error && "code" in error && error.code === "ENOENT") return null; + throw error; + } + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch { + throw new Error(`Managed gateway runtime binding '${bindingPath}' is malformed.`); + } + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { + throw new Error(`Managed gateway runtime binding '${bindingPath}' is malformed.`); + } + const candidate = parsed as Partial; + if ( + candidate.version !== MANAGED_GATEWAY_RUNTIME_BINDING_VERSION || + typeof candidate.driverName !== "string" || + !/^[a-z0-9][a-z0-9._-]{0,63}$/u.test(candidate.driverName) || + typeof candidate.configSha256 !== "string" || + !/^[0-9a-f]{64}$/u.test(candidate.configSha256) || + !candidate.values || + typeof candidate.values !== "object" || + Array.isArray(candidate.values) + ) { + throw new Error(`Managed gateway runtime binding '${bindingPath}' is malformed.`); + } + const values: Record = {}; + for (const [key, value] of Object.entries(candidate.values)) { + if ( + !/^[a-z][a-z0-9_]*$/u.test(key) || + (typeof value !== "string" && typeof value !== "number" && typeof value !== "boolean") + ) { + throw new Error(`Managed gateway runtime binding '${bindingPath}' is malformed.`); + } + values[key] = value; + } + const configPath = path.join(stateDir, DOCKER_DRIVER_GATEWAY_CONFIG_NAME); + const configText = readRestrictedRuntimeFile(configPath); + if (sha256(configText) !== candidate.configSha256) { + throw new Error("Managed gateway runtime binding does not match its gateway configuration."); + } + return { + version: MANAGED_GATEWAY_RUNTIME_BINDING_VERSION, + driverName: candidate.driverName, + configSha256: candidate.configSha256, + values, + }; +} + +export function clearManagedGatewayRuntimeBinding(stateDir: string): void { + fs.rmSync(path.join(stateDir, MANAGED_GATEWAY_RUNTIME_BINDING_NAME), { force: true }); +} + function cleanupStaleAtomicFileTemps(dir: string, basename: string): void { const prefix = `.${basename}.tmp-`; for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { @@ -67,35 +230,45 @@ function gatewayLocalTlsDir(gatewayEnv: Record): string { return localTlsDir; } -export function buildDockerDriverGatewayConfigToml( +export interface ManagedGatewayDriverConfig { + readonly driverName: string; + readonly entries: readonly (readonly [string, string | number | boolean | undefined])[]; + /** + * Explicit non-secret identity required to recover this runtime. + * + * Driver config entries are not persisted by default: future adapters may + * add credentials or tokens to their gateway config, and those must never + * leak into the host-side recovery binding accidentally. + */ + readonly persistedRuntimeKeys: readonly string[]; +} + +function renderManagedGatewayDriverConfig(config: ManagedGatewayDriverConfig): string { + return config.entries + .filter( + (entry): entry is readonly [string, string | number | boolean] => + entry[1] !== undefined && (typeof entry[1] !== "string" || entry[1].trim() !== ""), + ) + .map( + ([key, value]) => `${key} = ${typeof value === "string" ? tomlString(value) : String(value)}`, + ) + .join("\n"); +} + +export function buildManagedDriverGatewayConfigToml( gatewayEnv: Record, - sandboxBin?: string | null, + driver: ManagedGatewayDriverConfig, jwtBundle?: DockerDriverGatewayJwtBundle | null, gatewayId = "nemoclaw", ): string { const localTlsDir = jwtBundle ? gatewayLocalTlsDir(gatewayEnv) : undefined; - const dockerEntries: [string, string | undefined][] = [ - ["grpc_endpoint", gatewayEnv.OPENSHELL_GRPC_ENDPOINT], - ["network_name", gatewayEnv.OPENSHELL_DOCKER_NETWORK_NAME], - ["supervisor_image", gatewayEnv.OPENSHELL_DOCKER_SUPERVISOR_IMAGE], - ["supervisor_bin", sandboxBin ?? undefined], - ["guest_tls_ca", localTlsDir ? path.join(localTlsDir, "ca.crt") : undefined], - ["guest_tls_cert", localTlsDir ? path.join(localTlsDir, "client", "tls.crt") : undefined], - ["guest_tls_key", localTlsDir ? path.join(localTlsDir, "client", "tls.key") : undefined], - ]; - const dockerConfig = dockerEntries - .filter( - (entry): entry is [string, string] => typeof entry[1] === "string" && entry[1].trim() !== "", - ) - .map(([key, value]) => `${key} = ${tomlString(value)}`) - .join("\n"); - + const driverConfig = renderManagedGatewayDriverConfig(driver); const sections = [ "[openshell]", "version = 1", "", "[openshell.gateway]", - 'compute_drivers = ["docker"]', + `compute_drivers = [${tomlString(driver.driverName)}]`, "disable_tls = false", "", ]; @@ -125,33 +298,75 @@ export function buildDockerDriverGatewayConfigToml( ); } - sections.push("[openshell.drivers.docker]"); - if (dockerConfig) sections.push(dockerConfig); + sections.push(`[openshell.drivers.${driver.driverName}]`); + if (driverConfig) sections.push(driverConfig); sections.push(""); - return sections.join("\n"); } -export function writeDockerDriverGatewayConfig( - stateDir: string, +export function buildDockerDriverGatewayConfigToml( gatewayEnv: Record, sandboxBin?: string | null, + jwtBundle?: DockerDriverGatewayJwtBundle | null, + gatewayId = "nemoclaw", +): string { + const localTlsDir = jwtBundle ? gatewayLocalTlsDir(gatewayEnv) : undefined; + const dockerEntries: readonly (readonly [string, string | undefined])[] = [ + ["grpc_endpoint", gatewayEnv.OPENSHELL_GRPC_ENDPOINT], + ["network_name", gatewayEnv.OPENSHELL_DOCKER_NETWORK_NAME], + ["supervisor_image", gatewayEnv.OPENSHELL_DOCKER_SUPERVISOR_IMAGE], + ["supervisor_bin", sandboxBin ?? undefined], + ["guest_tls_ca", localTlsDir ? path.join(localTlsDir, "ca.crt") : undefined], + ["guest_tls_cert", localTlsDir ? path.join(localTlsDir, "client", "tls.crt") : undefined], + ["guest_tls_key", localTlsDir ? path.join(localTlsDir, "client", "tls.key") : undefined], + ]; + return buildManagedDriverGatewayConfigToml( + gatewayEnv, + { driverName: "docker", entries: dockerEntries, persistedRuntimeKeys: [] }, + jwtBundle, + gatewayId, + ); +} + +export function writeManagedDriverGatewayConfig( + stateDir: string, + gatewayEnv: Record, + driver: ManagedGatewayDriverConfig, ): string { const configPath = path.join(stateDir, DOCKER_DRIVER_GATEWAY_CONFIG_NAME); const jwtBundle = ensureDockerDriverGatewayJwtBundle(stateDir); - writeRestrictedFileAtomic( - configPath, - buildDockerDriverGatewayConfigToml( - gatewayEnv, - sandboxBin, - jwtBundle, - gatewayIdForStateDir(stateDir), - ), - 0o600, + const configText = buildManagedDriverGatewayConfigToml( + gatewayEnv, + driver, + jwtBundle, + gatewayIdForStateDir(stateDir), ); + writeRestrictedFileAtomic(configPath, configText, 0o600); + writeManagedGatewayRuntimeBinding(stateDir, driver, configText); return configPath; } +export function writeDockerDriverGatewayConfig( + stateDir: string, + gatewayEnv: Record, + sandboxBin?: string | null, +): string { + const localTlsDir = gatewayLocalTlsDir(gatewayEnv); + return writeManagedDriverGatewayConfig(stateDir, gatewayEnv, { + driverName: "docker", + persistedRuntimeKeys: [], + entries: [ + ["grpc_endpoint", gatewayEnv.OPENSHELL_GRPC_ENDPOINT], + ["network_name", gatewayEnv.OPENSHELL_DOCKER_NETWORK_NAME], + ["supervisor_image", gatewayEnv.OPENSHELL_DOCKER_SUPERVISOR_IMAGE], + ["supervisor_bin", sandboxBin ?? undefined], + ["guest_tls_ca", path.join(localTlsDir, "ca.crt")], + ["guest_tls_cert", path.join(localTlsDir, "client", "tls.crt")], + ["guest_tls_key", path.join(localTlsDir, "client", "tls.key")], + ], + }); +} + export function prepareDockerDriverGatewayConfigEnv( gatewayEnv: Record, stateDir: string, diff --git a/src/lib/onboard/docker-driver-gateway-cutover.ts b/src/lib/onboard/docker-driver-gateway-cutover.ts index bec2b8f7cf..963c589427 100644 --- a/src/lib/onboard/docker-driver-gateway-cutover.ts +++ b/src/lib/onboard/docker-driver-gateway-cutover.ts @@ -23,6 +23,7 @@ export function readDockerDriverGatewayHealth( } export interface DockerDriverGatewayCutoverInput { + driverLabel?: string; gatewayBin: string | null; identityGatewayBin: string | null; driftGatewayBin: string | null; @@ -71,14 +72,15 @@ export interface DockerDriverGatewayCutoverDeps { } /** - * Resolve reuse, adoption, or replacement for the host Docker-driver gateway. + * Resolve reuse, adoption, or replacement for a NemoClaw-managed host gateway. * Every reuse path requires a complete listener scan; replacement reaps only * port-observed PIDs before the fresh-launch callback is allowed to run. */ -export async function runDockerDriverGatewayCutover( +export async function runManagedDriverGatewayCutover( input: DockerDriverGatewayCutoverInput, deps: DockerDriverGatewayCutoverDeps, ): Promise<"reused" | "launch"> { + const driverLabel = input.driverLabel?.trim() || "Docker"; const portListenerPids = input.portListenerScan.pids; const portListenerPid = input.portListenerScan.complete ? (portListenerPids[0] ?? null) : null; @@ -118,11 +120,11 @@ export async function runDockerDriverGatewayCutover( await deps.verifySandboxBridgeGatewayReachableOrExit(input.exitOnFailure, { skip: input.skipSandboxBridgeReachability, }); - deps.log(" ✓ Reusing existing Docker-driver gateway"); + deps.log(` ✓ Reusing existing ${driverLabel}-driver gateway`); return "reused"; } else { deps.log( - " Docker-driver gateway metadata reports healthy but its HTTP endpoint is not responding. Starting a fresh gateway...", + ` ${driverLabel}-driver gateway metadata reports healthy but its HTTP endpoint is not responding. Starting a fresh gateway...`, ); } } @@ -154,7 +156,9 @@ export async function runDockerDriverGatewayCutover( await deps.verifySandboxBridgeGatewayReachableOrExit(input.exitOnFailure, { skip: input.skipSandboxBridgeReachability, }); - deps.log(` ✓ Reusing existing Docker-driver gateway process (PID ${portListenerPid})`); + deps.log( + ` ✓ Reusing existing ${driverLabel}-driver gateway process (PID ${portListenerPid})`, + ); return "reused"; } } @@ -176,3 +180,11 @@ export async function runDockerDriverGatewayCutover( } return "launch"; } + +/** Backward-compatible Docker entry point for existing callers and tests. */ +export function runDockerDriverGatewayCutover( + input: DockerDriverGatewayCutoverInput, + deps: DockerDriverGatewayCutoverDeps, +): Promise<"reused" | "launch"> { + return runManagedDriverGatewayCutover({ driverLabel: "Docker", ...input }, deps); +} diff --git a/src/lib/onboard/docker-driver-gateway-env-service.test.ts b/src/lib/onboard/docker-driver-gateway-env-service.test.ts index 5cf5c83953..f6dcef455b 100644 --- a/src/lib/onboard/docker-driver-gateway-env-service.test.ts +++ b/src/lib/onboard/docker-driver-gateway-env-service.test.ts @@ -7,13 +7,67 @@ import path from "node:path"; import { describe, expect, it, vi } from "vitest"; import { writeSafeGatewayAuthConfig } from "../../../test/support/docker-driver-gateway-env-test-support"; -import { startPackageManagedDockerDriverGatewayWithEnvOverride } from "./docker-driver-gateway-env"; +import { + startPackageManagedDockerDriverGatewayWithEnvOverride, + startPackageManagedDriverGatewayWithEnvOverride, +} from "./docker-driver-gateway-env"; function homeEnv(home: string, xdgConfigHome = ""): NodeJS.ProcessEnv { return { HOME: home, XDG_CONFIG_HOME: xdgConfigHome } as NodeJS.ProcessEnv; } describe("package-managed Docker-driver gateway env service", () => { + it("writes a native Podman service environment without inheriting DOCKER_HOST", async () => { + const tempHome = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-gateway-env-")); + const envFile = path.join(tempHome, ".config", "openshell", "gateway.env"); + const log = vi.spyOn(console, "log").mockImplementation(() => {}); + try { + await expect( + startPackageManagedDriverGatewayWithEnvOverride({ + allowWildcardBind: true, + clearDockerDriverGatewayRuntimeFiles: vi.fn(), + driverLabel: "Podman", + driverName: "podman", + env: { + ...homeEnv(tempHome), + DOCKER_HOST: "unix:///should/not/be/inherited.sock", + }, + exitOnFailure: false, + gatewayEnv: { + OPENSHELL_BIND_ADDRESS: "0.0.0.0", + OPENSHELL_GATEWAY_CONFIG: writeSafeGatewayAuthConfig(tempHome), + OPENSHELL_PODMAN_SOCKET: "/run/user/1000/podman/podman.sock", + OPENSHELL_SERVER_PORT: "8080", + }, + gatewayName: "nemoclaw", + hasOpenShellGatewayUserService: () => true, + isDockerDriverGatewayReady: async () => true, + registerDockerDriverGatewayEndpoint: () => true, + runCaptureOpenshell: (args) => + args[0] === "status" + ? "Gateway: nemoclaw\nConnected" + : "Gateway: nemoclaw\nGateway endpoint: https://127.0.0.1:8080/", + skipSandboxBridgeReachability: false, + startOpenShellGatewayUserService: (opts) => { + opts?.prepareServiceEnv?.(); + return { attempted: true, fallbackAllowed: false, started: true }; + }, + verifySandboxBridgeGatewayReachableOrExit: async () => undefined, + }), + ).resolves.toBe(true); + + const serviceEnv = fs.readFileSync(envFile, "utf-8"); + expect(serviceEnv).toContain("OPENSHELL_PODMAN_SOCKET='/run/user/1000/podman/podman.sock'\n"); + expect(serviceEnv).not.toContain("DOCKER_HOST="); + expect(log).toHaveBeenCalledWith( + " Starting OpenShell Podman-driver gateway via managed service...", + ); + } finally { + log.mockRestore(); + fs.rmSync(tempHome, { recursive: true, force: true }); + } + }); + it("stages the service and writes its env under one XDG config root (#6903)", async () => { const tempHome = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-gateway-env-")); const configHome = path.join(tempHome, "xdg-config"); diff --git a/src/lib/onboard/docker-driver-gateway-env.ts b/src/lib/onboard/docker-driver-gateway-env.ts index a44ac85c4a..3b0e5a7c99 100644 --- a/src/lib/onboard/docker-driver-gateway-env.ts +++ b/src/lib/onboard/docker-driver-gateway-env.ts @@ -24,12 +24,14 @@ import { hasOpenShellGatewayUserService, type PackageManagedDockerDriverGatewayOptions, startPackageManagedDockerDriverGateway, + startPackageManagedDriverGateway, } from "./docker-driver-gateway-service"; export { getGatewayHttpsEndpoint, startPackageManagedDockerDriverGateway }; export const DOCKER_DRIVER_GATEWAY_RUNTIME_ENV_KEYS = [ "DOCKER_HOST", + "OPENSHELL_PODMAN_SOCKET", "OPENSHELL_DRIVERS", "OPENSHELL_BIND_ADDRESS", "OPENSHELL_SERVER_PORT", @@ -43,7 +45,9 @@ export const DOCKER_DRIVER_GATEWAY_RUNTIME_ENV_KEYS = [ "OPENSHELL_DOCKER_NETWORK_NAME", "OPENSHELL_DOCKER_SUPERVISOR_IMAGE", "OPENSHELL_DOCKER_SUPERVISOR_BIN", + "OPENSHELL_SUPERVISOR_IMAGE", "OPENSHELL_GATEWAY_CONFIG", + "NEMOCLAW_MANAGED_GATEWAY_CONFIG_SHA256", "OPENSHELL_VM_DRIVER_STATE_DIR", "OPENSHELL_DRIVER_DIR", ] as const; @@ -61,6 +65,9 @@ export type PackageManagedDockerDriverGatewayWithEnvOverrideOptions = Omit< PackageManagedDockerDriverGatewayOptions, "prepareOpenShellGatewayUserServiceEnv" > & { + allowWildcardBind?: boolean; + driverLabel?: string; + driverName?: string; env?: NodeJS.ProcessEnv; gatewayEnv: Record; home?: string; @@ -167,10 +174,22 @@ function assertGatewayJwtFile(key: string, filePath: string): void { } export function assertDockerDriverGatewayAuthConfigSafe(gatewayEnv: Record): void { - assertDockerDriverGatewayBindAddressSafe(gatewayEnv); + assertManagedDriverGatewayAuthConfigSafe(gatewayEnv, { + allowWildcardBind: false, + driverName: "Docker", + }); +} + +export function assertManagedDriverGatewayAuthConfigSafe( + gatewayEnv: Record, + options: { allowWildcardBind: boolean; driverName: string }, +): void { + if (!options.allowWildcardBind) assertDockerDriverGatewayBindAddressSafe(gatewayEnv); const configPath = gatewayEnv.OPENSHELL_GATEWAY_CONFIG?.trim(); if (!configPath) { - throw new Error("OpenShell Docker-driver gateway requires OPENSHELL_GATEWAY_CONFIG"); + throw new Error( + `OpenShell ${options.driverName}-driver gateway requires OPENSHELL_GATEWAY_CONFIG`, + ); } const toml = fs.readFileSync(configPath, "utf-8"); const values = parseTomlScalarValues(toml); @@ -251,6 +270,15 @@ function formatEnvironmentFileAssignment(key: string, value: string): string { if (!dockerHost) throw new Error("Invalid empty DOCKER_HOST for the OpenShell gateway service"); return `${key}='${dockerHost}'`; } + if (key === "OPENSHELL_PODMAN_SOCKET") { + const socketPath = value.trim(); + if (!path.isAbsolute(socketPath) || /[\0\r\n']/.test(socketPath)) { + throw new Error( + "Invalid OPENSHELL_PODMAN_SOCKET for the OpenShell gateway service; expected a safely serializable absolute path.", + ); + } + return `${key}='${socketPath}'`; + } return `${key}=${value}`; } @@ -337,22 +365,42 @@ export function writeDockerGatewayDebEnvOverrideOrThrow( export function startPackageManagedDockerDriverGatewayWithEnvOverride( optionsWithEnv: PackageManagedDockerDriverGatewayWithEnvOverrideOptions, ): Promise { - const { env: _env, gatewayEnv, home, ...options } = optionsWithEnv; + return startPackageManagedDriverGatewayWithEnvOverride(optionsWithEnv); +} + +export function startPackageManagedDriverGatewayWithEnvOverride( + optionsWithEnv: PackageManagedDockerDriverGatewayWithEnvOverrideOptions, +): Promise { + const { + driverName = "docker", + driverLabel = driverName === "podman" ? "Podman" : "Docker", + allowWildcardBind = driverName === "podman", + env: _env, + gatewayEnv, + home, + ...options + } = optionsWithEnv; const env = optionsWithEnv.env ?? process.env; const gatewayPort = Number(gatewayEnv.OPENSHELL_SERVER_PORT ?? GATEWAY_PORT); if (gatewayPort !== DEFAULT_GATEWAY_PORT) return Promise.resolve(false); - assertDockerDriverGatewayAuthConfigSafe(gatewayEnv); + assertManagedDriverGatewayAuthConfigSafe(gatewayEnv, { + allowWildcardBind, + driverName: driverLabel, + }); const effectiveHome = home ?? optionsWithEnv.env?.HOME ?? os.homedir(); - return startPackageManagedDockerDriverGateway({ + return startPackageManagedDriverGateway({ ...options, + driverLabel, hasOpenShellGatewayUserService: options.hasOpenShellGatewayUserService ?? (() => hasOpenShellGatewayUserService({ env, home: effectiveHome })), prepareOpenShellGatewayUserServiceEnv: () => { const serviceGatewayEnv = { ...gatewayEnv }; delete serviceGatewayEnv.DOCKER_HOST; - const dockerHost = normalizePackageServiceDockerHost(env.DOCKER_HOST); - if (dockerHost) serviceGatewayEnv.DOCKER_HOST = dockerHost; + if (driverName === "docker") { + const dockerHost = normalizePackageServiceDockerHost(env.DOCKER_HOST); + if (dockerHost) serviceGatewayEnv.DOCKER_HOST = dockerHost; + } writeDockerGatewayDebEnvOverrideFile(() => serviceGatewayEnv, { env, home: effectiveHome, diff --git a/src/lib/onboard/docker-driver-gateway-failure.ts b/src/lib/onboard/docker-driver-gateway-failure.ts index 4326733b99..13ead66d98 100644 --- a/src/lib/onboard/docker-driver-gateway-failure.ts +++ b/src/lib/onboard/docker-driver-gateway-failure.ts @@ -39,6 +39,12 @@ export type ReportDockerDriverGatewayStartFailureOpts = { exitOnFailure: boolean; }; +export type ReportManagedDriverGatewayStartFailureOpts = + ReportDockerDriverGatewayStartFailureOpts & { + driverLabel: string; + runtimeDiagnostics?: readonly string[]; + }; + /** * Print the standard Docker-driver-gateway-start failure diagnostic set * to stderr and either exit or return. Always prints: @@ -49,16 +55,20 @@ export type ReportDockerDriverGatewayStartFailureOpts = { * - a short Troubleshooting footer with the log path and a docker CDI * inspection command. */ -export function reportDockerDriverGatewayStartFailure( +export function reportManagedDriverGatewayStartFailure( logPath: string, childExit: ChildExitState, - { exitOnFailure }: ReportDockerDriverGatewayStartFailureOpts, + { + driverLabel, + exitOnFailure, + runtimeDiagnostics = [], + }: ReportManagedDriverGatewayStartFailureOpts, ): void { const tail = fs.existsSync(logPath) ? fs.readFileSync(logPath, "utf-8").split("\n").filter(Boolean).slice(-20).join("\n") : ""; - console.error(" Docker-driver gateway failed to start."); + console.error(` ${driverLabel}-driver gateway failed to start.`); if (childExit.exited) { console.error(` Gateway process ${childExit.describeExit()} before becoming ready.`); } else { @@ -74,16 +84,28 @@ export function reportDockerDriverGatewayStartFailure( console.error(" Gateway log tail:"); for (const line of tail.split("\n")) console.error(` ${redact(line)}`); } - if (classifyGatewayStartFailure(tail).kind === "docker_unreachable") { + if (driverLabel === "Docker" && classifyGatewayStartFailure(tail).kind === "docker_unreachable") { printDockerDaemonRecovery(console.error); } console.error(" Troubleshooting:"); console.error(` tail -100 ${logPath}`); console.error(" openshell status"); console.error(" openshell gateway info"); - console.error(" docker info --format '{{json .CDISpecDirs}}'"); + for (const command of runtimeDiagnostics) console.error(` ${command}`); if (exitOnFailure) { process.exit(1); } } + +export function reportDockerDriverGatewayStartFailure( + logPath: string, + childExit: ChildExitState, + options: ReportDockerDriverGatewayStartFailureOpts, +): void { + reportManagedDriverGatewayStartFailure(logPath, childExit, { + ...options, + driverLabel: "Docker", + runtimeDiagnostics: ["docker info --format '{{json .CDISpecDirs}}'"], + }); +} diff --git a/src/lib/onboard/docker-driver-gateway-launch.test.ts b/src/lib/onboard/docker-driver-gateway-launch.test.ts index 0938f654d8..2e81b14a6c 100644 --- a/src/lib/onboard/docker-driver-gateway-launch.test.ts +++ b/src/lib/onboard/docker-driver-gateway-launch.test.ts @@ -11,6 +11,7 @@ import { buildDockerDriverGatewayConfigToml, buildDockerDriverGatewayLaunch, buildDockerDriverGatewayRuntimeIdentity, + buildHostManagedGatewayRuntimeIdentity, parseGlibcVersionsFromBinaryText, resolveDriftGatewayBin, shouldUseContainerizedGateway, @@ -33,6 +34,35 @@ function withTempBinaries( } describe("docker-driver-gateway-launch", () => { + it("builds a host-only runtime identity without incompatible driver environment", () => { + const identity = buildHostManagedGatewayRuntimeIdentity({ + gatewayBin: "/tmp/openshell-gateway", + gatewayEnv: { + OPENSHELL_DRIVERS: "podman", + OPENSHELL_PODMAN_SOCKET: "/run/user/1000/podman/podman.sock", + }, + env: { + DOCKER_HOST: "unix:///var/run/docker.sock", + OPENSHELL_DOCKER_SUPERVISOR_BIN: "/tmp/openshell-sandbox", + }, + removeEnvironmentKeys: ["DOCKER_HOST", "OPENSHELL_DOCKER_SUPERVISOR_BIN"], + runtimeEnvironmentKeys: ["OPENSHELL_PODMAN_SOCKET"], + }); + + expect(identity.launch).toMatchObject({ + command: "/tmp/openshell-gateway", + args: [], + mode: "host", + processGatewayBin: "/tmp/openshell-gateway", + }); + expect(identity.launch?.env).not.toHaveProperty("DOCKER_HOST"); + expect(identity.launch?.env).not.toHaveProperty("OPENSHELL_DOCKER_SUPERVISOR_BIN"); + expect(identity.desiredEnv).toMatchObject({ + OPENSHELL_DRIVERS: "podman", + OPENSHELL_PODMAN_SOCKET: "/run/user/1000/podman/podman.sock", + }); + }); + it("extracts GLIBC versions from binary text", () => { expect(parseGlibcVersionsFromBinaryText("GLIBC_2.35\0GLIBC_2.39\0GLIBC_2.39")).toEqual([ "2.35", @@ -210,13 +240,16 @@ describe("docker-driver-gateway-launch", () => { }); }); - it("scrubs stale auth-disable env from direct host gateway launches", () => { + it("scrubs stale TLS/auth-disable env from direct host gateway launches", () => { withTempBinaries(({ dir, gatewayBin }) => { const launch = buildDockerDriverGatewayLaunch({ gatewayBin, stateDir: dir, platform: "linux", - env: { OPENSHELL_DISABLE_GATEWAY_AUTH: "true" }, + env: { + OPENSHELL_DISABLE_GATEWAY_AUTH: "true", + OPENSHELL_DISABLE_TLS: "true", + }, hostGlibcVersion: "2.39", requiredGlibcVersions: ["2.39"], gatewayEnv: { OPENSHELL_DRIVERS: "docker" }, @@ -224,6 +257,7 @@ describe("docker-driver-gateway-launch", () => { expect(launch.mode).toBe("host"); expect(launch.env.OPENSHELL_DISABLE_GATEWAY_AUTH).toBeUndefined(); + expect(launch.env.OPENSHELL_DISABLE_TLS).toBeUndefined(); }); }); }); diff --git a/src/lib/onboard/docker-driver-gateway-launch.ts b/src/lib/onboard/docker-driver-gateway-launch.ts index 271875c8ee..f93ed46c72 100644 --- a/src/lib/onboard/docker-driver-gateway-launch.ts +++ b/src/lib/onboard/docker-driver-gateway-launch.ts @@ -53,9 +53,12 @@ export type DockerDriverGatewayRuntimeIdentity = { identityGatewayBin: string | null; }; -export function openDockerDriverGatewayLog( +export type ManagedDriverGatewayLaunch = DockerDriverGatewayLaunch; +export type ManagedDriverGatewayRuntimeIdentity = DockerDriverGatewayRuntimeIdentity; + +export function openManagedDriverGatewayLog( logPath: string, - options: { exitOnFailure?: boolean } = {}, + options: { driverLabel?: string; exitOnFailure?: boolean } = {}, ): number { const appendNoFollow = fs.constants.O_APPEND | fs.constants.O_CREAT | fs.constants.O_WRONLY | fs.constants.O_NOFOLLOW; @@ -63,13 +66,20 @@ export function openDockerDriverGatewayLog( return fs.openSync(logPath, appendNoFollow, 0o600); } catch (error) { console.error( - ` Failed to open OpenShell Docker-driver gateway log '${logPath}': ${String(error)}`, + ` Failed to open OpenShell ${options.driverLabel ?? "managed"}-driver gateway log '${logPath}': ${String(error)}`, ); if (options.exitOnFailure) process.exit(1); throw error; } } +export function openDockerDriverGatewayLog( + logPath: string, + options: { exitOnFailure?: boolean } = {}, +): number { + return openManagedDriverGatewayLog(logPath, { ...options, driverLabel: "Docker" }); +} + export function spawnDockerDriverGateway( launch: DockerDriverGatewayLaunch, logFd: number, @@ -95,6 +105,7 @@ type BuildGatewayLaunchOptions = { env?: NodeJS.ProcessEnv; hostGlibcVersion?: string | null; requiredGlibcVersions?: string[]; + removeEnvironmentKeys?: readonly string[]; ensureLocalTlsBundle?: boolean; // Multi-gateway callers pass the selected name. The hardened config derives // its JWT gateway identity from the already gateway-scoped state directory. @@ -111,12 +122,65 @@ function buildGatewayProcessEnv( gatewayEnv: Record, ): NodeJS.ProcessEnv { const env = { ...baseEnv, ...gatewayEnv }; - if (!("OPENSHELL_DISABLE_GATEWAY_AUTH" in gatewayEnv)) { - delete env.OPENSHELL_DISABLE_GATEWAY_AUTH; + for (const key of ["OPENSHELL_DISABLE_GATEWAY_AUTH", "OPENSHELL_DISABLE_TLS"] as const) { + if (!(key in gatewayEnv)) delete env[key]; } return env; } +export function buildHostManagedGatewayLaunch(options: { + gatewayBin: string; + gatewayEnv: Record; + gatewayName?: string; + env?: NodeJS.ProcessEnv; + removeEnvironmentKeys?: readonly string[]; +}): DockerDriverGatewayLaunch { + const env = buildGatewayProcessEnv(options.env ?? process.env, options.gatewayEnv); + for (const key of options.removeEnvironmentKeys ?? []) delete env[key]; + return { + command: options.gatewayBin, + args: [], + argv0: buildOwnedHostGatewayArgv0(options.gatewayName) ?? undefined, + env, + mode: "host", + processGatewayBin: options.gatewayBin, + }; +} + +function buildManagedGatewayRuntimeIdentity( + launch: ManagedDriverGatewayLaunch, + desiredKeys: Iterable, +): ManagedDriverGatewayRuntimeIdentity { + const desiredKeySet = new Set(desiredKeys); + const desiredEnv = Object.fromEntries( + Object.entries(launch.env).filter( + ([key, value]) => desiredKeySet.has(key) && typeof value === "string", + ) as [string, string][], + ); + return { + launch, + desiredEnv, + driftGatewayBin: launch.processGatewayBin, + identityGatewayBin: launch.processGatewayBin, + }; +} + +export function buildHostManagedGatewayRuntimeIdentity(options: { + gatewayBin: string; + gatewayEnv: Record; + gatewayName?: string; + env?: NodeJS.ProcessEnv; + removeEnvironmentKeys?: readonly string[]; + runtimeEnvironmentKeys?: readonly string[]; +}): ManagedDriverGatewayRuntimeIdentity { + const launch = buildHostManagedGatewayLaunch(options); + return buildManagedGatewayRuntimeIdentity(launch, [ + ...Object.keys(options.gatewayEnv), + ...(options.runtimeEnvironmentKeys ?? []), + "OPENSHELL_GATEWAY_CONFIG", + ]); +} + export function buildDockerDriverGatewayLaunch( options: BuildGatewayLaunchOptions, ): DockerDriverGatewayLaunch { @@ -143,18 +207,16 @@ export function buildDockerDriverGatewayLaunch( const baseEnv = options.env ?? process.env; const compat = shouldUseContainerizedGateway(options); if (!compat.useContainer) { - const env = buildGatewayProcessEnv(baseEnv, gatewayEnv); - return { - command: options.gatewayBin, - args: [], - argv0: buildOwnedHostGatewayArgv0(options.gatewayName) ?? undefined, - env, - mode: "host", - processGatewayBin: options.gatewayBin, - }; + return buildHostManagedGatewayLaunch({ + gatewayBin: options.gatewayBin, + gatewayEnv, + gatewayName: options.gatewayName, + env: baseEnv, + removeEnvironmentKeys: options.removeEnvironmentKeys, + }); } - return buildContainerizedDockerDriverGatewayLaunch({ + const launch = buildContainerizedDockerDriverGatewayLaunch({ gatewayBin: options.gatewayBin, gatewayEnv, stateDir: options.stateDir, @@ -163,6 +225,8 @@ export function buildDockerDriverGatewayLaunch( baseEnv, reason: compat.reason, }); + for (const key of options.removeEnvironmentKeys ?? []) delete launch.env[key]; + return launch; } export function prepareDockerDriverGatewayLaunch(launch: DockerDriverGatewayLaunch): void { @@ -173,19 +237,13 @@ export function buildDockerDriverGatewayRuntimeIdentity( options: BuildGatewayLaunchOptions, ): DockerDriverGatewayRuntimeIdentity { const launch = buildDockerDriverGatewayLaunch(options); - const desiredKeys = new Set([ + const identity = buildManagedGatewayRuntimeIdentity(launch, [ ...Object.keys(options.gatewayEnv), "OPENSHELL_DOCKER_SUPERVISOR_BIN", "OPENSHELL_GATEWAY_CONFIG", ]); - const desiredEnv = Object.fromEntries( - Object.entries(launch.env).filter( - ([key, value]) => desiredKeys.has(key) && typeof value === "string", - ) as [string, string][], - ); return { - launch, - desiredEnv, + ...identity, driftGatewayBin: launch.processGatewayBin, identityGatewayBin: launch.processGatewayBin || options.gatewayBin, }; diff --git a/src/lib/onboard/docker-driver-gateway-local-tls.test.ts b/src/lib/onboard/docker-driver-gateway-local-tls.test.ts index 78825ec6fe..6f8d7875d3 100644 --- a/src/lib/onboard/docker-driver-gateway-local-tls.test.ts +++ b/src/lib/onboard/docker-driver-gateway-local-tls.test.ts @@ -8,6 +8,7 @@ import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; import { + buildDockerDriverGatewayCertgenArgs, dockerDriverGatewayLocalTlsBundleIsComplete, ensureDockerDriverGatewayLocalTlsBundle, getDockerDriverGatewayLocalTlsBundle, @@ -224,6 +225,37 @@ describe("docker-driver-gateway-local-tls", () => { } }); + it("adds the Podman host callback name without changing the Docker default SAN set", () => { + expect(buildDockerDriverGatewayCertgenArgs("/state/tls")).toEqual([ + "generate-certs", + "--output-dir", + "/state/tls", + "--server-san", + "host.openshell.internal", + "--server-san", + "localhost", + "--server-san", + "127.0.0.1", + ]); + expect( + buildDockerDriverGatewayCertgenArgs("/state/tls", ["host.containers.internal"]), + ).toContain("host.containers.internal"); + }); + + it("does not accept a Docker-only certificate for the Podman callback hostname", () => { + const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-gateway-tls-")); + writeBundle(stateDir, TEST_CERT_PEM, TEST_KEY_PEM); + useTestCertificateClock(); + try { + expect(dockerDriverGatewayLocalTlsBundleIsComplete(stateDir)).toBe(true); + expect( + dockerDriverGatewayLocalTlsBundleIsComplete(stateDir, ["host.containers.internal"]), + ).toBe(false); + } finally { + fs.rmSync(stateDir, { recursive: true, force: true }); + } + }); + it("preserves an existing complete mTLS bundle without regenerating certs", () => { const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-gateway-tls-")); const contents = writeBundle(stateDir, TEST_CERT_PEM, TEST_KEY_PEM); diff --git a/src/lib/onboard/docker-driver-gateway-local-tls.ts b/src/lib/onboard/docker-driver-gateway-local-tls.ts index 703e1d5f46..7245f83981 100644 --- a/src/lib/onboard/docker-driver-gateway-local-tls.ts +++ b/src/lib/onboard/docker-driver-gateway-local-tls.ts @@ -23,6 +23,7 @@ export type DockerDriverGatewayLocalTlsBundle = { }; export interface EnsureDockerDriverGatewayLocalTlsBundleOptions { + additionalServerDnsSans?: readonly string[]; env?: NodeJS.ProcessEnv; gatewayBin: string; spawnSyncImpl?: typeof spawnSync; @@ -47,7 +48,10 @@ export function getDockerDriverGatewayLocalTlsBundle( }; } -export function dockerDriverGatewayLocalTlsBundleIsComplete(stateDir: string): boolean { +export function dockerDriverGatewayLocalTlsBundleIsComplete( + stateDir: string, + additionalServerDnsSans: readonly string[] = [], +): boolean { const bundle = getDockerDriverGatewayLocalTlsBundle(stateDir); const expectedFiles = [ bundle.caPath, @@ -74,7 +78,7 @@ export function dockerDriverGatewayLocalTlsBundleIsComplete(stateDir: string): b certificateMatchesPrivateKey(clientCert, clientKey) && certificateVerifiesAgainstCa(serverCert, ca) && certificateVerifiesAgainstCa(clientCert, ca) && - certificateHasRequiredServerSubjectAltNames(serverCert) + certificateHasRequiredServerSubjectAltNames(serverCert, additionalServerDnsSans) ); } @@ -84,6 +88,22 @@ export function buildDockerDriverGatewayLocalTlsEnv(stateDir: string): Record ["--server-san", subjectAltName]), + ]; +} + function text(value: Buffer | string | null | undefined): string { if (typeof value === "string") return value; if (Buffer.isBuffer(value)) return value.toString("utf-8"); @@ -156,7 +176,10 @@ function certificateVerifiesAgainstCa(certificate: X509Certificate, ca: X509Cert } } -function certificateHasRequiredServerSubjectAltNames(certificate: X509Certificate): boolean { +function certificateHasRequiredServerSubjectAltNames( + certificate: X509Certificate, + additionalServerDnsSans: readonly string[], +): boolean { const subjectAltNames = new Set( (certificate.subjectAltName ?? "") .split(",") @@ -164,7 +187,9 @@ function certificateHasRequiredServerSubjectAltNames(certificate: X509Certificat .filter(Boolean), ); return ( - REQUIRED_SERVER_DNS_SANS.every((dnsName) => subjectAltNames.has(`dns:${dnsName}`)) && + [...REQUIRED_SERVER_DNS_SANS, ...additionalServerDnsSans].every((dnsName) => + subjectAltNames.has(`dns:${dnsName.toLowerCase()}`), + ) && REQUIRED_SERVER_IP_SANS.every( (ipAddress) => subjectAltNames.has(`ip address:${ipAddress}`) || subjectAltNames.has(`ip:${ipAddress}`), @@ -180,6 +205,7 @@ function normalizeDockerDriverGatewayLocalTlsBundlePermissions( } export function ensureDockerDriverGatewayLocalTlsBundle({ + additionalServerDnsSans = [], env = process.env, gatewayBin, spawnSyncImpl = spawnSync, @@ -188,24 +214,14 @@ export function ensureDockerDriverGatewayLocalTlsBundle({ const bundle = getDockerDriverGatewayLocalTlsBundle(stateDir); fs.mkdirSync(stateDir, { recursive: true, mode: 0o700 }); fs.chmodSync(stateDir, 0o700); - if (dockerDriverGatewayLocalTlsBundleIsComplete(stateDir)) { + if (dockerDriverGatewayLocalTlsBundleIsComplete(stateDir, additionalServerDnsSans)) { normalizeDockerDriverGatewayLocalTlsBundlePermissions(bundle); return bundle; } const result = spawnSyncImpl( gatewayBin, - [ - "generate-certs", - "--output-dir", - bundle.localTlsDir, - "--server-san", - "host.openshell.internal", - "--server-san", - "localhost", - "--server-san", - "127.0.0.1", - ], + buildDockerDriverGatewayCertgenArgs(bundle.localTlsDir, additionalServerDnsSans), { encoding: "utf-8", env: { @@ -229,7 +245,7 @@ export function ensureDockerDriverGatewayLocalTlsBundle({ const detail = sanitizeDockerDriverGatewayLocalTlsErrorDetail(rawDetail, bundle, stateDir); throw new Error(`OpenShell gateway certificate generation failed: ${detail}`); } - if (!dockerDriverGatewayLocalTlsBundleIsComplete(stateDir)) { + if (!dockerDriverGatewayLocalTlsBundleIsComplete(stateDir, additionalServerDnsSans)) { const detail = sanitizeDockerDriverGatewayLocalTlsErrorDetail( `incomplete mTLS bundle in ${bundle.localTlsDir}`, bundle, diff --git a/src/lib/onboard/docker-driver-gateway-readiness.ts b/src/lib/onboard/docker-driver-gateway-readiness.ts index 03931df882..7d259426bb 100644 --- a/src/lib/onboard/docker-driver-gateway-readiness.ts +++ b/src/lib/onboard/docker-driver-gateway-readiness.ts @@ -8,7 +8,7 @@ type RunCaptureOpenshell = (args: string[], opts?: { ignoreError?: boolean }) => export type DockerDriverGatewayStartupResult = "healthy" | "exited" | "timeout"; -export async function waitForStandaloneDockerDriverGateway(options: { +export async function waitForStandaloneManagedDriverGateway(options: { childExited: () => boolean; childPid: number; gatewayName: string; @@ -63,3 +63,9 @@ export async function waitForStandaloneDockerDriverGateway(options: { return result; } + +export function waitForStandaloneDockerDriverGateway( + options: Parameters[0], +): Promise { + return waitForStandaloneManagedDriverGateway(options); +} diff --git a/src/lib/onboard/docker-driver-gateway-runtime.test.ts b/src/lib/onboard/docker-driver-gateway-runtime.test.ts index a6a90aa6fb..b4282663e1 100644 --- a/src/lib/onboard/docker-driver-gateway-runtime.test.ts +++ b/src/lib/onboard/docker-driver-gateway-runtime.test.ts @@ -114,6 +114,27 @@ describe("docker-driver gateway runtime helpers", () => { } }); + it("uses an explicit resolved state directory when HOME points elsewhere", () => { + const snapshotHome = "/snapshot-home"; + const resolvedStateDir = path.join(snapshotHome, ".local", "state", "nemoclaw", "gateway"); + const homedir = vi.spyOn(os, "homedir").mockReturnValue("/os-account-home"); + withEnv( + { + HOME: snapshotHome, + NEMOCLAW_OPENSHELL_GATEWAY_STATE_DIR: undefined, + }, + () => { + const { helpers } = makeHelpers({ stateDir: resolvedStateDir }); + + expect(helpers.getDockerDriverGatewayStateDir()).toBe(resolvedStateDir); + expect(helpers.getDockerDriverGatewayPidFile()).toBe( + path.join(resolvedStateDir, "openshell-gateway.pid"), + ); + }, + ); + expect(homedir).not.toHaveBeenCalled(); + }); + it("uses the moving dev supervisor image for an explicit or detected dev runtime", () => { const explicit = makeHelpers({ shouldUseOpenshellDevChannel: () => true }); expect( diff --git a/src/lib/onboard/docker-driver-gateway-runtime.ts b/src/lib/onboard/docker-driver-gateway-runtime.ts index 42047e4a2a..78cf675655 100644 --- a/src/lib/onboard/docker-driver-gateway-runtime.ts +++ b/src/lib/onboard/docker-driver-gateway-runtime.ts @@ -66,6 +66,7 @@ export interface DockerDriverGatewayRuntimeDeps { runCapture: RunCapture; runCaptureEx?: RunCaptureEx; shouldUseOpenshellDevChannel(): boolean; + stateDir?: string; supportedOpenshellFallbackVersion: string; } @@ -79,6 +80,7 @@ export function createDockerDriverGatewayRuntimeHelpers(deps: DockerDriverGatewa versionOutput?: string | null, platform?: NodeJS.Platform, ): Record; + getOpenShellSupervisorImage(versionOutput?: string | null): string; getDockerDriverGatewayPid(): number | null; getDockerDriverGatewayPidFile(): string; getDockerDriverGatewayPortListenerScan( @@ -146,6 +148,7 @@ export function createDockerDriverGatewayRuntimeHelpers(deps: DockerDriverGatewa typeof deps.gatewayPort === "function" ? deps.gatewayPort() : deps.gatewayPort; function getDockerDriverGatewayStateDir(): string { + if (deps.stateDir?.trim()) return path.resolve(deps.stateDir.trim()); const configured = process.env.NEMOCLAW_OPENSHELL_GATEWAY_STATE_DIR; if (configured && configured.trim()) return path.resolve(configured.trim()); const dir = gatewayBinding.resolveGatewayStateDirName(currentGatewayPort()); @@ -200,9 +203,11 @@ export function createDockerDriverGatewayRuntimeHelpers(deps: DockerDriverGatewa return null; } - function getOpenShellDockerSupervisorImage(versionOutput: string | null = null): string { - if (process.env.OPENSHELL_DOCKER_SUPERVISOR_IMAGE) { - return process.env.OPENSHELL_DOCKER_SUPERVISOR_IMAGE; + function getOpenShellSupervisorImage(versionOutput: string | null = null): string { + const configured = + process.env.OPENSHELL_SUPERVISOR_IMAGE || process.env.OPENSHELL_DOCKER_SUPERVISOR_IMAGE; + if (configured) { + return configured; } const installedVersion = deps.getInstalledOpenshellVersion(versionOutput); if (deps.shouldUseOpenshellDevChannel() || deps.isOpenshellDevVersion(versionOutput)) { @@ -227,7 +232,7 @@ export function createDockerDriverGatewayRuntimeHelpers(deps: DockerDriverGatewa gatewayPort: currentGatewayPort(), stateDir: getDockerDriverGatewayStateDir(), dockerNetworkName: process.env.OPENSHELL_DOCKER_NETWORK_NAME || "openshell-docker", - getDockerSupervisorImage: () => getOpenShellDockerSupervisorImage(versionOutput), + getDockerSupervisorImage: () => getOpenShellSupervisorImage(versionOutput), resolveSandboxBin: resolveOpenShellSandboxBinary, }); if (gatewayEnv.OPENSHELL_LOCAL_TLS_DIR) { @@ -273,12 +278,14 @@ export function createDockerDriverGatewayRuntimeHelpers(deps: DockerDriverGatewa return env; } - function hasDockerDriverGatewayEnv(pid: number): boolean { + function hasManagedDriverGatewayEnv(pid: number): boolean { const env = readProcessEnv(pid); if (!env) return false; return ( env.OPENSHELL_DRIVERS === "docker" || + env.OPENSHELL_DRIVERS === "podman" || Boolean(env.OPENSHELL_DOCKER_SUPERVISOR_IMAGE) || + Boolean(env.OPENSHELL_PODMAN_SOCKET) || env.OPENSHELL_GRPC_ENDPOINT === dockerDriverGatewayEnv.getDockerDriverGatewayEndpoint(currentGatewayPort()) ); @@ -446,7 +453,7 @@ export function createDockerDriverGatewayRuntimeHelpers(deps: DockerDriverGatewa if (!identity) return false; const matchesGatewayBinary = processIdentityMatchesGatewayBinary(identity, gatewayBin); if (!matchesGatewayBinary) return false; - if (opts.requireDockerDriverEnv && !hasDockerDriverGatewayEnv(pid)) return false; + if (opts.requireDockerDriverEnv && !hasManagedDriverGatewayEnv(pid)) return false; return true; } @@ -513,6 +520,7 @@ export function createDockerDriverGatewayRuntimeHelpers(deps: DockerDriverGatewa clearDockerDriverGatewayRuntimeFiles, createGatewayServicePortOwnership, getDockerDriverGatewayEnv, + getOpenShellSupervisorImage, getDockerDriverGatewayPid, getDockerDriverGatewayPidFile, getDockerDriverGatewayPortListenerScan, diff --git a/src/lib/onboard/docker-driver-gateway-service.test.ts b/src/lib/onboard/docker-driver-gateway-service.test.ts index 85a56eb217..b70545c12c 100644 --- a/src/lib/onboard/docker-driver-gateway-service.test.ts +++ b/src/lib/onboard/docker-driver-gateway-service.test.ts @@ -5,6 +5,9 @@ import { describe, expect, it, vi } from "vitest"; import { createVirtualClock } from "./__test-helpers__/virtual-clock"; import { + assertTrustedOpenShellGatewayUserServiceInactive, + captureTrustedActiveOpenShellGatewayUserService, + captureTrustedOpenShellGatewayUserServiceIfActive, getNemoclawOpenShellGatewayUserServicePath, getOpenShellGatewayUserServiceBinaryPaths, getOpenShellGatewayUserServicePaths, @@ -12,9 +15,11 @@ import { getTrustedActiveOpenShellGatewayUserServicePid, hasOpenShellGatewayUserService, NEMOCLAW_OPENSHELL_GATEWAY_USER_SERVICE_MARKER, + resumeTrustedOpenShellGatewayUserServiceAndProveActive, type SpawnSyncLikeResult, startOpenShellGatewayUserService, startPackageManagedDockerDriverGateway, + stopTrustedOpenShellGatewayUserServiceAndProveInactive, } from "./docker-driver-gateway-service"; const STATUS_CONNECTED = ` @@ -32,6 +37,10 @@ Gateway: nemoclaw Gateway endpoint: https://127.0.0.1:8080/ `; +const SYSTEMD_INVOCATION_A = "11111111111111111111111111111111"; +const SYSTEMD_INVOCATION_B = "22222222222222222222222222222222"; +const GATEWAY_ARGV = ["/usr/bin/openshell-gateway", "--port", "8080"] as const; + function spawnResult(status = 0, stderr = "", stdout = ""): SpawnSyncLikeResult { return { status, stderr, stdout }; } @@ -39,13 +48,23 @@ function spawnResult(status = 0, stderr = "", stdout = ""): SpawnSyncLikeResult function trustedShowOutput( fragmentPath = "/lib/systemd/user/openshell-gateway.service", execPath = "/usr/bin/openshell-gateway", + execArgv = `${execPath} --port 8080`, + invocationId = SYSTEMD_INVOCATION_A, ): string { return [ `FragmentPath=${fragmentPath}`, - `ExecStart={ path=${execPath} ; argv[]=${execPath} ; }`, + `ExecStart={ path=${execPath} ; argv[]=${execArgv} ; }`, + `InvocationID=${invocationId}`, ].join("\n"); } +function processIdentity( + startIdentity = "process-start-a", + argv: readonly string[] = GATEWAY_ARGV, +) { + return { argv, startIdentity }; +} + function officialFormulaInfo(): SpawnSyncLikeResult { return spawnResult( 0, @@ -359,6 +378,466 @@ describe("docker-driver-gateway-service", () => { ).toBeNull(); }); + it("leases the exact trusted systemd service identity across stop and resume", () => { + const events: string[] = []; + let active = true; + let pid = 4242; + const opts = { + commandExists: (command: string) => command === "systemctl", + existsSync: (candidate: string) => + candidate === "/lib/systemd/user/openshell-gateway.service", + platform: "linux" as const, + readProcessIdentity: (candidate: number) => + active && candidate === pid + ? processIdentity(pid === 4242 ? "process-start-a" : "process-start-b") + : null, + spawnSyncImpl: vi.fn((_command: string, args: string[]) => { + const operation = args.slice(1).join(" "); + events.push(operation); + if (args.includes("show")) { + return spawnResult( + 0, + "", + [ + trustedShowOutput( + undefined, + undefined, + undefined, + pid === 4242 ? SYSTEMD_INVOCATION_A : SYSTEMD_INVOCATION_B, + ), + `ActiveState=${active ? "active" : "inactive"}`, + `MainPID=${active ? String(pid) : "0"}`, + ].join("\n"), + ); + } + if (args.includes("stop")) { + active = false; + return spawnResult(); + } + if (args.includes("start")) { + active = true; + pid = 5252; + return spawnResult(); + } + return spawnResult(1, "unexpected command"); + }), + }; + + const captured = captureTrustedActiveOpenShellGatewayUserService(opts); + expect(captured).toMatchObject({ + execStart: + "{ path=/usr/bin/openshell-gateway ; argv[]=/usr/bin/openshell-gateway --port 8080 ; }", + execStartPath: "/usr/bin/openshell-gateway", + invocationId: SYSTEMD_INVOCATION_A, + manager: "systemd", + pid: 4242, + processArgv: GATEWAY_ARGV, + processStartIdentity: "process-start-a", + serviceName: "openshell-gateway", + unitPath: "/lib/systemd/user/openshell-gateway.service", + }); + expect(Object.isFrozen(captured)).toBe(true); + + stopTrustedOpenShellGatewayUserServiceAndProveInactive(captured, opts); + assertTrustedOpenShellGatewayUserServiceInactive(captured, opts); + const resumed = resumeTrustedOpenShellGatewayUserServiceAndProveActive(captured, opts); + + expect(resumed).toMatchObject({ + execStartPath: "/usr/bin/openshell-gateway", + invocationId: SYSTEMD_INVOCATION_B, + manager: "systemd", + pid: 5252, + processArgv: GATEWAY_ARGV, + processStartIdentity: "process-start-b", + serviceName: "openshell-gateway", + unitPath: "/lib/systemd/user/openshell-gateway.service", + }); + expect(events).toEqual([ + "show openshell-gateway --property=FragmentPath --property=ExecStart --property=ActiveState --property=InvocationID --property=MainPID", + "show openshell-gateway --property=FragmentPath --property=ExecStart --property=ActiveState --property=InvocationID --property=MainPID", + "stop openshell-gateway", + "show openshell-gateway --property=FragmentPath --property=ExecStart --property=ActiveState --property=InvocationID --property=MainPID", + "show openshell-gateway --property=FragmentPath --property=ExecStart --property=ActiveState --property=InvocationID --property=MainPID", + "show openshell-gateway --property=FragmentPath --property=ExecStart --property=ActiveState --property=InvocationID --property=MainPID", + "start openshell-gateway", + "show openshell-gateway --property=FragmentPath --property=ExecStart --property=ActiveState --property=InvocationID --property=MainPID", + ]); + }); + + it("distinguishes a proven inactive installed systemd service from its active owner", () => { + const base = { + commandExists: (command: string) => command === "systemctl", + existsSync: (candidate: string) => + candidate === "/lib/systemd/user/openshell-gateway.service", + platform: "linux" as const, + }; + const show = (activeState: string, mainPid: number) => + vi.fn(() => + spawnResult( + 0, + "", + [trustedShowOutput(), `ActiveState=${activeState}`, `MainPID=${String(mainPid)}`].join( + "\n", + ), + ), + ); + + expect( + captureTrustedOpenShellGatewayUserServiceIfActive({ + ...base, + spawnSyncImpl: show("inactive", 0), + }), + ).toBeNull(); + expect(() => + captureTrustedOpenShellGatewayUserServiceIfActive({ + ...base, + spawnSyncImpl: show("activating", 4242), + }), + ).toThrow("state is transitional"); + }); + + it("refuses full systemd ExecStart drift before stopping the captured service", () => { + let execArgv = "/usr/bin/openshell-gateway --port 8080"; + const stop = vi.fn(); + const opts = { + commandExists: () => true, + existsSync: (candidate: string) => + candidate === "/lib/systemd/user/openshell-gateway.service", + platform: "linux" as const, + readProcessIdentity: () => processIdentity(), + spawnSyncImpl: vi.fn((_command: string, args: string[]) => { + if (args.includes("show")) { + return spawnResult( + 0, + "", + [ + trustedShowOutput( + "/lib/systemd/user/openshell-gateway.service", + "/usr/bin/openshell-gateway", + execArgv, + ), + "ActiveState=active", + "MainPID=4242", + ].join("\n"), + ); + } + stop(); + return spawnResult(); + }), + }; + const captured = captureTrustedActiveOpenShellGatewayUserService(opts); + execArgv = "/usr/bin/openshell-gateway --port 9090"; + + expect(() => stopTrustedOpenShellGatewayUserServiceAndProveInactive(captured, opts)).toThrow( + "unit identity drifted", + ); + expect(stop).not.toHaveBeenCalled(); + }); + + it.each([ + { + case: "InvocationID drift", + invocationId: SYSTEMD_INVOCATION_B, + process: processIdentity(), + }, + { + case: "PID reuse with a different process start", + invocationId: SYSTEMD_INVOCATION_A, + process: processIdentity("process-start-reused"), + }, + ])("refuses systemd $case before mutation", ({ invocationId, process }) => { + let drifted = false; + const stop = vi.fn(); + const opts = { + commandExists: () => true, + existsSync: (candidate: string) => + candidate === "/lib/systemd/user/openshell-gateway.service", + platform: "linux" as const, + readProcessIdentity: () => (drifted ? process : processIdentity()), + spawnSyncImpl: vi.fn((_command: string, args: string[]) => { + if (args.includes("show")) { + return spawnResult( + 0, + "", + [ + trustedShowOutput( + undefined, + undefined, + undefined, + drifted ? invocationId : SYSTEMD_INVOCATION_A, + ), + "ActiveState=active", + "MainPID=4242", + ].join("\n"), + ); + } + stop(); + return spawnResult(); + }), + }; + const captured = captureTrustedActiveOpenShellGatewayUserService(opts); + drifted = true; + + expect(() => stopTrustedOpenShellGatewayUserServiceAndProveInactive(captured, opts)).toThrow( + "process identity drifted", + ); + expect(stop).not.toHaveBeenCalled(); + }); + + it("fails closed when systemd respawns the service during stop proof", () => { + let pid = 4242; + const opts = { + commandExists: () => true, + existsSync: (candidate: string) => + candidate === "/lib/systemd/user/openshell-gateway.service", + platform: "linux" as const, + readProcessIdentity: () => processIdentity(), + spawnSyncImpl: vi.fn((_command: string, args: string[]) => { + if (args.includes("show")) { + return spawnResult( + 0, + "", + [trustedShowOutput(), "ActiveState=active", `MainPID=${String(pid)}`].join("\n"), + ); + } + if (args.includes("stop")) { + pid = 5252; + return spawnResult(); + } + return spawnResult(1, "unexpected command"); + }), + }; + const captured = captureTrustedActiveOpenShellGatewayUserService(opts); + + expect(() => stopTrustedOpenShellGatewayUserServiceAndProveInactive(captured, opts)).toThrow( + "not proven inactive", + ); + }); + + it("does not mutate when the captured systemd manager becomes unavailable", () => { + const captureOpts = { + commandExists: () => true, + existsSync: (candidate: string) => + candidate === "/lib/systemd/user/openshell-gateway.service", + platform: "linux" as const, + readProcessIdentity: () => processIdentity(), + spawnSyncImpl: () => + spawnResult(0, "", [trustedShowOutput(), "ActiveState=active", "MainPID=4242"].join("\n")), + }; + const captured = captureTrustedActiveOpenShellGatewayUserService(captureOpts); + const mutation = vi.fn(); + + expect(() => + stopTrustedOpenShellGatewayUserServiceAndProveInactive(captured, { + ...captureOpts, + commandExists: () => false, + spawnSyncImpl: mutation, + }), + ).toThrow("systemctl is unavailable"); + expect(mutation).not.toHaveBeenCalled(); + }); + + it("rejects a resumed systemd service that reuses the captured PID", () => { + let active = true; + let resumed = false; + const opts = { + commandExists: () => true, + existsSync: (candidate: string) => + candidate === "/lib/systemd/user/openshell-gateway.service", + platform: "linux" as const, + readProcessIdentity: () => + active ? processIdentity(resumed ? "process-start-b" : "process-start-a") : null, + spawnSyncImpl: vi.fn((_command: string, args: string[]) => { + if (args.includes("show")) { + return spawnResult( + 0, + "", + [ + trustedShowOutput( + undefined, + undefined, + undefined, + resumed ? SYSTEMD_INVOCATION_B : SYSTEMD_INVOCATION_A, + ), + `ActiveState=${active ? "active" : "inactive"}`, + `MainPID=${active ? "4242" : "0"}`, + ].join("\n"), + ); + } + if (args.includes("stop")) active = false; + if (args.includes("start")) { + active = true; + resumed = true; + } + return spawnResult(); + }), + }; + const captured = captureTrustedActiveOpenShellGatewayUserService(opts); + stopTrustedOpenShellGatewayUserServiceAndProveInactive(captured, opts); + + expect(() => resumeTrustedOpenShellGatewayUserServiceAndProveActive(captured, opts)).toThrow( + "reused its prior PID", + ); + }); + + it("leases the exact official Homebrew formula identity across stop and resume", () => { + const events: string[] = []; + let active = true; + let pid = 4242; + const opts = { + commandExists: (command: string) => command === "brew", + platform: "darwin" as const, + readProcessIdentity: (candidate: number) => + active && candidate === pid + ? processIdentity(pid === 4242 ? "process-start-a" : "process-start-b") + : null, + spawnSyncImpl: vi.fn((_command: string, args: string[]) => { + events.push(args.join(" ")); + if (args[0] === "list") return spawnResult(); + if (args[0] === "info") return officialFormulaInfo(); + if (args[0] === "services" && args[1] === "info") { + return officialRunningServiceInfo({ + loaded: active, + pid: active ? pid : 0, + running: active, + }); + } + if (args[0] === "services" && args[1] === "stop") { + active = false; + return spawnResult(); + } + if (args[0] === "services" && args[1] === "start") { + active = true; + pid = 5252; + return spawnResult(); + } + return spawnResult(1, "unexpected command"); + }), + }; + + const captured = captureTrustedActiveOpenShellGatewayUserService(opts); + expect(captured).toMatchObject({ + formulaName: "openshell", + formulaTap: "nvidia/openshell", + manager: "homebrew", + pid: 4242, + processArgv: GATEWAY_ARGV, + processStartIdentity: "process-start-a", + serviceIdentity: "homebrew.mxcl.openshell", + serviceName: "openshell", + }); + + stopTrustedOpenShellGatewayUserServiceAndProveInactive(captured, opts); + assertTrustedOpenShellGatewayUserServiceInactive(captured, opts); + const resumed = resumeTrustedOpenShellGatewayUserServiceAndProveActive(captured, opts); + + expect(resumed).toMatchObject({ + formulaName: "openshell", + formulaTap: "nvidia/openshell", + manager: "homebrew", + pid: 5252, + processArgv: GATEWAY_ARGV, + processStartIdentity: "process-start-b", + serviceIdentity: "homebrew.mxcl.openshell", + }); + expect(events.filter((event) => event === "info --json=v2 openshell")).toHaveLength(6); + expect(events).toContain("services stop openshell"); + expect(events).toContain("services start openshell"); + }); + + it("refuses Homebrew tap drift before stopping the captured service", () => { + const captureOpts = { + commandExists: () => true, + platform: "darwin" as const, + readProcessIdentity: () => processIdentity(), + spawnSyncImpl: vi.fn((_command: string, args: string[]) => { + if (args[0] === "info") return officialFormulaInfo(); + if (args[0] === "services") return officialRunningServiceInfo(); + return spawnResult(); + }), + }; + const captured = captureTrustedActiveOpenShellGatewayUserService(captureOpts); + const stop = vi.fn(); + + expect(() => + stopTrustedOpenShellGatewayUserServiceAndProveInactive(captured, { + ...captureOpts, + spawnSyncImpl: vi.fn((_command: string, args: string[]) => { + if (args[0] === "info") { + return spawnResult( + 0, + "", + JSON.stringify({ formulae: [{ name: "openshell", tap: "other/tap" }] }), + ); + } + if (args[0] === "services" && args[1] === "stop") stop(); + return spawnResult(); + }), + }), + ).toThrow("must come from nvidia/openshell"); + expect(stop).not.toHaveBeenCalled(); + }); + + it.each([ + ["argv drift", processIdentity("process-start-a", [...GATEWAY_ARGV, "--foreign"])], + ["PID reuse with a different process start", processIdentity("process-start-reused")], + ])("refuses Homebrew %s before mutation", (_case, driftedProcess) => { + let drifted = false; + const stop = vi.fn(); + const opts = { + commandExists: () => true, + platform: "darwin" as const, + readProcessIdentity: () => (drifted ? driftedProcess : processIdentity()), + spawnSyncImpl: vi.fn((_command: string, args: string[]) => { + if (args[0] === "info") return officialFormulaInfo(); + if (args[0] === "services" && args[1] === "info") { + return officialRunningServiceInfo(); + } + if (args[0] === "services" && args[1] === "stop") stop(); + return spawnResult(); + }), + }; + const captured = captureTrustedActiveOpenShellGatewayUserService(opts); + drifted = true; + + expect(() => stopTrustedOpenShellGatewayUserServiceAndProveInactive(captured, opts)).toThrow( + "process identity drifted", + ); + expect(stop).not.toHaveBeenCalled(); + }); + + it("fails closed when Homebrew reports a foreign service after stopping", () => { + let stopped = false; + const opts = { + commandExists: () => true, + platform: "darwin" as const, + readProcessIdentity: () => processIdentity(), + spawnSyncImpl: vi.fn((_command: string, args: string[]) => { + if (args[0] === "info") return officialFormulaInfo(); + if (args[0] === "services" && args[1] === "info") { + return stopped + ? officialRunningServiceInfo({ + loaded: false, + pid: 0, + running: false, + service_name: "foreign.openshell", + }) + : officialRunningServiceInfo(); + } + if (args[0] === "services" && args[1] === "stop") { + stopped = true; + return spawnResult(); + } + return spawnResult(); + }), + }; + const captured = captureTrustedActiveOpenShellGatewayUserService(opts); + + expect(() => stopTrustedOpenShellGatewayUserServiceAndProveInactive(captured, opts)).toThrow( + "foreign or incomplete", + ); + }); + it("removes a marked NemoClaw unit before activating an upstream systemd unit (#6903)", () => { const events: string[] = []; const removed: string[] = []; diff --git a/src/lib/onboard/docker-driver-gateway-service.ts b/src/lib/onboard/docker-driver-gateway-service.ts index 78e47ce04d..e13f76c627 100644 --- a/src/lib/onboard/docker-driver-gateway-service.ts +++ b/src/lib/onboard/docker-driver-gateway-service.ts @@ -8,6 +8,10 @@ import path from "node:path"; import { sleepSeconds, waitUntilAsync } from "../core/wait"; import { isGatewayHealthy } from "../state/gateway"; +import { + captureHostProcessIdentity, + type HostProcessIdentity, +} from "./compute/host-process-identity"; import { envInt } from "./env"; import { createGatewayHealthWaitOptions, @@ -32,6 +36,7 @@ export interface OpenShellGatewayUserServiceOptions { platform?: NodeJS.Platform; preparePortForServiceStart?: () => void; prepareServiceEnv?: () => void; + readProcessIdentity?: (pid: number) => OpenShellGatewayProcessIdentity | null; readFileSync?: (filePath: string, encoding: BufferEncoding) => string; rmSync?: typeof fs.rmSync; spawnSyncImpl?: SpawnSyncLike; @@ -61,8 +66,11 @@ export type SpawnSyncLike = ( options?: SpawnSyncOptions, ) => SpawnSyncLikeResult; -export interface PackageManagedDockerDriverGatewayOptions { +export type OpenShellGatewayProcessIdentity = HostProcessIdentity; + +export interface PackageManagedDriverGatewayOptions { clearDockerDriverGatewayRuntimeFiles: () => void; + driverLabel?: string; exitOnFailure: boolean; gatewayName: string; hasOpenShellGatewayUserService?: () => boolean; @@ -89,6 +97,8 @@ export interface PackageManagedDockerDriverGatewayOptions { ) => Promise; } +export type PackageManagedDockerDriverGatewayOptions = PackageManagedDriverGatewayOptions; + interface OpenShellGatewayUserServiceTarget { manager: "homebrew" | "systemd"; serviceName: string; @@ -97,6 +107,44 @@ interface OpenShellGatewayUserServiceTarget { trustedUnitPaths: string[]; } +const TRUSTED_GATEWAY_SERVICE_IDENTITY = Symbol("trusted-openshell-gateway-service-identity"); + +interface TrustedOpenShellGatewayUserServiceIdentityBase { + readonly [TRUSTED_GATEWAY_SERVICE_IDENTITY]: true; + readonly pid: number; + readonly processArgv: readonly string[]; + readonly processStartIdentity: string; + readonly serviceName: string; +} + +export interface TrustedHomebrewOpenShellGatewayUserServiceIdentity + extends TrustedOpenShellGatewayUserServiceIdentityBase { + readonly formulaName: typeof OPENSHELL_GATEWAY_HOMEBREW_SERVICE; + readonly formulaTap: typeof OPENSHELL_GATEWAY_HOMEBREW_TAP; + readonly manager: "homebrew"; + readonly serviceIdentity: string; +} + +export interface TrustedSystemdOpenShellGatewayUserServiceIdentity + extends TrustedOpenShellGatewayUserServiceIdentityBase { + readonly execStart: string; + readonly execStartPath: string; + readonly invocationId: string; + readonly manager: "systemd"; + readonly unitPath: string; +} + +/** + * Opaque, immutable evidence for one active trusted package-managed gateway. + * + * Callers may retain and inspect this receipt, but only this module can mint + * one. Every lifecycle mutation re-resolves the package-managed authority and + * proves that its exact unit/formula identity still matches this receipt. + */ +export type TrustedActiveOpenShellGatewayUserServiceIdentity = + | TrustedHomebrewOpenShellGatewayUserServiceIdentity + | TrustedSystemdOpenShellGatewayUserServiceIdentity; + export function getOpenShellGatewayUserServicePaths(): string[] { return [ "/usr/local/lib/systemd/user/openshell-gateway.service", @@ -355,6 +403,10 @@ function extractSystemdExecStartPath(execStart: string): string | null { return candidate && path.isAbsolute(candidate) ? path.normalize(candidate) : null; } +function isSystemdInvocationId(value: string): boolean { + return /^[0-9a-f]{32}$/i.test(value); +} + function validateSystemdServiceIdentity( service: OpenShellGatewayUserServiceTarget, opts: Required>, @@ -388,6 +440,498 @@ function validateSystemdServiceIdentityFromProperties( }; } +interface TrustedServiceContext { + commandExists: (command: string) => boolean; + env: NodeJS.ProcessEnv; + readProcessIdentity: (pid: number) => OpenShellGatewayProcessIdentity | null; + service: OpenShellGatewayUserServiceTarget; + spawnSyncImpl: SpawnSyncLike; +} + +interface SystemdServiceState { + activeState: string; + execStart: string; + execStartPath: string; + invocationId: string; + mainPid: number; + unitPath: string; +} + +interface HomebrewServiceState { + loaded: boolean; + pid: number | null; + running: boolean; + serviceIdentity: string; +} + +function freezeProcessIdentity( + identity: OpenShellGatewayProcessIdentity | null, +): OpenShellGatewayProcessIdentity | null { + if (identity === null) return null; + if ( + !Array.isArray(identity.argv) || + identity.argv.length === 0 || + identity.argv.some((value) => typeof value !== "string") || + typeof identity.startIdentity !== "string" || + !identity.startIdentity.trim() + ) { + throw new Error("OpenShell gateway process identity is incomplete"); + } + return Object.freeze({ + argv: Object.freeze([...identity.argv]), + startIdentity: identity.startIdentity, + }); +} + +function trustedServiceContext(opts: OpenShellGatewayUserServiceOptions): TrustedServiceContext { + const platform = opts.platform ?? process.platform; + if (platform !== "linux" && platform !== "darwin") { + throw new Error(`OpenShell gateway service lifecycle is unsupported on ${platform}`); + } + const env = opts.env ?? process.env; + const home = effectiveHome(opts.home, opts.env); + const commandExists = opts.commandExists ?? ((command) => defaultCommandExists(command, env)); + const spawnSyncImpl = opts.spawnSyncImpl ?? spawnSync; + const readProcessIdentity = + opts.readProcessIdentity ?? + ((pid: number) => + captureHostProcessIdentity(pid, { + env, + platform, + run: spawnSyncImpl, + })); + const service = resolveOpenShellGatewayUserService({ ...opts, env, home }); + if (!service) { + throw new Error("No trusted package-managed OpenShell gateway service is installed"); + } + const command = service.manager === "homebrew" ? "brew" : "systemctl"; + if (!commandExists(command)) { + throw new Error(`${command} is unavailable for the trusted OpenShell gateway service`); + } + return { commandExists, env, readProcessIdentity, service, spawnSyncImpl }; +} + +function queryTrustedSystemdServiceState(context: TrustedServiceContext): SystemdServiceState { + const result = runSystemctlUser( + [ + "show", + context.service.serviceName, + "--property=FragmentPath", + "--property=ExecStart", + "--property=ActiveState", + "--property=InvocationID", + "--property=MainPID", + ], + context, + ); + if (!result.ok) { + throw new Error( + `Failed to query trusted OpenShell gateway systemd service: ${result.reason ?? "unknown error"}`, + ); + } + const properties = parseSystemctlShow(result.stdout ?? ""); + const identity = validateSystemdServiceIdentityFromProperties(context.service, properties); + if (!identity.ok) { + throw new Error(identity.reason ?? "OpenShell gateway systemd service identity is invalid"); + } + const unitPath = path.normalize(properties.FragmentPath ?? ""); + const execStart = (properties.ExecStart ?? "").trim(); + const execStartPath = extractSystemdExecStartPath(execStart); + const invocationId = (properties.InvocationID ?? "").trim(); + const mainPid = Number(properties.MainPID); + if ( + !execStart || + execStartPath === null || + !Number.isSafeInteger(mainPid) || + mainPid < 0 || + !(properties.ActiveState ?? "").trim() + ) { + throw new Error("OpenShell gateway systemd service returned incomplete lifecycle state"); + } + return { + activeState: properties.ActiveState, + execStart, + execStartPath, + invocationId, + mainPid, + unitPath, + }; +} + +function queryTrustedHomebrewServiceState(context: TrustedServiceContext): HomebrewServiceState { + const result = runBrew(["services", "info", context.service.serviceName, "--json"], context); + if (!result.ok) { + throw new Error( + `Failed to query trusted OpenShell gateway Homebrew service: ${result.reason ?? "unknown error"}`, + ); + } + let records: unknown; + try { + records = JSON.parse(result.stdout ?? ""); + } catch { + throw new Error("OpenShell gateway Homebrew service state returned invalid JSON"); + } + if (!Array.isArray(records)) { + throw new Error("OpenShell gateway Homebrew service state did not return an array"); + } + const namedRecords = records.filter( + (candidate): candidate is Record => + candidate !== null && + typeof candidate === "object" && + (candidate as Record).name === context.service.serviceName, + ); + if (namedRecords.length !== 1) { + throw new Error("OpenShell gateway Homebrew service identity is missing or ambiguous"); + } + const record = namedRecords[0]; + const expectedServiceIdentity = `homebrew.mxcl.${context.service.serviceName}`; + if ( + record.service_name !== expectedServiceIdentity || + typeof record.loaded !== "boolean" || + typeof record.running !== "boolean" + ) { + throw new Error("OpenShell gateway Homebrew service identity is foreign or incomplete"); + } + const rawPid = record.pid; + const pid = + rawPid === null || rawPid === undefined + ? null + : typeof rawPid === "number" && Number.isSafeInteger(rawPid) && rawPid >= 0 + ? rawPid + : Number.NaN; + if (Number.isNaN(pid)) { + throw new Error("OpenShell gateway Homebrew service returned an invalid process ID"); + } + return { + loaded: record.loaded, + pid, + running: record.running, + serviceIdentity: expectedServiceIdentity, + }; +} + +function requireTrustedIdentityReceipt( + identity: TrustedActiveOpenShellGatewayUserServiceIdentity, +): void { + if ( + identity === null || + typeof identity !== "object" || + identity[TRUSTED_GATEWAY_SERVICE_IDENTITY] !== true || + !Object.isFrozen(identity) + ) { + throw new Error("OpenShell gateway service lifecycle requires a captured trusted identity"); + } +} + +function sameArgv(expected: readonly string[], actual: readonly string[]): boolean { + return ( + actual.length === expected.length && actual.every((value, index) => value === expected[index]) + ); +} + +function sameProcessIdentity( + expected: Pick< + TrustedActiveOpenShellGatewayUserServiceIdentity, + "processArgv" | "processStartIdentity" + >, + actual: OpenShellGatewayProcessIdentity | null, +): boolean { + return ( + actual !== null && + actual.startIdentity === expected.processStartIdentity && + sameArgv(expected.processArgv, actual.argv) + ); +} + +function requireActiveProcessIdentity( + context: TrustedServiceContext, + pid: number, +): OpenShellGatewayProcessIdentity { + const identity = freezeProcessIdentity(context.readProcessIdentity(pid)); + if (!identity) { + throw new Error("Trusted OpenShell gateway process identity is unavailable or incomplete"); + } + return identity; +} + +function assertCapturedProcessInactive( + identity: TrustedActiveOpenShellGatewayUserServiceIdentity, + context: TrustedServiceContext, +): void { + const current = freezeProcessIdentity(context.readProcessIdentity(identity.pid)); + if (sameProcessIdentity(identity, current)) { + throw new Error("Trusted OpenShell gateway captured process is still active"); + } +} + +function assertMatchingServiceTarget( + identity: TrustedActiveOpenShellGatewayUserServiceIdentity, + context: TrustedServiceContext, +): void { + if ( + context.service.manager !== identity.manager || + context.service.serviceName !== identity.serviceName + ) { + throw new Error("Trusted OpenShell gateway service authority drifted"); + } +} + +function requireMatchingSystemdState( + identity: TrustedSystemdOpenShellGatewayUserServiceIdentity, + context: TrustedServiceContext, +): SystemdServiceState { + const state = queryTrustedSystemdServiceState(context); + if ( + state.unitPath !== identity.unitPath || + state.execStart !== identity.execStart || + state.execStartPath !== identity.execStartPath + ) { + throw new Error("Trusted OpenShell gateway systemd unit identity drifted"); + } + return state; +} + +function requireMatchingHomebrewState( + identity: TrustedHomebrewOpenShellGatewayUserServiceIdentity, + context: TrustedServiceContext, +): HomebrewServiceState { + if ( + identity.formulaName !== OPENSHELL_GATEWAY_HOMEBREW_SERVICE || + identity.formulaTap !== OPENSHELL_GATEWAY_HOMEBREW_TAP + ) { + throw new Error("Trusted OpenShell gateway Homebrew formula identity drifted"); + } + const state = queryTrustedHomebrewServiceState(context); + if (state.serviceIdentity !== identity.serviceIdentity) { + throw new Error("Trusted OpenShell gateway Homebrew service identity drifted"); + } + return state; +} + +function trustedActiveIdentity( + context: TrustedServiceContext, +): TrustedActiveOpenShellGatewayUserServiceIdentity { + if (context.service.manager === "systemd") { + const state = queryTrustedSystemdServiceState(context); + if ( + state.activeState !== "active" || + state.mainPid <= 0 || + !isSystemdInvocationId(state.invocationId) + ) { + throw new Error("Trusted OpenShell gateway systemd service is not active"); + } + const process = requireActiveProcessIdentity(context, state.mainPid); + return Object.freeze({ + [TRUSTED_GATEWAY_SERVICE_IDENTITY]: true as const, + execStart: state.execStart, + execStartPath: state.execStartPath, + invocationId: state.invocationId, + manager: "systemd", + pid: state.mainPid, + processArgv: process.argv, + processStartIdentity: process.startIdentity, + serviceName: context.service.serviceName, + unitPath: state.unitPath, + }); + } + const state = queryTrustedHomebrewServiceState(context); + if (state.loaded !== true || state.running !== true || state.pid === null || state.pid <= 0) { + throw new Error("Trusted OpenShell gateway Homebrew service is not active"); + } + const process = requireActiveProcessIdentity(context, state.pid); + return Object.freeze({ + [TRUSTED_GATEWAY_SERVICE_IDENTITY]: true as const, + formulaName: OPENSHELL_GATEWAY_HOMEBREW_SERVICE, + formulaTap: OPENSHELL_GATEWAY_HOMEBREW_TAP, + manager: "homebrew", + pid: state.pid, + processArgv: process.argv, + processStartIdentity: process.startIdentity, + serviceIdentity: state.serviceIdentity, + serviceName: context.service.serviceName, + }); +} + +/** + * Capture the exact active package-managed gateway authority and process. + * + * This is intentionally strict: unavailable managers, inactive services, + * foreign units/formulae, malformed state, and ambiguous identities all throw. + */ +export function captureTrustedActiveOpenShellGatewayUserService( + opts: OpenShellGatewayUserServiceOptions = {}, +): TrustedActiveOpenShellGatewayUserServiceIdentity { + return trustedActiveIdentity(trustedServiceContext(opts)); +} + +/** + * Distinguish an exactly inactive installed service from the active lifecycle + * owner. Transitional, malformed, foreign, or unavailable manager state still + * fails closed; only a fully proven inactive service returns null. + */ +export function captureTrustedOpenShellGatewayUserServiceIfActive( + opts: OpenShellGatewayUserServiceOptions = {}, +): TrustedActiveOpenShellGatewayUserServiceIdentity | null { + const context = trustedServiceContext(opts); + if (context.service.manager === "systemd") { + const state = queryTrustedSystemdServiceState(context); + if (state.activeState === "inactive" && state.mainPid === 0) return null; + if ( + state.activeState !== "active" || + state.mainPid <= 0 || + !isSystemdInvocationId(state.invocationId) + ) { + throw new Error("Trusted OpenShell gateway systemd service state is transitional"); + } + } else { + const state = queryTrustedHomebrewServiceState(context); + if (!state.loaded && !state.running && (state.pid === null || state.pid === 0)) return null; + if (!state.loaded || !state.running || state.pid === null || state.pid <= 0) { + throw new Error("Trusted OpenShell gateway Homebrew service state is transitional"); + } + } + return trustedActiveIdentity(context); +} + +function requireInactiveServiceContext( + identity: TrustedActiveOpenShellGatewayUserServiceIdentity, + opts: OpenShellGatewayUserServiceOptions, +): TrustedServiceContext { + requireTrustedIdentityReceipt(identity); + const context = trustedServiceContext(opts); + assertMatchingServiceTarget(identity, context); + if (identity.manager === "systemd") { + const state = requireMatchingSystemdState(identity, context); + if (state.activeState !== "inactive" || state.mainPid !== 0) { + throw new Error("Trusted OpenShell gateway systemd service is not proven inactive"); + } + } else { + const state = requireMatchingHomebrewState(identity, context); + if (state.loaded || state.running || (state.pid !== null && state.pid !== 0)) { + throw new Error("Trusted OpenShell gateway Homebrew service is not proven inactive"); + } + } + assertCapturedProcessInactive(identity, context); + return context; +} + +/** + * Revalidate that the exact captured service authority remains inactive. + */ +export function assertTrustedOpenShellGatewayUserServiceInactive( + identity: TrustedActiveOpenShellGatewayUserServiceIdentity, + opts: OpenShellGatewayUserServiceOptions = {}, +): void { + requireInactiveServiceContext(identity, opts); +} + +function requireExactActiveServiceContext( + identity: TrustedActiveOpenShellGatewayUserServiceIdentity, + opts: OpenShellGatewayUserServiceOptions, +): TrustedServiceContext { + requireTrustedIdentityReceipt(identity); + const context = trustedServiceContext(opts); + assertMatchingServiceTarget(identity, context); + if (identity.manager === "systemd") { + const state = requireMatchingSystemdState(identity, context); + const process = + state.mainPid > 0 ? freezeProcessIdentity(context.readProcessIdentity(state.mainPid)) : null; + if ( + state.activeState !== "active" || + state.mainPid !== identity.pid || + state.invocationId !== identity.invocationId || + !sameProcessIdentity(identity, process) + ) { + throw new Error("Trusted OpenShell gateway systemd process identity drifted"); + } + } else { + const state = requireMatchingHomebrewState(identity, context); + const process = + state.pid !== null && state.pid > 0 + ? freezeProcessIdentity(context.readProcessIdentity(state.pid)) + : null; + if ( + !state.loaded || + !state.running || + state.pid === null || + state.pid !== identity.pid || + !sameProcessIdentity(identity, process) + ) { + throw new Error("Trusted OpenShell gateway Homebrew process identity drifted"); + } + } + return context; +} + +/** + * Stop only the exact captured manager/service identity, then independently + * prove that the same trusted authority is inactive and did not respawn. + */ +export function stopTrustedOpenShellGatewayUserServiceAndProveInactive( + identity: TrustedActiveOpenShellGatewayUserServiceIdentity, + opts: OpenShellGatewayUserServiceOptions = {}, +): void { + const context = requireExactActiveServiceContext(identity, opts); + const result = + identity.manager === "homebrew" + ? runBrew(["services", "stop", identity.serviceName], context) + : runSystemctlUser(["stop", identity.serviceName], context); + if (!result.ok) { + throw new Error( + `Failed to stop trusted OpenShell gateway ${identity.manager} service: ${ + result.reason ?? "unknown error" + }`, + ); + } + requireInactiveServiceContext(identity, opts); +} + +/** + * Start the same captured manager/service identity from a proven stopped state, + * then independently prove the exact unit/formula identity and a new PID. + */ +export function resumeTrustedOpenShellGatewayUserServiceAndProveActive( + identity: TrustedActiveOpenShellGatewayUserServiceIdentity, + opts: OpenShellGatewayUserServiceOptions = {}, +): TrustedActiveOpenShellGatewayUserServiceIdentity { + const context = requireInactiveServiceContext(identity, opts); + const result = + identity.manager === "homebrew" + ? runBrew(["services", "start", identity.serviceName], context) + : runSystemctlUser(["start", identity.serviceName], context); + if (!result.ok) { + throw new Error( + `Failed to resume trusted OpenShell gateway ${identity.manager} service: ${ + result.reason ?? "unknown error" + }`, + ); + } + const resumed = trustedActiveIdentity(trustedServiceContext(opts)); + if ( + resumed.manager !== identity.manager || + resumed.serviceName !== identity.serviceName || + resumed.pid === identity.pid || + (identity.manager === "systemd" && + (resumed.manager !== "systemd" || + resumed.unitPath !== identity.unitPath || + resumed.execStart !== identity.execStart || + resumed.execStartPath !== identity.execStartPath || + resumed.invocationId === identity.invocationId || + !sameArgv(identity.processArgv, resumed.processArgv) || + resumed.processStartIdentity === identity.processStartIdentity)) || + (identity.manager === "homebrew" && + (resumed.manager !== "homebrew" || + resumed.formulaName !== identity.formulaName || + resumed.formulaTap !== identity.formulaTap || + resumed.serviceIdentity !== identity.serviceIdentity || + !sameArgv(identity.processArgv, resumed.processArgv) || + resumed.processStartIdentity === identity.processStartIdentity)) + ) { + throw new Error("Resumed OpenShell gateway service identity drifted or reused its prior PID"); + } + return resumed; +} + export function getTrustedActiveOpenShellGatewayUserServicePid( opts: OpenShellGatewayUserServiceOptions = {}, ): number | null { @@ -642,8 +1186,9 @@ export function startOpenShellGatewayUserService( }; } -export async function startPackageManagedDockerDriverGateway({ +export async function startPackageManagedDriverGateway({ clearDockerDriverGatewayRuntimeFiles, + driverLabel = "Docker", exitOnFailure, gatewayName, hasOpenShellGatewayUserService: hasService = hasOpenShellGatewayUserService, @@ -660,10 +1205,10 @@ export async function startPackageManagedDockerDriverGateway({ startOpenShellGatewayUserService: startService = startOpenShellGatewayUserService, validatePortOwnerForOpenShellGatewayUserServiceStart, verifySandboxBridgeGatewayReachableOrExit, -}: PackageManagedDockerDriverGatewayOptions): Promise { +}: PackageManagedDriverGatewayOptions): Promise { if (!hasService()) return false; - console.log(" Starting OpenShell Docker-driver gateway via managed service..."); + console.log(` Starting OpenShell ${driverLabel}-driver gateway via managed service...`); const serviceStart = startService({ preparePortForServiceStart: preparePortForOpenShellGatewayUserServiceStart, prepareServiceEnv: prepareOpenShellGatewayUserServiceEnv, @@ -734,3 +1279,9 @@ export async function startPackageManagedDockerDriverGateway({ if (exitOnFailure) process.exit(1); throw new Error(message); } + +export function startPackageManagedDockerDriverGateway( + options: PackageManagedDockerDriverGatewayOptions, +): Promise { + return startPackageManagedDriverGateway({ ...options, driverLabel: "Docker" }); +} diff --git a/src/lib/onboard/docker-driver-platform.ts b/src/lib/onboard/docker-driver-platform.ts index 2dbc453cc4..bb0b075640 100644 --- a/src/lib/onboard/docker-driver-platform.ts +++ b/src/lib/onboard/docker-driver-platform.ts @@ -1,9 +1,13 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { resolveCurrentOpenShellComputePlan, usesManagedDockerGateway } from "./compute/plan"; + +export { resolveCurrentOpenShellComputePlan } from "./compute/plan"; + export function isLinuxDockerDriverGatewayEnabled( platform: NodeJS.Platform = process.platform, arch: NodeJS.Architecture = process.arch, ): boolean { - return platform === "linux" || (platform === "darwin" && arch === "arm64"); + return usesManagedDockerGateway(resolveCurrentOpenShellComputePlan(platform, arch)); } diff --git a/src/lib/onboard/docker-gpu-patch-clone.test.ts b/src/lib/onboard/docker-gpu-patch-clone.test.ts index 8ad39b3016..77079d33b0 100644 --- a/src/lib/onboard/docker-gpu-patch-clone.test.ts +++ b/src/lib/onboard/docker-gpu-patch-clone.test.ts @@ -95,6 +95,37 @@ describe("Docker GPU clone envelope", () => { ); }); + it("persists the image-owned hold without replacing the original supervisor entrypoint", () => { + const inspect = inspectFixture(); + inspect.Config!.Entrypoint = ["/opt/openshell/bin/openshell-sandbox", "--gateway-mode"]; + inspect.Config!.Cmd = ["serve", "--foreground"]; + const immutableImage = inspect.Image as string; + const args = buildDockerGpuCloneRunArgs(inspect, buildDockerGpuMode("startup-command"), { + image: immutableImage, + openshellSandboxCommand: [ + "env", + "/usr/local/bin/nemoclaw-managed-startup-hold", + "--agent", + "hermes", + "--profile-fingerprint", + "a".repeat(64), + ], + }); + + expect(args).toEqual( + expect.arrayContaining([ + "--env", + `OPENSHELL_SANDBOX_COMMAND=env /usr/local/bin/nemoclaw-managed-startup-hold --agent hermes --profile-fingerprint ${"a".repeat(64)}`, + "--entrypoint", + "/opt/openshell/bin/openshell-sandbox", + ]), + ); + const imageIndex = args.indexOf(immutableImage); + expect(imageIndex).toBeGreaterThan(0); + expect(args.slice(imageIndex)).toEqual([immutableImage]); + expect(args).not.toContain("openshell/sandbox:abc"); + }); + it("preserves inspected ulimits and overrides DCode's exact required limits", () => { const inspect = inspectFixture(); inspect.HostConfig!.Ulimits = [ diff --git a/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts b/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts index fb94eba175..3dbf57e674 100644 --- a/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts +++ b/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts @@ -42,7 +42,7 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { vi.restoreAllMocks(); }); - it("defers backup removal until waitForSupervisorReconnectIfNeeded sees supervisorReady=true", () => { + it("retains the backup after reconnect and removes it only after post-Ready commit", () => { const deps = makeDeps(); const result = deferredCreateResult(); const recreatePatch = vi.fn(() => result); @@ -84,6 +84,9 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { patch.waitForSupervisorReconnectIfNeeded(); expect(waitForSupervisor).toHaveBeenCalledTimes(1); + expect(finalizeBackup).not.toHaveBeenCalled(); + + patch.commitAfterReady(); expect(finalizeBackup).toHaveBeenCalledTimes(1); expect(finalizeBackup).toHaveBeenCalledWith({ result, supervisorReady: true }, deps); expect(capturePreRollbackDiagnostics).not.toHaveBeenCalled(); @@ -113,6 +116,9 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { patch.maybeApplyDuringCreate(); patch.waitForSupervisorReconnectIfNeeded(); + expect(onPatchFailureExit).not.toHaveBeenCalled(); + + patch.commitAfterReady(); expect(onPatchFailureExit).toHaveBeenCalledOnce(); expect(onPatchFailureExit.mock.calls[0]?.[1]).toEqual( diff --git a/src/lib/onboard/docker-gpu-sandbox-create.ts b/src/lib/onboard/docker-gpu-sandbox-create.ts index b683cbb1c7..a27c91bec4 100644 --- a/src/lib/onboard/docker-gpu-sandbox-create.ts +++ b/src/lib/onboard/docker-gpu-sandbox-create.ts @@ -28,7 +28,15 @@ import { type RecreateGpuPatchFn, type RecreateStartupPatchFn, } from "./docker-startup-command-sandbox-create"; +import { + applyDockerManagedStartupRootRequest, + type DockerManagedStartupTransaction, + getDockerManagedStartupFailureTransaction, +} from "./managed-startup/docker-root-apply"; +import { finalizeDockerManagedStartupSharedState } from "./managed-startup/docker-shared-state"; +import type { ManagedStartupRootApplyRequest } from "./managed-startup/root-apply"; import { findOpenShellDockerSandboxContainerIds } from "./openshell-docker-sandbox-containers"; +import type { SandboxCreateRuntimePatch } from "./sandbox-create-runtime/types"; export type { DockerGpuRoutePlan, @@ -42,13 +50,15 @@ export { type DockerGpuSandboxCreateDeps = Pick< DockerGpuPatchDeps, - "runOpenshell" | "runCaptureOpenshell" | "sleep" | "dockerCapture" + "runOpenshell" | "runCaptureOpenshell" | "sleep" | "dockerCapture" | "dockerRun" | "dockerStop" >; type WaitSupervisorFn = typeof waitForOpenShellSupervisorReconnect; type FindContainerIdsFn = typeof findOpenShellDockerSandboxContainerIds; type FinalizeBackupFn = typeof finalizeDockerGpuPatchBackup; type CapturePreRollbackDiagnosticsFn = typeof captureDockerGpuPreRollbackDiagnostics; +type FinalizeManagedStartupSharedStateFn = typeof finalizeDockerManagedStartupSharedState; +type ApplyManagedStartupRootRequestFn = typeof applyDockerManagedStartupRootRequest; // Loosen the override return type from `never` to `void` so tests can pass a // plain `vi.fn()` mock. Production wires `printDockerGpuPatchFailureAndExit` // which has return type `never`; that is assignable to `void`. @@ -58,9 +68,10 @@ type PatchFailureExitFn = ( deps: Parameters[2], ) => void; -type DockerGpuSandboxCreatePatchOptions = { +export type DockerGpuSandboxCreatePatchOptions = { route: SelectedDockerGpuRoute; persistStartupCommand?: boolean; + managedStartupRootApplyRequest?: ManagedStartupRootApplyRequest | null; sandboxName: string; gpuDevice?: string | null; openshellSandboxCommand?: readonly string[] | null; @@ -84,53 +95,47 @@ type DockerGpuSandboxCreatePatchOptions = { findContainerIds?: FindContainerIdsFn; recreatePatch?: RecreateGpuPatchFn; recreateStartupPatch?: RecreateStartupPatchFn; + applyManagedStartupRootRequest?: ApplyManagedStartupRootRequestFn; waitForSupervisor?: WaitSupervisorFn; finalizeBackup?: FinalizeBackupFn; + finalizeManagedStartupSharedState?: FinalizeManagedStartupSharedStateFn; capturePreRollbackDiagnostics?: CapturePreRollbackDiagnosticsFn; onPatchFailureExit?: PatchFailureExitFn; }; }; -export type DockerGpuSandboxCreatePatch = { - maybeApplyDuringCreate: () => void; - createFailureMessage: () => string | null; - exitOnPatchError: () => void; - ensureApplied: () => void; - waitForSupervisorReconnectIfNeeded: () => void; - selectedMode: () => DockerGpuPatchMode | null; - /** - * Print the Docker GPU readiness-failure block (including the Error-phase - * classification + patched container State diagnostics) when the - * post-create readiness wait times out. No-op when the patch is disabled. - */ - printReadinessFailureIfEnabled: () => void; - /** - * Run the GPU proof while distinguishing "sandbox in terminal phase" from - * "proof failed inside a live sandbox". Calls `process.exit(1)` for the - * former and rethrows after printing diagnostics for the latter so the - * onboarding flow surfaces the right failure cause (#4316). Returns the - * CUDA-usability proof result on success so callers can persist it (#4231). - */ - verifyGpuOrExit: ( +export interface DockerGpuSandboxCreateHooks { + selectedMode(): DockerGpuPatchMode | null; + printReadinessFailureIfEnabled(): void; + verifyGpuOrExit( verifyDirectSandboxGpu: (sandboxName: string) => SandboxGpuProofResult, - ) => SandboxGpuProofResult; -}; + ): SandboxGpuProofResult; +} + +export type DockerGpuSandboxCreatePatch = SandboxCreateRuntimePatch & DockerGpuSandboxCreateHooks; export function createDockerGpuSandboxCreatePatch( options: DockerGpuSandboxCreatePatchOptions, ): DockerGpuSandboxCreatePatch { const routeAdapter = adaptDockerGpuRouteForPatch(options.route); let result: DockerGpuPatchResult | null = null; + let managedStartupTransaction: DockerManagedStartupTransaction | null = null; + let managedStartupApplied = false; let patchError: unknown = null; let needsSupervisorWait = false; + let managedStartupFinalized = false; const findContainerIds = options.overrides?.findContainerIds ?? findOpenShellDockerSandboxContainerIds; const recreatePatch = options.overrides?.recreatePatch ?? recreateOpenShellDockerSandboxWithGpu; const recreateStartupPatch = options.overrides?.recreateStartupPatch; + const applyManagedStartupRootRequest = + options.overrides?.applyManagedStartupRootRequest ?? applyDockerManagedStartupRootRequest; const waitForSupervisor = options.overrides?.waitForSupervisor ?? waitForOpenShellSupervisorReconnect; const finalizeBackup = options.overrides?.finalizeBackup ?? finalizeDockerGpuPatchBackup; + const finalizeManagedStartupSharedState = + options.overrides?.finalizeManagedStartupSharedState ?? finalizeDockerManagedStartupSharedState; const captureFailedClone = options.overrides?.capturePreRollbackDiagnostics ?? captureDockerGpuPreRollbackDiagnostics; const onPatchFailureExit = @@ -145,8 +150,18 @@ export function createDockerGpuSandboxCreatePatch( backend: options.backend, dockerDesktopWsl: options.dockerDesktopWsl ?? isDockerDesktopWslRuntime(), }; - const patchEnabled = routeAdapter.enabled || options.persistStartupCommand === true; - const patchTarget = routeAdapter.enabled ? "NVIDIA GPU access" : "restart-safe startup"; + const recreationEnabled = routeAdapter.enabled || options.persistStartupCommand === true; + const managedStartupEnabled = options.managedStartupRootApplyRequest != null; + const patchEnabled = recreationEnabled || managedStartupEnabled; + const patchTarget = routeAdapter.enabled + ? managedStartupEnabled + ? "NVIDIA GPU access and managed startup" + : "NVIDIA GPU access" + : recreationEnabled + ? managedStartupEnabled + ? "restart-safe startup and managed startup" + : "restart-safe startup" + : "managed startup"; const recreateSelectedPatch = createDockerSandboxRecreator({ gpuEnabled: routeAdapter.enabled, gpuOptions: applyOptions, @@ -156,22 +171,105 @@ export function createDockerGpuSandboxCreatePatch( recreateStartup: recreateStartupPatch, }); + const applyManagedStartupToContainer = (containerId: string): void => { + const request = options.managedStartupRootApplyRequest; + if (!request) return; + console.log( + ` Applying the ${request.agent} managed startup profile to exact container ${containerId.slice(0, 12)}...`, + ); + managedStartupTransaction = applyManagedStartupRootRequest( + { containerId, request }, + { dockerCapture: options.deps.dockerCapture }, + ); + managedStartupApplied = true; + console.log( + managedStartupTransaction + ? ` ✓ Managed startup profile applied for ${request.agent}` + : ` ✓ Managed startup profile was already complete for ${request.agent}`, + ); + }; + + const applyPatch = (deps: DockerGpuPatchDeps): void => { + let targetContainerId: string; + if (recreationEnabled) { + result = recreateSelectedPatch(false, deps); + needsSupervisorWait = true; + targetContainerId = result.newContainerId; + console.log(` ✓ Docker container mode selected: ${result.mode.label}`); + } else { + const containerIds = findContainerIds(options.sandboxName); + if (containerIds.length !== 1) { + throw new Error( + `Managed startup requires exactly one OpenShell Docker container; found ${String( + containerIds.length, + )}.`, + ); + } + targetContainerId = containerIds[0] as string; + } + applyManagedStartupToContainer(targetContainerId); + }; + + const rollbackAfterFailure = (): Error | null => { + if (managedStartupFinalized || (!managedStartupTransaction && !result)) return null; + try { + if (managedStartupTransaction) { + finalizeManagedStartupSharedState( + { + transaction: managedStartupTransaction, + patchResult: result, + supervisorReady: false, + }, + options.deps, + ); + } + if (result) finalizeBackup({ result, supervisorReady: false }, options.deps); + managedStartupFinalized = true; + needsSupervisorWait = false; + return null; + } catch (error) { + return error instanceof Error ? error : new Error(String(error)); + } + }; + + const reportPatchErrorAndExit = (): void => { + if (!patchError) return; + const rollbackError = rollbackAfterFailure(); + if (rollbackError) { + patchError = new Error( + `${patchError instanceof Error ? patchError.message : String(patchError)}; managed startup rollback failed: ${rollbackError.message}`, + ); + } + onPatchFailureExit(options.sandboxName, patchError, { + runCaptureOpenshell: options.deps.runCaptureOpenshell, + dockerCapture: options.deps.dockerCapture, + additionalSummaryLines: routeAdapter.additionalSummaryLines, + }); + }; + return { + revalidateBeforeMutation() {}, + maybeApplyDuringCreate() { - if (!patchEnabled || result || patchError) return; + if (!patchEnabled || result || managedStartupApplied || patchError) return; const containerIds = findContainerIds(options.sandboxName); if (containerIds.length === 0) return; + if (containerIds.length !== 1) { + patchError = new Error( + `Managed startup observed ${String(containerIds.length)} matching Docker containers; refusing an ambiguous root application.`, + ); + return; + } console.log( - ` OpenShell Docker container detected; recreating it with ${patchTarget} before readiness wait...`, + ` OpenShell Docker container detected; applying ${patchTarget} before readiness wait...`, ); try { - result = recreateSelectedPatch(false, { + applyPatch({ runCaptureOpenshell: options.deps.runCaptureOpenshell, sleep: options.deps.sleep, }); - needsSupervisorWait = true; - console.log(` ✓ Docker container mode selected: ${result.mode.label}`); } catch (error) { + managedStartupTransaction ??= getDockerManagedStartupFailureTransaction(error); patchError = error; } }, @@ -180,12 +278,19 @@ export function createDockerGpuSandboxCreatePatch( if (!patchError) return null; return routeAdapter.enabled ? "Docker GPU patch failed while OpenShell sandbox create was still waiting." - : "Docker startup-command patch failed while OpenShell sandbox create was still waiting."; + : managedStartupEnabled + ? "Docker managed startup failed while OpenShell sandbox create was still waiting." + : "Docker startup-command patch failed while OpenShell sandbox create was still waiting."; }, exitOnPatchError() { - if (!patchError) return; - onPatchFailureExit(options.sandboxName, patchError, { + reportPatchErrorAndExit(); + }, + + rollbackManagedStartupAfterCreateFailure() { + const rollbackError = rollbackAfterFailure(); + if (!rollbackError) return; + onPatchFailureExit(options.sandboxName, rollbackError, { runCaptureOpenshell: options.deps.runCaptureOpenshell, dockerCapture: options.deps.dockerCapture, additionalSummaryLines: routeAdapter.additionalSummaryLines, @@ -193,23 +298,19 @@ export function createDockerGpuSandboxCreatePatch( }, ensureApplied() { - if (!patchEnabled || result) return; - console.log(` Recreating OpenShell Docker sandbox container with ${patchTarget}...`); + if (!patchEnabled || result || managedStartupApplied) return; + console.log(` Applying ${patchTarget} to the OpenShell Docker sandbox...`); try { - result = recreateSelectedPatch(false, options.deps); - needsSupervisorWait = true; - console.log(` ✓ Docker container mode selected: ${result.mode.label}`); + applyPatch(options.deps); } catch (error) { - onPatchFailureExit(options.sandboxName, error, { - runCaptureOpenshell: options.deps.runCaptureOpenshell, - dockerCapture: options.deps.dockerCapture, - additionalSummaryLines: routeAdapter.additionalSummaryLines, - }); + managedStartupTransaction ??= getDockerManagedStartupFailureTransaction(error); + patchError = error; + reportPatchErrorAndExit(); } }, waitForSupervisorReconnectIfNeeded() { - if (!needsSupervisorWait) return; + if (!needsSupervisorWait || managedStartupFinalized) return; const supervisorReconnectTimeoutSecs = getDockerGpuSupervisorReconnectTimeoutSecs( options.timeoutSecs, ); @@ -221,14 +322,17 @@ export function createDockerGpuSandboxCreatePatch( supervisorReconnectTimeoutSecs, { runOpenshell: options.deps.runOpenshell, - // Pass `runCaptureOpenshell` so the supervisor-reconnect wait can - // short-circuit on a terminal sandbox phase instead of burning - // the full reconnect timeout window when the patched container - // crashed on startup (#4316). runCaptureOpenshell: options.deps.runCaptureOpenshell, sleep: options.deps.sleep, }, ); + if (supervisorReady) { + // Reconnect is necessary but not sufficient for cutover. Keep both the + // managed shared-state receipt and recreation backup until the caller + // accepts authoritative Ready and any required GPU proof. + needsSupervisorWait = false; + return; + } if (!supervisorReady && result) { try { captureFailedClone(options.sandboxName, result, options.deps); @@ -238,15 +342,140 @@ export function createDockerGpuSandboxCreatePatch( ); } } + let managedSharedStateOutcome = { + supervisorReady: false, + failure: null as Error | null, + }; + if (managedStartupTransaction) { + try { + managedSharedStateOutcome = finalizeManagedStartupSharedState( + { + transaction: managedStartupTransaction, + patchResult: result, + supervisorReady: false, + }, + options.deps, + ); + } catch (error) { + onPatchFailureExit(options.sandboxName, error, { + runCaptureOpenshell: options.deps.runCaptureOpenshell, + dockerCapture: options.deps.dockerCapture, + additionalSummaryLines: routeAdapter.additionalSummaryLines, + context: { + sandboxName: options.sandboxName, + oldContainerId: result?.oldContainerId ?? null, + newContainerId: result?.newContainerId ?? managedStartupTransaction.containerId, + backupContainerName: result?.backupContainerName ?? null, + selectedMode: result?.mode ?? null, + rolledBack: false, + }, + }); + return; + } + } const finalizeOutcome = result - ? finalizeBackup({ result, supervisorReady }, options.deps) + ? finalizeBackup( + { result, supervisorReady: managedSharedStateOutcome.supervisorReady }, + options.deps, + ) : null; - if (supervisorReady) { + managedStartupFinalized = true; + needsSupervisorWait = false; + const failureMessage = (() => { + if (managedSharedStateOutcome.failure) { + if (!result) { + return `${managedSharedStateOutcome.failure.message}; failed container removed after shared-state rollback.`; + } + return finalizeOutcome?.rolledBack + ? `${managedSharedStateOutcome.failure.message}; pre-patch sandbox restored.` + : `${managedSharedStateOutcome.failure.message}; container rollback failed and pre-patch sandbox was NOT restored.`; + } + if (!finalizeOutcome) { + return "OpenShell supervisor did not reconnect to the recreated container."; + } + return finalizeOutcome.rolledBack + ? "OpenShell supervisor did not reconnect to the recreated container; pre-patch sandbox restored." + : "OpenShell supervisor did not reconnect to the recreated container and rollback failed; pre-patch sandbox was NOT restored."; + })(); + onPatchFailureExit(options.sandboxName, new Error(failureMessage), { + runCaptureOpenshell: options.deps.runCaptureOpenshell, + dockerCapture: options.deps.dockerCapture, + additionalSummaryLines: routeAdapter.additionalSummaryLines, + context: { + sandboxName: options.sandboxName, + oldContainerId: result?.oldContainerId, + newContainerId: result?.newContainerId, + backupContainerName: result?.backupContainerName, + selectedMode: result?.mode ?? null, + rolledBack: finalizeOutcome?.rolledBack ?? false, + }, + }); + }, + + commitAfterReady() { + if (managedStartupFinalized || (!managedStartupTransaction && !result)) return; + if (needsSupervisorWait) { + const error = new Error( + "Managed startup cannot commit before the recreated OpenShell supervisor reconnects.", + ); + const rollbackError = rollbackAfterFailure(); + onPatchFailureExit( + options.sandboxName, + rollbackError + ? new Error(`${error.message} Rollback failed: ${rollbackError.message}`) + : error, + { + runCaptureOpenshell: options.deps.runCaptureOpenshell, + dockerCapture: options.deps.dockerCapture, + additionalSummaryLines: routeAdapter.additionalSummaryLines, + }, + ); + return; + } + let managedSharedStateOutcome = { + supervisorReady: true, + failure: null as Error | null, + }; + if (managedStartupTransaction) { + try { + managedSharedStateOutcome = finalizeManagedStartupSharedState( + { + transaction: managedStartupTransaction, + patchResult: result, + supervisorReady: true, + }, + options.deps, + ); + } catch (error) { + onPatchFailureExit(options.sandboxName, error, { + runCaptureOpenshell: options.deps.runCaptureOpenshell, + dockerCapture: options.deps.dockerCapture, + additionalSummaryLines: routeAdapter.additionalSummaryLines, + context: { + sandboxName: options.sandboxName, + oldContainerId: result?.oldContainerId ?? null, + newContainerId: result?.newContainerId ?? managedStartupTransaction.containerId, + backupContainerName: result?.backupContainerName ?? null, + selectedMode: result?.mode ?? null, + rolledBack: false, + }, + }); + return; + } + } + const finalizeOutcome = result + ? finalizeBackup( + { result, supervisorReady: managedSharedStateOutcome.supervisorReady }, + options.deps, + ) + : null; + managedStartupFinalized = true; + if (managedSharedStateOutcome.supervisorReady) { if (finalizeOutcome && !finalizeOutcome.backupRemoved) { onPatchFailureExit( options.sandboxName, new Error( - "OpenShell supervisor reconnected, but the recreated backup container could not be removed.", + "Managed startup passed Ready, but the recreated backup container could not be removed.", ), { runCaptureOpenshell: options.deps.runCaptureOpenshell, @@ -266,12 +495,12 @@ export function createDockerGpuSandboxCreatePatch( return; } const failureMessage = (() => { - if (!finalizeOutcome) { - return "OpenShell supervisor did not reconnect to the recreated container."; + if (!result) { + return `${managedSharedStateOutcome.failure?.message ?? "Managed shared-state commit failed"}; failed container removed after shared-state rollback.`; } - return finalizeOutcome.rolledBack - ? "OpenShell supervisor did not reconnect to the recreated container; pre-patch sandbox restored." - : "OpenShell supervisor did not reconnect to the recreated container and rollback failed; pre-patch sandbox was NOT restored."; + return finalizeOutcome?.rolledBack + ? `${managedSharedStateOutcome.failure?.message ?? "Managed shared-state commit failed"}; pre-patch sandbox restored.` + : `${managedSharedStateOutcome.failure?.message ?? "Managed shared-state commit failed"}; container rollback failed and pre-patch sandbox was NOT restored.`; })(); onPatchFailureExit(options.sandboxName, new Error(failureMessage), { runCaptureOpenshell: options.deps.runCaptureOpenshell, @@ -334,6 +563,14 @@ export function createDockerGpuSandboxCreatePatch( additionalSummaryLines: routeAdapter.additionalSummaryLines, }, ); + const rollbackError = rollbackAfterFailure(); + if (rollbackError) { + onPatchFailureExit(options.sandboxName, rollbackError, { + runCaptureOpenshell: options.deps.runCaptureOpenshell, + dockerCapture: options.deps.dockerCapture, + additionalSummaryLines: routeAdapter.additionalSummaryLines, + }); + } process.exit(1); } } diff --git a/src/lib/onboard/docker-gpu-supervisor-reconnect.ts b/src/lib/onboard/docker-gpu-supervisor-reconnect.ts index 9db684642c..2f4a342109 100644 --- a/src/lib/onboard/docker-gpu-supervisor-reconnect.ts +++ b/src/lib/onboard/docker-gpu-supervisor-reconnect.ts @@ -23,9 +23,13 @@ * recovers to Ready is the runtime evidence required. */ -import { hasZeroDockerExitStatus } from "./docker-command-result"; import { DOCKER_GPU_PATCH_TIMEOUT_MS } from "./docker-gpu-patch-constants"; import { envInt } from "./env"; +import { + resolveSupervisorReconnectTimeoutSecs, + type SupervisorReconnectDeps, + waitForSupervisorReconnect, +} from "./sandbox-create-runtime/supervisor-reconnect"; const DOCKER_GPU_SUPERVISOR_RECONNECT_MIN_SECS = 900; // Default consecutive Error-phase polls required before fast-fail. With a @@ -54,101 +58,33 @@ export const DOCKER_GPU_SUPERVISOR_RECONNECT_TIMEOUT_ENV = export const DOCKER_GPU_SUPERVISOR_RECONNECT_ERROR_DEBOUNCE_ENV = "NEMOCLAW_DOCKER_GPU_SUPERVISOR_RECONNECT_ERROR_DEBOUNCE"; -const TERMINAL_SANDBOX_FAILURE_PHASES = new Set(["Error", "Failed", "CrashLoopBackOff"]); - -type DockerRunResult = { - status?: number | null; - stdout?: string | Buffer | null; - stderr?: string | Buffer | null; -}; - -type RunOpenshellFn = (args: string[], opts?: Record) => DockerRunResult; -type RunCaptureOpenshellFn = (args: string[], opts?: Record) => string; - -export type DockerGpuSupervisorReconnectDeps = { - runOpenshell?: RunOpenshellFn; - runCaptureOpenshell?: RunCaptureOpenshellFn; - sleep?: (seconds: number) => void; - errorPhaseDebouncePolls?: number; -}; - -function defaultSleep(seconds: number): void { - Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, Math.max(0, seconds) * 1000); -} - -const ANSI_RE = /\x1b\[[0-9;]*m/g; - -function parseSandboxListFailurePhase(output: string, sandboxName: string): string | null { - if (typeof output !== "string" || !output.includes(sandboxName)) return null; - for (const line of output.replace(ANSI_RE, "").split(/\r?\n/)) { - const cols = line.trim().split(/\s+/); - if (cols[0] === sandboxName) { - return cols.find((col) => TERMINAL_SANDBOX_FAILURE_PHASES.has(col)) ?? null; - } - } - return null; -} - -function sandboxListShowsErrorPhase( - sandboxName: string, - runCaptureOpenshell: RunCaptureOpenshellFn, -): boolean { - try { - const list = runCaptureOpenshell(["sandbox", "list"], { - ignoreError: true, - suppressOutput: true, - timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, - }); - return parseSandboxListFailurePhase(list, sandboxName) !== null; - } catch { - return false; - } -} +export type DockerGpuSupervisorReconnectDeps = SupervisorReconnectDeps; export function waitForOpenShellSupervisorReconnect( sandboxName: string, timeoutSecs: number, deps: DockerGpuSupervisorReconnectDeps, ): boolean { - if (!deps.runOpenshell) return true; - const sleep = deps.sleep ?? defaultSleep; - const deadline = Date.now() + Math.max(1, timeoutSecs) * 1000; const errorPhaseDebouncePolls = deps.errorPhaseDebouncePolls == null || !Number.isFinite(deps.errorPhaseDebouncePolls) ? getDockerGpuSupervisorReconnectErrorDebouncePolls() - : // Round (not truncate) to match the env-var path's envInt rounding and - // the sibling create/readiness debounce in sandbox-readiness-tracing.ts. - Math.max(1, Math.round(deps.errorPhaseDebouncePolls)); - let consecutiveErrorPolls = 0; - while (Date.now() <= deadline) { - const result = deps.runOpenshell(["sandbox", "exec", "-n", sandboxName, "--", "true"], { - ignoreError: true, - suppressOutput: true, - timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, - }); - if (hasZeroDockerExitStatus(result)) return true; - if ( - deps.runCaptureOpenshell && - sandboxListShowsErrorPhase(sandboxName, deps.runCaptureOpenshell) - ) { - consecutiveErrorPolls += 1; - if (consecutiveErrorPolls >= errorPhaseDebouncePolls) return false; - } else { - consecutiveErrorPolls = 0; - } - sleep(2); - } - return false; + : deps.errorPhaseDebouncePolls; + return waitForSupervisorReconnect( + sandboxName, + timeoutSecs, + { ...deps, errorPhaseDebouncePolls }, + { commandTimeoutMs: DOCKER_GPU_PATCH_TIMEOUT_MS }, + ); } export function getDockerGpuSupervisorReconnectTimeoutSecs( sandboxReadyTimeoutSecs: number, env: Record = process.env, ): number { - const readyTimeoutSecs = Number.isFinite(sandboxReadyTimeoutSecs) - ? Math.max(1, Math.round(sandboxReadyTimeoutSecs)) - : 1; - const fallback = Math.max(readyTimeoutSecs, DOCKER_GPU_SUPERVISOR_RECONNECT_MIN_SECS); + const fallback = resolveSupervisorReconnectTimeoutSecs( + sandboxReadyTimeoutSecs, + DOCKER_GPU_SUPERVISOR_RECONNECT_MIN_SECS, + ); return Math.max(1, envInt(DOCKER_GPU_SUPERVISOR_RECONNECT_TIMEOUT_ENV, fallback, env)); } diff --git a/src/lib/onboard/docker-startup-command-agent.test.ts b/src/lib/onboard/docker-startup-command-agent.test.ts new file mode 100644 index 0000000000..4de5e95ef6 --- /dev/null +++ b/src/lib/onboard/docker-startup-command-agent.test.ts @@ -0,0 +1,57 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; +import { resolveDockerStartupCommandPatch } from "./docker-startup-command-agent"; + +describe("Docker managed startup adapter selection", () => { + it("does not recreate OpenClaw solely to apply a managed profile", () => { + expect( + resolveDockerStartupCommandPatch( + { name: "openclaw" } as Parameters[0], + true, + ), + ).toEqual({ persistStartupCommand: false, requiredUlimits: null }); + }); + + it("does not select Docker lifecycle behavior for a native non-Docker driver", () => { + expect( + resolveDockerStartupCommandPatch( + { name: "openclaw" } as Parameters[0], + false, + ), + ).toEqual({ + persistStartupCommand: false, + requiredUlimits: null, + }); + }); + + it("leaves the legacy OpenClaw Dockerfile path unchanged", () => { + expect( + resolveDockerStartupCommandPatch( + { name: "openclaw" } as Parameters[0], + true, + ), + ).toEqual({ + persistStartupCommand: false, + requiredUlimits: null, + }); + }); + + it("retains resource-only restart recreation for Hermes and DCode", () => { + expect( + resolveDockerStartupCommandPatch( + { name: "hermes" } as Parameters[0], + true, + ).persistStartupCommand, + ).toBe(true); + const dcode = resolveDockerStartupCommandPatch( + { + name: "langchain-deepagents-code", + } as Parameters[0], + true, + ); + expect(dcode.persistStartupCommand).toBe(true); + expect(dcode.requiredUlimits).toHaveLength(2); + }); +}); diff --git a/src/lib/onboard/docker-startup-command-sandbox-create.test.ts b/src/lib/onboard/docker-startup-command-sandbox-create.test.ts index 17cd02f643..66bd8e9b8d 100644 --- a/src/lib/onboard/docker-startup-command-sandbox-create.test.ts +++ b/src/lib/onboard/docker-startup-command-sandbox-create.test.ts @@ -156,6 +156,190 @@ describe("Docker startup-command sandbox creation", () => { expect(context.rolledBack).toBe(true); }); + it("applies managed startup directly to the exact container without recreation", () => { + const deps = makeDeps(); + const containerId = "b".repeat(64); + const request = { + schemaVersion: 1, + agent: "openclaw", + encodedProfile: "profile", + profileFingerprint: "a".repeat(64), + corporateCaB64: null, + } as const; + const transaction = { + agent: "openclaw", + containerId, + image: `sha256:${"c".repeat(64)}`, + } as const; + const applyManagedStartupRootRequest = vi.fn(() => transaction); + const finalizeManagedStartupSharedState = vi.fn(() => ({ + supervisorReady: true, + failure: null, + })); + const recreateStartupPatch = vi.fn(); + const patch = createDockerGpuSandboxCreatePatch({ + route: "native", + persistStartupCommand: false, + managedStartupRootApplyRequest: request, + sandboxName: "alpha", + openshellSandboxCommand: ["env", "/usr/local/bin/nemoclaw-managed-startup-hold"], + timeoutSecs: 60, + deps, + overrides: { + findContainerIds: vi.fn(() => [containerId]), + recreateStartupPatch, + applyManagedStartupRootRequest, + finalizeManagedStartupSharedState, + }, + }); + + patch.maybeApplyDuringCreate(); + patch.waitForSupervisorReconnectIfNeeded(); + + expect(recreateStartupPatch).not.toHaveBeenCalled(); + expect(applyManagedStartupRootRequest).toHaveBeenCalledWith( + { containerId, request }, + { dockerCapture: deps.dockerCapture }, + ); + expect(finalizeManagedStartupSharedState).not.toHaveBeenCalled(); + + patch.commitAfterReady(); + expect(finalizeManagedStartupSharedState).toHaveBeenCalledWith( + { transaction, patchResult: null, supervisorReady: true }, + deps, + ); + }); + + it("treats an already-finalized same-profile replay as an applied no-op", () => { + const deps = makeDeps(); + const containerId = "c".repeat(64); + const request = { + schemaVersion: 1, + agent: "openclaw", + encodedProfile: "profile", + profileFingerprint: "a".repeat(64), + corporateCaB64: null, + } as const; + const applyManagedStartupRootRequest = vi.fn(() => null); + const finalizeManagedStartupSharedState = vi.fn(); + const patch = createDockerGpuSandboxCreatePatch({ + route: "native", + managedStartupRootApplyRequest: request, + sandboxName: "alpha", + timeoutSecs: 60, + deps, + overrides: { + findContainerIds: vi.fn(() => [containerId]), + applyManagedStartupRootRequest, + finalizeManagedStartupSharedState, + }, + }); + + patch.maybeApplyDuringCreate(); + patch.maybeApplyDuringCreate(); + patch.ensureApplied(); + patch.waitForSupervisorReconnectIfNeeded(); + patch.commitAfterReady(); + + expect(applyManagedStartupRootRequest).toHaveBeenCalledOnce(); + expect(finalizeManagedStartupSharedState).not.toHaveBeenCalled(); + }); + + it("finishes resource recreation before applying managed startup to the replacement", () => { + const deps = makeDeps(); + const result = { ...startupResult(), newContainerId: "d".repeat(64) }; + const request = { + schemaVersion: 1, + agent: "langchain-deepagents-code", + encodedProfile: "profile", + profileFingerprint: "a".repeat(64), + corporateCaB64: null, + } as const; + const transaction = { + agent: request.agent, + containerId: result.newContainerId, + image: `sha256:${"c".repeat(64)}`, + } as const; + const calls: string[] = []; + const recreateStartupPatch = vi.fn(() => { + calls.push("recreate"); + return result; + }); + const applyManagedStartupRootRequest = vi.fn(() => { + calls.push("apply"); + return transaction; + }); + const patch = createDockerGpuSandboxCreatePatch({ + route: "native", + persistStartupCommand: true, + managedStartupRootApplyRequest: request, + sandboxName: "alpha", + openshellSandboxCommand: ["env", "/usr/local/bin/nemoclaw-managed-startup-hold"], + timeoutSecs: 60, + deps, + overrides: { + findContainerIds: vi.fn(() => ["existing-container"]), + recreateStartupPatch, + applyManagedStartupRootRequest, + }, + }); + + patch.maybeApplyDuringCreate(); + + expect(calls).toEqual(["recreate", "apply"]); + expect(applyManagedStartupRootRequest).toHaveBeenCalledWith( + { containerId: result.newContainerId, request }, + { dockerCapture: deps.dockerCapture }, + ); + }); + + it("rolls back direct managed shared state when create fails after root application", () => { + const deps = makeDeps(); + const containerId = "e".repeat(64); + const request = { + schemaVersion: 1, + agent: "hermes", + encodedProfile: "profile", + profileFingerprint: "a".repeat(64), + corporateCaB64: null, + } as const; + const transaction = { + agent: request.agent, + containerId, + image: `sha256:${"f".repeat(64)}`, + } as const; + const finalizeManagedStartupSharedState = vi.fn(() => ({ + supervisorReady: false, + failure: null, + })); + const finalizeBackup = vi.fn(); + const patch = createDockerGpuSandboxCreatePatch({ + route: "native", + managedStartupRootApplyRequest: request, + sandboxName: "alpha", + timeoutSecs: 60, + deps, + overrides: { + findContainerIds: vi.fn(() => [containerId]), + applyManagedStartupRootRequest: vi.fn(() => transaction), + finalizeManagedStartupSharedState, + finalizeBackup, + }, + }); + + patch.maybeApplyDuringCreate(); + patch.waitForSupervisorReconnectIfNeeded(); + expect(finalizeManagedStartupSharedState).not.toHaveBeenCalled(); + + patch.rollbackManagedStartupAfterCreateFailure(); + + expect(finalizeManagedStartupSharedState).toHaveBeenCalledWith( + { transaction, patchResult: null, supervisorReady: false }, + deps, + ); + expect(finalizeBackup).not.toHaveBeenCalled(); + }); + it("reports startup-command creation failures through the composed patch boundary", () => { const deps = makeDeps(); const onPatchFailureExit = vi.fn(); diff --git a/src/lib/onboard/dockerfile-remote-dashboard-bind-contract.ts b/src/lib/onboard/dockerfile-remote-dashboard-bind-contract.ts index f63bef91cb..7f5ed5e0d1 100644 --- a/src/lib/onboard/dockerfile-remote-dashboard-bind-contract.ts +++ b/src/lib/onboard/dockerfile-remote-dashboard-bind-contract.ts @@ -13,7 +13,7 @@ const REMOTE_BIND_ARG_RE = /^ARG\s+NEMOCLAW_DASHBOARD_BIND=/; const REMOTE_BIND_PATCHED_ARG_RE = /^ARG\s+NEMOCLAW_DASHBOARD_BIND=0\.0\.0\.0$/; const REMOTE_BIND_PROMOTION_RE = /NEMOCLAW_DASHBOARD_BIND=\$\{NEMOCLAW_DASHBOARD_BIND\}/; const OPENCLAW_CONFIG_GENERATOR_RE = - /^RUN\s+(?:NEMOCLAW_OPENCLAW_MANAGED_PROXY=0\s+)?node\s+--experimental-strip-types\s+\/scripts\/generate-openclaw-config\.mts$/; + /^RUN\s+(?:NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION=0\s+)?(?:NEMOCLAW_OPENCLAW_MANAGED_PROXY=0\s+)?node\s+--experimental-strip-types\s+\/scripts\/generate-openclaw-config\.mts$/; const SAFE_VALIDATION_GENERATOR_RE = /^RUN\s+validation_home="\$validation_root\/progressive";\s+HOME=(?:"\$validation_home"|\$validation_home)\s+node\s+--experimental-strip-types\s+\/scripts\/generate-openclaw-config\.mts$/; const PASSIVE_FINAL_STAGE_INSTRUCTION_RE = /^(?:ARG|ENV|WORKDIR|USER|HEALTHCHECK|ENTRYPOINT|CMD)\b/; @@ -57,6 +57,10 @@ const CANONICAL_POST_GENERATOR_RUN_SHA256 = new Set([ "ca1f7b1cb9dd5d467f806792c4072a84ef1e6402c3e8650b6325b95cc186ccdf", "7e6a6879382f833f17be02ca7d287685b6afa1c423b1e087b3b05dd677d6e325", "22406cef76f7a66a3d527c17f2a5bc6a217c71c753c979406fb2c3fa7cd8f0eb", + "9300de0b56a7d8a1498fd36cb9c05313b6691d6756b455f91628ed989221afc2", + "6f457f365f5c0d128e5e3b549a630b5bd9ebd223919f2c2c8e6a31235d763781", + "dca7d3dbc030e4efa77c850b9d21a826358c69c7d2062f3eee2f5a57eeb07aa2", + "b01b5f5d2cba5778cd8eb87139f2c6a8174082a7f6775e443a1dbdc0629ce7e5", ]); function instructionSha256(text: string): string { diff --git a/src/lib/onboard/fatal-runtime-preflight.test.ts b/src/lib/onboard/fatal-runtime-preflight.test.ts index 93ca2b5352..387634512f 100644 --- a/src/lib/onboard/fatal-runtime-preflight.test.ts +++ b/src/lib/onboard/fatal-runtime-preflight.test.ts @@ -3,7 +3,13 @@ import { describe, expect, it, vi } from "vitest"; import { isLinuxDockerDriverGatewayEnabled } from "./docker-driver-platform"; -import { rejectUnsupportedContainerRuntime } from "./fatal-runtime-preflight"; +import { + assertSelectedContainerRuntimeReady, + type FatalRuntimePreflightDriverAdapter, + rejectUnsupportedContainerRuntime, + resolveFatalRuntimePreflightDriverBehavior, + runFatalOnboardRuntimePreflight, +} from "./fatal-runtime-preflight"; import type { HostAssessment } from "./preflight"; function hostWithRuntime(runtime: HostAssessment["runtime"]): HostAssessment { @@ -51,4 +57,161 @@ describe("rejectUnsupportedContainerRuntime (#7320)", () => { rejectUnsupportedContainerRuntime(hostWithRuntime("docker"), exit as never); expect(exit).not.toHaveBeenCalled(); }); + + it("preserves the registered Kubernetes readiness behavior", () => { + const exit = vi.fn(); + const receipt = assertSelectedContainerRuntimeReady( + hostWithRuntime("docker"), + { driverName: "kubernetes", gatewayLauncher: "openshell" }, + { exitProcess: exit as never }, + ); + + expect(receipt).toBeNull(); + expect(exit).not.toHaveBeenCalled(); + }); + + it("qualifies a selected native Podman driver without Docker reachability", () => { + const host = { ...hostWithRuntime("unknown"), dockerReachable: false }; + const env: NodeJS.ProcessEnv = {}; + const receipt = assertSelectedContainerRuntimeReady( + host, + { driverName: "podman", gatewayLauncher: "nemoclaw" }, + { + env, + nativePodmanDeps: { + platform: "linux", + architecture: "x64", + env: { OPENSHELL_PODMAN_SOCKET: "/run/user/1000/podman/podman.sock" }, + uid: 1000, + lstatSync: ((filePath: string) => { + const socket = filePath.endsWith("/podman.sock"); + const currentUserPath = filePath.startsWith("/run/user/1000"); + return { + dev: 8, + ino: socket ? 9001 : filePath.length, + mode: socket ? 0o600 : currentUserPath ? 0o700 : 0o755, + uid: socket || currentUserPath ? 1000 : 0, + isDirectory: () => !socket, + isSocket: () => socket, + }; + }) as never, + run: (_command, args) => { + if (args[0] === "--version") { + return { status: 0, stdout: "podman version 5.6.2", stderr: "" }; + } + if (args[0] === "unshare") { + return { status: 0, stdout: "0 1000 1\n1 100000 65536\n", stderr: "" }; + } + return { + status: 0, + stdout: JSON.stringify({ + host: { + arch: "amd64", + os: "linux", + cgroupVersion: "v2", + networkBackend: "netavark", + security: { rootless: true }, + }, + }), + stderr: "", + }; + }, + }, + }, + ); + + expect(receipt?.driverName).toBe("podman"); + expect(env.OPENSHELL_PODMAN_SOCKET).toBe("/run/user/1000/podman/podman.sock"); + }); + + it("dispatches an injected MXC fatal preflight without inherited runtime probes", () => { + const host = { + ...hostWithRuntime("unknown"), + dockerInstalled: false, + dockerRunning: false, + dockerReachable: false, + }; + const result = { + gpu: null, + host, + sandboxGpuConfig: { + mode: "0" as const, + hostGpuDetected: false, + hostGpuPlatform: null, + sandboxGpuEnabled: false, + sandboxGpuDevice: null, + errors: [], + }, + nativePodman: null, + }; + const mxcAdapter = { + driverName: "mxc", + behavior: { + checkContainerRuntimeResources: false, + checkDockerBridgeDns: false, + defaultSandboxGpuFlag: null, + sandboxGpuUnsupportedMessage: null, + skipDockerProbe: true, + }, + assertReady: vi.fn(() => null), + run: vi.fn(() => result), + } satisfies FatalRuntimePreflightDriverAdapter; + + expect( + assertSelectedContainerRuntimeReady( + host, + { driverName: "mxc", gatewayLauncher: "nemoclaw" }, + { driverPreflightAdapters: { mxc: mxcAdapter } }, + ), + ).toBeNull(); + + expect( + runFatalOnboardRuntimePreflight( + {}, + { + nonInteractive: true, + computePlan: { driverName: "mxc", gatewayLauncher: "nemoclaw" }, + driverPreflightAdapters: { mxc: mxcAdapter }, + nativePodmanDeps: { + lstatSync: () => { + throw new Error("Podman probe must not run"); + }, + run: () => { + throw new Error("Podman probe must not run"); + }, + }, + }, + ), + ).toBe(result); + expect(mxcAdapter.assertReady).toHaveBeenCalledOnce(); + expect(mxcAdapter.run).toHaveBeenCalledOnce(); + expect( + resolveFatalRuntimePreflightDriverBehavior( + { driverName: "mxc", gatewayLauncher: "nemoclaw" }, + { mxc: mxcAdapter }, + ), + ).toBe(mxcAdapter.behavior); + }); + + it("rejects a registry entry whose adapter identity does not match its driver key", () => { + const dockerAdapter = { + driverName: "docker", + behavior: { + checkContainerRuntimeResources: true, + checkDockerBridgeDns: true, + defaultSandboxGpuFlag: null, + sandboxGpuUnsupportedMessage: null, + skipDockerProbe: false, + }, + assertReady: vi.fn(() => null), + run: vi.fn(), + } satisfies FatalRuntimePreflightDriverAdapter; + + expect(() => + resolveFatalRuntimePreflightDriverBehavior( + { driverName: "mxc", gatewayLauncher: "nemoclaw" }, + { mxc: dockerAdapter }, + ), + ).toThrow("has no registered fatal runtime preflight adapter"); + }); }); diff --git a/src/lib/onboard/fatal-runtime-preflight.ts b/src/lib/onboard/fatal-runtime-preflight.ts index 72cda1cb19..3f20161176 100644 --- a/src/lib/onboard/fatal-runtime-preflight.ts +++ b/src/lib/onboard/fatal-runtime-preflight.ts @@ -3,7 +3,17 @@ import { detectGpu, type GpuDetection } from "../inference/nim"; import { assertDockerBridgeAndContainerDnsHealthy } from "./bridge-dns-preflight"; -import { isLinuxDockerDriverGatewayEnabled } from "./docker-driver-platform"; +import { + type OpenShellComputePlan, + resolveCurrentOpenShellComputePlan, + usesManagedDockerGateway, +} from "./compute/plan"; +import { + assessNativePodman, + type NativePodmanPreflightDeps, + NativePodmanPreflightError, + type NativePodmanPreflightReceipt, +} from "./compute/podman-preflight"; import { warnIfHostProxyMissesLoopback } from "./http-proxy-preflight"; import { assertCdiNvidiaGpuSpecPresent, @@ -30,58 +40,262 @@ export type FatalRuntimePreflightOptions = Pick< export interface FatalRuntimePreflightContext { nonInteractive: boolean; exitProcess?: (code: number) => never; + computePlan?: OpenShellComputePlan; + env?: NodeJS.ProcessEnv; + nativePodmanDeps?: NativePodmanPreflightDeps; + driverPreflightAdapters?: FatalRuntimePreflightDriverAdapterRegistry; } export interface FatalRuntimePreflightResult { gpu: GpuDetection | null; host: HostAssessment; sandboxGpuConfig: SandboxGpuConfig; + nativePodman: NativePodmanPreflightReceipt | null; } const exitProcessByDefault = (code: number): never => process.exit(code); +const PODMAN_SANDBOX_GPU_UNSUPPORTED_MESSAGE = + "Native Podman support currently covers CPU sandboxes with hosted inference; GPU passthrough is not yet supported."; + +export interface FatalRuntimePreflightDriverAdapterInput { + readonly options: FatalRuntimePreflightOptions; + readonly nonInteractive: boolean; + readonly exitProcess: (code: number) => never; + readonly computePlan: OpenShellComputePlan; + readonly env: NodeJS.ProcessEnv; + readonly nativePodmanDeps?: NativePodmanPreflightDeps; +} + +export interface FatalRuntimePreflightDriverReadinessInput { + readonly host: HostAssessment; + readonly computePlan: OpenShellComputePlan; + readonly exitProcess: (code: number) => never; + readonly env: NodeJS.ProcessEnv; + readonly nativePodmanDeps?: NativePodmanPreflightDeps; +} + +/** + * Runtime-specific fatal checks are registered by driver identity. A new + * driver must supply both entry points so it cannot silently inherit Docker + * or Podman host probes. + */ +export interface FatalRuntimePreflightDriverAdapter { + readonly driverName: string; + readonly behavior: FatalRuntimePreflightDriverBehavior; + assertReady( + input: FatalRuntimePreflightDriverReadinessInput, + ): NativePodmanPreflightReceipt | null; + run(input: FatalRuntimePreflightDriverAdapterInput): FatalRuntimePreflightResult; +} + +export interface FatalRuntimePreflightDriverBehavior { + readonly checkContainerRuntimeResources: boolean; + readonly checkDockerBridgeDns: boolean; + readonly defaultSandboxGpuFlag: "disable" | null; + readonly sandboxGpuUnsupportedMessage: string | null; + readonly skipDockerProbe: boolean; +} + +export type FatalRuntimePreflightDriverAdapterRegistry = Readonly< + Record +>; /** Reject runtimes that cannot support the OpenShell Docker-driver integration. */ export function rejectUnsupportedContainerRuntime( host: HostAssessment, exitProcess: (code: number) => never = exitProcessByDefault, + computePlan: OpenShellComputePlan = resolveCurrentOpenShellComputePlan(), ): void { - if (isLinuxDockerDriverGatewayEnabled() && host.runtime === "podman") { + if (usesManagedDockerGateway(computePlan) && host.runtime === "podman") { printUnsupportedRuntimeError(); exitProcess(1); } } -/** Run the non-mutating runtime gates shared by fresh, resume, and rebuild onboarding. */ -export function runFatalOnboardRuntimePreflight( - options: FatalRuntimePreflightOptions, - context: FatalRuntimePreflightContext, +function failNativePodman(error: unknown, exitProcess: (code: number) => never): never { + const message = + error instanceof NativePodmanPreflightError + ? error.message + : `Native Podman preflight failed: ${error instanceof Error ? error.message : String(error)}`; + console.error(` ${message}`); + exitProcess(1); +} + +function assertDockerFamilyRuntimeReady(input: FatalRuntimePreflightDriverReadinessInput): null { + rejectUnsupportedContainerRuntime(input.host, input.exitProcess, input.computePlan); + return null; +} + +function assertNativePodmanRuntimeReady( + input: FatalRuntimePreflightDriverReadinessInput, +): NativePodmanPreflightReceipt { + let receipt: NativePodmanPreflightReceipt; + try { + receipt = assessNativePodman(input.nativePodmanDeps); + } catch (error) { + return failNativePodman(error, input.exitProcess); + } + input.env.OPENSHELL_PODMAN_SOCKET = receipt.socketPath; + return receipt; +} + +function runDockerFamilyFatalRuntimePreflight( + input: FatalRuntimePreflightDriverAdapterInput, ): FatalRuntimePreflightResult { - const exitProcess = context.exitProcess ?? exitProcessByDefault; const host = assessHost(); + assertDockerFamilyRuntimeReady({ ...input, host }); if (!host.dockerReachable) { printDockerNotReachableError(); printRemediationActions(planHostRemediation(host)); - exitProcess(1); + input.exitProcess(1); } - rejectUnsupportedContainerRuntime(host, exitProcess); console.log(" ✓ Docker is running"); warnIfHostProxyMissesLoopback(); const gpu = detectGpu(); const sandboxGpuConfig = resolveSandboxGpuConfig(gpu, { - flag: resolveSandboxGpuFlagFromOptions(options), - device: options.sandboxGpuDevice ?? null, + flag: resolveSandboxGpuFlagFromOptions(input.options), + device: input.options.sandboxGpuDevice ?? null, }); const explicitlyOptedOutGpuPassthrough = - options.optedOutGpuPassthrough === true || options.noGpu === true; + input.options.optedOutGpuPassthrough === true || input.options.noGpu === true; assertCdiNvidiaGpuSpecPresent( host, explicitlyOptedOutGpuPassthrough, sandboxGpuConfig.hostGpuPlatform, - exitProcess, + input.exitProcess, ); - assertDockerBridgeAndContainerDnsHealthy(host, context.nonInteractive, exitProcess); - validateSandboxGpuPreflight(sandboxGpuConfig, {}, exitProcess); + assertDockerBridgeAndContainerDnsHealthy(host, input.nonInteractive, input.exitProcess); + validateSandboxGpuPreflight(sandboxGpuConfig, {}, input.exitProcess); if (host.runtime !== "unknown") console.log(` ✓ Container runtime: ${host.runtime}`); if (host.notes.includes("Running under WSL")) console.log(" ⓘ Running under WSL"); - return { gpu, host, sandboxGpuConfig }; + return { gpu, host, sandboxGpuConfig, nativePodman: null }; +} + +function runNativePodmanFatalRuntimePreflight( + input: FatalRuntimePreflightDriverAdapterInput, +): FatalRuntimePreflightResult { + const assessedHost = assessHost({ skipDockerProbe: true }); + const nativePodman = assertNativePodmanRuntimeReady({ ...input, host: assessedHost }); + const host = { ...assessedHost, runtime: "podman" as const, isUnsupportedRuntime: false }; + warnIfHostProxyMissesLoopback(); + const gpu = detectGpu(); + if (input.options.gpu === true || input.options.sandboxGpu === "enable") { + console.error(` ${PODMAN_SANDBOX_GPU_UNSUPPORTED_MESSAGE}`); + input.exitProcess(1); + } + const sandboxGpuConfig = resolveSandboxGpuConfig(gpu, { + flag: "disable", + device: null, + }); + validateSandboxGpuPreflight(sandboxGpuConfig, {}, input.exitProcess); + console.log( + ` ✓ Rootless Podman ${nativePodman.version} is reachable at ${nativePodman.socketPath}`, + ); + console.log(` ✓ Container runtime: podman (${nativePodman.networkBackend})`); + return { gpu, host, sandboxGpuConfig, nativePodman }; +} + +const dockerFatalRuntimePreflightAdapter: FatalRuntimePreflightDriverAdapter = { + driverName: "docker", + behavior: { + checkContainerRuntimeResources: true, + checkDockerBridgeDns: true, + defaultSandboxGpuFlag: null, + sandboxGpuUnsupportedMessage: null, + skipDockerProbe: false, + }, + assertReady: assertDockerFamilyRuntimeReady, + run: runDockerFamilyFatalRuntimePreflight, +}; + +const kubernetesFatalRuntimePreflightAdapter: FatalRuntimePreflightDriverAdapter = { + driverName: "kubernetes", + behavior: dockerFatalRuntimePreflightAdapter.behavior, + assertReady: assertDockerFamilyRuntimeReady, + run: runDockerFamilyFatalRuntimePreflight, +}; + +export const CURRENT_FATAL_RUNTIME_PREFLIGHT_DRIVER_ADAPTERS = { + docker: dockerFatalRuntimePreflightAdapter, + kubernetes: kubernetesFatalRuntimePreflightAdapter, + podman: { + driverName: "podman", + behavior: { + checkContainerRuntimeResources: false, + checkDockerBridgeDns: false, + defaultSandboxGpuFlag: "disable", + sandboxGpuUnsupportedMessage: PODMAN_SANDBOX_GPU_UNSUPPORTED_MESSAGE, + skipDockerProbe: true, + }, + assertReady: assertNativePodmanRuntimeReady, + run: runNativePodmanFatalRuntimePreflight, + }, +} as const satisfies FatalRuntimePreflightDriverAdapterRegistry; + +function resolveFatalRuntimePreflightDriverAdapter( + computePlan: OpenShellComputePlan, + adapters: FatalRuntimePreflightDriverAdapterRegistry, +): FatalRuntimePreflightDriverAdapter { + const adapter = Object.hasOwn(adapters, computePlan.driverName) + ? adapters[computePlan.driverName] + : undefined; + if (!adapter || adapter.driverName !== computePlan.driverName) { + throw new Error( + `OpenShell compute driver '${computePlan.driverName}' has no registered fatal runtime preflight adapter.`, + ); + } + return adapter; +} + +export function resolveFatalRuntimePreflightDriverBehavior( + computePlan: OpenShellComputePlan, + adapters: FatalRuntimePreflightDriverAdapterRegistry = CURRENT_FATAL_RUNTIME_PREFLIGHT_DRIVER_ADAPTERS, +): FatalRuntimePreflightDriverBehavior { + return resolveFatalRuntimePreflightDriverAdapter(computePlan, adapters).behavior; +} + +export function assertSelectedContainerRuntimeReady( + host: HostAssessment, + computePlan: OpenShellComputePlan, + options: { + exitProcess?: (code: number) => never; + env?: NodeJS.ProcessEnv; + nativePodmanDeps?: NativePodmanPreflightDeps; + driverPreflightAdapters?: FatalRuntimePreflightDriverAdapterRegistry; + } = {}, +): NativePodmanPreflightReceipt | null { + const exitProcess = options.exitProcess ?? exitProcessByDefault; + const env = options.env ?? process.env; + const adapter = resolveFatalRuntimePreflightDriverAdapter( + computePlan, + options.driverPreflightAdapters ?? CURRENT_FATAL_RUNTIME_PREFLIGHT_DRIVER_ADAPTERS, + ); + return adapter.assertReady({ + host, + computePlan, + exitProcess, + env, + nativePodmanDeps: options.nativePodmanDeps, + }); +} + +/** Run the non-mutating runtime gates shared by fresh, resume, and rebuild onboarding. */ +export function runFatalOnboardRuntimePreflight( + options: FatalRuntimePreflightOptions, + context: FatalRuntimePreflightContext, +): FatalRuntimePreflightResult { + const exitProcess = context.exitProcess ?? exitProcessByDefault; + const computePlan = context.computePlan ?? resolveCurrentOpenShellComputePlan(); + const adapter = resolveFatalRuntimePreflightDriverAdapter( + computePlan, + context.driverPreflightAdapters ?? CURRENT_FATAL_RUNTIME_PREFLIGHT_DRIVER_ADAPTERS, + ); + return adapter.run({ + options, + nonInteractive: context.nonInteractive, + exitProcess, + computePlan, + env: context.env ?? process.env, + nativePodmanDeps: context.nativePodmanDeps, + }); } diff --git a/src/lib/onboard/gateway-binding.test.ts b/src/lib/onboard/gateway-binding.test.ts index ad7094d037..80585225aa 100644 --- a/src/lib/onboard/gateway-binding.test.ts +++ b/src/lib/onboard/gateway-binding.test.ts @@ -21,9 +21,11 @@ import { createDynamicGatewayRuntimeHelpers, resolveCoreOnboardGatewayBinding, resolveGatewayCompatContainerName, + resolveGatewayComputeDriverBinding, resolveGatewayName, resolveGatewayPortFromName, resolveGatewayStateDirName, + resolveManagedGatewayStateDirectory, resolveSandboxGatewayName, } from "./gateway-binding"; @@ -177,6 +179,21 @@ describe("gateway-binding resolver (#4422)", () => { expect(resolveGatewayStateDirName(a)).not.toBe(resolveGatewayStateDirName(b)); expect(resolveGatewayCompatContainerName(a)).not.toBe(resolveGatewayCompatContainerName(b)); }); + + it("resolves the durable managed-runtime directory for a gateway", () => { + expect( + resolveManagedGatewayStateDirectory("nemoclaw-8081", { + env: {}, + home: "/home/tester", + }), + ).toBe("/home/tester/.local/state/nemoclaw/openshell-docker-gateway-8081"); + expect( + resolveManagedGatewayStateDirectory("nemoclaw", { + env: { NEMOCLAW_OPENSHELL_GATEWAY_STATE_DIR: "/runtime/gateway" }, + home: "/home/tester", + }), + ).toBe("/runtime/gateway"); + }); }); describe("resolveSandboxGatewayName", () => { @@ -276,6 +293,74 @@ describe("resolveSandboxGatewayName", () => { }); }); +describe("resolveGatewayComputeDriverBinding", () => { + it("returns the one driver shared by live sandboxes on the target gateway", () => { + expect( + resolveGatewayComputeDriverBinding({ + gatewayName: "nemoclaw-9090", + legacyDriverName: "docker", + sandboxes: [ + { + name: "podman-a", + gatewayPort: 9090, + openshellDriver: "podman", + }, + { + name: "podman-b", + gatewayName: "nemoclaw-9090", + openshellDriver: " podman ", + }, + { + name: "other-gateway", + gatewayPort: 9191, + openshellDriver: "docker", + }, + { + name: "route-only", + pendingRouteReservation: true, + openshellDriver: "docker", + }, + ], + }), + ).toBe("podman"); + }); + + it("treats a legacy live row as its platform-derived driver", () => { + expect( + resolveGatewayComputeDriverBinding({ + gatewayName: "nemoclaw", + legacyDriverName: "docker", + sandboxes: [{ name: "legacy" }], + }), + ).toBe("docker"); + }); + + it("returns null when no live sandbox is bound to the gateway", () => { + expect( + resolveGatewayComputeDriverBinding({ + gatewayName: "nemoclaw-9090", + legacyDriverName: "docker", + sandboxes: [{ name: "other", gatewayPort: 9191, openshellDriver: "docker" }], + }), + ).toBeNull(); + }); + + it("fails closed when one shared gateway contains mixed drivers", () => { + expect(() => + resolveGatewayComputeDriverBinding({ + gatewayName: "nemoclaw", + legacyDriverName: "docker", + sandboxes: [ + { name: "docker-agent", openshellDriver: "docker" }, + { name: "podman-agent", openshellDriver: "podman" }, + ], + }), + ).toThrow( + "Conflicting OpenShell compute drivers are bound to gateway 'nemoclaw': docker-agent='docker', podman-agent='podman'.", + ); + }); +}); + describe("resolveCoreOnboardGatewayBinding", () => { const currentGateway = { name: "nemoclaw", port: DEFAULT_GATEWAY_PORT }; diff --git a/src/lib/onboard/gateway-binding.ts b/src/lib/onboard/gateway-binding.ts index de3a43ace7..a4fc892b3d 100644 --- a/src/lib/onboard/gateway-binding.ts +++ b/src/lib/onboard/gateway-binding.ts @@ -19,6 +19,9 @@ * ports never collide. */ +import os from "node:os"; +import path from "node:path"; + import { DEFAULT_GATEWAY_PORT } from "../core/ports"; import type { GatewayReuseState } from "../state/gateway"; @@ -68,6 +71,12 @@ export interface SandboxGatewayBinding { gatewayPort?: number | null; } +export interface SandboxGatewayComputeBinding extends SandboxGatewayBinding { + name?: string; + openshellDriver?: string | null; + pendingRouteReservation?: true; +} + /** * Recognises a NemoClaw-namespaced gateway name. The persisted form is either * the bare `nemoclaw` or the per-port `nemoclaw-` derivation — anything @@ -141,6 +150,40 @@ export function resolveSandboxGatewayName( throw new Error(`Invalid persisted sandbox gateway binding (${detail.join(", ")})`); } +/** + * Resolve the durable compute driver shared by every live sandbox attached to + * one gateway. A NemoClaw-managed gateway is a shared runtime: selecting a + * different driver for a second sandbox would replace the runtime underneath + * the first sandbox. Legacy rows inherit the platform driver they used before + * explicit driver identity was persisted. + */ +export function resolveGatewayComputeDriverBinding(options: { + gatewayName: string; + sandboxes: readonly SandboxGatewayComputeBinding[]; + legacyDriverName: string; +}): string | null { + const evidence = options.sandboxes.flatMap((sandbox) => { + if (sandbox.pendingRouteReservation === true) return []; + if (resolveSandboxGatewayName(sandbox) !== options.gatewayName) return []; + const driverName = sandbox.openshellDriver?.trim() || options.legacyDriverName.trim(); + if (!driverName) { + throw new Error( + `Sandbox '${sandbox.name || ""}' has no durable OpenShell compute driver.`, + ); + } + return [{ sandboxName: sandbox.name || "", driverName }]; + }); + const drivers = new Set(evidence.map(({ driverName }) => driverName)); + if (drivers.size > 1) { + throw new Error( + `Conflicting OpenShell compute drivers are bound to gateway '${options.gatewayName}': ${evidence + .map(({ sandboxName, driverName }) => `${sandboxName}='${driverName}'`) + .join(", ")}.`, + ); + } + return evidence[0]?.driverName ?? null; +} + /** Resolve one attempt-wide onboarding target without overriding an authoritative rebuild. */ export function resolveCoreOnboardGatewayBinding(options: { authoritativeGateway?: { name: string; port: number } | null; @@ -168,6 +211,26 @@ export function resolveGatewayStateDirName(port: number): string { : `${BASE_GATEWAY_STATE_DIR_NAME}-${port}`; } +export function resolveManagedGatewayStateDirectory( + gatewayName: string, + options: { env?: NodeJS.ProcessEnv; home?: string } = {}, +): string { + const gatewayPort = resolveGatewayPortFromName(gatewayName); + if (gatewayPort === null) { + throw new Error(`Invalid NemoClaw gateway name '${gatewayName}'`); + } + const env = options.env ?? process.env; + const configured = env.NEMOCLAW_OPENSHELL_GATEWAY_STATE_DIR?.trim(); + if (configured) return path.resolve(configured); + return path.join( + options.home ?? env.HOME ?? os.homedir(), + ".local", + "state", + "nemoclaw", + resolveGatewayStateDirName(gatewayPort), + ); +} + /** * Resolve the Docker-driver gateway compatibility container name for a gateway * port. A per-port container name prevents the second onboard's diff --git a/src/lib/onboard/gateway-destroy.test.ts b/src/lib/onboard/gateway-destroy.test.ts index 7cf06a5726..8606cfdd00 100644 --- a/src/lib/onboard/gateway-destroy.test.ts +++ b/src/lib/onboard/gateway-destroy.test.ts @@ -3,7 +3,7 @@ import { describe, expect, it, vi } from "vitest"; -import { destroyGatewayWithVolumeCleanup, type DestroyGatewayDeps } from "./gateway-destroy"; +import { type DestroyGatewayDeps, destroyGatewayWithVolumeCleanup } from "./gateway-destroy"; function deps(overrides: Partial = {}): DestroyGatewayDeps { return { @@ -82,6 +82,20 @@ describe("destroyGatewayWithVolumeCleanup", () => { }); }); + it("stops a non-Docker managed gateway without invoking Docker volume cleanup", () => { + const d = deps({ + hasLifecycleCommands: vi.fn(() => false), + isManagedDriverGatewayEnabled: vi.fn(() => true), + shouldCleanupLegacyDockerVolumes: vi.fn(() => false), + }); + + expect(destroyGatewayWithVolumeCleanup(d)).toBe(true); + + expect(d.stopDockerDriverGatewayProcess).toHaveBeenCalledOnce(); + expect(d.removeDockerDriverGatewayRegistration).toHaveBeenCalledOnce(); + expect(d.dockerRemoveVolumesByPrefix).not.toHaveBeenCalled(); + }); + it("does not clear registry or remove volumes when gateway removal fails", () => { const d = deps({ runOpenshell: vi.fn(() => ({ status: 1 })) }); diff --git a/src/lib/onboard/gateway-destroy.ts b/src/lib/onboard/gateway-destroy.ts index 3dc447913f..bce28d7fd0 100644 --- a/src/lib/onboard/gateway-destroy.ts +++ b/src/lib/onboard/gateway-destroy.ts @@ -14,9 +14,11 @@ export type DestroyGatewayDeps = { gatewayName: string; hasLifecycleCommands: () => boolean; isDockerDriverGatewayEnabled: () => boolean; + isManagedDriverGatewayEnabled?: () => boolean; removeDockerDriverGatewayRegistration: () => boolean; runOpenshell: RunOpenshell; stopDockerDriverGatewayProcess: () => void; + shouldCleanupLegacyDockerVolumes?: () => boolean; }; export function destroyGatewayWithVolumeCleanup({ @@ -25,17 +27,21 @@ export function destroyGatewayWithVolumeCleanup({ gatewayName, hasLifecycleCommands, isDockerDriverGatewayEnabled, + isManagedDriverGatewayEnabled, removeDockerDriverGatewayRegistration, runOpenshell, stopDockerDriverGatewayProcess, + shouldCleanupLegacyDockerVolumes, }: DestroyGatewayDeps): boolean { const dockerDriver = isDockerDriverGatewayEnabled(); - if (dockerDriver) { + const managedDriver = isManagedDriverGatewayEnabled?.() ?? dockerDriver; + const cleanupLegacyDockerVolumes = shouldCleanupLegacyDockerVolumes?.() ?? dockerDriver; + if (managedDriver) { stopDockerDriverGatewayProcess(); } const lifecycleCommands = hasLifecycleCommands(); - const gatewayRemoved = dockerDriver + const gatewayRemoved = managedDriver ? removeDockerDriverGatewayRegistration() : (() => { const removeResult = runOpenshell(["gateway", "remove", gatewayName], { @@ -54,7 +60,7 @@ export function destroyGatewayWithVolumeCleanup({ clearRegistry(); } - if (gatewayRemoved && (dockerDriver || lifecycleCommands)) { + if (gatewayRemoved && (cleanupLegacyDockerVolumes || (!managedDriver && lifecycleCommands))) { dockerRemoveVolumesByPrefix(`openshell-cluster-${gatewayName}`, { ignoreError: true }); } diff --git a/src/lib/onboard/gateway-recovery.test.ts b/src/lib/onboard/gateway-recovery.test.ts index a24111674e..660f6378d4 100644 --- a/src/lib/onboard/gateway-recovery.test.ts +++ b/src/lib/onboard/gateway-recovery.test.ts @@ -65,6 +65,8 @@ describe("gateway recovery", () => { expect(deps.startGatewayWithOptions).toHaveBeenCalledWith(undefined, { exitOnFailure: false, + gatewayName: "nemoclaw", + gatewayPort: 8080, }); expect(deps.runOpenshell).not.toHaveBeenCalled(); }); @@ -292,15 +294,47 @@ describe("gateway recovery", () => { expect(deps.runOpenshell).not.toHaveBeenCalled(); }); - it("fails closed on cross-port recovery when the Linux Docker-driver gateway is enabled", async () => { + it("routes cross-port Linux Docker-driver recovery through the exact managed target", async () => { const deps = createDeps({ isLinuxDockerDriverGatewayEnabled: () => true }); - await expect(startGatewayForRecovery({ gatewayName: "nemoclaw-8090" }, deps)).rejects.toThrow( - /Cross-port recovery for Linux Docker-driver gateway 'nemoclaw-8090' is not safe/, - ); + await startGatewayForRecovery({ gatewayName: "nemoclaw-8090" }, deps); expect(deps.runOpenshell).not.toHaveBeenCalled(); - expect(deps.startGatewayWithOptions).not.toHaveBeenCalled(); + expect(deps.startGatewayWithOptions).toHaveBeenCalledWith(undefined, { + exitOnFailure: false, + gatewayName: "nemoclaw-8090", + gatewayPort: 8090, + }); + }); + + it("routes a non-Docker managed gateway through the shared recovery lifecycle", async () => { + const deps = createDeps({ + isLinuxDockerDriverGatewayEnabled: () => false, + isManagedDriverGatewayEnabled: () => true, + }); + + await startGatewayForRecovery({}, deps); + + expect(deps.startGatewayWithOptions).toHaveBeenCalledOnce(); + expect(deps.runOpenshell).not.toHaveBeenCalledWith( + expect.arrayContaining(["gateway", "start"]), + expect.anything(), + ); + }); + + it("routes cross-port recovery for every managed driver through the exact target", async () => { + const deps = createDeps({ + isLinuxDockerDriverGatewayEnabled: () => false, + isManagedDriverGatewayEnabled: () => true, + }); + + await startGatewayForRecovery({ gatewayName: "nemoclaw-8090" }, deps); + + expect(deps.startGatewayWithOptions).toHaveBeenCalledWith(undefined, { + exitOnFailure: false, + gatewayName: "nemoclaw-8090", + gatewayPort: 8090, + }); }); }); @@ -313,8 +347,8 @@ describe("gateway lifecycle authority during recovery", () => { }), }); - // The cross-port, non-default-name target is the branch that reaches a raw - // `openshell gateway start` without going through startGatewayWithOptions. + // The cross-port, non-default-name target must still pass the ownership + // guard before the exact managed target reaches startGatewayWithOptions. await expect( startGatewayForRecovery({ gatewayName: "nemoclaw-8090", gatewayPort: 8090 }, deps), ).rejects.toThrow(ownershipError); diff --git a/src/lib/onboard/gateway-recovery.ts b/src/lib/onboard/gateway-recovery.ts index b542e94b00..77d03d1ba3 100644 --- a/src/lib/onboard/gateway-recovery.ts +++ b/src/lib/onboard/gateway-recovery.ts @@ -36,6 +36,7 @@ import { } from "./readiness-wait"; export type StartGatewayForRecoveryOptions = { + computeDriver?: string; gatewayName?: string; gatewayPort?: number; }; @@ -67,8 +68,16 @@ export type GatewayRecoveryDeps = { getGatewayStartEnv(): Record; runCaptureOpenshell(args: string[], opts?: RunCaptureOpenshellOptions): string; runOpenshell(args: string[], opts?: RunOpenshellOptions): GatewayStartResult; - startGatewayWithOptions(gpu: never, options: { exitOnFailure: false }): Promise; + startGatewayWithOptions( + gpu: never, + options: { + exitOnFailure: false; + gatewayName: string; + gatewayPort: number; + }, + ): Promise; isLinuxDockerDriverGatewayEnabled?(): boolean; + isManagedDriverGatewayEnabled?(): boolean; sleepSeconds?(seconds: number): void; // Injected so caller-level tests can exercise the success + retry-success // paths at unit-test speed without standing up a real gateway. Defaults @@ -257,32 +266,24 @@ export async function startGatewayForRecovery( // startGatewayWithOptions. Resolve and bind the requested target first: the // process-global gateway can name a different port during sandbox recovery. deps.assertGatewayStartAllowed(false, target); - const linuxDockerDriverEnabled = ( - deps.isLinuxDockerDriverGatewayEnabled ?? isLinuxDockerDriverGatewayEnabled + const managedDriverEnabled = ( + deps.isManagedDriverGatewayEnabled ?? + deps.isLinuxDockerDriverGatewayEnabled ?? + isLinuxDockerDriverGatewayEnabled )(); - // The Docker-driver Linux startup path (startGatewayWithOptions → - // startDockerDriverGateway) restores the runtime-marker, package-managed + // The managed-driver startup path (startGatewayWithOptions → + // the selected runtime adapter) restores the runtime-marker, package-managed // registration, and sandbox-bridge reachability — none of which a plain // `openshell gateway start` produces. Route through it whenever the - // recovery target matches the current process's GATEWAY_PORT (the common - // case where the user re-runs with the same NEMOCLAW_GATEWAY_PORT). - if (target.gatewayPort === GATEWAY_PORT) { - if (target.gatewayName === resolveDefaultGatewayName() || linuxDockerDriverEnabled) { - return deps.startGatewayWithOptions(undefined as never, { exitOnFailure: false }); - } - } - // Cross-port recovery on a Linux Docker-driver gateway cannot share this - // process's module-globals: startDockerDriverGateway captures the port at - // load time, so a plain `openshell gateway start` would skip the - // runtime-marker / package registration / sandbox-bridge setup and leave - // the host in a half-recovered state. Fail closed instead and direct the - // operator to re-run with the matching NEMOCLAW_GATEWAY_PORT so the - // docker-driver path re-stamps the per-port artefacts. - if (linuxDockerDriverEnabled && target.gatewayPort !== GATEWAY_PORT) { - throw new Error( - `Cross-port recovery for Linux Docker-driver gateway '${target.gatewayName}' is not safe from a process bound to port ${GATEWAY_PORT}. ` + - `Re-run with NEMOCLAW_GATEWAY_PORT=${target.gatewayPort} so the docker-driver setup can restamp the runtime marker, registration, and sandbox bridge.`, - ); + // recovery target is the default gateway or any registered managed-driver + // gateway. The caller binds its process-local coordinator state to this + // exact target for the duration of startup, including non-default ports. + if (target.gatewayName === resolveDefaultGatewayName() || managedDriverEnabled) { + return deps.startGatewayWithOptions(undefined as never, { + exitOnFailure: false, + gatewayName: target.gatewayName, + gatewayPort: target.gatewayPort, + }); } return startTargetGatewayForRecovery(target, deps); } diff --git a/src/lib/onboard/host-proxy-env.ts b/src/lib/onboard/host-proxy-env.ts index 99bca5190e..664ee7f869 100644 --- a/src/lib/onboard/host-proxy-env.ts +++ b/src/lib/onboard/host-proxy-env.ts @@ -12,12 +12,30 @@ const HOST_PROXY_ENV_NAMES = [ "https_proxy", "no_proxy", ] as const; +const HOST_PROXY_URL_ENV_NAMES = [ + "HTTP_PROXY", + "HTTPS_PROXY", + "http_proxy", + "https_proxy", +] as const; +const MAX_HOST_PROXY_URL_BYTES = 4096; +const MAX_HOST_NO_PROXY_BYTES = 8192; +const MAX_HOST_PROXY_ENV_BYTES = 24 * 1024; type HostProxyEnvOptions = { dropCredentialBearingProxyUrls?: boolean; }; -function isCredentialBearingProxyUrl(value: string): boolean { +type HostProxyEnvironment = Record; + +export class HostProxyEnvironmentError extends Error { + constructor(message: string) { + super(`Invalid host proxy environment: ${message}`); + this.name = "HostProxyEnvironmentError"; + } +} + +export function isCredentialBearingProxyUrl(value: string): boolean { try { const parsed = new URL(value.includes("://") ? value : `http://${value}`); return parsed.username !== "" || parsed.password !== ""; @@ -26,12 +44,18 @@ function isCredentialBearingProxyUrl(value: string): boolean { } } -export function appendHostProxyEnvArgs( - envArgs: string[], +function maximumBytes(name: (typeof HOST_PROXY_ENV_NAMES)[number]): number { + return name === "NO_PROXY" || name === "no_proxy" + ? MAX_HOST_NO_PROXY_BYTES + : MAX_HOST_PROXY_URL_BYTES; +} + +export function resolveHostProxyEnvironment( env: NodeJS.ProcessEnv = process.env, options: HostProxyEnvOptions = {}, -): void { - const proxyEnv: Record = {}; +): HostProxyEnvironment { + const proxyEnv: HostProxyEnvironment = {}; + let totalBytes = 0; for (const name of HOST_PROXY_ENV_NAMES) { const value = env[name]; if (typeof value === "string") { @@ -43,11 +67,75 @@ export function appendHostProxyEnvArgs( trimmed !== "" && !(options.dropCredentialBearingProxyUrls && isCredentialBearingProxyUrl(trimmed)) ) { + const valueBytes = Buffer.byteLength(trimmed, "utf8"); + if ( + valueBytes > maximumBytes(name) || + /[\u0000-\u001f\u007f]/u.test(trimmed) || + totalBytes + valueBytes > MAX_HOST_PROXY_ENV_BYTES + ) { + throw new HostProxyEnvironmentError(`${name} is malformed or exceeds its size limit`); + } proxyEnv[name] = trimmed; + totalBytes += valueBytes; } } } + const hasProxy = HOST_PROXY_URL_ENV_NAMES.some((name) => proxyEnv[name] !== undefined); + if (!hasProxy) return {}; + + withLocalNoProxy(proxyEnv); + for (const name of ["NO_PROXY", "no_proxy"] as const) { + const value = proxyEnv[name]; + if (value && Buffer.byteLength(value, "utf8") > MAX_HOST_NO_PROXY_BYTES) { + throw new HostProxyEnvironmentError(`${name} exceeds its size limit after normalization`); + } + } + const normalizedBytes = Object.values(proxyEnv).reduce( + (total, value) => total + Buffer.byteLength(value, "utf8"), + 0, + ); + if (normalizedBytes > MAX_HOST_PROXY_ENV_BYTES) { + throw new HostProxyEnvironmentError("the normalized proxy environment exceeds its size limit"); + } + return proxyEnv; +} + +export function hasCredentialBearingHostProxyEnvironment( + env: NodeJS.ProcessEnv = process.env, +): boolean { + const proxyEnv = resolveHostProxyEnvironment(env); + return HOST_PROXY_URL_ENV_NAMES.some((name) => { + const value = proxyEnv[name]; + return value !== undefined && isCredentialBearingProxyUrl(value); + }); +} + +export function credentialHostProxyReplayEnvArgs(env: NodeJS.ProcessEnv = process.env): string[] { + const proxyEnv = resolveHostProxyEnvironment(env); + if ( + !HOST_PROXY_URL_ENV_NAMES.some((name) => { + const value = proxyEnv[name]; + return value !== undefined && isCredentialBearingProxyUrl(value); + }) + ) { + throw new HostProxyEnvironmentError( + "the source requires a credential-bearing proxy, but this command has none to replay", + ); + } + return HOST_PROXY_ENV_NAMES.flatMap((name) => { + const value = proxyEnv[name]; + return value === undefined ? [] : [formatEnvAssignment(name, value)]; + }); +} + +export function appendHostProxyEnvArgs( + envArgs: string[], + env: NodeJS.ProcessEnv = process.env, + options: HostProxyEnvOptions = {}, +): void { + const proxyEnv = resolveHostProxyEnvironment(env, options); + // #2598: NEMOCLAW_MINIMAL_BOOTSTRAP is a host-side opt-in flag (set to // "1") that the sandbox's nemoclaw-start.sh:seed_default_workspace_templates // reads to skip default workspace template seeding for new/pristine @@ -63,11 +151,9 @@ export function appendHostProxyEnvArgs( envArgs.push(formatEnvAssignment("NEMOCLAW_MINIMAL_BOOTSTRAP", "1")); } - const hasProxy = - proxyEnv.HTTP_PROXY || proxyEnv.HTTPS_PROXY || proxyEnv.http_proxy || proxyEnv.https_proxy; + const hasProxy = HOST_PROXY_URL_ENV_NAMES.some((name) => proxyEnv[name] !== undefined); if (!hasProxy) return; - withLocalNoProxy(proxyEnv); for (const name of HOST_PROXY_ENV_NAMES) { const value = proxyEnv[name]; if (value) envArgs.push(formatEnvAssignment(name, value)); diff --git a/src/lib/onboard/host-service-reachability.test.ts b/src/lib/onboard/host-service-reachability.test.ts index cc76797cc3..659d123c17 100644 --- a/src/lib/onboard/host-service-reachability.test.ts +++ b/src/lib/onboard/host-service-reachability.test.ts @@ -6,7 +6,7 @@ // See: https://github.com/NVIDIA/NemoClaw/issues/3340 (Ollama auth proxy) and // https://github.com/NVIDIA/NemoClaw/issues/4564 (Model Router port 4000). -import { describe, expect, it, vi } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; // Mock the docker adapter so the test never loads runner.ts (which requires // the compiled ./platform artifact unavailable in the test environment). @@ -16,6 +16,11 @@ vi.mock("../adapters/docker/run", () => ({ })); import { + configureHostContainerEngine, + resetHostContainerEngineForTests, +} from "../adapters/container-engine"; +import { + __test, DEFAULT_PROBE_NETWORK, formatHostServiceUnreachableMessage, probeHostServiceSandboxReachability, @@ -29,6 +34,10 @@ function makeNetwork(partial: { subnet?: string; gatewayIp?: string } = {}): { } describe("probeHostServiceSandboxReachability", () => { + afterEach(() => { + resetHostContainerEngineForTests(); + }); + it("returns ok and echoes the probed port when nc connects", async () => { const result = await probeHostServiceSandboxReachability({ port: 4000, @@ -85,6 +94,53 @@ describe("probeHostServiceSandboxReachability", () => { expect(capturedArgs).toContain("host.openshell.internal"); expect(capturedArgs).toContain("4000"); }); + + it("keeps the Ollama route on Podman's exact network and canonical sandbox alias", async () => { + const restore = configureHostContainerEngine({ + driverName: "podman", + executable: "podman", + prefixArgs: ["--url", "unix:///run/user/1000/podman/podman.sock"], + sandboxNetworkName: "openshell", + hostGatewayTarget: "host-gateway", + }); + let inspectedNetwork = ""; + let capturedArgs: readonly string[] = []; + try { + const result = await probeHostServiceSandboxReachability({ + port: 11435, + inspectNetworkImpl: (networkName) => { + inspectedNetwork = networkName; + return makeNetwork(); + }, + runImpl: (args) => { + capturedArgs = args; + return { status: 0 }; + }, + }); + expect(result.ok).toBe(true); + } finally { + restore(); + } + + expect(inspectedNetwork).toBe("openshell"); + expect(capturedArgs).toContain("openshell"); + expect(capturedArgs).toContain("host.openshell.internal:host-gateway"); + expect(capturedArgs).toContain("host.openshell.internal"); + expect(capturedArgs).not.toContain("openshell-docker"); + }); + + it("parses Podman network inspect subnets without Docker's IPAM template", () => { + expect( + __test.parseNetworkIpamConfig( + JSON.stringify([ + { + name: "openshell", + subnets: [{ subnet: "10.89.0.0/24", gateway: "10.89.0.1" }], + }, + ]), + ), + ).toEqual({ subnet: "10.89.0.0/24", gatewayIp: "10.89.0.1" }); + }); }); describe("formatHostServiceUnreachableMessage", () => { diff --git a/src/lib/onboard/host-service-reachability.ts b/src/lib/onboard/host-service-reachability.ts index 648a9a3363..31192d51be 100644 --- a/src/lib/onboard/host-service-reachability.ts +++ b/src/lib/onboard/host-service-reachability.ts @@ -17,7 +17,13 @@ * onboard successful. */ +import { + currentHostContainerEngineCommand, + hostContainerEngineArgv, + hostContainerEngineDisplayName, +} from "../adapters/container-engine"; import { dockerCapture, dockerRun } from "../adapters/docker/run"; +import { shellQuote } from "../core/shell-quote"; export const DEFAULT_PROBE_NETWORK = "openshell-docker"; const HOST_INTERNAL_NAME = "host.openshell.internal"; @@ -70,11 +76,22 @@ function parseNetworkIpamConfig(raw: string): { subnet?: string; gatewayIp?: str return undefined; } if (!Array.isArray(parsed)) return undefined; - for (const entry of parsed) { + const candidates = parsed.flatMap((entry) => { + if (!entry || typeof entry !== "object") return []; + const record = entry as Record; + return Array.isArray(record.subnets) ? record.subnets : [entry]; + }); + for (const entry of candidates) { if (!entry || typeof entry !== "object") continue; const r = entry as Record; - const subnet = typeof r.Subnet === "string" ? r.Subnet : undefined; - const gatewayIp = typeof r.Gateway === "string" ? r.Gateway : undefined; + const subnet = + typeof r.Subnet === "string" ? r.Subnet : typeof r.subnet === "string" ? r.subnet : undefined; + const gatewayIp = + typeof r.Gateway === "string" + ? r.Gateway + : typeof r.gateway === "string" + ? r.gateway + : undefined; // Skip IPv6-only entries (contain colons) if (gatewayIp && !gatewayIp.includes(":")) return { subnet, gatewayIp }; } @@ -84,10 +101,12 @@ function parseNetworkIpamConfig(raw: string): { subnet?: string; gatewayIp?: str function defaultInspectNetwork( networkName: string, ): { subnet?: string; gatewayIp?: string } | undefined { - const raw = dockerCapture( - ["network", "inspect", "--format", "{{json .IPAM.Config}}", networkName], - { ignoreError: true }, - ); + const command = currentHostContainerEngineCommand(); + const args = + command.driverName === "podman" + ? ["network", "inspect", networkName] + : ["network", "inspect", "--format", "{{json .IPAM.Config}}", networkName]; + const raw = dockerCapture(args, { ignoreError: true }); return parseNetworkIpamConfig(raw); } @@ -95,6 +114,7 @@ function defaultInspectNetwork( // than a specific bridge IP. UFW is not relevant on those platforms, so we // classify probes from those environments as probe_unavailable. function defaultUsesHostGatewayRoute(): boolean { + if (currentHostContainerEngineCommand().hostGatewayTarget) return true; if (process.platform !== "linux") return true; const info = dockerCapture( ["info", "--format", "{{.OperatingSystem}}\n{{range .Labels}}{{.}}\n{{end}}"], @@ -130,11 +150,21 @@ function isNameResolutionFailure(detail: string): boolean { ); } +function shellCommand(argv: readonly string[]): string { + return argv + .map((value) => (/^[A-Za-z0-9_./:@=+-]+$/u.test(value) ? value : shellQuote(value))) + .join(" "); +} + export async function probeHostServiceSandboxReachability( opts: HostServiceReachabilityOptions, ): Promise { + const engine = currentHostContainerEngineCommand(); const networkName = - opts.networkName ?? process.env.OPENSHELL_DOCKER_NETWORK_NAME ?? DEFAULT_PROBE_NETWORK; + opts.networkName ?? + (engine.driverName === "podman" + ? (process.env.OPENSHELL_PODMAN_NETWORK_NAME ?? engine.sandboxNetworkName ?? "openshell") + : (process.env.OPENSHELL_DOCKER_NETWORK_NAME ?? DEFAULT_PROBE_NETWORK)); const port = opts.port; const timeoutSec = opts.timeoutSec ?? PROBE_TIMEOUT_SEC; const probeImage = opts.probeImage ?? PROBE_IMAGE; @@ -149,7 +179,7 @@ export async function probeHostServiceSandboxReachability( reason: "probe_unavailable", port, networkName, - detail: `Docker network "${networkName}" not found`, + detail: `${hostContainerEngineDisplayName()} network "${networkName}" not found`, }; } @@ -166,7 +196,9 @@ export async function probeHostServiceSandboxReachability( }; } - const hostInternalTarget = isHostGateway ? "host-gateway" : (network.gatewayIp as string); + const hostInternalTarget = isHostGateway + ? (engine.hostGatewayTarget ?? "host-gateway") + : (network.gatewayIp as string); const probeArgs = [ "run", @@ -237,19 +269,34 @@ export function formatHostServiceUnreachableMessage( if (result.ok || result.reason !== "tcp_failed") return ""; const port = options.port ?? result.port; + const engine = currentHostContainerEngineCommand(); + const engineName = hostContainerEngineDisplayName(); + const inspectFormat = + engine.driverName === "podman" + ? "{{(index .Subnets 0).Subnet}}" + : "{{(index .IPAM.Config 0).Subnet}}"; + const inspectCommand = shellCommand( + hostContainerEngineArgv([ + "network", + "inspect", + result.networkName ?? DEFAULT_PROBE_NETWORK, + "--format", + inspectFormat, + ]), + ); const allowCmd = result.subnet && result.gatewayIp ? ` sudo ufw allow from ${result.subnet} to ${result.gatewayIp} port ${port} proto tcp` : result.subnet ? ` sudo ufw allow from ${result.subnet} to any port ${port} proto tcp` : [ - ` SUBNET=$(docker network inspect ${result.networkName ?? DEFAULT_PROBE_NETWORK} --format '{{(index .IPAM.Config 0).Subnet}}')`, + ` SUBNET=$(${inspectCommand})`, ` sudo ufw allow from "$SUBNET" to any port ${port} proto tcp`, ].join("\n"); return [ ` ✗ Sandbox containers cannot reach the ${options.serviceLabel} at ${HOST_INTERNAL_NAME}:${port}.`, - " A host firewall may be blocking traffic from the OpenShell Docker bridge.", + ` A host firewall may be blocking traffic from the OpenShell ${engineName} network.`, " To allow it:", allowCmd, " Then re-run `nemoclaw onboard`.", diff --git a/src/lib/onboard/inference-route.ts b/src/lib/onboard/inference-route.ts index 810f2464c5..347ee7121a 100644 --- a/src/lib/onboard/inference-route.ts +++ b/src/lib/onboard/inference-route.ts @@ -1,7 +1,11 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { parseGatewayInference } from "../inference/config"; +import { + getSandboxInferenceConfig, + parseGatewayInference, + resolveAgentInferenceApi, +} from "../inference/config"; import { type CurrentGatewayRouteCompatibilityCheck, type CurrentGatewayRouteDiscoveryPreflight, @@ -12,6 +16,20 @@ import { listSandboxes } from "../state/registry"; type RunCaptureOpenshell = (args: string[], options?: { ignoreError?: boolean }) => string | null; +/** Resolve the exact portable inference route shared by managed rebuild and clone paths. */ +export function resolveManagedStartupInferenceRoute( + agentName: string, + provider: string, + model: string, + preferredInferenceApi: string | null, +) { + const api = + agentName === "langchain-deepagents-code" + ? "openai-completions" + : resolveAgentInferenceApi(agentName, provider, preferredInferenceApi); + return getSandboxInferenceConfig(model, provider, api); +} + export function createInferenceRouteHelpers( runCaptureOpenshell: RunCaptureOpenshell, listSandboxesFn: typeof listSandboxes = listSandboxes, diff --git a/src/lib/onboard/initial-policy-real-policy.test.ts b/src/lib/onboard/initial-policy-real-policy.test.ts index 1e45923013..418d0e28c0 100644 --- a/src/lib/onboard/initial-policy-real-policy.test.ts +++ b/src/lib/onboard/initial-policy-real-policy.test.ts @@ -30,7 +30,7 @@ type PolicyEntry = { }; type PolicyDocument = { - filesystem_policy?: { read_write?: string[] }; + filesystem_policy?: { read_only?: string[]; read_write?: string[] }; network_policies?: Record; }; @@ -140,6 +140,37 @@ describe("initial sandbox policy real preset merge", () => { } }); + it("keeps the root-owned managed-startup handoff read-only in every shipping policy", () => { + const policyCases = [ + { path: ["nemoclaw-blueprint", "policies", "openclaw-sandbox.yaml"], agent: "openclaw" }, + { + path: ["nemoclaw-blueprint", "policies", "openclaw-sandbox-permissive.yaml"], + agent: "openclaw", + }, + { path: ["agents", "openclaw", "policy-permissive.yaml"], agent: "openclaw" }, + { path: ["agents", "hermes", "policy-additions.yaml"], agent: "hermes" }, + { path: ["agents", "hermes", "policy-permissive.yaml"], agent: "hermes" }, + { + path: ["agents", "langchain-deepagents-code", "policy-additions.yaml"], + agent: "langchain-deepagents-code", + }, + ]; + + for (const policyCase of policyCases) { + const effective = readPreparedPolicy( + prepareInitialSandboxCreatePolicy(repoPath(...policyCase.path), [], { + agentName: policyCase.agent, + }), + ); + expect(effective.filesystem_policy?.read_only, policyCase.path.join("/")).toContain( + "/run/nemoclaw", + ); + expect(effective.filesystem_policy?.read_write, policyCase.path.join("/")).not.toContain( + "/run/nemoclaw", + ); + } + }); + it("preserves baseline writable paths in effective OpenClaw permissive create policies", () => { const baseline = readPreparedPolicy( prepareInitialSandboxCreatePolicy( diff --git a/src/lib/onboard/machine/core-flow-phases.ts b/src/lib/onboard/machine/core-flow-phases.ts index 962e1871db..21f52a3673 100644 --- a/src/lib/onboard/machine/core-flow-phases.ts +++ b/src/lib/onboard/machine/core-flow-phases.ts @@ -162,6 +162,7 @@ export function createProviderInferenceOnboardFlowPhase< webSearchConfig: context.webSearchConfig, }, selectedMessagingChannels: context.selectedMessagingChannels, + customDockerfileRequested: context.fromDockerfile !== null, env: options.env, constants: options.constants, deps: options.deps, diff --git a/src/lib/onboard/machine/handlers/provider-inference-build-estimate.test.ts b/src/lib/onboard/machine/handlers/provider-inference-build-estimate.test.ts new file mode 100644 index 0000000000..d1bbd296df --- /dev/null +++ b/src/lib/onboard/machine/handlers/provider-inference-build-estimate.test.ts @@ -0,0 +1,31 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import { createSession } from "../../../state/onboard-session"; +import { handleProviderInferenceState } from "./provider-inference"; +import { baseOptions, createDeps } from "./provider-inference.test-support"; + +describe("handleProviderInferenceState build estimate", () => { + it("adds a build estimate for an explicit custom Dockerfile", async () => { + const formatSandboxBuildEstimateNote = vi.fn(() => "custom build estimate"); + const formatOnboardConfigSummary = vi.fn(() => "custom summary"); + const { deps, calls } = createDeps({ + formatSandboxBuildEstimateNote, + formatOnboardConfigSummary, + }); + const session = createSession(); + calls.complete.mockResolvedValue(session); + + await handleProviderInferenceState({ + ...baseOptions(deps, session), + customDockerfileRequested: true, + }); + + expect(formatSandboxBuildEstimateNote).toHaveBeenCalledWith({ cpus: 8 }, "custom-dockerfile"); + expect(formatOnboardConfigSummary).toHaveBeenCalledWith( + expect.objectContaining({ notes: ["custom build estimate"] }), + ); + }); +}); diff --git a/src/lib/onboard/machine/handlers/provider-inference.test-support.ts b/src/lib/onboard/machine/handlers/provider-inference.test-support.ts index e49f44914f..9df6dc39c1 100644 --- a/src/lib/onboard/machine/handlers/provider-inference.test-support.ts +++ b/src/lib/onboard/machine/handlers/provider-inference.test-support.ts @@ -195,7 +195,7 @@ export function createDeps( registryUpdateSandbox: calls.updateSandbox, promptValidatedSandboxName: calls.promptName, assessHost: () => ({ cpus: 8 }), - formatSandboxBuildEstimateNote: () => "estimate", + formatSandboxBuildEstimateNote: vi.fn(() => "estimate"), formatOnboardConfigSummary: (options: { provider: string; model: string; diff --git a/src/lib/onboard/machine/handlers/provider-inference.test.ts b/src/lib/onboard/machine/handlers/provider-inference.test.ts index ee99ffa16a..545cb3b12f 100644 --- a/src/lib/onboard/machine/handlers/provider-inference.test.ts +++ b/src/lib/onboard/machine/handlers/provider-inference.test.ts @@ -62,6 +62,7 @@ describe("handleProviderInferenceState", () => { expect(selectionUpdates).not.toHaveProperty("endpointSource"); expect(selectionUpdates).not.toHaveProperty("onboardEndpointUrl"); expect(calls.promptName).toHaveBeenCalledWith(null); + expect(deps.formatSandboxBuildEstimateNote).not.toHaveBeenCalled(); expect(calls.log).toHaveBeenCalledWith("summary:nvidia-prod/nvidia/test/my-assistant"); expect(calls.startStep).toHaveBeenNthCalledWith(2, "inference", { provider: "nvidia-prod", diff --git a/src/lib/onboard/machine/handlers/provider-inference.ts b/src/lib/onboard/machine/handlers/provider-inference.ts index ac0ccf6b02..41667627e6 100644 --- a/src/lib/onboard/machine/handlers/provider-inference.ts +++ b/src/lib/onboard/machine/handlers/provider-inference.ts @@ -28,6 +28,7 @@ import type { createProviderRecoveryReceiptLedger, ProviderRecoveryReceipt, } from "../../rebuild-route-handoff"; +import type { SandboxLocalBuildKind } from "../../summary"; import { withInferenceTrace, withProviderSelectionTrace } from "../../tracing"; import { advanceTo, type OnboardStateTransitionResult, retryTo } from "../result"; import { createRecovery, type RecoveryAuthority } from "./provider-inference-recovery"; @@ -123,6 +124,8 @@ export interface ProviderInferenceStateOptions { webSearchConfig: WebSearchConfig | null; }; selectedMessagingChannels: string[]; + /** True only for an explicit custom Dockerfile known before workload resolution. */ + customDockerfileRequested?: boolean; env: NodeJS.ProcessEnv; constants: { hermesProviderName: string; @@ -234,7 +237,7 @@ export interface ProviderInferenceStateOptions { registryUpdateSandbox(sandboxName: string, updates: { nimContainer?: string | null }): void; promptValidatedSandboxName(agent: Agent): Promise; assessHost(): Host; - formatSandboxBuildEstimateNote(host: Host): string | null; + formatSandboxBuildEstimateNote(host: Host, buildKind: SandboxLocalBuildKind): string | null; formatOnboardConfigSummary(options: { provider: string; model: string; @@ -373,6 +376,7 @@ export async function handleProviderInferenceState({ providerRecoveryReceiptLedger, initial, selectedMessagingChannels, + customDockerfileRequested = false, env, constants, deps, @@ -896,9 +900,9 @@ export async function handleProviderInferenceState({ if (!sandboxName) sandboxName = await deps.promptValidatedSandboxName(agent); const confirmedSandboxName = sandboxName; const buildEstimateNote = - env.NEMOCLAW_IGNORE_RUNTIME_RESOURCES === "1" + env.NEMOCLAW_IGNORE_RUNTIME_RESOURCES === "1" || !customDockerfileRequested ? null - : deps.formatSandboxBuildEstimateNote(deps.assessHost()); + : deps.formatSandboxBuildEstimateNote(deps.assessHost(), "custom-dockerfile"); deps.log( deps.formatOnboardConfigSummary({ provider, diff --git a/src/lib/onboard/managed-gateway-runtime-binding.test.ts b/src/lib/onboard/managed-gateway-runtime-binding.test.ts new file mode 100644 index 0000000000..3ce803bfc1 --- /dev/null +++ b/src/lib/onboard/managed-gateway-runtime-binding.test.ts @@ -0,0 +1,110 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { afterEach, describe, expect, it } from "vitest"; +import { buildPodmanDriverGatewayEnv } from "./compute/podman/gateway-env"; +import { + DOCKER_DRIVER_GATEWAY_CONFIG_NAME, + MANAGED_GATEWAY_RUNTIME_BINDING_NAME, + readManagedGatewayRuntimeBinding, + writeManagedDriverGatewayConfig, +} from "./docker-driver-gateway-config"; + +const directories: string[] = []; + +function createPodmanBinding() { + const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-runtime-binding-")); + directories.push(stateDir); + buildPodmanDriverGatewayEnv({ + gatewayPort: 8080, + stateDir, + podmanSocketPath: "/run/user/1000/podman/podman.sock", + supervisorImage: "ghcr.io/nvidia/openshell/supervisor@sha256:abc", + }); + return stateDir; +} + +afterEach(() => { + for (const directory of directories.splice(0)) { + fs.rmSync(directory, { recursive: true, force: true }); + } +}); + +describe("managed gateway runtime binding", () => { + it("persists the exact driver-native values beside the matching gateway config", () => { + const stateDir = createPodmanBinding(); + const binding = readManagedGatewayRuntimeBinding(stateDir); + + expect(binding).toMatchObject({ + version: 1, + driverName: "podman", + configSha256: expect.stringMatching(/^[0-9a-f]{64}$/), + values: { + socket_path: "/run/user/1000/podman/podman.sock", + network_name: "openshell", + }, + }); + expect(Object.keys(binding?.values ?? {}).sort()).toEqual([ + "network_name", + "socket_path", + "supervisor_image", + ]); + expect( + fs.statSync(path.join(stateDir, MANAGED_GATEWAY_RUNTIME_BINDING_NAME)).mode & 0o777, + ).toBe(0o600); + }); + + it("persists only an adapter-owned allowlist and never copies unrelated credentials", () => { + const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-runtime-allowlist-")); + directories.push(stateDir); + const credential = "must-not-enter-runtime-binding"; + writeManagedDriverGatewayConfig( + stateDir, + { OPENSHELL_LOCAL_TLS_DIR: path.join(stateDir, "tls") }, + { + driverName: "mxc", + entries: [ + ["endpoint", "unix:///run/mxc.sock"], + ["access_token", credential], + ], + persistedRuntimeKeys: ["endpoint"], + }, + ); + + const bindingPath = path.join(stateDir, MANAGED_GATEWAY_RUNTIME_BINDING_NAME); + const bindingText = fs.readFileSync(bindingPath, "utf8"); + expect(bindingText).not.toContain(credential); + expect(readManagedGatewayRuntimeBinding(stateDir)).toMatchObject({ + driverName: "mxc", + values: { endpoint: "unix:///run/mxc.sock" }, + }); + }); + + it("fails closed when the sidecar no longer matches the gateway config", () => { + const stateDir = createPodmanBinding(); + fs.appendFileSync( + path.join(stateDir, DOCKER_DRIVER_GATEWAY_CONFIG_NAME), + "\n# unexpected drift\n", + ); + + expect(() => readManagedGatewayRuntimeBinding(stateDir)).toThrow( + "does not match its gateway configuration", + ); + }); + + it("rejects a symlinked runtime sidecar", () => { + const stateDir = createPodmanBinding(); + const bindingPath = path.join(stateDir, MANAGED_GATEWAY_RUNTIME_BINDING_NAME); + const movedPath = path.join(stateDir, "moved-runtime.json"); + fs.renameSync(bindingPath, movedPath); + fs.symlinkSync(movedPath, bindingPath); + + expect(() => readManagedGatewayRuntimeBinding(stateDir)).toThrow( + "failed ownership or mode checks", + ); + }); +}); diff --git a/src/lib/onboard/managed-image-catalog.test.ts b/src/lib/onboard/managed-image-catalog.test.ts new file mode 100644 index 0000000000..1cef8cc7b9 --- /dev/null +++ b/src/lib/onboard/managed-image-catalog.test.ts @@ -0,0 +1,539 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; + +import { describe, expect, it, vi } from "vitest"; +import { + normalizeManagedImageRelease, + resolveManagedImageCatalogFromGhcr, + resolveManagedImageContractFromGhcr, +} from "./managed-image/catalog"; +import { + MANAGED_IMAGE_CAPABILITY_CONTRACT_VERSION, + MANAGED_IMAGE_CONTRACT_VERSION, + MANAGED_IMAGE_PLATFORM, + MANAGED_IMAGE_REPOSITORIES, + MANAGED_IMAGE_SOURCE_REPOSITORY, + MANAGED_IMAGE_STARTUP_PROFILE_CONTRACT_VERSION, + SHIPPED_MANAGED_IMAGE_AGENTS, + type ShippedManagedImageAgent, +} from "./managed-image/contract"; + +const RELEASE = "v0.0.97"; +const REVISION = "2f03907c3a7ec151d7f5d4bb2a73abafc2849f83"; +const COHORT = "ghrun-12345-1"; +const OCI_INDEX = "application/vnd.oci.image.index.v1+json"; +const OCI_MANIFEST = "application/vnd.oci.image.manifest.v1+json"; + +type RegistryFixtureOptions = { + readonly blobRedirectLocation?: string; + readonly blobRedirectStatus?: number; + readonly duplicatePlatform?: boolean; + readonly directManifest?: boolean; + readonly configBodyMismatch?: boolean; + readonly labels?: Readonly>; + readonly missingRoot?: boolean; + readonly oversizedRootBody?: boolean; + readonly rootBodyMismatch?: boolean; + readonly rootReference?: string; + readonly secondBlobRedirect?: boolean; +}; + +function digest(character: string): `sha256:${string}` { + return `sha256:${character.repeat(64)}`; +} + +function jsonBody(value: unknown): Buffer { + return Buffer.from(JSON.stringify(value), "utf8"); +} + +function bodyDigest(body: Buffer): `sha256:${string}` { + return `sha256:${createHash("sha256").update(body).digest("hex")}`; +} + +function jsonResponse(body: Buffer, digestHeader?: `sha256:${string}`): Response { + return new Response(body, { + headers: { + "content-type": "application/json", + ...(digestHeader === undefined ? {} : { "docker-content-digest": digestHeader }), + }, + }); +} + +function registryFixture(agent: ShippedManagedImageAgent, options: RegistryFixtureOptions = {}) { + const repository = MANAGED_IMAGE_REPOSITORIES[agent].replace(/^ghcr\.io\//u, ""); + const labels = { + "io.nvidia.nemoclaw.agent": agent, + "io.nvidia.nemoclaw.managed-image.contract": String(MANAGED_IMAGE_CONTRACT_VERSION), + "io.nvidia.nemoclaw.managed-image.platform": MANAGED_IMAGE_PLATFORM, + "io.nvidia.nemoclaw.managed-image.startup-profile": String( + MANAGED_IMAGE_STARTUP_PROFILE_CONTRACT_VERSION, + ), + "io.nvidia.nemoclaw.managed-image.capabilities": String( + MANAGED_IMAGE_CAPABILITY_CONTRACT_VERSION, + ), + "org.opencontainers.image.source": `https://github.com/${MANAGED_IMAGE_SOURCE_REPOSITORY}`, + "org.opencontainers.image.revision": REVISION, + "io.nvidia.nemoclaw.managed-image.cohort": COHORT, + ...options.labels, + }; + const imageConfig = { + architecture: "amd64", + os: "linux", + config: { Labels: labels }, + }; + const configBody = jsonBody(imageConfig); + const configDigest = bodyDigest(configBody); + const imageManifest = { + schemaVersion: 2, + mediaType: OCI_MANIFEST, + config: { + mediaType: "application/vnd.oci.image.config.v1+json", + digest: configDigest, + size: configBody.length, + }, + layers: [], + }; + const imageManifestBody = jsonBody(imageManifest); + const platformDigest = bodyDigest(imageManifestBody); + const rootDocument = options.directManifest + ? imageManifest + : { + schemaVersion: 2, + mediaType: OCI_INDEX, + manifests: [ + { + mediaType: OCI_MANIFEST, + digest: platformDigest, + size: imageManifestBody.length, + platform: { architecture: "amd64", os: "linux" }, + }, + ...(options.duplicatePlatform + ? [ + { + mediaType: OCI_MANIFEST, + digest: digest("f"), + size: 1024, + platform: { architecture: "amd64", os: "linux" }, + }, + ] + : []), + { + mediaType: "application/vnd.oci.image.manifest.v1+json", + digest: digest("e"), + size: 1024, + platform: { architecture: "unknown", os: "unknown" }, + }, + ], + }; + const rootBody = options.directManifest ? imageManifestBody : jsonBody(rootDocument); + const rootDigest = bodyDigest(rootBody); + const fetchMock = vi.fn(async (input: Parameters[0], init?: RequestInit) => { + const url = new URL(String(input)); + const authorization = new Headers(init?.headers).get("authorization"); + const manifestPrefix = `/v2/${repository}/manifests/`; + const rootReference = options.rootReference ?? RELEASE; + switch (true) { + case url.pathname === "/token": + expect(url.searchParams.get("service")).toBe("ghcr.io"); + expect(url.searchParams.get("scope")).toBe(`repository:${repository}:pull`); + return Response.json({ token: "anonymous-registry-token" }); + case url.pathname === `${manifestPrefix}${rootReference}` && !authorization: + return new Response("", { + status: 401, + headers: { + "www-authenticate": `Bearer realm="https://ghcr.io/token",service="ghcr.io",scope="repository:${repository}:pull"`, + }, + }); + case url.pathname === `${manifestPrefix}${rootReference}` && Boolean(authorization): { + switch (true) { + case options.missingRoot === true: + return new Response("not found", { status: 404 }); + case options.oversizedRootBody === true: + return new Response(Buffer.alloc(2 * 1024 * 1024 + 1, 0x20), { + headers: { "docker-content-digest": rootDigest }, + }); + } + const deliveredRootBody = options.rootBodyMismatch + ? Buffer.concat([rootBody, Buffer.from(" ", "utf8")]) + : rootBody; + return jsonResponse(deliveredRootBody, rootDigest); + } + case url.pathname === `${manifestPrefix}${platformDigest}` && Boolean(authorization): + return jsonResponse(imageManifestBody, platformDigest); + case url.pathname === `/v2/${repository}/blobs/${configDigest}` && Boolean(authorization): { + expect(init?.redirect).toBe("manual"); + return new Response(null, { + status: options.blobRedirectStatus ?? 307, + headers: { + location: + options.blobRedirectLocation ?? + `https://pkg-containers.githubusercontent.com/ghcrblobs13/blobs/${configDigest}?sig=fixture`, + }, + }); + } + case url.origin === "https://pkg-containers.githubusercontent.com" && + url.pathname === `/ghcrblobs13/blobs/${configDigest}`: + expect(authorization).toBeNull(); + expect(init?.redirect).toBe("manual"); + switch (options.secondBlobRedirect) { + case true: + return new Response(null, { + status: 307, + headers: { + location: `https://pkg-containers.githubusercontent.com/ghcrblobs14/blobs/${configDigest}?sig=second`, + }, + }); + } + return jsonResponse( + options.configBodyMismatch + ? Buffer.concat([configBody, Buffer.from(" ", "utf8")]) + : configBody, + ); + default: + return new Response("not found", { status: 404 }); + } + }); + const fetchImpl = fetchMock as unknown as typeof fetch; + + return { configDigest, fetchImpl, fetchMock, platformDigest, rootDigest }; +} + +function catalogFixture( + options: Partial> = {}, +) { + const fixtures = Object.fromEntries( + SHIPPED_MANAGED_IMAGE_AGENTS.map((agent) => [ + agent, + registryFixture(agent, { + rootReference: agent === "openclaw" ? RELEASE : `cohort-${COHORT}`, + ...options[agent], + }), + ]), + ) as Record>; + const fetchMock = vi.fn(async (input: Parameters[0], init?: RequestInit) => { + const url = new URL(String(input)); + const scope = url.searchParams.get("scope") ?? ""; + const agent = SHIPPED_MANAGED_IMAGE_AGENTS.find((candidate) => + [url.pathname, scope].some( + (value) => + value.includes(MANAGED_IMAGE_REPOSITORIES[candidate].replace("ghcr.io/", "")) || + value.includes(fixtures[candidate].configDigest), + ), + ); + return agent + ? fixtures[agent].fetchImpl(input, init) + : new Response("not found", { status: 404 }); + }); + const fetchImpl = fetchMock as unknown as typeof fetch; + + return { fetchImpl, fetchMock, fixtures }; +} + +describe("managed image GHCR catalog", () => { + it("resolves the OpenClaw release pointer to an exact validated identity (#7744)", async () => { + const fixture = registryFixture("openclaw"); + + await expect( + resolveManagedImageContractFromGhcr({ + agent: "openclaw", + release: RELEASE, + fetchImpl: fixture.fetchImpl, + }), + ).resolves.toEqual({ + contractVersion: MANAGED_IMAGE_CONTRACT_VERSION, + agent: "openclaw", + platform: MANAGED_IMAGE_PLATFORM, + image: MANAGED_IMAGE_REPOSITORIES.openclaw, + digest: fixture.platformDigest, + reference: `${MANAGED_IMAGE_REPOSITORIES.openclaw}@${fixture.platformDigest}`, + source: { + repository: MANAGED_IMAGE_SOURCE_REPOSITORY, + revision: REVISION, + release: RELEASE, + cohort: COHORT, + }, + startupProfileContractVersion: MANAGED_IMAGE_STARTUP_PROFILE_CONTRACT_VERSION, + capabilityContractVersion: MANAGED_IMAGE_CAPABILITY_CONTRACT_VERSION, + }); + }); + + it("rejects wrong manifest bytes even when the registry advertises the expected digest", async () => { + const fixture = registryFixture("openclaw", { rootBodyMismatch: true }); + + await expect( + resolveManagedImageContractFromGhcr({ + agent: "openclaw", + release: RELEASE, + fetchImpl: fixture.fetchImpl, + }), + ).rejects.toThrow(/GHCR manifest bytes do not match digest/); + }); + + it("hard-stops an oversized chunked manifest before parsing or allocating beyond the cap", async () => { + const fixture = registryFixture("openclaw", { oversizedRootBody: true }); + + await expect( + resolveManagedImageContractFromGhcr({ + agent: "openclaw", + release: RELEASE, + fetchImpl: fixture.fetchImpl, + }), + ).rejects.toThrow(/GHCR manifest exceeds the registry response limit/); + }); + + it("rejects image-config bytes that do not match the manifest descriptor digest", async () => { + const fixture = registryFixture("openclaw", { configBodyMismatch: true }); + + await expect( + resolveManagedImageContractFromGhcr({ + agent: "openclaw", + release: RELEASE, + fetchImpl: fixture.fetchImpl, + }), + ).rejects.toThrow(/GHCR image config bytes do not match digest/); + }); + + it.each([ + "hermes", + "langchain-deepagents-code", + ] as const)("refuses independent %s release-alias discovery", async (agent) => { + const fetchImpl = vi.fn(); + await expect( + resolveManagedImageContractFromGhcr({ + agent, + release: RELEASE, + fetchImpl: fetchImpl as typeof fetch, + }), + ).rejects.toThrow(/only be resolved from the OpenClaw cohort pointer/); + expect(fetchImpl).not.toHaveBeenCalled(); + }); + + it("resolves the complete three-agent catalog rather than an OpenClaw-only default (#7744)", async () => { + const fixture = catalogFixture(); + + const catalog = await resolveManagedImageCatalogFromGhcr({ + release: RELEASE, + fetchImpl: fixture.fetchImpl, + }); + + expect(Object.keys(catalog).sort()).toEqual([...SHIPPED_MANAGED_IMAGE_AGENTS].sort()); + expect( + SHIPPED_MANAGED_IMAGE_AGENTS.map( + (agent) => (catalog[agent] as { source: { cohort: string } }).source.cohort, + ), + ).toEqual([COHORT, COHORT, COHORT]); + const rootManifestRequests = fixture.fetchMock.mock.calls + .map(([input]) => new URL(String(input)).pathname) + .filter((pathname) => pathname.includes("/manifests/")); + expect(rootManifestRequests.filter((pathname) => pathname.endsWith(`/${RELEASE}`))).toEqual([ + `/v2/nvidia/nemoclaw/openclaw-sandbox/manifests/${RELEASE}`, + `/v2/nvidia/nemoclaw/openclaw-sandbox/manifests/${RELEASE}`, + ]); + for (const agent of ["hermes", "langchain-deepagents-code"] as const) { + const repository = MANAGED_IMAGE_REPOSITORIES[agent].replace("ghcr.io/", ""); + expect(rootManifestRequests).toContain(`/v2/${repository}/manifests/cohort-${COHORT}`); + expect(rootManifestRequests).not.toContain(`/v2/${repository}/manifests/${RELEASE}`); + } + }); + + it("fails closed when a dependent cohort alias is torn or absent", async () => { + const fixture = catalogFixture({ hermes: { missingRoot: true } }); + + await expect( + resolveManagedImageCatalogFromGhcr({ + release: RELEASE, + fetchImpl: fixture.fetchImpl, + }), + ).rejects.toThrow(/GHCR manifest request returned HTTP 404/); + + const hermesRepository = MANAGED_IMAGE_REPOSITORIES.hermes.replace("ghcr.io/", ""); + const hermesManifestRequests = fixture.fetchMock.mock.calls + .map(([input]) => new URL(String(input)).pathname) + .filter((pathname) => pathname.startsWith(`/v2/${hermesRepository}/manifests/`)); + expect(hermesManifestRequests).toEqual([ + `/v2/${hermesRepository}/manifests/cohort-${COHORT}`, + `/v2/${hermesRepository}/manifests/cohort-${COHORT}`, + ]); + }); + + it("rejects a dependent image whose label does not match the OpenClaw cohort", async () => { + const fixture = catalogFixture({ + hermes: { + labels: { "io.nvidia.nemoclaw.managed-image.cohort": "ghrun-99999-2" }, + }, + }); + + await expect( + resolveManagedImageCatalogFromGhcr({ + release: RELEASE, + fetchImpl: fixture.fetchImpl, + }), + ).rejects.toThrow(/publication cohort does not match the OpenClaw cohort/); + }); + + it("rejects a dependent image whose revision does not match the OpenClaw pointer", async () => { + const fixture = catalogFixture({ + "langchain-deepagents-code": { + labels: { + "org.opencontainers.image.revision": "f".repeat(40), + }, + }, + }); + + await expect( + resolveManagedImageCatalogFromGhcr({ + release: RELEASE, + fetchImpl: fixture.fetchImpl, + }), + ).rejects.toThrow(/source revision does not match the OpenClaw revision/); + }); + + it.each([ + "ghrun-0-1", + "ghrun-1-0", + "ghrun-01-1", + "run-1-1", + `ghrun-${"1".repeat(21)}-1`, + `ghrun-1-${"1".repeat(11)}`, + ])("rejects malformed OpenClaw publication cohort %j before dependent resolution", async (cohort) => { + const fixture = registryFixture("openclaw", { + labels: { "io.nvidia.nemoclaw.managed-image.cohort": cohort }, + }); + + await expect( + resolveManagedImageCatalogFromGhcr({ + release: RELEASE, + fetchImpl: fixture.fetchImpl, + }), + ).rejects.toThrow(/publication cohort is not a supported identity/); + }); + + it("accepts an OCI image manifest without requiring an index", async () => { + const fixture = registryFixture("openclaw", { directManifest: true }); + + await expect( + resolveManagedImageContractFromGhcr({ + agent: "openclaw", + release: RELEASE, + fetchImpl: fixture.fetchImpl, + }), + ).resolves.toMatchObject({ digest: fixture.rootDigest }); + }); + + it("pins the validated linux/amd64 child manifest rather than its mutable platform index", async () => { + const fixture = registryFixture("openclaw"); + + await expect( + resolveManagedImageContractFromGhcr({ + agent: "openclaw", + release: RELEASE, + fetchImpl: fixture.fetchImpl, + }), + ).resolves.toMatchObject({ + digest: fixture.platformDigest, + reference: `${MANAGED_IMAGE_REPOSITORIES.openclaw}@${fixture.platformDigest}`, + }); + }); + + it("rejects a GHCR blob redirect outside the exact GitHub container origin", async () => { + const fixture = registryFixture("openclaw", { + blobRedirectLocation: `https://example.com/ghcrblobs13/blobs/${digest("7")}?sig=fixture`, + }); + + await expect( + resolveManagedImageContractFromGhcr({ + agent: "openclaw", + release: RELEASE, + fetchImpl: fixture.fetchImpl, + }), + ).rejects.toThrow(/unexpected redirect target/); + }); + + it.each([ + `https://fixture@pkg-containers.githubusercontent.com/ghcrblobs13/blobs/${digest("7")}?sig=fixture`, + `https://pkg-containers.githubusercontent.com/ghcrblobs13/blobs/${digest("7")}?sig=fixture#fragment`, + ])("rejects blob redirect target credentials or fragments: %s", async (location) => { + const fixture = registryFixture("openclaw", { blobRedirectLocation: location }); + + await expect( + resolveManagedImageContractFromGhcr({ + agent: "openclaw", + release: RELEASE, + fetchImpl: fixture.fetchImpl, + }), + ).rejects.toThrow(/unexpected redirect target/); + }); + + it("rejects a second redirect from the validated GitHub container blob target", async () => { + const fixture = registryFixture("openclaw", { secondBlobRedirect: true }); + + await expect( + resolveManagedImageContractFromGhcr({ + agent: "openclaw", + release: RELEASE, + fetchImpl: fixture.fetchImpl, + }), + ).rejects.toThrow(/attempted another redirect/); + }); + + it("requires the first image-config response to be exactly HTTP 307", async () => { + const fixture = registryFixture("openclaw", { + blobRedirectStatus: 302, + }); + + await expect( + resolveManagedImageContractFromGhcr({ + agent: "openclaw", + release: RELEASE, + fetchImpl: fixture.fetchImpl, + }), + ).rejects.toThrow(/returned HTTP 302, expected 307/); + }); + + it.each([ + "latest", + "main", + "v0.0.97@sha256:abc", + "0", + ])("rejects mutable or malformed release selector %j before network access", async (release) => { + const fetchImpl = vi.fn(); + await expect( + resolveManagedImageContractFromGhcr({ + agent: "openclaw", + release, + fetchImpl: fetchImpl as typeof fetch, + }), + ).rejects.toThrow(/not a supported release version/); + expect(fetchImpl).not.toHaveBeenCalled(); + }); + + it("normalizes an installed version to the corresponding immutable release alias", () => { + expect(normalizeManagedImageRelease("0.0.97")).toBe(RELEASE); + }); + + it("fails closed when an index has multiple linux/amd64 workload manifests", async () => { + const fixture = registryFixture("openclaw", { duplicatePlatform: true }); + await expect( + resolveManagedImageContractFromGhcr({ + agent: "openclaw", + release: RELEASE, + fetchImpl: fixture.fetchImpl, + }), + ).rejects.toThrow(/contains 2 linux\/amd64/); + }); + + it("fails closed when the image does not advertise startup-profile v1", async () => { + const fixture = registryFixture("openclaw", { + labels: { "io.nvidia.nemoclaw.managed-image.startup-profile": "2" }, + }); + await expect( + resolveManagedImageContractFromGhcr({ + agent: "openclaw", + release: RELEASE, + fetchImpl: fixture.fetchImpl, + }), + ).rejects.toThrow(/startup-profile does not match/); + }); +}); diff --git a/src/lib/onboard/managed-image-contract.test.ts b/src/lib/onboard/managed-image-contract.test.ts new file mode 100644 index 0000000000..3e5486e3d8 --- /dev/null +++ b/src/lib/onboard/managed-image-contract.test.ts @@ -0,0 +1,189 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; +import { + MANAGED_IMAGE_CAPABILITY_CONTRACT_VERSION, + MANAGED_IMAGE_CONTRACT_VERSION, + MANAGED_IMAGE_PLATFORM, + MANAGED_IMAGE_REPOSITORIES, + MANAGED_IMAGE_SOURCE_REPOSITORY, + MANAGED_IMAGE_STARTUP_PROFILE_CONTRACT_VERSION, + type ManagedImageContractV1, + parseManagedImageContractV1, + SHIPPED_MANAGED_IMAGE_AGENTS, + type ShippedManagedImageAgent, +} from "./managed-image/contract"; +import { + MANAGED_STARTUP_AGENTS, + MANAGED_STARTUP_PROFILE_SCHEMA_VERSION, +} from "./managed-startup/profile"; + +const SOURCE_REVISION = "2f03907c37822ea6f1ac9d1bf5c82a4a4568585f"; +const SOURCE_RELEASE = "v0.0.89"; +const SOURCE_COHORT = "ghrun-7744-2"; +const DIGESTS = { + openclaw: `sha256:${"1a".repeat(32)}`, + hermes: `sha256:${"2b".repeat(32)}`, + "langchain-deepagents-code": `sha256:${"3c".repeat(32)}`, +} as const satisfies Record; + +function contractFor(agent: ShippedManagedImageAgent): ManagedImageContractV1 { + const image = MANAGED_IMAGE_REPOSITORIES[agent]; + const digest = DIGESTS[agent]; + return { + contractVersion: MANAGED_IMAGE_CONTRACT_VERSION, + agent, + platform: MANAGED_IMAGE_PLATFORM, + image, + digest, + reference: `${image}@${digest}`, + source: { + repository: MANAGED_IMAGE_SOURCE_REPOSITORY, + revision: SOURCE_REVISION, + release: SOURCE_RELEASE, + cohort: SOURCE_COHORT, + }, + startupProfileContractVersion: MANAGED_IMAGE_STARTUP_PROFILE_CONTRACT_VERSION, + capabilityContractVersion: MANAGED_IMAGE_CAPABILITY_CONTRACT_VERSION, + }; +} + +describe("managed image contract v1", () => { + it("advertises the exact startup-profile schema and all shipped agent consumers (#7744)", () => { + expect(MANAGED_IMAGE_STARTUP_PROFILE_CONTRACT_VERSION).toBe( + MANAGED_STARTUP_PROFILE_SCHEMA_VERSION, + ); + expect([...SHIPPED_MANAGED_IMAGE_AGENTS].sort()).toEqual([...MANAGED_STARTUP_AGENTS].sort()); + }); + + it.each( + SHIPPED_MANAGED_IMAGE_AGENTS, + )("maps %s to its immutable public GHCR identity (#7744)", (agent) => { + const parsed = parseManagedImageContractV1(contractFor(agent), agent); + + expect(parsed).toEqual(contractFor(agent)); + expect(parsed.image).toBe(MANAGED_IMAGE_REPOSITORIES[agent]); + expect(parsed.reference).toBe(`${parsed.image}@${parsed.digest}`); + expect(parsed.reference).toMatch( + /^ghcr\.io\/nvidia\/nemoclaw\/[a-z0-9-]+@sha256:[0-9a-f]{64}$/u, + ); + }); + + it("rejects a mutable tag in place of the exact digest reference (#7744)", () => { + const contract = { + ...contractFor("openclaw"), + reference: `${MANAGED_IMAGE_REPOSITORIES.openclaw}:${SOURCE_RELEASE}`, + }; + + expect(() => parseManagedImageContractV1(contract, "openclaw")).toThrow("contract.reference"); + }); + + it("rejects a digest that is not a lowercase sha256 identity (#7744)", () => { + const contract = { + ...contractFor("hermes"), + digest: `sha256:${"AB".repeat(32)}`, + }; + + expect(() => parseManagedImageContractV1(contract, "hermes")).toThrow("contract.digest"); + }); + + it("rejects a repository outside the agent's reviewed public mapping (#7744)", () => { + const contract = { + ...contractFor("langchain-deepagents-code"), + image: "ghcr.io/example/private/deepagents", + }; + + expect(() => parseManagedImageContractV1(contract, "langchain-deepagents-code")).toThrow( + "contract.image", + ); + }); + + it("rejects a source identity without a full revision (#7744)", () => { + const contract = { + ...contractFor("openclaw"), + source: { + repository: MANAGED_IMAGE_SOURCE_REPOSITORY, + revision: SOURCE_REVISION.slice(0, 12), + release: SOURCE_RELEASE, + cohort: SOURCE_COHORT, + }, + }; + + expect(() => parseManagedImageContractV1(contract, "openclaw")).toThrow( + "contract.source.revision", + ); + }); + + it("rejects a mutable source release alias (#7744)", () => { + const contract = { + ...contractFor("hermes"), + source: { + repository: MANAGED_IMAGE_SOURCE_REPOSITORY, + revision: SOURCE_REVISION, + release: "latest", + cohort: SOURCE_COHORT, + }, + }; + + expect(() => parseManagedImageContractV1(contract, "hermes")).toThrow( + "contract.source.release", + ); + }); + + it("rejects a source identity without an exact publication cohort (#7744)", () => { + const contract = { + ...contractFor("langchain-deepagents-code"), + source: { + repository: MANAGED_IMAGE_SOURCE_REPOSITORY, + revision: SOURCE_REVISION, + release: SOURCE_RELEASE, + cohort: "latest", + }, + }; + + expect(() => parseManagedImageContractV1(contract, "langchain-deepagents-code")).toThrow( + "contract.source.cohort", + ); + }); + + it.each([ + "ghrun-123456789012345678901-1", + "ghrun-1-12345678901", + ])("rejects an unbounded publication cohort %s (#7744)", (cohort) => { + expect(() => + parseManagedImageContractV1( + { + ...contractFor("openclaw"), + source: { ...contractFor("openclaw").source, cohort }, + }, + "openclaw", + ), + ).toThrow("contract.source.cohort"); + }); + + it.each([ + ["platform", { platform: "linux/arm64" }, "contract.platform"], + [ + "startup profile", + { startupProfileContractVersion: 2 }, + "contract.startupProfileContractVersion", + ], + ["capability", { capabilityContractVersion: 2 }, "contract.capabilityContractVersion"], + ])("rejects %s contract-version drift instead of guessing compatibility (#7744)", (_label, mutation, expectedField) => { + expect(() => + parseManagedImageContractV1({ ...contractFor("hermes"), ...mutation }, "hermes"), + ).toThrow(expectedField); + }); + + it("rejects unexpected identity fields so publication metadata cannot alter runtime semantics (#7744)", () => { + const contract = { + ...contractFor("openclaw"), + aliases: [`${MANAGED_IMAGE_REPOSITORIES.openclaw}:${SOURCE_RELEASE}`], + }; + + expect(() => parseManagedImageContractV1(contract, "openclaw")).toThrow( + "contract must contain exactly", + ); + }); +}); diff --git a/src/lib/onboard/managed-image-registry-fetch.test.ts b/src/lib/onboard/managed-image-registry-fetch.test.ts new file mode 100644 index 0000000000..cbab25470a --- /dev/null +++ b/src/lib/onboard/managed-image-registry-fetch.test.ts @@ -0,0 +1,127 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createServer, type Server } from "node:http"; +import type { AddressInfo } from "node:net"; + +import { afterEach, describe, expect, it } from "vitest"; + +import { PEM } from "./__test-helpers__/corporate-ca-fixtures"; +import { + createManagedImageRegistryFetchSession, + resolveManagedImageRegistryDispatcherOptions, +} from "./managed-image/registry-fetch"; + +const servers: Server[] = []; + +async function listen(server: Server): Promise { + servers.push(server); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", resolve); + }); + return (server.address() as AddressInfo).port; +} + +afterEach(async () => { + await Promise.all( + servers.splice(0).map( + (server) => + new Promise((resolve) => { + server.close(() => resolve()); + server.closeAllConnections(); + }), + ), + ); +}); + +describe("managed image registry transport", () => { + it("routes the default registry fetch through the bounded host proxy", async () => { + const tunnels: string[] = []; + const proxy = createServer(); + proxy.on("connect", (request, socket) => { + tunnels.push(request.url ?? ""); + socket.end("HTTP/1.1 502 Bad Gateway\r\nConnection: close\r\n\r\n"); + }); + const proxyPort = await listen(proxy); + const session = createManagedImageRegistryFetchSession({ + environment: { + HTTP_PROXY: `http://127.0.0.1:${proxyPort}`, + NEMOCLAW_CORPORATE_CA_IMPORT: "0", + }, + }); + + try { + await expect(session.fetchImpl("http://registry.invalid/v2/")).rejects.toThrow(); + expect(tunnels).toEqual(["registry.invalid:80"]); + } finally { + await session.close(); + } + }); + + it("honors normalized NO_PROXY without mutating the global dispatcher", async () => { + let proxyTunnels = 0; + const proxy = createServer(); + proxy.on("connect", (_request, socket) => { + proxyTunnels += 1; + socket.end("HTTP/1.1 502 Bad Gateway\r\nConnection: close\r\n\r\n"); + }); + const proxyPort = await listen(proxy); + const targetPort = await listen( + createServer((_request, response) => { + response.end("direct"); + }), + ); + const session = createManagedImageRegistryFetchSession({ + environment: { + HTTPS_PROXY: `http://127.0.0.1:${proxyPort}`, + NO_PROXY: "127.0.0.1", + NEMOCLAW_CORPORATE_CA_IMPORT: "0", + }, + }); + + try { + const response = await session.fetchImpl(`http://127.0.0.1:${targetPort}/health`); + expect(await response.text()).toBe("direct"); + expect(proxyTunnels).toBe(0); + } finally { + await session.close(); + } + }); + + it("adds the validated corporate CA to direct, request, and proxy TLS trust", () => { + const options = resolveManagedImageRegistryDispatcherOptions({ + environment: {}, + corporateCaOverride: { + pem: PEM, + sourceEnv: "fixture", + sourcePath: "/fixture/corporate-ca.pem", + }, + }); + + for (const tls of [options.connect, options.requestTls, options.proxyTls]) { + expect((tls as { ca?: readonly string[] }).ca).toContain(PEM); + } + }); + + it("gives lowercase proxy variables precedence and rejects unsupported proxy URLs", () => { + expect( + resolveManagedImageRegistryDispatcherOptions({ + environment: { + HTTP_PROXY: "http://ignored.example:8080", + http_proxy: "http://selected.example:8081", + NEMOCLAW_CORPORATE_CA_IMPORT: "0", + }, + }).httpProxy, + ).toBe("http://selected.example:8081/"); + + expect(() => + resolveManagedImageRegistryDispatcherOptions({ + environment: { + HTTPS_PROXY: "file:///tmp/not-a-proxy", + NEMOCLAW_CORPORATE_CA_IMPORT: "0", + }, + }), + ).toThrow("not a supported HTTP(S) proxy URL"); + }); +}); diff --git a/src/lib/onboard/managed-image/catalog.ts b/src/lib/onboard/managed-image/catalog.ts new file mode 100644 index 0000000000..80ee090d98 --- /dev/null +++ b/src/lib/onboard/managed-image/catalog.ts @@ -0,0 +1,541 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { Buffer } from "node:buffer"; +import { createHash } from "node:crypto"; + +import { + MANAGED_IMAGE_CAPABILITY_CONTRACT_VERSION, + MANAGED_IMAGE_CONTRACT_VERSION, + MANAGED_IMAGE_PLATFORM, + MANAGED_IMAGE_REPOSITORIES, + MANAGED_IMAGE_SOURCE_REPOSITORY, + MANAGED_IMAGE_STARTUP_PROFILE_CONTRACT_VERSION, + type ManagedImageContractCatalog, + type ManagedImageContractV1, + type ManagedImagePublicationCohort, + SHIPPED_MANAGED_IMAGE_AGENTS, + type ShippedManagedImageAgent, +} from "./contract"; +import type { ManagedImageRegistryFetchSession } from "./registry-fetch"; + +const GHCR_ORIGIN = "https://ghcr.io"; +const GHCR_TOKEN_PATH = "/token"; +const GHCR_BLOB_ORIGIN = "https://pkg-containers.githubusercontent.com"; +const MANIFEST_ACCEPT = [ + "application/vnd.oci.image.index.v1+json", + "application/vnd.oci.image.manifest.v1+json", + "application/vnd.docker.distribution.manifest.list.v2+json", + "application/vnd.docker.distribution.manifest.v2+json", +].join(", "); +const INDEX_MEDIA_TYPES = new Set([ + "application/vnd.oci.image.index.v1+json", + "application/vnd.docker.distribution.manifest.list.v2+json", +]); +const MANIFEST_MEDIA_TYPES = new Set([ + "application/vnd.oci.image.manifest.v1+json", + "application/vnd.docker.distribution.manifest.v2+json", +]); +const DIGEST_PATTERN = /^sha256:[0-9a-f]{64}$/u; +const RELEASE_PATTERN = /^v[0-9]+(?:\.[0-9]+){1,3}(?:[-.][0-9A-Za-z][0-9A-Za-z.-]*)?$/u; +const REVISION_PATTERN = /^[0-9a-f]{40}$/u; +const COHORT_PATTERN = /^ghrun-[1-9][0-9]{0,19}-[1-9][0-9]{0,9}$/u; +const MAX_REGISTRY_JSON_BYTES = 2 * 1024 * 1024; +const REQUEST_TIMEOUT_MS = 30_000; + +type Fetch = typeof fetch; +type RegistryRedirect = "error" | "manual"; + +type OciDescriptor = { + readonly digest?: unknown; + readonly mediaType?: unknown; + readonly platform?: { + readonly architecture?: unknown; + readonly os?: unknown; + }; +}; + +type OciIndex = { + readonly mediaType?: unknown; + readonly manifests?: unknown; +}; + +type OciManifest = { + readonly mediaType?: unknown; + readonly config?: OciDescriptor; +}; + +type OciImageConfig = { + readonly architecture?: unknown; + readonly os?: unknown; + readonly config?: { + readonly Labels?: unknown; + }; +}; + +export class ManagedImageCatalogError extends Error { + constructor(message: string, options?: ErrorOptions) { + super(`Managed image catalog resolution failed: ${message}`, options); + this.name = "ManagedImageCatalogError"; + } +} + +function invalid(message: string, options?: ErrorOptions): never { + throw new ManagedImageCatalogError(message, options); +} + +async function withRegistryFetch( + fetchImpl: Fetch | undefined, + environment: NodeJS.ProcessEnv | undefined, + operation: (resolvedFetch: Fetch) => Promise, +): Promise { + if (fetchImpl !== undefined) return operation(fetchImpl); + let session: ManagedImageRegistryFetchSession; + try { + const { createManagedImageRegistryFetchSession } = await import("./registry-fetch"); + session = createManagedImageRegistryFetchSession({ environment }); + } catch (error) { + return invalid("registry transport could not be configured", { cause: error }); + } + try { + return await operation(session.fetchImpl); + } finally { + await session.close(); + } +} + +function registryRepository(agent: ShippedManagedImageAgent): string { + const prefix = "ghcr.io/"; + const repository = MANAGED_IMAGE_REPOSITORIES[agent]; + if (!repository.startsWith(prefix)) { + return invalid(`repository for '${agent}' is not hosted on GHCR`); + } + return repository.slice(prefix.length); +} + +function requireDigest(value: unknown, field: string): `sha256:${string}` { + if (typeof value !== "string" || !DIGEST_PATTERN.test(value)) { + return invalid(`${field} is not a lowercase sha256 digest`); + } + return value as `sha256:${string}`; +} + +function requireObject(value: unknown, field: string): Record { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + return invalid(`${field} is not an object`); + } + return value as Record; +} + +async function readBoundedBody(response: Response, field: string): Promise { + const contentLength = response.headers.get("content-length"); + if ( + contentLength !== null && + (!/^[0-9]+$/u.test(contentLength) || Number(contentLength) > MAX_REGISTRY_JSON_BYTES) + ) { + return invalid(`${field} exceeds the registry response limit`); + } + if (response.body === null) return Buffer.alloc(0); + + const reader = response.body.getReader(); + const chunks: Buffer[] = []; + let bytes = 0; + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + bytes += value.byteLength; + if (bytes > MAX_REGISTRY_JSON_BYTES) { + try { + await reader.cancel(); + } catch { + // Preserve the bounded-response error. + } + return invalid(`${field} exceeds the registry response limit`); + } + chunks.push(Buffer.from(value)); + } + } finally { + reader.releaseLock(); + } + return Buffer.concat(chunks, bytes); +} + +function requireBodyDigest(body: Buffer, digest: `sha256:${string}`, field: string): void { + const actual = `sha256:${createHash("sha256").update(body).digest("hex")}`; + if (actual !== digest) { + invalid(`${field} bytes do not match digest ${digest}`); + } +} + +async function readBoundedJson( + response: Response, + field: string, + expectedDigest?: `sha256:${string}`, +): Promise { + const body = await readBoundedBody(response, field); + if (expectedDigest !== undefined) requireBodyDigest(body, expectedDigest, field); + try { + return JSON.parse(body.toString("utf8")) as unknown; + } catch (error) { + return invalid(`${field} is not JSON`, { cause: error }); + } +} + +function bearerChallenge(response: Response, expectedScope: string): URL { + const raw = response.headers.get("www-authenticate"); + if (!raw?.startsWith("Bearer ")) { + return invalid("GHCR did not return a Bearer authentication challenge"); + } + const parameters = new Map(); + for (const match of raw.slice("Bearer ".length).matchAll(/([a-z]+)="([^"]*)"/gu)) { + if (parameters.has(match[1])) return invalid("GHCR returned a duplicate challenge parameter"); + parameters.set(match[1], match[2]); + } + const realm = parameters.get("realm"); + const service = parameters.get("service"); + const scope = parameters.get("scope"); + let tokenUrl: URL; + try { + tokenUrl = new URL(realm ?? ""); + } catch (error) { + return invalid("GHCR returned an invalid token realm", { cause: error }); + } + if ( + tokenUrl.origin !== GHCR_ORIGIN || + tokenUrl.pathname !== GHCR_TOKEN_PATH || + tokenUrl.username || + tokenUrl.password || + tokenUrl.search || + tokenUrl.hash || + service !== "ghcr.io" || + scope !== expectedScope + ) { + return invalid("GHCR returned an unexpected authentication challenge"); + } + tokenUrl.searchParams.set("service", service); + tokenUrl.searchParams.set("scope", scope); + return tokenUrl; +} + +async function requestRegistry( + fetchImpl: Fetch, + url: URL, + init: RequestInit, + description: string, + redirect: RegistryRedirect = "error", +): Promise { + let response: Response; + try { + response = await fetchImpl(url, { + ...init, + redirect, + signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), + }); + } catch (error) { + return invalid(`${description} request failed`, { cause: error }); + } + return response; +} + +function redirectedBlobUrl(response: Response, digest: `sha256:${string}`): URL { + if (response.status !== 307) { + return invalid(`GHCR image config request returned HTTP ${response.status}, expected 307`); + } + const location = response.headers.get("location"); + let target: URL; + try { + target = new URL(location ?? ""); + } catch (error) { + return invalid("GHCR image config returned an invalid redirect target", { cause: error }); + } + const pathSegments = target.pathname.split("/"); + if ( + target.origin !== GHCR_BLOB_ORIGIN || + target.username || + target.password || + target.hash || + pathSegments.length !== 4 || + !/^[A-Za-z0-9._-]+$/u.test(pathSegments[1] ?? "") || + pathSegments[2] !== "blobs" || + pathSegments[3] !== digest + ) { + return invalid("GHCR image config returned an unexpected redirect target"); + } + return target; +} + +async function getAnonymousToken( + fetchImpl: Fetch, + challengeResponse: Response, + expectedScope: string, +): Promise { + const tokenUrl = bearerChallenge(challengeResponse, expectedScope); + const response = await requestRegistry(fetchImpl, tokenUrl, { method: "GET" }, "GHCR token"); + if (!response.ok) return invalid(`GHCR token request returned HTTP ${response.status}`); + const document = requireObject(await readBoundedJson(response, "GHCR token response"), "token"); + const token = document.token; + if (typeof token !== "string" || token.length < 16 || token.length > 16_384) { + return invalid("GHCR token response did not contain a bounded token"); + } + return token; +} + +async function getManifest( + fetchImpl: Fetch, + repository: string, + reference: string, + token?: string, +): Promise<{ + readonly digest: `sha256:${string}`; + readonly document: unknown; + readonly token: string; +}> { + const manifestUrl = new URL(`/v2/${repository}/manifests/${reference}`, GHCR_ORIGIN); + const headers: Record = { Accept: MANIFEST_ACCEPT }; + if (token) headers.Authorization = `Bearer ${token}`; + let response = await requestRegistry( + fetchImpl, + manifestUrl, + { headers, method: "GET" }, + "GHCR manifest", + ); + let resolvedToken = token; + if (response.status === 401 && !resolvedToken) { + const scope = `repository:${repository}:pull`; + resolvedToken = await getAnonymousToken(fetchImpl, response, scope); + response = await requestRegistry( + fetchImpl, + manifestUrl, + { + headers: { ...headers, Authorization: `Bearer ${resolvedToken}` }, + method: "GET", + }, + "authenticated GHCR manifest", + ); + } + if (!resolvedToken) return invalid("GHCR manifest did not establish anonymous pull identity"); + if (!response.ok) return invalid(`GHCR manifest request returned HTTP ${response.status}`); + const digest = requireDigest(response.headers.get("docker-content-digest"), "manifest digest"); + if (DIGEST_PATTERN.test(reference) && digest !== reference) { + return invalid("GHCR returned a manifest digest different from the requested digest"); + } + return { + digest, + document: await readBoundedJson(response, "GHCR manifest", digest), + token: resolvedToken, + }; +} + +function selectImageManifest(document: unknown): `sha256:${string}` | null { + const root = requireObject(document, "manifest") as OciIndex; + if (typeof root.mediaType !== "string") return invalid("manifest mediaType is missing"); + if (MANIFEST_MEDIA_TYPES.has(root.mediaType)) return null; + if (!INDEX_MEDIA_TYPES.has(root.mediaType) || !Array.isArray(root.manifests)) { + return invalid(`unsupported manifest mediaType '${root.mediaType}'`); + } + const candidates = (root.manifests as OciDescriptor[]).filter( + (descriptor) => + descriptor.platform?.os === "linux" && descriptor.platform.architecture === "amd64", + ); + if (candidates.length !== 1) { + return invalid(`manifest index contains ${candidates.length} linux/amd64 image manifests`); + } + return requireDigest(candidates[0]?.digest, "linux/amd64 manifest digest"); +} + +function configDigest(document: unknown): `sha256:${string}` { + const manifest = requireObject(document, "image manifest") as OciManifest; + if (typeof manifest.mediaType !== "string" || !MANIFEST_MEDIA_TYPES.has(manifest.mediaType)) { + return invalid("selected platform descriptor is not an OCI/Docker image manifest"); + } + return requireDigest(manifest.config?.digest, "image config digest"); +} + +async function getImageConfig( + fetchImpl: Fetch, + repository: string, + digest: `sha256:${string}`, + token: string, +): Promise { + const url = new URL(`/v2/${repository}/blobs/${digest}`, GHCR_ORIGIN); + const redirectResponse = await requestRegistry( + fetchImpl, + url, + { headers: { Authorization: `Bearer ${token}` }, method: "GET" }, + "GHCR image config", + "manual", + ); + const target = redirectedBlobUrl(redirectResponse, digest); + const response = await requestRegistry( + fetchImpl, + target, + { method: "GET" }, + "GHCR redirected image config", + "manual", + ); + if (response.status >= 300 && response.status < 400) { + return invalid("GHCR image config redirect target attempted another redirect"); + } + if (!response.ok) return invalid(`GHCR image config request returned HTTP ${response.status}`); + return requireObject( + await readBoundedJson(response, "GHCR image config", digest), + "image config", + ); +} + +function validateImageLabels( + agent: ShippedManagedImageAgent, + imageConfig: OciImageConfig, +): { + readonly cohort: ManagedImagePublicationCohort; + readonly revision: string; +} { + if (imageConfig.os !== "linux" || imageConfig.architecture !== "amd64") { + return invalid(`'${agent}' image config is not linux/amd64`); + } + const labels = requireObject(imageConfig.config?.Labels, "image labels"); + const expected: Readonly> = { + "io.nvidia.nemoclaw.agent": agent, + "io.nvidia.nemoclaw.managed-image.contract": String(MANAGED_IMAGE_CONTRACT_VERSION), + "io.nvidia.nemoclaw.managed-image.platform": MANAGED_IMAGE_PLATFORM, + "io.nvidia.nemoclaw.managed-image.startup-profile": String( + MANAGED_IMAGE_STARTUP_PROFILE_CONTRACT_VERSION, + ), + "io.nvidia.nemoclaw.managed-image.capabilities": String( + MANAGED_IMAGE_CAPABILITY_CONTRACT_VERSION, + ), + "org.opencontainers.image.source": `https://github.com/${MANAGED_IMAGE_SOURCE_REPOSITORY}`, + }; + for (const [key, value] of Object.entries(expected)) { + if (labels[key] !== value) return invalid(`'${agent}' image label ${key} does not match`); + } + const revision = labels["org.opencontainers.image.revision"]; + if (typeof revision !== "string" || !REVISION_PATTERN.test(revision)) { + return invalid(`'${agent}' image source revision is not a full lowercase SHA`); + } + const cohort = labels["io.nvidia.nemoclaw.managed-image.cohort"]; + if (typeof cohort !== "string" || !COHORT_PATTERN.test(cohort)) { + return invalid(`'${agent}' image publication cohort is not a supported identity`); + } + return { + cohort: cohort as ManagedImagePublicationCohort, + revision, + }; +} + +export function normalizeManagedImageRelease(version: string): string { + const release = version.startsWith("v") ? version : `v${version}`; + if (!RELEASE_PATTERN.test(release)) { + return invalid(`'${version}' is not a supported release version`); + } + return release; +} + +async function resolveManagedImageContractAtReferenceFromGhcr(options: { + readonly agent: ShippedManagedImageAgent; + readonly reference: string; + readonly release: string; + readonly fetchImpl: Fetch; + readonly expectedCohort?: ManagedImagePublicationCohort; + readonly expectedRevision?: string; +}): Promise { + const { agent, reference } = options; + const release = normalizeManagedImageRelease(options.release); + const fetchImpl = options.fetchImpl; + const repository = registryRepository(agent); + const root = await getManifest(fetchImpl, repository, reference); + const platformDigest = selectImageManifest(root.document); + const workloadDigest = platformDigest ?? root.digest; + const imageManifest = + platformDigest === null + ? root.document + : (await getManifest(fetchImpl, repository, platformDigest, root.token)).document; + const imageConfig = await getImageConfig( + fetchImpl, + repository, + configDigest(imageManifest), + root.token, + ); + const identity = validateImageLabels(agent, imageConfig); + if (options.expectedCohort !== undefined && identity.cohort !== options.expectedCohort) { + return invalid(`'${agent}' image publication cohort does not match the OpenClaw cohort`); + } + if (options.expectedRevision !== undefined && identity.revision !== options.expectedRevision) { + return invalid(`'${agent}' image source revision does not match the OpenClaw revision`); + } + const image = MANAGED_IMAGE_REPOSITORIES[agent]; + return { + contractVersion: MANAGED_IMAGE_CONTRACT_VERSION, + agent, + platform: MANAGED_IMAGE_PLATFORM, + image, + digest: workloadDigest, + reference: `${image}@${workloadDigest}`, + source: { + repository: MANAGED_IMAGE_SOURCE_REPOSITORY, + revision: identity.revision, + release, + cohort: identity.cohort, + }, + startupProfileContractVersion: MANAGED_IMAGE_STARTUP_PROFILE_CONTRACT_VERSION, + capabilityContractVersion: MANAGED_IMAGE_CAPABILITY_CONTRACT_VERSION, + }; +} + +/** + * Resolve the single public release pointer. + * + * Only OpenClaw carries the consumer-facing release alias. The complete + * catalog must be resolved through {@link resolveManagedImageCatalogFromGhcr} + * so the other shipped agents are bound to OpenClaw's immutable cohort. + */ +export async function resolveManagedImageContractFromGhcr(options: { + readonly agent: ShippedManagedImageAgent; + readonly release: string; + readonly fetchImpl?: Fetch; + readonly environment?: NodeJS.ProcessEnv; +}): Promise { + if (options.agent !== "openclaw") { + return invalid("release aliases may only be resolved from the OpenClaw cohort pointer"); + } + const release = normalizeManagedImageRelease(options.release); + return withRegistryFetch(options.fetchImpl, options.environment, (fetchImpl) => + resolveManagedImageContractAtReferenceFromGhcr({ + agent: options.agent, + reference: release, + release, + fetchImpl, + }), + ); +} + +export async function resolveManagedImageCatalogFromGhcr(options: { + readonly release: string; + readonly fetchImpl?: Fetch; + readonly environment?: NodeJS.ProcessEnv; +}): Promise { + const release = normalizeManagedImageRelease(options.release); + return withRegistryFetch(options.fetchImpl, options.environment, async (fetchImpl) => { + const openclaw = await resolveManagedImageContractFromGhcr({ + agent: "openclaw", + release, + fetchImpl, + }); + const cohortReference = `cohort-${openclaw.source.cohort}`; + const dependentEntries = await Promise.all( + SHIPPED_MANAGED_IMAGE_AGENTS.filter((agent) => agent !== "openclaw").map(async (agent) => [ + agent, + await resolveManagedImageContractAtReferenceFromGhcr({ + agent, + reference: cohortReference, + release, + fetchImpl, + expectedCohort: openclaw.source.cohort, + expectedRevision: openclaw.source.revision, + }), + ]), + ); + return Object.fromEntries([["openclaw", openclaw], ...dependentEntries]); + }); +} diff --git a/src/lib/onboard/managed-image/contract.ts b/src/lib/onboard/managed-image/contract.ts new file mode 100644 index 0000000000..70dcba7d6b --- /dev/null +++ b/src/lib/onboard/managed-image/contract.ts @@ -0,0 +1,193 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export const MANAGED_IMAGE_CONTRACT_VERSION = 1 as const; +export const MANAGED_IMAGE_PLATFORM = "linux/amd64" as const; +export const MANAGED_IMAGE_STARTUP_PROFILE_CONTRACT_VERSION = 1 as const; +export const MANAGED_IMAGE_CAPABILITY_CONTRACT_VERSION = 1 as const; +export const MANAGED_IMAGE_SOURCE_REPOSITORY = "NVIDIA/NemoClaw" as const; + +export const SHIPPED_MANAGED_IMAGE_AGENTS = [ + "openclaw", + "hermes", + "langchain-deepagents-code", +] as const; + +export type ShippedManagedImageAgent = (typeof SHIPPED_MANAGED_IMAGE_AGENTS)[number]; + +export const MANAGED_IMAGE_REPOSITORIES = { + openclaw: "ghcr.io/nvidia/nemoclaw/openclaw-sandbox", + hermes: "ghcr.io/nvidia/nemoclaw/hermes-sandbox", + "langchain-deepagents-code": "ghcr.io/nvidia/nemoclaw/langchain-deepagents-code-sandbox", +} as const satisfies Record; + +export type PublicManagedImageRepository = + (typeof MANAGED_IMAGE_REPOSITORIES)[ShippedManagedImageAgent]; +export type ManagedImageDigest = `sha256:${string}`; +export type ManagedImageReference = `${PublicManagedImageRepository}@${ManagedImageDigest}`; +export type ManagedImagePublicationCohort = `ghrun-${number}-${number}`; + +export interface ManagedImageSourceIdentity { + readonly repository: typeof MANAGED_IMAGE_SOURCE_REPOSITORY; + readonly revision: string; + readonly release: string; + readonly cohort: ManagedImagePublicationCohort; +} + +/** + * Immutable identity consumed by buildless onboarding. + * + * The validated cohort binds all shipped agent images to one publication. + * Other publication evidence (mutable aliases and base-image provenance) stays + * outside this runtime identity. + */ +export interface ManagedImageContractV1 { + readonly contractVersion: typeof MANAGED_IMAGE_CONTRACT_VERSION; + readonly agent: ShippedManagedImageAgent; + readonly platform: typeof MANAGED_IMAGE_PLATFORM; + readonly image: PublicManagedImageRepository; + readonly digest: ManagedImageDigest; + readonly reference: ManagedImageReference; + readonly source: ManagedImageSourceIdentity; + readonly startupProfileContractVersion: typeof MANAGED_IMAGE_STARTUP_PROFILE_CONTRACT_VERSION; + readonly capabilityContractVersion: typeof MANAGED_IMAGE_CAPABILITY_CONTRACT_VERSION; +} + +export type ManagedImageContractCatalog = Readonly>; + +const DIGEST_PATTERN = /^sha256:[0-9a-f]{64}$/u; +const REVISION_PATTERN = /^[0-9a-f]{40}$/u; +const RELEASE_PATTERN = /^v[0-9]+(?:\.[0-9]+){1,3}(?:[-.][0-9A-Za-z][0-9A-Za-z.-]*)?$/u; +const COHORT_PATTERN = /^ghrun-[1-9][0-9]{0,19}-[1-9][0-9]{0,9}$/u; + +export class ManagedImageContractError extends Error { + constructor(message: string) { + super(`Invalid managed image contract: ${message}`); + this.name = "ManagedImageContractError"; + } +} + +function isRecord(value: unknown): value is Record { + if (typeof value !== "object" || value === null || Array.isArray(value)) return false; + const prototype = Object.getPrototypeOf(value); + return prototype === Object.prototype || prototype === null; +} + +function requireRecord(value: unknown, field: string): Record { + if (!isRecord(value)) { + throw new ManagedImageContractError(`${field} must be an object`); + } + return value; +} + +function requireExactKeys( + value: Record, + expectedKeys: readonly string[], + field: string, +): void { + const actual = Object.keys(value).sort(); + const expected = [...expectedKeys].sort(); + if (actual.length !== expected.length || actual.some((key, index) => key !== expected[index])) { + throw new ManagedImageContractError(`${field} must contain exactly: ${expected.join(", ")}`); + } +} + +function requireLiteral(value: unknown, expected: T, field: string): T { + if (value !== expected) { + throw new ManagedImageContractError(`${field} must be ${JSON.stringify(expected)}`); + } + return expected; +} + +function requirePattern(value: unknown, pattern: RegExp, field: string): string { + if (typeof value !== "string" || !pattern.test(value)) { + throw new ManagedImageContractError(`${field} has an unsupported format`); + } + return value; +} + +export function isShippedManagedImageAgent(value: string): value is ShippedManagedImageAgent { + return (SHIPPED_MANAGED_IMAGE_AGENTS as readonly string[]).includes(value); +} + +export function parseManagedImageContractV1( + value: unknown, + expectedAgent?: ShippedManagedImageAgent, +): ManagedImageContractV1 { + const contract = requireRecord(value, "contract"); + requireExactKeys( + contract, + [ + "agent", + "capabilityContractVersion", + "contractVersion", + "digest", + "image", + "platform", + "reference", + "source", + "startupProfileContractVersion", + ], + "contract", + ); + + requireLiteral( + contract.contractVersion, + MANAGED_IMAGE_CONTRACT_VERSION, + "contract.contractVersion", + ); + if (typeof contract.agent !== "string" || !isShippedManagedImageAgent(contract.agent)) { + throw new ManagedImageContractError("contract.agent is not a shipped managed agent"); + } + const agent = contract.agent; + if (expectedAgent !== undefined && agent !== expectedAgent) { + throw new ManagedImageContractError(`contract.agent must be ${JSON.stringify(expectedAgent)}`); + } + + const platform = requireLiteral(contract.platform, MANAGED_IMAGE_PLATFORM, "contract.platform"); + const image = requireLiteral(contract.image, MANAGED_IMAGE_REPOSITORIES[agent], "contract.image"); + const digest = requirePattern(contract.digest, DIGEST_PATTERN, "contract.digest"); + const reference = requireLiteral(contract.reference, `${image}@${digest}`, "contract.reference"); + + const source = requireRecord(contract.source, "contract.source"); + requireExactKeys(source, ["cohort", "release", "repository", "revision"], "contract.source"); + const sourceRepository = requireLiteral( + source.repository, + MANAGED_IMAGE_SOURCE_REPOSITORY, + "contract.source.repository", + ); + const sourceRevision = requirePattern( + source.revision, + REVISION_PATTERN, + "contract.source.revision", + ); + const sourceRelease = requirePattern(source.release, RELEASE_PATTERN, "contract.source.release"); + const sourceCohort = requirePattern(source.cohort, COHORT_PATTERN, "contract.source.cohort"); + const startupProfileContractVersion = requireLiteral( + contract.startupProfileContractVersion, + MANAGED_IMAGE_STARTUP_PROFILE_CONTRACT_VERSION, + "contract.startupProfileContractVersion", + ); + const capabilityContractVersion = requireLiteral( + contract.capabilityContractVersion, + MANAGED_IMAGE_CAPABILITY_CONTRACT_VERSION, + "contract.capabilityContractVersion", + ); + + return { + contractVersion: MANAGED_IMAGE_CONTRACT_VERSION, + agent, + platform, + image, + digest: digest as ManagedImageDigest, + reference: reference as ManagedImageReference, + source: { + repository: sourceRepository, + revision: sourceRevision, + release: sourceRelease, + cohort: sourceCohort as ManagedImagePublicationCohort, + }, + startupProfileContractVersion, + capabilityContractVersion, + }; +} diff --git a/src/lib/onboard/managed-image/registry-fetch.ts b/src/lib/onboard/managed-image/registry-fetch.ts new file mode 100644 index 0000000000..cff0c547a1 --- /dev/null +++ b/src/lib/onboard/managed-image/registry-fetch.ts @@ -0,0 +1,98 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { rootCertificates } from "node:tls"; + +import { EnvHttpProxyAgent, fetch as undiciFetch } from "undici"; + +import { resolveCorporateCa } from "../corporate-ca"; +import type { ResolvedCorporateCa } from "../corporate-ca-types"; +import { resolveHostProxyEnvironment } from "../host-proxy-env"; + +type RegistryFetch = typeof fetch; +type RegistryDispatcherOptions = NonNullable[0]>; + +export interface ManagedImageRegistryFetchSession { + readonly fetchImpl: RegistryFetch; + close(): Promise; +} + +export interface ManagedImageRegistryTransportOptions { + readonly environment?: NodeJS.ProcessEnv; + /** Test seam; production resolves the same validated host CA policy as onboarding. */ + readonly corporateCaOverride?: ResolvedCorporateCa | null; +} + +function proxyUrl(value: string | undefined, name: string): string { + if (value === undefined) return ""; + let parsed: URL; + try { + parsed = new URL(value.includes("://") ? value : `http://${value}`); + } catch (error) { + throw new Error(`${name} is not a valid proxy URL`, { cause: error }); + } + if ( + (parsed.protocol !== "http:" && parsed.protocol !== "https:") || + parsed.hostname === "" || + (parsed.pathname !== "" && parsed.pathname !== "/") || + parsed.search !== "" || + parsed.hash !== "" + ) { + throw new Error(`${name} is not a supported HTTP(S) proxy URL`); + } + return parsed.href; +} + +/** + * Build a scoped Undici transport. This never replaces Node's global + * dispatcher and never persists or logs credential-bearing proxy URLs. + */ +export function resolveManagedImageRegistryDispatcherOptions( + options: ManagedImageRegistryTransportOptions = {}, +): RegistryDispatcherOptions { + const environment = options.environment ?? process.env; + const proxy = resolveHostProxyEnvironment(environment); + const httpProxy = proxyUrl(proxy.http_proxy ?? proxy.HTTP_PROXY, "HTTP_PROXY"); + const httpsProxy = proxyUrl( + proxy.https_proxy ?? proxy.HTTPS_PROXY ?? proxy.http_proxy ?? proxy.HTTP_PROXY, + "HTTPS_PROXY", + ); + const noProxy = proxy.no_proxy ?? proxy.NO_PROXY ?? ""; + const corporateCa = + options.corporateCaOverride === undefined + ? resolveCorporateCa(environment) + : options.corporateCaOverride; + if (corporateCa === null) return { httpProxy, httpsProxy, noProxy }; + + const ca = [...rootCertificates, corporateCa.pem]; + return { + httpProxy, + httpsProxy, + noProxy, + connect: { ca }, + requestTls: { ca }, + proxyTls: { ca }, + }; +} + +export function createManagedImageRegistryFetchSession( + options: ManagedImageRegistryTransportOptions = {}, +): ManagedImageRegistryFetchSession { + const dispatcher = new EnvHttpProxyAgent(resolveManagedImageRegistryDispatcherOptions(options)); + const fetchImpl = (async (input, init) => { + const undiciInit = { + ...(init as object), + dispatcher, + } as Parameters[1]; + return (await undiciFetch( + input as Parameters[0], + undiciInit, + )) as unknown as Response; + }) as RegistryFetch; + return { + fetchImpl, + close: async () => { + await dispatcher.close(); + }, + }; +} diff --git a/src/lib/onboard/managed-startup-agent-environment.test.ts b/src/lib/onboard/managed-startup-agent-environment.test.ts new file mode 100644 index 0000000000..c3e3aaeb20 --- /dev/null +++ b/src/lib/onboard/managed-startup-agent-environment.test.ts @@ -0,0 +1,782 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import assert from "node:assert/strict"; +import { Buffer } from "node:buffer"; + +import { describe, expect, it } from "vitest"; + +import { readHermesBuildSettings } from "../../../agents/hermes/config/build-env"; +import { buildConfig as buildOpenClawConfig } from "../../../scripts/generate-openclaw-config.mts"; +import { + type ManagedStartupAgentEnvironment, + ManagedStartupAgentEnvironmentError, + mapManagedStartupProfileToAgentEnvironment, +} from "./managed-startup/agent-environment"; +import { + MANAGED_STARTUP_AGENTS, + MANAGED_STARTUP_PROFILE_AFFORDANCE_INVENTORY, + MANAGED_STARTUP_PROFILE_SCHEMA_VERSION, + type ManagedStartupAgent, + type ManagedStartupJsonObject, + type ManagedStartupProfile, +} from "./managed-startup/profile"; + +const CA_SHA256 = "a".repeat(64); + +function messagingPlan(agent: "openclaw" | "hermes"): ManagedStartupJsonObject { + return { + schemaVersion: 1, + sandboxName: `${agent}-sandbox`, + agent, + workflow: "onboard", + channels: [], + disabledChannels: [], + credentialBindings: [], + networkPolicy: { presets: [], entries: [] }, + agentRender: [], + buildSteps: [], + runtimeSetup: { + nodePreloads: [], + envAliases: [], + secretScans: [], + }, + stateUpdates: [], + healthChecks: [], + }; +} + +function openClawProfile(): ManagedStartupProfile { + return { + schemaVersion: MANAGED_STARTUP_PROFILE_SCHEMA_VERSION, + agent: "openclaw", + agentConfig: { + agent: "openclaw", + webSearch: { enabled: true, provider: "brave" }, + otel: { + enabled: true, + endpointUrl: "http://host.openshell.internal:4318", + serviceName: "openclaw-gateway", + sampleRate: 0.5, + }, + agentTimeoutSeconds: 900, + heartbeatEvery: "30m", + extraAgents: { + agents: [ + { + id: "reviewer", + workspace: "/sandbox/.openclaw/workspace-reviewer", + agentDir: "/sandbox/.openclaw/agents/reviewer", + tools: { profile: "minimal", allow: ["read"], deny: ["exec"] }, + }, + ], + defaults: { subagents: { maxSpawnDepth: 3 } }, + main: { tools: { profile: "minimal", allow: ["read"], deny: ["exec"] } }, + }, + deviceAuth: { disabled: true, optOutSource: "managed-onboard" }, + minimalBootstrap: true, + }, + inference: { + routeProvider: "inference", + upstreamProvider: "nvidia-prod", + model: "nvidia/nemotron-3-ultra-550b-a55b", + routedBaseUrl: "https://inference.local/v1", + upstreamEndpointUrl: null, + api: "openai-responses", + primaryModelRef: "inference/nvidia/nemotron-3-ultra-550b-a55b", + compatibility: { maxRetries: 2, supportsDeveloperRole: true }, + inputModalities: ["text", "image"], + }, + proxy: { + managedHost: "10.200.0.1", + managedPort: 3128, + hostHttpUrl: "http://proxy.example.test:8080", + hostHttpsUrl: "https://connect-proxy.example.test:8443", + hostNoProxy: ["localhost", "inference.local", "127.0.0.1"], + }, + dashboard: { + agent: "openclaw", + mode: "remote", + url: "https://dashboard.example.test:18789", + port: 18_789, + bindAddress: "0.0.0.0", + wslExposure: true, + }, + tools: { + disclosure: "progressive", + enabledGateways: [], + }, + messaging: { plan: messagingPlan("openclaw") }, + tuning: { + contextWindow: 131_072, + maxTokens: 8192, + reasoning: true, + reasoningEffort: "high", + }, + corporateCa: { bundleSha256: CA_SHA256 }, + }; +} + +function hermesProfile(): ManagedStartupProfile { + return { + schemaVersion: MANAGED_STARTUP_PROFILE_SCHEMA_VERSION, + agent: "hermes", + agentConfig: { + agent: "hermes", + webSearch: { enabled: true, provider: "tavily" }, + }, + inference: { + routeProvider: "custom", + upstreamProvider: "anthropic-prod", + model: "claude-sonnet-4-5", + routedBaseUrl: "https://inference.local/v1", + upstreamEndpointUrl: null, + api: "anthropic-messages", + primaryModelRef: null, + compatibility: null, + inputModalities: null, + }, + proxy: { + managedHost: "proxy_name", + managedPort: 43128, + hostHttpUrl: "http://proxy.example.test:8080", + hostHttpsUrl: "http://proxy.example.test:3128", + hostNoProxy: ["localhost", "127.0.0.1"], + }, + dashboard: { + agent: "hermes", + mode: "loopback-forwarded", + url: "http://127.0.0.1:19189", + publicPort: 19_189, + internalPort: 29_189, + tuiEnabled: true, + }, + tools: { + disclosure: "direct", + enabledGateways: ["nous-web", "nous-image", "nous-audio", "nous-browser", "nous-code"], + }, + messaging: { plan: messagingPlan("hermes") }, + tuning: { + contextWindow: 65_536, + maxTokens: null, + reasoning: null, + reasoningEffort: null, + }, + corporateCa: { bundleSha256: CA_SHA256 }, + }; +} + +function dcodeProfile(): ManagedStartupProfile { + return { + schemaVersion: MANAGED_STARTUP_PROFILE_SCHEMA_VERSION, + agent: "langchain-deepagents-code", + agentConfig: { + agent: "langchain-deepagents-code", + autoApprovalMode: "thread-opt-in", + observabilityEnabled: true, + }, + inference: { + routeProvider: "inference", + upstreamProvider: "openrouter", + model: "openai/gpt-5.4", + routedBaseUrl: "https://inference.local/v1", + upstreamEndpointUrl: "https://openrouter.ai/api/v1", + api: "openai-completions", + primaryModelRef: null, + compatibility: null, + inputModalities: null, + }, + proxy: { + managedHost: "10.200.0.1", + managedPort: 3128, + hostHttpUrl: null, + hostHttpsUrl: null, + hostNoProxy: [], + }, + dashboard: { + agent: "langchain-deepagents-code", + mode: "disabled", + }, + tools: { + disclosure: "progressive", + enabledGateways: [], + }, + messaging: { plan: null }, + tuning: { + contextWindow: null, + maxTokens: null, + reasoning: null, + reasoningEffort: null, + }, + corporateCa: { bundleSha256: CA_SHA256 }, + }; +} + +function decodeBase64Json(encoded: string): unknown { + return JSON.parse(Buffer.from(encoded, "base64").toString("utf8")) as unknown; +} + +function representedLegacyInputs(result: ManagedStartupAgentEnvironment): string[] { + return [ + ...new Set([ + ...Object.keys(result.configurationEnvironment), + ...Object.keys(result.runtimeEnvironment), + ...result.materials.map((material) => material.legacyInput), + ]), + ].sort(); +} + +const PROFILES: Readonly ManagedStartupProfile>> = { + openclaw: openClawProfile, + hermes: hermesProfile, + "langchain-deepagents-code": dcodeProfile, +}; + +describe("managed startup agent environment", () => { + it("maps every OpenClaw profile field to the existing generator and entrypoint contracts", () => { + const result = mapManagedStartupProfileToAgentEnvironment(openClawProfile()); + + expect(result.schemaVersion).toBe(1); + expect(result.agent).toBe("openclaw"); + expect(result.configurationEnvironment).toEqual({ + CHAT_UI_URL: "https://dashboard.example.test:18789", + NEMOCLAW_AGENT_HEARTBEAT_EVERY: "30m", + NEMOCLAW_AGENT_TIMEOUT: "900", + NEMOCLAW_CONTEXT_WINDOW: "131072", + NEMOCLAW_DASHBOARD_BIND: "0.0.0.0", + NEMOCLAW_DISABLE_DEVICE_AUTH: "1", + NEMOCLAW_DEVICE_AUTH_OPT_OUT_SOURCE: "managed-onboard", + NEMOCLAW_EXTRA_AGENTS_JSON_B64: expect.any(String), + NEMOCLAW_INFERENCE_API: "openai-responses", + NEMOCLAW_INFERENCE_BASE_URL: "https://inference.local/v1", + NEMOCLAW_INFERENCE_COMPAT_B64: expect.any(String), + NEMOCLAW_INFERENCE_INPUTS: "image,text", + NEMOCLAW_INFERENCE_PROVIDER_ID: "inference", + NEMOCLAW_MAX_TOKENS: "8192", + NEMOCLAW_MESSAGING_PLAN_B64: expect.any(String), + NEMOCLAW_MODEL: "nvidia/nemotron-3-ultra-550b-a55b", + NEMOCLAW_OPENCLAW_OTEL: "1", + NEMOCLAW_OPENCLAW_OTEL_ENDPOINT: "http://host.openshell.internal:4318", + NEMOCLAW_OPENCLAW_OTEL_SAMPLE_RATE: "0.5", + NEMOCLAW_OPENCLAW_OTEL_SERVICE_NAME: "openclaw-gateway", + NEMOCLAW_PRIMARY_MODEL_REF: "inference/nvidia/nemotron-3-ultra-550b-a55b", + NEMOCLAW_PROXY_HOST: "10.200.0.1", + NEMOCLAW_PROXY_PORT: "3128", + NEMOCLAW_REASONING: "true", + NEMOCLAW_REASONING_EFFORT: "high", + NEMOCLAW_TOOL_DISCLOSURE: "progressive", + NEMOCLAW_UPSTREAM_PROVIDER: "nvidia-prod", + NEMOCLAW_WEB_SEARCH_ENABLED: "1", + NEMOCLAW_WEB_SEARCH_PROVIDER: "brave", + NEMOCLAW_WSL_DASHBOARD_EXPOSURE: "1", + }); + const expectedOpenClawRuntime = { ...result.configurationEnvironment }; + delete expectedOpenClawRuntime.NEMOCLAW_MESSAGING_PLAN_B64; + expect(result.runtimeEnvironment).toEqual({ + ...expectedOpenClawRuntime, + HTTP_PROXY: "http://proxy.example.test:8080", + HTTPS_PROXY: "https://connect-proxy.example.test:8443", + NO_PROXY: "127.0.0.1,inference.local,localhost", + NEMOCLAW_DASHBOARD_PORT: "18789", + NEMOCLAW_MINIMAL_BOOTSTRAP: "1", + http_proxy: "http://proxy.example.test:8080", + https_proxy: "https://connect-proxy.example.test:8443", + no_proxy: "127.0.0.1,inference.local,localhost", + }); + expect(Object.hasOwn(result.runtimeEnvironment, "NEMOCLAW_MESSAGING_PLAN_B64")).toBe(false); + + expect( + decodeBase64Json(result.configurationEnvironment.NEMOCLAW_INFERENCE_COMPAT_B64 ?? ""), + ).toEqual({ + maxRetries: 2, + supportsDeveloperRole: true, + }); + expect( + decodeBase64Json(result.configurationEnvironment.NEMOCLAW_EXTRA_AGENTS_JSON_B64 ?? ""), + ).toEqual({ + agents: [ + { + agentDir: "/sandbox/.openclaw/agents/reviewer", + id: "reviewer", + tools: { allow: ["read"], deny: ["exec"], profile: "minimal" }, + workspace: "/sandbox/.openclaw/workspace-reviewer", + }, + ], + defaults: { subagents: { maxSpawnDepth: 3 } }, + main: { tools: { allow: ["read"], deny: ["exec"], profile: "minimal" } }, + }); + const encodedPlan = result.configurationEnvironment.NEMOCLAW_MESSAGING_PLAN_B64 ?? ""; + expect(encodedPlan).toMatch(/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/); + expect(decodeBase64Json(encodedPlan)).toMatchObject({ + schemaVersion: 1, + sandboxName: "openclaw-sandbox", + agent: "openclaw", + }); + expect(decodeBase64Json(encodedPlan)).not.toHaveProperty("workflow"); + + expect(result.materials).toEqual([ + { + kind: "corporate-ca-handoff", + legacyInput: "NEMOCLAW_CORPORATE_CA_B64", + expectedSha256: CA_SHA256, + }, + ]); + expect(result.actions).toEqual([ + { + kind: "apply-messaging-plan", + agent: "openclaw", + mode: "apply", + phase: "runtime-setup", + runAs: "root", + }, + { kind: "generate-agent-config", agent: "openclaw", runAs: "sandbox" }, + { + kind: "apply-messaging-plan", + agent: "openclaw", + mode: "apply", + phase: "post-agent-install", + runAs: "sandbox", + }, + { + kind: "configure-dashboard", + dashboard: openClawProfile().dashboard, + }, + ]); + }); + + it("maps every Hermes profile field, including gateway presets and dashboard forwarding", () => { + const result = mapManagedStartupProfileToAgentEnvironment(hermesProfile()); + + expect(result.configurationEnvironment).toEqual({ + CHAT_UI_URL: "http://127.0.0.1:19189", + NEMOCLAW_CONTEXT_WINDOW: "65536", + NEMOCLAW_HERMES_TOOL_GATEWAY_BROKER: "1", + NEMOCLAW_HERMES_TOOL_GATEWAY_PRESETS_B64: expect.any(String), + NEMOCLAW_INFERENCE_API: "anthropic-messages", + NEMOCLAW_INFERENCE_BASE_URL: "https://inference.local/v1", + NEMOCLAW_INFERENCE_PROVIDER_ID: "custom", + NEMOCLAW_MESSAGING_PLAN_B64: expect.any(String), + NEMOCLAW_MODEL: "claude-sonnet-4-5", + NEMOCLAW_TOOL_DISCLOSURE: "direct", + NEMOCLAW_UPSTREAM_PROVIDER: "anthropic-prod", + NEMOCLAW_WEB_SEARCH_ENABLED: "1", + NEMOCLAW_WEB_SEARCH_PROVIDER: "tavily", + }); + expect( + decodeBase64Json( + result.configurationEnvironment.NEMOCLAW_HERMES_TOOL_GATEWAY_PRESETS_B64 ?? "", + ), + ).toEqual(["nous-audio", "nous-browser", "nous-code", "nous-image", "nous-web"]); + expect(result.runtimeEnvironment).toEqual({ + CHAT_UI_URL: "http://127.0.0.1:19189", + HTTP_PROXY: "http://proxy.example.test:8080", + HTTPS_PROXY: "http://proxy.example.test:3128", + NO_PROXY: "127.0.0.1,localhost", + NEMOCLAW_CONTEXT_WINDOW: "65536", + NEMOCLAW_HERMES_DASHBOARD: "1", + NEMOCLAW_HERMES_DASHBOARD_INTERNAL_PORT: "29189", + NEMOCLAW_HERMES_DASHBOARD_PORT: "19189", + NEMOCLAW_HERMES_DASHBOARD_TUI: "1", + NEMOCLAW_HERMES_TOOL_GATEWAY_BROKER: "1", + NEMOCLAW_HERMES_TOOL_GATEWAY_PRESETS_B64: + result.configurationEnvironment.NEMOCLAW_HERMES_TOOL_GATEWAY_PRESETS_B64, + NEMOCLAW_INFERENCE_API: "anthropic-messages", + NEMOCLAW_INFERENCE_BASE_URL: "https://inference.local/v1", + NEMOCLAW_INFERENCE_PROVIDER_ID: "custom", + NEMOCLAW_MODEL: "claude-sonnet-4-5", + NEMOCLAW_PROXY_HOST: "proxy_name", + NEMOCLAW_PROXY_PORT: "43128", + NEMOCLAW_TOOL_DISCLOSURE: "direct", + NEMOCLAW_UPSTREAM_PROVIDER: "anthropic-prod", + NEMOCLAW_WEB_SEARCH_ENABLED: "1", + NEMOCLAW_WEB_SEARCH_PROVIDER: "tavily", + http_proxy: "http://proxy.example.test:8080", + https_proxy: "http://proxy.example.test:3128", + no_proxy: "127.0.0.1,localhost", + }); + expect(result.actions).toContainEqual({ + kind: "apply-messaging-plan", + agent: "hermes", + mode: "apply", + phase: "runtime-setup", + runAs: "root", + }); + expect(result.actions).toContainEqual({ + kind: "apply-messaging-plan", + agent: "hermes", + mode: "apply", + phase: "post-agent-install", + runAs: "sandbox", + }); + expect(result.actions).toContainEqual({ + kind: "configure-dashboard", + dashboard: hermesProfile().dashboard, + }); + }); + + it("keeps DCode routing and auto-approval in root-owned files instead of ambient runtime env", () => { + const result = mapManagedStartupProfileToAgentEnvironment(dcodeProfile()); + + expect(result.configurationEnvironment).toEqual({ + HTTP_PROXY: "", + HTTPS_PROXY: "", + NEMOCLAW_INFERENCE_API: "openai-completions", + NEMOCLAW_INFERENCE_BASE_URL: "https://inference.local/v1", + NEMOCLAW_INFERENCE_PROVIDER_ID: "inference", + NEMOCLAW_MODEL: "openai/gpt-5.4", + NEMOCLAW_TOOL_DISCLOSURE: "progressive", + NEMOCLAW_UPSTREAM_ENDPOINT_URL: "https://openrouter.ai/api/v1", + NEMOCLAW_UPSTREAM_PROVIDER: "openrouter", + NO_PROXY: "", + http_proxy: "", + https_proxy: "", + no_proxy: "", + }); + const expectedDcodeRuntime = { ...result.configurationEnvironment }; + delete expectedDcodeRuntime.NEMOCLAW_INFERENCE_BASE_URL; + for (const name of [ + "HTTP_PROXY", + "HTTPS_PROXY", + "NO_PROXY", + "http_proxy", + "https_proxy", + "no_proxy", + ]) { + delete expectedDcodeRuntime[name]; + } + expect(result.runtimeEnvironment).toEqual({ + ...expectedDcodeRuntime, + NEMOCLAW_OBSERVABILITY: "1", + }); + for (const environment of [result.configurationEnvironment, result.runtimeEnvironment]) { + expect(environment).not.toHaveProperty("NEMOCLAW_DCODE_AUTO_APPROVAL"); + expect(environment).not.toHaveProperty("NEMOCLAW_MESSAGING_PLAN_B64"); + expect(environment).not.toHaveProperty("NEMOCLAW_PROXY_HOST"); + expect(environment).not.toHaveProperty("NEMOCLAW_PROXY_PORT"); + } + expect(result.runtimeEnvironment).not.toHaveProperty("HTTP_PROXY"); + expect(result.runtimeEnvironment).not.toHaveProperty("HTTPS_PROXY"); + expect(result.runtimeEnvironment).not.toHaveProperty("NEMOCLAW_INFERENCE_BASE_URL"); + + expect(result.materials).toEqual([ + { + kind: "corporate-ca-handoff", + legacyInput: "NEMOCLAW_CORPORATE_CA_B64", + expectedSha256: CA_SHA256, + }, + { + kind: "root-owned-file", + legacyInput: "NEMOCLAW_DCODE_AUTO_APPROVAL", + path: "/usr/local/share/nemoclaw/dcode-auto-approval", + contents: "thread-opt-in\n", + owner: "root", + group: "root", + mode: 0o444, + }, + { + kind: "root-owned-file", + legacyInput: "NEMOCLAW_INFERENCE_BASE_URL", + path: "/usr/local/share/nemoclaw/dcode-inference-base-url", + contents: "https://inference.local/v1\n", + owner: "root", + group: "root", + mode: 0o444, + }, + { + kind: "root-owned-file", + legacyInput: "NEMOCLAW_PROXY_HOST", + path: "/usr/local/share/nemoclaw/dcode-proxy-host", + contents: "10.200.0.1\n", + owner: "root", + group: "root", + mode: 0o444, + }, + { + kind: "root-owned-file", + legacyInput: "NEMOCLAW_PROXY_PORT", + path: "/usr/local/share/nemoclaw/dcode-proxy-port", + contents: "3128\n", + owner: "root", + group: "root", + mode: 0o444, + }, + ]); + expect(result.actions).toEqual([ + { + kind: "generate-agent-config", + agent: "langchain-deepagents-code", + runAs: "sandbox", + }, + { + kind: "configure-dashboard", + dashboard: { agent: "langchain-deepagents-code", mode: "disabled" }, + }, + ]); + }); + + it("feeds the existing OpenClaw and Hermes config consumers without translation", () => { + const openclaw = mapManagedStartupProfileToAgentEnvironment(openClawProfile()); + const openclawConfig = buildOpenClawConfig({ + ...openclaw.configurationEnvironment, + ...openclaw.runtimeEnvironment, + }); + expect(openclawConfig).toMatchObject({ + agents: { + defaults: { + heartbeat: { every: "30m" }, + subagents: { maxSpawnDepth: 3 }, + timeoutSeconds: 900, + }, + list: [{ default: true, id: "main" }, { id: "reviewer" }], + }, + models: { + providers: { + inference: { + api: "openai-responses", + baseUrl: "https://inference.local/v1", + }, + }, + }, + }); + + const hermes = mapManagedStartupProfileToAgentEnvironment(hermesProfile()); + const hermesSettings = readHermesBuildSettings({ + ...hermes.configurationEnvironment, + ...hermes.runtimeEnvironment, + }); + expect(hermesSettings).toMatchObject({ + model: "claude-sonnet-4-5", + baseUrl: "https://inference.local/v1", + providerKey: "custom", + upstreamProvider: "anthropic-prod", + inferenceApi: "anthropic-messages", + contextWindow: 65_536, + toolDisclosure: "direct", + webSearchProvider: "tavily", + managedToolGateways: { + brokerEnabled: true, + presets: ["nous-audio", "nous-browser", "nous-code", "nous-image", "nous-web"], + }, + }); + }); + + it.each( + MANAGED_STARTUP_AGENTS, + )("represents the complete $0 Docker/start affordance inventory", (agent) => { + const result = mapManagedStartupProfileToAgentEnvironment(PROFILES[agent]()); + expect(representedLegacyInputs(result)).toEqual( + MANAGED_STARTUP_PROFILE_AFFORDANCE_INVENTORY[agent] + .map((affordance) => affordance.input) + .sort(), + ); + const messagingActions = result.actions.filter( + (action) => action.kind === "apply-messaging-plan", + ); + expect(messagingActions.map(({ phase, runAs }) => [phase, runAs])).toEqual( + agent === "langchain-deepagents-code" + ? [] + : [ + ["runtime-setup", "root"], + ["post-agent-install", "sandbox"], + ], + ); + expect(messagingActions.map((action) => String(action.phase))).not.toContain("agent-install"); + }); + + it("uses explicit clear states without erasing launch-only ambient proxy credentials", () => { + const openclawBase = openClawProfile(); + assert(openclawBase.agentConfig.agent === "openclaw", "fixture mismatch"); + const openclaw: ManagedStartupProfile = { + ...openclawBase, + agentConfig: { + ...openclawBase.agentConfig, + heartbeatEvery: null, + minimalBootstrap: false, + }, + proxy: { + ...openclawBase.proxy, + hostHttpUrl: null, + hostHttpsUrl: null, + hostNoProxy: [], + }, + dashboard: { + agent: "openclaw", + mode: "loopback", + url: "http://127.0.0.1:18789", + port: 18_789, + bindAddress: "127.0.0.1", + wslExposure: false, + }, + messaging: { plan: null }, + corporateCa: { bundleSha256: null }, + }; + + const openclawResult = mapManagedStartupProfileToAgentEnvironment(openclaw); + expect(openclawResult.configurationEnvironment.NEMOCLAW_AGENT_HEARTBEAT_EVERY).toBe(""); + expect(openclawResult.configurationEnvironment.NEMOCLAW_DASHBOARD_BIND).toBe(""); + expect(openclawResult.runtimeEnvironment.NEMOCLAW_MINIMAL_BOOTSTRAP).toBe("0"); + for (const name of [ + "HTTP_PROXY", + "HTTPS_PROXY", + "NO_PROXY", + "http_proxy", + "https_proxy", + "no_proxy", + ]) { + expect(openclawResult.runtimeEnvironment).not.toHaveProperty(name); + } + expect(openclawResult.configurationEnvironment).not.toHaveProperty( + "NEMOCLAW_MESSAGING_PLAN_B64", + ); + expect(openclawResult.actions).toContainEqual({ + kind: "apply-messaging-plan", + agent: "openclaw", + mode: "clear", + phase: "runtime-setup", + runAs: "root", + }); + expect(openclawResult.actions).toContainEqual({ + kind: "apply-messaging-plan", + agent: "openclaw", + mode: "clear", + phase: "post-agent-install", + runAs: "sandbox", + }); + expect(openclawResult.materials[0]).toMatchObject({ expectedSha256: null }); + + const hermes: ManagedStartupProfile = { + ...hermesProfile(), + proxy: { + ...hermesProfile().proxy, + hostHttpUrl: null, + hostHttpsUrl: null, + hostNoProxy: [], + }, + dashboard: { + agent: "hermes", + mode: "disabled", + url: "http://127.0.0.1:18789", + publicPort: null, + internalPort: null, + tuiEnabled: false, + }, + tuning: { + contextWindow: null, + maxTokens: null, + reasoning: null, + reasoningEffort: null, + }, + }; + const hermesResult = mapManagedStartupProfileToAgentEnvironment(hermes); + expect(hermesResult.configurationEnvironment.NEMOCLAW_CONTEXT_WINDOW).toBe(""); + expect(hermesResult.runtimeEnvironment).toMatchObject({ + NEMOCLAW_HERMES_DASHBOARD: "0", + NEMOCLAW_HERMES_DASHBOARD_INTERNAL_PORT: "", + NEMOCLAW_HERMES_DASHBOARD_PORT: "", + NEMOCLAW_HERMES_DASHBOARD_TUI: "0", + }); + for (const name of [ + "HTTP_PROXY", + "HTTPS_PROXY", + "NO_PROXY", + "http_proxy", + "https_proxy", + "no_proxy", + ]) { + expect(hermesResult.runtimeEnvironment).not.toHaveProperty(name); + } + + const dcodeBase = dcodeProfile(); + const dcode: ManagedStartupProfile = { + ...dcodeBase, + inference: { ...dcodeBase.inference, upstreamEndpointUrl: null }, + }; + const dcodeResult = mapManagedStartupProfileToAgentEnvironment(dcode); + expect(dcodeResult.configurationEnvironment.NEMOCLAW_UPSTREAM_ENDPOINT_URL).toBe(""); + for (const name of [ + "HTTP_PROXY", + "HTTPS_PROXY", + "NO_PROXY", + "http_proxy", + "https_proxy", + "no_proxy", + ]) { + expect(dcodeResult.configurationEnvironment).toHaveProperty(name, ""); + expect(dcodeResult.runtimeEnvironment).not.toHaveProperty(name); + } + }); + + it("is deterministic across profile key order and never emits certificate or credential bytes", () => { + const profile = openClawProfile(); + const cloned = JSON.parse(JSON.stringify(profile)) as ManagedStartupProfile; + const reordered: ManagedStartupProfile = { + ...cloned, + inference: { + api: profile.inference.api, + upstreamEndpointUrl: profile.inference.upstreamEndpointUrl, + compatibility: profile.inference.compatibility, + inputModalities: profile.inference.inputModalities, + routeProvider: profile.inference.routeProvider, + upstreamProvider: profile.inference.upstreamProvider, + primaryModelRef: profile.inference.primaryModelRef, + routedBaseUrl: profile.inference.routedBaseUrl, + model: profile.inference.model, + }, + }; + const first = mapManagedStartupProfileToAgentEnvironment(profile); + const second = mapManagedStartupProfileToAgentEnvironment(reordered); + + expect(JSON.stringify(first)).toBe(JSON.stringify(second)); + const serialized = JSON.stringify(first); + expect(serialized).not.toContain("BEGIN CERTIFICATE"); + expect(serialized).not.toContain("nvapi-"); + expect(serialized).not.toContain("NVIDIA_API_KEY"); + expect(serialized).toContain(CA_SHA256); + }); + + it("fails closed on a mismatched or unsupported messaging profile", () => { + const profile: ManagedStartupProfile = { + ...openClawProfile(), + messaging: { plan: messagingPlan("hermes") }, + }; + expect(() => mapManagedStartupProfileToAgentEnvironment(profile)).toThrow( + ManagedStartupAgentEnvironmentError, + ); + expect(() => mapManagedStartupProfileToAgentEnvironment(profile)).toThrow( + /validated openclaw messaging plan/, + ); + + const dcode: ManagedStartupProfile = { + ...dcodeProfile(), + messaging: { plan: messagingPlan("openclaw") }, + }; + expect(() => mapManagedStartupProfileToAgentEnvironment(dcode)).toThrow( + /messaging.plan must be null for langchain-deepagents-code/, + ); + }); + + it("revalidates typed input while keeping DCode host proxy intent outside its pinned runtime", () => { + const dcodeBase = dcodeProfile(); + const profile: ManagedStartupProfile = { + ...dcodeBase, + proxy: { + ...dcodeBase.proxy, + hostHttpUrl: "http://proxy.example.test:8080", + }, + }; + const mappedDcode = mapManagedStartupProfileToAgentEnvironment(profile); + expect(mappedDcode.runtimeEnvironment.HTTP_PROXY).toBeUndefined(); + + const openclawBase = openClawProfile(); + const credentialBearing: ManagedStartupProfile = { + ...openclawBase, + inference: { + ...openclawBase.inference, + routedBaseUrl: "https://user:password@inference.local/v1", + }, + }; + expect(() => mapManagedStartupProfileToAgentEnvironment(credentialBearing)).toThrow( + /credential/, + ); + }); +}); diff --git a/src/lib/onboard/managed-startup-application.test.ts b/src/lib/onboard/managed-startup-application.test.ts new file mode 100644 index 0000000000..cd40764ba9 --- /dev/null +++ b/src/lib/onboard/managed-startup-application.test.ts @@ -0,0 +1,407 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { Buffer } from "node:buffer"; +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { LEAF_PEM, PEM } from "./__test-helpers__/corporate-ca-fixtures"; +import { + commitManagedStartupApplication, + type ManagedStartupApplicationTestRuntime, + prepareManagedStartupApplication, +} from "./managed-startup/application"; +import { + encodeManagedStartupProfile, + MANAGED_STARTUP_PROFILE_SCHEMA_VERSION, + type ManagedStartupAgent, + type ManagedStartupAgentConfig, + type ManagedStartupProfile, +} from "./managed-startup/profile"; + +function sha256(bytes: string | Buffer): string { + return createHash("sha256").update(bytes).digest("hex"); +} + +function agentConfigFor(agent: ManagedStartupAgent): ManagedStartupAgentConfig { + switch (agent) { + case "openclaw": + return { + agent, + webSearch: { enabled: false, provider: "brave" }, + otel: { + enabled: false, + endpointUrl: "http://host.openshell.internal:4318", + serviceName: "openclaw-gateway", + sampleRate: 1, + }, + agentTimeoutSeconds: 900, + heartbeatEvery: null, + extraAgents: { agents: [], defaults: {}, main: {} }, + deviceAuth: { disabled: true, optOutSource: "managed-onboard" }, + minimalBootstrap: true, + }; + case "hermes": + return { agent, webSearch: { enabled: false, provider: "tavily" } }; + case "langchain-deepagents-code": + return { agent, autoApprovalMode: "thread-opt-in", observabilityEnabled: true }; + } +} + +function profileFor( + agent: ManagedStartupAgent, + corporateCa: string | null = PEM, +): ManagedStartupProfile { + const inference = + agent === "openclaw" + ? { + routeProvider: "inference", + upstreamProvider: "nvidia", + model: "nvidia/nemotron-3-ultra-550b-a55b", + routedBaseUrl: "https://inference.local/v1", + upstreamEndpointUrl: null, + api: "openai-responses" as const, + primaryModelRef: "inference/nvidia/nemotron-3-ultra-550b-a55b", + compatibility: null, + inputModalities: ["text"] as const, + } + : { + routeProvider: "inference", + upstreamProvider: agent === "hermes" ? "nvidia" : "openrouter", + model: "nvidia/nemotron-3-ultra-550b-a55b", + routedBaseUrl: "https://inference.local/v1", + upstreamEndpointUrl: + agent === "langchain-deepagents-code" ? "https://openrouter.ai/api/v1" : null, + api: "openai-completions" as const, + primaryModelRef: null, + compatibility: null, + inputModalities: null, + }; + const dashboard = + agent === "openclaw" + ? { + agent, + mode: "loopback" as const, + url: "http://127.0.0.1:18789", + port: 18_789, + bindAddress: "127.0.0.1" as const, + wslExposure: false, + } + : agent === "hermes" + ? { + agent, + mode: "disabled" as const, + url: "http://127.0.0.1:19189", + publicPort: null, + internalPort: null, + tuiEnabled: false as const, + } + : { + agent, + mode: "disabled" as const, + }; + return { + schemaVersion: MANAGED_STARTUP_PROFILE_SCHEMA_VERSION, + agent, + agentConfig: agentConfigFor(agent), + inference, + proxy: { + managedHost: "10.200.0.1", + managedPort: 3128, + hostHttpUrl: null, + hostHttpsUrl: null, + hostNoProxy: [], + }, + dashboard, + tools: { + disclosure: "progressive", + enabledGateways: [], + }, + messaging: { plan: null }, + tuning: { + contextWindow: agent === "langchain-deepagents-code" ? null : 65_536, + maxTokens: agent === "openclaw" ? 8192 : null, + reasoning: agent === "openclaw" ? true : null, + reasoningEffort: agent === "openclaw" ? "default" : null, + }, + corporateCa: { + bundleSha256: corporateCa === null ? null : sha256(corporateCa), + }, + }; +} + +describe("managed startup application", () => { + let fixtureRoot: string; + let stateDirectory: string; + let runtime: ManagedStartupApplicationTestRuntime; + + beforeEach(() => { + vi.spyOn(process, "geteuid").mockReturnValue(0); + fixtureRoot = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-managed-startup-")); + fs.chmodSync(fixtureRoot, 0o700); + stateDirectory = path.join(fixtureRoot, "state"); + runtime = { + rootUid: process.getuid?.() ?? 0, + rootGid: process.getgid?.() ?? 0, + }; + }); + + afterEach(() => { + vi.restoreAllMocks(); + fs.rmSync(fixtureRoot, { force: true, recursive: true }); + }); + + function prepare( + agent: ManagedStartupAgent, + corporateCa: string | null = PEM, + corporateCaB64: string | undefined = corporateCa === null + ? undefined + : Buffer.from(corporateCa, "utf8").toString("base64"), + ) { + return prepareManagedStartupApplication( + { + encodedProfile: encodeManagedStartupProfile(profileFor(agent, corporateCa)), + expectedAgent: agent, + corporateCaB64, + stateDirectory, + }, + runtime, + ); + } + + it("requires effective uid 0 before touching state", () => { + vi.mocked(process.geteuid as () => number).mockReturnValue(1000); + expect(() => + prepareManagedStartupApplication( + { + encodedProfile: encodeManagedStartupProfile(profileFor("openclaw")), + expectedAgent: "openclaw", + corporateCaB64: Buffer.from(PEM).toString("base64"), + stateDirectory, + }, + runtime, + ), + ).toThrow(/effective uid 0/u); + expect(fs.existsSync(stateDirectory)).toBe(false); + }); + + it.each([ + "openclaw", + "hermes", + "langchain-deepagents-code", + ] as const)("prepares and commits a root-owned envelope for %s", (agent) => { + const prepared = prepare(agent); + + expect(prepared.status).toBe("prepared"); + expect(prepared.profile.agent).toBe(agent); + expect(fs.existsSync(path.join(stateDirectory, "committed.json"))).toBe(false); + expect(fs.existsSync(path.join(stateDirectory, "pending.json"))).toBe(true); + expect(fs.readFileSync(prepared.profilePath, "utf8")).toBe( + JSON.stringify(JSON.parse(fs.readFileSync(prepared.profilePath, "utf8"))), + ); + expect(fs.readFileSync(prepared.corporateCaPath as string)).toEqual(Buffer.from(PEM)); + + const stateStat = fs.statSync(stateDirectory); + const profileStat = fs.statSync(prepared.profilePath); + expect(stateStat.mode & 0o777).toBe(0o700); + expect(profileStat.mode & 0o777).toBe(0o600); + expect(profileStat.uid).toBe(runtime.rootUid); + expect(profileStat.gid).toBe(runtime.rootGid); + + const committed = commitManagedStartupApplication(prepared, runtime); + expect(committed.status).toBe("committed"); + expect(fs.existsSync(path.join(stateDirectory, "committed.json"))).toBe(true); + expect(fs.existsSync(path.join(stateDirectory, "pending.json"))).toBe(false); + }); + + it("rejects a canonical profile for the wrong image agent", () => { + expect(() => + prepareManagedStartupApplication( + { + encodedProfile: encodeManagedStartupProfile(profileFor("hermes")), + expectedAgent: "openclaw", + corporateCaB64: Buffer.from(PEM).toString("base64"), + stateDirectory, + }, + runtime, + ), + ).toThrow(/targets hermes, expected openclaw/u); + }); + + it("requires the CA transport exactly when the profile records a digest", () => { + expect(() => + prepareManagedStartupApplication( + { + encodedProfile: encodeManagedStartupProfile(profileFor("openclaw")), + expectedAgent: "openclaw", + stateDirectory, + }, + runtime, + ), + ).toThrow(/canonical standard base64/u); + + expect(() => + prepareManagedStartupApplication( + { + encodedProfile: encodeManagedStartupProfile(profileFor("openclaw", null)), + expectedAgent: "openclaw", + corporateCaB64: Buffer.from(PEM).toString("base64"), + stateDirectory, + }, + runtime, + ), + ).toThrow(/must be absent/u); + + const prepared = prepare("openclaw", null); + expect(prepared.corporateCaPath).toBeNull(); + }); + + it.each([ + { + label: "non-canonical standard base64", + pem: PEM, + encoded: `${Buffer.from(PEM).toString("base64")}\n`, + message: /canonical standard base64/u, + }, + { + label: "wrong digest", + pem: PEM, + profilePem: LEAF_PEM, + encoded: Buffer.from(PEM).toString("base64"), + message: /SHA-256 digest/u, + }, + { + label: "invalid X.509", + pem: "-----BEGIN CERTIFICATE-----\nMIIBfake\n-----END CERTIFICATE-----\n", + message: /invalid X\.509/u, + }, + { + label: "non-CA certificate", + pem: LEAF_PEM, + message: /CA:TRUE/u, + }, + { + label: "trailing material", + pem: `${PEM}not-a-certificate`, + message: /trailing non-PEM material/u, + }, + { + label: "too many certificates", + pem: PEM.repeat(25), + message: /1-24 PEM CA certificates/u, + }, + ])("rejects a corporate CA with $label", ({ pem, encoded, message, ...testCase }) => { + const profilePem = + "profilePem" in testCase && typeof testCase.profilePem === "string" + ? testCase.profilePem + : encoded === undefined + ? pem + : PEM; + expect(() => + prepareManagedStartupApplication( + { + encodedProfile: encodeManagedStartupProfile(profileFor("openclaw", profilePem)), + expectedAgent: "openclaw", + corporateCaB64: encoded ?? Buffer.from(pem).toString("base64"), + stateDirectory, + }, + runtime, + ), + ).toThrow(message); + }); + + it("rejects a symlinked state directory", () => { + const redirected = path.join(fixtureRoot, "redirected"); + fs.mkdirSync(redirected, { mode: 0o700 }); + fs.symlinkSync(redirected, stateDirectory); + + expect(() => prepare("openclaw")).toThrow(/real directory/u); + }); + + it("rejects permissive or non-root-owned state components", () => { + fs.mkdirSync(stateDirectory, { mode: 0o700 }); + fs.chmodSync(stateDirectory, 0o755); + expect(() => prepare("openclaw")).toThrow(/mode 0700/u); + + fs.chmodSync(stateDirectory, 0o700); + expect(() => + prepareManagedStartupApplication( + { + encodedProfile: encodeManagedStartupProfile(profileFor("openclaw")), + expectedAgent: "openclaw", + corporateCaB64: Buffer.from(PEM).toString("base64"), + stateDirectory, + }, + { ...runtime, rootUid: runtime.rootUid + 1 }, + ), + ).toThrow(/root:root/u); + }); + + it("rejects hardlinked generation files before commit", () => { + const prepared = prepare("openclaw"); + const outside = path.join(fixtureRoot, "outside-profile"); + fs.writeFileSync(outside, fs.readFileSync(prepared.profilePath), { mode: 0o600 }); + fs.unlinkSync(prepared.profilePath); + fs.linkSync(outside, prepared.profilePath); + + expect(() => commitManagedStartupApplication(prepared, runtime)).toThrow(/hardlinked/u); + }); + + it("is idempotent for one committed fingerprint and rejects profile changes", () => { + const first = prepare("openclaw"); + commitManagedStartupApplication(first, runtime); + + const repeated = prepare("openclaw"); + expect(repeated.status).toBe("already-committed"); + expect(() => commitManagedStartupApplication(repeated, runtime)).not.toThrow(); + + const changed = { + ...profileFor("openclaw"), + inference: { + ...profileFor("openclaw").inference, + model: "nvidia/a-different-model", + }, + }; + expect(() => + prepareManagedStartupApplication( + { + encodedProfile: encodeManagedStartupProfile(changed), + expectedAgent: "openclaw", + corporateCaB64: Buffer.from(PEM).toString("base64"), + stateDirectory, + }, + runtime, + ), + ).toThrow(/recreate the sandbox/u); + }); + + it("recovers a crash before commit without accepting the profile as applied", () => { + const first = prepare("hermes"); + const abandoned = path.join(stateDirectory, `.prepare-999-${"a".repeat(24)}`); + fs.mkdirSync(abandoned, { mode: 0o700 }); + fs.writeFileSync(path.join(abandoned, "profile.json"), "partial", { mode: 0o600 }); + + const recovered = prepare("hermes"); + expect(recovered.status).toBe("prepared"); + expect(recovered.fingerprint).toBe(first.fingerprint); + expect(fs.existsSync(abandoned)).toBe(false); + expect(fs.existsSync(path.join(stateDirectory, "committed.json"))).toBe(false); + + commitManagedStartupApplication(recovered, runtime); + expect(fs.existsSync(path.join(stateDirectory, "committed.json"))).toBe(true); + }); + + it("never accepts a partial committed generation", () => { + const prepared = prepare("langchain-deepagents-code"); + commitManagedStartupApplication(prepared, runtime); + fs.truncateSync(prepared.profilePath, 10); + + expect(() => prepare("langchain-deepagents-code")).toThrow( + /not valid JSON|canonical managed startup profile/u, + ); + }); +}); diff --git a/src/lib/onboard/managed-startup-clone-rebinder.test.ts b/src/lib/onboard/managed-startup-clone-rebinder.test.ts new file mode 100644 index 0000000000..d147570ae7 --- /dev/null +++ b/src/lib/onboard/managed-startup-clone-rebinder.test.ts @@ -0,0 +1,354 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import type { SandboxMessagingPlan } from "../messaging/manifest"; +import { PEM } from "./__test-helpers__/corporate-ca-fixtures"; +import { + type ManagedStartupCloneCurrentState, + ManagedStartupCloneRebindError, + rebindManagedStartupProfileForClone, +} from "./managed-startup/clone-rebinder"; +import { + buildManagedStartupProfile, + type ManagedStartupProfileBuilderInput, +} from "./managed-startup/profile-builder"; + +function messagingPlan(agent: "openclaw" | "hermes", sandboxName = "source"): SandboxMessagingPlan { + return { + schemaVersion: 1, + sandboxName, + agent, + workflow: "onboard", + channels: [ + { + channelId: "telegram", + configured: true, + active: true, + disabled: false, + inputs: [ + { inputId: "botToken", credentialAvailable: true }, + { inputId: "allowedIds", value: ["123456"] }, + ], + }, + ], + disabledChannels: [], + credentialBindings: [], + networkPolicy: { presets: [], entries: [] }, + agentRender: [], + buildSteps: [], + stateUpdates: [], + healthChecks: [], + } as unknown as SandboxMessagingPlan; +} + +function openClawInput(): ManagedStartupProfileBuilderInput { + return { + agent: "openclaw", + inference: { + routeProvider: "inference", + upstreamProvider: "openai-api", + model: "gpt-5.4", + routedBaseUrl: "https://inference.local/v1", + upstreamEndpointUrl: null, + api: "openai-responses", + primaryModelRef: "inference/gpt-5.4", + compatibility: {}, + }, + dashboard: { + agent: "openclaw", + mode: "loopback", + url: "http://127.0.0.1:18789", + port: 18_789, + bindAddress: "127.0.0.1", + wslExposure: false, + }, + webSearch: null, + toolDisclosure: "progressive", + hermesToolGateways: [], + messagingPlan: messagingPlan("openclaw"), + dcodeAutoApprovalMode: null, + observabilityEnabled: null, + environment: {}, + corporateCa: null, + }; +} + +function hermesInput(): ManagedStartupProfileBuilderInput { + return { + agent: "hermes", + inference: { + routeProvider: "inference", + upstreamProvider: "compatible-anthropic-endpoint", + model: "claude-sonnet-4-6", + routedBaseUrl: "https://inference.local/v1", + upstreamEndpointUrl: null, + api: "openai-completions", + primaryModelRef: null, + compatibility: null, + }, + dashboard: { + agent: "hermes", + mode: "loopback-forwarded", + url: "http://127.0.0.1:19189", + publicPort: 19_189, + internalPort: 29_189, + tuiEnabled: true, + }, + webSearch: null, + toolDisclosure: "progressive", + hermesToolGateways: [], + messagingPlan: messagingPlan("hermes"), + dcodeAutoApprovalMode: null, + observabilityEnabled: null, + environment: {}, + corporateCa: null, + }; +} + +function dcodeInput(): ManagedStartupProfileBuilderInput { + return { + agent: "langchain-deepagents-code", + inference: { + routeProvider: "inference", + upstreamProvider: "openrouter", + model: "openai/gpt-5.4", + routedBaseUrl: "https://inference.local/v1", + upstreamEndpointUrl: "https://openrouter.ai/api/v1", + api: "openai-completions", + primaryModelRef: null, + compatibility: null, + }, + dashboard: { agent: "langchain-deepagents-code", mode: "disabled" }, + webSearch: null, + toolDisclosure: "progressive", + hermesToolGateways: [], + messagingPlan: null, + dcodeAutoApprovalMode: "disabled", + observabilityEnabled: false, + environment: {}, + corporateCa: null, + }; +} + +function rebind( + built: ReturnType, + expectedAgent: ManagedStartupProfileBuilderInput["agent"], + destinationDashboardPort: number | null, + currentOverrides: Partial = {}, +) { + const profile = built.profile; + const webSearch = + profile.agentConfig.agent === "langchain-deepagents-code" + ? null + : profile.agentConfig.webSearch; + const hermesDashboard = profile.dashboard.agent === "hermes" ? profile.dashboard : null; + const dcodeConfig = + profile.agentConfig.agent === "langchain-deepagents-code" ? profile.agentConfig : null; + return rebindManagedStartupProfileForClone({ + sourceSandboxName: "source", + destinationSandboxName: "destination", + expectedAgent, + destinationDashboardPort, + encodedProfile: built.encodedProfile, + startupProfileSha256: built.startupProfileSha256, + ...(built.corporateCaB64 === undefined ? {} : { corporateCaB64: built.corporateCaB64 }), + currentSource: { + provider: profile.inference.upstreamProvider, + model: profile.inference.model, + endpointUrl: profile.inference.upstreamEndpointUrl, + preferredInferenceApi: profile.inference.api, + compatibleEndpointReasoning: + profile.agent === "openclaw" && profile.inference.upstreamProvider === "compatible-endpoint" + ? profile.tuning.reasoning + ? "true" + : "false" + : null, + compatibleEndpointReasoningEffort: + profile.agent === "openclaw" && + profile.inference.upstreamProvider === "compatible-endpoint" && + profile.tuning.reasoningEffort !== "default" + ? profile.tuning.reasoningEffort + : null, + toolDisclosure: profile.tools.disclosure, + webSearchEnabled: webSearch?.enabled, + webSearchProvider: webSearch?.provider, + messaging: + profile.messaging.plan === null + ? undefined + : { schemaVersion: 1, plan: profile.messaging.plan }, + hermesToolGateways: profile.tools.enabledGateways, + hermesDashboardEnabled: hermesDashboard?.mode === "loopback-forwarded", + hermesDashboardPort: + hermesDashboard?.mode === "loopback-forwarded" ? hermesDashboard.publicPort : undefined, + hermesDashboardInternalPort: + hermesDashboard?.mode === "loopback-forwarded" ? hermesDashboard.internalPort : undefined, + hermesDashboardTui: + hermesDashboard?.mode === "loopback-forwarded" ? hermesDashboard.tuiEnabled : undefined, + dashboardPort: + profile.dashboard.agent === "openclaw" + ? profile.dashboard.port + : hermesDashboard?.mode === "loopback-forwarded" + ? hermesDashboard.publicPort + : undefined, + dashboardRemoteBindPrepared: + profile.dashboard.agent === "openclaw" + ? profile.dashboard.bindAddress === "0.0.0.0" + : undefined, + dcodeAutoApprovalMode: dcodeConfig?.autoApprovalMode, + observabilityEnabled: dcodeConfig?.observabilityEnabled, + ...currentOverrides, + }, + }); +} + +describe("rebindManagedStartupProfileForClone", () => { + it("rebinds OpenClaw dashboard and manifest-derived provider identity without ambient tokens", () => { + const built = buildManagedStartupProfile(openClawInput()); + const previousToken = process.env.TELEGRAM_BOT_TOKEN; + process.env.TELEGRAM_BOT_TOKEN = "ambient-token-must-not-be-read"; + try { + const rebound = rebind(built, "openclaw", 20_789); + expect(rebound.profile.dashboard).toMatchObject({ + agent: "openclaw", + url: "http://127.0.0.1:20789", + port: 20_789, + }); + expect(rebound.profile.messaging.plan).toMatchObject({ + sandboxName: "destination", + credentialBindings: [ + { + providerName: "destination-telegram-bridge", + credentialAvailable: true, + }, + ], + }); + expect(JSON.stringify(rebound.profile.messaging.plan)).not.toContain( + "ambient-token-must-not-be-read", + ); + expect( + (rebound.profile.messaging.plan as unknown as SandboxMessagingPlan).credentialBindings[0], + ).not.toHaveProperty("credentialHash"); + expect(rebound.startupProfileSha256).not.toBe(built.startupProfileSha256); + } finally { + previousToken === undefined + ? Reflect.deleteProperty(process.env, "TELEGRAM_BOT_TOKEN") + : Reflect.set(process.env, "TELEGRAM_BOT_TOKEN", previousToken); + } + }); + + it("rebinds the current compatible-endpoint reasoning effort instead of stale receipt tuning", () => { + const built = buildManagedStartupProfile({ + ...openClawInput(), + inference: { + ...openClawInput().inference, + upstreamProvider: "compatible-endpoint", + api: "openai-completions", + }, + environment: { + NEMOCLAW_REASONING: "true", + NEMOCLAW_REASONING_EFFORT: "low", + }, + }); + + const rebound = rebind(built, "openclaw", 20_789, { + compatibleEndpointReasoning: "true", + compatibleEndpointReasoningEffort: "high", + }); + + expect(built.profile.tuning.reasoningEffort).toBe("low"); + expect(rebound.profile.tuning).toMatchObject({ + reasoning: true, + reasoningEffort: "high", + }); + }); + + it("rebinds Hermes public dashboard and provider identity while retaining its internal port", () => { + const rebound = rebind(buildManagedStartupProfile(hermesInput()), "hermes", 21_189); + + expect(rebound.profile.dashboard).toEqual({ + agent: "hermes", + mode: "loopback-forwarded", + url: "http://127.0.0.1:21189", + publicPort: 21_189, + internalPort: 29_189, + tuiEnabled: true, + }); + expect(rebound.profile.messaging.plan).toMatchObject({ + sandboxName: "destination", + credentialBindings: [{ providerName: "destination-telegram-bridge" }], + }); + }); + + it("rebinds DCode without inventing dashboard or messaging state", () => { + const rebound = rebind( + buildManagedStartupProfile(dcodeInput()), + "langchain-deepagents-code", + null, + ); + + expect(rebound.profile.dashboard).toEqual({ + agent: "langchain-deepagents-code", + mode: "disabled", + }); + expect(rebound.profile.messaging).toEqual({ plan: null }); + expect(rebound.encodedProfile).toBe(buildManagedStartupProfile(dcodeInput()).encodedProfile); + }); + + it("retains and revalidates the exact public corporate CA transport", () => { + const built = buildManagedStartupProfile({ + ...openClawInput(), + corporateCa: { + pem: PEM, + sourcePath: "/public/corporate-ca.pem", + sourceEnv: "NEMOCLAW_CORPORATE_CA_BUNDLE", + }, + }); + + const rebound = rebind(built, "openclaw", 20_789); + + expect(rebound.corporateCaB64).toBe(built.corporateCaB64); + expect(rebound.profile.corporateCa).toEqual(built.profile.corporateCa); + }); + + it("fails closed on receipt hash, source messaging identity, and unexpected CA transport", () => { + const built = buildManagedStartupProfile(openClawInput()); + expect(() => + rebindManagedStartupProfileForClone({ + sourceSandboxName: "source", + destinationSandboxName: "destination", + expectedAgent: "openclaw", + destinationDashboardPort: 20_789, + encodedProfile: built.encodedProfile, + startupProfileSha256: "0".repeat(64), + currentSource: { + provider: built.profile.inference.upstreamProvider, + model: built.profile.inference.model, + }, + }), + ).toThrow(ManagedStartupCloneRebindError); + + const wrongIdentity = buildManagedStartupProfile({ + ...openClawInput(), + messagingPlan: messagingPlan("openclaw", "other-source"), + }); + expect(() => rebind(wrongIdentity, "openclaw", 20_789)).toThrow(/source plan is invalid/u); + + expect(() => + rebindManagedStartupProfileForClone({ + sourceSandboxName: "source", + destinationSandboxName: "destination", + expectedAgent: "openclaw", + destinationDashboardPort: 20_789, + encodedProfile: built.encodedProfile, + startupProfileSha256: built.startupProfileSha256, + corporateCaB64: "eA==", + currentSource: { + provider: built.profile.inference.upstreamProvider, + model: built.profile.inference.model, + }, + }), + ).toThrow(/corporate CA transport/u); + }); +}); diff --git a/src/lib/onboard/managed-startup-coordinator.test.ts b/src/lib/onboard/managed-startup-coordinator.test.ts new file mode 100644 index 0000000000..dc99e569ca --- /dev/null +++ b/src/lib/onboard/managed-startup-coordinator.test.ts @@ -0,0 +1,254 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import { + type CommittedManagedStartupApplication, + type PreparedManagedStartupApplication, + type PrepareManagedStartupApplicationInput, +} from "./managed-startup/application"; +import { + coordinateManagedStartupApplication, + type ManagedStartupAgentAdapter, + type ManagedStartupCoordinatorDependencies, +} from "./managed-startup/coordinator"; +import { type ManagedStartupAgent, type ManagedStartupProfile } from "./managed-startup/profile"; + +function inputFor(agent: ManagedStartupAgent): PrepareManagedStartupApplicationInput { + return { + encodedProfile: `encoded-${agent}`, + expectedAgent: agent, + }; +} + +function preparedFor( + agent: ManagedStartupAgent, + status: PreparedManagedStartupApplication["status"] = "prepared", +): PreparedManagedStartupApplication { + return { + status, + stateDirectory: "/var/lib/nemoclaw/startup-profile", + generationDirectory: `/var/lib/nemoclaw/startup-profile/generation-${"a".repeat(64)}`, + profilePath: `/var/lib/nemoclaw/startup-profile/generation-${"a".repeat(64)}/profile.json`, + corporateCaPath: null, + fingerprint: "a".repeat(64), + expectedAgent: agent, + profile: { agent } as ManagedStartupProfile, + }; +} + +function committedFrom( + prepared: PreparedManagedStartupApplication, +): CommittedManagedStartupApplication { + const { status: _status, ...application } = prepared; + return { ...application, status: "committed" }; +} + +function dependenciesFor( + prepared: PreparedManagedStartupApplication, + order: string[] = [], +): ManagedStartupCoordinatorDependencies & { + prepareApplication: ReturnType; + commitApplication: ReturnType; +} { + return { + prepareApplication: vi.fn(async () => { + order.push("prepare"); + return prepared; + }), + commitApplication: vi.fn(async (application: PreparedManagedStartupApplication) => { + order.push("commit"); + return committedFrom(application); + }), + }; +} + +function adaptersFor(order: string[] = []): { + readonly adapters: ManagedStartupAgentAdapter[]; + readonly applyByAgent: Record>; +} { + const applyByAgent = { + openclaw: vi.fn(async () => { + order.push("apply:openclaw"); + }), + hermes: vi.fn(async () => { + order.push("apply:hermes"); + }), + "langchain-deepagents-code": vi.fn(async () => { + order.push("apply:langchain-deepagents-code"); + }), + }; + return { + adapters: [ + { agent: "openclaw", apply: applyByAgent.openclaw }, + { agent: "hermes", apply: applyByAgent.hermes }, + { + agent: "langchain-deepagents-code", + apply: applyByAgent["langchain-deepagents-code"], + }, + ], + applyByAgent, + }; +} + +describe("managed startup coordinator", () => { + it.each([ + "openclaw", + "hermes", + "langchain-deepagents-code", + ] as const)("dispatches exactly the %s adapter before commit", async (agent) => { + const order: string[] = []; + const prepared = preparedFor(agent); + const dependencies = dependenciesFor(prepared, order); + const { adapters, applyByAgent } = adaptersFor(order); + + const result = await coordinateManagedStartupApplication( + inputFor(agent), + adapters, + dependencies, + ); + + expect(result.adapterApplied).toBe(true); + expect(result.application.status).toBe("committed"); + expect(order).toEqual(["prepare", `apply:${agent}`, "commit"]); + expect(applyByAgent[agent]).toHaveBeenCalledExactlyOnceWith( + expect.objectContaining({ + agent, + profile: prepared.profile, + fingerprint: prepared.fingerprint, + }), + ); + for (const otherAgent of ["openclaw", "hermes", "langchain-deepagents-code"] as const) { + expect(applyByAgent[otherAgent]).toHaveBeenCalledTimes(otherAgent === agent ? 1 : 0); + } + expect(dependencies.commitApplication).toHaveBeenCalledWith(prepared); + }); + + it("does not reapply mutable config for an already committed profile", async () => { + const prepared = preparedFor("openclaw", "already-committed"); + const dependencies = dependenciesFor(prepared); + const { adapters, applyByAgent } = adaptersFor(); + + const result = await coordinateManagedStartupApplication( + inputFor("openclaw"), + adapters, + dependencies, + ); + + expect(result.adapterApplied).toBe(false); + expect(dependencies.commitApplication).toHaveBeenCalledExactlyOnceWith(prepared); + for (const apply of Object.values(applyByAgent)) { + expect(apply).not.toHaveBeenCalled(); + } + }); + + it("rejects a missing adapter before preparing state", async () => { + const prepared = preparedFor("openclaw"); + const dependencies = dependenciesFor(prepared); + const { adapters } = adaptersFor(); + + await expect( + coordinateManagedStartupApplication( + inputFor("openclaw"), + adapters.filter((adapter) => adapter.agent !== "hermes"), + dependencies, + ), + ).rejects.toThrow(/missing adapter for hermes/u); + expect(dependencies.prepareApplication).not.toHaveBeenCalled(); + }); + + it("rejects a duplicate adapter before preparing state", async () => { + const prepared = preparedFor("openclaw"); + const dependencies = dependenciesFor(prepared); + const { adapters } = adaptersFor(); + + await expect( + coordinateManagedStartupApplication( + inputFor("openclaw"), + [...adapters, adapters[0] as ManagedStartupAgentAdapter], + dependencies, + ), + ).rejects.toThrow(/duplicate adapter registered for openclaw/u); + expect(dependencies.prepareApplication).not.toHaveBeenCalled(); + }); + + it("rejects an adapter for an unshipped agent before preparing state", async () => { + const prepared = preparedFor("openclaw"); + const dependencies = dependenciesFor(prepared); + const { adapters } = adaptersFor(); + const wrong = { + agent: "not-a-shipped-agent", + apply: vi.fn(), + } as unknown as ManagedStartupAgentAdapter; + + await expect( + coordinateManagedStartupApplication(inputFor("openclaw"), [...adapters, wrong], dependencies), + ).rejects.toThrow(/one shipped agent/u); + expect(dependencies.prepareApplication).not.toHaveBeenCalled(); + }); + + it("fails closed instead of cross-dispatching a mismatched prepared profile", async () => { + const prepared = { + ...preparedFor("openclaw"), + profile: { agent: "hermes" } as ManagedStartupProfile, + }; + const dependencies = dependenciesFor(prepared); + const { adapters, applyByAgent } = adaptersFor(); + + await expect( + coordinateManagedStartupApplication(inputFor("openclaw"), adapters, dependencies), + ).rejects.toThrow(/targets hermes, expected openclaw/u); + expect(dependencies.commitApplication).not.toHaveBeenCalled(); + for (const apply of Object.values(applyByAgent)) { + expect(apply).not.toHaveBeenCalled(); + } + }); + + it("does not commit an adapter failure and can retry the pending profile", async () => { + const prepared = preparedFor("hermes"); + const dependencies = dependenciesFor(prepared); + const { adapters, applyByAgent } = adaptersFor(); + applyByAgent.hermes.mockRejectedValueOnce(new Error("adapter failed")); + + await expect( + coordinateManagedStartupApplication(inputFor("hermes"), adapters, dependencies), + ).rejects.toThrow("adapter failed"); + expect(dependencies.commitApplication).not.toHaveBeenCalled(); + + const retried = await coordinateManagedStartupApplication( + inputFor("hermes"), + adapters, + dependencies, + ); + expect(retried.application.status).toBe("committed"); + expect(applyByAgent.hermes).toHaveBeenCalledTimes(2); + expect(dependencies.commitApplication).toHaveBeenCalledTimes(1); + }); + + it("reapplies a pending adapter after a crash at the commit boundary", async () => { + const prepared = preparedFor("langchain-deepagents-code"); + const dependencies = dependenciesFor(prepared); + const { adapters, applyByAgent } = adaptersFor(); + dependencies.commitApplication.mockRejectedValueOnce( + new Error("simulated process interruption"), + ); + + await expect( + coordinateManagedStartupApplication( + inputFor("langchain-deepagents-code"), + adapters, + dependencies, + ), + ).rejects.toThrow("simulated process interruption"); + + const retried = await coordinateManagedStartupApplication( + inputFor("langchain-deepagents-code"), + adapters, + dependencies, + ); + expect(retried.application.status).toBe("committed"); + expect(applyByAgent["langchain-deepagents-code"]).toHaveBeenCalledTimes(2); + expect(dependencies.commitApplication).toHaveBeenCalledTimes(2); + }); +}); diff --git a/src/lib/onboard/managed-startup-image-runtime.test.ts b/src/lib/onboard/managed-startup-image-runtime.test.ts new file mode 100644 index 0000000000..8436449bdd --- /dev/null +++ b/src/lib/onboard/managed-startup-image-runtime.test.ts @@ -0,0 +1,434 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash, X509Certificate } from "node:crypto"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { + MANAGED_STARTUP_E2E_CORPORATE_CA_PEM, + managedStartupE2eProfile, +} from "../../../scripts/checks/generate-managed-startup-profile-fixture.mts"; +import { mapManagedStartupProfileToAgentEnvironment } from "./managed-startup/agent-environment"; +import { + buildManagedStartupImageActionPlan, + MANAGED_STARTUP_MERGED_CA_FILE, + normalizeHermesManagedConfigDescriptor, + readStableRegularFile, + serializeManagedStartupCompletionMarker, + serializeManagedStartupRuntimeEnvironment, + verifyManagedStartupImageCompletion, +} from "./managed-startup/image-runtime"; +import { + fingerprintManagedStartupProfile, + MANAGED_STARTUP_AGENTS, + type ManagedStartupAgent, + type ManagedStartupProfile, + validateManagedStartupProfile, +} from "./managed-startup/profile"; + +const PROXY_ENV_NAMES = [ + "HTTP_PROXY", + "HTTPS_PROXY", + "NO_PROXY", + "http_proxy", + "https_proxy", + "no_proxy", +] as const; + +describe("managed startup image runtime", () => { + let temporaryDirectoryPath = ""; + + beforeEach(() => { + temporaryDirectoryPath = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-managed-startup-")); + }); + afterEach(() => { + vi.restoreAllMocks(); + fs.rmSync(temporaryDirectoryPath, { force: true, recursive: true }); + }); + + function temporaryDirectory(): string { + return temporaryDirectoryPath; + } + + function mockDescriptorOwnership(uid: bigint, gid: bigint): void { + const realFstatSync = fs.fstatSync.bind(fs); + const realLstatSync = fs.lstatSync.bind(fs); + const ownership = new Map([ + ["uid", uid], + ["gid", gid], + ]); + const owned = (stat: fs.BigIntStats): fs.BigIntStats => + new Proxy(stat, { + get(inner, property) { + const value = ownership.has(property) + ? ownership.get(property) + : (Reflect.get(inner, property, inner) as unknown); + return typeof value === "function" ? value.bind(inner) : value; + }, + }); + vi.spyOn(fs, "fstatSync").mockImplementation(((descriptor: number, options: { bigint: true }) => + owned(realFstatSync(descriptor, options))) as typeof fs.fstatSync); + vi.spyOn(fs, "lstatSync").mockImplementation(((file: fs.PathLike, options: { bigint: true }) => + owned(realLstatSync(file, options))) as typeof fs.lstatSync); + } + + function writeCompletionFixture( + profile: ManagedStartupProfile, + corporateCaMerged = false, + ): { + readonly agent: ManagedStartupAgent; + readonly completionFile: string; + readonly fingerprint: string; + readonly runtimeEnvironmentFile: string; + } { + const mapped = mapManagedStartupProfileToAgentEnvironment(profile); + const runtimeEnvironment = serializeManagedStartupRuntimeEnvironment( + mapped.runtimeEnvironment, + corporateCaMerged, + mapped.configurationEnvironment, + ); + const fingerprint = fingerprintManagedStartupProfile(profile); + const completionFile = path.join(temporaryDirectory(), "managed-startup-complete.json"); + const runtimeEnvironmentFile = path.join(temporaryDirectory(), "managed-startup-runtime.env"); + fs.writeFileSync(runtimeEnvironmentFile, runtimeEnvironment, { mode: 0o444 }); + fs.chmodSync(runtimeEnvironmentFile, 0o444); + fs.writeFileSync( + completionFile, + serializeManagedStartupCompletionMarker({ + schemaVersion: 1, + agent: profile.agent, + profileFingerprint: fingerprint, + runtimeEnvironmentSha256: createHash("sha256") + .update(runtimeEnvironment, "utf8") + .digest("hex"), + corporateCaMerged, + }), + { mode: 0o444 }, + ); + fs.chmodSync(completionFile, 0o444); + return { + agent: profile.agent, + completionFile, + fingerprint, + runtimeEnvironmentFile, + }; + } + + it.each([ + "openclaw", + "hermes", + ] as const)("maps the complete %s profile to an offline messaging and config action plan", (agent) => { + const profile = managedStartupE2eProfile(agent); + const mapped = mapManagedStartupProfileToAgentEnvironment(profile); + const plan = buildManagedStartupImageActionPlan(mapped); + + expect(plan.some((command) => command.argv.includes("agent-install"))).toBe(false); + expect( + plan.some((command) => + command.argv.some((argument) => /^(?:npm|npx|pip|pip3|uv)$/u.test(argument)), + ), + ).toBe(false); + expect(plan.map(({ action, runAs }) => ({ action, runAs }))).toEqual([ + { action: "messaging-runtime-setup", runAs: "root" }, + { action: "generate-agent-config", runAs: "sandbox" }, + { action: "messaging-post-agent-install", runAs: "sandbox" }, + ]); + expect(plan[0]?.argv).toContain("runtime-setup"); + expect(plan[2]?.argv).toContain("post-agent-install"); + expect(plan[0]?.argv).not.toContain("--managed-startup-runtime"); + expect(plan[2]?.argv).toContain("--managed-startup-runtime"); + }); + + it("maps the complete DCode profile to its offline config action plan", () => { + const agent = "langchain-deepagents-code"; + const profile = managedStartupE2eProfile(agent); + const mapped = mapManagedStartupProfileToAgentEnvironment(profile); + const plan = buildManagedStartupImageActionPlan(mapped); + + expect(plan.some((command) => command.argv.includes("agent-install"))).toBe(false); + expect( + plan.some((command) => + command.argv.some((argument) => /^(?:npm|npx|pip|pip3|uv)$/u.test(argument)), + ), + ).toBe(false); + expect(plan).toEqual([ + { + action: "generate-agent-config", + runAs: "sandbox", + argv: [ + "/usr/local/bin/node", + "--experimental-strip-types", + "/opt/nemoclaw-deepagents-code/generate-config.ts", + ], + }, + ]); + }); + + it.each( + MANAGED_STARTUP_AGENTS, + )("provides valid same-profile and changed-profile fixtures for %s recreation checks", (agent) => { + const initial = validateManagedStartupProfile(managedStartupE2eProfile(agent)); + const same = validateManagedStartupProfile(managedStartupE2eProfile(agent)); + const changed = validateManagedStartupProfile(managedStartupE2eProfile(agent, true)); + + expect(fingerprintManagedStartupProfile(same)).toBe(fingerprintManagedStartupProfile(initial)); + expect(fingerprintManagedStartupProfile(changed)).not.toBe( + fingerprintManagedStartupProfile(initial), + ); + }); + + it.each( + MANAGED_STARTUP_AGENTS, + )("accepts the root completion marker and exact runtime handoff for %s", (agent) => { + const fixture = writeCompletionFixture(managedStartupE2eProfile(agent)); + mockDescriptorOwnership(0n, 0n); + expect( + verifyManagedStartupImageCompletion( + agent, + fixture.fingerprint, + fixture.completionFile, + fixture.runtimeEnvironmentFile, + ), + ).toEqual({ agent, fingerprint: fixture.fingerprint }); + }); + + it("rejects a changed profile against the root completion fingerprint", () => { + const initial = writeCompletionFixture(managedStartupE2eProfile("openclaw")); + const changedProfile = managedStartupE2eProfile("openclaw", true); + mockDescriptorOwnership(0n, 0n); + expect(() => + verifyManagedStartupImageCompletion( + "openclaw", + fingerprintManagedStartupProfile(changedProfile), + initial.completionFile, + initial.runtimeEnvironmentFile, + ), + ).toThrow(/completion marker does not match the requested profile/u); + }); + + it("rejects runtime handoff drift after a matching completion", () => { + const fixture = writeCompletionFixture(managedStartupE2eProfile("hermes")); + mockDescriptorOwnership(0n, 0n); + fs.chmodSync(fixture.runtimeEnvironmentFile, 0o644); + fs.appendFileSync(fixture.runtimeEnvironmentFile, "export NEMOCLAW_MODEL='tampered/model'\n"); + fs.chmodSync(fixture.runtimeEnvironmentFile, 0o444); + + expect(() => + verifyManagedStartupImageCompletion( + "hermes", + fixture.fingerprint, + fixture.completionFile, + fixture.runtimeEnvironmentFile, + ), + ).toThrow(/runtime environment digest mismatch/u); + }); + + it("accepts merged CA paths without putting the CA payload in the readable handoff", () => { + const fixture = writeCompletionFixture( + managedStartupE2eProfile("langchain-deepagents-code", false, true), + true, + ); + mockDescriptorOwnership(0n, 0n); + expect( + verifyManagedStartupImageCompletion( + "langchain-deepagents-code", + fixture.fingerprint, + fixture.completionFile, + fixture.runtimeEnvironmentFile, + ), + ).toEqual({ + agent: "langchain-deepagents-code", + fingerprint: fixture.fingerprint, + }); + expect(fs.readFileSync(fixture.runtimeEnvironmentFile, "utf8")).not.toContain( + "NEMOCLAW_CORPORATE_CA_B64", + ); + }); + + it("binds the real corporate-CA fixture into every agent profile by exact digest", () => { + expect(() => new X509Certificate(MANAGED_STARTUP_E2E_CORPORATE_CA_PEM)).not.toThrow(); + const digest = createHash("sha256").update(MANAGED_STARTUP_E2E_CORPORATE_CA_PEM).digest("hex"); + + for (const agent of MANAGED_STARTUP_AGENTS) { + expect(managedStartupE2eProfile(agent, false, true).corporateCa.bundleSha256).toBe(digest); + } + }); + + it("writes a deterministic root-sourced runtime environment without profile transport", () => { + const script = serializeManagedStartupRuntimeEnvironment( + { + NEMOCLAW_MODEL: "model-with-'quote", + NEMOCLAW_OBSERVABILITY: "0", + }, + true, + { + NEMOCLAW_INFERENCE_BASE_URL: "https://inference.local/v1", + NEMOCLAW_MODEL: "model-with-'quote", + }, + ); + + expect(script).toContain("unset NEMOCLAW_INFERENCE_BASE_URL"); + expect(script).toContain("export NEMOCLAW_MANAGED_STARTUP_APPLIED='1'"); + expect(script).toContain("export NEMOCLAW_MODEL='model-with-'\"'\"'quote'"); + expect(script).toContain(`export SSL_CERT_FILE='${MANAGED_STARTUP_MERGED_CA_FILE}'`); + expect(script).toContain("export _NEMOCLAW_CORPORATE_CA_MERGED='1'"); + expect(script).not.toContain("NEMOCLAW_STARTUP_PROFILE_B64"); + expect(script).not.toContain("NEMOCLAW_CORPORATE_CA_B64"); + expect(script.endsWith("\n")).toBe(true); + }); + + it.each(["openclaw", "hermes"] as const)("preserves launch-only proxy env for %s", (agent) => { + const mapped = mapManagedStartupProfileToAgentEnvironment( + managedStartupE2eProfile(agent, false, false, true), + ); + const script = serializeManagedStartupRuntimeEnvironment( + mapped.runtimeEnvironment, + false, + mapped.configurationEnvironment, + ); + for (const name of PROXY_ENV_NAMES) { + expect(script).not.toMatch(new RegExp(`(?:export|unset) ${name}(?:=|$)`, "mu")); + } + }); + + it("clears launch-only proxy env when DCode pins managed routing", () => { + const mapped = mapManagedStartupProfileToAgentEnvironment( + managedStartupE2eProfile("langchain-deepagents-code", false, false, true), + ); + const script = serializeManagedStartupRuntimeEnvironment( + mapped.runtimeEnvironment, + false, + mapped.configurationEnvironment, + ); + for (const name of PROXY_ENV_NAMES) { + expect(script).toContain(`unset ${name}`); + } + }); + + it("rejects multiline runtime values before producing a sourceable file", () => { + expect(() => + serializeManagedStartupRuntimeEnvironment({ NEMOCLAW_MODEL: "bad\nvalue" }, false), + ).toThrow(/single-line/u); + }); + + it("refuses a symlink instead of opening its target", () => { + const directory = temporaryDirectory(); + const target = path.join(directory, "target"); + const link = path.join(directory, "link"); + fs.writeFileSync(target, "trusted\n"); + fs.symlinkSync(target, link); + + expect(() => readStableRegularFile(link, 1024)).toThrow(/unsafe or unreadable/u); + }); + + it("rejects descriptor metadata drift after a bounded read", () => { + const directory = temporaryDirectory(); + const target = path.join(directory, "material"); + fs.writeFileSync(target, "trusted\n", { mode: 0o600 }); + const realReadSync = fs.readSync.bind(fs); + vi.spyOn(fs, "readSync") + .mockImplementationOnce((( + descriptor: number, + buffer: NodeJS.ArrayBufferView, + offset: number, + length: number, + position: number | null, + ) => { + const bytesRead = realReadSync(descriptor, buffer, offset, length, position); + fs.chmodSync(target, 0o644); + return bytesRead; + }) as typeof fs.readSync) + .mockImplementation(realReadSync as typeof fs.readSync); + + expect(() => readStableRegularFile(target, 1024)).toThrow(/changed while it was read/u); + }); + + it("normalizes mutable sandbox-owned Hermes config descriptors to mode 0640", () => { + const directory = temporaryDirectory(); + const target = path.join(directory, "config.yaml"); + fs.writeFileSync(target, "model: managed\n", { mode: 0o600 }); + mockDescriptorOwnership(501n, 20n); + + normalizeHermesManagedConfigDescriptor(target, { + uid: 501, + gid: 20, + }); + + expect(fs.readFileSync(target, "utf8")).toBe("model: managed\n"); + expect(fs.statSync(target).mode & 0o777).toBe(0o640); + }); + + it("preserves a root-owned shields-up Hermes descriptor without chmod", () => { + const directory = temporaryDirectory(); + const target = path.join(directory, ".env"); + fs.writeFileSync(target, "OPENAI_API_KEY=managed\n", { mode: 0o444 }); + mockDescriptorOwnership(0n, 0n); + const chmod = vi.spyOn(fs, "fchmodSync"); + + normalizeHermesManagedConfigDescriptor(target, { + uid: 501, + gid: 20, + }); + + expect(chmod).not.toHaveBeenCalled(); + expect(fs.readFileSync(target, "utf8")).toBe("OPENAI_API_KEY=managed\n"); + }); + + it.each([0o440, 0o644, 0o660])("fails closed on unexpected mutable Hermes mode %s", (mode) => { + const directory = temporaryDirectory(); + const target = path.join(directory, "config.yaml"); + fs.writeFileSync(target, "model: managed\n", { mode }); + fs.chmodSync(target, mode); + mockDescriptorOwnership(501n, 20n); + + expect(() => + normalizeHermesManagedConfigDescriptor(target, { + uid: 501, + gid: 20, + }), + ).toThrow(/unexpected Hermes managed config descriptor/u); + expect(fs.statSync(target).mode & 0o777).toBe(mode); + }); + + it("fails closed on an unexpected Hermes descriptor owner", () => { + const directory = temporaryDirectory(); + const target = path.join(directory, "config.yaml"); + fs.writeFileSync(target, "model: managed\n", { mode: 0o600 }); + mockDescriptorOwnership(502n, 21n); + + expect(() => + normalizeHermesManagedConfigDescriptor(target, { + uid: 501, + gid: 20, + }), + ).toThrow(/unexpected Hermes managed config descriptor/u); + expect(fs.statSync(target).mode & 0o777).toBe(0o600); + }); + + it("detects a path replacement while normalizing through the trusted descriptor", () => { + const directory = temporaryDirectory(); + const target = path.join(directory, "config.yaml"); + const displaced = path.join(directory, "displaced.yaml"); + const replacement = path.join(directory, "replacement.yaml"); + fs.writeFileSync(target, "model: managed\n", { mode: 0o600 }); + fs.writeFileSync(replacement, "model: replaced\n", { mode: 0o640 }); + mockDescriptorOwnership(501n, 20n); + const realFchmodSync = fs.fchmodSync.bind(fs); + vi.spyOn(fs, "fchmodSync").mockImplementation((descriptor, mode) => { + realFchmodSync(descriptor, mode); + fs.renameSync(target, displaced); + fs.renameSync(replacement, target); + }); + + expect(() => + normalizeHermesManagedConfigDescriptor(target, { + uid: 501, + gid: 20, + }), + ).toThrow(/changed during normalization/u); + expect(fs.readFileSync(target, "utf8")).toBe("model: replaced\n"); + }); +}); diff --git a/src/lib/onboard/managed-startup-onboard-profile.test.ts b/src/lib/onboard/managed-startup-onboard-profile.test.ts new file mode 100644 index 0000000000..5383810634 --- /dev/null +++ b/src/lib/onboard/managed-startup-onboard-profile.test.ts @@ -0,0 +1,423 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { + buildManagedStartupOnboardProfile, + type ManagedStartupOnboardProfileInput, +} from "./managed-startup/onboard-profile"; + +const CA_DISABLED_ENVIRONMENT: NodeJS.ProcessEnv = { + NEMOCLAW_CORPORATE_CA_IMPORT: "0", +}; + +function messagingPlan(agent: "openclaw" | "hermes") { + return { + schemaVersion: 1, + sandboxName: "adapter-test", + agent, + workflow: "onboard", + channels: [], + disabledChannels: [], + credentialBindings: [], + networkPolicy: { presets: [], entries: [] }, + agentRender: [], + buildSteps: [], + stateUpdates: [], + healthChecks: [], + } as const; +} + +function openClawInput( + overrides: Partial = {}, +): ManagedStartupOnboardProfileInput { + return { + agentName: "openclaw", + inference: { + routeProvider: "openai", + upstreamProvider: "openai-api", + model: "gpt-5.4", + routedBaseUrl: "https://inference.local/v1", + upstreamEndpointUrl: null, + api: "openai-responses", + primaryModelRef: "openai/gpt-5.4", + compatibility: {}, + }, + chatUiUrl: "http://127.0.0.1:18789", + effectiveDashboardPort: 18_789, + manageDashboard: true, + dashboardBindAddress: undefined, + wslExposure: false, + hermesDashboardState: { config: null, enabled: false }, + webSearch: null, + toolDisclosure: "progressive", + hermesToolGateways: [], + messagingPlan: null, + dcodeAutoApprovalMode: "disabled", + observabilityEnabled: false, + environment: CA_DISABLED_ENVIRONMENT, + ...overrides, + }; +} + +function hermesInput( + overrides: Partial = {}, +): ManagedStartupOnboardProfileInput { + return { + agentName: "hermes", + inference: { + routeProvider: "inference", + upstreamProvider: "hermes-provider", + model: "moonshotai/kimi-k2.6", + routedBaseUrl: "https://inference.local/v1", + upstreamEndpointUrl: null, + api: "openai-completions", + primaryModelRef: null, + compatibility: null, + }, + chatUiUrl: "http://127.0.0.1:18789", + effectiveDashboardPort: 18_789, + manageDashboard: true, + dashboardBindAddress: undefined, + wslExposure: false, + hermesDashboardState: { + config: { + enabled: false, + port: 9119, + internalPort: 19_119, + tuiEnabled: false, + }, + enabled: false, + }, + webSearch: null, + toolDisclosure: "progressive", + hermesToolGateways: [], + messagingPlan: null, + dcodeAutoApprovalMode: "disabled", + observabilityEnabled: false, + environment: CA_DISABLED_ENVIRONMENT, + ...overrides, + }; +} + +function dcodeInput( + overrides: Partial = {}, +): ManagedStartupOnboardProfileInput { + return { + agentName: "langchain-deepagents-code", + inference: { + routeProvider: "inference", + upstreamProvider: "openrouter", + model: "openai/gpt-5.4", + routedBaseUrl: "https://inference.local/v1", + upstreamEndpointUrl: "https://openrouter.ai/api/v1", + api: "openai-completions", + primaryModelRef: null, + compatibility: null, + }, + chatUiUrl: "", + effectiveDashboardPort: 0, + manageDashboard: false, + dashboardBindAddress: undefined, + wslExposure: false, + hermesDashboardState: { config: null, enabled: false }, + webSearch: null, + toolDisclosure: "progressive", + hermesToolGateways: [], + messagingPlan: null, + dcodeAutoApprovalMode: "disabled", + observabilityEnabled: false, + environment: CA_DISABLED_ENVIRONMENT, + ...overrides, + }; +} + +describe("buildManagedStartupOnboardProfile", () => { + it("maps a remote OpenClaw dashboard and its complete agent-owned state", () => { + const plan = messagingPlan("openclaw"); + const built = buildManagedStartupOnboardProfile( + openClawInput({ + chatUiUrl: "https://dashboard.example.test:19443", + effectiveDashboardPort: 19_443, + dashboardBindAddress: "0.0.0.0", + wslExposure: true, + webSearch: { fetchEnabled: true, provider: "tavily" }, + toolDisclosure: "direct", + messagingPlan: plan, + }), + ); + + expect(built.profile).toMatchObject({ + agent: "openclaw", + dashboard: { + agent: "openclaw", + mode: "remote", + url: "https://dashboard.example.test:19443", + port: 19_443, + bindAddress: "0.0.0.0", + wslExposure: true, + }, + agentConfig: { + agent: "openclaw", + webSearch: { enabled: true, provider: "tavily" }, + deviceAuth: { disabled: true, optOutSource: "managed-onboard" }, + }, + tools: { disclosure: "direct", enabledGateways: [] }, + }); + expect(built.profile.messaging.plan).not.toBeNull(); + expect(built.profile.inference.upstreamEndpointUrl).toBeNull(); + }); + + it("keeps bracketed IPv6 loopback OpenClaw dashboards in loopback mode", () => { + const built = buildManagedStartupOnboardProfile( + openClawInput({ + chatUiUrl: "http://[::1]:18789", + }), + ); + + expect(built.profile.dashboard).toEqual({ + agent: "openclaw", + mode: "loopback", + url: "http://[::1]:18789", + port: 18_789, + bindAddress: "127.0.0.1", + wslExposure: false, + }); + }); + + it("rejects an OpenClaw bind value that would otherwise be silently downgraded", () => { + expect(() => + buildManagedStartupOnboardProfile( + openClawInput({ + dashboardBindAddress: "127.0.0.1", + }), + ), + ).toThrow(/dashboard bind address must be empty or 0\.0\.0\.0/); + }); + + it("maps Hermes with its dashboard disabled and Tavily retained", () => { + const built = buildManagedStartupOnboardProfile(hermesInput()); + + expect(built.profile).toMatchObject({ + agent: "hermes", + dashboard: { + agent: "hermes", + mode: "disabled", + url: "http://127.0.0.1:18789", + publicPort: null, + internalPort: null, + tuiEnabled: false, + }, + agentConfig: { + agent: "hermes", + webSearch: { enabled: false, provider: "tavily" }, + }, + inference: { + upstreamEndpointUrl: null, + primaryModelRef: null, + compatibility: null, + inputModalities: null, + }, + }); + }); + + it("maps Hermes forwarding, tool gateways, messaging, and context independently", () => { + const plan = messagingPlan("hermes"); + const built = buildManagedStartupOnboardProfile( + hermesInput({ + chatUiUrl: "http://127.0.0.1:19189", + effectiveDashboardPort: 19_189, + hermesDashboardState: { + config: { + enabled: true, + port: 19_189, + internalPort: 29_189, + tuiEnabled: true, + }, + enabled: true, + }, + webSearch: { fetchEnabled: true, provider: "tavily" }, + hermesToolGateways: ["nous-web", "nous-image"], + messagingPlan: plan, + environment: { + ...CA_DISABLED_ENVIRONMENT, + NEMOCLAW_CONTEXT_WINDOW: "65536", + }, + }), + ); + + expect(built.profile.dashboard).toEqual({ + agent: "hermes", + mode: "loopback-forwarded", + url: "http://127.0.0.1:19189", + publicPort: 19_189, + internalPort: 29_189, + tuiEnabled: true, + }); + expect(built.profile.tools.enabledGateways).toEqual(["nous-image", "nous-web"]); + expect(built.profile.messaging.plan).not.toBeNull(); + expect(built.profile.tuning).toEqual({ + contextWindow: 65_536, + maxTokens: null, + reasoning: null, + reasoningEffort: null, + }); + }); + + it("maps DCode without dashboard, messaging, web-search, gateway, or tuning state", () => { + const built = buildManagedStartupOnboardProfile( + dcodeInput({ + dcodeAutoApprovalMode: "thread-opt-in", + observabilityEnabled: true, + }), + ); + + expect(built.profile).toMatchObject({ + agent: "langchain-deepagents-code", + agentConfig: { + agent: "langchain-deepagents-code", + autoApprovalMode: "thread-opt-in", + observabilityEnabled: true, + }, + dashboard: { agent: "langchain-deepagents-code", mode: "disabled" }, + inference: { + upstreamEndpointUrl: "https://openrouter.ai/api/v1", + api: "openai-completions", + primaryModelRef: null, + compatibility: null, + inputModalities: null, + }, + tools: { disclosure: "progressive", enabledGateways: [] }, + messaging: { plan: null }, + tuning: { + contextWindow: null, + maxTokens: null, + reasoning: null, + reasoningEffort: null, + }, + }); + }); + + it("rejects a DCode dashboard while retaining credential-free host-proxy intent", () => { + expect(() => buildManagedStartupOnboardProfile(dcodeInput({ manageDashboard: true }))).toThrow( + /DCode must not enable a dashboard/, + ); + + const built = buildManagedStartupOnboardProfile( + dcodeInput({ + environment: { + ...CA_DISABLED_ENVIRONMENT, + HTTP_PROXY: "http://proxy.example.test:8080", + }, + }), + ); + expect(built.profile.proxy.hostHttpUrl).toBe("http://proxy.example.test:8080"); + expect(built.profile.proxy.hostNoProxy).toContain("inference.local"); + }); + + it("preserves an explicitly resolved routed base URL for every agent", () => { + const route = "https://portable-route.example.test/v1"; + + expect( + buildManagedStartupOnboardProfile( + openClawInput({ + inference: { ...openClawInput().inference, routedBaseUrl: route }, + }), + ).profile.inference.routedBaseUrl, + ).toBe(route); + expect( + buildManagedStartupOnboardProfile( + hermesInput({ + inference: { ...hermesInput().inference, routedBaseUrl: route }, + }), + ).profile.inference.routedBaseUrl, + ).toBe(route); + expect( + buildManagedStartupOnboardProfile( + dcodeInput({ + inference: { ...dcodeInput().inference, routedBaseUrl: route }, + }), + ).profile.inference.routedBaseUrl, + ).toBe(route); + }); + + it.each([ + ["openclaw", openClawInput], + ["hermes", hermesInput], + ["langchain-deepagents-code", dcodeInput], + ] as const)("keeps credential-bearing proxy aliases out of the %s secret-free profile", (agent, input) => { + const built = buildManagedStartupOnboardProfile( + input({ + environment: { + ...CA_DISABLED_ENVIRONMENT, + HTTP_PROXY: "http://upper:upper-secret@upper.example.test:8080", + HTTPS_PROXY: "http://upper-tls:upper-secret@upper-tls.example.test:8443", + NO_PROXY: "upper.internal", + http_proxy: "http://lower:lower-secret@lower.example.test:8081", + https_proxy: "http://lower-tls:lower-secret@lower-tls.example.test:8444", + no_proxy: "lower.internal", + }, + }), + ); + + expect(built.profile.proxy).toMatchObject({ + hostHttpUrl: null, + hostHttpsUrl: null, + hostNoProxy: [], + }); + expect(built.credentialProxyReplayRequired).toBe(agent !== "langchain-deepagents-code"); + const serialized = JSON.stringify(built); + expect(serialized).not.toContain("upper-secret"); + expect(serialized).not.toContain("lower-secret"); + expect(serialized).not.toContain("upper.internal"); + expect(serialized).not.toContain("lower.internal"); + }); + + it("filters stale semantic env while preserving environment-owned tuning and proxy knobs", () => { + const built = buildManagedStartupOnboardProfile( + openClawInput({ + toolDisclosure: "direct", + webSearch: { fetchEnabled: true, provider: "tavily" }, + environment: { + ...CA_DISABLED_ENVIRONMENT, + NEMOCLAW_MODEL: "stale-model", + NEMOCLAW_INFERENCE_API: "anthropic-messages", + NEMOCLAW_TOOL_DISCLOSURE: "progressive", + NEMOCLAW_WEB_SEARCH_ENABLED: "0", + NEMOCLAW_WEB_SEARCH_PROVIDER: "brave", + CHAT_UI_URL: "https://stale.example.test:19999", + NEMOCLAW_CONTEXT_WINDOW: "262144", + NEMOCLAW_MAX_TOKENS: "8192", + NEMOCLAW_PROXY_HOST: "host.containers.internal", + NEMOCLAW_PROXY_PORT: "3129", + HTTP_PROXY: "http://proxy.example.test:8080", + }, + }), + ); + + expect(built.profile.inference).toMatchObject({ + model: "gpt-5.4", + api: "openai-responses", + }); + expect(built.profile.tools.disclosure).toBe("direct"); + expect(built.profile.agentConfig).toMatchObject({ + agent: "openclaw", + webSearch: { enabled: true, provider: "tavily" }, + }); + expect(built.profile.dashboard).toMatchObject({ + url: "http://127.0.0.1:18789", + }); + expect(built.profile.tuning).toEqual({ + contextWindow: 262_144, + maxTokens: 8192, + reasoning: false, + reasoningEffort: "default", + }); + expect(built.profile.proxy).toMatchObject({ + managedHost: "host.containers.internal", + managedPort: 3129, + hostHttpUrl: "http://proxy.example.test:8080", + }); + }); +}); diff --git a/src/lib/onboard/managed-startup-profile-builder.test.ts b/src/lib/onboard/managed-startup-profile-builder.test.ts new file mode 100644 index 0000000000..4bf45fe249 --- /dev/null +++ b/src/lib/onboard/managed-startup-profile-builder.test.ts @@ -0,0 +1,567 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { Buffer } from "node:buffer"; +import { createHash } from "node:crypto"; + +import { describe, expect, it } from "vitest"; + +import { PEM } from "./__test-helpers__/corporate-ca-fixtures"; +import { decodeManagedStartupProfile } from "./managed-startup/profile"; +import { + assertManagedStartupProfileBuilderInventoryCoverage, + buildManagedStartupProfile, + type ManagedStartupProfileBuilderInput, +} from "./managed-startup/profile-builder"; + +function messagingPlan(agent: "openclaw" | "hermes") { + return { + schemaVersion: 1, + sandboxName: "portable-agent", + agent, + workflow: "onboard", + channels: [], + disabledChannels: [], + credentialBindings: [], + networkPolicy: { presets: [], entries: [] }, + agentRender: [], + buildSteps: [], + stateUpdates: [], + healthChecks: [], + } as const; +} + +function encodeJson(value: unknown): string { + return Buffer.from(JSON.stringify(value), "utf8").toString("base64"); +} + +function openClawInput( + overrides: Partial = {}, +): ManagedStartupProfileBuilderInput { + return { + agent: "openclaw", + inference: { + routeProvider: "openai", + upstreamProvider: "openai-api", + model: "gpt-5.4", + routedBaseUrl: "https://inference.local/v1", + upstreamEndpointUrl: null, + api: "openai-responses", + primaryModelRef: "openai/gpt-5.4", + compatibility: {}, + }, + dashboard: { + agent: "openclaw", + mode: "loopback", + url: "http://127.0.0.1:18789", + port: 18_789, + bindAddress: "127.0.0.1", + wslExposure: false, + }, + webSearch: null, + toolDisclosure: "progressive", + hermesToolGateways: [], + messagingPlan: null, + dcodeAutoApprovalMode: null, + observabilityEnabled: null, + environment: {}, + corporateCa: null, + ...overrides, + }; +} + +function hermesInput( + overrides: Partial = {}, +): ManagedStartupProfileBuilderInput { + return { + agent: "hermes", + inference: { + routeProvider: "inference", + upstreamProvider: "compatible-anthropic-endpoint", + model: "claude-sonnet-4-6", + routedBaseUrl: "https://inference.local/v1", + upstreamEndpointUrl: null, + api: "openai-completions", + primaryModelRef: null, + compatibility: null, + }, + dashboard: { + agent: "hermes", + mode: "disabled", + url: "http://127.0.0.1:18789", + publicPort: null, + internalPort: null, + tuiEnabled: false, + }, + webSearch: null, + toolDisclosure: "progressive", + hermesToolGateways: [], + messagingPlan: null, + dcodeAutoApprovalMode: null, + observabilityEnabled: null, + environment: {}, + corporateCa: null, + ...overrides, + }; +} + +function dcodeInput( + overrides: Partial = {}, +): ManagedStartupProfileBuilderInput { + return { + agent: "langchain-deepagents-code", + inference: { + routeProvider: "inference", + upstreamProvider: "openrouter", + model: "openai/gpt-5.4", + routedBaseUrl: "https://inference.local/v1", + upstreamEndpointUrl: "https://openrouter.ai/api/v1", + api: "openai-completions", + primaryModelRef: null, + compatibility: null, + }, + dashboard: { + agent: "langchain-deepagents-code", + mode: "disabled", + }, + webSearch: null, + toolDisclosure: "progressive", + hermesToolGateways: [], + messagingPlan: null, + dcodeAutoApprovalMode: "disabled", + observabilityEnabled: false, + environment: {}, + corporateCa: null, + ...overrides, + }; +} + +describe("buildManagedStartupProfile", () => { + it("builds OpenClaw with every stock behavior knob and canonical transport", () => { + const plan = messagingPlan("openclaw"); + const extraAgents = { + agents: [{ id: "reviewer", workspace: "/sandbox/reviewer" }], + defaults: { subagents: { maxSpawnDepth: 2 } }, + main: { tools: { profile: "coding" } }, + }; + const input = openClawInput({ + dashboard: { + agent: "openclaw", + mode: "remote", + url: "https://dashboard.example.test:19443", + port: 19_443, + bindAddress: "0.0.0.0", + wslExposure: true, + }, + webSearch: { fetchEnabled: true, provider: "tavily" }, + toolDisclosure: "direct", + messagingPlan: plan, + environment: { + NEMOCLAW_MODEL: "gpt-5.4", + NEMOCLAW_INFERENCE_PROVIDER_ID: "openai", + NEMOCLAW_UPSTREAM_PROVIDER: "openai-api", + NEMOCLAW_PRIMARY_MODEL_REF: "openai/gpt-5.4", + NEMOCLAW_INFERENCE_BASE_URL: "https://inference.local/v1", + NEMOCLAW_INFERENCE_API: "openai-responses", + NEMOCLAW_INFERENCE_COMPAT_B64: encodeJson({}), + NEMOCLAW_INFERENCE_INPUTS: "text,image", + NEMOCLAW_CONTEXT_WINDOW: "262144", + NEMOCLAW_MAX_TOKENS: "8192", + NEMOCLAW_REASONING: "true", + NEMOCLAW_REASONING_EFFORT: " HIGH ", + NEMOCLAW_TOOL_DISCLOSURE: "direct", + NEMOCLAW_AGENT_TIMEOUT: "900", + NEMOCLAW_AGENT_HEARTBEAT_EVERY: "30m", + NEMOCLAW_EXTRA_AGENTS_JSON: JSON.stringify(extraAgents), + NEMOCLAW_DISABLE_DEVICE_AUTH: "1", + NEMOCLAW_DEVICE_AUTH_OPT_OUT_SOURCE: "managed-onboard", + NEMOCLAW_WEB_SEARCH_ENABLED: "1", + NEMOCLAW_WEB_SEARCH_PROVIDER: "tavily", + NEMOCLAW_OPENCLAW_OTEL: "yes", + NEMOCLAW_OPENCLAW_OTEL_ENDPOINT: "https://otel.example.test:4318", + NEMOCLAW_OPENCLAW_OTEL_SERVICE_NAME: "nemoclaw-openclaw", + NEMOCLAW_OPENCLAW_OTEL_SAMPLE_RATE: "0.25", + CHAT_UI_URL: "https://dashboard.example.test:19443", + NEMOCLAW_DASHBOARD_BIND: "0.0.0.0", + NEMOCLAW_WSL_DASHBOARD_EXPOSURE: "1", + NEMOCLAW_DASHBOARD_PORT: "19443", + NEMOCLAW_PROXY_HOST: "host.containers.internal", + NEMOCLAW_PROXY_PORT: "3129", + NEMOCLAW_MESSAGING_PLAN_B64: encodeJson(plan), + NEMOCLAW_MINIMAL_BOOTSTRAP: "1", + HTTP_PROXY: "http://proxy.example.test:8080", + http_proxy: "http://proxy.example.test:8080", + HTTPS_PROXY: "https://connect.example.test:8443", + https_proxy: "https://connect.example.test:8443", + NO_PROXY: "metadata.example.test", + no_proxy: "metadata.example.test", + }, + }); + + const built = buildManagedStartupProfile(input); + + expect(built.profile.agent).toBe("openclaw"); + expect(built.profile.inference).toMatchObject({ + routeProvider: "openai", + upstreamProvider: "openai-api", + model: "gpt-5.4", + api: "openai-responses", + primaryModelRef: "openai/gpt-5.4", + compatibility: {}, + inputModalities: ["image", "text"], + }); + expect(built.profile.agentConfig).toMatchObject({ + agent: "openclaw", + webSearch: { enabled: true, provider: "tavily" }, + otel: { + enabled: true, + endpointUrl: "https://otel.example.test:4318", + serviceName: "nemoclaw-openclaw", + sampleRate: 0.25, + }, + agentTimeoutSeconds: 900, + heartbeatEvery: "30m", + extraAgents, + deviceAuth: { disabled: true, optOutSource: "managed-onboard" }, + minimalBootstrap: true, + }); + expect(built.profile.proxy).toMatchObject({ + managedHost: "host.containers.internal", + managedPort: 3129, + hostHttpUrl: "http://proxy.example.test:8080", + hostHttpsUrl: "https://connect.example.test:8443", + }); + expect(built.profile.proxy.hostNoProxy).toEqual( + expect.arrayContaining([ + "metadata.example.test", + "localhost", + "host.docker.internal", + "host.containers.internal", + "inference.local", + ]), + ); + expect(built.profile.messaging.plan).not.toBeNull(); + expect(built.profile.tuning).toEqual({ + contextWindow: 262_144, + maxTokens: 8192, + reasoning: true, + reasoningEffort: "high", + }); + expect(built.corporateCaB64).toBeUndefined(); + expect(decodeManagedStartupProfile(built.encodedProfile)).toEqual(built.profile); + expect(built.startupProfileSha256).toBe( + createHash("sha256").update(built.encodedProfile, "utf8").digest("hex"), + ); + }); + + it("builds Hermes with Tavily, gateway presets, messaging, context, and forwarding", () => { + const plan = messagingPlan("hermes"); + const gateways = ["nous-web", "nous-image"] as const; + const built = buildManagedStartupProfile( + hermesInput({ + dashboard: { + agent: "hermes", + mode: "loopback-forwarded", + url: "http://127.0.0.1:19189", + publicPort: 19_189, + internalPort: 29_189, + tuiEnabled: true, + }, + webSearch: { fetchEnabled: true, provider: "tavily" }, + toolDisclosure: "direct", + hermesToolGateways: gateways, + messagingPlan: plan, + environment: { + NEMOCLAW_MODEL: "claude-sonnet-4-6", + NEMOCLAW_INFERENCE_PROVIDER_ID: "inference", + NEMOCLAW_UPSTREAM_PROVIDER: "compatible-anthropic-endpoint", + NEMOCLAW_INFERENCE_BASE_URL: "https://inference.local/v1", + NEMOCLAW_INFERENCE_API: "openai-completions", + NEMOCLAW_CONTEXT_WINDOW: "65536", + NEMOCLAW_TOOL_DISCLOSURE: "direct", + NEMOCLAW_HERMES_TOOL_GATEWAY_BROKER: "1", + NEMOCLAW_HERMES_TOOL_GATEWAY_PRESETS_B64: encodeJson(gateways), + NEMOCLAW_WEB_SEARCH_ENABLED: "1", + NEMOCLAW_WEB_SEARCH_PROVIDER: "tavily", + NEMOCLAW_MESSAGING_PLAN_B64: encodeJson(plan), + CHAT_UI_URL: "http://127.0.0.1:19189", + NEMOCLAW_DASHBOARD_PORT: "19189", + NEMOCLAW_HERMES_DASHBOARD: "true", + NEMOCLAW_HERMES_DASHBOARD_PORT: "19189", + NEMOCLAW_HERMES_DASHBOARD_INTERNAL_PORT: "29189", + NEMOCLAW_HERMES_DASHBOARD_TUI: "yes", + NEMOCLAW_PROXY_HOST: "10.200.0.1", + NEMOCLAW_PROXY_PORT: "3128", + HTTP_PROXY: "http://proxy.example.test:8080", + NO_PROXY: "internal.example.test", + }, + }), + ); + + expect(built.profile.agentConfig).toEqual({ + agent: "hermes", + webSearch: { enabled: true, provider: "tavily" }, + }); + expect(built.profile.inference).toMatchObject({ + routeProvider: "inference", + upstreamProvider: "compatible-anthropic-endpoint", + api: "openai-completions", + primaryModelRef: null, + compatibility: null, + inputModalities: null, + }); + expect(built.profile.dashboard).toEqual({ + agent: "hermes", + mode: "loopback-forwarded", + url: "http://127.0.0.1:19189", + publicPort: 19_189, + internalPort: 29_189, + tuiEnabled: true, + }); + expect(built.profile.tools).toEqual({ + disclosure: "direct", + enabledGateways: ["nous-image", "nous-web"], + }); + expect(built.profile.tuning).toEqual({ + contextWindow: 65_536, + maxTokens: null, + reasoning: null, + reasoningEffort: null, + }); + expect(decodeManagedStartupProfile(built.encodedProfile)).toEqual(built.profile); + }); + + it("builds DCode with its direct upstream, approval, and observability contract", () => { + const built = buildManagedStartupProfile( + dcodeInput({ + dcodeAutoApprovalMode: "thread-opt-in", + observabilityEnabled: true, + toolDisclosure: "direct", + environment: { + NEMOCLAW_MODEL: "openai/gpt-5.4", + NEMOCLAW_INFERENCE_PROVIDER_ID: "inference", + NEMOCLAW_UPSTREAM_PROVIDER: "openrouter", + NEMOCLAW_UPSTREAM_ENDPOINT_URL: "https://openrouter.ai/api/v1", + NEMOCLAW_INFERENCE_BASE_URL: "https://inference.local/v1", + NEMOCLAW_INFERENCE_API: "openai-completions", + NEMOCLAW_TOOL_DISCLOSURE: "direct", + NEMOCLAW_DCODE_AUTO_APPROVAL: "thread-opt-in", + NEMOCLAW_PROXY_HOST: "10.200.0.1", + NEMOCLAW_PROXY_PORT: "3128", + NEMOCLAW_OBSERVABILITY: "1", + }, + }), + ); + + expect(built.profile).toMatchObject({ + agent: "langchain-deepagents-code", + agentConfig: { + agent: "langchain-deepagents-code", + autoApprovalMode: "thread-opt-in", + observabilityEnabled: true, + }, + inference: { + routeProvider: "inference", + upstreamProvider: "openrouter", + upstreamEndpointUrl: "https://openrouter.ai/api/v1", + api: "openai-completions", + primaryModelRef: null, + compatibility: null, + inputModalities: null, + }, + proxy: { + managedHost: "10.200.0.1", + managedPort: 3128, + hostHttpUrl: null, + hostHttpsUrl: null, + hostNoProxy: [], + }, + dashboard: { + agent: "langchain-deepagents-code", + mode: "disabled", + }, + messaging: { plan: null }, + tuning: { + contextWindow: null, + maxTokens: null, + reasoning: null, + reasoningEffort: null, + }, + }); + expect(decodeManagedStartupProfile(built.encodedProfile)).toEqual(built.profile); + }); + + it("keeps validated corporate CA bytes out of the profile and returns a digest-bound transport", () => { + const built = buildManagedStartupProfile( + openClawInput({ + corporateCa: { + pem: PEM, + sourcePath: "/public/corporate-ca.pem", + sourceEnv: "NEMOCLAW_CORPORATE_CA_BUNDLE", + }, + }), + ); + + const normalizedPem = PEM.endsWith("\n") ? PEM : `${PEM}\n`; + expect(built.corporateCaB64).toBe(Buffer.from(normalizedPem, "utf8").toString("base64")); + expect(built.profile.corporateCa.bundleSha256).toBe( + createHash("sha256").update(normalizedPem, "utf8").digest("hex"), + ); + const decodedProfile = Buffer.from(built.encodedProfile, "base64url").toString("utf8"); + expect(decodedProfile).not.toContain("BEGIN CERTIFICATE"); + expect(decodedProfile).not.toContain("/public/corporate-ca.pem"); + }); + + it("uses managed OpenClaw defaults without relying on Dockerfile defaults", () => { + const built = buildManagedStartupProfile(openClawInput()); + + expect(built.profile.agentConfig).toMatchObject({ + agent: "openclaw", + webSearch: { enabled: false, provider: "brave" }, + otel: { + enabled: false, + endpointUrl: "http://host.openshell.internal:4318", + serviceName: "openclaw-gateway", + sampleRate: 1, + }, + agentTimeoutSeconds: 600, + heartbeatEvery: null, + extraAgents: { + agents: [], + defaults: { subagents: {} }, + main: {}, + }, + deviceAuth: { disabled: true, optOutSource: "managed-onboard" }, + minimalBootstrap: false, + }); + expect(built.profile.tuning).toEqual({ + contextWindow: 131_072, + maxTokens: 4096, + reasoning: false, + reasoningEffort: "default", + }); + }); + + it("retains Hermes Tavily selection while web search is disabled", () => { + const built = buildManagedStartupProfile(hermesInput()); + expect(built.profile.agentConfig).toEqual({ + agent: "hermes", + webSearch: { enabled: false, provider: "tavily" }, + }); + }); + + it.each([ + ["DCode messaging", dcodeInput({ messagingPlan: messagingPlan("openclaw") }), /messagingPlan/], + [ + "DCode web search", + dcodeInput({ webSearch: { fetchEnabled: false, provider: "brave" } }), + /web-search/, + ], + [ + "OpenClaw Hermes gateways", + openClawInput({ hermesToolGateways: ["nous-web"] }), + /another agent/, + ], + [ + "Hermes Brave", + hermesInput({ webSearch: { fetchEnabled: true, provider: "brave" } }), + /only the Tavily/, + ], + [ + "Hermes OpenClaw OTEL knob", + hermesInput({ environment: { NEMOCLAW_OPENCLAW_OTEL: "1" } }), + /not supported by hermes/, + ], + ])("rejects unsupported cross-agent intent: %s", (_label, input, message) => { + expect(() => buildManagedStartupProfile(input)).toThrow(message); + }); + + it.each([ + [ + "oversized context", + openClawInput({ environment: { NEMOCLAW_CONTEXT_WINDOW: "9999999999" } }), + /NEMOCLAW_CONTEXT_WINDOW/, + ], + [ + "malformed reasoning", + openClawInput({ environment: { NEMOCLAW_REASONING: "sometimes" } }), + /NEMOCLAW_REASONING/, + ], + [ + "malformed reasoning effort", + openClawInput({ environment: { NEMOCLAW_REASONING_EFFORT: "maximum" } }), + /NEMOCLAW_REASONING_EFFORT/, + ], + [ + "conflicting proxy aliases", + openClawInput({ + environment: { + HTTP_PROXY: "http://one.example.test:8080", + http_proxy: "http://two.example.test:8080", + }, + }), + /conflicting values/, + ], + [ + "bare no-proxy intent", + openClawInput({ environment: { NO_PROXY: "localhost" } }), + /requires an HTTP_PROXY or HTTPS_PROXY/, + ], + [ + "raw CA transport", + openClawInput({ environment: { NEMOCLAW_CORPORATE_CA_B64: "ZmFrZQ==" } }), + /separate corporateCa input/, + ], + [ + "conflicting semantic model", + openClawInput({ environment: { NEMOCLAW_MODEL: "another-model" } }), + /conflicts with the resolved semantic/, + ], + ])("fails closed for malformed or conflicting stock input: %s", (_label, input, message) => { + expect(() => buildManagedStartupProfile(input)).toThrow(message); + }); + + it.each([ + openClawInput({ + environment: { HTTP_PROXY: "http://operator:password@proxy.example.test:8080" }, + }), + openClawInput({ + inference: { + ...openClawInput().inference, + model: "sk-proj-secret-material-1234567890", + }, + }), + openClawInput({ + environment: { + NEMOCLAW_EXTRA_AGENTS_JSON: JSON.stringify({ + agents: [ + { + id: "reviewer", + api_key: ["sk", "secret", "material", "1234567890"].join("-"), + }, + ], + }), + }, + }), + ])("rejects secret material instead of serializing it", (input) => { + expect(() => buildManagedStartupProfile(input)).toThrow(); + }); + + it("rejects malformed or non-CA certificate material", () => { + expect(() => + buildManagedStartupProfile( + openClawInput({ + corporateCa: { + pem: "-----BEGIN CERTIFICATE-----\nMIIBfake\n-----END CERTIFICATE-----\n", + sourcePath: "/public/bad.pem", + sourceEnv: "test", + }, + }), + ), + ).toThrow(/invalid X.509/); + }); + + it("audits the exact all-agent affordance inventory", () => { + expect(() => assertManagedStartupProfileBuilderInventoryCoverage()).not.toThrow(); + }); +}); diff --git a/src/lib/onboard/managed-startup-profile.test.ts b/src/lib/onboard/managed-startup-profile.test.ts new file mode 100644 index 0000000000..a9a73eeff0 --- /dev/null +++ b/src/lib/onboard/managed-startup-profile.test.ts @@ -0,0 +1,759 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { Buffer } from "node:buffer"; +import { createHash } from "node:crypto"; +import { readFileSync } from "node:fs"; + +import { describe, expect, it } from "vitest"; + +import { + decodeManagedStartupProfile, + encodeManagedStartupProfile, + fingerprintManagedStartupProfile, + MANAGED_STARTUP_AGENTS, + MANAGED_STARTUP_PROFILE_AFFORDANCE_INVENTORY, + MANAGED_STARTUP_PROFILE_CAPABILITIES, + MANAGED_STARTUP_PROFILE_EXCLUDED_DOCKER_INPUTS, + MANAGED_STARTUP_PROFILE_MAX_BYTES, + MANAGED_STARTUP_PROFILE_SCHEMA_VERSION, + type ManagedStartupAgent, + type ManagedStartupProfile, + serializeManagedStartupProfile, + validateManagedStartupProfile, +} from "./managed-startup/profile"; + +const CA_SHA256 = "a".repeat(64); + +const MESSAGING_PLAN = { + schemaVersion: 1, + sandboxName: "demo", + agent: "portable", + workflow: "onboard", + channels: [], + disabledChannels: [], + credentialBindings: [ + { + credentialId: "slackBotToken", + providerEnvKey: "SLACK_BOT_TOKEN", + placeholder: "openshell:resolve:env:SLACK_BOT_TOKEN", + credentialAvailable: true, + }, + ], + networkPolicy: { presets: [], entries: [] }, + agentRender: [], + buildSteps: [], + stateUpdates: [], + healthChecks: [], +} as const; + +const OPENCLAW_PROFILE = { + schemaVersion: MANAGED_STARTUP_PROFILE_SCHEMA_VERSION, + agent: "openclaw", + agentConfig: { + agent: "openclaw", + webSearch: { enabled: true, provider: "brave" }, + otel: { + enabled: true, + endpointUrl: "http://host.openshell.internal:4318", + serviceName: "openclaw-gateway", + sampleRate: 0.75, + }, + agentTimeoutSeconds: 900, + heartbeatEvery: "30m", + extraAgents: { + agents: [ + { + id: "reviewer", + workspace: "/sandbox/reviewer", + model: "inference/nvidia/nemotron-3-ultra-550b-a55b", + }, + ], + defaults: { subagents: { maxSpawnDepth: 3 } }, + main: { tools: { profile: "coding" } }, + }, + deviceAuth: { disabled: true, optOutSource: "managed-onboard" }, + minimalBootstrap: true, + }, + inference: { + routeProvider: "inference", + upstreamProvider: "nvidia-prod", + model: "nvidia/nemotron-3-ultra-550b-a55b", + routedBaseUrl: "https://inference.local/v1", + upstreamEndpointUrl: null, + api: "openai-responses", + primaryModelRef: "inference/nvidia/nemotron-3-ultra-550b-a55b", + compatibility: { supportsDeveloperRole: true, maxRetries: 2 }, + inputModalities: ["text", "image"], + }, + proxy: { + managedHost: "10.200.0.1", + managedPort: 3128, + hostHttpUrl: "http://proxy.example.test:8080", + hostHttpsUrl: "http://connect-proxy.example.test:3128", + hostNoProxy: ["inference.local", "127.0.0.1", "localhost"], + }, + dashboard: { + agent: "openclaw", + mode: "remote", + url: "https://dashboard.example.test:18789", + port: 18_789, + bindAddress: "0.0.0.0", + wslExposure: true, + }, + tools: { + disclosure: "progressive", + enabledGateways: [], + }, + messaging: { plan: MESSAGING_PLAN }, + tuning: { + contextWindow: 131_072, + maxTokens: 8192, + reasoning: true, + reasoningEffort: "high", + }, + corporateCa: { bundleSha256: CA_SHA256 }, +} as const satisfies ManagedStartupProfile; + +const HERMES_PROFILE = { + schemaVersion: MANAGED_STARTUP_PROFILE_SCHEMA_VERSION, + agent: "hermes", + agentConfig: { + agent: "hermes", + webSearch: { enabled: true, provider: "tavily" }, + }, + inference: { + routeProvider: "custom", + upstreamProvider: "anthropic-prod", + model: "claude-sonnet-4-5", + routedBaseUrl: "https://inference.local/v1", + upstreamEndpointUrl: null, + api: "anthropic-messages", + primaryModelRef: null, + compatibility: null, + inputModalities: null, + }, + proxy: { + managedHost: "10.200.0.1", + managedPort: 3128, + hostHttpUrl: "http://proxy.example.test:8080", + hostHttpsUrl: "https://proxy.example.test:8443", + hostNoProxy: ["localhost", "127.0.0.1"], + }, + dashboard: { + agent: "hermes", + mode: "loopback-forwarded", + url: "http://127.0.0.1:19189", + publicPort: 19_189, + internalPort: 29_189, + tuiEnabled: true, + }, + tools: { + disclosure: "direct", + enabledGateways: ["nous-web", "nous-image", "nous-audio", "nous-browser", "nous-code"], + }, + messaging: { plan: MESSAGING_PLAN }, + tuning: { + contextWindow: 65_536, + maxTokens: null, + reasoning: null, + reasoningEffort: null, + }, + corporateCa: { bundleSha256: CA_SHA256 }, +} as const satisfies ManagedStartupProfile; + +const DCODE_PROFILE = { + schemaVersion: MANAGED_STARTUP_PROFILE_SCHEMA_VERSION, + agent: "langchain-deepagents-code", + agentConfig: { + agent: "langchain-deepagents-code", + autoApprovalMode: "thread-opt-in", + observabilityEnabled: true, + }, + inference: { + routeProvider: "inference", + upstreamProvider: "openrouter", + model: "openai/gpt-5.4", + routedBaseUrl: "https://inference.local/v1", + upstreamEndpointUrl: "https://openrouter.ai/api/v1", + api: "openai-completions", + primaryModelRef: null, + compatibility: null, + inputModalities: null, + }, + proxy: { + managedHost: "10.200.0.1", + managedPort: 3128, + hostHttpUrl: null, + hostHttpsUrl: null, + hostNoProxy: [], + }, + dashboard: { + agent: "langchain-deepagents-code", + mode: "disabled", + }, + tools: { + disclosure: "progressive", + enabledGateways: [], + }, + messaging: { plan: null }, + tuning: { + contextWindow: null, + maxTokens: null, + reasoning: null, + reasoningEffort: null, + }, + corporateCa: { bundleSha256: CA_SHA256 }, +} as const satisfies ManagedStartupProfile; + +const VALID_PROFILES = [OPENCLAW_PROFILE, HERMES_PROFILE, DCODE_PROFILE] as const; + +function encodeUnknown(value: unknown): string { + return Buffer.from(JSON.stringify(value), "utf8").toString("base64url"); +} + +function dockerArgs(relativePath: string): Set { + const source = readFileSync(relativePath, "utf8"); + return new Set( + [...source.matchAll(/^ARG\s+([A-Z][A-Z0-9_]*)/gmu)].map((match) => match[1] as string), + ); +} + +describe("managed startup profile", () => { + it.each( + VALID_PROFILES, + )("round-trips every $agent affordance without secret material", (profile) => { + const validated = validateManagedStartupProfile(profile); + const encoded = encodeManagedStartupProfile(profile); + + expect(decodeManagedStartupProfile(encoded)).toEqual(validated); + expect(encoded).not.toContain(profile.inference.model); + expect(fingerprintManagedStartupProfile(profile)).toMatch(/^[a-f0-9]{64}$/); + }); + + it("round-trips every OpenClaw-only startup knob", () => { + const profile = decodeManagedStartupProfile(encodeManagedStartupProfile(OPENCLAW_PROFILE)); + expect(profile).toMatchObject({ + agentConfig: { + webSearch: { enabled: true, provider: "brave" }, + otel: { + enabled: true, + endpointUrl: "http://host.openshell.internal:4318", + serviceName: "openclaw-gateway", + sampleRate: 0.75, + }, + agentTimeoutSeconds: 900, + heartbeatEvery: "30m", + minimalBootstrap: true, + deviceAuth: { disabled: true, optOutSource: "managed-onboard" }, + }, + inference: { + api: "openai-responses", + primaryModelRef: "inference/nvidia/nemotron-3-ultra-550b-a55b", + compatibility: { supportsDeveloperRole: true, maxRetries: 2 }, + inputModalities: ["image", "text"], + }, + dashboard: { + mode: "remote", + bindAddress: "0.0.0.0", + wslExposure: true, + }, + tuning: { + contextWindow: 131_072, + maxTokens: 8192, + reasoning: true, + reasoningEffort: "high", + }, + }); + expect(profile.agentConfig.agent === "openclaw" && profile.agentConfig.extraAgents).toEqual( + OPENCLAW_PROFILE.agentConfig.extraAgents, + ); + }); + + it("round-trips Hermes forwarding, reviewed gateways, messaging, and context tuning", () => { + const profile = decodeManagedStartupProfile(encodeManagedStartupProfile(HERMES_PROFILE)); + expect(profile).toMatchObject({ + dashboard: { + mode: "loopback-forwarded", + publicPort: 19_189, + internalPort: 29_189, + tuiEnabled: true, + }, + agentConfig: { webSearch: { enabled: true, provider: "tavily" } }, + tuning: { + contextWindow: 65_536, + maxTokens: null, + reasoning: null, + reasoningEffort: null, + }, + }); + expect(profile.tools.enabledGateways).toEqual([ + "nous-audio", + "nous-browser", + "nous-code", + "nous-image", + "nous-web", + ]); + expect(profile.messaging.plan).not.toBeNull(); + }); + + it("round-trips DCode upstream metadata, trusted proxy, approval, and observability", () => { + const profile = decodeManagedStartupProfile(encodeManagedStartupProfile(DCODE_PROFILE)); + expect(profile).toMatchObject({ + inference: { + routeProvider: "inference", + upstreamProvider: "openrouter", + upstreamEndpointUrl: "https://openrouter.ai/api/v1", + routedBaseUrl: "https://inference.local/v1", + api: "openai-completions", + }, + proxy: { + managedHost: "10.200.0.1", + managedPort: 3128, + hostHttpUrl: null, + hostHttpsUrl: null, + hostNoProxy: [], + }, + agentConfig: { + autoApprovalMode: "thread-opt-in", + observabilityEnabled: true, + }, + dashboard: { mode: "disabled" }, + }); + }); + + it("canonicalizes object keys and set-like lists before fingerprinting", () => { + const reordered = { + ...HERMES_PROFILE, + proxy: { + ...HERMES_PROFILE.proxy, + hostNoProxy: [...HERMES_PROFILE.proxy.hostNoProxy].reverse(), + }, + tools: { + ...HERMES_PROFILE.tools, + enabledGateways: [...HERMES_PROFILE.tools.enabledGateways].reverse(), + }, + }; + const serialized = serializeManagedStartupProfile(HERMES_PROFILE); + + expect(serializeManagedStartupProfile(reordered)).toBe(serialized); + expect(fingerprintManagedStartupProfile(reordered)).toBe( + createHash("sha256").update(serialized, "utf8").digest("hex"), + ); + }); + + it("exports complete, fail-closed capabilities for every supported agent", () => { + expect(Object.keys(MANAGED_STARTUP_PROFILE_CAPABILITIES).sort()).toEqual( + [...MANAGED_STARTUP_AGENTS].sort(), + ); + expect(MANAGED_STARTUP_PROFILE_CAPABILITIES.openclaw.dashboardModes).toEqual([ + "loopback", + "remote", + ]); + expect(MANAGED_STARTUP_PROFILE_CAPABILITIES.hermes.dashboardModes).toEqual([ + "disabled", + "loopback-forwarded", + ]); + expect(MANAGED_STARTUP_PROFILE_CAPABILITIES.hermes.inputModalities).toEqual([]); + expect(MANAGED_STARTUP_PROFILE_CAPABILITIES["langchain-deepagents-code"].inferenceApis).toEqual( + ["openai-completions"], + ); + expect( + MANAGED_STARTUP_PROFILE_CAPABILITIES["langchain-deepagents-code"].inputModalities, + ).toEqual([]); + }); + + it.each([ + ["openclaw", "Dockerfile"], + ["hermes", "agents/hermes/Dockerfile"], + ["langchain-deepagents-code", "agents/langchain-deepagents-code/Dockerfile"], + ] as const)("classifies every stock %s Docker ARG as startup-affordance or deliberate exclusion", (agent, dockerfile) => { + const classified = new Set([ + ...MANAGED_STARTUP_PROFILE_AFFORDANCE_INVENTORY[agent].map(({ input }) => input), + ...MANAGED_STARTUP_PROFILE_EXCLUDED_DOCKER_INPUTS[agent].map(({ input }) => input), + ]); + expect([...dockerArgs(dockerfile)].filter((input) => !classified.has(input))).toEqual([]); + }); + + it.each(MANAGED_STARTUP_AGENTS)("keeps the %s affordance inventory unambiguous", (agent) => { + const inventory = MANAGED_STARTUP_PROFILE_AFFORDANCE_INVENTORY[agent]; + const uniqueMappings = new Set( + inventory.map(({ input, profilePath, source }) => `${source}:${input}:${profilePath}`), + ); + expect(uniqueMappings.size).toBe(inventory.length); + expect(inventory.every(({ profilePath }) => !profilePath.startsWith("env."))).toBe(true); + }); + + it.each( + VALID_PROFILES, + )("maps every $agent inventory entry to an explicit profile field", (profile) => { + for (const { profilePath } of MANAGED_STARTUP_PROFILE_AFFORDANCE_INVENTORY[profile.agent]) { + let current: unknown = profile; + for (const segment of profilePath.split(".")) { + expect(current).not.toBeNull(); + expect(typeof current).toBe("object"); + expect(Object.hasOwn(current as object, segment)).toBe(true); + current = (current as Record)[segment]; + } + } + }); + + it("rejects non-canonical transports instead of accepting ambiguous fingerprints", () => { + const raw = JSON.stringify(OPENCLAW_PROFILE); + expect(() => decodeManagedStartupProfile(Buffer.from(raw).toString("base64url"))).toThrow( + /canonical form/, + ); + }); + + it.each([ + { + label: "top level", + mutate: (profile: ManagedStartupProfile) => ({ ...profile, extension: true }), + }, + { + label: "inference", + mutate: (profile: ManagedStartupProfile) => ({ + ...profile, + inference: { ...profile.inference, extension: true }, + }), + }, + { + label: "proxy", + mutate: (profile: ManagedStartupProfile) => ({ + ...profile, + proxy: { ...profile.proxy, extension: true }, + }), + }, + { + label: "dashboard", + mutate: (profile: ManagedStartupProfile) => ({ + ...profile, + dashboard: { ...profile.dashboard, extension: true }, + }), + }, + { + label: "messaging wrapper", + mutate: (profile: ManagedStartupProfile) => ({ + ...profile, + messaging: { ...profile.messaging, extension: true }, + }), + }, + { + label: "agent config", + mutate: (profile: ManagedStartupProfile) => ({ + ...profile, + agentConfig: { ...profile.agentConfig, extension: true }, + }), + }, + { + label: "CA digest", + mutate: (profile: ManagedStartupProfile) => ({ + ...profile, + corporateCa: { ...profile.corporateCa, extension: true }, + }), + }, + ])("rejects recursively unknown keys at $label", ({ mutate }) => { + expect(() => validateManagedStartupProfile(mutate(OPENCLAW_PROFILE))).toThrow( + /unsupported fields/, + ); + }); + + it.each([ + ["agentConfig", { ...OPENCLAW_PROFILE, agentConfig: HERMES_PROFILE.agentConfig }], + ["dashboard", { ...OPENCLAW_PROFILE, dashboard: HERMES_PROFILE.dashboard }], + ])("rejects a mismatched %s agent discriminator", (_label, profile) => { + expect(() => validateManagedStartupProfile(profile)).toThrow(/must match agent/); + }); + + it.each([ + ["credential-named compatibility field", { accessToken: "not-even-a-real-token" }], + [ + "credential-named field hidden under a non-messaging plan", + { plan: { accessToken: "not-even-a-real-token" } }, + ], + ["provider token", { note: `nvapi-${"a".repeat(32)}` }], + ["bearer value", { note: `Bearer ${"a".repeat(32)}` }], + [ + "private key", + { + note: `-----BEGIN ${"PRIVATE"} KEY-----\nabc\n-----END ${"PRIVATE"} KEY-----`, + }, + ], + ])("rejects %s anywhere in the profile", (_label, compatibility) => { + expect(() => + validateManagedStartupProfile({ + ...OPENCLAW_PROFILE, + inference: { ...OPENCLAW_PROFILE.inference, compatibility }, + }), + ).toThrow(/credential-shaped/); + }); + + it("rejects raw credentials nested inside an otherwise opaque messaging plan", () => { + expect(() => + validateManagedStartupProfile({ + ...OPENCLAW_PROFILE, + messaging: { + plan: { + ...OPENCLAW_PROFILE.messaging.plan, + credentialBindings: [ + { + providerEnvKey: "SLACK_BOT_TOKEN", + value: `xoxb-${"a".repeat(32)}`, + }, + ], + }, + }, + }), + ).toThrow(/credential-shaped string data/); + }); + + it.each([ + ["routed inference", "inference", "routedBaseUrl"], + ["upstream inference", "inference", "upstreamEndpointUrl"], + ["OTEL", "otel", "endpointUrl"], + ["dashboard", "dashboard", "url"], + ] as const)("rejects credentials embedded in the %s URL", (_label, scope, field) => { + const profile = + scope === "inference" + ? { + ...DCODE_PROFILE, + inference: { + ...DCODE_PROFILE.inference, + [field]: "https://user:password@example.test/v1", + }, + } + : scope === "otel" + ? { + ...OPENCLAW_PROFILE, + agentConfig: { + ...OPENCLAW_PROFILE.agentConfig, + otel: { + ...OPENCLAW_PROFILE.agentConfig.otel, + [field]: "https://user:password@example.test/v1", + }, + }, + } + : { + ...OPENCLAW_PROFILE, + dashboard: { + ...OPENCLAW_PROFILE.dashboard, + [field]: "https://user:password@example.test/v1", + }, + }; + expect(() => validateManagedStartupProfile(profile)).toThrow( + /embedded credentials|credential-free/, + ); + }); + + it("accepts an HTTP CONNECT origin for host HTTPS proxy intent", () => { + expect(validateManagedStartupProfile(OPENCLAW_PROFILE).proxy.hostHttpsUrl).toBe( + "http://connect-proxy.example.test:3128", + ); + }); + + it("keeps DCode messaging, dashboards, and tuning fail-closed while retaining host proxy intent", () => { + expect(() => + validateManagedStartupProfile({ + ...DCODE_PROFILE, + messaging: { plan: { schemaVersion: 1 } }, + }), + ).toThrow(/messaging\.plan must be null/); + expect( + validateManagedStartupProfile({ + ...DCODE_PROFILE, + proxy: { ...DCODE_PROFILE.proxy, hostHttpUrl: "http://proxy.example.test:8080" }, + }).proxy.hostHttpUrl, + ).toBe("http://proxy.example.test:8080"); + expect(() => + validateManagedStartupProfile({ + ...DCODE_PROFILE, + tuning: { ...DCODE_PROFILE.tuning, contextWindow: 65_536 }, + }), + ).toThrow(/does not support startup tuning/); + expect(() => + validateManagedStartupProfile({ + ...DCODE_PROFILE, + dashboard: { agent: "langchain-deepagents-code", mode: "remote" }, + }), + ).toThrow(/dashboard\.mode must be disabled/); + }); + + it("rejects unsupported DCode inference APIs and OpenClaw-only inference fields", () => { + expect(() => + validateManagedStartupProfile({ + ...DCODE_PROFILE, + inference: { ...DCODE_PROFILE.inference, api: "openai-responses" }, + }), + ).toThrow(/not supported/); + expect(() => + validateManagedStartupProfile({ + ...HERMES_PROFILE, + inference: { + ...HERMES_PROFILE.inference, + compatibility: { supportsDeveloperRole: true }, + }, + }), + ).toThrow(/does not support/); + }); + + it("accepts only reviewed Hermes gateway IDs and rejects gateways for other agents", () => { + expect(validateManagedStartupProfile(HERMES_PROFILE).tools.enabledGateways).toHaveLength(5); + expect(() => + validateManagedStartupProfile({ + ...HERMES_PROFILE, + tools: { ...HERMES_PROFILE.tools, enabledGateways: ["filesystem"] }, + }), + ).toThrow(/unsupported value/); + expect(() => + validateManagedStartupProfile({ + ...OPENCLAW_PROFILE, + tools: { ...OPENCLAW_PROFILE.tools, enabledGateways: ["nous-web"] }, + }), + ).toThrow(/supported only by hermes/); + }); + + it("enforces adapter-specific web-search providers", () => { + expect(() => + validateManagedStartupProfile({ + ...HERMES_PROFILE, + agentConfig: { + ...HERMES_PROFILE.agentConfig, + webSearch: { enabled: true, provider: "brave" }, + }, + }), + ).toThrow(/not supported/); + }); + + it("enforces resolved OpenClaw dashboard exposure and device-auth semantics", () => { + expect(() => + validateManagedStartupProfile({ + ...OPENCLAW_PROFILE, + dashboard: { ...OPENCLAW_PROFILE.dashboard, mode: "loopback" }, + }), + ).toThrow(/must reflect/); + expect(() => + validateManagedStartupProfile({ + ...OPENCLAW_PROFILE, + agentConfig: { + ...OPENCLAW_PROFILE.agentConfig, + deviceAuth: { disabled: false, optOutSource: "operator" }, + }, + }), + ).toThrow(/requires device auth to be disabled/); + expect(() => + validateManagedStartupProfile({ + ...OPENCLAW_PROFILE, + dashboard: { ...OPENCLAW_PROFILE.dashboard, port: 18_790 }, + }), + ).toThrow(/must match dashboard\.url/); + }); + + it("supports disabled or loopback-forwarded Hermes dashboards only", () => { + const disabled = validateManagedStartupProfile({ + ...HERMES_PROFILE, + dashboard: { + agent: "hermes", + mode: "disabled", + url: "http://127.0.0.1:18789", + publicPort: null, + internalPort: null, + tuiEnabled: false, + }, + }); + expect(disabled.dashboard.mode).toBe("disabled"); + expect(() => + validateManagedStartupProfile({ + ...HERMES_PROFILE, + dashboard: { ...HERMES_PROFILE.dashboard, url: "https://dashboard.example.test" }, + }), + ).toThrow(/must remain loopback/); + expect(() => + validateManagedStartupProfile({ + ...HERMES_PROFILE, + dashboard: { ...HERMES_PROFILE.dashboard, publicPort: 8642 }, + }), + ).toThrow(/reserved API port/); + expect(() => + validateManagedStartupProfile({ + ...HERMES_PROFILE, + dashboard: { ...HERMES_PROFILE.dashboard, publicPort: 19_190 }, + }), + ).toThrow(/must match dashboard\.url/); + }); + + it.each([ + "-----BEGIN CERTIFICATE-----\nMIIB\n-----END CERTIFICATE-----", + `MII${"A".repeat(300)}`, + ])("rejects raw CA material while accepting only its digest", (rawCa) => { + expect(() => + validateManagedStartupProfile({ + ...OPENCLAW_PROFILE, + inference: { ...OPENCLAW_PROFILE.inference, model: rawCa }, + }), + ).toThrow(/raw certificate data/); + }); + + it.each([ + ["bad schema", { ...OPENCLAW_PROFILE, schemaVersion: 2 }], + [ + "bad DCode mode", + { + ...DCODE_PROFILE, + agentConfig: { + ...DCODE_PROFILE.agentConfig, + autoApprovalMode: "always", + }, + }, + ], + [ + "bad heartbeat", + { + ...OPENCLAW_PROFILE, + agentConfig: { ...OPENCLAW_PROFILE.agentConfig, heartbeatEvery: "every hour" }, + }, + ], + [ + "invalid CA digest", + { + ...OPENCLAW_PROFILE, + corporateCa: { bundleSha256: "not-a-digest" }, + }, + ], + ])("rejects malformed profile: %s", (_label, profile) => { + expect(() => validateManagedStartupProfile(profile)).toThrow(/Invalid managed startup profile/); + }); + + it.each([ + ["empty", ""], + ["not base64url", "%%%"], + ["invalid base64url quantum", "a"], + ["invalid JSON", Buffer.from("{", "utf8").toString("base64url")], + ["invalid UTF-8", Buffer.from([0xc3, 0x28]).toString("base64url")], + ])("rejects malformed encoded payload: %s", (_label, encoded) => { + expect(() => decodeManagedStartupProfile(encoded)).toThrow(/Invalid managed startup profile/); + }); + + it("rejects decoded payloads over the bounded profile size", () => { + const encoded = Buffer.alloc(MANAGED_STARTUP_PROFILE_MAX_BYTES + 1, 0x61).toString("base64url"); + expect(() => decodeManagedStartupProfile(encoded)).toThrow(/size limit/); + }); + + it("rejects structurally deep payloads within the byte cap", () => { + const deep = JSON.parse(`${'{"nested":'.repeat(40)}null${"}".repeat(40)}`) as Record< + string, + unknown + >; + expect(() => validateManagedStartupProfile(deep)).toThrow(/complexity limit/); + }); + + it("rejects a noncanonical payload even when its profile values are otherwise valid", () => { + const encoded = encodeUnknown({ + ...HERMES_PROFILE, + tools: { + ...HERMES_PROFILE.tools, + enabledGateways: [...HERMES_PROFILE.tools.enabledGateways].reverse(), + }, + }); + expect(() => decodeManagedStartupProfile(encoded)).toThrow(/canonical form/); + }); +}); diff --git a/src/lib/onboard/managed-startup-root-apply.test.ts b/src/lib/onboard/managed-startup-root-apply.test.ts new file mode 100644 index 0000000000..5ee4e59e6d --- /dev/null +++ b/src/lib/onboard/managed-startup-root-apply.test.ts @@ -0,0 +1,96 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { + MANAGED_STARTUP_E2E_CORPORATE_CA_PEM, + managedStartupE2eProfile, +} from "../../../scripts/checks/generate-managed-startup-profile-fixture.mts"; +import { encodeManagedStartupProfile } from "./managed-startup/profile"; +import { + createManagedStartupRootApplyRequest, + MANAGED_STARTUP_ROOT_APPLY_MAX_BYTES, + parseManagedStartupRootApplyRequest, + serializeManagedStartupRootApplyRequest, +} from "./managed-startup/root-apply"; + +function requestFor( + agent: "openclaw" | "hermes" | "langchain-deepagents-code", + withCorporateCa = false, +) { + return createManagedStartupRootApplyRequest({ + agent, + encodedProfile: encodeManagedStartupProfile( + managedStartupE2eProfile(agent, false, withCorporateCa), + ), + ...(withCorporateCa + ? { corporateCaB64: Buffer.from(MANAGED_STARTUP_E2E_CORPORATE_CA_PEM).toString("base64") } + : {}), + }); +} + +describe("managed startup root-application envelope", () => { + it.each([ + "openclaw", + "hermes", + "langchain-deepagents-code", + ] as const)("round-trips one canonical bounded %s request", (agent) => { + const request = requestFor(agent, true); + const serialized = serializeManagedStartupRootApplyRequest(request); + + expect(Buffer.byteLength(serialized, "utf8")).toBeLessThan( + MANAGED_STARTUP_ROOT_APPLY_MAX_BYTES, + ); + expect(parseManagedStartupRootApplyRequest(serialized)).toEqual(request); + }); + + it("rejects non-canonical JSON and unknown fields", () => { + const request = requestFor("openclaw"); + const reordered = `${JSON.stringify({ + schemaVersion: request.schemaVersion, + profileFingerprint: request.profileFingerprint, + encodedProfile: request.encodedProfile, + corporateCaB64: request.corporateCaB64, + agent: request.agent, + })}\n`; + const withUnknownField = `${JSON.stringify({ + ...JSON.parse(serializeManagedStartupRootApplyRequest(request)), + surprise: true, + })}\n`; + + expect(() => parseManagedStartupRootApplyRequest(reordered)).toThrow(/not canonical/u); + expect(() => parseManagedStartupRootApplyRequest(withUnknownField)).toThrow(/invalid schema/u); + }); + + it("rejects a tampered profile fingerprint", () => { + const request = requestFor("hermes"); + const parsed = JSON.parse(serializeManagedStartupRootApplyRequest(request)) as Record< + string, + unknown + >; + parsed.profileFingerprint = "f".repeat(64); + + expect(() => parseManagedStartupRootApplyRequest(`${JSON.stringify(parsed)}\n`)).toThrow( + /fingerprint does not match/u, + ); + }); + + it("rejects a canonical CA payload that does not match the profile digest", () => { + const profile = managedStartupE2eProfile("langchain-deepagents-code", false, true); + + expect(() => + createManagedStartupRootApplyRequest({ + agent: profile.agent, + encodedProfile: encodeManagedStartupProfile(profile), + corporateCaB64: Buffer.from("different-ca").toString("base64"), + }), + ).toThrow(/does not match the profile digest/u); + }); + + it("rejects an oversized serialized transport before parsing JSON", () => { + expect(() => + parseManagedStartupRootApplyRequest("x".repeat(MANAGED_STARTUP_ROOT_APPLY_MAX_BYTES + 1)), + ).toThrow(/empty or too large/u); + }); +}); diff --git a/src/lib/onboard/managed-startup-shared-state-transaction.test.ts b/src/lib/onboard/managed-startup-shared-state-transaction.test.ts new file mode 100644 index 0000000000..0d28be2cbe --- /dev/null +++ b/src/lib/onboard/managed-startup-shared-state-transaction.test.ts @@ -0,0 +1,328 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { managedStartupE2eProfile } from "../../../scripts/checks/generate-managed-startup-profile-fixture.mts"; +import type { SandboxMessagingPlan } from "../messaging/manifest"; +import type { ManagedStartupAgent, ManagedStartupProfile } from "./managed-startup/profile"; +import { + beginManagedStartupSharedStateTransaction, + commitManagedStartupSharedStateTransaction, + type ManagedStartupSharedTransactionOptions, + rollbackManagedStartupSharedStateTransaction, +} from "./managed-startup/shared-state-transaction"; + +function mode(target: string): number { + return fs.lstatSync(target).mode & 0o7777; +} + +function unavailableIdentity(name: string): never { + throw new Error(`effective ${name} is unavailable`); +} + +function effectiveUid(): number { + return process.geteuid?.() ?? unavailableIdentity("uid"); +} + +function effectiveGid(): number { + return process.getegid?.() ?? unavailableIdentity("gid"); +} + +describe("managed startup shared-state transaction", () => { + let temporaryRoot = ""; + let sandboxRoot = ""; + let transactionDirectory = ""; + let options: ManagedStartupSharedTransactionOptions; + + beforeEach(() => { + temporaryRoot = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-shared-transaction-")); + sandboxRoot = path.join(temporaryRoot, "sandbox"); + const transactionParent = path.join(temporaryRoot, "root-state"); + transactionDirectory = path.join(transactionParent, "transaction"); + fs.mkdirSync(sandboxRoot, { mode: 0o755 }); + fs.mkdirSync(transactionParent, { mode: 0o755 }); + fs.chmodSync(transactionParent, 0o755); + options = { + sandboxRoot, + transactionDirectory, + trustedUid: effectiveUid(), + trustedGid: effectiveGid(), + }; + }); + + afterEach(() => { + vi.restoreAllMocks(); + fs.rmSync(temporaryRoot, { force: true, recursive: true }); + }); + + function agentRoot(agent: ManagedStartupAgent): string { + return path.join( + sandboxRoot, + agent === "openclaw" ? ".openclaw" : agent === "hermes" ? ".hermes" : ".deepagents", + ); + } + + it.each([ + "openclaw", + "hermes", + "langchain-deepagents-code", + ] as const)("restores exact %s bytes, ownership, modes, and absence receipts", (agent) => { + const root = agentRoot(agent); + fs.mkdirSync(root, { mode: 0o750 }); + fs.chmodSync(root, 0o750); + const originalFiles = + agent === "openclaw" + ? [["openclaw.json", "openclaw-original\n", 0o640] as const] + : agent === "hermes" + ? [ + ["config.yaml", "hermes-original\n", 0o640] as const, + [".env", "TOKEN=original\n", 0o600] as const, + ] + : [["config.toml", "dcode-original\n", 0o660] as const]; + for (const [name, contents, fileMode] of originalFiles) { + const target = path.join(root, name); + fs.writeFileSync(target, contents); + fs.chmodSync(target, fileMode); + } + + beginManagedStartupSharedStateTransaction(managedStartupE2eProfile(agent), options); + for (const [name] of originalFiles) { + const target = path.join(root, name); + fs.writeFileSync(target, "changed\n"); + fs.chmodSync(target, 0o600); + } + const createManagedDrift: Record void> = { + openclaw: () => fs.writeFileSync(path.join(root, ".config-hash"), "new\n"), + hermes: () => fs.writeFileSync(path.join(root, ".config-hash"), "new\n"), + "langchain-deepagents-code": () => { + fs.mkdirSync(path.join(root, ".state")); + fs.mkdirSync(path.join(root, "skills")); + }, + }; + createManagedDrift[agent](); + + expect(rollbackManagedStartupSharedStateTransaction(agent, options)).toBe(true); + for (const [name, contents, fileMode] of originalFiles) { + const target = path.join(root, name); + expect(fs.readFileSync(target, "utf8")).toBe(contents); + expect(mode(target)).toBe(fileMode); + expect(fs.lstatSync(target).uid).toBe(effectiveUid()); + expect(fs.lstatSync(target).gid).toBe(effectiveGid()); + } + expect(mode(root)).toBe(0o750); + const absentManagedPaths: Record = { + openclaw: [".config-hash"], + hermes: [".config-hash"], + "langchain-deepagents-code": [".state", "skills"], + }; + for (const relativePath of absentManagedPaths[agent]) { + expect(fs.existsSync(path.join(root, relativePath))).toBe(false); + } + expect(fs.existsSync(transactionDirectory)).toBe(false); + }); + + it("tracks only active post-install messaging outputs and leaves disabled targets alone", () => { + const root = agentRoot("openclaw"); + fs.mkdirSync(root); + fs.writeFileSync(path.join(root, "openclaw.json"), "{}\n"); + const plan: SandboxMessagingPlan = { + schemaVersion: 1, + sandboxName: "managed", + agent: "openclaw", + workflow: "onboard", + channels: [ + { + channelId: "wechat", + displayName: "WeChat", + authMode: "host-qr", + active: true, + selected: true, + configured: true, + disabled: false, + inputs: [], + hooks: [ + { + channelId: "wechat", + id: "seed", + phase: "post-agent-install", + handler: "wechat.seedOpenClawAccount", + }, + ], + }, + { + channelId: "telegram", + displayName: "Telegram", + authMode: "token-paste", + active: false, + selected: true, + configured: true, + disabled: true, + inputs: [], + hooks: [], + }, + ], + disabledChannels: ["telegram"], + credentialBindings: [], + networkPolicy: { presets: [], entries: [] }, + agentRender: [ + { + channelId: "wechat", + agent: "openclaw", + target: "~/.openclaw/active-render.json", + kind: "json-fragment", + path: "channels.wechat", + value: { enabled: true }, + templateRefs: [], + }, + { + channelId: "telegram", + agent: "openclaw", + target: "~/.openclaw/disabled-render.json", + kind: "json-fragment", + path: "channels.telegram", + value: { enabled: true }, + templateRefs: [], + }, + ], + buildSteps: [ + { + channelId: "wechat", + kind: "build-file", + hookId: "seed", + outputId: "accounts", + required: true, + value: { + path: "openclaw-weixin/accounts.json", + content: ["primary"], + }, + }, + { + channelId: "telegram", + kind: "build-file", + outputId: "disabled", + required: true, + value: { path: "disabled/new.json", content: {} }, + }, + ], + stateUpdates: [], + healthChecks: [], + }; + const profile = { + ...managedStartupE2eProfile("openclaw"), + messaging: { plan: plan as unknown as ManagedStartupProfile["messaging"]["plan"] }, + }; + beginManagedStartupSharedStateTransaction(profile, options); + + fs.writeFileSync(path.join(root, "active-render.json"), "active\n"); + fs.mkdirSync(path.join(root, "openclaw-weixin")); + fs.writeFileSync(path.join(root, "openclaw-weixin", "accounts.json"), "active\n"); + fs.writeFileSync(path.join(root, "disabled-render.json"), "keep\n"); + fs.mkdirSync(path.join(root, "disabled")); + fs.writeFileSync(path.join(root, "disabled", "new.json"), "keep\n"); + + expect(rollbackManagedStartupSharedStateTransaction("openclaw", options)).toBe(true); + expect(fs.existsSync(path.join(root, "active-render.json"))).toBe(false); + expect(fs.existsSync(path.join(root, "openclaw-weixin"))).toBe(false); + expect(fs.readFileSync(path.join(root, "disabled-render.json"), "utf8")).toBe("keep\n"); + expect(fs.readFileSync(path.join(root, "disabled", "new.json"), "utf8")).toBe("keep\n"); + }); + + it("commits applied output while removing its private backups", () => { + const root = agentRoot("openclaw"); + fs.mkdirSync(root); + const config = path.join(root, "openclaw.json"); + fs.writeFileSync(config, "before\n"); + beginManagedStartupSharedStateTransaction(managedStartupE2eProfile("openclaw"), options); + fs.writeFileSync(config, "after\n"); + + expect(commitManagedStartupSharedStateTransaction("openclaw", options)).toBe(true); + expect(fs.readFileSync(config, "utf8")).toBe("after\n"); + expect(fs.existsSync(transactionDirectory)).toBe(false); + expect(commitManagedStartupSharedStateTransaction("openclaw", options)).toBe(false); + }); + + it("resumes the same pending profile idempotently and rejects profile drift", () => { + const root = agentRoot("openclaw"); + fs.mkdirSync(root); + fs.writeFileSync(path.join(root, "openclaw.json"), "{}\n"); + const profile = managedStartupE2eProfile("openclaw"); + expect(beginManagedStartupSharedStateTransaction(profile, options)).toBe(true); + expect(beginManagedStartupSharedStateTransaction(profile, options)).toBe(false); + expect(() => + beginManagedStartupSharedStateTransaction( + managedStartupE2eProfile("openclaw", true), + options, + ), + ).toThrow(/belongs to a different profile/u); + expect(rollbackManagedStartupSharedStateTransaction("openclaw", options)).toBe(true); + }); + + it("rejects planted target and ancestor symlinks before creating a receipt", () => { + const outside = path.join(temporaryRoot, "outside"); + fs.mkdirSync(outside); + const root = agentRoot("openclaw"); + fs.symlinkSync(outside, root); + expect(() => + beginManagedStartupSharedStateTransaction(managedStartupE2eProfile("openclaw"), options), + ).toThrow(/ancestor is unsafe/u); + expect(fs.existsSync(transactionDirectory)).toBe(false); + + fs.unlinkSync(root); + fs.mkdirSync(root); + const outsideConfig = path.join(outside, "openclaw.json"); + fs.writeFileSync(outsideConfig, "outside\n"); + fs.symlinkSync(outsideConfig, path.join(root, "openclaw.json")); + expect(() => + beginManagedStartupSharedStateTransaction(managedStartupE2eProfile("openclaw"), options), + ).toThrow(/not a safe regular file/u); + expect(fs.readFileSync(outsideConfig, "utf8")).toBe("outside\n"); + expect(fs.existsSync(transactionDirectory)).toBe(false); + }); + + it("does not rewrite unchanged shield-like files or directory metadata", () => { + const root = agentRoot("openclaw"); + fs.mkdirSync(root, { mode: 0o755 }); + const config = path.join(root, "openclaw.json"); + fs.writeFileSync(config, "shielded\n"); + fs.chmodSync(config, 0o444); + beginManagedStartupSharedStateTransaction(managedStartupE2eProfile("openclaw"), options); + const rename = vi.spyOn(fs, "renameSync"); + const chown = vi.spyOn(fs, "chownSync"); + + expect(rollbackManagedStartupSharedStateTransaction("openclaw", options)).toBe(true); + expect(rename).not.toHaveBeenCalled(); + expect(chown).not.toHaveBeenCalled(); + expect(fs.readFileSync(config, "utf8")).toBe("shielded\n"); + expect(mode(config)).toBe(0o444); + }); + + it("refuses to operate on a pending transaction for a different agent", () => { + const root = agentRoot("openclaw"); + fs.mkdirSync(root); + fs.writeFileSync(path.join(root, "openclaw.json"), "{}\n"); + beginManagedStartupSharedStateTransaction(managedStartupE2eProfile("openclaw"), options); + expect(() => rollbackManagedStartupSharedStateTransaction("hermes", options)).toThrow( + /targets openclaw, expected hermes/u, + ); + expect(fs.existsSync(transactionDirectory)).toBe(true); + expect(rollbackManagedStartupSharedStateTransaction("openclaw", options)).toBe(true); + }); + + it("rejects an oversized managed output before creating a receipt", () => { + const root = agentRoot("openclaw"); + fs.mkdirSync(root); + const config = path.join(root, "openclaw.json"); + fs.writeFileSync(config, ""); + fs.truncateSync(config, 8 * 1024 * 1024 + 1); + + expect(() => + beginManagedStartupSharedStateTransaction(managedStartupE2eProfile("openclaw"), options), + ).toThrow(/unsafe or oversized/u); + expect(fs.existsSync(transactionDirectory)).toBe(false); + }); +}); diff --git a/src/lib/onboard/managed-startup/agent-environment.ts b/src/lib/onboard/managed-startup/agent-environment.ts new file mode 100644 index 0000000000..4a15fb41a7 --- /dev/null +++ b/src/lib/onboard/managed-startup/agent-environment.ts @@ -0,0 +1,466 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { Buffer } from "node:buffer"; + +import { parseSandboxMessagingPlan } from "../../messaging/plan-validation"; +import { + type ManagedStartupAgent, + type ManagedStartupDashboard, + type ManagedStartupProfile, + validateManagedStartupProfile, +} from "./profile"; + +export type ManagedStartupConfigAgent = ManagedStartupAgent; +export type ManagedStartupMessagingAgent = "openclaw" | "hermes"; + +export interface ManagedStartupCorporateCaMaterial { + readonly kind: "corporate-ca-handoff"; + readonly legacyInput: "NEMOCLAW_CORPORATE_CA_B64"; + /** + * The certificate bytes use a separate bounded transport. Keeping only its + * digest here prevents the driver-neutral profile mapper from becoming a + * secret or arbitrary-file transport. + */ + readonly expectedSha256: string | null; +} + +export interface ManagedStartupRootOwnedFileMaterial { + readonly kind: "root-owned-file"; + readonly legacyInput: + | "NEMOCLAW_DCODE_AUTO_APPROVAL" + | "NEMOCLAW_INFERENCE_BASE_URL" + | "NEMOCLAW_PROXY_HOST" + | "NEMOCLAW_PROXY_PORT"; + readonly path: + | "/usr/local/share/nemoclaw/dcode-auto-approval" + | "/usr/local/share/nemoclaw/dcode-inference-base-url" + | "/usr/local/share/nemoclaw/dcode-proxy-host" + | "/usr/local/share/nemoclaw/dcode-proxy-port"; + readonly contents: string; + readonly owner: "root"; + readonly group: "root"; + readonly mode: 0o444; +} + +export type ManagedStartupAgentMaterial = + | ManagedStartupCorporateCaMaterial + | ManagedStartupRootOwnedFileMaterial; + +export interface ManagedStartupGenerateConfigAction { + readonly kind: "generate-agent-config"; + readonly agent: ManagedStartupConfigAgent; + readonly runAs: "sandbox"; +} + +interface ManagedStartupApplyMessagingActionBase { + readonly kind: "apply-messaging-plan"; + readonly agent: ManagedStartupMessagingAgent; + readonly mode: "apply" | "clear"; + /** + * Complete managed images already contain the reviewed dependency union. + * The runtime action vocabulary intentionally cannot express the + * package-install phase. + */ + readonly phase: "runtime-setup" | "post-agent-install"; +} + +export interface ManagedStartupApplyMessagingRuntimeAction + extends ManagedStartupApplyMessagingActionBase { + readonly phase: "runtime-setup"; + /** Writes the reduced, root-owned messaging runtime-plan artifact. */ + readonly runAs: "root"; +} + +export interface ManagedStartupApplyMessagingConfigAction + extends ManagedStartupApplyMessagingActionBase { + readonly phase: "post-agent-install"; + /** Renders only sandbox-owned agent configuration from preinstalled assets. */ + readonly runAs: "sandbox"; +} + +export type ManagedStartupApplyMessagingAction = + | ManagedStartupApplyMessagingRuntimeAction + | ManagedStartupApplyMessagingConfigAction; + +export interface ManagedStartupConfigureDashboardAction { + readonly kind: "configure-dashboard"; + readonly dashboard: ManagedStartupDashboard; +} + +export type ManagedStartupAgentAction = + | ManagedStartupGenerateConfigAction + | ManagedStartupApplyMessagingAction + | ManagedStartupConfigureDashboardAction; + +export interface ManagedStartupAgentEnvironment { + readonly schemaVersion: ManagedStartupProfile["schemaVersion"]; + readonly agent: ManagedStartupAgent; + /** + * Inputs scoped to the trusted configuration-application phase. The + * application boundary must not blindly retain this whole map in the agent + * process environment. + */ + readonly configurationEnvironment: Readonly>; + /** + * Non-secret values intentionally retained for existing entrypoints and + * agent runtime adapters after generated configuration is committed. + */ + readonly runtimeEnvironment: Readonly>; + readonly materials: readonly ManagedStartupAgentMaterial[]; + readonly actions: readonly ManagedStartupAgentAction[]; +} + +export class ManagedStartupAgentEnvironmentError extends Error { + constructor(message: string) { + super(`Cannot map managed startup profile: ${message}`); + this.name = "ManagedStartupAgentEnvironmentError"; + } +} + +type MutableEnvironment = Record; + +function booleanFlag(value: boolean): "0" | "1" { + return value ? "1" : "0"; +} + +function canonicalizeJson(value: unknown): unknown { + if (Array.isArray(value)) return value.map((item) => canonicalizeJson(item)); + if (value === null || typeof value !== "object") return value; + const record = value as Record; + return Object.fromEntries( + Object.keys(record) + .sort() + .map((key) => [key, canonicalizeJson(record[key])]), + ); +} + +function encodeCanonicalJson(value: unknown): string { + return Buffer.from(JSON.stringify(canonicalizeJson(value)), "utf8").toString("base64"); +} + +function sortedEnvironment(environment: MutableEnvironment): Readonly> { + return Object.freeze( + Object.fromEntries( + Object.entries(environment).sort(([left], [right]) => + left < right ? -1 : left > right ? 1 : 0, + ), + ), + ); +} + +function commonConfigurationEnvironment(profile: ManagedStartupProfile): MutableEnvironment { + return { + NEMOCLAW_INFERENCE_API: profile.inference.api, + NEMOCLAW_INFERENCE_BASE_URL: profile.inference.routedBaseUrl, + NEMOCLAW_INFERENCE_PROVIDER_ID: profile.inference.routeProvider, + NEMOCLAW_MODEL: profile.inference.model, + NEMOCLAW_TOOL_DISCLOSURE: profile.tools.disclosure, + NEMOCLAW_UPSTREAM_PROVIDER: profile.inference.upstreamProvider, + }; +} + +function appendHostProxyEnvironment( + environment: MutableEnvironment, + profile: ManagedStartupProfile, + options: { readonly preserveAmbientWhenAbsent?: boolean } = {}, +): void { + if ( + options.preserveAmbientWhenAbsent === true && + profile.proxy.hostHttpUrl === null && + profile.proxy.hostHttpsUrl === null && + profile.proxy.hostNoProxy.length === 0 + ) { + return; + } + const httpProxy = profile.proxy.hostHttpUrl ?? ""; + const httpsProxy = profile.proxy.hostHttpsUrl ?? ""; + const noProxy = profile.proxy.hostNoProxy.join(","); + environment.HTTP_PROXY = httpProxy; + environment.HTTPS_PROXY = httpsProxy; + environment.NO_PROXY = noProxy; + environment.http_proxy = httpProxy; + environment.https_proxy = httpsProxy; + environment.no_proxy = noProxy; +} + +function messagingEnvironment( + profile: ManagedStartupProfile, + expectedAgent: ManagedStartupMessagingAgent, +): MutableEnvironment { + if (profile.messaging.plan === null) return {}; + const plan = parseSandboxMessagingPlan(profile.messaging.plan, { agent: expectedAgent }); + if (!plan) { + throw new ManagedStartupAgentEnvironmentError( + `messaging.plan must contain a validated ${expectedAgent} messaging plan`, + ); + } + const { workflow: _workflow, ...imageBuildPlan } = plan; + return { + NEMOCLAW_MESSAGING_PLAN_B64: encodeCanonicalJson(imageBuildPlan), + }; +} + +function corporateCaMaterial(profile: ManagedStartupProfile): ManagedStartupCorporateCaMaterial { + return Object.freeze({ + kind: "corporate-ca-handoff", + legacyInput: "NEMOCLAW_CORPORATE_CA_B64", + expectedSha256: profile.corporateCa.bundleSha256, + }); +} + +function rootOwnedFile( + legacyInput: ManagedStartupRootOwnedFileMaterial["legacyInput"], + path: ManagedStartupRootOwnedFileMaterial["path"], + value: string, +): ManagedStartupRootOwnedFileMaterial { + return Object.freeze({ + kind: "root-owned-file", + legacyInput, + path, + contents: `${value}\n`, + owner: "root", + group: "root", + mode: 0o444, + }); +} + +function dashboardAction( + dashboard: ManagedStartupDashboard, +): ManagedStartupConfigureDashboardAction { + return Object.freeze({ + kind: "configure-dashboard", + dashboard: Object.freeze(structuredClone(dashboard)), + }); +} + +function applicationActions( + profile: ManagedStartupProfile, + messagingAgent: ManagedStartupMessagingAgent | null, +): readonly ManagedStartupAgentAction[] { + const actions: ManagedStartupAgentAction[] = []; + if (messagingAgent !== null) { + actions.push( + Object.freeze({ + kind: "apply-messaging-plan", + agent: messagingAgent, + mode: profile.messaging.plan === null ? "clear" : "apply", + phase: "runtime-setup", + runAs: "root", + }), + ); + } + actions.push( + Object.freeze({ + kind: "generate-agent-config", + agent: profile.agent, + runAs: "sandbox", + }), + ); + if (messagingAgent !== null) { + actions.push( + Object.freeze({ + kind: "apply-messaging-plan", + agent: messagingAgent, + mode: profile.messaging.plan === null ? "clear" : "apply", + phase: "post-agent-install", + runAs: "sandbox", + }), + ); + } + actions.push(dashboardAction(profile.dashboard)); + return Object.freeze(actions); +} + +function mapOpenClawProfile(profile: ManagedStartupProfile): ManagedStartupAgentEnvironment { + if ( + profile.agent !== "openclaw" || + profile.agentConfig.agent !== "openclaw" || + profile.dashboard.agent !== "openclaw" || + profile.inference.primaryModelRef === null || + profile.inference.inputModalities === null || + profile.tuning.contextWindow === null || + profile.tuning.maxTokens === null || + profile.tuning.reasoning === null || + profile.tuning.reasoningEffort === null + ) { + throw new ManagedStartupAgentEnvironmentError("OpenClaw profile state is inconsistent"); + } + + const configurationEnvironment: MutableEnvironment = { + ...commonConfigurationEnvironment(profile), + ...messagingEnvironment(profile, "openclaw"), + CHAT_UI_URL: profile.dashboard.url, + NEMOCLAW_AGENT_HEARTBEAT_EVERY: profile.agentConfig.heartbeatEvery ?? "", + NEMOCLAW_AGENT_TIMEOUT: String(profile.agentConfig.agentTimeoutSeconds), + NEMOCLAW_CONTEXT_WINDOW: String(profile.tuning.contextWindow), + NEMOCLAW_DASHBOARD_BIND: + profile.dashboard.bindAddress === "0.0.0.0" ? profile.dashboard.bindAddress : "", + NEMOCLAW_DISABLE_DEVICE_AUTH: booleanFlag(profile.agentConfig.deviceAuth.disabled), + NEMOCLAW_DEVICE_AUTH_OPT_OUT_SOURCE: profile.agentConfig.deviceAuth.optOutSource, + NEMOCLAW_EXTRA_AGENTS_JSON_B64: encodeCanonicalJson(profile.agentConfig.extraAgents), + NEMOCLAW_INFERENCE_COMPAT_B64: encodeCanonicalJson(profile.inference.compatibility), + NEMOCLAW_INFERENCE_INPUTS: profile.inference.inputModalities.join(","), + NEMOCLAW_MAX_TOKENS: String(profile.tuning.maxTokens), + NEMOCLAW_OPENCLAW_OTEL: booleanFlag(profile.agentConfig.otel.enabled), + NEMOCLAW_OPENCLAW_OTEL_ENDPOINT: profile.agentConfig.otel.endpointUrl, + NEMOCLAW_OPENCLAW_OTEL_SAMPLE_RATE: String(profile.agentConfig.otel.sampleRate), + NEMOCLAW_OPENCLAW_OTEL_SERVICE_NAME: profile.agentConfig.otel.serviceName, + NEMOCLAW_PRIMARY_MODEL_REF: profile.inference.primaryModelRef, + NEMOCLAW_PROXY_HOST: profile.proxy.managedHost, + NEMOCLAW_PROXY_PORT: String(profile.proxy.managedPort), + NEMOCLAW_REASONING: String(profile.tuning.reasoning), + NEMOCLAW_REASONING_EFFORT: profile.tuning.reasoningEffort, + NEMOCLAW_WEB_SEARCH_ENABLED: booleanFlag(profile.agentConfig.webSearch.enabled), + NEMOCLAW_WEB_SEARCH_PROVIDER: profile.agentConfig.webSearch.provider, + NEMOCLAW_WSL_DASHBOARD_EXPOSURE: booleanFlag(profile.dashboard.wslExposure), + }; + + const runtimeEnvironment: MutableEnvironment = { ...configurationEnvironment }; + delete runtimeEnvironment.NEMOCLAW_MESSAGING_PLAN_B64; + runtimeEnvironment.NEMOCLAW_DASHBOARD_PORT = String(profile.dashboard.port); + runtimeEnvironment.NEMOCLAW_MINIMAL_BOOTSTRAP = booleanFlag(profile.agentConfig.minimalBootstrap); + appendHostProxyEnvironment(runtimeEnvironment, profile, { preserveAmbientWhenAbsent: true }); + + return Object.freeze({ + schemaVersion: profile.schemaVersion, + agent: profile.agent, + configurationEnvironment: sortedEnvironment(configurationEnvironment), + runtimeEnvironment: sortedEnvironment(runtimeEnvironment), + materials: Object.freeze([corporateCaMaterial(profile)]), + actions: applicationActions(profile, "openclaw"), + }); +} + +function mapHermesProfile(profile: ManagedStartupProfile): ManagedStartupAgentEnvironment { + if ( + profile.agent !== "hermes" || + profile.agentConfig.agent !== "hermes" || + profile.dashboard.agent !== "hermes" + ) { + throw new ManagedStartupAgentEnvironmentError("Hermes profile state is inconsistent"); + } + + const configurationEnvironment: MutableEnvironment = { + ...commonConfigurationEnvironment(profile), + ...messagingEnvironment(profile, "hermes"), + CHAT_UI_URL: profile.dashboard.url, + NEMOCLAW_CONTEXT_WINDOW: + profile.tuning.contextWindow === null ? "" : String(profile.tuning.contextWindow), + NEMOCLAW_HERMES_TOOL_GATEWAY_BROKER: booleanFlag(profile.tools.enabledGateways.length > 0), + NEMOCLAW_HERMES_TOOL_GATEWAY_PRESETS_B64: encodeCanonicalJson(profile.tools.enabledGateways), + NEMOCLAW_WEB_SEARCH_ENABLED: booleanFlag(profile.agentConfig.webSearch.enabled), + NEMOCLAW_WEB_SEARCH_PROVIDER: profile.agentConfig.webSearch.provider, + }; + + const runtimeEnvironment: MutableEnvironment = { ...configurationEnvironment }; + delete runtimeEnvironment.NEMOCLAW_MESSAGING_PLAN_B64; + runtimeEnvironment.NEMOCLAW_HERMES_DASHBOARD = + profile.dashboard.mode === "loopback-forwarded" ? "1" : "0"; + runtimeEnvironment.NEMOCLAW_HERMES_DASHBOARD_INTERNAL_PORT = + profile.dashboard.internalPort === null ? "" : String(profile.dashboard.internalPort); + runtimeEnvironment.NEMOCLAW_HERMES_DASHBOARD_PORT = + profile.dashboard.publicPort === null ? "" : String(profile.dashboard.publicPort); + runtimeEnvironment.NEMOCLAW_HERMES_DASHBOARD_TUI = booleanFlag(profile.dashboard.tuiEnabled); + runtimeEnvironment.NEMOCLAW_PROXY_HOST = profile.proxy.managedHost; + runtimeEnvironment.NEMOCLAW_PROXY_PORT = String(profile.proxy.managedPort); + appendHostProxyEnvironment(runtimeEnvironment, profile, { preserveAmbientWhenAbsent: true }); + + return Object.freeze({ + schemaVersion: profile.schemaVersion, + agent: profile.agent, + configurationEnvironment: sortedEnvironment(configurationEnvironment), + runtimeEnvironment: sortedEnvironment(runtimeEnvironment), + materials: Object.freeze([corporateCaMaterial(profile)]), + actions: applicationActions(profile, "hermes"), + }); +} + +function mapDcodeProfile(profile: ManagedStartupProfile): ManagedStartupAgentEnvironment { + if ( + profile.agent !== "langchain-deepagents-code" || + profile.agentConfig.agent !== "langchain-deepagents-code" || + profile.dashboard.agent !== "langchain-deepagents-code" || + profile.messaging.plan !== null + ) { + throw new ManagedStartupAgentEnvironmentError( + "LangChain Deep Agents Code profile state is inconsistent", + ); + } + + const configurationEnvironment: MutableEnvironment = { + ...commonConfigurationEnvironment(profile), + NEMOCLAW_UPSTREAM_ENDPOINT_URL: profile.inference.upstreamEndpointUrl ?? "", + }; + appendHostProxyEnvironment(configurationEnvironment, profile); + const runtimeEnvironment: MutableEnvironment = { + ...configurationEnvironment, + NEMOCLAW_OBSERVABILITY: booleanFlag(profile.agentConfig.observabilityEnabled), + }; + // The config generator needs the routed base URL, but the long-running + // DCode process trusts only the root-owned file consumed by + // managed-dcode-runtime.py. + delete runtimeEnvironment.NEMOCLAW_INFERENCE_BASE_URL; + for (const name of [ + "HTTP_PROXY", + "HTTPS_PROXY", + "NO_PROXY", + "http_proxy", + "https_proxy", + "no_proxy", + ]) { + delete runtimeEnvironment[name]; + } + const materials: readonly ManagedStartupAgentMaterial[] = Object.freeze([ + corporateCaMaterial(profile), + rootOwnedFile( + "NEMOCLAW_DCODE_AUTO_APPROVAL", + "/usr/local/share/nemoclaw/dcode-auto-approval", + profile.agentConfig.autoApprovalMode, + ), + rootOwnedFile( + "NEMOCLAW_INFERENCE_BASE_URL", + "/usr/local/share/nemoclaw/dcode-inference-base-url", + profile.inference.routedBaseUrl, + ), + rootOwnedFile( + "NEMOCLAW_PROXY_HOST", + "/usr/local/share/nemoclaw/dcode-proxy-host", + profile.proxy.managedHost, + ), + rootOwnedFile( + "NEMOCLAW_PROXY_PORT", + "/usr/local/share/nemoclaw/dcode-proxy-port", + String(profile.proxy.managedPort), + ), + ]); + + return Object.freeze({ + schemaVersion: profile.schemaVersion, + agent: profile.agent, + configurationEnvironment: sortedEnvironment(configurationEnvironment), + runtimeEnvironment: sortedEnvironment(runtimeEnvironment), + materials, + actions: applicationActions(profile, null), + }); +} + +/** + * Convert a secret-free validated profile into existing agent-generator and + * entrypoint inputs without depending on Docker, Podman, or another compute + * driver. Validation is repeated at this trust boundary so callers cannot use + * a TypeScript assertion to bypass agent capability checks. + */ +export function mapManagedStartupProfileToAgentEnvironment( + profile: ManagedStartupProfile, +): ManagedStartupAgentEnvironment { + const validated = validateManagedStartupProfile(profile); + switch (validated.agent) { + case "openclaw": + return mapOpenClawProfile(validated); + case "hermes": + return mapHermesProfile(validated); + case "langchain-deepagents-code": + return mapDcodeProfile(validated); + } +} diff --git a/src/lib/onboard/managed-startup/application.ts b/src/lib/onboard/managed-startup/application.ts new file mode 100644 index 0000000000..07e5d9adf1 --- /dev/null +++ b/src/lib/onboard/managed-startup/application.ts @@ -0,0 +1,826 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { Buffer } from "node:buffer"; +import { createHash, randomBytes, X509Certificate } from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; +import { TextDecoder } from "node:util"; + +import { + decodeManagedStartupProfile, + fingerprintManagedStartupProfile, + MANAGED_STARTUP_PROFILE_MAX_BYTES, + type ManagedStartupAgent, + type ManagedStartupProfile, + serializeManagedStartupProfile, + validateManagedStartupProfile, +} from "./profile"; + +export const MANAGED_STARTUP_APPLICATION_STATE_DIR = "/var/lib/nemoclaw/startup-profile"; +export const MANAGED_STARTUP_CA_MAX_BYTES = 128 * 1024; +export const MANAGED_STARTUP_CA_MAX_CERTIFICATES = 24; + +const STATE_SCHEMA_VERSION = 1 as const; +const STATE_DIRECTORY_MODE = 0o700; +const STATE_FILE_MODE = 0o600; +const MAX_CONTROL_FILE_BYTES = 512; +const MAX_STATE_ENTRIES = 32; +const SHA256_RE = /^[a-f0-9]{64}$/u; +const GENERATION_RE = /^generation-([a-f0-9]{64})$/u; +const PREPARE_TEMP_RE = /^\.prepare-[0-9]+-[a-f0-9]{24}$/u; +const CONTROL_TEMP_RE = /^\.(?:committed|pending)\.json-[a-f0-9]{24}\.tmp$/u; +const PEM_CERTIFICATE_RE = + /-----BEGIN CERTIFICATE-----\r?\n[A-Za-z0-9+/=\r\n]+?-----END CERTIFICATE-----/gu; +const UTF8_DECODER = new TextDecoder("utf-8", { fatal: true }); + +interface ManagedStartupApplicationRuntime { + readonly rootUid: number; + readonly rootGid: number; +} + +const DEFAULT_RUNTIME: ManagedStartupApplicationRuntime = { + rootUid: 0, + rootGid: 0, +}; + +/** + * Explicit filesystem seam for unit tests that cannot create uid-0 files. + * Image entrypoints must omit this argument so uid/gid 0 remain mandatory. + */ +export interface ManagedStartupApplicationTestRuntime { + readonly rootUid: number; + readonly rootGid: number; +} + +export interface PrepareManagedStartupApplicationInput { + readonly encodedProfile: string; + readonly expectedAgent: ManagedStartupAgent; + readonly corporateCaB64?: string; + readonly stateDirectory?: string; +} + +export interface PreparedManagedStartupApplication { + readonly status: "prepared" | "already-committed"; + readonly stateDirectory: string; + readonly generationDirectory: string; + readonly profilePath: string; + readonly corporateCaPath: string | null; + readonly fingerprint: string; + readonly expectedAgent: ManagedStartupAgent; + readonly profile: ManagedStartupProfile; +} + +export interface CommittedManagedStartupApplication + extends Omit { + readonly status: "committed"; +} + +interface StateControl { + readonly schemaVersion: typeof STATE_SCHEMA_VERSION; + readonly fingerprint: string; + readonly generation: string; +} + +interface ValidatedGeneration { + readonly directory: string; + readonly profilePath: string; + readonly corporateCaPath: string | null; + readonly profile: ManagedStartupProfile; + readonly fingerprint: string; +} + +export class ManagedStartupApplicationError extends Error { + constructor(message: string) { + super(`Managed startup application failed: ${message}`); + this.name = "ManagedStartupApplicationError"; + } +} + +function fail(message: string): never { + throw new ManagedStartupApplicationError(message); +} + +function runtimeFor( + override: ManagedStartupApplicationTestRuntime | undefined, +): ManagedStartupApplicationRuntime { + return override ?? DEFAULT_RUNTIME; +} + +function requireContainerRoot(): void { + if (process.geteuid?.() !== 0) { + fail("the image-side applicator must run with effective uid 0"); + } +} + +function modeOf(stat: fs.Stats): number { + return stat.mode & 0o777; +} + +function requireOwner(stat: fs.Stats, target: string, runtime: ManagedStartupApplicationRuntime) { + if (stat.uid !== runtime.rootUid || stat.gid !== runtime.rootGid) { + fail(`${target} must be owned by root:root`); + } +} + +function requireSecureDirectory( + target: string, + runtime: ManagedStartupApplicationRuntime, + exactMode: boolean, +): void { + let stat: fs.Stats; + try { + stat = fs.lstatSync(target); + } catch { + fail(`state directory component is missing or unreadable: ${target}`); + } + if (stat.isSymbolicLink() || !stat.isDirectory()) { + fail(`state directory component must be a real directory: ${target}`); + } + requireOwner(stat, target, runtime); + const mode = modeOf(stat); + if ((exactMode && mode !== STATE_DIRECTORY_MODE) || (!exactMode && (mode & 0o022) !== 0)) { + fail( + exactMode + ? `${target} must have mode 0700` + : `${target} must not be group- or world-writable`, + ); + } +} + +function ensureStateDirectory( + rawStateDirectory: string | undefined, + runtime: ManagedStartupApplicationRuntime, +): string { + const stateDirectory = rawStateDirectory ?? MANAGED_STARTUP_APPLICATION_STATE_DIR; + if (!path.isAbsolute(stateDirectory) || stateDirectory.includes("\0")) { + fail("stateDirectory must be an absolute path"); + } + const normalized = path.resolve(stateDirectory); + const parent = path.dirname(normalized); + requireSecureDirectory(parent, runtime, false); + try { + fs.mkdirSync(normalized, { mode: STATE_DIRECTORY_MODE }); + fs.chownSync(normalized, runtime.rootUid, runtime.rootGid); + fs.chmodSync(normalized, STATE_DIRECTORY_MODE); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EEXIST") { + fail(`could not create the managed startup state directory: ${normalized}`); + } + } + requireSecureDirectory(normalized, runtime, true); + return normalized; +} + +function requireSecureRegularFileStat( + stat: fs.Stats, + target: string, + runtime: ManagedStartupApplicationRuntime, +): void { + if (!stat.isFile() || stat.isSymbolicLink()) { + fail(`${target} must be a regular file`); + } + if (stat.nlink !== 1) { + fail(`${target} must not be hardlinked`); + } + requireOwner(stat, target, runtime); + if (modeOf(stat) !== STATE_FILE_MODE) { + fail(`${target} must have mode 0600`); + } +} + +function readSecureFile( + target: string, + maxBytes: number, + runtime: ManagedStartupApplicationRuntime, +): Buffer { + let descriptor: number; + try { + descriptor = fs.openSync(target, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW); + } catch { + fail(`state file is missing, unreadable, or a symlink: ${target}`); + } + try { + const stat = fs.fstatSync(descriptor); + requireSecureRegularFileStat(stat, target, runtime); + if (stat.size < 1 || stat.size > maxBytes) { + fail(`${target} is empty or exceeds its size limit`); + } + const content = fs.readFileSync(descriptor); + if (content.length !== stat.size) { + fail(`${target} changed while it was being read`); + } + return content; + } finally { + fs.closeSync(descriptor); + } +} + +function writeSecureNewFile( + target: string, + content: string | Buffer, + runtime: ManagedStartupApplicationRuntime, +): void { + let descriptor: number; + try { + descriptor = fs.openSync( + target, + fs.constants.O_CREAT | fs.constants.O_EXCL | fs.constants.O_WRONLY | fs.constants.O_NOFOLLOW, + STATE_FILE_MODE, + ); + } catch { + fail(`refused to replace an existing state file: ${target}`); + } + try { + fs.fchownSync(descriptor, runtime.rootUid, runtime.rootGid); + fs.fchmodSync(descriptor, STATE_FILE_MODE); + fs.writeFileSync(descriptor, content); + fs.fsyncSync(descriptor); + } finally { + fs.closeSync(descriptor); + } +} + +function syncDirectory(target: string): void { + const descriptor = fs.openSync(target, fs.constants.O_RDONLY); + try { + fs.fsyncSync(descriptor); + } finally { + fs.closeSync(descriptor); + } +} + +function randomToken(): string { + return randomBytes(12).toString("hex"); +} + +function stateControl(fingerprint: string): StateControl { + return { + schemaVersion: STATE_SCHEMA_VERSION, + fingerprint, + generation: `generation-${fingerprint}`, + }; +} + +function serializeStateControl(control: StateControl): string { + return JSON.stringify({ + fingerprint: control.fingerprint, + generation: control.generation, + schemaVersion: control.schemaVersion, + }); +} + +function parseStateControl( + target: string, + runtime: ManagedStartupApplicationRuntime, +): StateControl { + const bytes = readSecureFile(target, MAX_CONTROL_FILE_BYTES, runtime); + let raw: string; + try { + raw = UTF8_DECODER.decode(bytes); + } catch { + fail(`${target} is not valid UTF-8`); + } + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch { + fail(`${target} is not valid JSON`); + } + if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) { + fail(`${target} does not contain a valid state control`); + } + const record = parsed as Record; + if ( + Object.keys(record).sort().join(",") !== "fingerprint,generation,schemaVersion" || + record.schemaVersion !== STATE_SCHEMA_VERSION || + typeof record.fingerprint !== "string" || + !SHA256_RE.test(record.fingerprint) || + record.generation !== `generation-${record.fingerprint}` + ) { + fail(`${target} does not contain a valid state control`); + } + const control = stateControl(record.fingerprint); + if (serializeStateControl(control) !== raw) { + fail(`${target} is not in canonical form`); + } + return control; +} + +function atomicWriteStateControl( + stateDirectory: string, + basename: "committed.json" | "pending.json", + control: StateControl, + runtime: ManagedStartupApplicationRuntime, +): void { + const target = path.join(stateDirectory, basename); + const temporary = path.join(stateDirectory, `.${basename}-${randomToken()}.tmp`); + writeSecureNewFile(temporary, serializeStateControl(control), runtime); + try { + fs.renameSync(temporary, target); + syncDirectory(stateDirectory); + } catch { + try { + fs.unlinkSync(temporary); + } catch { + // Preserve the primary atomic-write error. + } + fail(`could not atomically write ${basename}`); + } +} + +function validateCorporateCaBytes(bytes: Buffer): void { + if (bytes.length < 1 || bytes.length > MANAGED_STARTUP_CA_MAX_BYTES) { + fail(`corporate CA bundle must contain 1-${String(MANAGED_STARTUP_CA_MAX_BYTES)} bytes`); + } + let pem: string; + try { + pem = UTF8_DECODER.decode(bytes); + } catch { + fail("corporate CA bundle must be valid UTF-8 PEM"); + } + + const matches = [...pem.matchAll(PEM_CERTIFICATE_RE)]; + if ( + matches.length < 1 || + matches.length > MANAGED_STARTUP_CA_MAX_CERTIFICATES || + matches[0]?.index !== 0 + ) { + fail( + `corporate CA bundle must contain 1-${String( + MANAGED_STARTUP_CA_MAX_CERTIFICATES, + )} PEM CA certificates`, + ); + } + + let cursor = 0; + for (const match of matches) { + const index = match.index; + if (index === undefined || (!/^(?:\r?\n)+$/u.test(pem.slice(cursor, index)) && index !== 0)) { + fail("corporate CA bundle contains non-PEM material between certificates"); + } + const block = match[0]; + let certificate: X509Certificate; + try { + certificate = new X509Certificate(block); + } catch { + fail("corporate CA bundle contains an invalid X.509 certificate"); + } + if (!certificate.ca) { + fail("corporate CA bundle contains a certificate without basicConstraints CA:TRUE"); + } + cursor = index + block.length; + } + if (!/^(?:\r?\n)?$/u.test(pem.slice(cursor))) { + fail("corporate CA bundle contains trailing non-PEM material"); + } +} + +export function validateManagedStartupCorporateCaTransport( + encoded: string | undefined, + profile: ManagedStartupProfile, +): Buffer | null { + const expectedDigest = profile.corporateCa.bundleSha256; + if (expectedDigest === null) { + if (encoded !== undefined) { + fail("corporate CA transport must be absent when the profile has no CA digest"); + } + return null; + } + if ( + typeof encoded !== "string" || + encoded.length === 0 || + encoded.length > Math.ceil(MANAGED_STARTUP_CA_MAX_BYTES / 3) * 4 || + !/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/u.test(encoded) + ) { + fail("corporate CA transport must be canonical standard base64"); + } + const bytes = Buffer.from(encoded, "base64"); + if (bytes.toString("base64") !== encoded) { + fail("corporate CA transport must be canonical standard base64"); + } + validateCorporateCaBytes(bytes); + const actualDigest = createHash("sha256").update(bytes).digest("hex"); + if (actualDigest !== expectedDigest) { + fail("corporate CA bundle does not match the profile SHA-256 digest"); + } + return bytes; +} + +function readCanonicalProfile( + profilePath: string, + runtime: ManagedStartupApplicationRuntime, +): { + profile: ManagedStartupProfile; + fingerprint: string; +} { + const bytes = readSecureFile(profilePath, MANAGED_STARTUP_PROFILE_MAX_BYTES, runtime); + let raw: string; + try { + raw = UTF8_DECODER.decode(bytes); + } catch { + fail(`${profilePath} is not valid UTF-8`); + } + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch { + fail(`${profilePath} is not valid JSON`); + } + let profile: ManagedStartupProfile; + try { + profile = validateManagedStartupProfile(parsed); + } catch (error) { + fail(`${profilePath} is invalid: ${(error as Error).message}`); + } + if (serializeManagedStartupProfile(profile) !== raw) { + fail(`${profilePath} is not a canonical managed startup profile`); + } + return { + profile, + fingerprint: fingerprintManagedStartupProfile(profile), + }; +} + +function validateGeneration( + stateDirectory: string, + control: StateControl, + runtime: ManagedStartupApplicationRuntime, + expectedAgent?: ManagedStartupAgent, +): ValidatedGeneration { + if (!GENERATION_RE.test(control.generation)) { + fail("state control names an invalid generation"); + } + const directory = path.join(stateDirectory, control.generation); + requireSecureDirectory(directory, runtime, true); + const entries = fs.readdirSync(directory).sort(); + if ( + entries.some((entry) => entry !== "profile.json" && entry !== "corporate-ca.pem") || + !entries.includes("profile.json") + ) { + fail(`${directory} contains missing or unsupported state files`); + } + const profilePath = path.join(directory, "profile.json"); + const { profile, fingerprint } = readCanonicalProfile(profilePath, runtime); + if (fingerprint !== control.fingerprint) { + fail(`${directory} does not match its recorded profile fingerprint`); + } + if (expectedAgent !== undefined && profile.agent !== expectedAgent) { + fail(`managed startup profile targets ${profile.agent}, expected ${expectedAgent}`); + } + + const caPath = path.join(directory, "corporate-ca.pem"); + let corporateCaPath: string | null = null; + if (profile.corporateCa.bundleSha256 === null) { + if (entries.includes("corporate-ca.pem")) { + fail(`${directory} contains a CA bundle that is absent from the profile`); + } + } else { + if (!entries.includes("corporate-ca.pem")) { + fail(`${directory} is missing the CA bundle recorded by the profile`); + } + const caBytes = readSecureFile(caPath, MANAGED_STARTUP_CA_MAX_BYTES, runtime); + validateCorporateCaBytes(caBytes); + if (createHash("sha256").update(caBytes).digest("hex") !== profile.corporateCa.bundleSha256) { + fail(`${directory} contains a CA bundle with the wrong SHA-256 digest`); + } + corporateCaPath = caPath; + } + + return { + directory, + profilePath, + corporateCaPath, + profile, + fingerprint, + }; +} + +function validateDisposableDirectory( + target: string, + runtime: ManagedStartupApplicationRuntime, +): void { + requireSecureDirectory(target, runtime, true); + const entries = fs.readdirSync(target); + if ( + entries.length > 2 || + entries.some((entry) => entry !== "profile.json" && entry !== "corporate-ca.pem") + ) { + fail(`${target} is not a recognized disposable generation`); + } + for (const entry of entries) { + const file = path.join(target, entry); + const stat = fs.lstatSync(file); + requireSecureRegularFileStat(stat, file, runtime); + } +} + +function discardDirectory(target: string, runtime: ManagedStartupApplicationRuntime): void { + validateDisposableDirectory(target, runtime); + fs.rmSync(target, { recursive: true }); +} + +function unlinkSecureControlOrTemp( + target: string, + runtime: ManagedStartupApplicationRuntime, +): void { + const stat = fs.lstatSync(target); + requireSecureRegularFileStat(stat, target, runtime); + if (stat.size > MAX_CONTROL_FILE_BYTES) { + fail(`${target} exceeds the state-control size limit`); + } + fs.unlinkSync(target); +} + +function listStateEntries(stateDirectory: string): string[] { + const entries = fs.readdirSync(stateDirectory).sort(); + if (entries.length > MAX_STATE_ENTRIES) { + fail(`state directory exceeds ${String(MAX_STATE_ENTRIES)} entries`); + } + return entries; +} + +function cleanAtomicTemps( + stateDirectory: string, + entries: readonly string[], + runtime: ManagedStartupApplicationRuntime, +): void { + for (const entry of entries) { + const target = path.join(stateDirectory, entry); + if (PREPARE_TEMP_RE.test(entry)) { + discardDirectory(target, runtime); + } else if (CONTROL_TEMP_RE.test(entry)) { + unlinkSecureControlOrTemp(target, runtime); + } + } +} + +function requireKnownStateEntries(stateDirectory: string, entries: readonly string[]): void { + for (const entry of entries) { + if ( + entry === "committed.json" || + entry === "pending.json" || + GENERATION_RE.test(entry) || + PREPARE_TEMP_RE.test(entry) || + CONTROL_TEMP_RE.test(entry) + ) { + continue; + } + fail(`${stateDirectory} contains unsupported state component ${entry}`); + } +} + +function discardGenerationsExcept( + stateDirectory: string, + keepGeneration: string | null, + runtime: ManagedStartupApplicationRuntime, +): void { + for (const entry of listStateEntries(stateDirectory)) { + if (GENERATION_RE.test(entry) && entry !== keepGeneration) { + discardDirectory(path.join(stateDirectory, entry), runtime); + } + } +} + +function optionalStateControl( + stateDirectory: string, + basename: "committed.json" | "pending.json", + runtime: ManagedStartupApplicationRuntime, +): StateControl | null { + const target = path.join(stateDirectory, basename); + try { + fs.lstatSync(target); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; + fail(`could not inspect ${target}`); + } + return parseStateControl(target, runtime); +} + +function removePendingControl( + stateDirectory: string, + runtime: ManagedStartupApplicationRuntime, +): void { + unlinkSecureControlOrTemp(path.join(stateDirectory, "pending.json"), runtime); + syncDirectory(stateDirectory); +} + +function recoverState( + stateDirectory: string, + requested: StateControl, + expectedAgent: ManagedStartupAgent, + runtime: ManagedStartupApplicationRuntime, +): { + committed: ValidatedGeneration | null; + pending: ValidatedGeneration | null; +} { + const initialEntries = listStateEntries(stateDirectory); + requireKnownStateEntries(stateDirectory, initialEntries); + cleanAtomicTemps(stateDirectory, initialEntries, runtime); + + const committedControl = optionalStateControl(stateDirectory, "committed.json", runtime); + const pendingControl = optionalStateControl(stateDirectory, "pending.json", runtime); + if (committedControl) { + const committed = validateGeneration(stateDirectory, committedControl, runtime, expectedAgent); + if ( + committedControl.fingerprint !== requested.fingerprint || + committedControl.generation !== requested.generation + ) { + fail("a different startup profile is already committed; recreate the sandbox to change it"); + } + if (pendingControl) { + if ( + pendingControl.fingerprint !== committedControl.fingerprint || + pendingControl.generation !== committedControl.generation + ) { + fail("committed and pending startup state disagree"); + } + removePendingControl(stateDirectory, runtime); + } + discardGenerationsExcept(stateDirectory, committedControl.generation, runtime); + return { committed, pending: null }; + } + + if (pendingControl) { + if ( + pendingControl.fingerprint === requested.fingerprint && + pendingControl.generation === requested.generation + ) { + const pending = validateGeneration(stateDirectory, pendingControl, runtime, expectedAgent); + discardGenerationsExcept(stateDirectory, pendingControl.generation, runtime); + return { committed: null, pending }; + } + discardGenerationsExcept(stateDirectory, null, runtime); + removePendingControl(stateDirectory, runtime); + return { committed: null, pending: null }; + } + + discardGenerationsExcept(stateDirectory, null, runtime); + return { committed: null, pending: null }; +} + +function createGeneration( + stateDirectory: string, + control: StateControl, + profileJson: string, + corporateCa: Buffer | null, + runtime: ManagedStartupApplicationRuntime, +): ValidatedGeneration { + const temporaryName = `.prepare-${String(process.pid)}-${randomToken()}`; + const temporary = path.join(stateDirectory, temporaryName); + const generation = path.join(stateDirectory, control.generation); + try { + fs.mkdirSync(temporary, { mode: STATE_DIRECTORY_MODE }); + fs.chownSync(temporary, runtime.rootUid, runtime.rootGid); + fs.chmodSync(temporary, STATE_DIRECTORY_MODE); + writeSecureNewFile(path.join(temporary, "profile.json"), profileJson, runtime); + if (corporateCa) { + writeSecureNewFile(path.join(temporary, "corporate-ca.pem"), corporateCa, runtime); + } + syncDirectory(temporary); + fs.renameSync(temporary, generation); + syncDirectory(stateDirectory); + } catch (error) { + try { + fs.lstatSync(temporary); + discardDirectory(temporary, runtime); + } catch { + // Preserve the generation error. + } + if (error instanceof ManagedStartupApplicationError) throw error; + fail(`could not atomically prepare generation ${control.generation}`); + } + return validateGeneration(stateDirectory, control, runtime); +} + +function toPrepared( + status: PreparedManagedStartupApplication["status"], + stateDirectory: string, + generation: ValidatedGeneration, + expectedAgent: ManagedStartupAgent, +): PreparedManagedStartupApplication { + return { + status, + stateDirectory, + generationDirectory: generation.directory, + profilePath: generation.profilePath, + corporateCaPath: generation.corporateCaPath, + fingerprint: generation.fingerprint, + expectedAgent, + profile: generation.profile, + }; +} + +/** + * Validate the secret-free envelope and atomically prepare immutable state. + * + * Agent-specific adapters may read the returned generation, make their own + * configuration changes, and then call commitManagedStartupApplication. A + * prepared generation is deliberately not treated as applied. + */ +export function prepareManagedStartupApplication( + input: PrepareManagedStartupApplicationInput, + testRuntime?: ManagedStartupApplicationTestRuntime, +): PreparedManagedStartupApplication { + const runtime = runtimeFor(testRuntime); + requireContainerRoot(); + + let profile: ManagedStartupProfile; + try { + profile = decodeManagedStartupProfile(input.encodedProfile); + } catch (error) { + fail((error as Error).message); + } + if (profile.agent !== input.expectedAgent) { + fail(`managed startup profile targets ${profile.agent}, expected ${input.expectedAgent}`); + } + const corporateCa = validateManagedStartupCorporateCaTransport(input.corporateCaB64, profile); + const profileJson = serializeManagedStartupProfile(profile); + const control = stateControl(fingerprintManagedStartupProfile(profile)); + const stateDirectory = ensureStateDirectory(input.stateDirectory, runtime); + const recovered = recoverState(stateDirectory, control, input.expectedAgent, runtime); + if (recovered.committed) { + return toPrepared( + "already-committed", + stateDirectory, + recovered.committed, + input.expectedAgent, + ); + } + if (recovered.pending) { + return toPrepared("prepared", stateDirectory, recovered.pending, input.expectedAgent); + } + + const generation = createGeneration(stateDirectory, control, profileJson, corporateCa, runtime); + atomicWriteStateControl(stateDirectory, "pending.json", control, runtime); + return toPrepared("prepared", stateDirectory, generation, input.expectedAgent); +} + +function validatePreparedHandle(handle: PreparedManagedStartupApplication): StateControl { + if ( + !path.isAbsolute(handle.stateDirectory) || + !SHA256_RE.test(handle.fingerprint) || + handle.generationDirectory !== + path.join(handle.stateDirectory, `generation-${handle.fingerprint}`) || + handle.profilePath !== path.join(handle.generationDirectory, "profile.json") || + (handle.corporateCaPath !== null && + handle.corporateCaPath !== path.join(handle.generationDirectory, "corporate-ca.pem")) + ) { + fail("prepared startup handle is malformed"); + } + return stateControl(handle.fingerprint); +} + +/** + * Mark a prepared profile applied only after every agent-specific adapter has + * completed. The marker rename is the sole commit point. + */ +export function commitManagedStartupApplication( + prepared: PreparedManagedStartupApplication, + testRuntime?: ManagedStartupApplicationTestRuntime, +): CommittedManagedStartupApplication { + const runtime = runtimeFor(testRuntime); + requireContainerRoot(); + const requested = validatePreparedHandle(prepared); + const stateDirectory = ensureStateDirectory(prepared.stateDirectory, runtime); + const committedControl = optionalStateControl(stateDirectory, "committed.json", runtime); + if (committedControl) { + if ( + committedControl.fingerprint !== requested.fingerprint || + committedControl.generation !== requested.generation + ) { + fail("a different startup profile is already committed"); + } + const generation = validateGeneration( + stateDirectory, + committedControl, + runtime, + prepared.expectedAgent, + ); + return { + ...toPrepared("already-committed", stateDirectory, generation, prepared.expectedAgent), + status: "committed", + }; + } + + const pendingControl = optionalStateControl(stateDirectory, "pending.json", runtime); + if ( + !pendingControl || + pendingControl.fingerprint !== requested.fingerprint || + pendingControl.generation !== requested.generation + ) { + fail("the prepared startup generation is not the active pending generation"); + } + const generation = validateGeneration( + stateDirectory, + pendingControl, + runtime, + prepared.expectedAgent, + ); + atomicWriteStateControl(stateDirectory, "committed.json", pendingControl, runtime); + removePendingControl(stateDirectory, runtime); + return { + ...toPrepared("already-committed", stateDirectory, generation, prepared.expectedAgent), + status: "committed", + }; +} diff --git a/src/lib/onboard/managed-startup/clone-rebinder.ts b/src/lib/onboard/managed-startup/clone-rebinder.ts new file mode 100644 index 0000000000..afdbb032b7 --- /dev/null +++ b/src/lib/onboard/managed-startup/clone-rebinder.ts @@ -0,0 +1,477 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; + +import { resolveContextWindowForModel } from "../../inference/context-window"; +import { rebindSandboxMessagingPlanForClone } from "../../messaging/clone-rebind"; +import { DEFAULT_TOOL_DISCLOSURE } from "../../tool-disclosure"; +import { resolveManagedStartupInferenceRoute } from "../inference-route"; +import { validateManagedStartupCorporateCaTransport } from "./application"; +import { + decodeManagedStartupProfile, + encodeManagedStartupProfile, + type ManagedStartupAgent, + type ManagedStartupDashboard, + type ManagedStartupJsonObject, + type ManagedStartupProfile, + validateManagedStartupProfile, +} from "./profile"; + +const SHA256_PATTERN = /^[a-f0-9]{64}$/u; +const MANAGED_INFERENCE_API_SET = new Set([ + "openai-completions", + "openai-responses", + "anthropic-messages", +]); + +/** + * Current durable state for the source sandbox. The managed-image receipt owns + * immutable image affordances (proxy route, tuning, CA, OpenClaw bootstrap + * knobs); mutable operator intent is re-read from this state before a clone can + * replace an existing destination. + */ +export interface ManagedStartupCloneCurrentState { + readonly provider?: string | null; + readonly model?: string | null; + readonly endpointUrl?: string | null; + readonly preferredInferenceApi?: string | null; + readonly compatibleEndpointReasoning?: "true" | "false" | string | null; + readonly compatibleEndpointReasoningEffort?: "low" | "medium" | "high" | string | null; + readonly toolDisclosure?: "progressive" | "direct" | string; + readonly webSearchEnabled?: boolean; + readonly webSearchProvider?: "brave" | "tavily" | string | null; + readonly messaging?: { readonly schemaVersion?: number; readonly plan?: unknown } | null; + readonly hermesToolGateways?: readonly string[]; + readonly hermesDashboardEnabled?: boolean; + readonly hermesDashboardPort?: number | null; + readonly hermesDashboardInternalPort?: number | null; + readonly hermesDashboardTui?: boolean; + readonly dashboardPort?: number | null; + readonly dashboardRemoteBindPrepared?: boolean; + readonly dcodeAutoApprovalMode?: "disabled" | "thread-opt-in" | string; + readonly observabilityEnabled?: boolean; +} + +export interface ManagedStartupCloneRebindInput { + readonly sourceSandboxName: string; + readonly destinationSandboxName: string; + readonly expectedAgent: ManagedStartupAgent; + readonly destinationDashboardPort: number | null; + readonly encodedProfile: string; + readonly startupProfileSha256: string; + readonly corporateCaB64?: string; + readonly currentSource: ManagedStartupCloneCurrentState; +} + +export interface ReboundManagedStartupClone { + readonly profile: ManagedStartupProfile; + readonly encodedProfile: string; + readonly startupProfileSha256: string; + readonly corporateCaB64?: string; +} + +export class ManagedStartupCloneRebindError extends Error { + constructor(message: string, options?: ErrorOptions) { + super(`Cannot prepare managed snapshot clone: ${message}`, options); + this.name = "ManagedStartupCloneRebindError"; + } +} + +function fail(message: string, cause?: unknown): never { + throw new ManagedStartupCloneRebindError(message, cause === undefined ? undefined : { cause }); +} + +function requireDestinationPort(port: number | null, agent: ManagedStartupAgent): number { + if (!Number.isInteger(port) || port === null || port < 1024 || port > 65_535) { + fail(`${agent} requires an allocated destination dashboard port`); + } + return port; +} + +function urlAtPort(raw: string, port: number): string { + let parsed: URL; + try { + parsed = new URL(raw); + } catch (error) { + fail("source dashboard URL is invalid", error); + } + parsed.port = String(port); + return parsed.toString(); +} + +function requireCurrentString(value: unknown, label: string): string { + if (typeof value !== "string" || value.trim() === "" || value !== value.trim()) { + fail(`current source ${label} is missing or invalid`); + } + return value; +} + +function optionalCurrentString(value: unknown, label: string): string | null { + if (value === null || value === undefined) return null; + return requireCurrentString(value, label); +} + +function currentInference( + profile: ManagedStartupProfile, + current: ManagedStartupCloneCurrentState, +): ManagedStartupProfile["inference"] { + const provider = requireCurrentString(current.provider, "inference provider"); + const model = requireCurrentString(current.model, "inference model"); + const preferredApi = optionalCurrentString( + current.preferredInferenceApi, + "preferred inference API", + ); + if (preferredApi !== null && !MANAGED_INFERENCE_API_SET.has(preferredApi)) { + fail("current source preferred inference API is unsupported"); + } + const resolved = resolveManagedStartupInferenceRoute( + profile.agent, + provider, + model, + preferredApi, + ); + if (!MANAGED_INFERENCE_API_SET.has(resolved.inferenceApi)) { + fail("current source inference route resolved an unsupported API"); + } + const upstreamEndpointUrl = + profile.agent === "langchain-deepagents-code" + ? optionalCurrentString(current.endpointUrl, "upstream endpoint URL") + : null; + return { + routeProvider: resolved.providerKey, + upstreamProvider: provider, + model, + routedBaseUrl: resolved.inferenceBaseUrl, + upstreamEndpointUrl, + api: resolved.inferenceApi as ManagedStartupProfile["inference"]["api"], + primaryModelRef: profile.agent === "openclaw" ? resolved.primaryModelRef : null, + compatibility: + profile.agent === "openclaw" + ? (JSON.parse(JSON.stringify(resolved.inferenceCompat ?? {})) as ManagedStartupJsonObject) + : null, + // Input-modality detection is a stock-image affordance with no durable + // post-onboard mutator. Keep the receipt-owned, already validated value. + inputModalities: profile.agent === "openclaw" ? profile.inference.inputModalities : null, + }; +} + +function currentToolDisclosure( + current: ManagedStartupCloneCurrentState, +): ManagedStartupProfile["tools"]["disclosure"] { + const value = current.toolDisclosure ?? DEFAULT_TOOL_DISCLOSURE; + if (value !== "progressive" && value !== "direct") { + fail("current source tool disclosure is invalid"); + } + return value; +} + +function currentWebSearch( + profile: ManagedStartupProfile, + current: ManagedStartupCloneCurrentState, +): Extract["webSearch"] { + if (profile.agentConfig.agent === "langchain-deepagents-code") { + fail("DCode cannot carry web-search state"); + } + const enabled = current.webSearchEnabled === true; + const configuredProvider = current.webSearchProvider; + if ( + configuredProvider !== undefined && + configuredProvider !== null && + configuredProvider !== "brave" && + configuredProvider !== "tavily" + ) { + fail("current source web-search provider is invalid"); + } + const provider = enabled + ? configuredProvider + : (configuredProvider ?? profile.agentConfig.webSearch.provider); + if (provider !== "brave" && provider !== "tavily") { + fail("enabled current source web search has no valid provider"); + } + if (profile.agent === "hermes" && provider !== "tavily") { + fail("current Hermes web search must use Tavily"); + } + return { enabled, provider }; +} + +function currentAgentConfig( + profile: ManagedStartupProfile, + current: ManagedStartupCloneCurrentState, +): ManagedStartupProfile["agentConfig"] { + if (profile.agentConfig.agent === "openclaw") { + return { + ...profile.agentConfig, + webSearch: currentWebSearch(profile, current), + }; + } + if (profile.agentConfig.agent === "hermes") { + return { + ...profile.agentConfig, + webSearch: currentWebSearch(profile, current), + }; + } + const autoApprovalMode = current.dcodeAutoApprovalMode ?? "disabled"; + if (autoApprovalMode !== "disabled" && autoApprovalMode !== "thread-opt-in") { + fail("current DCode auto-approval mode is invalid"); + } + return { + agent: "langchain-deepagents-code", + autoApprovalMode, + // Registry persistence represents false as absence for this flag. + observabilityEnabled: current.observabilityEnabled === true, + }; +} + +function currentSourceDashboard( + profile: ManagedStartupProfile, + current: ManagedStartupCloneCurrentState, +): ManagedStartupDashboard { + if (profile.dashboard.agent === "openclaw") { + const port = + current.dashboardPort === undefined || current.dashboardPort === null + ? profile.dashboard.port + : requireDestinationPort(current.dashboardPort, profile.agent); + const remoteBind = current.dashboardRemoteBindPrepared === true; + if (remoteBind !== (profile.dashboard.bindAddress === "0.0.0.0")) { + fail("current OpenClaw dashboard bind state conflicts with its managed receipt"); + } + return { + ...profile.dashboard, + url: urlAtPort(profile.dashboard.url, port), + port, + }; + } + if (profile.dashboard.agent === "hermes") { + if (current.hermesDashboardEnabled !== true) { + return { + agent: "hermes", + mode: "disabled", + url: profile.dashboard.url, + publicPort: null, + internalPort: null, + tuiEnabled: false, + }; + } + const publicPort = requireDestinationPort( + current.hermesDashboardPort ?? current.dashboardPort ?? null, + profile.agent, + ); + const internalPort = requireDestinationPort( + current.hermesDashboardInternalPort ?? null, + profile.agent, + ); + return { + agent: "hermes", + mode: "loopback-forwarded", + url: urlAtPort(profile.dashboard.url, publicPort), + publicPort, + internalPort, + tuiEnabled: current.hermesDashboardTui === true, + }; + } + return profile.dashboard; +} + +function currentMessagingPlan(current: ManagedStartupCloneCurrentState): unknown | null { + if (current.messaging === undefined || current.messaging === null) return null; + if (current.messaging.schemaVersion !== 1 || current.messaging.plan === undefined) { + fail("current source messaging state is invalid"); + } + return current.messaging.plan; +} + +function reconcileCurrentSourceProfile( + profile: ManagedStartupProfile, + current: ManagedStartupCloneCurrentState, +): ManagedStartupProfile { + const inference = currentInference(profile, current); + const hermesToolGateways = current.hermesToolGateways ?? []; + if ( + !Array.isArray(hermesToolGateways) || + !hermesToolGateways.every((value) => typeof value === "string") + ) { + fail("current Hermes tool gateways are invalid"); + } + let reasoning = profile.tuning.reasoning; + let reasoningEffort = profile.tuning.reasoningEffort; + let contextWindow = profile.tuning.contextWindow; + if (profile.agent === "openclaw") { + const currentReasoning = current.compatibleEndpointReasoning; + if ( + currentReasoning !== undefined && + currentReasoning !== null && + currentReasoning !== "true" && + currentReasoning !== "false" + ) { + fail("current compatible-endpoint reasoning state is invalid"); + } + reasoning = current.provider === "compatible-endpoint" ? currentReasoning === "true" : false; + const currentReasoningEffort = current.compatibleEndpointReasoningEffort; + if ( + currentReasoningEffort !== undefined && + currentReasoningEffort !== null && + currentReasoningEffort !== "low" && + currentReasoningEffort !== "medium" && + currentReasoningEffort !== "high" + ) { + fail("current compatible-endpoint reasoning-effort state is invalid"); + } + reasoningEffort = + inference.upstreamProvider === "compatible-endpoint" && inference.api === "openai-completions" + ? (currentReasoningEffort ?? "default") + : "default"; + if ( + profile.inference.upstreamProvider !== current.provider || + profile.inference.model !== current.model + ) { + contextWindow = resolveContextWindowForModel( + requireCurrentString(current.provider, "inference provider"), + requireCurrentString(current.model, "inference model"), + ); + if (contextWindow === null) { + fail("current OpenClaw inference route has no verifiable context window"); + } + } + } + return validateManagedStartupProfile({ + ...profile, + agentConfig: currentAgentConfig(profile, current), + inference, + dashboard: currentSourceDashboard(profile, current), + tools: { + disclosure: currentToolDisclosure(current), + enabledGateways: profile.agent === "hermes" ? [...hermesToolGateways] : [], + }, + messaging: { plan: currentMessagingPlan(current) }, + tuning: { ...profile.tuning, contextWindow, reasoning, reasoningEffort }, + }); +} + +function destinationDashboard( + profile: ManagedStartupProfile, + destinationDashboardPort: number | null, +): ManagedStartupDashboard { + const dashboard = profile.dashboard; + if (dashboard.agent === "openclaw") { + const port = requireDestinationPort(destinationDashboardPort, profile.agent); + return { + ...dashboard, + url: urlAtPort(dashboard.url, port), + port, + }; + } + if (dashboard.agent === "hermes") { + if (dashboard.mode === "disabled") { + if (destinationDashboardPort === null) { + return { + ...dashboard, + // Disabled Hermes carries CHAT_UI_URL as a stock-image input only. + // Remove the source sandbox's host-port identity even though no + // destination forward is allocated. + url: "http://127.0.0.1/", + }; + } + return { + ...dashboard, + url: urlAtPort(dashboard.url, destinationDashboardPort), + }; + } + const port = requireDestinationPort(destinationDashboardPort, profile.agent); + return { + ...dashboard, + url: urlAtPort(dashboard.url, port), + publicPort: port, + }; + } + if (destinationDashboardPort !== null) { + fail("langchain-deepagents-code cannot accept a destination dashboard port"); + } + return dashboard; +} + +function destinationMessagingPlan( + profile: ManagedStartupProfile, + sourceSandboxName: string, + destinationSandboxName: string, +): ManagedStartupJsonObject | null { + if (profile.messaging.plan === null) return null; + if (profile.agent === "langchain-deepagents-code") { + fail("langchain-deepagents-code cannot carry a messaging plan"); + } + const rebound = rebindSandboxMessagingPlanForClone({ + sourceSandboxName, + destinationSandboxName, + agent: profile.agent, + sourcePlan: profile.messaging.plan, + environment: { + NEMOCLAW_PROXY_HOST: profile.proxy.managedHost, + NEMOCLAW_PROXY_PORT: String(profile.proxy.managedPort), + }, + }); + return JSON.parse(JSON.stringify(rebound)) as ManagedStartupJsonObject; +} + +/** + * Verify a source managed receipt transport and bind its secret-free intent to + * a newly allocated destination identity before any snapshot mutation occurs. + */ +export function rebindManagedStartupProfileForClone( + input: ManagedStartupCloneRebindInput, +): ReboundManagedStartupClone { + if ( + !SHA256_PATTERN.test(input.startupProfileSha256) || + createHash("sha256").update(input.encodedProfile, "utf8").digest("hex") !== + input.startupProfileSha256 + ) { + fail("source profile transport does not match its receipt SHA-256 digest"); + } + + let sourceProfile: ManagedStartupProfile; + try { + sourceProfile = decodeManagedStartupProfile(input.encodedProfile); + } catch (error) { + fail("source profile transport is not canonical and valid", error); + } + if (sourceProfile.agent !== input.expectedAgent) { + fail(`source profile targets ${sourceProfile.agent}, expected ${input.expectedAgent}`); + } + try { + validateManagedStartupCorporateCaTransport(input.corporateCaB64, sourceProfile); + } catch (error) { + fail("source corporate CA transport does not match the profile", error); + } + + let profile: ManagedStartupProfile; + try { + const currentSourceProfile = reconcileCurrentSourceProfile(sourceProfile, input.currentSource); + profile = validateManagedStartupProfile({ + ...currentSourceProfile, + dashboard: destinationDashboard(currentSourceProfile, input.destinationDashboardPort), + messaging: { + plan: destinationMessagingPlan( + currentSourceProfile, + input.sourceSandboxName, + input.destinationSandboxName, + ), + }, + }); + } catch (error) { + if (error instanceof ManagedStartupCloneRebindError) throw error; + const detail = error instanceof Error ? `: ${error.message}` : ""; + fail(`destination profile could not be validated${detail}`, error); + } + + const encodedProfile = encodeManagedStartupProfile(profile); + const startupProfileSha256 = createHash("sha256").update(encodedProfile, "utf8").digest("hex"); + return Object.freeze( + input.corporateCaB64 === undefined + ? { profile, encodedProfile, startupProfileSha256 } + : { + profile, + encodedProfile, + startupProfileSha256, + corporateCaB64: input.corporateCaB64, + }, + ); +} diff --git a/src/lib/onboard/managed-startup/coordinator.ts b/src/lib/onboard/managed-startup/coordinator.ts new file mode 100644 index 0000000000..a2b99281c2 --- /dev/null +++ b/src/lib/onboard/managed-startup/coordinator.ts @@ -0,0 +1,154 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + type CommittedManagedStartupApplication, + commitManagedStartupApplication, + type PreparedManagedStartupApplication, + type PrepareManagedStartupApplicationInput, + prepareManagedStartupApplication, +} from "./application"; +import { + MANAGED_STARTUP_AGENTS, + type ManagedStartupAgent, + type ManagedStartupProfile, +} from "./profile"; + +export interface ManagedStartupAdapterContext { + readonly agent: ManagedStartupAgent; + readonly profile: ManagedStartupProfile; + readonly fingerprint: string; + readonly generationDirectory: string; + readonly profilePath: string; + readonly corporateCaPath: string | null; +} + +export interface ManagedStartupAgentAdapter { + readonly agent: ManagedStartupAgent; + readonly apply: (context: ManagedStartupAdapterContext) => void | Promise; +} + +export interface ManagedStartupCoordinatorDependencies { + readonly prepareApplication: ( + input: PrepareManagedStartupApplicationInput, + ) => PreparedManagedStartupApplication | Promise; + readonly commitApplication: ( + prepared: PreparedManagedStartupApplication, + ) => CommittedManagedStartupApplication | Promise; +} + +export interface ManagedStartupCoordinationResult { + readonly adapterApplied: boolean; + readonly application: CommittedManagedStartupApplication; +} + +type AdapterRegistry = Readonly>; + +const SHIPPED_AGENT_SET = new Set(MANAGED_STARTUP_AGENTS); + +const DEFAULT_DEPENDENCIES: ManagedStartupCoordinatorDependencies = { + prepareApplication: (input) => prepareManagedStartupApplication(input), + commitApplication: (prepared) => commitManagedStartupApplication(prepared), +}; + +export class ManagedStartupCoordinatorError extends Error { + constructor(message: string) { + super(`Managed startup coordination failed: ${message}`); + this.name = "ManagedStartupCoordinatorError"; + } +} + +function fail(message: string): never { + throw new ManagedStartupCoordinatorError(message); +} + +function createAdapterRegistry(adapters: readonly ManagedStartupAgentAdapter[]): AdapterRegistry { + const byAgent = new Map(); + for (const adapter of adapters) { + if ( + typeof adapter !== "object" || + adapter === null || + !SHIPPED_AGENT_SET.has(adapter.agent) || + typeof adapter.apply !== "function" + ) { + fail("every adapter must identify one shipped agent and provide an apply function"); + } + if (byAgent.has(adapter.agent)) { + fail(`duplicate adapter registered for ${adapter.agent}`); + } + byAgent.set(adapter.agent, adapter); + } + + const missing = MANAGED_STARTUP_AGENTS.filter((agent) => !byAgent.has(agent)); + if (missing.length > 0) { + fail(`missing adapter for ${missing.join(", ")}`); + } + if (byAgent.size !== MANAGED_STARTUP_AGENTS.length) { + fail("adapter registry must contain exactly the shipped agents"); + } + + return Object.freeze( + Object.fromEntries( + MANAGED_STARTUP_AGENTS.map((agent) => { + const adapter = byAgent.get(agent); + if (!adapter) fail(`missing adapter for ${agent}`); + return [agent, adapter]; + }), + ), + ) as AdapterRegistry; +} + +function requirePreparedIdentity( + prepared: PreparedManagedStartupApplication, + requestedAgent: ManagedStartupAgent, +): void { + if (prepared.expectedAgent !== requestedAgent || prepared.profile.agent !== requestedAgent) { + fail(`prepared profile targets ${prepared.profile.agent}, expected ${requestedAgent}`); + } +} + +function adapterContext(prepared: PreparedManagedStartupApplication): ManagedStartupAdapterContext { + return Object.freeze({ + agent: prepared.profile.agent, + profile: prepared.profile, + fingerprint: prepared.fingerprint, + generationDirectory: prepared.generationDirectory, + profilePath: prepared.profilePath, + corporateCaPath: prepared.corporateCaPath, + }); +} + +/** + * Coordinate one managed startup without depending on a host container driver. + * + * A complete, duplicate-free adapter registry is required before application + * state is prepared. New pending profiles dispatch exactly their matching + * adapter and commit only after it succeeds. An already committed profile is + * revalidated through commit without reapplying mutable agent configuration. + */ +export async function coordinateManagedStartupApplication( + input: PrepareManagedStartupApplicationInput, + adapters: readonly ManagedStartupAgentAdapter[], + dependencies: ManagedStartupCoordinatorDependencies = DEFAULT_DEPENDENCIES, +): Promise { + const registry = createAdapterRegistry(adapters); + const prepared = await dependencies.prepareApplication(input); + requirePreparedIdentity(prepared, input.expectedAgent); + + if (prepared.status === "already-committed") { + return { + adapterApplied: false, + application: await dependencies.commitApplication(prepared), + }; + } + + const adapter = registry[prepared.profile.agent]; + if (adapter.agent !== prepared.profile.agent) { + fail(`adapter registry cross-dispatch detected for ${prepared.profile.agent}`); + } + await adapter.apply(adapterContext(prepared)); + return { + adapterApplied: true, + application: await dependencies.commitApplication(prepared), + }; +} diff --git a/src/lib/onboard/managed-startup/docker-root-apply.test.ts b/src/lib/onboard/managed-startup/docker-root-apply.test.ts new file mode 100644 index 0000000000..ed536e0116 --- /dev/null +++ b/src/lib/onboard/managed-startup/docker-root-apply.test.ts @@ -0,0 +1,260 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import { managedStartupE2eProfile } from "../../../../scripts/checks/generate-managed-startup-profile-fixture.mts"; +import { + applyDockerManagedStartupRootRequest, + getDockerManagedStartupFailureTransaction, +} from "./docker-root-apply"; +import { encodeManagedStartupProfile } from "./profile"; +import { + createManagedStartupRootApplyRequest, + parseManagedStartupRootApplyRequest, +} from "./root-apply"; +import { MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY } from "./shared-state-transaction"; + +const CONTAINER_ID = "b".repeat(64); +const IMAGE_ID = `sha256:${"c".repeat(64)}`; + +function requestFor(agent: "openclaw" | "hermes" | "langchain-deepagents-code") { + return createManagedStartupRootApplyRequest({ + agent, + encodedProfile: encodeManagedStartupProfile(managedStartupE2eProfile(agent)), + }); +} + +function stableInspect(overrides: Record = {}): string { + return JSON.stringify([ + { + Id: CONTAINER_ID, + Image: IMAGE_ID, + State: { + Running: true, + Paused: false, + Restarting: false, + Dead: false, + }, + ...overrides, + }, + ]); +} + +function successfulSpawnResult() { + return { + status: 0, + signal: null, + stdout: "", + stderr: "", + output: [null, "", ""], + pid: 1, + error: undefined, + }; +} + +describe("Docker managed-startup root applicator", () => { + it.each([ + "openclaw", + "hermes", + "langchain-deepagents-code", + ] as const)("pins exact container/image identity and uses fixed root stdin for %s", (agent) => { + const request = requestFor(agent); + const dockerCapture = vi.fn(() => stableInspect()); + const dockerSpawnSync = vi.fn(() => successfulSpawnResult()); + + expect( + applyDockerManagedStartupRootRequest( + { containerId: CONTAINER_ID, request }, + { dockerCapture, dockerSpawnSync }, + ), + ).toEqual({ agent, containerId: CONTAINER_ID, image: IMAGE_ID }); + + expect(dockerCapture).toHaveBeenCalledWith(["inspect", "--type", "container", CONTAINER_ID], { + ignoreError: false, + timeout: 30_000, + }); + expect(dockerSpawnSync).toHaveBeenCalledTimes(2); + const [argv, options] = dockerSpawnSync.mock.calls[0] as unknown as [ + string[], + { input: string; timeout: number; encoding: string }, + ]; + expect(argv).toEqual([ + "exec", + "--interactive", + "--user", + "0:0", + "--workdir", + "/", + CONTAINER_ID, + "/usr/bin/env", + "-i", + "HOME=/root", + "LANG=C.UTF-8", + "LC_ALL=C.UTF-8", + "NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION=1", + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + "/usr/local/bin/node", + "/usr/local/lib/nemoclaw/managed-startup-image-runtime.cjs", + "--apply-root-stdin", + "--agent", + agent, + ]); + expect(argv.join(" ")).not.toContain(request.encodedProfile); + expect(parseManagedStartupRootApplyRequest(options.input)).toEqual(request); + expect(options).toMatchObject({ encoding: "utf8", timeout: 300_000 }); + expect(dockerSpawnSync.mock.calls[1]).toEqual([ + [ + "exec", + "--user", + "0:0", + "--workdir", + "/", + CONTAINER_ID, + "/usr/bin/env", + "-i", + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + "/bin/sh", + "-c", + 'if [ -d "$1" ] && [ ! -L "$1" ]; then exit 0; fi; if [ ! -e "$1" ] && [ ! -L "$1" ]; then exit 1; fi; exit 2', + "nemoclaw-transaction-probe", + MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY, + ], + { encoding: "utf8", timeout: 30_000 }, + ]); + }); + + it("retries one lost acknowledgement with the identical pinned command and payload", () => { + const request = requestFor("openclaw"); + const dockerSpawnSync = vi + .fn() + .mockReturnValueOnce({ ...successfulSpawnResult(), status: 1, stderr: "lost ack" }) + .mockReturnValueOnce(successfulSpawnResult()) + .mockReturnValueOnce(successfulSpawnResult()); + + applyDockerManagedStartupRootRequest( + { containerId: CONTAINER_ID, request }, + { dockerCapture: vi.fn(() => stableInspect()), dockerSpawnSync }, + ); + + expect(dockerSpawnSync).toHaveBeenCalledTimes(3); + expect(dockerSpawnSync.mock.calls[1]).toEqual(dockerSpawnSync.mock.calls[0]); + }); + + it("returns no pending transaction when the same profile was already finalized", () => { + const request = requestFor("openclaw"); + const dockerSpawnSync = vi + .fn() + .mockReturnValueOnce(successfulSpawnResult()) + .mockReturnValueOnce({ ...successfulSpawnResult(), status: 1 }); + + expect( + applyDockerManagedStartupRootRequest( + { containerId: CONTAINER_ID, request }, + { dockerCapture: vi.fn(() => stableInspect()), dockerSpawnSync }, + ), + ).toBeNull(); + expect(dockerSpawnSync).toHaveBeenCalledTimes(2); + }); + + it("fails with rollback context when transaction state cannot be verified", () => { + const request = requestFor("hermes"); + const dockerSpawnSync = vi + .fn() + .mockReturnValueOnce(successfulSpawnResult()) + .mockReturnValueOnce({ + ...successfulSpawnResult(), + status: null, + error: new Error("probe unavailable"), + }); + + let failure: unknown; + try { + applyDockerManagedStartupRootRequest( + { containerId: CONTAINER_ID, request }, + { dockerCapture: vi.fn(() => stableInspect()), dockerSpawnSync }, + ); + } catch (error) { + failure = error; + } + + expect(failure).toEqual( + expect.objectContaining({ + message: expect.stringContaining("transaction state could not be verified"), + }), + ); + expect(getDockerManagedStartupFailureTransaction(failure)).toEqual({ + agent: "hermes", + containerId: CONTAINER_ID, + image: IMAGE_ID, + }); + }); + + it("attaches the pinned transaction when both idempotent attempts fail", () => { + const request = requestFor("hermes"); + const dockerSpawnSync = vi.fn(() => ({ + ...successfulSpawnResult(), + status: 1, + stderr: "exec failed", + })); + + let failure: unknown; + try { + applyDockerManagedStartupRootRequest( + { containerId: CONTAINER_ID, request }, + { dockerCapture: vi.fn(() => stableInspect()), dockerSpawnSync }, + ); + } catch (error) { + failure = error; + } + + expect(failure).toEqual( + expect.objectContaining({ message: expect.stringContaining("exec failed") }), + ); + expect(getDockerManagedStartupFailureTransaction(failure)).toEqual({ + agent: "hermes", + containerId: CONTAINER_ID, + image: IMAGE_ID, + }); + expect(dockerSpawnSync).toHaveBeenCalledTimes(2); + }); + + it.each([ + { + label: "changed exact identity", + containerId: CONTAINER_ID, + inspect: stableInspect({ Id: "d".repeat(64) }), + error: /identity changed/u, + }, + { + label: "mutable image identity", + containerId: CONTAINER_ID, + inspect: stableInspect({ Image: "registry.example/image:latest" }), + error: /immutable image identity/u, + }, + { + label: "unstable running state", + containerId: CONTAINER_ID, + inspect: stableInspect({ + State: { Running: true, Paused: false, Restarting: true, Dead: false }, + }), + error: /not stably running/u, + }, + { + label: "short caller identity", + containerId: "b".repeat(12), + inspect: stableInspect(), + error: /full lowercase Docker container ID/u, + }, + ])("rejects $label before root exec", ({ containerId, inspect, error }) => { + const dockerSpawnSync = vi.fn(); + + expect(() => + applyDockerManagedStartupRootRequest( + { containerId, request: requestFor("openclaw") }, + { dockerCapture: vi.fn(() => inspect), dockerSpawnSync }, + ), + ).toThrow(error); + expect(dockerSpawnSync).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/onboard/managed-startup/docker-root-apply.ts b/src/lib/onboard/managed-startup/docker-root-apply.ts new file mode 100644 index 0000000000..836d43c782 --- /dev/null +++ b/src/lib/onboard/managed-startup/docker-root-apply.ts @@ -0,0 +1,204 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { dockerSpawnSync } from "../../adapters/docker/exec"; +import { dockerCapture } from "../../adapters/docker/run"; +import { isImmutableDockerImageId } from "../openshell-docker-sandbox-containers"; +import { MANAGED_STARTUP_RUNTIME_EXECUTABLE } from "./image-runtime"; +import { + type ManagedStartupRootApplyRequest, + serializeManagedStartupRootApplyRequest, +} from "./root-apply"; +import { MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY } from "./shared-state-transaction"; + +const FULL_CONTAINER_ID_RE = /^[a-f0-9]{64}$/u; +const ROOT_APPLY_TIMEOUT_MS = 300_000; +const FIXED_ROOT_ENV = [ + "HOME=/root", + "LANG=C.UTF-8", + "LC_ALL=C.UTF-8", + "NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION=1", + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", +] as const; + +interface DockerManagedStartupInspect { + readonly Id?: string; + readonly Image?: string; + readonly State?: { + readonly Running?: boolean; + readonly Paused?: boolean; + readonly Restarting?: boolean; + readonly Dead?: boolean; + } | null; +} + +export interface DockerManagedStartupTransaction { + readonly agent: ManagedStartupRootApplyRequest["agent"]; + readonly containerId: string; + readonly image: string; +} + +export interface DockerManagedStartupRootApplyDeps { + readonly dockerCapture?: typeof dockerCapture; + readonly dockerSpawnSync?: typeof dockerSpawnSync; +} + +export function getDockerManagedStartupFailureTransaction( + error: unknown, +): DockerManagedStartupTransaction | null { + if (typeof error === "object" && error !== null && "managedStartupTransaction" in error) { + return ( + (error as { managedStartupTransaction?: DockerManagedStartupTransaction }) + .managedStartupTransaction ?? null + ); + } + return null; +} + +function commandDetail(result: { + readonly status?: number | null; + readonly stdout?: string | Buffer | null; + readonly stderr?: string | Buffer | null; + readonly error?: Error | null; +}): string { + return `${String(result.stderr ?? "")} ${String(result.stdout ?? "")} ${String( + result.error?.message ?? "", + )}` + .trim() + .slice(-1200); +} + +function inspectExactContainer( + containerId: string, + capture: typeof dockerCapture, +): { readonly containerId: string; readonly image: string } { + if (!FULL_CONTAINER_ID_RE.test(containerId)) { + throw new Error("Managed startup requires one full lowercase Docker container ID."); + } + const output = capture(["inspect", "--type", "container", containerId], { + ignoreError: false, + timeout: 30_000, + }); + let parsed: unknown; + try { + parsed = JSON.parse(output); + } catch { + throw new Error("Docker returned malformed inspect output for the managed-startup container."); + } + if (!Array.isArray(parsed) || parsed.length !== 1) { + throw new Error("Docker inspect did not resolve exactly one managed-startup container."); + } + const inspect = parsed[0] as DockerManagedStartupInspect; + const exactId = String(inspect.Id ?? "").toLowerCase(); + const image = String(inspect.Image ?? "").toLowerCase(); + if (exactId !== containerId) { + throw new Error("Managed-startup container identity changed before root application."); + } + if (!isImmutableDockerImageId(image)) { + throw new Error("Managed-startup container does not have an immutable image identity."); + } + if ( + inspect.State?.Running !== true || + inspect.State.Paused === true || + inspect.State.Restarting === true || + inspect.State.Dead === true + ) { + throw new Error("Managed-startup container is not stably running for root application."); + } + return { containerId: exactId, image }; +} + +export function applyDockerManagedStartupRootRequest( + input: { + readonly containerId: string; + readonly request: ManagedStartupRootApplyRequest; + }, + deps: DockerManagedStartupRootApplyDeps = {}, +): DockerManagedStartupTransaction | null { + const capture = deps.dockerCapture ?? dockerCapture; + const spawn = deps.dockerSpawnSync ?? dockerSpawnSync; + const pinned = inspectExactContainer(input.containerId, capture); + const transaction = { + agent: input.request.agent, + containerId: pinned.containerId, + image: pinned.image, + } satisfies DockerManagedStartupTransaction; + const payload = serializeManagedStartupRootApplyRequest(input.request); + const argv = [ + "exec", + "--interactive", + "--user", + "0:0", + "--workdir", + "/", + pinned.containerId, + "/usr/bin/env", + "-i", + ...FIXED_ROOT_ENV, + "/usr/local/bin/node", + MANAGED_STARTUP_RUNTIME_EXECUTABLE, + "--apply-root-stdin", + "--agent", + input.request.agent, + ]; + const receiptProbeArgv = [ + "exec", + "--user", + "0:0", + "--workdir", + "/", + pinned.containerId, + "/usr/bin/env", + "-i", + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + "/bin/sh", + "-c", + 'if [ -d "$1" ] && [ ! -L "$1" ]; then exit 0; fi; if [ ! -e "$1" ] && [ ! -L "$1" ]; then exit 1; fi; exit 2', + "nemoclaw-transaction-probe", + MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY, + ]; + + let lastFailure = ""; + // The image-side coordinator and transaction are idempotent. One retry + // reconciles the only ambiguous case: Docker lost the first exec + // acknowledgement after the completion marker was already published. + for (let attempt = 0; attempt < 2; attempt += 1) { + const result = spawn(argv, { + encoding: "utf8", + input: payload, + timeout: ROOT_APPLY_TIMEOUT_MS, + }); + if (result.status === 0) { + const receiptProbe = spawn(receiptProbeArgv, { + encoding: "utf8", + timeout: 30_000, + }); + if (receiptProbe.status === 0) return transaction; + if (receiptProbe.status === 1) return null; + const receiptProbeDetail = commandDetail(receiptProbe); + const error = new Error( + `Managed startup root application completed, but transaction state could not be verified in exact container ${pinned.containerId.slice(0, 12)}${ + receiptProbeDetail ? `: ${receiptProbeDetail}` : "" + }`, + ); + ( + error as Error & { + managedStartupTransaction?: DockerManagedStartupTransaction; + } + ).managedStartupTransaction = transaction; + throw error; + } + lastFailure = commandDetail(result); + } + const error = new Error( + `Managed startup root application failed in exact container ${pinned.containerId.slice(0, 12)}${ + lastFailure ? `: ${lastFailure}` : "" + }`, + ); + ( + error as Error & { + managedStartupTransaction?: DockerManagedStartupTransaction; + } + ).managedStartupTransaction = transaction; + throw error; +} diff --git a/src/lib/onboard/managed-startup/docker-shared-state.test.ts b/src/lib/onboard/managed-startup/docker-shared-state.test.ts new file mode 100644 index 0000000000..a604a19656 --- /dev/null +++ b/src/lib/onboard/managed-startup/docker-shared-state.test.ts @@ -0,0 +1,287 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import path from "node:path"; + +import { afterEach, describe, expect, it, vi } from "vitest"; + +import type { DockerGpuPatchResult } from "../docker-gpu-patch-types"; +import type { DockerManagedStartupTransaction } from "./docker-root-apply"; +import { finalizeDockerManagedStartupSharedState } from "./docker-shared-state"; +import { MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY } from "./shared-state-transaction"; + +const IMMUTABLE_IMAGE = `sha256:${"a".repeat(64)}`; + +function result(): DockerGpuPatchResult { + return { + applied: true, + oldContainerId: "old", + newContainerId: "new", + originalName: "openshell-alpha", + backupContainerName: "openshell-alpha-backup", + mode: { + kind: "startup-command", + label: "restart-safe startup command", + device: "", + args: [], + }, + backupRemoved: false, + }; +} + +function transaction(): DockerManagedStartupTransaction { + return { + agent: "openclaw", + containerId: "new", + image: IMMUTABLE_IMAGE, + }; +} + +function removeReceiptParents(...receiptPaths: readonly string[]): void { + for (const receiptPath of receiptPaths.filter((candidate) => candidate.length > 0)) { + fs.rmSync(path.dirname(receiptPath), { force: true, recursive: true }); + } +} + +describe("Docker managed-startup shared-state finalization", () => { + afterEach(() => { + vi.restoreAllMocks(); + }); + + it("copies the bounded receipt before commit and removes the host copy on success", () => { + const calls: string[] = []; + let receiptPath = ""; + const dockerRun = vi + .fn() + .mockImplementationOnce((args: readonly string[]) => { + calls.push("copy"); + receiptPath = String(args[2]); + expect(args[1]).toBe(`new:${MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY}`); + return { status: 0 }; + }) + .mockImplementationOnce((args: readonly string[]) => { + calls.push("commit"); + expect(args).toEqual([ + "exec", + "--user", + "0:0", + "--env", + "NODE_OPTIONS=", + "--env", + "NODE_PATH=", + "--env", + "BASH_ENV=", + "--env", + "ENV=", + "new", + "/usr/bin/env", + "-i", + "HOME=/root", + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + "/usr/local/bin/node", + "/usr/local/lib/nemoclaw/managed-startup-image-runtime.cjs", + "--commit-shared-state-transaction", + "--agent", + "openclaw", + ]); + return { status: 0 }; + }); + const dockerStop = vi.fn(); + + expect( + finalizeDockerManagedStartupSharedState( + { transaction: transaction(), patchResult: result(), supervisorReady: true }, + { dockerRun, dockerStop }, + ), + ).toEqual({ supervisorReady: true, failure: null }); + expect(calls).toEqual(["copy", "commit"]); + expect(dockerStop).not.toHaveBeenCalled(); + expect(fs.existsSync(path.dirname(receiptPath))).toBe(false); + }); + + it("uses the preserved pre-commit receipt after a lost commit acknowledgement", () => { + const calls: string[] = []; + let receiptPath = ""; + const dockerRun = vi + .fn() + .mockImplementationOnce((args: readonly string[]) => { + calls.push("copy"); + receiptPath = String(args[2]); + return { status: 0 }; + }) + .mockImplementationOnce(() => { + calls.push("commit-lost-ack"); + return { status: 1, stderr: "daemon acknowledgement lost" }; + }) + .mockImplementationOnce((args: readonly string[]) => { + calls.push("rollback-helper"); + expect(args).toEqual([ + "run", + "--rm", + "--pull", + "never", + "--network", + "none", + "--read-only", + "--user", + "0:0", + "--security-opt", + "no-new-privileges", + "--cap-drop", + "ALL", + "--cap-add", + "CHOWN", + "--cap-add", + "DAC_OVERRIDE", + "--cap-add", + "FOWNER", + "--env", + "NODE_OPTIONS=", + "--env", + "NODE_PATH=", + "--env", + "BASH_ENV=", + "--env", + "ENV=", + "--volumes-from", + "new", + "--mount", + expect.stringMatching( + /^type=bind,src=.+,dst=\/run\/nemoclaw\/managed-startup-shared-rollback-receipt-v1,readonly$/u, + ), + "--entrypoint", + "/usr/local/bin/node", + IMMUTABLE_IMAGE, + "/usr/local/lib/nemoclaw/managed-startup-image-runtime.cjs", + "--rollback-shared-state-transaction", + "--agent", + "openclaw", + "--read-only-receipt", + ]); + return { status: 0 }; + }); + const dockerStop = vi.fn(() => { + calls.push("stop"); + return { status: 0 }; + }); + + const outcome = finalizeDockerManagedStartupSharedState( + { transaction: transaction(), patchResult: result(), supervisorReady: true }, + { dockerRun, dockerStop }, + ); + expect(outcome.supervisorReady).toBe(false); + expect(outcome.failure?.message).toContain("commit failed"); + expect(calls).toEqual(["copy", "commit-lost-ack", "stop", "rollback-helper"]); + expect(fs.existsSync(path.dirname(receiptPath))).toBe(false); + }); + + it("uses unique receipt paths and treats already-completed cleanup idempotently", () => { + const receiptPaths: string[] = []; + const copyReceipt = (args: readonly string[]) => { + expect(args[0]).toBe("cp"); + const receiptPath = String(args[2]); + receiptPaths.push(receiptPath); + return { status: 0 }; + }; + const completeCommit = (args: readonly string[]) => { + expect(args[0]).toBe("exec"); + removeReceiptParents(receiptPaths.at(-1)!); + return { status: 0 }; + }; + const dockerRun = vi + .fn() + .mockImplementationOnce(copyReceipt) + .mockImplementationOnce(completeCommit) + .mockImplementationOnce(copyReceipt) + .mockImplementationOnce(completeCommit); + + for (let attempt = 0; attempt < 2; attempt += 1) { + expect( + finalizeDockerManagedStartupSharedState( + { transaction: transaction(), patchResult: result(), supervisorReady: true }, + { dockerRun }, + ), + ).toEqual({ supervisorReady: true, failure: null }); + } + expect(new Set(receiptPaths).size).toBe(2); + expect(receiptPaths.every((receiptPath) => !fs.existsSync(path.dirname(receiptPath)))).toBe( + true, + ); + }); + + it("quiesces a failed supervisor before copying and replaying the receipt", () => { + const calls: string[] = []; + const dockerStop = vi.fn(() => { + calls.push("stop"); + return { status: 0 }; + }); + const dockerRun = vi.fn((args: readonly string[]) => { + calls.push(args[0] === "cp" ? "copy" : "rollback-helper"); + return { status: 0 }; + }); + + expect( + finalizeDockerManagedStartupSharedState( + { transaction: transaction(), patchResult: result(), supervisorReady: false }, + { dockerRun, dockerStop }, + ), + ).toEqual({ supervisorReady: false, failure: null }); + expect(calls).toEqual(["stop", "copy", "rollback-helper"]); + }); + + it("stops a live workload when pre-commit receipt preservation fails", () => { + const dockerRun = vi.fn(() => ({ status: 1, stderr: "copy failed" })); + const dockerStop = vi.fn(() => ({ status: 0 })); + + expect(() => + finalizeDockerManagedStartupSharedState( + { transaction: transaction(), patchResult: result(), supervisorReady: true }, + { dockerRun, dockerStop }, + ), + ).toThrow(/Could not copy/u); + expect(dockerStop).toHaveBeenCalledOnce(); + expect(dockerRun).toHaveBeenCalledOnce(); + }); + + it("fails before container rollback when the immutable helper cannot verify restoration", () => { + const dockerStop = vi.fn(() => ({ status: 0 })); + let receiptPath = ""; + const dockerRun = vi + .fn() + .mockImplementationOnce((args: readonly string[]) => { + receiptPath = String(args[2]); + return { status: 0 }; + }) + .mockImplementationOnce(() => ({ + status: 1, + stderr: "receipt verification failed", + })); + + try { + expect(() => + finalizeDockerManagedStartupSharedState( + { transaction: transaction(), patchResult: result(), supervisorReady: false }, + { dockerRun, dockerStop }, + ), + ).toThrow(/could not restore and verify/u); + expect(dockerStop).toHaveBeenCalledOnce(); + expect(fs.existsSync(path.dirname(receiptPath))).toBe(true); + } finally { + removeReceiptParents(receiptPath); + } + }); + + it("is a no-op for non-managed container patches", () => { + const dockerRun = vi.fn(); + const dockerStop = vi.fn(); + expect( + finalizeDockerManagedStartupSharedState( + { transaction: null, patchResult: result(), supervisorReady: true }, + { dockerRun, dockerStop }, + ), + ).toEqual({ supervisorReady: true, failure: null }); + expect(dockerRun).not.toHaveBeenCalled(); + expect(dockerStop).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/onboard/managed-startup/docker-shared-state.ts b/src/lib/onboard/managed-startup/docker-shared-state.ts new file mode 100644 index 0000000000..41c36aea85 --- /dev/null +++ b/src/lib/onboard/managed-startup/docker-shared-state.ts @@ -0,0 +1,271 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + dockerRm as defaultDockerRm, + dockerStop as defaultDockerStop, +} from "../../adapters/docker/container"; +import { dockerRun as defaultDockerRun } from "../../adapters/docker/run"; +import { hasZeroDockerExitStatus } from "../docker-command-result"; +import { + DOCKER_GPU_PATCH_STOP_TIMEOUT_MS, + DOCKER_GPU_PATCH_TIMEOUT_MS, +} from "../docker-gpu-patch-constants"; +import type { DockerGpuPatchDeps, DockerGpuPatchResult } from "../docker-gpu-patch-types"; +import { cleanupTempDir, secureTempFile } from "../temp-files"; +import type { DockerManagedStartupTransaction } from "./docker-root-apply"; +import { MANAGED_STARTUP_RUNTIME_EXECUTABLE } from "./image-runtime"; +import { + MANAGED_STARTUP_SHARED_ROLLBACK_RECEIPT_DIRECTORY, + MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY, +} from "./shared-state-transaction"; + +const RECEIPT_TEMP_PREFIX = "nemoclaw-managed-startup-receipt"; +const NEUTRALIZED_PROCESS_INJECTION_ENV = [ + "--env", + "NODE_OPTIONS=", + "--env", + "NODE_PATH=", + "--env", + "BASH_ENV=", + "--env", + "ENV=", +] as const; + +export interface DockerManagedStartupSharedStateOutcome { + /** + * True only when the new supervisor is still eligible for successful + * container cutover. A commit failure forces shared-state rollback first. + */ + readonly supervisorReady: boolean; + /** Original commit failure after a successful shared-state rollback. */ + readonly failure: Error | null; +} + +function commandDetail(result: { + readonly stderr?: string | Buffer | null; + readonly stdout?: string | Buffer | null; + readonly error?: Error | null; +}): string { + return `${String(result.stderr ?? "")} ${String(result.stdout ?? "")} ${String( + result.error?.message ?? "", + )}` + .trim() + .slice(-800); +} + +function cleanupReceiptBestEffort(receiptPath: string): void { + try { + cleanupTempDir(receiptPath, RECEIPT_TEMP_PREFIX); + } catch (error) { + console.warn( + ` ⚠ Managed-startup shared state is finalized, but its protected host receipt could not be removed (${receiptPath}): ${ + error instanceof Error ? error.message : String(error) + }`, + ); + } +} + +function transactionCommand(action: "commit" | "rollback", agent: string): string[] { + return [ + MANAGED_STARTUP_RUNTIME_EXECUTABLE, + `--${action}-shared-state-transaction`, + "--agent", + agent, + ]; +} + +const DOCKER_MUTATION_OPTIONS = { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, +} as const; + +function quiesceManagedStartupContainer( + transaction: DockerManagedStartupTransaction, + deps: DockerGpuPatchDeps, +): void { + const dockerStop = deps.dockerStop ?? defaultDockerStop; + const stopped = dockerStop(transaction.containerId, { + ...DOCKER_MUTATION_OPTIONS, + timeout: DOCKER_GPU_PATCH_STOP_TIMEOUT_MS, + }); + if (!hasZeroDockerExitStatus(stopped)) { + throw new Error( + `Could not quiesce the failed managed-startup container before shared-state rollback: ${commandDetail(stopped)}`, + ); + } +} + +function copyManagedStartupReceipt( + transaction: DockerManagedStartupTransaction, + deps: DockerGpuPatchDeps, +): string { + const dockerRun = deps.dockerRun ?? defaultDockerRun; + const receiptPath = secureTempFile(RECEIPT_TEMP_PREFIX); + try { + const copy = dockerRun( + [ + "cp", + `${transaction.containerId}:${MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY}`, + receiptPath, + ], + DOCKER_MUTATION_OPTIONS, + ); + if (!hasZeroDockerExitStatus(copy)) { + throw new Error( + `Could not copy the managed-startup rollback receipt from the failed container: ${commandDetail(copy)}`, + ); + } + if (receiptPath.includes(",") || /[\r\n\0]/u.test(receiptPath)) { + throw new Error("Managed-startup rollback receipt path is unsafe for a Docker bind mount"); + } + return receiptPath; + } catch (error) { + cleanupReceiptBestEffort(receiptPath); + throw error; + } +} + +function rollbackManagedStartupSharedState( + transaction: DockerManagedStartupTransaction, + receiptPath: string, + deps: DockerGpuPatchDeps, +): void { + const dockerRun = deps.dockerRun ?? defaultDockerRun; + let restored = false; + try { + const helper = dockerRun( + [ + "run", + "--rm", + "--pull", + "never", + "--network", + "none", + "--read-only", + "--user", + "0:0", + "--security-opt", + "no-new-privileges", + "--cap-drop", + "ALL", + "--cap-add", + "CHOWN", + "--cap-add", + "DAC_OVERRIDE", + "--cap-add", + "FOWNER", + ...NEUTRALIZED_PROCESS_INJECTION_ENV, + "--volumes-from", + transaction.containerId, + "--mount", + `type=bind,src=${receiptPath},dst=${MANAGED_STARTUP_SHARED_ROLLBACK_RECEIPT_DIRECTORY},readonly`, + "--entrypoint", + "/usr/local/bin/node", + transaction.image, + ...transactionCommand("rollback", transaction.agent), + "--read-only-receipt", + ], + DOCKER_MUTATION_OPTIONS, + ); + if (!hasZeroDockerExitStatus(helper)) { + throw new Error( + `Immutable managed-startup helper could not restore and verify shared state: ${commandDetail(helper)}. ` + + `Protected receipt retained at ${receiptPath}`, + ); + } + restored = true; + } finally { + if (restored) { + cleanupReceiptBestEffort(receiptPath); + } + } +} + +function removeFailedUnbackedContainer( + transaction: DockerManagedStartupTransaction, + deps: DockerGpuPatchDeps, +): void { + const dockerRm = deps.dockerRm ?? defaultDockerRm; + const removed = dockerRm(transaction.containerId, DOCKER_MUTATION_OPTIONS); + if (!hasZeroDockerExitStatus(removed)) { + throw new Error( + `Could not remove the failed managed-startup container after shared-state rollback: ${commandDetail(removed)}`, + ); + } +} + +/** + * Finalize the shared-state half of managed container cutover before generic + * backup removal or rollback. A shared-state rollback failure deliberately + * throws so callers cannot remove the new container or restart the old one + * while `/sandbox` remains partially applied. + */ +export function finalizeDockerManagedStartupSharedState( + input: { + readonly transaction: DockerManagedStartupTransaction | null; + readonly patchResult?: DockerGpuPatchResult | null; + readonly supervisorReady: boolean; + }, + deps: DockerGpuPatchDeps = {}, +): DockerManagedStartupSharedStateOutcome { + const transaction = input.transaction; + if (!transaction) { + return { supervisorReady: input.supervisorReady, failure: null }; + } + const dockerRun = deps.dockerRun ?? defaultDockerRun; + if (input.supervisorReady) { + // Preserve a verified rollback source before commit deletes the + // container-local receipt. If Docker loses the exec acknowledgement after + // deletion, this copy still makes the cutover reversible. + let receiptPath: string; + try { + receiptPath = copyManagedStartupReceipt(transaction, deps); + } catch (error) { + try { + quiesceManagedStartupContainer(transaction, deps); + } catch (stopError) { + throw new Error( + `Managed-startup receipt preservation failed and the new workload could not be quiesced: ${ + error instanceof Error ? error.message : String(error) + }; ${stopError instanceof Error ? stopError.message : String(stopError)}`, + ); + } + throw error; + } + const commit = dockerRun( + [ + "exec", + "--user", + "0:0", + ...NEUTRALIZED_PROCESS_INJECTION_ENV, + transaction.containerId, + "/usr/bin/env", + "-i", + "HOME=/root", + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + "/usr/local/bin/node", + ...transactionCommand("commit", transaction.agent), + ], + DOCKER_MUTATION_OPTIONS, + ); + if (hasZeroDockerExitStatus(commit)) { + cleanupReceiptBestEffort(receiptPath); + return { supervisorReady: true, failure: null }; + } + const failure = new Error( + `OpenShell supervisor reconnected, but managed shared-state commit failed: ${commandDetail(commit)}`, + ); + quiesceManagedStartupContainer(transaction, deps); + rollbackManagedStartupSharedState(transaction, receiptPath, deps); + if (!input.patchResult) removeFailedUnbackedContainer(transaction, deps); + return { supervisorReady: false, failure }; + } + + quiesceManagedStartupContainer(transaction, deps); + const receiptPath = copyManagedStartupReceipt(transaction, deps); + rollbackManagedStartupSharedState(transaction, receiptPath, deps); + if (!input.patchResult) removeFailedUnbackedContainer(transaction, deps); + return { supervisorReady: false, failure: null }; +} diff --git a/src/lib/onboard/managed-startup/hold.ts b/src/lib/onboard/managed-startup/hold.ts new file mode 100644 index 0000000000..36015e8475 --- /dev/null +++ b/src/lib/onboard/managed-startup/hold.ts @@ -0,0 +1,4 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export const MANAGED_STARTUP_HOLD_EXECUTABLE = "/usr/local/bin/nemoclaw-managed-startup-hold"; diff --git a/src/lib/onboard/managed-startup/image-runtime.ts b/src/lib/onboard/managed-startup/image-runtime.ts new file mode 100644 index 0000000000..440d937b90 --- /dev/null +++ b/src/lib/onboard/managed-startup/image-runtime.ts @@ -0,0 +1,1342 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import { createHash, randomBytes } from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; + +import { + type ManagedStartupAgentEnvironment, + type ManagedStartupAgentMaterial, + mapManagedStartupProfileToAgentEnvironment, +} from "./agent-environment"; +import { + coordinateManagedStartupApplication, + type ManagedStartupAdapterContext, + type ManagedStartupAgentAdapter, +} from "./coordinator"; +import { + decodeManagedStartupProfile, + fingerprintManagedStartupProfile, + MANAGED_STARTUP_AGENTS, + type ManagedStartupAgent, + type ManagedStartupProfile, +} from "./profile"; +import { + MANAGED_STARTUP_ROOT_APPLY_MAX_BYTES, + type ManagedStartupRootApplyRequest, + parseManagedStartupRootApplyRequest, +} from "./root-apply"; +import { + beginManagedStartupSharedStateTransaction, + commitManagedStartupSharedStateTransaction, + MANAGED_STARTUP_SHARED_ROLLBACK_RECEIPT_DIRECTORY, + rollbackManagedStartupSharedStateTransaction, +} from "./shared-state-transaction"; +import { MANAGED_STARTUP_CA_ENV, MANAGED_STARTUP_PROFILE_ENV } from "./transport"; + +export { MANAGED_STARTUP_CA_ENV, MANAGED_STARTUP_PROFILE_ENV } from "./transport"; +export const MANAGED_STARTUP_RUNTIME_ENV_FILE = "/run/nemoclaw/managed-startup-runtime.env"; +export const MANAGED_STARTUP_RUNTIME_EXECUTABLE = + "/usr/local/lib/nemoclaw/managed-startup-image-runtime.cjs"; +export const MANAGED_STARTUP_MERGED_CA_FILE = "/run/nemoclaw/managed-startup-ca-bundle.pem"; +export const MANAGED_STARTUP_COMPLETION_FILE = "/run/nemoclaw/managed-startup-complete.json"; + +const MANAGED_STARTUP_CORPORATE_CA_FILE = "/usr/local/share/nemoclaw/corporate-ca.pem"; +const MESSAGING_RUNTIME_PLAN_FILE = "/usr/local/share/nemoclaw/messaging-runtime-plan.json"; +const ROOT_STATE_PARENT = "/var/lib/nemoclaw"; +const ROOT_RUNTIME_DIRECTORY = "/run/nemoclaw"; +const ROOT_OWNED_DIRECTORY_MODE = 0o755; +const MAX_TRUST_BUNDLE_BYTES = 4 * 1024 * 1024; +const HERMES_MANAGED_CONFIG_FILES = [ + "/sandbox/.hermes/config.yaml", + "/sandbox/.hermes/.env", +] as const; +const FIXED_PATH = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"; +const SHA256_RE = /^[a-f0-9]{64}$/u; +const MANAGED_STARTUP_COMPLETION_SCHEMA_VERSION = 1; +const MAX_MANAGED_STARTUP_COMPLETION_BYTES = 4096; +const MAX_MANAGED_STARTUP_RUNTIME_ENVIRONMENT_BYTES = 512 * 1024; + +export type ManagedStartupImageIdentity = "root" | "sandbox"; + +export interface ManagedStartupImageActionCommand { + readonly action: + | "generate-agent-config" + | "messaging-runtime-setup" + | "messaging-post-agent-install"; + readonly runAs: ManagedStartupImageIdentity; + readonly argv: readonly string[]; +} + +export interface ManagedStartupImageApplyResult { + readonly agent: ManagedStartupAgent; + readonly adapterApplied: boolean; + readonly fingerprint: string; + readonly runtimeEnvironmentFile: string; +} + +export interface ManagedStartupRootApplyResult extends ManagedStartupImageApplyResult { + readonly transactionPending: boolean; +} + +export interface ManagedStartupCompletionMarker { + readonly schemaVersion: typeof MANAGED_STARTUP_COMPLETION_SCHEMA_VERSION; + readonly agent: ManagedStartupAgent; + readonly profileFingerprint: string; + readonly runtimeEnvironmentSha256: string; + readonly corporateCaMerged: boolean; +} + +export class ManagedStartupImageRuntimeError extends Error { + constructor(message: string) { + super(`Managed startup image application failed: ${message}`); + this.name = "ManagedStartupImageRuntimeError"; + } +} + +type Environment = Record; + +interface CommandResult { + readonly status: number; + readonly stdout: string; + readonly stderr: string; +} + +function fail(message: string): never { + throw new ManagedStartupImageRuntimeError(message); +} + +function exactAgent(value: string): ManagedStartupAgent { + if ((MANAGED_STARTUP_AGENTS as readonly string[]).includes(value)) { + return value as ManagedStartupAgent; + } + return fail(`unsupported agent ${JSON.stringify(value)}`); +} + +function managedTransactionProfile( + expectedAgentInput: string, + env: Environment = process.env, +): ManagedStartupProfile { + requireRoot(); + const expectedAgent = exactAgent(expectedAgentInput); + if (env.NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION !== "1") { + fail("shared-state transactions require a complete managed image"); + } + const encodedProfile = env[MANAGED_STARTUP_PROFILE_ENV]; + if (!encodedProfile) fail(`${MANAGED_STARTUP_PROFILE_ENV} is required`); + const profile = decodeManagedStartupProfile(encodedProfile); + if (profile.agent !== expectedAgent) { + fail(`shared-state transaction profile targets ${profile.agent}, expected ${expectedAgent}`); + } + return profile; +} + +function requireRoot(): void { + if (process.geteuid?.() !== 0) { + fail("managed startup requires container effective uid 0"); + } +} + +function modeOf(stat: fs.Stats): number { + return stat.mode & 0o777; +} + +function requireRootOwnedDirectory(target: string, mode: number): void { + let stat: fs.Stats; + try { + stat = fs.lstatSync(target); + } catch { + fail(`required root-owned directory is missing: ${target}`); + } + if ( + stat.isSymbolicLink() || + !stat.isDirectory() || + stat.uid !== 0 || + stat.gid !== 0 || + modeOf(stat) !== mode + ) { + fail(`${target} must be a root:root directory with mode ${mode.toString(8)}`); + } +} + +function ensureRootOwnedDirectory(target: string, mode = ROOT_OWNED_DIRECTORY_MODE): void { + const parent = path.dirname(target); + const parentStat = fs.lstatSync(parent); + if ( + parentStat.isSymbolicLink() || + !parentStat.isDirectory() || + parentStat.uid !== 0 || + parentStat.gid !== 0 || + (modeOf(parentStat) & 0o022) !== 0 + ) { + fail(`refusing unsafe parent directory for ${target}`); + } + try { + fs.mkdirSync(target, { mode }); + fs.chownSync(target, 0, 0); + fs.chmodSync(target, mode); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EEXIST") { + fail(`could not create ${target}`); + } + } + requireRootOwnedDirectory(target, mode); +} + +function requireSafeExistingRootTarget(target: string): void { + let stat: fs.Stats; + try { + stat = fs.lstatSync(target); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return; + fail(`could not inspect ${target}`); + } + if ( + stat.isSymbolicLink() || + !stat.isFile() || + stat.nlink !== 1 || + stat.uid !== 0 || + stat.gid !== 0 + ) { + fail(`refusing to replace unsafe root-owned file ${target}`); + } +} + +function atomicWriteRootFile(target: string, contents: string | Buffer, mode: number): void { + const parent = path.dirname(target); + const parentStat = fs.lstatSync(parent); + if ( + parentStat.isSymbolicLink() || + !parentStat.isDirectory() || + parentStat.uid !== 0 || + parentStat.gid !== 0 || + (modeOf(parentStat) & 0o022) !== 0 + ) { + fail(`refusing unsafe root-owned file parent ${parent}`); + } + requireSafeExistingRootTarget(target); + const temporary = path.join( + parent, + `.${path.basename(target)}.${randomBytes(12).toString("hex")}`, + ); + let descriptor: number | undefined; + try { + descriptor = fs.openSync( + temporary, + fs.constants.O_CREAT | fs.constants.O_EXCL | fs.constants.O_WRONLY | fs.constants.O_NOFOLLOW, + 0o600, + ); + fs.fchownSync(descriptor, 0, 0); + fs.writeFileSync(descriptor, contents); + fs.fchmodSync(descriptor, mode); + fs.fsyncSync(descriptor); + fs.closeSync(descriptor); + descriptor = undefined; + fs.renameSync(temporary, target); + } catch (error) { + if (descriptor !== undefined) fs.closeSync(descriptor); + try { + fs.unlinkSync(temporary); + } catch { + // Preserve the primary write failure. + } + fail(`could not atomically write ${target}: ${(error as Error).message}`); + } + const stat = fs.lstatSync(target); + if ( + stat.isSymbolicLink() || + !stat.isFile() || + stat.nlink !== 1 || + stat.uid !== 0 || + stat.gid !== 0 || + modeOf(stat) !== mode + ) { + fail(`root-owned output failed metadata verification: ${target}`); + } +} + +function removeSafeRootFile(target: string): void { + requireSafeExistingRootTarget(target); + try { + fs.unlinkSync(target); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") { + fail(`could not remove ${target}`); + } + } +} + +function trustedExecutable(target: string): boolean { + try { + const stat = fs.lstatSync(target); + return ( + !stat.isSymbolicLink() && + stat.isFile() && + stat.uid === 0 && + stat.gid === 0 && + (modeOf(stat) & 0o022) === 0 && + (modeOf(stat) & 0o111) !== 0 + ); + } catch { + return false; + } +} + +function readSandboxIdentity(): { readonly uid: string; readonly gid: string } { + const readId = (flag: "-u" | "-g"): string => { + const result = spawnSync("/usr/bin/id", [flag, "sandbox"], { + encoding: "utf8", + env: { PATH: FIXED_PATH }, + }); + const value = result.stdout.trim(); + if (result.status !== 0 || !/^[1-9][0-9]*$/u.test(value)) { + fail("could not resolve the sandbox account"); + } + return value; + }; + return { uid: readId("-u"), gid: readId("-g") }; +} + +function sandboxPrefix(): readonly string[] { + if (trustedExecutable("/usr/local/bin/gosu")) { + return ["/usr/local/bin/gosu", "sandbox"]; + } + if (trustedExecutable("/usr/bin/setpriv")) { + const identity = readSandboxIdentity(); + return [ + "/usr/bin/setpriv", + `--reuid=${identity.uid}`, + `--regid=${identity.gid}`, + "--init-groups", + "--", + ]; + } + return fail("a trusted gosu or setpriv executable is required"); +} + +function commandEnvironment( + configurationEnvironment: Readonly>, +): NodeJS.ProcessEnv { + const env: NodeJS.ProcessEnv = { + ...process.env, + ...configurationEnvironment, + HOME: "/sandbox", + PATH: FIXED_PATH, + NPM_CONFIG_OFFLINE: "true", + npm_config_offline: "true", + PIP_DISABLE_PIP_VERSION_CHECK: "1", + PIP_NO_INDEX: "1", + UV_OFFLINE: "1", + }; + delete env[MANAGED_STARTUP_PROFILE_ENV]; + delete env[MANAGED_STARTUP_CA_ENV]; + return env; +} + +function execute( + argv: readonly string[], + runAs: ManagedStartupImageIdentity, + configurationEnvironment: Readonly>, + capture = false, +): CommandResult { + if (argv.length === 0) fail("refusing an empty managed startup command"); + const command = runAs === "sandbox" ? [...sandboxPrefix(), ...argv] : [...argv]; + const result = spawnSync(command[0] as string, command.slice(1), { + encoding: "utf8", + env: commandEnvironment(configurationEnvironment), + stdio: capture ? "pipe" : "inherit", + }); + if (result.error) { + fail(`could not execute ${argv[0]}: ${result.error.message}`); + } + if (result.status !== 0) { + const detail = capture ? `: ${(result.stderr || result.stdout).trim()}` : ""; + fail(`${argv[0]} exited with status ${String(result.status ?? "unknown")}${detail}`); + } + return { + status: result.status, + stdout: result.stdout ?? "", + stderr: result.stderr ?? "", + }; +} + +function generatorCommand(agent: ManagedStartupAgent): readonly string[] { + switch (agent) { + case "openclaw": + return [ + "/usr/local/bin/node", + "--experimental-strip-types", + "/scripts/generate-openclaw-config.mts", + ]; + case "hermes": + return [ + "/usr/local/bin/node", + "--experimental-strip-types", + "/opt/nemoclaw-hermes-config/generate-config.ts", + ]; + case "langchain-deepagents-code": + return [ + "/usr/local/bin/node", + "--experimental-strip-types", + "/opt/nemoclaw-deepagents-code/generate-config.ts", + ]; + } +} + +function messagingCommand( + agent: "openclaw" | "hermes", + phase: "runtime-setup" | "post-agent-install", +): readonly string[] { + return [ + "/usr/local/bin/node", + "--experimental-strip-types", + "/src/lib/messaging/applier/build/messaging-build-applier.mts", + "--agent", + agent, + "--phase", + phase, + ...(phase === "post-agent-install" ? ["--managed-startup-runtime"] : []), + ]; +} + +/** + * Convert the mapper's closed action vocabulary into executable image steps. + * There is deliberately no mapping for agent-install or any package manager. + */ +export function buildManagedStartupImageActionPlan( + environment: ManagedStartupAgentEnvironment, +): readonly ManagedStartupImageActionCommand[] { + const commands: ManagedStartupImageActionCommand[] = []; + for (const action of environment.actions) { + if (action.kind === "configure-dashboard") continue; + if (action.kind === "generate-agent-config") { + commands.push({ + action: "generate-agent-config", + runAs: action.runAs, + argv: generatorCommand(action.agent), + }); + continue; + } + commands.push({ + action: + action.phase === "runtime-setup" + ? "messaging-runtime-setup" + : "messaging-post-agent-install", + runAs: action.runAs, + argv: messagingCommand(action.agent, action.phase), + }); + } + return Object.freeze(commands.map((command) => Object.freeze(command))); +} + +function prepareMessagingRuntimeTarget(mode: "apply" | "clear"): void { + if (mode === "clear") { + removeSafeRootFile(MESSAGING_RUNTIME_PLAN_FILE); + return; + } + requireSafeExistingRootTarget(MESSAGING_RUNTIME_PLAN_FILE); + try { + fs.unlinkSync(MESSAGING_RUNTIME_PLAN_FILE); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") { + fail("could not prepare the messaging runtime-plan target"); + } + } +} + +function verifyMessagingRuntimeTarget(mode: "apply" | "clear"): void { + if (mode === "clear") { + if (fs.existsSync(MESSAGING_RUNTIME_PLAN_FILE)) { + fail("clear messaging profile left a runtime-plan artifact"); + } + return; + } + const stat = fs.lstatSync(MESSAGING_RUNTIME_PLAN_FILE); + if ( + stat.isSymbolicLink() || + !stat.isFile() || + stat.nlink !== 1 || + stat.uid !== 0 || + stat.gid !== 0 || + modeOf(stat) !== 0o644 + ) { + fail("messaging runtime-plan artifact failed root ownership validation"); + } +} + +function runInternalSandboxAction( + action: "write-openclaw-hash" | "write-hermes-compat-hash", + configurationEnvironment: Readonly>, + extraEnvironment: Readonly> = {}, +): void { + execute( + ["/usr/local/bin/node", MANAGED_STARTUP_RUNTIME_EXECUTABLE, `--internal-${action}`], + "sandbox", + { ...configurationEnvironment, ...extraEnvironment }, + ); +} + +function sealOpenClawConfiguration( + configurationEnvironment: Readonly>, +): void { + const validation = execute( + ["/usr/local/bin/openclaw", "config", "validate", "--json"], + "sandbox", + { + ...configurationEnvironment, + OPENCLAW_CONFIG_PATH: "/sandbox/.openclaw/openclaw.json", + }, + true, + ); + let parsed: unknown; + try { + parsed = JSON.parse(validation.stdout); + } catch { + fail("OpenClaw config validation did not emit JSON"); + } + if ( + typeof parsed !== "object" || + parsed === null || + (parsed as Record).valid !== true + ) { + fail("OpenClaw rejected the generated managed startup config"); + } + runInternalSandboxAction("write-openclaw-hash", configurationEnvironment); +} + +interface StableRegularFile { + readonly bytes: Buffer; + readonly stat: fs.BigIntStats; +} + +interface NumericIdentity { + readonly uid: number; + readonly gid: number; +} + +function sameStableFileMetadata(left: fs.BigIntStats, right: fs.BigIntStats): boolean { + return ( + left.dev === right.dev && + left.ino === right.ino && + left.mode === right.mode && + left.nlink === right.nlink && + left.uid === right.uid && + left.gid === right.gid && + left.size === right.size && + left.mtimeNs === right.mtimeNs && + left.ctimeNs === right.ctimeNs + ); +} + +function readStableRegularFileSnapshot(target: string, maxBytes: number): StableRegularFile { + if (typeof fs.constants.O_NOFOLLOW !== "number") { + fail("O_NOFOLLOW is unavailable for managed startup file reads"); + } + const nonblock = typeof fs.constants.O_NONBLOCK === "number" ? fs.constants.O_NONBLOCK : 0; + let descriptor: number; + try { + descriptor = fs.openSync(target, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW | nonblock); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") throw error; + fail(`refusing unsafe or unreadable file ${target}`); + } + + try { + const before = fs.fstatSync(descriptor, { bigint: true }); + if ( + !before.isFile() || + before.nlink !== 1n || + before.size < 1n || + before.size > BigInt(maxBytes) + ) { + fail(`refusing unsafe or oversized file ${target}`); + } + + const bytes = Buffer.alloc(Number(before.size)); + let offset = 0; + while (offset < bytes.length) { + const bytesRead = fs.readSync(descriptor, bytes, offset, bytes.length - offset, offset); + if (bytesRead === 0) break; + offset += bytesRead; + } + const overflow = Buffer.alloc(1); + const overflowBytes = fs.readSync(descriptor, overflow, 0, 1, offset); + const after = fs.fstatSync(descriptor, { bigint: true }); + if (offset !== bytes.length || overflowBytes !== 0 || !sameStableFileMetadata(before, after)) { + fail(`${target} changed while it was read`); + } + return { bytes, stat: before }; + } finally { + try { + fs.closeSync(descriptor); + } catch { + fail(`could not close safely opened file ${target}`); + } + } +} + +export function readStableRegularFile(target: string, maxBytes: number): Buffer { + return readStableRegularFileSnapshot(target, maxBytes).bytes; +} + +/** + * Restore the mutable Hermes image contract after its sandbox-side generator + * atomically replaces config.yaml or .env with mode 0600. The mode transition + * is performed through the already-authenticated descriptor, never by path. + * + * Shields-up turns these files into root:root 0444 trust anchors. That state is + * valid on an already-committed replay and must not be made mutable again. + */ +export function normalizeHermesManagedConfigDescriptor( + target: string, + sandboxIdentity: NumericIdentity, +): void { + if ( + !Number.isSafeInteger(sandboxIdentity.uid) || + sandboxIdentity.uid <= 0 || + !Number.isSafeInteger(sandboxIdentity.gid) || + sandboxIdentity.gid <= 0 + ) { + fail("invalid sandbox identity for Hermes descriptor normalization"); + } + if (typeof fs.constants.O_NOFOLLOW !== "number") { + fail("O_NOFOLLOW is unavailable for Hermes descriptor normalization"); + } + const nonblock = typeof fs.constants.O_NONBLOCK === "number" ? fs.constants.O_NONBLOCK : 0; + let descriptor: number; + try { + descriptor = fs.openSync(target, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW | nonblock); + } catch { + fail(`refusing unsafe Hermes managed config descriptor ${target}`); + } + + try { + const before = fs.fstatSync(descriptor, { bigint: true }); + const beforeMode = Number(before.mode & 0o777n); + const mutable = + before.uid === BigInt(sandboxIdentity.uid) && + before.gid === BigInt(sandboxIdentity.gid) && + (beforeMode === 0o600 || beforeMode === 0o640); + const shielded = before.uid === 0n && before.gid === 0n && beforeMode === 0o444; + if (!before.isFile() || before.nlink !== 1n || (!mutable && !shielded)) { + fail(`refusing unexpected Hermes managed config descriptor ${target}`); + } + + const expectedMode = mutable ? 0o640 : 0o444; + if (mutable && beforeMode === 0o600) { + try { + fs.fchmodSync(descriptor, expectedMode); + } catch { + fail(`could not normalize Hermes managed config descriptor ${target}`); + } + } + + const after = fs.fstatSync(descriptor, { bigint: true }); + let pathAfter: fs.BigIntStats; + try { + pathAfter = fs.lstatSync(target, { bigint: true }); + } catch { + fail(`Hermes managed config descriptor disappeared during normalization: ${target}`); + } + const expectedUid = mutable ? BigInt(sandboxIdentity.uid) : 0n; + const expectedGid = mutable ? BigInt(sandboxIdentity.gid) : 0n; + if ( + !after.isFile() || + after.nlink !== 1n || + after.dev !== before.dev || + after.ino !== before.ino || + after.uid !== expectedUid || + after.gid !== expectedGid || + Number(after.mode & 0o777n) !== expectedMode || + after.size !== before.size || + after.mtimeNs !== before.mtimeNs || + pathAfter.isSymbolicLink() || + !pathAfter.isFile() || + !sameStableFileMetadata(after, pathAfter) + ) { + fail(`Hermes managed config descriptor changed during normalization: ${target}`); + } + } finally { + try { + fs.closeSync(descriptor); + } catch { + fail(`could not close Hermes managed config descriptor ${target}`); + } + } +} + +function normalizeHermesManagedConfiguration(): void { + const identity = readSandboxIdentity(); + const sandboxIdentity = { + uid: Number(identity.uid), + gid: Number(identity.gid), + }; + for (const target of HERMES_MANAGED_CONFIG_FILES) { + normalizeHermesManagedConfigDescriptor(target, sandboxIdentity); + } +} + +function sealHermesConfiguration(configurationEnvironment: Readonly>): void { + const configPath = "/sandbox/.hermes/config.yaml"; + const envPath = "/sandbox/.hermes/.env"; + const config = readStableRegularFile(configPath, 4 * 1024 * 1024); + const env = readStableRegularFile(envPath, 512 * 1024); + const digest = execute( + [ + "/opt/hermes/.venv/bin/python3", + "-I", + "/usr/local/lib/nemoclaw/build-hermes-mcp-digest.py", + "--guard", + "/usr/local/lib/nemoclaw/hermes-runtime-config-guard.py", + "--config", + configPath, + ], + "root", + configurationEnvironment, + true, + ).stdout.trim(); + if (!SHA256_RE.test(digest)) { + fail("Hermes MCP digest helper returned an invalid digest"); + } + const hashText = [ + `${createHash("sha256").update(config).digest("hex")} ${configPath}`, + `${createHash("sha256").update(env).digest("hex")} ${envPath}`, + `# nemoclaw-hermes-mcp-state-v1 intended=${digest} applied=${digest}`, + "", + ].join("\n"); + atomicWriteRootFile("/etc/nemoclaw/hermes.config-hash", hashText, 0o444); + runInternalSandboxAction("write-hermes-compat-hash", configurationEnvironment, { + NEMOCLAW_MANAGED_HERMES_HASH_B64: Buffer.from(hashText, "utf8").toString("base64"), + }); +} + +function installRootOwnedMaterials(materials: readonly ManagedStartupAgentMaterial[]): void { + for (const material of materials) { + if (material.kind !== "root-owned-file") continue; + if (material.owner !== "root" || material.group !== "root" || material.mode !== 0o444) { + fail(`unsupported root-owned material contract for ${material.path}`); + } + atomicWriteRootFile(material.path, material.contents, material.mode); + } +} + +function verifyRootOwnedMaterials(materials: readonly ManagedStartupAgentMaterial[]): void { + for (const material of materials) { + if (material.kind !== "root-owned-file") continue; + const expected = Buffer.from(material.contents, "utf8"); + const { bytes, stat } = readStableRegularFileSnapshot(material.path, expected.length); + if ( + stat.nlink !== 1n || + stat.uid !== 0n || + stat.gid !== 0n || + Number(stat.mode & 0o777n) !== material.mode || + !bytes.equals(expected) + ) { + fail(`committed root-owned material drifted: ${material.path}`); + } + } +} + +function installCorporateCa(corporateCaPath: string | null): void { + if (corporateCaPath === null) { + removeSafeRootFile(MANAGED_STARTUP_CORPORATE_CA_FILE); + return; + } + const bytes = readStableRegularFile(corporateCaPath, 128 * 1024); + atomicWriteRootFile(MANAGED_STARTUP_CORPORATE_CA_FILE, bytes, 0o444); +} + +function safeTrustBundle(target: string): Buffer | null { + try { + const { bytes, stat } = readStableRegularFileSnapshot(target, MAX_TRUST_BUNDLE_BYTES); + if (Number(stat.mode & 0o022n) !== 0) { + fail(`refusing unsafe trust bundle ${target}`); + } + return bytes; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; + throw error; + } +} + +function mergeCorporateCa(corporateCaPath: string | null): boolean { + if (corporateCaPath === null) { + removeSafeRootFile(MANAGED_STARTUP_MERGED_CA_FILE); + return false; + } + const corporate = readStableRegularFile(corporateCaPath, 128 * 1024); + const candidates = [ + "/etc/openshell-tls/ca-bundle.pem", + process.env.SSL_CERT_FILE ?? "", + "/etc/ssl/certs/ca-certificates.crt", + ].filter( + (candidate, index, values) => + candidate && + candidate !== MANAGED_STARTUP_MERGED_CA_FILE && + values.indexOf(candidate) === index, + ); + let base: Buffer | null = null; + for (const candidate of candidates) { + base = safeTrustBundle(candidate); + if (base) break; + } + const merged = Buffer.concat([ + ...(base ? [base, Buffer.from("\n", "utf8")] : []), + corporate, + ...(corporate.at(-1) === 0x0a ? [] : [Buffer.from("\n", "utf8")]), + ]); + atomicWriteRootFile(MANAGED_STARTUP_MERGED_CA_FILE, merged, 0o444); + return true; +} + +function shellSingleQuote(value: string): string { + if (value.includes("\0") || /[\r\n]/u.test(value)) { + fail("runtime environment values must be single-line text"); + } + return `'${value.replaceAll("'", `'\"'\"'`)}'`; +} + +export function serializeManagedStartupRuntimeEnvironment( + environment: Readonly>, + corporateCaMerged: boolean, + configurationEnvironment: Readonly> = {}, +): string { + const { output, unsetNames } = materializeManagedStartupRuntimeEnvironment( + environment, + corporateCaMerged, + configurationEnvironment, + ); + const unsetLines = unsetNames.map((name) => `unset ${name}`); + const exportLines = Object.entries(output) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([name, value]) => `export ${name}=${shellSingleQuote(value)}`); + return `${[...unsetLines, ...exportLines].join("\n")}\n`; +} + +function materializeManagedStartupRuntimeEnvironment( + environment: Readonly>, + corporateCaMerged: boolean, + configurationEnvironment: Readonly> = {}, +): { output: Record; unsetNames: string[] } { + const output: Record = { + ...environment, + NEMOCLAW_MANAGED_STARTUP_APPLIED: "1", + }; + if (corporateCaMerged) { + for (const name of [ + "CURL_CA_BUNDLE", + "GIT_SSL_CAINFO", + "NODE_EXTRA_CA_CERTS", + "REQUESTS_CA_BUNDLE", + "SSL_CERT_FILE", + ]) { + output[name] = MANAGED_STARTUP_MERGED_CA_FILE; + } + output._NEMOCLAW_CORPORATE_CA_MERGED = "1"; + } + for (const name of [...Object.keys(configurationEnvironment), ...Object.keys(output)]) { + if (!/^[A-Za-z_][A-Za-z0-9_]*$/u.test(name)) { + fail(`invalid runtime environment key ${JSON.stringify(name)}`); + } + } + const unsetNames = Object.keys(configurationEnvironment) + .filter((name) => !Object.hasOwn(environment, name)) + .sort(); + return { output, unsetNames }; +} + +export function serializeManagedStartupCompletionMarker( + marker: ManagedStartupCompletionMarker, +): string { + if ( + marker.schemaVersion !== MANAGED_STARTUP_COMPLETION_SCHEMA_VERSION || + !(MANAGED_STARTUP_AGENTS as readonly string[]).includes(marker.agent) || + !SHA256_RE.test(marker.profileFingerprint) || + !SHA256_RE.test(marker.runtimeEnvironmentSha256) || + typeof marker.corporateCaMerged !== "boolean" + ) { + fail("managed startup completion marker is invalid"); + } + return `${JSON.stringify({ + agent: marker.agent, + corporateCaMerged: marker.corporateCaMerged, + profileFingerprint: marker.profileFingerprint, + runtimeEnvironmentSha256: marker.runtimeEnvironmentSha256, + schemaVersion: marker.schemaVersion, + })}\n`; +} + +function parseManagedStartupCompletionMarker(text: string): ManagedStartupCompletionMarker { + let parsed: unknown; + try { + parsed = JSON.parse(text); + } catch { + fail("managed startup completion marker is not valid JSON"); + } + if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) { + fail("managed startup completion marker must be an object"); + } + const record = parsed as Record; + const expectedKeys = [ + "agent", + "corporateCaMerged", + "profileFingerprint", + "runtimeEnvironmentSha256", + "schemaVersion", + ]; + if ( + Object.keys(record).sort().join(",") !== expectedKeys.sort().join(",") || + record.schemaVersion !== MANAGED_STARTUP_COMPLETION_SCHEMA_VERSION || + typeof record.agent !== "string" || + !(MANAGED_STARTUP_AGENTS as readonly string[]).includes(record.agent) || + typeof record.profileFingerprint !== "string" || + !SHA256_RE.test(record.profileFingerprint) || + typeof record.runtimeEnvironmentSha256 !== "string" || + !SHA256_RE.test(record.runtimeEnvironmentSha256) || + typeof record.corporateCaMerged !== "boolean" + ) { + fail("managed startup completion marker has an invalid schema"); + } + const marker = { + schemaVersion: MANAGED_STARTUP_COMPLETION_SCHEMA_VERSION, + agent: record.agent as ManagedStartupAgent, + profileFingerprint: record.profileFingerprint, + runtimeEnvironmentSha256: record.runtimeEnvironmentSha256, + corporateCaMerged: record.corporateCaMerged, + } as const; + if (serializeManagedStartupCompletionMarker(marker) !== text) { + fail("managed startup completion marker is not canonical"); + } + return marker; +} + +export function verifyManagedStartupImageCompletion( + expectedAgentInput: string, + expectedFingerprint: string, + completionFile: string = MANAGED_STARTUP_COMPLETION_FILE, + runtimeEnvironmentFile: string = MANAGED_STARTUP_RUNTIME_ENV_FILE, +): { readonly agent: ManagedStartupAgent; readonly fingerprint: string } { + const expectedAgent = exactAgent(expectedAgentInput); + if (!SHA256_RE.test(expectedFingerprint)) { + fail("startup completion expected profile fingerprint is invalid"); + } + const { bytes, stat } = readStableRegularFileSnapshot( + completionFile, + MAX_MANAGED_STARTUP_COMPLETION_BYTES, + ); + if ( + stat.nlink !== 1n || + stat.uid !== 0n || + stat.gid !== 0n || + Number(stat.mode & 0o777n) !== 0o444 + ) { + fail("managed startup completion marker must be root:root mode 0444"); + } + const marker = parseManagedStartupCompletionMarker(bytes.toString("utf8")); + if (marker.agent !== expectedAgent || marker.profileFingerprint !== expectedFingerprint) { + fail("managed startup completion marker does not match the requested profile"); + } + const runtimeEnvironment = readStableRegularFileSnapshot( + runtimeEnvironmentFile, + MAX_MANAGED_STARTUP_RUNTIME_ENVIRONMENT_BYTES, + ); + if ( + runtimeEnvironment.stat.nlink !== 1n || + runtimeEnvironment.stat.uid !== 0n || + runtimeEnvironment.stat.gid !== 0n || + Number(runtimeEnvironment.stat.mode & 0o777n) !== 0o444 + ) { + fail("managed startup runtime environment must be root:root mode 0444"); + } + const runtimeEnvironmentSha256 = createHash("sha256") + .update(runtimeEnvironment.bytes) + .digest("hex"); + if (runtimeEnvironmentSha256 !== marker.runtimeEnvironmentSha256) { + fail("managed startup completion marker runtime environment digest mismatch"); + } + return { agent: expectedAgent, fingerprint: expectedFingerprint }; +} + +export function waitForManagedStartupImageCompletion( + expectedAgentInput: string, + expectedFingerprint: string, + timeoutSeconds = 600, +): { readonly agent: ManagedStartupAgent; readonly fingerprint: string } { + if (!Number.isSafeInteger(timeoutSeconds) || timeoutSeconds < 1 || timeoutSeconds > 3600) { + fail("startup completion wait timeout must be an integer from 1 to 3600 seconds"); + } + const deadline = Date.now() + timeoutSeconds * 1000; + while (true) { + try { + return verifyManagedStartupImageCompletion(expectedAgentInput, expectedFingerprint); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + if (Date.now() >= deadline) { + fail(`startup completion was not published within ${String(timeoutSeconds)} seconds`); + } + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 250); + } + } +} + +function applyAdapter(context: ManagedStartupAdapterContext): void { + const mapped = mapManagedStartupProfileToAgentEnvironment(context.profile); + if (mapped.agent !== context.agent) { + fail(`mapped ${mapped.agent} environment for ${context.agent}`); + } + for (const action of mapped.actions) { + if (action.kind === "configure-dashboard") continue; + if (action.kind === "apply-messaging-plan") { + if (action.phase === "runtime-setup") { + prepareMessagingRuntimeTarget(action.mode); + } + execute( + messagingCommand(action.agent, action.phase), + action.runAs, + mapped.configurationEnvironment, + ); + if (action.phase === "runtime-setup") { + verifyMessagingRuntimeTarget(action.mode); + } + continue; + } + execute(generatorCommand(action.agent), action.runAs, mapped.configurationEnvironment); + } + + switch (context.agent) { + case "openclaw": + sealOpenClawConfiguration(mapped.configurationEnvironment); + break; + case "hermes": + sealHermesConfiguration(mapped.configurationEnvironment); + // Normalize before the coordinator commits a newly applied profile so + // the durable transaction never records generator-created 0600 files as + // a completed mutable image contract. + normalizeHermesManagedConfiguration(); + break; + case "langchain-deepagents-code": + break; + } + installRootOwnedMaterials(mapped.materials); + installCorporateCa(context.corporateCaPath); + mergeCorporateCa(context.corporateCaPath); +} + +function adapters(): readonly ManagedStartupAgentAdapter[] { + return MANAGED_STARTUP_AGENTS.map((agent) => ({ + agent, + apply: (context: ManagedStartupAdapterContext) => applyAdapter(context), + })); +} + +export async function applyManagedStartupImageProfile( + expectedAgentInput: string, + env: Environment = process.env, +): Promise { + requireRoot(); + const expectedAgent = exactAgent(expectedAgentInput); + if (env.NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION !== "1") { + fail("startup profiles require a complete managed image"); + } + const encodedProfile = env[MANAGED_STARTUP_PROFILE_ENV]; + if (!encodedProfile) fail(`${MANAGED_STARTUP_PROFILE_ENV} is required`); + + ensureRootOwnedDirectory(ROOT_STATE_PARENT); + ensureRootOwnedDirectory(ROOT_RUNTIME_DIRECTORY); + const result = await coordinateManagedStartupApplication( + { + encodedProfile, + expectedAgent, + ...(env[MANAGED_STARTUP_CA_ENV] === undefined + ? {} + : { corporateCaB64: env[MANAGED_STARTUP_CA_ENV] }), + }, + adapters(), + ); + const mapped = mapManagedStartupProfileToAgentEnvironment(result.application.profile); + if (expectedAgent === "hermes" && !result.adapterApplied) { + // Committed startup replays still repair generator-created 0600 files, + // while the descriptor guard preserves root-owned shields-up files. + normalizeHermesManagedConfiguration(); + } + let corporateCaMerged: boolean; + if (result.adapterApplied) { + corporateCaMerged = result.application.corporateCaPath !== null; + } else { + verifyRootOwnedMaterials(mapped.materials); + if (result.application.corporateCaPath === null) { + if (fs.existsSync(MANAGED_STARTUP_CORPORATE_CA_FILE)) { + fail("committed profile without a corporate CA has a stale CA material"); + } + } else { + const expected = readStableRegularFile(result.application.corporateCaPath, 128 * 1024); + const installed = readStableRegularFile(MANAGED_STARTUP_CORPORATE_CA_FILE, 128 * 1024); + if (!expected.equals(installed)) { + fail("committed corporate CA material drifted"); + } + } + corporateCaMerged = mergeCorporateCa(result.application.corporateCaPath); + } + const runtimeEnvironment = serializeManagedStartupRuntimeEnvironment( + mapped.runtimeEnvironment, + corporateCaMerged, + mapped.configurationEnvironment, + ); + // This handoff is intentionally readable by the sandbox account only after + // the root-owned completion marker authenticates its exact digest. It + // contains the secret-free mapped profile environment, never provider + // credentials or the raw corporate-CA transport. + atomicWriteRootFile(MANAGED_STARTUP_RUNTIME_ENV_FILE, runtimeEnvironment, 0o444); + atomicWriteRootFile( + MANAGED_STARTUP_COMPLETION_FILE, + serializeManagedStartupCompletionMarker({ + schemaVersion: MANAGED_STARTUP_COMPLETION_SCHEMA_VERSION, + agent: expectedAgent, + profileFingerprint: result.application.fingerprint, + runtimeEnvironmentSha256: createHash("sha256") + .update(runtimeEnvironment, "utf8") + .digest("hex"), + corporateCaMerged, + }), + 0o444, + ); + return { + agent: expectedAgent, + adapterApplied: result.adapterApplied, + fingerprint: result.application.fingerprint, + runtimeEnvironmentFile: MANAGED_STARTUP_RUNTIME_ENV_FILE, + }; +} + +function completionAlreadyPublished(request: ManagedStartupRootApplyRequest): boolean { + try { + verifyManagedStartupImageCompletion(request.agent, request.profileFingerprint); + return true; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return false; + throw error; + } +} + +export async function applyManagedStartupRootRequest( + request: ManagedStartupRootApplyRequest, +): Promise { + requireRoot(); + if (completionAlreadyPublished(request)) { + return { + agent: request.agent, + adapterApplied: false, + fingerprint: request.profileFingerprint, + runtimeEnvironmentFile: MANAGED_STARTUP_RUNTIME_ENV_FILE, + transactionPending: false, + }; + } + const profile = decodeManagedStartupProfile(request.encodedProfile); + if ( + profile.agent !== request.agent || + fingerprintManagedStartupProfile(profile) !== request.profileFingerprint + ) { + fail("root application request identity does not match its profile"); + } + ensureRootOwnedDirectory(ROOT_STATE_PARENT); + beginManagedStartupSharedStateTransaction(profile); + const result = await applyManagedStartupImageProfile(request.agent, { + HOME: "/root", + PATH: FIXED_PATH, + NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION: "1", + [MANAGED_STARTUP_PROFILE_ENV]: request.encodedProfile, + ...(request.corporateCaB64 === null + ? {} + : { [MANAGED_STARTUP_CA_ENV]: request.corporateCaB64 }), + }); + return { ...result, transactionPending: true }; +} + +function readBoundedRootApplyStdin(): string { + const chunks: Buffer[] = []; + let total = 0; + while (true) { + const chunk = Buffer.alloc(16 * 1024); + const read = fs.readSync(0, chunk, 0, chunk.length, null); + if (read === 0) break; + total += read; + if (total > MANAGED_STARTUP_ROOT_APPLY_MAX_BYTES) { + fail("root application stdin exceeds its bounded transport"); + } + chunks.push(chunk.subarray(0, read)); + } + const bytes = Buffer.concat(chunks, total); + const text = bytes.toString("utf8"); + if (text.includes("\0") || !Buffer.from(text, "utf8").equals(bytes)) { + fail("root application stdin must be valid UTF-8 without NUL bytes"); + } + return text; +} + +function writeSandboxFileAtomically(target: string, contents: string, mode: number): void { + const parent = path.dirname(target); + const parentStat = fs.lstatSync(parent); + if ( + parentStat.isSymbolicLink() || + !parentStat.isDirectory() || + parentStat.uid !== process.geteuid?.() || + parentStat.gid !== process.getegid?.() + ) { + fail(`refusing unsafe sandbox-owned directory ${parent}`); + } + const temporary = path.join( + parent, + `.${path.basename(target)}.${randomBytes(12).toString("hex")}`, + ); + let descriptor: number | undefined; + try { + descriptor = fs.openSync( + temporary, + fs.constants.O_CREAT | fs.constants.O_EXCL | fs.constants.O_WRONLY | fs.constants.O_NOFOLLOW, + 0o600, + ); + fs.writeFileSync(descriptor, contents); + fs.fchmodSync(descriptor, mode); + fs.fsyncSync(descriptor); + fs.closeSync(descriptor); + descriptor = undefined; + fs.renameSync(temporary, target); + } catch (error) { + if (descriptor !== undefined) fs.closeSync(descriptor); + try { + fs.unlinkSync(temporary); + } catch { + // Preserve the primary write failure. + } + fail(`could not write sandbox-owned file ${target}: ${(error as Error).message}`); + } +} + +function internalWriteOpenClawHash(): void { + if (process.geteuid?.() === 0) fail("sandbox hash writer must not run as root"); + const configPath = "/sandbox/.openclaw/openclaw.json"; + const config = readStableRegularFile(configPath, 16 * 1024 * 1024); + const text = `${createHash("sha256").update(config).digest("hex")} openclaw.json\n`; + writeSandboxFileAtomically("/sandbox/.openclaw/.config-hash", text, 0o660); +} + +function internalWriteHermesCompatHash(): void { + if (process.geteuid?.() === 0) fail("sandbox hash writer must not run as root"); + const encoded = process.env.NEMOCLAW_MANAGED_HERMES_HASH_B64 ?? ""; + if ( + encoded.length === 0 || + encoded.length > 4096 || + !/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/u.test(encoded) + ) { + fail("Hermes compatibility hash transport is invalid"); + } + const decoded = Buffer.from(encoded, "base64"); + if (decoded.toString("base64") !== encoded) { + fail("Hermes compatibility hash transport is non-canonical"); + } + writeSandboxFileAtomically("/sandbox/.hermes/.config-hash", decoded.toString("utf8"), 0o640); +} + +function readCliAgent(argv: readonly string[], expectedLength = 2): string { + const index = argv.indexOf("--agent"); + if (index < 0 || index + 1 >= argv.length || argv.length !== expectedLength) { + fail( + "usage: managed-startup-image-runtime [--apply-root-stdin|--wait-for-completion|--verify-completion|--begin-shared-state-transaction|--commit-shared-state-transaction] --agent ", + ); + } + return argv[index + 1] as string; +} + +function readCliFingerprint(argv: readonly string[]): string { + const index = argv.indexOf("--profile-fingerprint"); + if (index < 0 || index + 1 >= argv.length) { + fail("managed startup profile fingerprint argument is missing"); + } + return argv[index + 1] as string; +} + +export async function main(argv: readonly string[] = process.argv.slice(2)): Promise { + if (argv.length === 1 && argv[0] === "--internal-write-openclaw-hash") { + internalWriteOpenClawHash(); + return; + } + if (argv.length === 1 && argv[0] === "--internal-write-hermes-compat-hash") { + internalWriteHermesCompatHash(); + return; + } + if (argv.length === 3 && argv[0] === "--apply-root-stdin") { + const expectedAgent = exactAgent(readCliAgent(argv, 3)); + const request = parseManagedStartupRootApplyRequest(readBoundedRootApplyStdin()); + if (request.agent !== expectedAgent) { + fail(`root application request targets ${request.agent}, expected ${expectedAgent}`); + } + const result = await applyManagedStartupRootRequest(request); + console.log( + result.transactionPending + ? `[managed-startup] applied ${result.agent} profile ${result.fingerprint}; transaction pending` + : `[managed-startup] ${result.agent} profile ${result.fingerprint} was already complete`, + ); + return; + } + if ( + argv.length === 5 && + (argv[0] === "--verify-completion" || argv[0] === "--wait-for-completion") + ) { + const agent = readCliAgent(argv, 5); + const fingerprint = readCliFingerprint(argv); + const result = + argv[0] === "--wait-for-completion" + ? waitForManagedStartupImageCompletion(agent, fingerprint) + : verifyManagedStartupImageCompletion(agent, fingerprint); + console.log( + `[managed-startup] verified ${result.agent} profile ${result.fingerprint} completion`, + ); + return; + } + if (argv.length === 3 && argv[0] === "--begin-shared-state-transaction") { + const profile = managedTransactionProfile(readCliAgent(argv, 3)); + ensureRootOwnedDirectory(ROOT_STATE_PARENT); + const created = beginManagedStartupSharedStateTransaction(profile); + process.stdout.write(created ? "created\n" : "pending\n"); + return; + } + if ( + argv.length === 4 && + argv[0] === "--rollback-shared-state-transaction" && + argv[3] === "--read-only-receipt" + ) { + requireRoot(); + const agent = exactAgent(readCliAgent(argv, 4)); + const rolledBack = rollbackManagedStartupSharedStateTransaction(agent, { + transactionDirectory: MANAGED_STARTUP_SHARED_ROLLBACK_RECEIPT_DIRECTORY, + readOnlyReceipt: true, + }); + if (!rolledBack) fail("read-only shared-state rollback receipt is missing"); + console.log(`[managed-startup] verified and restored ${agent} shared state`); + return; + } + if (argv.length === 3 && argv[0] === "--commit-shared-state-transaction") { + requireRoot(); + const agent = exactAgent(readCliAgent(argv, 3)); + if (!commitManagedStartupSharedStateTransaction(agent)) { + fail("managed startup transaction is missing at commit"); + } + console.log(`[managed-startup] committed ${agent} shared state`); + return; + } + const result = await applyManagedStartupImageProfile(readCliAgent(argv)); + console.log( + result.adapterApplied + ? `[managed-startup] applied ${result.agent} profile ${result.fingerprint}` + : `[managed-startup] ${result.agent} profile ${result.fingerprint} is already committed`, + ); +} + +if (require.main === module) { + main().catch((error: unknown) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + }); +} diff --git a/src/lib/onboard/managed-startup/onboard-profile.ts b/src/lib/onboard/managed-startup/onboard-profile.ts new file mode 100644 index 0000000000..d822ae8a3b --- /dev/null +++ b/src/lib/onboard/managed-startup/onboard-profile.ts @@ -0,0 +1,224 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { SandboxMessagingPlan } from "../../messaging/manifest"; +import type { ToolDisclosure } from "../../tool-disclosure"; +import { resolveCorporateCa } from "../corporate-ca"; +import type { ResolvedCorporateCa } from "../corporate-ca-types"; +import type { DcodeAutoApprovalMode } from "../dcode-auto-approval"; +import type { HermesDashboardOnboardState } from "../hermes-dashboard"; +import { hasCredentialBearingHostProxyEnvironment } from "../host-proxy-env"; +import { + MANAGED_STARTUP_AGENTS, + type ManagedStartupAgent, + type ManagedStartupDashboard, +} from "./profile"; +import { + type BuiltManagedStartupProfile, + buildManagedStartupProfile, + type ManagedStartupResolvedInferenceInput, +} from "./profile-builder"; + +const PROFILE_ENVIRONMENT_INPUTS = { + openclaw: [ + "NEMOCLAW_AGENT_HEARTBEAT_EVERY", + "NEMOCLAW_AGENT_TIMEOUT", + "NEMOCLAW_CONTEXT_WINDOW", + "NEMOCLAW_EXTRA_AGENTS_JSON", + "NEMOCLAW_EXTRA_AGENTS_JSON_B64", + "NEMOCLAW_INFERENCE_INPUTS", + "NEMOCLAW_MAX_TOKENS", + "NEMOCLAW_MINIMAL_BOOTSTRAP", + "NEMOCLAW_OPENCLAW_OTEL", + "NEMOCLAW_OPENCLAW_OTEL_ENDPOINT", + "NEMOCLAW_OPENCLAW_OTEL_SAMPLE_RATE", + "NEMOCLAW_OPENCLAW_OTEL_SERVICE_NAME", + "NEMOCLAW_PROXY_HOST", + "NEMOCLAW_PROXY_PORT", + "NEMOCLAW_REASONING", + "NEMOCLAW_REASONING_EFFORT", + ], + hermes: ["NEMOCLAW_CONTEXT_WINDOW", "NEMOCLAW_PROXY_HOST", "NEMOCLAW_PROXY_PORT"], + "langchain-deepagents-code": ["NEMOCLAW_PROXY_HOST", "NEMOCLAW_PROXY_PORT"], +} as const satisfies Record; + +const HOST_PROXY_URL_INPUTS = ["HTTP_PROXY", "HTTPS_PROXY", "http_proxy", "https_proxy"] as const; +const HOST_NO_PROXY_INPUTS = ["NO_PROXY", "no_proxy"] as const; + +export interface ManagedStartupOnboardProfileInput { + readonly agentName: string; + readonly inference: ManagedStartupResolvedInferenceInput; + readonly chatUiUrl: string; + readonly effectiveDashboardPort: number; + readonly manageDashboard: boolean; + readonly dashboardBindAddress: string | undefined; + readonly wslExposure: boolean; + readonly hermesDashboardState: HermesDashboardOnboardState; + readonly webSearch: { + readonly fetchEnabled: boolean; + readonly provider?: "brave" | "tavily"; + } | null; + readonly toolDisclosure: ToolDisclosure; + readonly hermesToolGateways: readonly string[]; + readonly messagingPlan: SandboxMessagingPlan | null; + readonly dcodeAutoApprovalMode: DcodeAutoApprovalMode; + readonly observabilityEnabled: boolean; + readonly environment: NodeJS.ProcessEnv; + /** Validated durable CA material retained by an authoritative managed rebuild. */ + readonly corporateCaOverride?: ResolvedCorporateCa | null; +} + +export type BuiltManagedStartupOnboardProfile = BuiltManagedStartupProfile & { + /** + * Non-secret replay intent. Proxy credentials remain launch-only and never + * enter the canonical startup profile or durable receipt. + */ + readonly credentialProxyReplayRequired: boolean; +}; + +export class ManagedStartupOnboardProfileError extends Error { + constructor(message: string) { + super(`Cannot prepare managed onboarding profile: ${message}`); + this.name = "ManagedStartupOnboardProfileError"; + } +} + +function exactManagedAgent(agentName: string): ManagedStartupAgent { + if ((MANAGED_STARTUP_AGENTS as readonly string[]).includes(agentName)) { + return agentName as ManagedStartupAgent; + } + throw new ManagedStartupOnboardProfileError(`unsupported agent '${agentName}'`); +} + +function requireDashboardPort(port: number): number { + if (!Number.isInteger(port) || port < 1024 || port > 65_535) { + throw new ManagedStartupOnboardProfileError("dashboard port is missing or invalid"); + } + return port; +} + +function dashboardForInput( + agent: ManagedStartupAgent, + input: ManagedStartupOnboardProfileInput, +): ManagedStartupDashboard { + if (agent === "langchain-deepagents-code") { + if (input.manageDashboard) { + throw new ManagedStartupOnboardProfileError("DCode must not enable a dashboard"); + } + return { agent, mode: "disabled" }; + } + + if (!input.manageDashboard) { + throw new ManagedStartupOnboardProfileError(`${agent} requires managed dashboard state`); + } + + if (agent === "openclaw") { + const requestedBind = input.dashboardBindAddress?.trim(); + if (requestedBind && requestedBind !== "0.0.0.0") { + throw new ManagedStartupOnboardProfileError( + "dashboard bind address must be empty or 0.0.0.0", + ); + } + const bindAddress = requestedBind === "0.0.0.0" ? "0.0.0.0" : "127.0.0.1"; + const remote = + bindAddress === "0.0.0.0" || + input.wslExposure || + !["127.0.0.1", "localhost", "::1", "[::1]"].includes(new URL(input.chatUiUrl).hostname); + return { + agent, + mode: remote ? "remote" : "loopback", + url: input.chatUiUrl, + port: requireDashboardPort(input.effectiveDashboardPort), + bindAddress, + wslExposure: input.wslExposure, + }; + } + + const config = input.hermesDashboardState.config; + if (!input.hermesDashboardState.enabled) { + return { + agent, + mode: "disabled", + url: input.chatUiUrl, + publicPort: null, + internalPort: null, + tuiEnabled: false, + }; + } + if (!config?.enabled) { + throw new ManagedStartupOnboardProfileError( + "Hermes dashboard is enabled without resolved configuration", + ); + } + return { + agent, + mode: "loopback-forwarded", + url: input.chatUiUrl, + publicPort: requireDashboardPort(config.port), + internalPort: requireDashboardPort(config.internalPort), + tuiEnabled: config.tuiEnabled, + }; +} + +/** + * Copy only knobs whose values remain environment-owned at the managed-image + * boundary. Model, route, dashboard, tool, messaging, and agent-specific + * selections are passed explicitly so CLI precedence cannot be reversed by a + * stale ambient variable. + */ +function profileEnvironment( + agent: ManagedStartupAgent, + environment: NodeJS.ProcessEnv, +): NodeJS.ProcessEnv { + const selected: NodeJS.ProcessEnv = {}; + for (const name of PROFILE_ENVIRONMENT_INPUTS[agent]) { + const value = environment[name]; + if (value !== undefined) selected[name] = value; + } + const credentialProxy = hasCredentialBearingHostProxyEnvironment(environment); + if (!credentialProxy) { + for (const name of HOST_PROXY_URL_INPUTS) { + const value = environment[name]?.trim(); + if (!value) continue; + selected[name] = value; + } + const hasCredentialFreeProxy = HOST_PROXY_URL_INPUTS.some((name) => selected[name]); + if (hasCredentialFreeProxy) { + for (const name of HOST_NO_PROXY_INPUTS) { + const value = environment[name]; + if (value !== undefined) selected[name] = value; + } + } + } + return selected; +} + +export function buildManagedStartupOnboardProfile( + input: ManagedStartupOnboardProfileInput, +): BuiltManagedStartupOnboardProfile { + const agent = exactManagedAgent(input.agentName); + const dashboard = dashboardForInput(agent, input); + const environment = profileEnvironment(agent, input.environment); + const corporateCa = + input.corporateCaOverride === undefined + ? resolveCorporateCa(input.environment) + : input.corporateCaOverride; + const credentialProxyReplayRequired = + agent !== "langchain-deepagents-code" && + hasCredentialBearingHostProxyEnvironment(input.environment); + const built = buildManagedStartupProfile({ + agent, + inference: input.inference, + dashboard, + webSearch: agent === "langchain-deepagents-code" ? null : input.webSearch, + toolDisclosure: input.toolDisclosure, + hermesToolGateways: agent === "hermes" ? input.hermesToolGateways : [], + messagingPlan: agent === "langchain-deepagents-code" ? null : input.messagingPlan, + dcodeAutoApprovalMode: + agent === "langchain-deepagents-code" ? input.dcodeAutoApprovalMode : null, + observabilityEnabled: agent === "langchain-deepagents-code" ? input.observabilityEnabled : null, + environment, + corporateCa, + }); + return Object.freeze({ ...built, credentialProxyReplayRequired }); +} diff --git a/src/lib/onboard/managed-startup/podman-root-apply.test.ts b/src/lib/onboard/managed-startup/podman-root-apply.test.ts new file mode 100644 index 0000000000..722cb1158d --- /dev/null +++ b/src/lib/onboard/managed-startup/podman-root-apply.test.ts @@ -0,0 +1,464 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import { managedStartupE2eProfile } from "../../../../scripts/checks/generate-managed-startup-profile-fixture.mts"; +import { + applyPodmanManagedStartupRootRequest, + getPodmanManagedStartupFailureTransaction, +} from "./podman-root-apply"; +import type { + PodmanManagedStartupCommandResult, + PodmanManagedStartupRuntimeDeps, + RunManagedStartupPodmanCommand, +} from "./podman-runtime"; +import { encodeManagedStartupProfile } from "./profile"; +import { + createManagedStartupRootApplyRequest, + parseManagedStartupRootApplyRequest, +} from "./root-apply"; +import { MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY } from "./shared-state-transaction"; + +const SOCKET_PATH = "/run/user/1000/podman/podman.sock"; +const SOCKET_URL = `unix://${SOCKET_PATH}`; +const SOCKET_AUTHORITY = { + directoryChain: [ + { + device: "8", + inode: "7000", + mode: "448", + ownerUid: "1000", + path: "/run/user/1000/podman", + }, + ], + device: "8", + inode: "9001", + ownerUid: "1000", + socketPath: SOCKET_PATH, +} as const; +const CONTAINER_ID = "b".repeat(64); +const IMAGE_ID = `sha256:${"c".repeat(64)}`; + +function testDeps( + run: RunManagedStartupPodmanCommand, + assertSocketAuthority: NonNullable< + PodmanManagedStartupRuntimeDeps["assertSocketAuthority"] + > = vi.fn(), +) { + return { assertSocketAuthority, run }; +} + +function requestFor(agent: "openclaw" | "hermes" | "langchain-deepagents-code") { + return createManagedStartupRootApplyRequest({ + agent, + encodedProfile: encodeManagedStartupProfile(managedStartupE2eProfile(agent)), + }); + + it("does not execute root mutation after socket authority changes", () => { + const baseline = successfulRunner(); + applyPodmanManagedStartupRootRequest( + { + containerId: CONTAINER_ID, + request: requestFor("openclaw"), + socketAuthority: SOCKET_AUTHORITY, + socketPath: SOCKET_PATH, + }, + testDeps(baseline), + ); + const mutationIndex = vi + .mocked(baseline) + .mock.calls.findIndex((call) => call[1].includes("--apply-root-stdin")); + expect(mutationIndex).toBeGreaterThan(0); + + const run = successfulRunner(); + expect(() => + applyPodmanManagedStartupRootRequest( + { + containerId: CONTAINER_ID, + request: requestFor("openclaw"), + socketAuthority: SOCKET_AUTHORITY, + socketPath: SOCKET_PATH, + }, + testDeps(run, () => { + if (vi.mocked(run).mock.calls.length >= mutationIndex) { + throw new Error("socket authority changed"); + } + }), + ), + ).toThrow("socket authority changed"); + expect(vi.mocked(run).mock.calls.some((call) => call[1].includes("--apply-root-stdin"))).toBe( + false, + ); + }); +} + +function podmanInfo(overrides: Record = {}): string { + return JSON.stringify({ + host: { security: { rootless: true } }, + store: { + graphRoot: "/home/test/.local/share/containers/storage", + runRoot: "/run/user/1000/containers", + }, + ...overrides, + }); +} + +function stableInspect(overrides: Record = {}): string { + return JSON.stringify([ + { + Id: CONTAINER_ID, + Image: IMAGE_ID, + State: { Dead: false, Paused: false, Restarting: false, Running: true }, + ...overrides, + }, + ]); +} + +function result( + status: number | null, + overrides: Partial = {}, +): PodmanManagedStartupCommandResult { + return { status, stderr: "", stdout: "", ...overrides }; +} + +function successfulRunner( + mutation?: ( + args: readonly string[], + options: Parameters[2], + ) => PodmanManagedStartupCommandResult, +): RunManagedStartupPodmanCommand { + return vi.fn((_command, args, options) => { + const operation = args.slice(2); + if (operation[0] === "info") return result(0, { stdout: podmanInfo() }); + if (operation[0] === "container" && operation[1] === "inspect") { + return result(0, { stdout: stableInspect() }); + } + return mutation?.(operation, options) ?? result(0); + }); +} + +describe("Podman managed-startup root applicator", () => { + it.each([ + "openclaw", + "hermes", + "langchain-deepagents-code", + ] as const)("pins the rootless socket and exact identities before fixed root stdin for %s", (agent) => { + const request = requestFor(agent); + const run = successfulRunner(); + + const transaction = applyPodmanManagedStartupRootRequest( + { + containerId: CONTAINER_ID, + request, + socketAuthority: SOCKET_AUTHORITY, + socketPath: SOCKET_PATH, + }, + testDeps(run), + ); + + expect(transaction).toMatchObject({ + agent, + containerId: CONTAINER_ID, + image: IMAGE_ID, + runtime: { + fingerprint: expect.stringMatching(/^[a-f0-9]{64}$/u), + socketAuthority: SOCKET_AUTHORITY, + socketPath: SOCKET_PATH, + }, + }); + expect(run).toHaveBeenCalledTimes(8); + for (const call of vi.mocked(run).mock.calls) { + expect(call[0]).toBe("podman"); + expect(call[1].slice(0, 2)).toEqual(["--url", SOCKET_URL]); + } + const applyCall = vi + .mocked(run) + .mock.calls.find((call) => call[1].includes("--apply-root-stdin")); + expect(applyCall).toBeDefined(); + const applyArgs = applyCall?.[1].slice(2); + expect(applyArgs).toEqual([ + "exec", + "--interactive", + "--user", + "0:0", + "--workdir", + "/", + CONTAINER_ID, + "/usr/bin/env", + "-i", + "HOME=/root", + "LANG=C.UTF-8", + "LC_ALL=C.UTF-8", + "NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION=1", + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + "/usr/local/bin/node", + "/usr/local/lib/nemoclaw/managed-startup-image-runtime.cjs", + "--apply-root-stdin", + "--agent", + agent, + ]); + expect(applyArgs?.join(" ")).not.toContain(request.encodedProfile); + const applyOptions = applyCall?.[2]; + expect(parseManagedStartupRootApplyRequest(String(applyOptions?.input))).toEqual(request); + expect(applyOptions).toMatchObject({ + encoding: "utf8", + stdio: ["pipe", "pipe", "pipe"], + timeout: 300_000, + }); + + const receiptCall = vi + .mocked(run) + .mock.calls.find((call) => call[1].includes("nemoclaw-transaction-probe")); + expect(receiptCall?.[1].slice(2)).toEqual([ + "exec", + "--user", + "0:0", + "--workdir", + "/", + CONTAINER_ID, + "/usr/bin/env", + "-i", + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + "/bin/sh", + "-c", + 'if [ -d "$1" ] && [ ! -L "$1" ]; then exit 0; fi; if [ ! -e "$1" ] && [ ! -L "$1" ]; then exit 1; fi; exit 2', + "nemoclaw-transaction-probe", + MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY, + ]); + }); + + it("retries a lost acknowledgement with identical root command and stdin", () => { + let applies = 0; + const run = successfulRunner((args) => { + if (!args.includes("--apply-root-stdin")) return result(0); + applies += 1; + return applies === 1 ? result(1, { stderr: "lost ack" }) : result(0); + }); + + applyPodmanManagedStartupRootRequest( + { + containerId: CONTAINER_ID, + request: requestFor("openclaw"), + socketAuthority: SOCKET_AUTHORITY, + socketPath: SOCKET_PATH, + }, + testDeps(run), + ); + + const applyCalls = vi + .mocked(run) + .mock.calls.filter((call) => call[1].includes("--apply-root-stdin")); + expect(applyCalls).toHaveLength(2); + expect(applyCalls[1]).toEqual(applyCalls[0]); + }); + + it("returns no transaction when the canonical receipt proves an already-finalized profile", () => { + const run = successfulRunner((args) => + args.includes("nemoclaw-transaction-probe") ? result(1) : result(0), + ); + expect( + applyPodmanManagedStartupRootRequest( + { + containerId: CONTAINER_ID, + request: requestFor("openclaw"), + socketAuthority: SOCKET_AUTHORITY, + socketPath: SOCKET_PATH, + }, + testDeps(run), + ), + ).toBeNull(); + }); + + it("attaches the exact rollback transaction when both attempts fail", () => { + const run = successfulRunner((args) => + args.includes("--apply-root-stdin") ? result(1, { stderr: "exec failed" }) : result(0), + ); + let failure: unknown; + try { + applyPodmanManagedStartupRootRequest( + { + containerId: CONTAINER_ID, + request: requestFor("hermes"), + socketAuthority: SOCKET_AUTHORITY, + socketPath: SOCKET_PATH, + }, + testDeps(run), + ); + } catch (error) { + failure = error; + } + + expect(failure).toEqual( + expect.objectContaining({ message: expect.stringContaining("exec failed") }), + ); + expect(getPodmanManagedStartupFailureTransaction(failure)).toMatchObject({ + agent: "hermes", + containerId: CONTAINER_ID, + image: IMAGE_ID, + runtime: { socketPath: SOCKET_PATH }, + }); + }); + + it("retains rollback context when a lost acknowledgement is followed by runtime drift", () => { + let infoCalls = 0; + const run = vi.fn((_command, args: readonly string[]) => { + const operation = args.slice(2); + if (operation[0] === "info") { + infoCalls += 1; + return result(0, { + stdout: + infoCalls < 3 + ? podmanInfo() + : podmanInfo({ + store: { + graphRoot: "/different/storage", + runRoot: "/run/user/1000/containers", + }, + }), + }); + } + if (operation[0] === "container") return result(0, { stdout: stableInspect() }); + if (operation.includes("--apply-root-stdin")) { + return result(1, { stderr: "ack lost" }); + } + return result(0); + }); + let failure: unknown; + try { + applyPodmanManagedStartupRootRequest( + { + containerId: CONTAINER_ID, + request: requestFor("openclaw"), + socketAuthority: SOCKET_AUTHORITY, + socketPath: SOCKET_PATH, + }, + testDeps(run), + ); + } catch (error) { + failure = error; + } + + expect(failure).toEqual( + expect.objectContaining({ message: expect.stringContaining("runtime identity changed") }), + ); + expect(getPodmanManagedStartupFailureTransaction(failure)).toMatchObject({ + containerId: CONTAINER_ID, + image: IMAGE_ID, + runtime: { socketPath: SOCKET_PATH }, + }); + }); + + it("fails closed when the receipt proof is unavailable", () => { + const run = successfulRunner((args) => + args.includes("nemoclaw-transaction-probe") + ? result(null, { error: new Error("probe unavailable") }) + : result(0), + ); + let failure: unknown; + try { + applyPodmanManagedStartupRootRequest( + { + containerId: CONTAINER_ID, + request: requestFor("langchain-deepagents-code"), + socketAuthority: SOCKET_AUTHORITY, + socketPath: SOCKET_PATH, + }, + testDeps(run), + ); + } catch (error) { + failure = error; + } + expect(failure).toEqual( + expect.objectContaining({ + message: expect.stringContaining("transaction state could not be verified"), + }), + ); + expect(getPodmanManagedStartupFailureTransaction(failure)).not.toBeNull(); + }); + + it.each([ + { + label: "relative socket", + socketPath: "run/podman.sock", + info: podmanInfo(), + inspect: stableInspect(), + containerId: CONTAINER_ID, + error: /safe normalized absolute path/u, + }, + { + label: "non-rootless API", + socketPath: SOCKET_PATH, + info: podmanInfo({ host: { security: { rootless: false } } }), + inspect: stableInspect(), + containerId: CONTAINER_ID, + error: /rootless Podman API/u, + }, + { + label: "short container identity", + socketPath: SOCKET_PATH, + info: podmanInfo(), + inspect: stableInspect(), + containerId: "b".repeat(12), + error: /full lowercase Podman container ID/u, + }, + { + label: "changed container identity", + socketPath: SOCKET_PATH, + info: podmanInfo(), + inspect: stableInspect({ Id: "d".repeat(64) }), + containerId: CONTAINER_ID, + error: /identity changed/u, + }, + { + label: "mutable image identity", + socketPath: SOCKET_PATH, + info: podmanInfo(), + inspect: stableInspect({ Image: "registry.example/image:latest" }), + containerId: CONTAINER_ID, + error: /full immutable Podman image ID/u, + }, + { + label: "stopped container", + socketPath: SOCKET_PATH, + info: podmanInfo(), + inspect: stableInspect({ + State: { Dead: false, Paused: false, Restarting: false, Running: false }, + }), + containerId: CONTAINER_ID, + error: /not running/u, + }, + { + label: "unstable running container", + socketPath: SOCKET_PATH, + info: podmanInfo(), + inspect: stableInspect({ + State: { Dead: false, Paused: true, Restarting: false, Running: true }, + }), + containerId: CONTAINER_ID, + error: /not stably running/u, + }, + ])("rejects $label before root exec", ({ socketPath, info, inspect, containerId, error }) => { + const run = vi.fn((_command, args: readonly string[]) => { + const operation = args.slice(2); + if (operation[0] === "info") return result(0, { stdout: info }); + if (operation[0] === "container") return result(0, { stdout: inspect }); + return result(0); + }); + + expect(() => + applyPodmanManagedStartupRootRequest( + { + containerId, + request: requestFor("openclaw"), + socketAuthority: SOCKET_AUTHORITY, + socketPath, + }, + testDeps(run), + ), + ).toThrow(error); + expect(vi.mocked(run).mock.calls.some((call) => call[1].includes("--apply-root-stdin"))).toBe( + false, + ); + }); +}); diff --git a/src/lib/onboard/managed-startup/podman-root-apply.ts b/src/lib/onboard/managed-startup/podman-root-apply.ts new file mode 100644 index 0000000000..b76cc13cde --- /dev/null +++ b/src/lib/onboard/managed-startup/podman-root-apply.ts @@ -0,0 +1,181 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { PodmanSocketAuthority } from "../compute/podman/socket-authority"; +import { MANAGED_STARTUP_RUNTIME_EXECUTABLE } from "./image-runtime"; +import { + assertPodmanManagedStartupRuntime, + inspectExactPodmanManagedStartupContainer, + type PodmanManagedStartupRuntimeDeps, + type PodmanManagedStartupRuntimeIdentity, + pinPodmanManagedStartupRuntime, + podmanManagedStartupCommandDetail, + runManagedStartupPodman, +} from "./podman-runtime"; +import { + type ManagedStartupRootApplyRequest, + serializeManagedStartupRootApplyRequest, +} from "./root-apply"; +import { MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY } from "./shared-state-transaction"; + +const ROOT_APPLY_TIMEOUT_MS = 300_000; +const FIXED_ROOT_ENV = [ + "HOME=/root", + "LANG=C.UTF-8", + "LC_ALL=C.UTF-8", + "NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION=1", + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", +] as const; + +export interface PodmanManagedStartupTransaction { + readonly agent: ManagedStartupRootApplyRequest["agent"]; + readonly containerId: string; + readonly image: string; + readonly runtime: PodmanManagedStartupRuntimeIdentity; +} + +export type PodmanManagedStartupRootApplyDeps = PodmanManagedStartupRuntimeDeps; + +export function getPodmanManagedStartupFailureTransaction( + error: unknown, +): PodmanManagedStartupTransaction | null { + if (typeof error === "object" && error !== null && "managedStartupTransaction" in error) { + return ( + (error as { managedStartupTransaction?: PodmanManagedStartupTransaction }) + .managedStartupTransaction ?? null + ); + } + return null; +} + +function failureWithTransaction( + message: string, + transaction: PodmanManagedStartupTransaction, +): Error { + const error = new Error(message); + ( + error as Error & { + managedStartupTransaction?: PodmanManagedStartupTransaction; + } + ).managedStartupTransaction = transaction; + return error; +} + +export function applyPodmanManagedStartupRootRequest( + input: { + readonly containerId: string; + readonly request: ManagedStartupRootApplyRequest; + readonly socketAuthority: PodmanSocketAuthority; + readonly socketPath: string; + }, + deps: PodmanManagedStartupRootApplyDeps = {}, +): PodmanManagedStartupTransaction | null { + deps = { ...deps, socketAuthority: input.socketAuthority }; + const runtime = pinPodmanManagedStartupRuntime(input.socketPath, input.socketAuthority, deps); + const pinned = inspectExactPodmanManagedStartupContainer( + runtime, + { containerId: input.containerId, requireRunning: true }, + deps, + ); + const transaction = Object.freeze({ + agent: input.request.agent, + containerId: pinned.containerId, + image: pinned.image, + runtime, + }) satisfies PodmanManagedStartupTransaction; + const payload = serializeManagedStartupRootApplyRequest(input.request); + const argv = [ + "exec", + "--interactive", + "--user", + "0:0", + "--workdir", + "/", + pinned.containerId, + "/usr/bin/env", + "-i", + ...FIXED_ROOT_ENV, + "/usr/local/bin/node", + MANAGED_STARTUP_RUNTIME_EXECUTABLE, + "--apply-root-stdin", + "--agent", + input.request.agent, + ]; + const receiptProbeArgv = [ + "exec", + "--user", + "0:0", + "--workdir", + "/", + pinned.containerId, + "/usr/bin/env", + "-i", + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + "/bin/sh", + "-c", + 'if [ -d "$1" ] && [ ! -L "$1" ]; then exit 0; fi; if [ ! -e "$1" ] && [ ! -L "$1" ]; then exit 1; fi; exit 2', + "nemoclaw-transaction-probe", + MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY, + ]; + + let lastFailure = ""; + // Root application is idempotent. Retry only the ambiguous lost-ack window, + // using the same socket, full container identity, command, and stdin bytes. + for (let attempt = 0; attempt < 2; attempt += 1) { + try { + assertPodmanManagedStartupRuntime(runtime, deps); + inspectExactPodmanManagedStartupContainer( + runtime, + { containerId: pinned.containerId, image: pinned.image, requireRunning: true }, + deps, + ); + } catch (error) { + throw failureWithTransaction( + `Managed startup could not revalidate the exact Podman runtime and container before root application: ${ + error instanceof Error ? error.message : String(error) + }`, + transaction, + ); + } + const result = runManagedStartupPodman( + runtime.socketPath, + argv, + { input: payload, timeout: ROOT_APPLY_TIMEOUT_MS }, + deps, + ); + if (result.status === 0) { + try { + assertPodmanManagedStartupRuntime(runtime, deps); + inspectExactPodmanManagedStartupContainer( + runtime, + { containerId: pinned.containerId, image: pinned.image, requireRunning: true }, + deps, + ); + } catch (error) { + throw failureWithTransaction( + `Managed startup could not revalidate the exact Podman runtime and container before transaction proof: ${ + error instanceof Error ? error.message : String(error) + }`, + transaction, + ); + } + const receiptProbe = runManagedStartupPodman(runtime.socketPath, receiptProbeArgv, {}, deps); + if (receiptProbe.status === 0) return transaction; + if (receiptProbe.status === 1) return null; + const detail = podmanManagedStartupCommandDetail(receiptProbe); + throw failureWithTransaction( + `Managed startup root application completed, but transaction state could not be verified in exact Podman container ${pinned.containerId.slice(0, 12)}${ + detail ? `: ${detail}` : "" + }`, + transaction, + ); + } + lastFailure = podmanManagedStartupCommandDetail(result); + } + throw failureWithTransaction( + `Managed startup root application failed in exact Podman container ${pinned.containerId.slice(0, 12)}${ + lastFailure ? `: ${lastFailure}` : "" + }`, + transaction, + ); +} diff --git a/src/lib/onboard/managed-startup/podman-runtime.ts b/src/lib/onboard/managed-startup/podman-runtime.ts new file mode 100644 index 0000000000..c7f1fb4d71 --- /dev/null +++ b/src/lib/onboard/managed-startup/podman-runtime.ts @@ -0,0 +1,289 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { type SpawnSyncOptions, spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import path from "node:path"; +import { + assertPodmanSocketAuthority, + type PodmanSocketAuthority, +} from "../compute/podman/socket-authority"; + +const FULL_CONTAINER_ID_RE = /^[a-f0-9]{64}$/u; +const FULL_IMAGE_ID_RE = /^(?:sha256:)?([a-f0-9]{64})$/u; +const PODMAN_PROBE_TIMEOUT_MS = 30_000; + +export interface PodmanManagedStartupCommandResult { + readonly error?: Error; + readonly status: number | null; + readonly stderr?: Buffer | string | null; + readonly stdout?: Buffer | string | null; +} + +export type RunManagedStartupPodmanCommand = ( + command: "podman", + args: readonly string[], + options: SpawnSyncOptions, +) => PodmanManagedStartupCommandResult; + +export interface PodmanManagedStartupRuntimeDeps { + readonly assertSocketAuthority?: (expected: PodmanSocketAuthority) => void; + readonly run?: RunManagedStartupPodmanCommand; + readonly socketAuthority?: PodmanSocketAuthority; +} + +export interface PodmanManagedStartupRuntimeIdentity { + readonly fingerprint: string; + readonly socketAuthority: PodmanSocketAuthority; + readonly socketPath: string; +} + +export interface PodmanManagedStartupContainerIdentity { + readonly containerId: string; + readonly image: string; + readonly running: boolean; +} + +type JsonRecord = Record; + +function output(value: Buffer | string | null | undefined): string { + if (typeof value === "string") return value; + return Buffer.isBuffer(value) ? value.toString("utf8") : ""; +} + +function record(value: unknown, label: string): JsonRecord { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + throw new Error(`${label} must be an object.`); + } + return value as JsonRecord; +} + +function optionalRecord(value: unknown): JsonRecord | null { + return typeof value === "object" && value !== null && !Array.isArray(value) + ? (value as JsonRecord) + : null; +} + +function absolutePath(value: unknown, label: string): string { + if ( + typeof value !== "string" || + value.length === 0 || + value !== value.trim() || + !path.isAbsolute(value) || + path.normalize(value) !== value || + /[\0\r\n]/u.test(value) + ) { + throw new Error(`${label} must be a safe normalized absolute path.`); + } + return value; +} + +function fullContainerId(value: unknown, label: string): string { + if (typeof value !== "string" || !FULL_CONTAINER_ID_RE.test(value)) { + throw new Error(`${label} must be one full lowercase Podman container ID.`); + } + return value; +} + +function immutableImage(value: unknown, label: string): string { + if (typeof value !== "string") { + throw new Error(`${label} must be a full immutable Podman image ID.`); + } + const match = value.match(FULL_IMAGE_ID_RE); + if (!match?.[1]) { + throw new Error(`${label} must be a full immutable Podman image ID.`); + } + return `sha256:${match[1]}`; +} + +export function podmanManagedStartupCommandDetail( + result: PodmanManagedStartupCommandResult, + maxLength = 1200, +): string { + return `${output(result.stderr)} ${output(result.stdout)} ${result.error?.message ?? ""}` + .replace(/\s+/gu, " ") + .trim() + .slice(-maxLength); +} + +export function podmanManagedStartupSocketUrl(socketPath: string): string { + return `unix://${absolutePath(socketPath, "Managed-startup Podman socket path")}`; +} + +export function runManagedStartupPodman( + socketPath: string, + args: readonly string[], + options: { + readonly input?: string; + readonly timeout?: number; + }, + deps: PodmanManagedStartupRuntimeDeps, +): PodmanManagedStartupCommandResult { + const run = + deps.run ?? + ((command: "podman", commandArgs: readonly string[], spawnOptions: SpawnSyncOptions) => + spawnSync(command, [...commandArgs], spawnOptions)); + try { + if (deps.socketAuthority) { + if (deps.socketAuthority.socketPath !== socketPath) { + throw new Error( + "Managed-startup Podman socket authority does not match the requested socket.", + ); + } + (deps.assertSocketAuthority ?? assertPodmanSocketAuthority)(deps.socketAuthority); + } + return run("podman", ["--url", podmanManagedStartupSocketUrl(socketPath), ...args], { + encoding: "utf8", + input: options.input, + stdio: [options.input === undefined ? "ignore" : "pipe", "pipe", "pipe"], + timeout: options.timeout ?? PODMAN_PROBE_TIMEOUT_MS, + }); + } catch (error) { + return { + error: error instanceof Error ? error : new Error(String(error)), + status: null, + }; + } +} + +function requireZero( + result: PodmanManagedStartupCommandResult, + action: string, +): PodmanManagedStartupCommandResult { + if (result.status === 0) return result; + const detail = podmanManagedStartupCommandDetail(result); + throw new Error( + `${action} failed with non-zero or unavailable Podman status${detail ? `: ${detail}` : "."}`, + ); +} + +function parseRootlessRuntimeInfo( + text: string, +): Pick { + let parsed: unknown; + try { + parsed = JSON.parse(text); + } catch { + throw new Error("Managed-startup Podman API identity proof returned unreadable JSON."); + } + const root = record(parsed, "Managed-startup Podman API identity"); + const host = optionalRecord(root.host ?? root.Host); + const security = optionalRecord(host?.security ?? host?.Security); + const store = optionalRecord(root.store ?? root.Store); + const rootless = security?.rootless ?? security?.Rootless; + const graphRoot = absolutePath( + store?.graphRoot ?? store?.GraphRoot, + "Managed-startup Podman graph root", + ); + const runRoot = absolutePath(store?.runRoot ?? store?.RunRoot, "Managed-startup Podman run root"); + if (rootless !== true) { + throw new Error("Managed startup requires a rootless Podman API."); + } + return { + fingerprint: createHash("sha256").update(`${graphRoot}\0${runRoot}`).digest("hex"), + }; +} + +export function pinPodmanManagedStartupRuntime( + socketPath: string, + socketAuthority: PodmanSocketAuthority, + deps: PodmanManagedStartupRuntimeDeps, +): PodmanManagedStartupRuntimeIdentity { + const normalizedSocketPath = absolutePath(socketPath, "Managed-startup Podman socket path"); + if (socketAuthority.socketPath !== normalizedSocketPath) { + throw new Error("Managed-startup Podman socket authority does not match the pinned socket."); + } + const qualifiedDeps = { ...deps, socketAuthority }; + const result = requireZero( + runManagedStartupPodman(normalizedSocketPath, ["info", "--format", "json"], {}, qualifiedDeps), + "Managed-startup rootless Podman API identity proof", + ); + return Object.freeze({ + ...parseRootlessRuntimeInfo(output(result.stdout)), + socketAuthority, + socketPath: normalizedSocketPath, + }); +} + +export function assertPodmanManagedStartupRuntime( + expected: PodmanManagedStartupRuntimeIdentity, + deps: PodmanManagedStartupRuntimeDeps, +): void { + const actual = pinPodmanManagedStartupRuntime( + expected.socketPath, + expected.socketAuthority, + deps, + ); + if (actual.fingerprint !== expected.fingerprint) { + throw new Error("Managed-startup Podman runtime identity changed after it was pinned."); + } +} + +function parseContainerInspect(text: string): PodmanManagedStartupContainerIdentity { + let parsed: unknown; + try { + parsed = JSON.parse(text); + } catch { + throw new Error("Podman returned malformed inspect output for the managed-startup container."); + } + if (!Array.isArray(parsed) || parsed.length !== 1) { + throw new Error("Podman inspect did not resolve exactly one managed-startup container."); + } + const inspect = record(parsed[0], "Managed-startup Podman container inspect"); + const state = record(inspect.State, "Managed-startup Podman container State"); + if (typeof state.Running !== "boolean") { + throw new Error("Managed-startup Podman container State.Running must be a boolean."); + } + for (const field of ["Paused", "Restarting", "Dead"] as const) { + if (state[field] !== undefined && typeof state[field] !== "boolean") { + throw new Error(`Managed-startup Podman container State.${field} must be a boolean.`); + } + } + if ( + state.Running === true && + (state.Paused === true || state.Restarting === true || state.Dead === true) + ) { + throw new Error("Managed-startup Podman container is not stably running."); + } + return { + containerId: fullContainerId(inspect.Id, "Managed-startup Podman inspect Id"), + image: immutableImage(inspect.Image, "Managed-startup Podman inspect Image"), + running: state.Running, + }; +} + +export function inspectExactPodmanManagedStartupContainer( + runtime: PodmanManagedStartupRuntimeIdentity, + expected: { + readonly containerId: string; + readonly image?: string; + readonly requireRunning?: boolean; + }, + deps: PodmanManagedStartupRuntimeDeps, +): PodmanManagedStartupContainerIdentity { + const containerId = fullContainerId(expected.containerId, "Managed-startup Podman container ID"); + const qualifiedDeps = { ...deps, socketAuthority: runtime.socketAuthority }; + const result = requireZero( + runManagedStartupPodman( + runtime.socketPath, + ["container", "inspect", containerId], + {}, + qualifiedDeps, + ), + "Managed-startup Podman container inspect", + ); + const actual = parseContainerInspect(output(result.stdout)); + if (actual.containerId !== containerId) { + throw new Error("Managed-startup Podman container identity changed after it was pinned."); + } + if ( + expected.image !== undefined && + actual.image !== immutableImage(expected.image, "Pinned image") + ) { + throw new Error("Managed-startup Podman image identity changed after it was pinned."); + } + if (expected.requireRunning === true && !actual.running) { + throw new Error("Managed-startup Podman container is not running."); + } + return actual; +} diff --git a/src/lib/onboard/managed-startup/podman-shared-state.test.ts b/src/lib/onboard/managed-startup/podman-shared-state.test.ts new file mode 100644 index 0000000000..c81b982b27 --- /dev/null +++ b/src/lib/onboard/managed-startup/podman-shared-state.test.ts @@ -0,0 +1,442 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; + +import { afterEach, describe, expect, it, vi } from "vitest"; + +import type { PodmanManagedSandboxRecreateTransaction } from "../compute/podman/sandbox-recreate"; +import type { PodmanManagedStartupTransaction } from "./podman-root-apply"; +import type { + PodmanManagedStartupCommandResult, + RunManagedStartupPodmanCommand, +} from "./podman-runtime"; +import { finalizePodmanManagedStartupSharedState } from "./podman-shared-state"; +import { + MANAGED_STARTUP_SHARED_ROLLBACK_RECEIPT_DIRECTORY, + MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY, +} from "./shared-state-transaction"; + +const SOCKET_PATH = "/run/user/1000/podman/podman.sock"; +const SOCKET_URL = `unix://${SOCKET_PATH}`; +const SOCKET_AUTHORITY = { + directoryChain: [ + { + device: "8", + inode: "7000", + mode: "448", + ownerUid: "1000", + path: "/run/user/1000/podman", + }, + ], + device: "8", + inode: "9001", + ownerUid: "1000", + socketPath: SOCKET_PATH, +} as const; +const CONTAINER_ID = "b".repeat(64); +const IMAGE_ID = `sha256:${"c".repeat(64)}`; +const GRAPH_ROOT = "/home/test/.local/share/containers/storage"; +const RUN_ROOT = "/run/user/1000/containers"; + +function transaction( + agent: PodmanManagedStartupTransaction["agent"] = "openclaw", +): PodmanManagedStartupTransaction { + return { + agent, + containerId: CONTAINER_ID, + image: IMAGE_ID, + runtime: { + fingerprint: createHash("sha256").update(`${GRAPH_ROOT}\0${RUN_ROOT}`).digest("hex"), + socketAuthority: SOCKET_AUTHORITY, + socketPath: SOCKET_PATH, + }, + }; +} + +function rollbackAuthority( + overrides: Partial = {}, +): PodmanManagedSandboxRecreateTransaction { + return { + applied: true, + driverName: "podman", + immutableImage: IMAGE_ID, + newContainerId: CONTAINER_ID, + socketAuthority: SOCKET_AUTHORITY, + socketPath: SOCKET_PATH, + ...overrides, + } as PodmanManagedSandboxRecreateTransaction; +} + +function testDeps(run: RunManagedStartupPodmanCommand) { + return { assertSocketAuthority: vi.fn(), run }; +} + +function result( + status: number | null, + overrides: Partial = {}, +): PodmanManagedStartupCommandResult { + return { status, stderr: "", stdout: "", ...overrides }; +} + +function identityRunner( + mutation: ( + args: readonly string[], + options: Parameters[2], + ) => PodmanManagedStartupCommandResult, +): { + readonly calls: string[]; + readonly receiptPaths: string[]; + readonly run: RunManagedStartupPodmanCommand; +} { + const calls: string[] = []; + const receiptPaths: string[] = []; + let running = true; + const run = vi.fn((_command, args, options) => { + expect(args.slice(0, 2)).toEqual(["--url", SOCKET_URL]); + const operation = args.slice(2); + if (operation[0] === "info") { + return result(0, { + stdout: JSON.stringify({ + host: { security: { rootless: true } }, + store: { graphRoot: GRAPH_ROOT, runRoot: RUN_ROOT }, + }), + }); + } + if (operation[0] === "container" && operation[1] === "inspect") { + return result(0, { + stdout: JSON.stringify([ + { + Id: CONTAINER_ID, + Image: IMAGE_ID, + State: { Dead: false, Paused: false, Restarting: false, Running: running }, + }, + ]), + }); + } + if (operation[0] === "stop") { + calls.push("stop"); + const mutationResult = mutation(operation, options); + if (mutationResult.status === 0) running = false; + return mutationResult; + } + if (operation[0] === "cp") { + calls.push("copy"); + receiptPaths.push(String(operation[2])); + } else if (operation[0] === "exec") { + calls.push("commit"); + } else if (operation[0] === "run") { + calls.push("rollback"); + } else if (operation[0] === "rm") { + calls.push("remove"); + } else if (operation[0] === "container" && operation[1] === "exists") { + calls.push("exists"); + } + return mutation(operation, options); + }); + return { calls, receiptPaths, run }; +} + +describe("Podman managed-startup shared-state finalization", () => { + afterEach(() => { + vi.restoreAllMocks(); + }); + + it.each([ + "openclaw", + "hermes", + "langchain-deepagents-code", + ] as const)("copies before committing %s through the exact rootless runtime", (agent) => { + const harness = identityRunner(() => result(0)); + const outcome = finalizePodmanManagedStartupSharedState( + { + containerRollbackAuthority: rollbackAuthority(), + supervisorReady: true, + transaction: transaction(agent), + }, + testDeps(harness.run), + ); + + expect(outcome).toEqual({ failure: null, supervisorReady: true }); + expect(harness.calls).toEqual(["copy", "commit"]); + expect(harness.receiptPaths).toHaveLength(1); + expect(fs.existsSync(path.dirname(harness.receiptPaths[0] as string))).toBe(false); + + const copyCall = vi.mocked(harness.run).mock.calls.find((call) => call[1].slice(2)[0] === "cp"); + expect(copyCall?.[1].slice(2, 4)).toEqual([ + "cp", + `${CONTAINER_ID}:${MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY}`, + ]); + const commitCall = vi + .mocked(harness.run) + .mock.calls.find((call) => call[1].includes("--commit-shared-state-transaction")); + expect(commitCall?.[1].slice(2)).toEqual([ + "exec", + "--user", + "0:0", + "--env", + "NODE_OPTIONS=", + "--env", + "NODE_PATH=", + "--env", + "BASH_ENV=", + "--env", + "ENV=", + CONTAINER_ID, + "/usr/bin/env", + "-i", + "HOME=/root", + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + "/usr/local/bin/node", + "/usr/local/lib/nemoclaw/managed-startup-image-runtime.cjs", + "--commit-shared-state-transaction", + "--agent", + agent, + ]); + }); + + it("uses the preserved receipt after a lost commit acknowledgement", () => { + const harness = identityRunner((args) => { + if (args[0] === "exec") return result(1, { stderr: "ack lost" }); + return result(0); + }); + const outcome = finalizePodmanManagedStartupSharedState( + { + containerRollbackAuthority: rollbackAuthority(), + supervisorReady: true, + transaction: transaction("hermes"), + }, + testDeps(harness.run), + ); + + expect(outcome.supervisorReady).toBe(false); + expect(outcome.failure?.message).toContain("commit failed"); + expect(harness.calls).toEqual(["copy", "commit", "stop", "rollback"]); + expect(fs.existsSync(path.dirname(harness.receiptPaths[0] as string))).toBe(false); + + const rollbackCall = vi + .mocked(harness.run) + .mock.calls.find((call) => call[1].includes("--rollback-shared-state-transaction")); + expect(rollbackCall?.[1].slice(2)).toEqual([ + "run", + "--rm", + "--pull", + "never", + "--network", + "none", + "--read-only", + "--user", + "0:0", + "--security-opt", + "no-new-privileges", + "--cap-drop", + "ALL", + "--cap-add", + "CHOWN", + "--cap-add", + "DAC_OVERRIDE", + "--cap-add", + "FOWNER", + "--env", + "NODE_OPTIONS=", + "--env", + "NODE_PATH=", + "--env", + "BASH_ENV=", + "--env", + "ENV=", + "--volumes-from", + CONTAINER_ID, + "--mount", + expect.stringMatching( + new RegExp( + `^type=bind,src=.+,dst=${MANAGED_STARTUP_SHARED_ROLLBACK_RECEIPT_DIRECTORY},readonly$`, + "u", + ), + ), + "--entrypoint", + "/usr/local/bin/node", + IMAGE_ID, + "/usr/local/lib/nemoclaw/managed-startup-image-runtime.cjs", + "--rollback-shared-state-transaction", + "--agent", + "hermes", + "--read-only-receipt", + ]); + }); + + it("quiesces before copying and removes an unbacked failed replacement after rollback", () => { + const harness = identityRunner((args) => { + if (args[0] === "container" && args[1] === "exists") return result(1); + return result(0); + }); + + expect( + finalizePodmanManagedStartupSharedState( + { supervisorReady: false, transaction: transaction() }, + testDeps(harness.run), + ), + ).toEqual({ failure: null, supervisorReady: false }); + expect(harness.calls).toEqual(["stop", "copy", "rollback", "remove", "exists"]); + expect(fs.existsSync(path.dirname(harness.receiptPaths[0] as string))).toBe(false); + }); + + it("retains the protected receipt when immutable rollback verification fails", () => { + const harness = identityRunner((args) => + args[0] === "run" ? result(1, { stderr: "receipt verification failed" }) : result(0), + ); + + try { + expect(() => + finalizePodmanManagedStartupSharedState( + { + containerRollbackAuthority: rollbackAuthority(), + supervisorReady: false, + transaction: transaction(), + }, + testDeps(harness.run), + ), + ).toThrow(/Protected receipt retained/u); + expect(harness.calls).toEqual(["stop", "copy", "rollback"]); + expect(fs.existsSync(path.dirname(harness.receiptPaths[0] as string))).toBe(true); + } finally { + const receiptPath = harness.receiptPaths[0]; + if (receiptPath) { + fs.rmSync(path.dirname(receiptPath), { force: true, recursive: true }); + } + } + }); + + it("stops the live workload when receipt preservation fails", () => { + const harness = identityRunner((args) => + args[0] === "cp" ? result(1, { stderr: "copy failed" }) : result(0), + ); + + expect(() => + finalizePodmanManagedStartupSharedState( + { + containerRollbackAuthority: rollbackAuthority(), + supervisorReady: true, + transaction: transaction(), + }, + testDeps(harness.run), + ), + ).toThrow(/Could not copy/u); + expect(harness.calls).toEqual(["copy", "stop"]); + }); + + it("fails closed before mutation when the rootless runtime fingerprint changed", () => { + const run = vi.fn((_command, args: readonly string[]) => { + expect(args.slice(0, 2)).toEqual(["--url", SOCKET_URL]); + return result(0, { + stdout: JSON.stringify({ + host: { security: { rootless: true } }, + store: { graphRoot: "/different/storage", runRoot: RUN_ROOT }, + }), + }); + }); + + expect(() => + finalizePodmanManagedStartupSharedState( + { + containerRollbackAuthority: rollbackAuthority(), + supervisorReady: true, + transaction: transaction(), + }, + testDeps(run), + ), + ).toThrow(/runtime identity changed/u); + expect(run).toHaveBeenCalledOnce(); + }); + + it("fails closed before mutation when the exact image identity changed", () => { + const run = vi.fn((_command, args: readonly string[]) => { + const operation = args.slice(2); + if (operation[0] === "info") { + return result(0, { + stdout: JSON.stringify({ + host: { security: { rootless: true } }, + store: { graphRoot: GRAPH_ROOT, runRoot: RUN_ROOT }, + }), + }); + } + return result(0, { + stdout: JSON.stringify([ + { + Id: CONTAINER_ID, + Image: `sha256:${"d".repeat(64)}`, + State: { Dead: false, Paused: false, Restarting: false, Running: true }, + }, + ]), + }); + }); + + expect(() => + finalizePodmanManagedStartupSharedState( + { + containerRollbackAuthority: rollbackAuthority(), + supervisorReady: true, + transaction: transaction(), + }, + testDeps(run), + ), + ).toThrow(/image identity changed/u); + expect(vi.mocked(run).mock.calls.some((call) => call[1].slice(2)[0] === "cp")).toBe(false); + }); + + it.each([ + { + label: "driver", + authority: rollbackAuthority({ driverName: "docker" as "podman" }), + }, + { + label: "application state", + authority: rollbackAuthority({ applied: false as true }), + }, + { + label: "runtime socket", + authority: rollbackAuthority({ socketPath: "/run/user/1000/podman/other.sock" }), + }, + { + label: "socket identity", + authority: rollbackAuthority({ + socketAuthority: { ...SOCKET_AUTHORITY, inode: "9002" }, + }), + }, + { + label: "replacement container", + authority: rollbackAuthority({ newContainerId: "d".repeat(64) }), + }, + { + label: "immutable image", + authority: rollbackAuthority({ immutableImage: `sha256:${"e".repeat(64)}` }), + }, + ])("rejects mismatched $label rollback authority before mutation", ({ authority }) => { + const run = vi.fn(); + + expect(() => + finalizePodmanManagedStartupSharedState( + { + containerRollbackAuthority: authority, + supervisorReady: false, + transaction: transaction(), + }, + testDeps(run), + ), + ).toThrow(/rollback authority does not match/u); + expect(run).not.toHaveBeenCalled(); + }); + + it("is a no-op without a managed-startup transaction", () => { + const run = vi.fn(); + expect( + finalizePodmanManagedStartupSharedState( + { supervisorReady: true, transaction: null }, + testDeps(run), + ), + ).toEqual({ failure: null, supervisorReady: true }); + expect(run).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/onboard/managed-startup/podman-shared-state.ts b/src/lib/onboard/managed-startup/podman-shared-state.ts new file mode 100644 index 0000000000..059253acc1 --- /dev/null +++ b/src/lib/onboard/managed-startup/podman-shared-state.ts @@ -0,0 +1,373 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { PodmanManagedSandboxRecreateTransaction } from "../compute/podman/sandbox-recreate"; +import { cleanupTempDir, secureTempFile } from "../temp-files"; +import { MANAGED_STARTUP_RUNTIME_EXECUTABLE } from "./image-runtime"; +import type { PodmanManagedStartupTransaction } from "./podman-root-apply"; +import { + assertPodmanManagedStartupRuntime, + inspectExactPodmanManagedStartupContainer, + type PodmanManagedStartupRuntimeDeps, + podmanManagedStartupCommandDetail, + runManagedStartupPodman, +} from "./podman-runtime"; +import { + MANAGED_STARTUP_SHARED_ROLLBACK_RECEIPT_DIRECTORY, + MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY, +} from "./shared-state-transaction"; + +const RECEIPT_TEMP_PREFIX = "nemoclaw-managed-startup-podman-receipt"; +const MUTATION_TIMEOUT_MS = 30_000; +const STOP_TIMEOUT_MS = 90_000; +const STOP_GRACE_SECONDS = "30"; +const NEUTRALIZED_PROCESS_INJECTION_ENV = [ + "--env", + "NODE_OPTIONS=", + "--env", + "NODE_PATH=", + "--env", + "BASH_ENV=", + "--env", + "ENV=", +] as const; + +export interface PodmanManagedStartupSharedStateOutcome { + readonly failure: Error | null; + readonly supervisorReady: boolean; +} + +export type PodmanManagedStartupSharedStateDeps = PodmanManagedStartupRuntimeDeps; + +function sameSocketAuthority( + left: PodmanManagedStartupTransaction["runtime"]["socketAuthority"], + right: PodmanManagedSandboxRecreateTransaction["socketAuthority"], +): boolean { + return ( + left.device === right.device && + left.inode === right.inode && + left.ownerUid === right.ownerUid && + left.socketPath === right.socketPath && + left.directoryChain.length === right.directoryChain.length && + left.directoryChain.every((component, index) => { + const candidate = right.directoryChain[index]; + return ( + candidate?.device === component.device && + candidate.inode === component.inode && + candidate.mode === component.mode && + candidate.ownerUid === component.ownerUid && + candidate.path === component.path + ); + }) + ); +} + +function hasMatchingContainerRollbackAuthority( + transaction: PodmanManagedStartupTransaction, + authority: PodmanManagedSandboxRecreateTransaction | null | undefined, +): boolean { + if (!authority) return false; + const transactionImage = transaction.image.replace(/^sha256:/u, ""); + const authorityImage = authority.immutableImage.replace(/^sha256:/u, ""); + if ( + authority.driverName !== "podman" || + authority.applied !== true || + authority.socketPath !== transaction.runtime.socketPath || + !sameSocketAuthority(transaction.runtime.socketAuthority, authority.socketAuthority) || + authority.newContainerId !== transaction.containerId || + authorityImage !== transactionImage + ) { + throw new Error( + "Podman managed-startup rollback authority does not match the exact runtime, image, and replacement container.", + ); + } + return true; +} + +function cleanupReceiptBestEffort(receiptPath: string): void { + try { + cleanupTempDir(receiptPath, RECEIPT_TEMP_PREFIX); + } catch (error) { + console.warn( + ` ⚠ Managed-startup shared state is finalized, but its protected Podman receipt could not be removed (${receiptPath}): ${ + error instanceof Error ? error.message : String(error) + }`, + ); + } +} + +function requireZero(result: ReturnType, action: string): void { + if (result.status === 0) return; + const detail = podmanManagedStartupCommandDetail(result, 800); + throw new Error(`${action}${detail ? `: ${detail}` : ""}`); +} + +function assertPinnedContainer( + transaction: PodmanManagedStartupTransaction, + deps: PodmanManagedStartupSharedStateDeps, + requireRunning: boolean, +): void { + assertPodmanManagedStartupRuntime(transaction.runtime, deps); + inspectExactPodmanManagedStartupContainer( + transaction.runtime, + { + containerId: transaction.containerId, + image: transaction.image, + requireRunning, + }, + deps, + ); +} + +function quiesceManagedStartupContainer( + transaction: PodmanManagedStartupTransaction, + deps: PodmanManagedStartupSharedStateDeps, +): void { + assertPinnedContainer(transaction, deps, false); + requireZero( + runManagedStartupPodman( + transaction.runtime.socketPath, + ["stop", "--time", STOP_GRACE_SECONDS, transaction.containerId], + { timeout: STOP_TIMEOUT_MS }, + deps, + ), + "Could not quiesce the failed Podman managed-startup container before shared-state rollback", + ); + assertPodmanManagedStartupRuntime(transaction.runtime, deps); + const stopped = inspectExactPodmanManagedStartupContainer( + transaction.runtime, + { containerId: transaction.containerId, image: transaction.image }, + deps, + ); + if (stopped.running) { + throw new Error( + "Podman reported success without quiescing the managed-startup container before rollback.", + ); + } +} + +function copyManagedStartupReceipt( + transaction: PodmanManagedStartupTransaction, + deps: PodmanManagedStartupSharedStateDeps, +): string { + assertPinnedContainer(transaction, deps, false); + const receiptPath = secureTempFile(RECEIPT_TEMP_PREFIX); + try { + requireZero( + runManagedStartupPodman( + transaction.runtime.socketPath, + [ + "cp", + `${transaction.containerId}:${MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY}`, + receiptPath, + ], + { timeout: MUTATION_TIMEOUT_MS }, + deps, + ), + "Could not copy the managed-startup rollback receipt from the Podman container", + ); + if (receiptPath.includes(",") || /[\0\r\n]/u.test(receiptPath)) { + throw new Error("Managed-startup rollback receipt path is unsafe for a Podman bind mount."); + } + return receiptPath; + } catch (error) { + cleanupReceiptBestEffort(receiptPath); + throw error; + } +} + +function transactionCommand(action: "commit" | "rollback", agent: string): string[] { + return [ + MANAGED_STARTUP_RUNTIME_EXECUTABLE, + `--${action}-shared-state-transaction`, + "--agent", + agent, + ]; +} + +function rollbackManagedStartupSharedState( + transaction: PodmanManagedStartupTransaction, + receiptPath: string, + deps: PodmanManagedStartupSharedStateDeps, +): void { + assertPodmanManagedStartupRuntime(transaction.runtime, deps); + let restored = false; + try { + requireZero( + runManagedStartupPodman( + transaction.runtime.socketPath, + [ + "run", + "--rm", + "--pull", + "never", + "--network", + "none", + "--read-only", + "--user", + "0:0", + "--security-opt", + "no-new-privileges", + "--cap-drop", + "ALL", + "--cap-add", + "CHOWN", + "--cap-add", + "DAC_OVERRIDE", + "--cap-add", + "FOWNER", + ...NEUTRALIZED_PROCESS_INJECTION_ENV, + "--volumes-from", + transaction.containerId, + "--mount", + `type=bind,src=${receiptPath},dst=${MANAGED_STARTUP_SHARED_ROLLBACK_RECEIPT_DIRECTORY},readonly`, + "--entrypoint", + "/usr/local/bin/node", + transaction.image, + ...transactionCommand("rollback", transaction.agent), + "--read-only-receipt", + ], + { timeout: MUTATION_TIMEOUT_MS }, + deps, + ), + `Immutable Podman managed-startup helper could not restore and verify shared state. Protected receipt retained at ${receiptPath}`, + ); + assertPodmanManagedStartupRuntime(transaction.runtime, deps); + restored = true; + } finally { + if (restored) cleanupReceiptBestEffort(receiptPath); + } +} + +function removeFailedUnbackedContainer( + transaction: PodmanManagedStartupTransaction, + deps: PodmanManagedStartupSharedStateDeps, +): void { + assertPinnedContainer(transaction, deps, false); + requireZero( + runManagedStartupPodman( + transaction.runtime.socketPath, + ["rm", transaction.containerId], + { timeout: MUTATION_TIMEOUT_MS }, + deps, + ), + "Could not remove the failed Podman managed-startup container after shared-state rollback", + ); + assertPodmanManagedStartupRuntime(transaction.runtime, deps); + const exists = runManagedStartupPodman( + transaction.runtime.socketPath, + ["container", "exists", transaction.containerId], + {}, + deps, + ); + if (exists.status !== 1) { + throw new Error( + `Could not prove removal of the failed Podman managed-startup container${ + exists.status === 0 + ? ": the exact container still exists." + : `: ${podmanManagedStartupCommandDetail(exists, 800)}` + }`, + ); + } +} + +/** + * Finalize the shared-state half of a Podman managed-container cutover. + * + * `containerRollbackAuthority` is the exact runtime-level recreation + * transaction retained by the caller. It is cross-checked against the + * managed-startup transaction before any mutation. Without that authority this + * layer removes a failed replacement after shared state is restored. + */ +export function finalizePodmanManagedStartupSharedState( + input: { + readonly containerRollbackAuthority?: PodmanManagedSandboxRecreateTransaction | null; + readonly supervisorReady: boolean; + readonly transaction: PodmanManagedStartupTransaction | null; + }, + deps: PodmanManagedStartupSharedStateDeps = {}, +): PodmanManagedStartupSharedStateOutcome { + const transaction = input.transaction; + if (!transaction) { + return { failure: null, supervisorReady: input.supervisorReady }; + } + deps = { ...deps, socketAuthority: transaction.runtime.socketAuthority }; + const containerRollbackArmed = hasMatchingContainerRollbackAuthority( + transaction, + input.containerRollbackAuthority, + ); + assertPinnedContainer(transaction, deps, input.supervisorReady); + if (input.supervisorReady) { + let receiptPath: string; + try { + receiptPath = copyManagedStartupReceipt(transaction, deps); + } catch (error) { + try { + quiesceManagedStartupContainer(transaction, deps); + } catch (stopError) { + throw new Error( + `Managed-startup Podman receipt preservation failed and the new workload could not be quiesced: ${ + error instanceof Error ? error.message : String(error) + }; ${stopError instanceof Error ? stopError.message : String(stopError)}`, + ); + } + throw error; + } + assertPodmanManagedStartupRuntime(transaction.runtime, deps); + const commit = runManagedStartupPodman( + transaction.runtime.socketPath, + [ + "exec", + "--user", + "0:0", + ...NEUTRALIZED_PROCESS_INJECTION_ENV, + transaction.containerId, + "/usr/bin/env", + "-i", + "HOME=/root", + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + "/usr/local/bin/node", + ...transactionCommand("commit", transaction.agent), + ], + { timeout: MUTATION_TIMEOUT_MS }, + deps, + ); + let failure: Error | null = null; + if (commit.status === 0) { + try { + assertPinnedContainer(transaction, deps, true); + } catch (error) { + failure = new Error( + `Podman managed shared-state commit returned success, but exact runtime and container identity could not be revalidated: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + } + if (!failure) { + cleanupReceiptBestEffort(receiptPath); + return { failure: null, supervisorReady: true }; + } + } else { + failure = new Error( + `OpenShell supervisor reconnected, but Podman managed shared-state commit failed: ${podmanManagedStartupCommandDetail( + commit, + 800, + )}`, + ); + } + quiesceManagedStartupContainer(transaction, deps); + rollbackManagedStartupSharedState(transaction, receiptPath, deps); + if (!containerRollbackArmed) { + removeFailedUnbackedContainer(transaction, deps); + } + return { failure, supervisorReady: false }; + } + + quiesceManagedStartupContainer(transaction, deps); + const receiptPath = copyManagedStartupReceipt(transaction, deps); + rollbackManagedStartupSharedState(transaction, receiptPath, deps); + if (!containerRollbackArmed) { + removeFailedUnbackedContainer(transaction, deps); + } + return { failure: null, supervisorReady: false }; +} diff --git a/src/lib/onboard/managed-startup/profile-builder.ts b/src/lib/onboard/managed-startup/profile-builder.ts new file mode 100644 index 0000000000..feb5c82fb1 --- /dev/null +++ b/src/lib/onboard/managed-startup/profile-builder.ts @@ -0,0 +1,935 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { Buffer } from "node:buffer"; +import { createHash, X509Certificate } from "node:crypto"; + +import { MAX_AUTODETECTED_OLLAMA_CONTEXT_WINDOW } from "../../inference/ollama-runtime-context"; +import { hydrateDerivedSandboxMessagingPlanFields } from "../../messaging/hydration"; +import { parseSandboxMessagingPlan } from "../../messaging/plan-validation"; +import { withLocalNoProxy } from "../../proxy/local-no-proxy"; +import { + MAX_CORPORATE_CA_BYTES, + MAX_CORPORATE_CA_CERTS, + PEM_CERTIFICATE_RE_GLOBAL, +} from "../corporate-ca-policy"; +import type { ResolvedCorporateCa } from "../corporate-ca-types"; +import { normalizeCertificateBlocks } from "../corporate-ca-validation"; +import { + encodeManagedStartupProfile, + MANAGED_STARTUP_PROFILE_AFFORDANCE_INVENTORY, + MANAGED_STARTUP_PROFILE_SCHEMA_VERSION, + MANAGED_STARTUP_REASONING_EFFORTS, + type ManagedStartupAgent, + type ManagedStartupDashboard, + type ManagedStartupDcodeAutoApprovalMode, + type ManagedStartupExtraAgents, + type ManagedStartupHermesToolGateway, + type ManagedStartupInputModality, + type ManagedStartupJsonObject, + type ManagedStartupProfile, + type ManagedStartupReasoningEffort, + type ManagedStartupToolDisclosure, + type ManagedStartupWebSearch, + validateManagedStartupProfile, +} from "./profile"; + +const DEFAULT_MANAGED_PROXY_HOST = "10.200.0.1"; +const DEFAULT_MANAGED_PROXY_PORT = 3128; +const DEFAULT_CONTEXT_WINDOW = 131_072; +const DEFAULT_OPENCLAW_MAX_TOKENS = 4096; +const DEFAULT_OPENCLAW_AGENT_TIMEOUT_SECONDS = 600; +const DEFAULT_OPENCLAW_OTEL_ENDPOINT = "http://host.openshell.internal:4318"; +const DEFAULT_OPENCLAW_OTEL_SERVICE_NAME = "openclaw-gateway"; +const MIN_HERMES_CONTEXT_WINDOW = 64_000; +const MAX_PROFILE_TUNING_INTEGER = 1_000_000_000; +const FALSE_VALUES = new Set(["0", "false", "no", "off"]); +const TRUE_VALUES = new Set(["1", "true", "yes", "on"]); +const STANDARD_BASE64_RE = /^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/u; + +/** + * These digests deliberately bind the builder to every classified stock + * Docker/runtime affordance, including its profile path and representation. + * Adding or reclassifying an affordance must update the builder in the same + * change; otherwise construction fails before a sandbox is launched. + */ +const EXPECTED_AFFORDANCE_INVENTORY_SHA256 = { + openclaw: "9b722441e33f0b0d7580f74cd185c0174979de9c1a784556ff56ff931b2c9904", + hermes: "eb3c3a42395256d6228a828a483362b6534b4774be515f14fa3b5c936d00c6d6", + "langchain-deepagents-code": "780ec15dd97efaea5b75614d657a4baf93cba4c7933e50f78fb72814cb427c85", +} as const satisfies Record; + +const INVENTORY_INPUTS = new Set( + Object.values(MANAGED_STARTUP_PROFILE_AFFORDANCE_INVENTORY).flatMap((entries) => + entries.map((entry) => entry.input), + ), +); + +export interface ManagedStartupResolvedInferenceInput { + readonly routeProvider: string; + readonly upstreamProvider: string; + readonly model: string; + readonly routedBaseUrl: string; + readonly upstreamEndpointUrl: string | null; + readonly api: ManagedStartupProfile["inference"]["api"]; + readonly primaryModelRef: string | null; + readonly compatibility: Readonly> | null; +} + +export interface ManagedStartupProfileBuilderInput { + readonly agent: ManagedStartupAgent; + /** Fully resolved host semantics; the portable builder never selects providers. */ + readonly inference: ManagedStartupResolvedInferenceInput; + readonly dashboard: ManagedStartupDashboard; + readonly webSearch: { + readonly fetchEnabled: boolean; + readonly provider?: "brave" | "tavily"; + } | null; + readonly toolDisclosure: ManagedStartupToolDisclosure; + readonly hermesToolGateways: readonly string[]; + readonly messagingPlan: unknown | null; + readonly dcodeAutoApprovalMode: ManagedStartupDcodeAutoApprovalMode | null; + readonly observabilityEnabled: boolean | null; + /** + * Host onboarding knobs only. The builder reads an exact allowlist and never + * copies this object wholesale, so ambient credentials cannot enter a profile. + */ + readonly environment: NodeJS.ProcessEnv; + /** Validated host CA material returned by the existing corporate-CA resolver. */ + readonly corporateCa?: ResolvedCorporateCa | null; +} + +export interface BuiltManagedStartupProfile { + readonly profile: ManagedStartupProfile; + readonly encodedProfile: string; + /** Digest of the canonical secret-free profile transport. */ + readonly startupProfileSha256: string; + /** Exact normalized PEM bytes, encoded separately from the secret-free profile. */ + readonly corporateCaB64?: string; +} + +export class ManagedStartupProfileBuilderError extends Error { + constructor(message: string) { + super(`Cannot build managed startup profile: ${message}`); + this.name = "ManagedStartupProfileBuilderError"; + } +} + +function fail(message: string): never { + throw new ManagedStartupProfileBuilderError(message); +} + +function presentEnvironmentValue(environment: NodeJS.ProcessEnv, name: string): string | null { + const value = environment[name]; + if (typeof value !== "string") return null; + const trimmed = value.trim(); + return trimmed === "" ? null : trimmed; +} + +function parsePositiveInteger( + environment: NodeJS.ProcessEnv, + name: string, + fallback: number | null, + options: { readonly minimum?: number; readonly maximum?: number } = {}, +): number | null { + const raw = presentEnvironmentValue(environment, name); + if (raw === null) return fallback; + if (!/^[1-9][0-9]*$/u.test(raw)) { + fail(`${name} must be a positive integer`); + } + const parsed = Number(raw); + const minimum = options.minimum ?? 1; + const maximum = options.maximum ?? MAX_PROFILE_TUNING_INTEGER; + if (!Number.isSafeInteger(parsed) || parsed < minimum || parsed > maximum) { + fail(`${name} must be between ${String(minimum)} and ${String(maximum)}`); + } + return parsed; +} + +function parsePort(environment: NodeJS.ProcessEnv, name: string, fallback: number): number { + const port = parsePositiveInteger(environment, name, fallback, { maximum: 65_535 }); + if (port === null) fail(`${name} must resolve to a TCP port`); + return port; +} + +function parseZeroOneFlag( + environment: NodeJS.ProcessEnv, + name: string, + fallback: boolean, +): boolean { + const raw = presentEnvironmentValue(environment, name); + if (raw === null) return fallback; + if (raw === "1") return true; + if (raw === "0") return false; + fail(`${name} must be "0" or "1"`); +} + +function parseHumanBoolean( + environment: NodeJS.ProcessEnv, + name: string, + fallback: boolean, +): boolean { + const raw = presentEnvironmentValue(environment, name); + if (raw === null) return fallback; + const normalized = raw.toLowerCase(); + if (TRUE_VALUES.has(normalized)) return true; + if (FALSE_VALUES.has(normalized)) return false; + fail(`${name} must be a boolean value`); +} + +function parseOpenClawOtelEnabled(environment: NodeJS.ProcessEnv): boolean { + const raw = presentEnvironmentValue(environment, "NEMOCLAW_OPENCLAW_OTEL"); + return raw !== null && !FALSE_VALUES.has(raw.toLowerCase()); +} + +function parseReasoning(environment: NodeJS.ProcessEnv): boolean { + const raw = presentEnvironmentValue(environment, "NEMOCLAW_REASONING"); + if (raw === null) return false; + if (raw === "true") return true; + if (raw === "false") return false; + fail('NEMOCLAW_REASONING must be "true" or "false"'); +} + +function parseReasoningEffort(environment: NodeJS.ProcessEnv): ManagedStartupReasoningEffort { + const raw = presentEnvironmentValue(environment, "NEMOCLAW_REASONING_EFFORT"); + const normalized = raw === null ? "default" : raw.toLowerCase(); + if ((MANAGED_STARTUP_REASONING_EFFORTS as readonly string[]).includes(normalized)) { + return normalized as ManagedStartupReasoningEffort; + } + fail(`NEMOCLAW_REASONING_EFFORT must be one of: ${MANAGED_STARTUP_REASONING_EFFORTS.join(", ")}`); +} + +function parseInputModalities( + environment: NodeJS.ProcessEnv, +): readonly ManagedStartupInputModality[] { + const raw = presentEnvironmentValue(environment, "NEMOCLAW_INFERENCE_INPUTS"); + if (raw === null) return ["text"]; + const values = raw.split(",").map((value) => value.trim()); + if ( + values.length === 0 || + values.some((value) => value !== "text" && value !== "image") || + new Set(values).size !== values.length + ) { + fail("NEMOCLAW_INFERENCE_INPUTS must be a unique comma-separated list of text and image"); + } + return values as ManagedStartupInputModality[]; +} + +function parseHeartbeat(environment: NodeJS.ProcessEnv): string | null { + const raw = presentEnvironmentValue(environment, "NEMOCLAW_AGENT_HEARTBEAT_EVERY"); + if (raw === null) return null; + if (!/^\d+(?:s|m|h)$/u.test(raw)) { + fail("NEMOCLAW_AGENT_HEARTBEAT_EVERY must be a duration ending in s, m, or h"); + } + return raw; +} + +function parseStrictBase64Json(raw: string, name: string): unknown { + if (!STANDARD_BASE64_RE.test(raw)) fail(`${name} must be canonical base64`); + const bytes = Buffer.from(raw, "base64"); + if (bytes.toString("base64") !== raw) fail(`${name} must be canonical base64`); + try { + return JSON.parse(bytes.toString("utf8")) as unknown; + } catch { + fail(`${name} must contain valid UTF-8 JSON`); + } +} + +function parseRawJson(raw: string, name: string): unknown { + try { + return JSON.parse(raw) as unknown; + } catch { + fail(`${name} must contain valid JSON`); + } +} + +function isPlainObject(value: unknown): value is Record { + if (value === null || typeof value !== "object" || Array.isArray(value)) return false; + const prototype = Object.getPrototypeOf(value); + return prototype === Object.prototype || prototype === null; +} + +function normalizeExtraAgentsCandidate(value: unknown): ManagedStartupExtraAgents { + const emptyDefaults = { subagents: {} }; + if (value === null || value === undefined) { + return { agents: [], defaults: emptyDefaults, main: {} }; + } + if (Array.isArray(value)) { + return { + agents: value as ManagedStartupJsonObject[], + defaults: emptyDefaults, + main: {}, + }; + } + if (!isPlainObject(value)) { + fail( + "NEMOCLAW_EXTRA_AGENTS_JSON must be an array or an object with agents, defaults, and main", + ); + } + const unknownKeys = Object.keys(value).filter( + (key) => key !== "agents" && key !== "defaults" && key !== "main", + ); + if (unknownKeys.length > 0) { + fail("NEMOCLAW_EXTRA_AGENTS_JSON contains unsupported top-level fields"); + } + const agents = value.agents ?? []; + const defaults = value.defaults ?? emptyDefaults; + const main = value.main ?? {}; + if (!Array.isArray(agents) || !agents.every((agent) => isPlainObject(agent))) { + fail("NEMOCLAW_EXTRA_AGENTS_JSON.agents must be an object list"); + } + if (!isPlainObject(defaults) || !isPlainObject(main)) { + fail("NEMOCLAW_EXTRA_AGENTS_JSON defaults and main must be objects"); + } + return { + agents: agents as ManagedStartupJsonObject[], + defaults: defaults as ManagedStartupJsonObject, + main: main as ManagedStartupJsonObject, + }; +} + +function parseExtraAgents(environment: NodeJS.ProcessEnv): ManagedStartupExtraAgents { + const raw = presentEnvironmentValue(environment, "NEMOCLAW_EXTRA_AGENTS_JSON"); + const encoded = presentEnvironmentValue(environment, "NEMOCLAW_EXTRA_AGENTS_JSON_B64"); + if (raw !== null && encoded !== null) { + fail("NEMOCLAW_EXTRA_AGENTS_JSON and NEMOCLAW_EXTRA_AGENTS_JSON_B64 must not both be set"); + } + return normalizeExtraAgentsCandidate( + raw !== null + ? parseRawJson(raw, "NEMOCLAW_EXTRA_AGENTS_JSON") + : encoded !== null + ? parseStrictBase64Json(encoded, "NEMOCLAW_EXTRA_AGENTS_JSON_B64") + : null, + ); +} + +function normalizeWebSearch( + agent: ManagedStartupAgent, + config: ManagedStartupProfileBuilderInput["webSearch"], +): ManagedStartupWebSearch | null { + if (agent === "langchain-deepagents-code") { + if (config !== null) { + fail("langchain-deepagents-code does not support web-search profile intent"); + } + return null; + } + if (config !== null) { + if (!isPlainObject(config)) fail("webSearch must be null or a configuration object"); + const unknownKeys = Object.keys(config).filter( + (key) => key !== "fetchEnabled" && key !== "provider", + ); + if (unknownKeys.length > 0 || typeof config.fetchEnabled !== "boolean") { + fail("webSearch contains unsupported or malformed fields"); + } + if ( + config.provider !== undefined && + config.provider !== "brave" && + config.provider !== "tavily" + ) { + fail("webSearch.provider must be brave or tavily"); + } + } + const provider = + agent === "hermes" && config?.provider === undefined ? "tavily" : (config?.provider ?? "brave"); + if (agent === "hermes" && provider !== "tavily") { + fail("Hermes supports only the Tavily web-search provider"); + } + return { enabled: config?.fetchEnabled === true, provider }; +} + +function normalizeMessagingPlan( + agent: ManagedStartupAgent, + value: unknown | null, +): ManagedStartupJsonObject | null { + if (agent === "langchain-deepagents-code") { + if (value !== null) { + fail("langchain-deepagents-code messagingPlan must be null"); + } + return null; + } + if (value === null) return null; + const plan = parseSandboxMessagingPlan(value, { agent }); + if (!plan) fail(`messagingPlan must be a valid ${agent} SandboxMessagingPlan`); + const hydrated = hydrateDerivedSandboxMessagingPlanFields(plan); + const reparsed = parseSandboxMessagingPlan(hydrated, { agent }); + if (!reparsed) fail(`messagingPlan hydration produced an invalid ${agent} plan`); + return JSON.parse(JSON.stringify(reparsed)) as ManagedStartupJsonObject; +} + +function resolveAliasedEnvironmentValue( + environment: NodeJS.ProcessEnv, + upper: string, + lower: string, +): string | null { + const upperValue = presentEnvironmentValue(environment, upper); + const lowerValue = presentEnvironmentValue(environment, lower); + if (upperValue !== null && lowerValue !== null && upperValue !== lowerValue) { + fail(`${upper} and ${lower} must not express conflicting values`); + } + return upperValue ?? lowerValue; +} + +function normalizeNoProxyList(raw: string | null): string[] { + if (raw === null) return []; + const values = raw + .split(",") + .map((value) => value.trim()) + .filter(Boolean); + return [...new Set(values)]; +} + +function resolveHostProxy( + _agent: ManagedStartupAgent, + environment: NodeJS.ProcessEnv, +): Pick { + const hostHttpUrl = resolveAliasedEnvironmentValue(environment, "HTTP_PROXY", "http_proxy"); + const hostHttpsUrl = resolveAliasedEnvironmentValue(environment, "HTTPS_PROXY", "https_proxy"); + const noProxy = resolveAliasedEnvironmentValue(environment, "NO_PROXY", "no_proxy"); + if (hostHttpUrl === null && hostHttpsUrl === null) { + if (noProxy !== null) { + fail("NO_PROXY/no_proxy requires an HTTP_PROXY or HTTPS_PROXY intent"); + } + return { hostHttpUrl: null, hostHttpsUrl: null, hostNoProxy: [] }; + } + + const normalizedEnvironment: Record = {}; + if (hostHttpUrl !== null) { + normalizedEnvironment.HTTP_PROXY = hostHttpUrl; + normalizedEnvironment.http_proxy = hostHttpUrl; + } + if (hostHttpsUrl !== null) { + normalizedEnvironment.HTTPS_PROXY = hostHttpsUrl; + normalizedEnvironment.https_proxy = hostHttpsUrl; + } + if (noProxy !== null) { + normalizedEnvironment.NO_PROXY = noProxy; + normalizedEnvironment.no_proxy = noProxy; + } + withLocalNoProxy(normalizedEnvironment); + const upperNoProxy = normalizeNoProxyList(normalizedEnvironment.NO_PROXY ?? null); + const lowerNoProxy = normalizeNoProxyList(normalizedEnvironment.no_proxy ?? null); + return { + hostHttpUrl, + hostHttpsUrl, + hostNoProxy: [...new Set([...upperNoProxy, ...lowerNoProxy])], + }; +} + +function resolveCorporateCaMaterial(corporateCa: ResolvedCorporateCa | null | undefined): { + readonly bundleSha256: string | null; + readonly corporateCaB64?: string; +} { + if (!corporateCa) return { bundleSha256: null }; + const bytes = Buffer.byteLength(corporateCa.pem, "utf8"); + if (bytes === 0 || bytes > MAX_CORPORATE_CA_BYTES) { + fail(`corporate CA material must be between 1 and ${String(MAX_CORPORATE_CA_BYTES)} bytes`); + } + const blocks = corporateCa.pem.match(PEM_CERTIFICATE_RE_GLOBAL); + if (!blocks || blocks.length === 0 || blocks.length > MAX_CORPORATE_CA_CERTS) { + fail( + `corporate CA material must contain between 1 and ${String(MAX_CORPORATE_CA_CERTS)} certificates`, + ); + } + for (const block of blocks) { + let certificate: X509Certificate; + try { + certificate = new X509Certificate(block); + } catch { + fail("corporate CA material contains an invalid X.509 certificate"); + } + if (!certificate.ca) { + fail("corporate CA material contains a certificate without CA basic constraints"); + } + } + const normalizedPem = normalizeCertificateBlocks(blocks); + if (corporateCa.pem.trim() !== normalizedPem.trim()) { + fail("corporate CA material must contain only normalized PEM certificate blocks"); + } + return { + bundleSha256: createHash("sha256").update(normalizedPem, "utf8").digest("hex"), + corporateCaB64: Buffer.from(normalizedPem, "utf8").toString("base64"), + }; +} + +function assertAgentSpecificInput(input: ManagedStartupProfileBuilderInput): void { + if (input.dashboard.agent !== input.agent) { + fail("dashboard.agent must match agent"); + } + if (input.agent === "openclaw") { + if ( + input.inference.upstreamEndpointUrl !== null || + input.hermesToolGateways.length > 0 || + input.dcodeAutoApprovalMode !== null || + input.observabilityEnabled !== null + ) { + fail("OpenClaw input contains state owned by another agent"); + } + return; + } + if (input.agent === "hermes") { + if ( + input.inference.upstreamEndpointUrl !== null || + input.dcodeAutoApprovalMode !== null || + input.observabilityEnabled !== null + ) { + fail("Hermes input contains state owned by another agent"); + } + return; + } + if (input.webSearch !== null) { + fail("langchain-deepagents-code does not support web-search profile intent"); + } + if (input.hermesToolGateways.length > 0) { + fail("langchain-deepagents-code does not support Hermes tool gateways"); + } + if (input.messagingPlan !== null) { + fail("langchain-deepagents-code messagingPlan must be null"); + } + if ( + input.dcodeAutoApprovalMode !== "disabled" && + input.dcodeAutoApprovalMode !== "thread-opt-in" + ) { + fail("DCode approval state must be explicit"); + } + if (typeof input.observabilityEnabled !== "boolean") { + fail("DCode observability state must be explicit"); + } +} + +function assertNoWrongAgentEnvironment( + agent: ManagedStartupAgent, + environment: NodeJS.ProcessEnv, +): void { + const supported = new Set( + MANAGED_STARTUP_PROFILE_AFFORDANCE_INVENTORY[agent].map((entry) => entry.input), + ); + for (const name of INVENTORY_INPUTS) { + if (presentEnvironmentValue(environment, name) === null || supported.has(name)) continue; + // NEMOCLAW_DASHBOARD_PORT is a host-side input used to resolve the explicit + // Hermes dashboard state even though the sandbox consumes its Hermes alias. + if (agent === "hermes" && name === "NEMOCLAW_DASHBOARD_PORT") continue; + fail(`${name} is not supported by ${agent}`); + } + const rawExtraAgents = presentEnvironmentValue(environment, "NEMOCLAW_EXTRA_AGENTS_JSON"); + if (agent !== "openclaw" && rawExtraAgents !== null) { + fail(`NEMOCLAW_EXTRA_AGENTS_JSON is not supported by ${agent}`); + } +} + +function inventoryDigest(agent: ManagedStartupAgent): string { + return createHash("sha256") + .update(JSON.stringify(MANAGED_STARTUP_PROFILE_AFFORDANCE_INVENTORY[agent])) + .digest("hex"); +} + +/** Fail closed if the authoritative affordance inventory changes without this builder. */ +export function assertManagedStartupProfileBuilderInventoryCoverage(): void { + for (const agent of Object.keys(EXPECTED_AFFORDANCE_INVENTORY_SHA256) as ManagedStartupAgent[]) { + if (inventoryDigest(agent) !== EXPECTED_AFFORDANCE_INVENTORY_SHA256[agent]) { + fail(`${agent} affordance inventory changed without a builder mapping update`); + } + } +} + +function profilePathExists(profile: ManagedStartupProfile, profilePath: string): boolean { + let value: unknown = profile; + for (const segment of profilePath.split(".")) { + if (!isPlainObject(value) || !Object.hasOwn(value, segment)) return false; + value = value[segment]; + } + return true; +} + +function assertInventoryPathsResolved(profile: ManagedStartupProfile): void { + for (const affordance of MANAGED_STARTUP_PROFILE_AFFORDANCE_INVENTORY[profile.agent]) { + if (!profilePathExists(profile, affordance.profilePath)) { + fail(`${affordance.input} has no resolved value at ${affordance.profilePath}`); + } + } +} + +function canonicalJson(value: unknown): string { + if (Array.isArray(value)) { + return `[${value.map((item) => canonicalJson(item)).join(",")}]`; + } + if (isPlainObject(value)) { + return `{${Object.keys(value) + .sort() + .map((key) => `${JSON.stringify(key)}:${canonicalJson(value[key])}`) + .join(",")}}`; + } + return JSON.stringify(value); +} + +function assertEquivalent(name: string, actual: unknown, expected: unknown): void { + if (canonicalJson(actual) !== canonicalJson(expected)) { + fail(`${name} conflicts with the resolved semantic onboarding state`); + } +} + +function parseEnvironmentJson(environment: NodeJS.ProcessEnv, name: string): unknown | undefined { + const raw = presentEnvironmentValue(environment, name); + return raw === null ? undefined : parseStrictBase64Json(raw, name); +} + +/** + * Environment may carry legacy Docker inputs while callers migrate. Parse and + * compare each supplied value instead of silently preferring one source. + */ +function assertEnvironmentConsistency( + profile: ManagedStartupProfile, + environment: NodeJS.ProcessEnv, +): void { + const stringValues: Readonly> = { + NEMOCLAW_MODEL: profile.inference.model, + NEMOCLAW_INFERENCE_PROVIDER_ID: profile.inference.routeProvider, + NEMOCLAW_UPSTREAM_PROVIDER: profile.inference.upstreamProvider, + NEMOCLAW_PRIMARY_MODEL_REF: profile.inference.primaryModelRef, + NEMOCLAW_INFERENCE_BASE_URL: profile.inference.routedBaseUrl, + NEMOCLAW_INFERENCE_API: profile.inference.api, + NEMOCLAW_TOOL_DISCLOSURE: profile.tools.disclosure, + CHAT_UI_URL: + profile.dashboard.agent === "langchain-deepagents-code" ? null : profile.dashboard.url, + }; + for (const [name, expected] of Object.entries(stringValues)) { + const raw = presentEnvironmentValue(environment, name); + if (raw !== null) assertEquivalent(name, raw, expected); + } + + const numericValues: Readonly> = { + NEMOCLAW_CONTEXT_WINDOW: profile.tuning.contextWindow, + NEMOCLAW_MAX_TOKENS: profile.tuning.maxTokens, + NEMOCLAW_PROXY_PORT: profile.proxy.managedPort, + NEMOCLAW_AGENT_TIMEOUT: + profile.agentConfig.agent === "openclaw" ? profile.agentConfig.agentTimeoutSeconds : null, + NEMOCLAW_OPENCLAW_OTEL_SAMPLE_RATE: + profile.agentConfig.agent === "openclaw" ? profile.agentConfig.otel.sampleRate : null, + }; + for (const [name, expected] of Object.entries(numericValues)) { + const raw = presentEnvironmentValue(environment, name); + if (raw !== null) assertEquivalent(name, Number(raw), expected); + } + + const managedHost = presentEnvironmentValue(environment, "NEMOCLAW_PROXY_HOST"); + if (managedHost !== null) + assertEquivalent("NEMOCLAW_PROXY_HOST", managedHost, profile.proxy.managedHost); + + const upstreamEndpoint = presentEnvironmentValue(environment, "NEMOCLAW_UPSTREAM_ENDPOINT_URL"); + if (upstreamEndpoint !== null) { + assertEquivalent( + "NEMOCLAW_UPSTREAM_ENDPOINT_URL", + upstreamEndpoint, + profile.inference.upstreamEndpointUrl, + ); + } + + if (profile.agentConfig.agent === "openclaw") { + const config = profile.agentConfig; + const dashboard = profile.dashboard; + if (dashboard.agent !== "openclaw") fail("OpenClaw dashboard state is inconsistent"); + const directValues: Readonly> = { + NEMOCLAW_REASONING: String(profile.tuning.reasoning), + NEMOCLAW_AGENT_HEARTBEAT_EVERY: config.heartbeatEvery, + NEMOCLAW_DISABLE_DEVICE_AUTH: config.deviceAuth.disabled ? "1" : "0", + NEMOCLAW_DEVICE_AUTH_OPT_OUT_SOURCE: config.deviceAuth.optOutSource, + NEMOCLAW_WEB_SEARCH_ENABLED: config.webSearch.enabled ? "1" : "0", + NEMOCLAW_WEB_SEARCH_PROVIDER: config.webSearch.provider, + NEMOCLAW_OPENCLAW_OTEL_ENDPOINT: config.otel.endpointUrl, + NEMOCLAW_OPENCLAW_OTEL_SERVICE_NAME: config.otel.serviceName, + NEMOCLAW_DASHBOARD_BIND: dashboard.bindAddress === "0.0.0.0" ? "0.0.0.0" : null, + NEMOCLAW_WSL_DASHBOARD_EXPOSURE: dashboard.wslExposure ? "1" : "0", + NEMOCLAW_DASHBOARD_PORT: dashboard.port, + }; + for (const [name, expected] of Object.entries(directValues)) { + const raw = presentEnvironmentValue(environment, name); + if (raw === null) continue; + assertEquivalent(name, typeof expected === "number" ? Number(raw) : raw, expected); + } + const reasoningEffort = presentEnvironmentValue(environment, "NEMOCLAW_REASONING_EFFORT"); + if (reasoningEffort !== null) { + assertEquivalent( + "NEMOCLAW_REASONING_EFFORT", + reasoningEffort.toLowerCase(), + profile.tuning.reasoningEffort, + ); + } + if (presentEnvironmentValue(environment, "NEMOCLAW_OPENCLAW_OTEL") !== null) { + assertEquivalent( + "NEMOCLAW_OPENCLAW_OTEL", + parseOpenClawOtelEnabled(environment), + config.otel.enabled, + ); + } + if (presentEnvironmentValue(environment, "NEMOCLAW_MINIMAL_BOOTSTRAP") !== null) { + assertEquivalent( + "NEMOCLAW_MINIMAL_BOOTSTRAP", + parseZeroOneFlag(environment, "NEMOCLAW_MINIMAL_BOOTSTRAP", false), + config.minimalBootstrap, + ); + } + if (presentEnvironmentValue(environment, "NEMOCLAW_INFERENCE_INPUTS") !== null) { + assertEquivalent( + "NEMOCLAW_INFERENCE_INPUTS", + [...parseInputModalities(environment)].sort(), + profile.inference.inputModalities, + ); + } + const compatibility = parseEnvironmentJson(environment, "NEMOCLAW_INFERENCE_COMPAT_B64"); + if (compatibility !== undefined) { + assertEquivalent( + "NEMOCLAW_INFERENCE_COMPAT_B64", + compatibility, + profile.inference.compatibility, + ); + } + const extraAgents = parseEnvironmentJson(environment, "NEMOCLAW_EXTRA_AGENTS_JSON_B64"); + if (extraAgents !== undefined) { + assertEquivalent( + "NEMOCLAW_EXTRA_AGENTS_JSON_B64", + normalizeExtraAgentsCandidate(extraAgents), + config.extraAgents, + ); + } + } else if (profile.agentConfig.agent === "hermes") { + const config = profile.agentConfig; + const dashboard = profile.dashboard; + if (dashboard.agent !== "hermes") fail("Hermes dashboard state is inconsistent"); + const directValues: Readonly> = { + NEMOCLAW_WEB_SEARCH_ENABLED: config.webSearch.enabled ? "1" : "0", + NEMOCLAW_WEB_SEARCH_PROVIDER: config.webSearch.provider, + NEMOCLAW_HERMES_TOOL_GATEWAY_BROKER: profile.tools.enabledGateways.length > 0 ? "1" : "0", + NEMOCLAW_HERMES_DASHBOARD_PORT: dashboard.publicPort, + NEMOCLAW_HERMES_DASHBOARD_INTERNAL_PORT: dashboard.internalPort, + }; + for (const [name, expected] of Object.entries(directValues)) { + const raw = presentEnvironmentValue(environment, name); + if (raw === null) continue; + assertEquivalent(name, typeof expected === "number" ? Number(raw) : raw, expected); + } + for (const [name, expected] of [ + ["NEMOCLAW_HERMES_DASHBOARD", dashboard.mode === "loopback-forwarded"], + ["NEMOCLAW_HERMES_DASHBOARD_TUI", dashboard.tuiEnabled], + ] as const) { + if (presentEnvironmentValue(environment, name) !== null) { + assertEquivalent(name, parseHumanBoolean(environment, name, false), expected); + } + } + const hostDashboardPort = presentEnvironmentValue(environment, "NEMOCLAW_DASHBOARD_PORT"); + if (hostDashboardPort !== null) { + assertEquivalent("NEMOCLAW_DASHBOARD_PORT", Number(hostDashboardPort), dashboard.publicPort); + } + const presets = parseEnvironmentJson(environment, "NEMOCLAW_HERMES_TOOL_GATEWAY_PRESETS_B64"); + if (presets !== undefined) { + if (!Array.isArray(presets)) { + fail("NEMOCLAW_HERMES_TOOL_GATEWAY_PRESETS_B64 must encode an array"); + } + assertEquivalent( + "NEMOCLAW_HERMES_TOOL_GATEWAY_PRESETS_B64", + [...presets].sort(), + profile.tools.enabledGateways, + ); + } + } else { + const config = profile.agentConfig; + const approval = presentEnvironmentValue(environment, "NEMOCLAW_DCODE_AUTO_APPROVAL"); + if (approval !== null) { + assertEquivalent("NEMOCLAW_DCODE_AUTO_APPROVAL", approval, config.autoApprovalMode); + } + const observability = presentEnvironmentValue(environment, "NEMOCLAW_OBSERVABILITY"); + if (observability !== null) { + assertEquivalent( + "NEMOCLAW_OBSERVABILITY", + parseZeroOneFlag(environment, "NEMOCLAW_OBSERVABILITY", false), + config.observabilityEnabled, + ); + } + } + + const messaging = parseEnvironmentJson(environment, "NEMOCLAW_MESSAGING_PLAN_B64"); + if (messaging !== undefined) { + assertEquivalent( + "NEMOCLAW_MESSAGING_PLAN_B64", + normalizeMessagingPlan(profile.agent, messaging), + profile.messaging.plan, + ); + } + if (presentEnvironmentValue(environment, "NEMOCLAW_CORPORATE_CA_B64") !== null) { + fail("NEMOCLAW_CORPORATE_CA_B64 must use the separate corporateCa input"); + } +} + +function buildCandidate(input: ManagedStartupProfileBuilderInput): { + readonly profile: ManagedStartupProfile; + readonly corporateCaB64?: string; +} { + assertManagedStartupProfileBuilderInventoryCoverage(); + assertAgentSpecificInput(input); + assertNoWrongAgentEnvironment(input.agent, input.environment); + + const inference = input.inference; + const hostProxy = resolveHostProxy(input.agent, input.environment); + const managedHost = + presentEnvironmentValue(input.environment, "NEMOCLAW_PROXY_HOST") ?? DEFAULT_MANAGED_PROXY_HOST; + const managedPort = parsePort( + input.environment, + "NEMOCLAW_PROXY_PORT", + DEFAULT_MANAGED_PROXY_PORT, + ); + const messagingPlan = normalizeMessagingPlan(input.agent, input.messagingPlan); + const corporateCa = resolveCorporateCaMaterial(input.corporateCa); + const webSearch = normalizeWebSearch(input.agent, input.webSearch); + + let agentConfig: ManagedStartupProfile["agentConfig"]; + let tuning: ManagedStartupProfile["tuning"]; + if (input.agent === "openclaw") { + if (!webSearch) fail("OpenClaw web-search state is missing"); + const otelSampleRaw = + presentEnvironmentValue(input.environment, "NEMOCLAW_OPENCLAW_OTEL_SAMPLE_RATE") ?? "1.0"; + const otelSampleRate = Number(otelSampleRaw); + if (!Number.isFinite(otelSampleRate) || otelSampleRate < 0 || otelSampleRate > 1) { + fail("NEMOCLAW_OPENCLAW_OTEL_SAMPLE_RATE must be between 0 and 1"); + } + const otelEndpoint = + presentEnvironmentValue(input.environment, "NEMOCLAW_OPENCLAW_OTEL_ENDPOINT") ?? + DEFAULT_OPENCLAW_OTEL_ENDPOINT; + const otelServiceName = + presentEnvironmentValue(input.environment, "NEMOCLAW_OPENCLAW_OTEL_SERVICE_NAME") ?? + DEFAULT_OPENCLAW_OTEL_SERVICE_NAME; + agentConfig = { + agent: "openclaw", + webSearch, + otel: { + enabled: parseOpenClawOtelEnabled(input.environment), + endpointUrl: otelEndpoint, + serviceName: otelServiceName, + sampleRate: otelSampleRate, + }, + agentTimeoutSeconds: + parsePositiveInteger( + input.environment, + "NEMOCLAW_AGENT_TIMEOUT", + DEFAULT_OPENCLAW_AGENT_TIMEOUT_SECONDS, + ) ?? DEFAULT_OPENCLAW_AGENT_TIMEOUT_SECONDS, + heartbeatEvery: parseHeartbeat(input.environment), + extraAgents: parseExtraAgents(input.environment), + // Managed onboarding currently applies this compatibility opt-out to + // every stock OpenClaw image, independently of dashboard exposure. + deviceAuth: { disabled: true, optOutSource: "managed-onboard" }, + minimalBootstrap: parseZeroOneFlag(input.environment, "NEMOCLAW_MINIMAL_BOOTSTRAP", false), + }; + tuning = { + contextWindow: + parsePositiveInteger(input.environment, "NEMOCLAW_CONTEXT_WINDOW", DEFAULT_CONTEXT_WINDOW, { + maximum: MAX_AUTODETECTED_OLLAMA_CONTEXT_WINDOW, + }) ?? DEFAULT_CONTEXT_WINDOW, + maxTokens: + parsePositiveInteger( + input.environment, + "NEMOCLAW_MAX_TOKENS", + DEFAULT_OPENCLAW_MAX_TOKENS, + ) ?? DEFAULT_OPENCLAW_MAX_TOKENS, + reasoning: parseReasoning(input.environment), + reasoningEffort: parseReasoningEffort(input.environment), + }; + } else if (input.agent === "hermes") { + if (!webSearch) fail("Hermes web-search state is missing"); + agentConfig = { agent: "hermes", webSearch }; + tuning = { + contextWindow: parsePositiveInteger(input.environment, "NEMOCLAW_CONTEXT_WINDOW", null, { + minimum: MIN_HERMES_CONTEXT_WINDOW, + maximum: MAX_AUTODETECTED_OLLAMA_CONTEXT_WINDOW, + }), + maxTokens: null, + reasoning: null, + reasoningEffort: null, + }; + } else { + if (input.dcodeAutoApprovalMode === null || input.observabilityEnabled === null) { + fail("DCode approval and observability state must be explicit"); + } + agentConfig = { + agent: "langchain-deepagents-code", + autoApprovalMode: input.dcodeAutoApprovalMode, + observabilityEnabled: input.observabilityEnabled, + }; + tuning = { + contextWindow: null, + maxTokens: null, + reasoning: null, + reasoningEffort: null, + }; + } + + const candidate: ManagedStartupProfile = { + schemaVersion: MANAGED_STARTUP_PROFILE_SCHEMA_VERSION, + agent: input.agent, + agentConfig, + inference: { + routeProvider: inference.routeProvider, + upstreamProvider: inference.upstreamProvider, + model: inference.model, + routedBaseUrl: inference.routedBaseUrl, + upstreamEndpointUrl: inference.upstreamEndpointUrl, + api: inference.api, + primaryModelRef: inference.primaryModelRef, + // Docker's legacy JSON encoder maps a null compatibility result to {}, + // and the OpenClaw generator consumes an object in all cases. + compatibility: + input.agent === "openclaw" + ? (JSON.parse(JSON.stringify(inference.compatibility ?? {})) as ManagedStartupJsonObject) + : null, + inputModalities: input.agent === "openclaw" ? parseInputModalities(input.environment) : null, + }, + proxy: { + managedHost, + managedPort, + ...hostProxy, + }, + dashboard: input.dashboard, + tools: { + disclosure: input.toolDisclosure, + enabledGateways: + input.agent === "hermes" + ? (input.hermesToolGateways as readonly ManagedStartupHermesToolGateway[]) + : [], + }, + messaging: { plan: messagingPlan }, + tuning, + corporateCa: { bundleSha256: corporateCa.bundleSha256 }, + }; + + const profile = validateManagedStartupProfile(candidate); + assertInventoryPathsResolved(profile); + assertEnvironmentConsistency(profile, input.environment); + return corporateCa.corporateCaB64 === undefined + ? { profile } + : { profile, corporateCaB64: corporateCa.corporateCaB64 }; +} + +/** + * Build the one driver-neutral startup profile used by Docker, Podman, and + * future OpenShell compute drivers. + */ +export function buildManagedStartupProfile( + input: ManagedStartupProfileBuilderInput, +): BuiltManagedStartupProfile { + try { + const built = buildCandidate(input); + const encodedProfile = encodeManagedStartupProfile(built.profile); + const startupProfileSha256 = createHash("sha256").update(encodedProfile, "utf8").digest("hex"); + return Object.freeze( + built.corporateCaB64 === undefined + ? { profile: built.profile, encodedProfile, startupProfileSha256 } + : { + profile: built.profile, + encodedProfile, + startupProfileSha256, + corporateCaB64: built.corporateCaB64, + }, + ); + } catch (error) { + if (error instanceof ManagedStartupProfileBuilderError) throw error; + const message = error instanceof Error ? error.message : "unknown validation failure"; + throw new ManagedStartupProfileBuilderError(message); + } +} diff --git a/src/lib/onboard/managed-startup/profile.ts b/src/lib/onboard/managed-startup/profile.ts new file mode 100644 index 0000000000..aebfd915b3 --- /dev/null +++ b/src/lib/onboard/managed-startup/profile.ts @@ -0,0 +1,1491 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { Buffer } from "node:buffer"; +import { createHash } from "node:crypto"; +import { TextDecoder } from "node:util"; + +/** + * Versioned, secret-free configuration applied when a complete managed image + * starts. This is the portable replacement for build-time Dockerfile ARG/env + * mutation: Docker, Podman, and future OpenShell compute drivers consume the + * same bounded profile. + */ +export const MANAGED_STARTUP_PROFILE_SCHEMA_VERSION = 1 as const; + +/** Profiles are configuration, not a general-purpose transport. */ +export const MANAGED_STARTUP_PROFILE_MAX_BYTES = 64 * 1024; + +/** Maximum canonical base64url size for a profile at the decoded byte cap. */ +export const MANAGED_STARTUP_PROFILE_MAX_ENCODED_BYTES = + Math.ceil(MANAGED_STARTUP_PROFILE_MAX_BYTES / 3) * 4; + +const MAX_IDENTIFIER_BYTES = 256; +const MAX_MODEL_BYTES = 1024; +const MAX_URL_BYTES = 2048; +const MAX_LIST_ITEMS = 128; +const MAX_JSON_NODES = 4096; +const MAX_JSON_DEPTH = 32; +const MAX_TUNING_INTEGER = 1_000_000_000; +const SHA256_RE = /^[a-f0-9]{64}$/; +const CONTROL_CHARACTER_RE = /[\u0000-\u001f\u007f-\u009f]/u; +const BASE64URL_RE = /^[A-Za-z0-9_-]+$/; +const RAW_CA_PEM_RE = /-----BEGIN (?:TRUSTED )?CERTIFICATE-----/iu; +const RAW_CA_DER_BASE64_RE = /^MII[A-Za-z0-9+/=\r\n]{253,}$/u; +const URL_CANDIDATE_RE = /[A-Za-z][A-Za-z0-9+.-]*:\/\/[^\s"'<>]+/gu; +const UTF8_DECODER = new TextDecoder("utf-8", { fatal: true }); +const CREDENTIAL_SHAPED_NAME_PATTERN = + /(?:^|[_-])(?:api[_-]?key|access[_-]?key|secret[_-]?key|auth[_-]?token|refresh[_-]?token|access[_-]?token|client[_-]?secret|private[_-]?key|pass[_-]?code|personal[_-]?access[_-]?token|connection[_-]?string|webhook(?:[_-]?url)?|key|secret|token|password|passwd|passcode|auth|authorization|credential|credentials|bearer|bearer[_-]?token|cookie|cookies|pat|private|privatekey|pin|webhookurl|dsn|connectionstring)(?:$|[_-])/iu; +const CREDENTIAL_COMPOUND_NAME_PATTERN = + /^(?:access|refresh|client|bearer|auth|api|private|signing|session|bot|app|resolved)(?:token|key|secret|password)$/iu; +const CREDENTIAL_ENV_NAME_PATTERN = + /^(?:[A-Z0-9]+_)*(?:TOKEN|KEY|SECRET|PASSWORD|PASSWD|PASS|PASSPHRASE|CREDENTIAL)S?$/u; +const CREDENTIAL_HEADER_NAME_PATTERN = + /^(?:authorization|proxy-authorization|cookie|set-cookie|.+-(?:key|token|secret|password|passphrase|credential|auth)s?)$/iu; +const PUBLIC_KEY_NAME_PATTERN = /(?:^|[-_])public[-_]?keys?$/iu; +const PASS_CREDENTIAL_NAME_PATTERN = /(?:^|[-_])pass(?:wd)?$/iu; +const SECRET_VALUE_PATTERNS: readonly RegExp[] = [ + /nvapi-[A-Za-z0-9_-]{10,}/u, + /nvcf-[A-Za-z0-9_-]{10,}/u, + /ghp_[A-Za-z0-9_-]{10,}/u, + /github_pat_[A-Za-z0-9_]{30,}/u, + /sk-(?:proj-|ant-)?[A-Za-z0-9_-]{10,}/u, + /(?:xox[bpas]|xapp)-[A-Za-z0-9-]{10,}/u, + /A(?:K|S)IA[A-Z0-9]{16}/u, + /hf_[A-Za-z0-9]{10,}/u, + /glpat-[A-Za-z0-9_-]{10,}/u, + /gsk_[A-Za-z0-9]{10,}/u, + /pypi-[A-Za-z0-9_-]{10,}/u, + /tvly-[A-Za-z0-9_-]{10,}/u, + /lsv2_(?:pt|sk)_[A-Za-z0-9]{10,}(?:_[A-Za-z0-9]+)*/u, + /\bbot\d{8,10}:[A-Za-z0-9_-]{35}\b/u, + /\b\d{8,10}:[A-Za-z0-9_-]{35}\b/u, + /\b[A-Za-z0-9]{24}\.[A-Za-z0-9_-]{6}\.[A-Za-z0-9_-]{27,}\b/u, + /\beyJ[A-Za-z0-9_-]{5,}\.[A-Za-z0-9_-]{2,}\.[A-Za-z0-9_-]{10,}\b/u, + /\bBearer\s+[A-Za-z0-9_.+/=-]{10,}/iu, + /-----BEGIN (?:[A-Z0-9]+ )?PRIVATE KEY-----/u, +]; + +export const MANAGED_STARTUP_INFERENCE_APIS = [ + "openai-completions", + "openai-responses", + "anthropic-messages", +] as const; +export type ManagedStartupInferenceApi = (typeof MANAGED_STARTUP_INFERENCE_APIS)[number]; +export type ManagedStartupToolDisclosure = "progressive" | "direct"; +export const MANAGED_STARTUP_REASONING_EFFORTS = ["default", "low", "medium", "high"] as const; +export type ManagedStartupReasoningEffort = (typeof MANAGED_STARTUP_REASONING_EFFORTS)[number]; +export const MANAGED_STARTUP_DCODE_AUTO_APPROVAL_MODES = ["disabled", "thread-opt-in"] as const; +export type ManagedStartupDcodeAutoApprovalMode = + (typeof MANAGED_STARTUP_DCODE_AUTO_APPROVAL_MODES)[number]; +export const MANAGED_STARTUP_HERMES_TOOL_GATEWAYS = [ + "nous-web", + "nous-image", + "nous-audio", + "nous-browser", + "nous-code", +] as const; +export type ManagedStartupHermesToolGateway = (typeof MANAGED_STARTUP_HERMES_TOOL_GATEWAYS)[number]; +export type ManagedStartupInputModality = "text" | "image"; +export type ManagedStartupWebSearchProvider = "brave" | "tavily"; +export type ManagedStartupDeviceAuthOptOutSource = "operator" | "managed-onboard"; + +export const MANAGED_STARTUP_AGENTS = ["openclaw", "hermes", "langchain-deepagents-code"] as const; + +export type ManagedStartupAgent = (typeof MANAGED_STARTUP_AGENTS)[number]; + +export type ManagedStartupJsonScalar = string | number | boolean | null; +export type ManagedStartupJsonValue = + | ManagedStartupJsonScalar + | ManagedStartupJsonObject + | readonly ManagedStartupJsonValue[]; +export interface ManagedStartupJsonObject { + readonly [key: string]: ManagedStartupJsonValue; +} + +export interface ManagedStartupInference { + /** Stable route name installed in the sandbox-facing inference config. */ + readonly routeProvider: string; + /** User-selected provider upstream of the managed inference route. */ + readonly upstreamProvider: string; + readonly model: string; + /** Sandbox-facing managed inference route (normally inference.local). */ + readonly routedBaseUrl: string; + /** Direct upstream metadata needed by DCode; never used as the sandbox route. */ + readonly upstreamEndpointUrl: string | null; + readonly api: ManagedStartupInferenceApi; + /** OpenClaw's provider/model reference. Other adapters require null. */ + readonly primaryModelRef: string | null; + /** OpenClaw provider compatibility options. Other adapters require null. */ + readonly compatibility: ManagedStartupJsonObject | null; + /** OpenClaw model inputs. Other adapters require null. */ + readonly inputModalities: readonly ManagedStartupInputModality[] | null; +} + +export interface ManagedStartupProxy { + /** Managed OpenShell policy-proxy route consumed by every adapter. */ + readonly managedHost: string; + readonly managedPort: number; + /** + * Optional host-proxy intent is represented separately from the trusted + * managed route. DCode retains this intent for OpenShell's create boundary, + * while its long-running runtime still pins the root-owned managed route. + */ + readonly hostHttpUrl: string | null; + readonly hostHttpsUrl: string | null; + readonly hostNoProxy: readonly string[]; +} + +export interface ManagedStartupOpenClawDashboard { + readonly agent: "openclaw"; + readonly mode: "loopback" | "remote"; + readonly url: string; + readonly port: number; + readonly bindAddress: "127.0.0.1" | "0.0.0.0"; + readonly wslExposure: boolean; +} + +export interface ManagedStartupHermesDashboardDisabled { + readonly agent: "hermes"; + readonly mode: "disabled"; + /** CHAT_UI_URL remains a stock image input even when host forwarding is off. */ + readonly url: string; + readonly publicPort: null; + readonly internalPort: null; + readonly tuiEnabled: false; +} + +export interface ManagedStartupHermesDashboardForwarded { + readonly agent: "hermes"; + readonly mode: "loopback-forwarded"; + readonly url: string; + readonly publicPort: number; + readonly internalPort: number; + readonly tuiEnabled: boolean; +} + +export type ManagedStartupHermesDashboard = + | ManagedStartupHermesDashboardDisabled + | ManagedStartupHermesDashboardForwarded; + +export interface ManagedStartupDcodeDashboard { + readonly agent: "langchain-deepagents-code"; + readonly mode: "disabled"; +} + +export type ManagedStartupDashboard = + | ManagedStartupOpenClawDashboard + | ManagedStartupHermesDashboard + | ManagedStartupDcodeDashboard; + +export interface ManagedStartupWebSearch { + readonly enabled: boolean; + /** + * The selected provider is retained even when disabled because the stock + * Dockerfiles currently materialize both build inputs independently. + */ + readonly provider: ManagedStartupWebSearchProvider; +} + +export interface ManagedStartupTools { + readonly disclosure: ManagedStartupToolDisclosure; + /** Reviewed Hermes gateway preset IDs; required empty for other adapters. */ + readonly enabledGateways: readonly ManagedStartupHermesToolGateway[]; +} + +export interface ManagedStartupMessaging { + /** + * A host-prevalidated, secretless SandboxMessagingPlan. The existing + * messaging validator/applier owns its versioned nested schema; this portable + * module owns JSON shape, size, and secret scanning only. + */ + readonly plan: ManagedStartupJsonObject | null; +} + +export interface ManagedStartupTuning { + readonly contextWindow: number | null; + readonly maxTokens: number | null; + readonly reasoning: boolean | null; + readonly reasoningEffort: ManagedStartupReasoningEffort | null; +} + +/** + * Only the digest crosses the startup-profile boundary. The host-side CA + * applicator owns the actual certificate bytes and verifies this digest before + * making them available to the sandbox. + */ +export interface ManagedStartupCorporateCa { + readonly bundleSha256: string | null; +} + +export interface ManagedStartupExtraAgents { + /** + * Canonical form of NEMOCLAW_EXTRA_AGENTS_JSON. The existing OpenClaw + * validator owns the nested agent schema; this boundary preserves all + * prevalidated values without accepting the legacy array/object ambiguity. + */ + readonly agents: readonly ManagedStartupJsonObject[]; + readonly defaults: ManagedStartupJsonObject; + readonly main: ManagedStartupJsonObject; +} + +export interface ManagedStartupOpenClawOtel { + readonly enabled: boolean; + readonly endpointUrl: string; + readonly serviceName: string; + readonly sampleRate: number; +} + +export interface ManagedStartupDeviceAuth { + readonly disabled: boolean; + readonly optOutSource: ManagedStartupDeviceAuthOptOutSource; +} + +export interface ManagedStartupOpenClawConfig { + readonly agent: "openclaw"; + readonly webSearch: ManagedStartupWebSearch; + readonly otel: ManagedStartupOpenClawOtel; + readonly agentTimeoutSeconds: number; + readonly heartbeatEvery: string | null; + readonly extraAgents: ManagedStartupExtraAgents; + readonly deviceAuth: ManagedStartupDeviceAuth; + readonly minimalBootstrap: boolean; +} + +export interface ManagedStartupHermesConfig { + readonly agent: "hermes"; + readonly webSearch: ManagedStartupWebSearch; +} + +export interface ManagedStartupDcodeConfig { + readonly agent: "langchain-deepagents-code"; + readonly autoApprovalMode: ManagedStartupDcodeAutoApprovalMode; + readonly observabilityEnabled: boolean; +} + +export type ManagedStartupAgentConfig = + | ManagedStartupOpenClawConfig + | ManagedStartupHermesConfig + | ManagedStartupDcodeConfig; + +export interface ManagedStartupProfile { + readonly schemaVersion: typeof MANAGED_STARTUP_PROFILE_SCHEMA_VERSION; + readonly agent: ManagedStartupAgent; + readonly agentConfig: ManagedStartupAgentConfig; + readonly inference: ManagedStartupInference; + readonly proxy: ManagedStartupProxy; + readonly dashboard: ManagedStartupDashboard; + readonly tools: ManagedStartupTools; + readonly messaging: ManagedStartupMessaging; + readonly tuning: ManagedStartupTuning; + readonly corporateCa: ManagedStartupCorporateCa; +} + +export type ManagedStartupDashboardMode = "disabled" | "loopback" | "remote" | "loopback-forwarded"; + +export interface ManagedStartupAgentCapabilities { + readonly inferenceApis: readonly ManagedStartupInferenceApi[]; + readonly dashboardModes: readonly ManagedStartupDashboardMode[]; + readonly inputModalities: readonly ManagedStartupInputModality[]; + readonly webSearchProviders: readonly ManagedStartupWebSearchProvider[]; + readonly toolGateways: readonly ManagedStartupHermesToolGateway[]; + readonly tuningFields: readonly ( + | "contextWindow" + | "maxTokens" + | "reasoning" + | "reasoningEffort" + )[]; + readonly supportsMessaging: boolean; + readonly supportsInferenceCompatibility: boolean; + readonly supportsUpstreamEndpoint: boolean; + readonly supportsHostProxyIntent: boolean; + readonly supportsPrimaryModelRef: boolean; + readonly supportsAgentTimeout: boolean; + readonly supportsHeartbeat: boolean; + readonly supportsExtraAgents: boolean; + readonly supportsDeviceAuth: boolean; + readonly observability: "openclaw-otel" | "dcode-marker" | "none"; + readonly supportsMinimalBootstrap: boolean; +} + +/** + * Host-side negotiation must use this table before dispatch. A runtime that + * does not advertise the requested semantic capability is rejected instead of + * silently dropping a field. + */ +export const MANAGED_STARTUP_PROFILE_CAPABILITIES = { + openclaw: { + inferenceApis: MANAGED_STARTUP_INFERENCE_APIS, + dashboardModes: ["loopback", "remote"], + inputModalities: ["text", "image"], + webSearchProviders: ["brave", "tavily"], + toolGateways: [], + tuningFields: ["contextWindow", "maxTokens", "reasoning", "reasoningEffort"], + supportsMessaging: true, + supportsInferenceCompatibility: true, + supportsUpstreamEndpoint: false, + supportsHostProxyIntent: true, + supportsPrimaryModelRef: true, + supportsAgentTimeout: true, + supportsHeartbeat: true, + supportsExtraAgents: true, + supportsDeviceAuth: true, + observability: "openclaw-otel", + supportsMinimalBootstrap: true, + }, + hermes: { + inferenceApis: MANAGED_STARTUP_INFERENCE_APIS, + dashboardModes: ["disabled", "loopback-forwarded"], + inputModalities: [], + webSearchProviders: ["tavily"], + toolGateways: MANAGED_STARTUP_HERMES_TOOL_GATEWAYS, + tuningFields: ["contextWindow"], + supportsMessaging: true, + supportsInferenceCompatibility: false, + supportsUpstreamEndpoint: false, + supportsHostProxyIntent: true, + supportsPrimaryModelRef: false, + supportsAgentTimeout: false, + supportsHeartbeat: false, + supportsExtraAgents: false, + supportsDeviceAuth: false, + observability: "none", + supportsMinimalBootstrap: false, + }, + "langchain-deepagents-code": { + inferenceApis: ["openai-completions"], + dashboardModes: ["disabled"], + inputModalities: [], + webSearchProviders: [], + toolGateways: [], + tuningFields: [], + supportsMessaging: false, + supportsInferenceCompatibility: false, + supportsUpstreamEndpoint: true, + supportsHostProxyIntent: true, + supportsPrimaryModelRef: false, + supportsAgentTimeout: false, + supportsHeartbeat: false, + supportsExtraAgents: false, + supportsDeviceAuth: false, + observability: "dcode-marker", + supportsMinimalBootstrap: false, + }, +} as const satisfies Record; + +export type ManagedStartupAffordanceSource = "docker-arg" | "runtime-env" | "host-material"; +export type ManagedStartupAffordanceRepresentation = "value" | "derived" | "digest-handoff"; + +export interface ManagedStartupAffordance { + readonly input: string; + readonly profilePath: string; + readonly source: ManagedStartupAffordanceSource; + readonly representation: ManagedStartupAffordanceRepresentation; +} + +function affordance( + input: string, + profilePath: string, + source: ManagedStartupAffordanceSource = "docker-arg", + representation: ManagedStartupAffordanceRepresentation = "value", +): ManagedStartupAffordance { + return { input, profilePath, source, representation }; +} + +const HOST_PROXY_AFFORDANCES = [ + affordance("HTTP_PROXY", "proxy.hostHttpUrl", "runtime-env"), + affordance("http_proxy", "proxy.hostHttpUrl", "runtime-env", "derived"), + affordance("HTTPS_PROXY", "proxy.hostHttpsUrl", "runtime-env"), + affordance("https_proxy", "proxy.hostHttpsUrl", "runtime-env", "derived"), + affordance("NO_PROXY", "proxy.hostNoProxy", "runtime-env"), + affordance("no_proxy", "proxy.hostNoProxy", "runtime-env", "derived"), +] as const; + +/** + * Complete v1 mapping from the deployment-specific Docker/start inputs that + * managed-image startup replaces to typed profile fields. This is deliberately + * data rather than prose so Dockerfile drift tests can fail closed. + */ +export const MANAGED_STARTUP_PROFILE_AFFORDANCE_INVENTORY = { + openclaw: [ + affordance("NEMOCLAW_MODEL", "inference.model"), + affordance("NEMOCLAW_INFERENCE_PROVIDER_ID", "inference.routeProvider"), + affordance("NEMOCLAW_UPSTREAM_PROVIDER", "inference.upstreamProvider"), + affordance("NEMOCLAW_PRIMARY_MODEL_REF", "inference.primaryModelRef"), + affordance("NEMOCLAW_INFERENCE_BASE_URL", "inference.routedBaseUrl"), + affordance("NEMOCLAW_INFERENCE_API", "inference.api"), + affordance("NEMOCLAW_INFERENCE_COMPAT_B64", "inference.compatibility"), + affordance("NEMOCLAW_INFERENCE_INPUTS", "inference.inputModalities"), + affordance("NEMOCLAW_CONTEXT_WINDOW", "tuning.contextWindow"), + affordance("NEMOCLAW_MAX_TOKENS", "tuning.maxTokens"), + affordance("NEMOCLAW_REASONING", "tuning.reasoning"), + affordance("NEMOCLAW_REASONING_EFFORT", "tuning.reasoningEffort"), + affordance("NEMOCLAW_TOOL_DISCLOSURE", "tools.disclosure"), + affordance("NEMOCLAW_AGENT_TIMEOUT", "agentConfig.agentTimeoutSeconds"), + affordance("NEMOCLAW_AGENT_HEARTBEAT_EVERY", "agentConfig.heartbeatEvery"), + affordance("NEMOCLAW_EXTRA_AGENTS_JSON_B64", "agentConfig.extraAgents"), + affordance("NEMOCLAW_DISABLE_DEVICE_AUTH", "agentConfig.deviceAuth.disabled"), + affordance("NEMOCLAW_DEVICE_AUTH_OPT_OUT_SOURCE", "agentConfig.deviceAuth.optOutSource"), + affordance("NEMOCLAW_WEB_SEARCH_ENABLED", "agentConfig.webSearch.enabled"), + affordance("NEMOCLAW_WEB_SEARCH_PROVIDER", "agentConfig.webSearch.provider"), + affordance("NEMOCLAW_OPENCLAW_OTEL", "agentConfig.otel.enabled"), + affordance("NEMOCLAW_OPENCLAW_OTEL_ENDPOINT", "agentConfig.otel.endpointUrl"), + affordance("NEMOCLAW_OPENCLAW_OTEL_SERVICE_NAME", "agentConfig.otel.serviceName"), + affordance("NEMOCLAW_OPENCLAW_OTEL_SAMPLE_RATE", "agentConfig.otel.sampleRate"), + affordance("CHAT_UI_URL", "dashboard.url"), + affordance("NEMOCLAW_DASHBOARD_BIND", "dashboard.bindAddress"), + affordance("NEMOCLAW_WSL_DASHBOARD_EXPOSURE", "dashboard.wslExposure"), + affordance("NEMOCLAW_DASHBOARD_PORT", "dashboard.port", "runtime-env"), + affordance("NEMOCLAW_PROXY_HOST", "proxy.managedHost"), + affordance("NEMOCLAW_PROXY_PORT", "proxy.managedPort"), + affordance("NEMOCLAW_MESSAGING_PLAN_B64", "messaging.plan"), + affordance("NEMOCLAW_MINIMAL_BOOTSTRAP", "agentConfig.minimalBootstrap", "runtime-env"), + affordance( + "NEMOCLAW_CORPORATE_CA_B64", + "corporateCa.bundleSha256", + "host-material", + "digest-handoff", + ), + ...HOST_PROXY_AFFORDANCES, + ], + hermes: [ + affordance("NEMOCLAW_MODEL", "inference.model"), + affordance("NEMOCLAW_INFERENCE_PROVIDER_ID", "inference.routeProvider"), + affordance("NEMOCLAW_UPSTREAM_PROVIDER", "inference.upstreamProvider"), + affordance("NEMOCLAW_INFERENCE_BASE_URL", "inference.routedBaseUrl"), + affordance("NEMOCLAW_INFERENCE_API", "inference.api"), + affordance("NEMOCLAW_CONTEXT_WINDOW", "tuning.contextWindow"), + affordance("NEMOCLAW_TOOL_DISCLOSURE", "tools.disclosure"), + affordance( + "NEMOCLAW_HERMES_TOOL_GATEWAY_BROKER", + "tools.enabledGateways", + "docker-arg", + "derived", + ), + affordance("NEMOCLAW_HERMES_TOOL_GATEWAY_PRESETS_B64", "tools.enabledGateways"), + affordance("NEMOCLAW_WEB_SEARCH_ENABLED", "agentConfig.webSearch.enabled"), + affordance("NEMOCLAW_WEB_SEARCH_PROVIDER", "agentConfig.webSearch.provider"), + affordance("NEMOCLAW_MESSAGING_PLAN_B64", "messaging.plan"), + affordance("CHAT_UI_URL", "dashboard.url"), + affordance("NEMOCLAW_HERMES_DASHBOARD", "dashboard.mode", "runtime-env"), + affordance("NEMOCLAW_HERMES_DASHBOARD_PORT", "dashboard.publicPort", "runtime-env"), + affordance("NEMOCLAW_HERMES_DASHBOARD_INTERNAL_PORT", "dashboard.internalPort", "runtime-env"), + affordance("NEMOCLAW_HERMES_DASHBOARD_TUI", "dashboard.tuiEnabled", "runtime-env"), + affordance("NEMOCLAW_PROXY_HOST", "proxy.managedHost", "runtime-env"), + affordance("NEMOCLAW_PROXY_PORT", "proxy.managedPort", "runtime-env"), + affordance( + "NEMOCLAW_CORPORATE_CA_B64", + "corporateCa.bundleSha256", + "host-material", + "digest-handoff", + ), + ...HOST_PROXY_AFFORDANCES, + ], + "langchain-deepagents-code": [ + affordance("NEMOCLAW_MODEL", "inference.model"), + affordance("NEMOCLAW_INFERENCE_PROVIDER_ID", "inference.routeProvider"), + affordance("NEMOCLAW_UPSTREAM_PROVIDER", "inference.upstreamProvider"), + affordance("NEMOCLAW_UPSTREAM_ENDPOINT_URL", "inference.upstreamEndpointUrl"), + affordance("NEMOCLAW_INFERENCE_BASE_URL", "inference.routedBaseUrl"), + affordance("NEMOCLAW_INFERENCE_API", "inference.api"), + affordance("NEMOCLAW_TOOL_DISCLOSURE", "tools.disclosure"), + affordance("NEMOCLAW_DCODE_AUTO_APPROVAL", "agentConfig.autoApprovalMode"), + affordance("NEMOCLAW_PROXY_HOST", "proxy.managedHost"), + affordance("NEMOCLAW_PROXY_PORT", "proxy.managedPort"), + affordance("NEMOCLAW_OBSERVABILITY", "agentConfig.observabilityEnabled", "runtime-env"), + affordance( + "NEMOCLAW_CORPORATE_CA_B64", + "corporateCa.bundleSha256", + "host-material", + "digest-handoff", + ), + ...HOST_PROXY_AFFORDANCES, + ], +} as const satisfies Record; + +export interface ManagedStartupExcludedDockerInput { + readonly input: string; + readonly reason: + | "release-composition" + | "integrity-pin" + | "build-provenance" + | "platform-build" + | "fixed-image-contract"; +} + +/** + * Docker inputs intentionally outside a startup profile. None are resolved + * deployment behavior: they select/pin release artifacts, invalidate build + * caches, or perform a platform-specific image ownership rewrite. + */ +export const MANAGED_STARTUP_PROFILE_EXCLUDED_DOCKER_INPUTS = { + openclaw: [ + { input: "BASE_IMAGE", reason: "release-composition" }, + { input: "OPENCLAW_VERSION", reason: "release-composition" }, + { input: "OPENCLAW_2026_7_1_INTEGRITY", reason: "integrity-pin" }, + { input: "OPENCLAW_2026_7_1_TARBALL", reason: "release-composition" }, + { input: "OPENCLAW_DIAGNOSTICS_OTEL_2026_7_1_INTEGRITY", reason: "integrity-pin" }, + { input: "OPENCLAW_BRAVE_PLUGIN_2026_7_1_INTEGRITY", reason: "integrity-pin" }, + { input: "NEMOCLAW_E2E_FIXTURE_LEGACY_OPENCLAW", reason: "release-composition" }, + { input: "NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION", reason: "release-composition" }, + { input: "OPENCLAW_2026_3_11_INTEGRITY", reason: "integrity-pin" }, + { input: "OPENCLAW_2026_3_11_TARBALL", reason: "release-composition" }, + { input: "OPENCLAW_2026_4_24_INTEGRITY", reason: "integrity-pin" }, + { input: "OPENCLAW_2026_4_24_TARBALL", reason: "release-composition" }, + { input: "CODEX_ACP_0_11_1_INTEGRITY", reason: "integrity-pin" }, + { input: "MCPORTER_VERSION", reason: "release-composition" }, + { input: "MCPORTER_0_7_3_INTEGRITY", reason: "integrity-pin" }, + { input: "MCPORTER_0_7_3_TARBALL", reason: "release-composition" }, + { input: "NEMOCLAW_BUILD_ID", reason: "build-provenance" }, + { input: "NEMOCLAW_DARWIN_VM_COMPAT", reason: "platform-build" }, + { input: "NEMOCLAW_MANAGED_IMAGE_RUNTIME_USER", reason: "fixed-image-contract" }, + ], + hermes: [ + { input: "BASE_IMAGE", reason: "release-composition" }, + { input: "SSL_CERT_FILE", reason: "fixed-image-contract" }, + { input: "NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION", reason: "release-composition" }, + { input: "NEMOCLAW_HERMES_PROFILE_POLICY_PATCHER_SHA256", reason: "integrity-pin" }, + { input: "NEMOCLAW_HERMES_GATEWAY_RUNTIME_METADATA_PATCHER_SHA256", reason: "integrity-pin" }, + { input: "NEMOCLAW_HERMES_DISCORD_RECOVERY_PATCHER_SHA256", reason: "integrity-pin" }, + { input: "NEMOCLAW_HERMES_LANGFUSE_PATCHER_SHA256", reason: "integrity-pin" }, + { input: "NEMOCLAW_HERMES_WRAPPER_SHA256", reason: "integrity-pin" }, + { input: "NEMOCLAW_HERMES_VALIDATOR_SHA256", reason: "integrity-pin" }, + { input: "NEMOCLAW_HERMES_TIRITH_FINALIZER_SHA256", reason: "integrity-pin" }, + { input: "NEMOCLAW_BUILD_ID", reason: "build-provenance" }, + { input: "NEMOCLAW_DARWIN_VM_COMPAT", reason: "platform-build" }, + { input: "NEMOCLAW_MANAGED_IMAGE_RUNTIME_USER", reason: "fixed-image-contract" }, + ], + "langchain-deepagents-code": [ + { input: "BASE_IMAGE", reason: "release-composition" }, + { input: "NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION", reason: "release-composition" }, + { input: "NEMOCLAW_BUILD_ID", reason: "build-provenance" }, + { input: "NEMOCLAW_DARWIN_VM_COMPAT", reason: "platform-build" }, + { input: "NEMOCLAW_MANAGED_IMAGE_RUNTIME_USER", reason: "fixed-image-contract" }, + ], +} as const satisfies Record; + +export class ManagedStartupProfileError extends Error { + constructor(message: string) { + super(`Invalid managed startup profile: ${message}`); + this.name = "ManagedStartupProfileError"; + } +} + +const PROFILE_KEYS = new Set([ + "schemaVersion", + "agent", + "agentConfig", + "inference", + "proxy", + "dashboard", + "tools", + "messaging", + "tuning", + "corporateCa", +]); +const INFERENCE_KEYS = new Set([ + "routeProvider", + "upstreamProvider", + "model", + "routedBaseUrl", + "upstreamEndpointUrl", + "api", + "primaryModelRef", + "compatibility", + "inputModalities", +]); +const PROXY_KEYS = new Set([ + "managedHost", + "managedPort", + "hostHttpUrl", + "hostHttpsUrl", + "hostNoProxy", +]); +const OPENCLAW_DASHBOARD_KEYS = new Set([ + "agent", + "mode", + "url", + "port", + "bindAddress", + "wslExposure", +]); +const HERMES_DASHBOARD_KEYS = new Set([ + "agent", + "mode", + "url", + "publicPort", + "internalPort", + "tuiEnabled", +]); +const DCODE_DASHBOARD_KEYS = new Set(["agent", "mode"]); +const TOOLS_KEYS = new Set(["disclosure", "enabledGateways"]); +const MESSAGING_KEYS = new Set(["plan"]); +const TUNING_KEYS = new Set(["contextWindow", "maxTokens", "reasoning", "reasoningEffort"]); +const CORPORATE_CA_KEYS = new Set(["bundleSha256"]); +const OPENCLAW_CONFIG_KEYS = new Set([ + "agent", + "webSearch", + "otel", + "agentTimeoutSeconds", + "heartbeatEvery", + "extraAgents", + "deviceAuth", + "minimalBootstrap", +]); +const HERMES_CONFIG_KEYS = new Set(["agent", "webSearch"]); +const DCODE_CONFIG_KEYS = new Set(["agent", "autoApprovalMode", "observabilityEnabled"]); +const WEB_SEARCH_KEYS = new Set(["enabled", "provider"]); +const OTEL_KEYS = new Set(["enabled", "endpointUrl", "serviceName", "sampleRate"]); +const DEVICE_AUTH_KEYS = new Set(["disabled", "optOutSource"]); +const EXTRA_AGENTS_KEYS = new Set(["agents", "defaults", "main"]); +const MANAGED_STARTUP_AGENT_SET = new Set(MANAGED_STARTUP_AGENTS); +const DCODE_AUTO_APPROVAL_MODE_SET = new Set(MANAGED_STARTUP_DCODE_AUTO_APPROVAL_MODES); +const INFERENCE_API_SET = new Set(MANAGED_STARTUP_INFERENCE_APIS); +const REASONING_EFFORT_SET = new Set(MANAGED_STARTUP_REASONING_EFFORTS); +const HERMES_GATEWAY_SET = new Set(MANAGED_STARTUP_HERMES_TOOL_GATEWAYS); + +function isPlainObject(value: unknown): value is Record { + if (typeof value !== "object" || value === null || Array.isArray(value)) return false; + const prototype = Object.getPrototypeOf(value); + return prototype === Object.prototype || prototype === null; +} + +function isCredentialShapedName(name: string): boolean { + if (PUBLIC_KEY_NAME_PATTERN.test(name)) return false; + return ( + CREDENTIAL_SHAPED_NAME_PATTERN.test(name) || + CREDENTIAL_COMPOUND_NAME_PATTERN.test(name) || + CREDENTIAL_ENV_NAME_PATTERN.test(name) || + CREDENTIAL_HEADER_NAME_PATTERN.test(name) || + PASS_CREDENTIAL_NAME_PATTERN.test(name) + ); +} + +function valueLooksLikeSecret(value: string): boolean { + return SECRET_VALUE_PATTERNS.some((pattern) => pattern.test(value)); +} + +function containsUrlWithUserinfo(value: string): boolean { + for (const candidate of value.match(URL_CANDIDATE_RE) ?? []) { + try { + const url = new URL(candidate); + if (url.username || url.password) return true; + } catch { + // A field-level URL validator owns malformed strings where a URL is expected. + } + } + return false; +} + +function invalid(reason: string): never { + throw new ManagedStartupProfileError(reason); +} + +function requireRecord(value: unknown, where: string): Record { + if (!isPlainObject(value)) invalid(`${where} must be an object`); + return value; +} + +function rejectUnknownKeys( + value: Record, + allowed: ReadonlySet, + where: string, +): void { + if (Object.keys(value).some((key) => !allowed.has(key))) { + invalid(`${where} contains unsupported fields`); + } +} + +function requireBoolean(value: unknown, where: string): boolean { + if (typeof value !== "boolean") invalid(`${where} must be a boolean`); + return value; +} + +function requireNullableBoolean(value: unknown, where: string): boolean | null { + if (value === null) return null; + return requireBoolean(value, where); +} + +function requireBoundedString( + value: unknown, + where: string, + maxBytes = MAX_IDENTIFIER_BYTES, +): string { + if ( + typeof value !== "string" || + value.length === 0 || + value !== value.trim() || + Buffer.byteLength(value, "utf8") > maxBytes || + CONTROL_CHARACTER_RE.test(value) + ) { + invalid(`${where} must be a bounded, non-empty string without control characters`); + } + return value; +} + +function requireStringEnum( + value: unknown, + allowed: ReadonlySet, + where: string, +): T { + const normalized = requireBoundedString(value, where); + if (!allowed.has(normalized)) invalid(`${where} is not supported`); + return normalized as T; +} + +function requireNullablePositiveInteger(value: unknown, where: string): number | null { + if (value === null) return null; + if ( + typeof value !== "number" || + !Number.isSafeInteger(value) || + value < 1 || + value > MAX_TUNING_INTEGER + ) { + invalid(`${where} must be null or a bounded positive integer`); + } + return value; +} + +function requirePositiveInteger( + value: unknown, + where: string, + maximum = MAX_TUNING_INTEGER, +): number { + if (typeof value !== "number" || !Number.isSafeInteger(value) || value < 1 || value > maximum) { + invalid(`${where} must be a bounded positive integer`); + } + return value; +} + +function requirePort(value: unknown, where: string, minimum = 1): number { + if (typeof value !== "number" || !Number.isInteger(value) || value < 1 || value > 65_535) { + invalid(`${where} must be a valid TCP port`); + } + if (value < minimum) invalid(`${where} must be at least ${String(minimum)}`); + return value; +} + +function requireStringList(value: unknown, where: string): readonly string[] { + if (!Array.isArray(value) || value.length > MAX_LIST_ITEMS) { + invalid(`${where} must be a bounded string list`); + } + const items = value.map((item) => requireBoundedString(item, `${where} item`)); + if (new Set(items).size !== items.length) invalid(`${where} must not contain duplicates`); + return items.sort((left, right) => (left < right ? -1 : left > right ? 1 : 0)); +} + +function requireEnumList( + value: unknown, + allowed: ReadonlySet, + where: string, + options: { readonly allowEmpty: boolean }, +): readonly T[] { + const items = requireStringList(value, where); + if (!options.allowEmpty && items.length === 0) invalid(`${where} must not be empty`); + for (const item of items) { + if (!allowed.has(item)) invalid(`${where} contains an unsupported value`); + } + return items as readonly T[]; +} + +function cloneJsonValue(value: unknown, where: string): ManagedStartupJsonValue { + const clone = (current: unknown, depth: number): ManagedStartupJsonValue => { + if (depth > MAX_JSON_DEPTH) invalid(`${where} exceeds the JSON depth limit`); + if (current === null || typeof current === "string" || typeof current === "boolean") { + return current; + } + if (typeof current === "number") { + if (!Number.isFinite(current)) invalid(`${where} contains a non-finite number`); + return current; + } + if (Array.isArray(current)) { + return current.map((item) => clone(item, depth + 1)); + } + if (!isPlainObject(current)) invalid(`${where} contains a non-JSON value`); + return Object.fromEntries( + Object.entries(current).map(([key, child]) => { + if ( + key.length === 0 || + Buffer.byteLength(key, "utf8") > MAX_IDENTIFIER_BYTES || + CONTROL_CHARACTER_RE.test(key) + ) { + invalid(`${where} contains an invalid object key`); + } + return [key, clone(child, depth + 1)]; + }), + ); + }; + return clone(value, 0); +} + +function requireJsonObjectOrNull(value: unknown, where: string): ManagedStartupJsonObject | null { + if (value === null) return null; + if (!isPlainObject(value)) invalid(`${where} must be null or a plain JSON object`); + return cloneJsonValue(value, where) as ManagedStartupJsonObject; +} + +function requireJsonObject(value: unknown, where: string): ManagedStartupJsonObject { + const object = requireJsonObjectOrNull(value, where); + if (object === null) invalid(`${where} must be a plain JSON object`); + return object; +} + +function requireHttpUrl(value: unknown, where: string): string { + const raw = requireBoundedString(value, where, MAX_URL_BYTES); + let parsed: URL; + try { + parsed = new URL(raw); + } catch { + invalid(`${where} must be a valid HTTP(S) URL`); + } + if ( + (parsed.protocol !== "http:" && parsed.protocol !== "https:") || + parsed.username || + parsed.password || + parsed.search || + parsed.hash + ) { + invalid(`${where} must be a credential-free HTTP(S) URL without query or fragment data`); + } + const pathname = parsed.pathname.replace(/\/+$/u, ""); + return pathname === "" ? parsed.origin : `${parsed.origin}${pathname}`; +} + +function requireProxyUrl( + value: unknown, + allowedSchemes: ReadonlySet, + where: string, +): string | null { + if (value === null) return null; + const raw = requireBoundedString(value, where, MAX_URL_BYTES); + let parsed: URL; + try { + parsed = new URL(raw); + } catch { + invalid(`${where} must be a valid HTTP(S) proxy URL`); + } + if ( + !allowedSchemes.has(parsed.protocol) || + parsed.username || + parsed.password || + parsed.pathname !== "/" || + parsed.search || + parsed.hash + ) { + invalid(`${where} must be a credential-free HTTP(S) proxy origin`); + } + return parsed.origin; +} + +function requireManagedProxyHost(value: unknown, where: string): string { + const host = requireBoundedString(value, where); + if (!/^[A-Za-z0-9._-]+$/u.test(host)) { + invalid(`${where} must be a hostname or IPv4 address without a scheme or separators`); + } + return host; +} + +function isLoopbackUrl(value: string): boolean { + const hostname = new URL(value).hostname.toLowerCase(); + return ( + hostname === "localhost" || + hostname === "127.0.0.1" || + hostname === "::1" || + hostname === "[::1]" + ); +} + +function configuredDashboardPort(value: string): number { + const explicit = new URL(value).port; + return explicit === "" ? 18_789 : Number(explicit); +} + +function requireSampleRate(value: unknown, where: string): number { + if (typeof value !== "number" || !Number.isFinite(value) || value < 0 || value > 1) { + invalid(`${where} must be a number between 0 and 1`); + } + return value; +} + +function assertNoSecretMaterial(root: unknown): void { + const pending: Array<{ + value: unknown; + depth: number; + path: readonly string[]; + allowCredentialFieldNames: boolean; + }> = [{ value: root, depth: 0, path: [], allowCredentialFieldNames: false }]; + let visited = 0; + while (pending.length > 0) { + const current = pending.pop(); + if (!current) break; + visited += 1; + if (visited > MAX_JSON_NODES || current.depth > MAX_JSON_DEPTH) { + invalid("payload structure exceeds the complexity limit"); + } + + if (typeof current.value === "string") { + if (valueLooksLikeSecret(current.value)) { + invalid("payload contains credential-shaped string data"); + } + if (RAW_CA_PEM_RE.test(current.value) || RAW_CA_DER_BASE64_RE.test(current.value)) { + invalid("payload contains raw certificate data; provide only the CA SHA-256 digest"); + } + if (containsUrlWithUserinfo(current.value)) { + invalid("payload contains a URL with embedded credentials"); + } + continue; + } + if (Array.isArray(current.value)) { + for (const item of current.value) { + pending.push({ + value: item, + depth: current.depth + 1, + path: current.path, + allowCredentialFieldNames: current.allowCredentialFieldNames, + }); + } + continue; + } + if (current.value !== null && typeof current.value === "object") { + if (!isPlainObject(current.value)) invalid("payload must contain only plain JSON objects"); + for (const [key, child] of Object.entries(current.value)) { + if (!current.allowCredentialFieldNames && isCredentialShapedName(key)) { + invalid("payload contains a credential-shaped field name"); + } + pending.push({ + value: child, + depth: current.depth + 1, + path: [...current.path, key], + // A messaging plan legitimately contains schema-owned names such as + // credentialBindings/providerEnvKey. Its dedicated validator proves + // those bindings are placeholders rather than secrets; this module + // still scans every nested value for raw credential material. + allowCredentialFieldNames: + current.allowCredentialFieldNames || + (current.path.length === 1 && current.path[0] === "messaging" && key === "plan"), + }); + } + } + } +} + +function assertPayloadWithinByteLimit(value: unknown): void { + let serialized: string | undefined; + try { + serialized = JSON.stringify(value); + } catch { + invalid("payload is not serializable JSON"); + } + if ( + serialized === undefined || + Buffer.byteLength(serialized, "utf8") > MANAGED_STARTUP_PROFILE_MAX_BYTES + ) { + invalid(`payload exceeds ${String(MANAGED_STARTUP_PROFILE_MAX_BYTES)} bytes`); + } +} + +function validateWebSearch(value: unknown, agent: "openclaw" | "hermes"): ManagedStartupWebSearch { + const webSearch = requireRecord(value, "agentConfig.webSearch"); + rejectUnknownKeys(webSearch, WEB_SEARCH_KEYS, "agentConfig.webSearch"); + const provider = requireStringEnum( + webSearch.provider, + new Set(agent === "openclaw" ? ["brave", "tavily"] : ["tavily"]), + "agentConfig.webSearch.provider", + ); + return { + enabled: requireBoolean(webSearch.enabled, "agentConfig.webSearch.enabled"), + provider, + }; +} + +function validateOpenClawOtel(value: unknown): ManagedStartupOpenClawOtel { + const otel = requireRecord(value, "agentConfig.otel"); + rejectUnknownKeys(otel, OTEL_KEYS, "agentConfig.otel"); + return { + enabled: requireBoolean(otel.enabled, "agentConfig.otel.enabled"), + endpointUrl: requireHttpUrl(otel.endpointUrl, "agentConfig.otel.endpointUrl"), + serviceName: requireBoundedString( + otel.serviceName, + "agentConfig.otel.serviceName", + MAX_IDENTIFIER_BYTES, + ), + sampleRate: requireSampleRate(otel.sampleRate, "agentConfig.otel.sampleRate"), + }; +} + +function validateExtraAgents(value: unknown): ManagedStartupExtraAgents { + const extraAgents = requireRecord(value, "agentConfig.extraAgents"); + rejectUnknownKeys(extraAgents, EXTRA_AGENTS_KEYS, "agentConfig.extraAgents"); + if (!Array.isArray(extraAgents.agents) || extraAgents.agents.length > MAX_LIST_ITEMS) { + invalid("agentConfig.extraAgents.agents must be a bounded JSON object list"); + } + return { + agents: extraAgents.agents.map((agent, index) => + requireJsonObject(agent, `agentConfig.extraAgents.agents[${String(index)}]`), + ), + defaults: requireJsonObject(extraAgents.defaults, "agentConfig.extraAgents.defaults"), + main: requireJsonObject(extraAgents.main, "agentConfig.extraAgents.main"), + }; +} + +function validateDeviceAuth(value: unknown): ManagedStartupDeviceAuth { + const deviceAuth = requireRecord(value, "agentConfig.deviceAuth"); + rejectUnknownKeys(deviceAuth, DEVICE_AUTH_KEYS, "agentConfig.deviceAuth"); + return { + disabled: requireBoolean(deviceAuth.disabled, "agentConfig.deviceAuth.disabled"), + optOutSource: requireStringEnum( + deviceAuth.optOutSource, + new Set(["operator", "managed-onboard"]), + "agentConfig.deviceAuth.optOutSource", + ), + }; +} + +function validateAgentConfig( + value: unknown, + expectedAgent: ManagedStartupAgent, +): ManagedStartupAgentConfig { + const config = requireRecord(value, "agentConfig"); + const agent = requireStringEnum( + config.agent, + MANAGED_STARTUP_AGENT_SET, + "agentConfig.agent", + ); + if (agent !== expectedAgent) invalid("agentConfig.agent must match agent"); + + if (agent === "openclaw") { + rejectUnknownKeys(config, OPENCLAW_CONFIG_KEYS, "agentConfig"); + const heartbeatEvery = + config.heartbeatEvery === null + ? null + : requireBoundedString( + config.heartbeatEvery, + "agentConfig.heartbeatEvery", + MAX_IDENTIFIER_BYTES, + ); + if (heartbeatEvery !== null && !/^\d+(?:s|m|h)$/u.test(heartbeatEvery)) { + invalid("agentConfig.heartbeatEvery must be null or a duration ending in s, m, or h"); + } + return { + agent, + webSearch: validateWebSearch(config.webSearch, agent), + otel: validateOpenClawOtel(config.otel), + agentTimeoutSeconds: requirePositiveInteger( + config.agentTimeoutSeconds, + "agentConfig.agentTimeoutSeconds", + ), + heartbeatEvery, + extraAgents: validateExtraAgents(config.extraAgents), + deviceAuth: validateDeviceAuth(config.deviceAuth), + minimalBootstrap: requireBoolean(config.minimalBootstrap, "agentConfig.minimalBootstrap"), + }; + } + if (agent === "hermes") { + rejectUnknownKeys(config, HERMES_CONFIG_KEYS, "agentConfig"); + return { agent, webSearch: validateWebSearch(config.webSearch, agent) }; + } + + rejectUnknownKeys(config, DCODE_CONFIG_KEYS, "agentConfig"); + return { + agent, + autoApprovalMode: requireStringEnum( + config.autoApprovalMode, + DCODE_AUTO_APPROVAL_MODE_SET, + "agentConfig.autoApprovalMode", + ), + observabilityEnabled: requireBoolean( + config.observabilityEnabled, + "agentConfig.observabilityEnabled", + ), + }; +} + +function validateDashboard( + value: unknown, + expectedAgent: ManagedStartupAgent, +): ManagedStartupDashboard { + const dashboard = requireRecord(value, "dashboard"); + const agent = requireStringEnum( + dashboard.agent, + MANAGED_STARTUP_AGENT_SET, + "dashboard.agent", + ); + if (agent !== expectedAgent) invalid("dashboard.agent must match agent"); + + if (agent === "openclaw") { + rejectUnknownKeys(dashboard, OPENCLAW_DASHBOARD_KEYS, "dashboard"); + const mode = requireStringEnum<"loopback" | "remote">( + dashboard.mode, + new Set(["loopback", "remote"]), + "dashboard.mode", + ); + const url = requireHttpUrl(dashboard.url, "dashboard.url"); + const bindAddress = requireStringEnum<"127.0.0.1" | "0.0.0.0">( + dashboard.bindAddress, + new Set(["127.0.0.1", "0.0.0.0"]), + "dashboard.bindAddress", + ); + const wslExposure = requireBoolean(dashboard.wslExposure, "dashboard.wslExposure"); + const hasRemoteExposure = !isLoopbackUrl(url) || bindAddress === "0.0.0.0" || wslExposure; + if ((mode === "remote") !== hasRemoteExposure) { + invalid("OpenClaw dashboard.mode must reflect its URL, bind address, and WSL exposure"); + } + const port = requirePort(dashboard.port, "dashboard.port", 1024); + if (port === 8642) + invalid("OpenClaw dashboard.port must not use reserved Hermes API port 8642"); + if (configuredDashboardPort(url) !== port) { + invalid("OpenClaw dashboard.port must match dashboard.url"); + } + return { + agent, + mode, + url, + port, + bindAddress, + wslExposure, + }; + } + if (agent === "hermes") { + rejectUnknownKeys(dashboard, HERMES_DASHBOARD_KEYS, "dashboard"); + const mode = requireStringEnum<"disabled" | "loopback-forwarded">( + dashboard.mode, + new Set(["disabled", "loopback-forwarded"]), + "dashboard.mode", + ); + const url = requireHttpUrl(dashboard.url, "dashboard.url"); + if (!isLoopbackUrl(url)) { + invalid("Hermes dashboard.url must remain loopback; OpenShell owns the host forward"); + } + if (mode === "disabled") { + if ( + dashboard.publicPort !== null || + dashboard.internalPort !== null || + dashboard.tuiEnabled !== false + ) { + invalid("disabled Hermes dashboard must not configure ports or TUI"); + } + return { + agent, + mode, + url, + publicPort: null, + internalPort: null, + tuiEnabled: false, + }; + } + const publicPort = requirePort(dashboard.publicPort, "dashboard.publicPort", 1024); + const internalPort = requirePort(dashboard.internalPort, "dashboard.internalPort", 1024); + if (publicPort === internalPort) { + invalid("Hermes dashboard publicPort and internalPort must differ"); + } + if (publicPort === 8642) { + invalid("Hermes dashboard publicPort must not use reserved API port 8642"); + } + if (configuredDashboardPort(url) !== publicPort) { + invalid("Hermes dashboard.publicPort must match dashboard.url"); + } + return { + agent, + mode, + url, + publicPort, + internalPort, + tuiEnabled: requireBoolean(dashboard.tuiEnabled, "dashboard.tuiEnabled"), + }; + } + + rejectUnknownKeys(dashboard, DCODE_DASHBOARD_KEYS, "dashboard"); + if (dashboard.mode !== "disabled") { + invalid("langchain-deepagents-code dashboard.mode must be disabled"); + } + return { agent, mode: "disabled" }; +} + +function validateInference(value: unknown, agent: ManagedStartupAgent): ManagedStartupInference { + const inference = requireRecord(value, "inference"); + rejectUnknownKeys(inference, INFERENCE_KEYS, "inference"); + const api = requireStringEnum( + inference.api, + INFERENCE_API_SET, + "inference.api", + ); + const supportedInferenceApis: readonly string[] = + MANAGED_STARTUP_PROFILE_CAPABILITIES[agent].inferenceApis; + if (!supportedInferenceApis.includes(api)) { + invalid(`inference.api is not supported by ${agent}`); + } + const upstreamEndpointUrl = + inference.upstreamEndpointUrl === null + ? null + : requireHttpUrl(inference.upstreamEndpointUrl, "inference.upstreamEndpointUrl"); + const primaryModelRef = + inference.primaryModelRef === null + ? null + : requireBoundedString( + inference.primaryModelRef, + "inference.primaryModelRef", + MAX_MODEL_BYTES, + ); + const compatibility = requireJsonObjectOrNull(inference.compatibility, "inference.compatibility"); + const inputModalities = + inference.inputModalities === null + ? null + : requireEnumList( + inference.inputModalities, + new Set(["text", "image"]), + "inference.inputModalities", + { allowEmpty: false }, + ); + + if (agent === "openclaw") { + if (upstreamEndpointUrl !== null) { + invalid("inference.upstreamEndpointUrl must be null for openclaw"); + } + if (primaryModelRef === null || inputModalities === null) { + invalid("openclaw requires primaryModelRef and inputModalities"); + } + } else { + if (primaryModelRef !== null || compatibility !== null || inputModalities !== null) { + invalid(`${agent} does not support primaryModelRef, compatibility, or inputModalities`); + } + if (agent === "hermes" && upstreamEndpointUrl !== null) { + invalid("inference.upstreamEndpointUrl must be null for hermes"); + } + } + + return { + routeProvider: requireBoundedString(inference.routeProvider, "inference.routeProvider"), + upstreamProvider: requireBoundedString( + inference.upstreamProvider, + "inference.upstreamProvider", + ), + model: requireBoundedString(inference.model, "inference.model", MAX_MODEL_BYTES), + routedBaseUrl: requireHttpUrl(inference.routedBaseUrl, "inference.routedBaseUrl"), + upstreamEndpointUrl, + api, + primaryModelRef, + compatibility, + inputModalities, + }; +} + +function validateProxy(value: unknown, agent: ManagedStartupAgent): ManagedStartupProxy { + const proxy = requireRecord(value, "proxy"); + rejectUnknownKeys(proxy, PROXY_KEYS, "proxy"); + const hostHttpUrl = requireProxyUrl(proxy.hostHttpUrl, new Set(["http:"]), "proxy.hostHttpUrl"); + // HTTPS_PROXY conventionally names an HTTP CONNECT proxy, so either scheme + // is valid while credentials and non-origin paths remain forbidden. + const hostHttpsUrl = requireProxyUrl( + proxy.hostHttpsUrl, + new Set(["http:", "https:"]), + "proxy.hostHttpsUrl", + ); + const hostNoProxy = requireStringList(proxy.hostNoProxy, "proxy.hostNoProxy"); + if ( + !MANAGED_STARTUP_PROFILE_CAPABILITIES[agent].supportsHostProxyIntent && + (hostHttpUrl !== null || hostHttpsUrl !== null || hostNoProxy.length > 0) + ) { + invalid(`${agent} rejects host proxy intent and uses only its trusted managed route`); + } + return { + managedHost: requireManagedProxyHost(proxy.managedHost, "proxy.managedHost"), + managedPort: requirePort(proxy.managedPort, "proxy.managedPort"), + hostHttpUrl, + hostHttpsUrl, + hostNoProxy, + }; +} + +function validateTools(value: unknown, agent: ManagedStartupAgent): ManagedStartupTools { + const tools = requireRecord(value, "tools"); + rejectUnknownKeys(tools, TOOLS_KEYS, "tools"); + const enabledGateways = requireEnumList( + tools.enabledGateways, + HERMES_GATEWAY_SET, + "tools.enabledGateways", + { allowEmpty: true }, + ); + if (agent !== "hermes" && enabledGateways.length > 0) { + invalid("tools.enabledGateways is supported only by hermes"); + } + return { + disclosure: requireStringEnum( + tools.disclosure, + new Set(["progressive", "direct"]), + "tools.disclosure", + ), + enabledGateways, + }; +} + +function validateTuning(value: unknown, agent: ManagedStartupAgent): ManagedStartupTuning { + const tuning = requireRecord(value, "tuning"); + rejectUnknownKeys(tuning, TUNING_KEYS, "tuning"); + const result: ManagedStartupTuning = { + contextWindow: requireNullablePositiveInteger(tuning.contextWindow, "tuning.contextWindow"), + maxTokens: requireNullablePositiveInteger(tuning.maxTokens, "tuning.maxTokens"), + reasoning: requireNullableBoolean(tuning.reasoning, "tuning.reasoning"), + reasoningEffort: + tuning.reasoningEffort === null + ? null + : requireStringEnum( + tuning.reasoningEffort, + REASONING_EFFORT_SET, + "tuning.reasoningEffort", + ), + }; + if (agent === "openclaw") { + if ( + result.contextWindow === null || + result.maxTokens === null || + result.reasoning === null || + result.reasoningEffort === null + ) { + invalid("openclaw requires contextWindow, maxTokens, reasoning, and reasoningEffort tuning"); + } + } else if (agent === "hermes") { + if (result.maxTokens !== null || result.reasoning !== null || result.reasoningEffort !== null) { + invalid("hermes supports only contextWindow tuning"); + } + } else if ( + result.contextWindow !== null || + result.maxTokens !== null || + result.reasoning !== null || + result.reasoningEffort !== null + ) { + invalid("langchain-deepagents-code does not support startup tuning fields"); + } + return result; +} + +/** + * Validate unknown input and return a canonical, deeply rebuilt profile. + * Unknown keys are rejected at every object boundary, and unordered set-like + * lists are sorted so all producers fingerprint the same resolved intent. + */ +export function validateManagedStartupProfile(value: unknown): ManagedStartupProfile { + assertPayloadWithinByteLimit(value); + assertNoSecretMaterial(value); + const profile = requireRecord(value, "profile"); + rejectUnknownKeys(profile, PROFILE_KEYS, "profile"); + if (profile.schemaVersion !== MANAGED_STARTUP_PROFILE_SCHEMA_VERSION) { + invalid(`schemaVersion must be ${String(MANAGED_STARTUP_PROFILE_SCHEMA_VERSION)}`); + } + const agent = requireStringEnum( + profile.agent, + MANAGED_STARTUP_AGENT_SET, + "agent", + ); + + const messaging = requireRecord(profile.messaging, "messaging"); + rejectUnknownKeys(messaging, MESSAGING_KEYS, "messaging"); + const messagingPlan = requireJsonObjectOrNull(messaging.plan, "messaging.plan"); + if (agent === "langchain-deepagents-code" && messagingPlan !== null) { + invalid("messaging.plan must be null for langchain-deepagents-code"); + } + + const corporateCa = requireRecord(profile.corporateCa, "corporateCa"); + rejectUnknownKeys(corporateCa, CORPORATE_CA_KEYS, "corporateCa"); + const bundleSha256 = corporateCa.bundleSha256; + if ( + bundleSha256 !== null && + (typeof bundleSha256 !== "string" || !SHA256_RE.test(bundleSha256)) + ) { + invalid("corporateCa.bundleSha256 must be null or a lowercase SHA-256 digest"); + } + + const agentConfig = validateAgentConfig(profile.agentConfig, agent); + const dashboard = validateDashboard(profile.dashboard, agent); + if ( + agentConfig.agent === "openclaw" && + dashboard.agent === "openclaw" && + dashboard.mode === "remote" && + !agentConfig.deviceAuth.disabled + ) { + invalid("remote OpenClaw dashboard exposure requires device auth to be disabled"); + } + + return { + schemaVersion: MANAGED_STARTUP_PROFILE_SCHEMA_VERSION, + agent, + agentConfig, + inference: validateInference(profile.inference, agent), + proxy: validateProxy(profile.proxy, agent), + dashboard, + tools: validateTools(profile.tools, agent), + messaging: { + plan: messagingPlan, + }, + tuning: validateTuning(profile.tuning, agent), + corporateCa: { bundleSha256 }, + }; +} + +function canonicalizeJson(value: unknown): unknown { + if (Array.isArray(value)) return value.map((item) => canonicalizeJson(item)); + if (!isPlainObject(value)) return value; + return Object.fromEntries( + Object.keys(value) + .sort() + .map((key) => [key, canonicalizeJson(value[key])]), + ); +} + +/** Canonical JSON used by both the transport and fingerprint. */ +export function serializeManagedStartupProfile(profile: ManagedStartupProfile): string { + const validated = validateManagedStartupProfile(profile); + const serialized = JSON.stringify(canonicalizeJson(validated)); + if (Buffer.byteLength(serialized, "utf8") > MANAGED_STARTUP_PROFILE_MAX_BYTES) { + invalid(`canonical payload exceeds ${String(MANAGED_STARTUP_PROFILE_MAX_BYTES)} bytes`); + } + return serialized; +} + +/** Encode canonical JSON as unpadded base64url for an argv/env-safe handoff. */ +export function encodeManagedStartupProfile(profile: ManagedStartupProfile): string { + return Buffer.from(serializeManagedStartupProfile(profile), "utf8").toString("base64url"); +} + +/** Decode only the canonical representation produced by encodeManagedStartupProfile. */ +export function decodeManagedStartupProfile(encoded: string): ManagedStartupProfile { + if ( + typeof encoded !== "string" || + encoded.length === 0 || + Buffer.byteLength(encoded, "ascii") > MANAGED_STARTUP_PROFILE_MAX_ENCODED_BYTES || + !BASE64URL_RE.test(encoded) || + encoded.length % 4 === 1 + ) { + invalid("encoded payload is malformed or exceeds the size limit"); + } + const bytes = Buffer.from(encoded, "base64url"); + if ( + bytes.length === 0 || + bytes.length > MANAGED_STARTUP_PROFILE_MAX_BYTES || + bytes.toString("base64url") !== encoded + ) { + invalid("encoded payload is malformed or exceeds the size limit"); + } + + let raw: string; + try { + raw = UTF8_DECODER.decode(bytes); + } catch { + invalid("payload is not valid UTF-8"); + } + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch { + invalid("payload is not valid JSON"); + } + const profile = validateManagedStartupProfile(parsed); + if (serializeManagedStartupProfile(profile) !== raw) { + invalid("payload is not in canonical form"); + } + return profile; +} + +/** SHA-256 over canonical decoded JSON, independent of object key insertion order. */ +export function fingerprintManagedStartupProfile(profile: ManagedStartupProfile): string { + return createHash("sha256").update(serializeManagedStartupProfile(profile), "utf8").digest("hex"); +} diff --git a/src/lib/onboard/managed-startup/root-apply.ts b/src/lib/onboard/managed-startup/root-apply.ts new file mode 100644 index 0000000000..83ed3d7fe4 --- /dev/null +++ b/src/lib/onboard/managed-startup/root-apply.ts @@ -0,0 +1,161 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; + +import { + decodeManagedStartupProfile, + fingerprintManagedStartupProfile, + MANAGED_STARTUP_AGENTS, + MANAGED_STARTUP_PROFILE_MAX_ENCODED_BYTES, + type ManagedStartupAgent, +} from "./profile"; + +export const MANAGED_STARTUP_ROOT_APPLY_SCHEMA_VERSION = 1 as const; + +// One canonical profile (64 KiB decoded), one bounded corporate CA bundle +// (128 KiB decoded), and a small fixed JSON envelope. +export const MANAGED_STARTUP_ROOT_APPLY_MAX_BYTES = 320 * 1024; + +const MAX_CORPORATE_CA_ENCODED_BYTES = 4 * Math.ceil((128 * 1024) / 3); +const SHA256_RE = /^[a-f0-9]{64}$/u; +const STANDARD_BASE64_RE = /^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/u; + +export interface ManagedStartupRootApplyRequest { + readonly schemaVersion: typeof MANAGED_STARTUP_ROOT_APPLY_SCHEMA_VERSION; + readonly agent: ManagedStartupAgent; + readonly encodedProfile: string; + readonly profileFingerprint: string; + readonly corporateCaB64: string | null; +} + +function fail(message: string): never { + throw new Error(`Managed startup root application request is invalid: ${message}`); +} + +function exactAgent(value: unknown): ManagedStartupAgent { + if (typeof value === "string" && (MANAGED_STARTUP_AGENTS as readonly string[]).includes(value)) { + return value as ManagedStartupAgent; + } + return fail("agent is unsupported"); +} + +export function createManagedStartupRootApplyRequest(input: { + readonly agent: ManagedStartupAgent; + readonly encodedProfile: string; + readonly corporateCaB64?: string; +}): ManagedStartupRootApplyRequest { + const agent = exactAgent(input.agent); + if ( + input.encodedProfile.length === 0 || + input.encodedProfile.length > MANAGED_STARTUP_PROFILE_MAX_ENCODED_BYTES + ) { + fail("encoded profile exceeds its bounded transport"); + } + const profile = decodeManagedStartupProfile(input.encodedProfile); + if (profile.agent !== agent) { + fail(`profile targets ${profile.agent}, expected ${agent}`); + } + const corporateCaB64 = input.corporateCaB64 ?? null; + if ( + corporateCaB64 !== null && + (corporateCaB64.length === 0 || + corporateCaB64.length > MAX_CORPORATE_CA_ENCODED_BYTES || + !STANDARD_BASE64_RE.test(corporateCaB64) || + Buffer.from(corporateCaB64, "base64").toString("base64") !== corporateCaB64) + ) { + fail("corporate CA is not canonical bounded base64"); + } + if ((profile.corporateCa.bundleSha256 !== null) !== (corporateCaB64 !== null)) { + fail("corporate CA transport does not match the profile"); + } + if ( + corporateCaB64 !== null && + createHash("sha256").update(Buffer.from(corporateCaB64, "base64")).digest("hex") !== + profile.corporateCa.bundleSha256 + ) { + fail("corporate CA does not match the profile digest"); + } + return Object.freeze({ + schemaVersion: MANAGED_STARTUP_ROOT_APPLY_SCHEMA_VERSION, + agent, + encodedProfile: input.encodedProfile, + profileFingerprint: fingerprintManagedStartupProfile(profile), + corporateCaB64, + }); +} + +export function serializeManagedStartupRootApplyRequest( + request: ManagedStartupRootApplyRequest, +): string { + const normalized = createManagedStartupRootApplyRequest({ + agent: request.agent, + encodedProfile: request.encodedProfile, + ...(request.corporateCaB64 === null ? {} : { corporateCaB64: request.corporateCaB64 }), + }); + if ( + request.schemaVersion !== MANAGED_STARTUP_ROOT_APPLY_SCHEMA_VERSION || + request.profileFingerprint !== normalized.profileFingerprint || + !SHA256_RE.test(request.profileFingerprint) + ) { + fail("schema version or profile fingerprint is invalid"); + } + const serialized = `${JSON.stringify({ + agent: normalized.agent, + corporateCaB64: normalized.corporateCaB64, + encodedProfile: normalized.encodedProfile, + profileFingerprint: normalized.profileFingerprint, + schemaVersion: normalized.schemaVersion, + })}\n`; + if (Buffer.byteLength(serialized, "utf8") > MANAGED_STARTUP_ROOT_APPLY_MAX_BYTES) { + fail("serialized request exceeds its bounded transport"); + } + return serialized; +} + +export function parseManagedStartupRootApplyRequest(text: string): ManagedStartupRootApplyRequest { + if (text.length === 0 || Buffer.byteLength(text, "utf8") > MANAGED_STARTUP_ROOT_APPLY_MAX_BYTES) { + fail("serialized request is empty or too large"); + } + let parsed: unknown; + try { + parsed = JSON.parse(text); + } catch { + fail("serialized request is not valid JSON"); + } + if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) { + fail("serialized request must be an object"); + } + const record = parsed as Record; + const expectedKeys = [ + "agent", + "corporateCaB64", + "encodedProfile", + "profileFingerprint", + "schemaVersion", + ]; + if ( + Object.keys(record).sort().join(",") !== expectedKeys.sort().join(",") || + record.schemaVersion !== MANAGED_STARTUP_ROOT_APPLY_SCHEMA_VERSION || + typeof record.encodedProfile !== "string" || + typeof record.profileFingerprint !== "string" || + (record.corporateCaB64 !== null && typeof record.corporateCaB64 !== "string") + ) { + fail("serialized request has an invalid schema"); + } + const request = createManagedStartupRootApplyRequest({ + agent: exactAgent(record.agent), + encodedProfile: record.encodedProfile, + ...(record.corporateCaB64 === null ? {} : { corporateCaB64: record.corporateCaB64 as string }), + }); + if ( + record.profileFingerprint !== request.profileFingerprint || + !SHA256_RE.test(record.profileFingerprint) + ) { + fail("profile fingerprint does not match the encoded profile"); + } + if (serializeManagedStartupRootApplyRequest(request) !== text) { + fail("serialized request is not canonical"); + } + return request; +} diff --git a/src/lib/onboard/managed-startup/shared-state-transaction.ts b/src/lib/onboard/managed-startup/shared-state-transaction.ts new file mode 100644 index 0000000000..d921ec36d8 --- /dev/null +++ b/src/lib/onboard/managed-startup/shared-state-transaction.ts @@ -0,0 +1,1025 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash, randomBytes } from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; +import { parseSandboxMessagingPlan } from "../../messaging/plan-validation"; +import { + selectEnabledMessagingAgentRender, + selectEnabledPostAgentInstallBuildFiles, +} from "../../messaging/post-agent-install-selection"; +import { + fingerprintManagedStartupProfile, + type ManagedStartupAgent, + type ManagedStartupProfile, +} from "./profile"; + +const TRANSACTION_SCHEMA_VERSION = 1; +const MAX_TRANSACTION_FILES = 128; +const MAX_TRANSACTION_FILE_BYTES = 8 * 1024 * 1024; +const MAX_TRANSACTION_TOTAL_BYTES = 32 * 1024 * 1024; +const MAX_MANIFEST_BYTES = 256 * 1024; +const TRANSACTION_PARENT_DIRECTORY_MODE = 0o755; +const TRANSACTION_DIRECTORY_MODE = 0o700; +const TRANSACTION_FILE_MODE = 0o400; + +export const MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY = + "/var/lib/nemoclaw/managed-startup-shared-state-transaction-v1"; +export const MANAGED_STARTUP_SHARED_ROLLBACK_RECEIPT_DIRECTORY = + "/run/nemoclaw/managed-startup-shared-rollback-receipt-v1"; + +interface FilePresentReceipt { + readonly path: string; + readonly state: "file"; + readonly backup: string; + readonly sha256: string; + readonly size: number; + readonly uid: number; + readonly gid: number; + readonly mode: number; +} + +interface FileAbsentReceipt { + readonly path: string; + readonly state: "absent"; +} + +type FileReceipt = FilePresentReceipt | FileAbsentReceipt; + +interface DirectoryPresentReceipt { + readonly path: string; + readonly state: "directory"; + readonly uid: number; + readonly gid: number; + readonly mode: number; +} + +interface DirectoryAbsentReceipt { + readonly path: string; + readonly state: "absent"; +} + +type DirectoryReceipt = DirectoryPresentReceipt | DirectoryAbsentReceipt; + +interface TransactionManifest { + readonly schemaVersion: typeof TRANSACTION_SCHEMA_VERSION; + readonly agent: ManagedStartupAgent; + readonly profileFingerprint: string; + readonly files: readonly FileReceipt[]; + readonly directories: readonly DirectoryReceipt[]; +} + +export interface ManagedStartupSharedTransactionOptions { + readonly sandboxRoot?: string; + readonly transactionDirectory?: string; + /** Test seam. Production always retains the root:root defaults. */ + readonly trustedUid?: number; + /** Test seam. Production always retains the root:root defaults. */ + readonly trustedGid?: number; + /** + * Rollback-helper seam. The host copy is mounted read-only at a fixed path, + * so ownership may reflect the Docker CLI user instead of container root. + */ + readonly readOnlyReceipt?: boolean; +} + +interface ResolvedOptions { + readonly sandboxRoot: string; + readonly transactionParentDirectory: string; + readonly transactionDirectory: string; + readonly backupDirectory: string; + readonly manifestFile: string; + readonly trustedUid: number; + readonly trustedGid: number; + readonly readOnlyReceipt: boolean; +} + +interface StableFile { + readonly bytes: Buffer; + readonly stat: fs.BigIntStats; +} + +function fail(message: string): never { + throw new Error(`Managed startup shared-state transaction failed: ${message}`); +} + +function resolveOptions(options: ManagedStartupSharedTransactionOptions = {}): ResolvedOptions { + const sandboxRoot = path.resolve(options.sandboxRoot ?? "/sandbox"); + const transactionDirectory = path.resolve( + options.transactionDirectory ?? MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY, + ); + if ( + transactionDirectory === sandboxRoot || + transactionDirectory.startsWith(`${sandboxRoot}${path.sep}`) + ) { + fail("transaction receipts must not be stored in sandbox-shared state"); + } + return { + sandboxRoot, + transactionParentDirectory: path.dirname(transactionDirectory), + transactionDirectory, + backupDirectory: path.join(transactionDirectory, "backups"), + manifestFile: path.join(transactionDirectory, "manifest.json"), + trustedUid: options.trustedUid ?? 0, + trustedGid: options.trustedGid ?? 0, + readOnlyReceipt: options.readOnlyReceipt ?? false, + }; +} + +function modeOf(stat: fs.Stats | fs.BigIntStats): number { + if (typeof stat.mode === "bigint") { + return Number(stat.mode & 0o7777n); + } + return stat.mode & 0o7777; +} + +function requireTransactionIdentity(options: ResolvedOptions): void { + const expectedUid = options.readOnlyReceipt ? 0 : options.trustedUid; + const expectedGid = options.readOnlyReceipt ? 0 : options.trustedGid; + if (process.geteuid?.() !== expectedUid || process.getegid?.() !== expectedGid) { + fail("transaction control requires the trusted effective identity"); + } +} + +function pathExistsNoFollow(target: string): boolean { + try { + fs.lstatSync(target); + return true; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return false; + fail(`could not inspect ${target}`); + } +} + +function requireDirectory( + target: string, + options: ResolvedOptions, + expectedMode: number | null = null, +): fs.Stats { + let stat: fs.Stats; + try { + stat = fs.lstatSync(target); + } catch { + fail(`required directory is missing: ${target}`); + } + if (stat.isSymbolicLink() || !stat.isDirectory()) { + fail(`required directory is unsafe: ${target}`); + } + if ( + expectedMode !== null && + (stat.uid !== options.trustedUid || + stat.gid !== options.trustedGid || + modeOf(stat) !== expectedMode) + ) { + fail( + `${target} must be ${options.trustedUid}:${options.trustedGid} mode ${expectedMode.toString(8)}`, + ); + } + return stat; +} + +function requireTransactionBoundaries(options: ResolvedOptions): void { + requireDirectory(options.sandboxRoot, options); + requireDirectory(options.transactionParentDirectory, options, TRANSACTION_PARENT_DIRECTORY_MODE); +} + +function sameStableMetadata(left: fs.BigIntStats, right: fs.BigIntStats): boolean { + return ( + left.dev === right.dev && + left.ino === right.ino && + left.mode === right.mode && + left.nlink === right.nlink && + left.uid === right.uid && + left.gid === right.gid && + left.size === right.size && + left.mtimeNs === right.mtimeNs && + left.ctimeNs === right.ctimeNs + ); +} + +function readStableFile(target: string, maxBytes: number): StableFile { + const noFollow = fs.constants.O_NOFOLLOW; + if (typeof noFollow !== "number") fail("O_NOFOLLOW is unavailable"); + let descriptor: number; + try { + descriptor = fs.openSync(target, fs.constants.O_RDONLY | noFollow); + } catch { + fail(`could not safely open ${target}`); + } + try { + const before = fs.fstatSync(descriptor, { bigint: true }); + if ( + !before.isFile() || + before.nlink !== 1n || + before.size < 0n || + before.size > BigInt(maxBytes) + ) { + fail(`refusing unsafe or oversized transaction file ${target}`); + } + const bytes = Buffer.alloc(Number(before.size)); + let offset = 0; + while (offset < bytes.length) { + const count = fs.readSync(descriptor, bytes, offset, bytes.length - offset, offset); + if (count === 0) break; + offset += count; + } + const overflow = Buffer.alloc(1); + const overflowCount = fs.readSync(descriptor, overflow, 0, 1, offset); + const after = fs.fstatSync(descriptor, { bigint: true }); + if (offset !== bytes.length || overflowCount !== 0 || !sameStableMetadata(before, after)) { + fail(`${target} changed while it was captured`); + } + return { bytes, stat: before }; + } finally { + fs.closeSync(descriptor); + } +} + +function safeRelativePath(value: string): string { + if ( + value.length === 0 || + value.startsWith("/") || + value.includes("\\") || + /[\0-\x1f\x7f]/u.test(value) + ) { + fail(`unsafe transaction path ${JSON.stringify(value)}`); + } + const segments = value.split("/"); + if (segments.some((segment) => !segment || segment === "." || segment === "..")) { + fail(`unsafe transaction path ${JSON.stringify(value)}`); + } + return segments.join("/"); +} + +function absoluteTarget(relativePath: string, options: ResolvedOptions): string { + const safe = safeRelativePath(relativePath); + const target = path.resolve(options.sandboxRoot, safe); + if (!target.startsWith(`${options.sandboxRoot}${path.sep}`)) { + fail(`transaction target escapes the sandbox root: ${relativePath}`); + } + return target; +} + +function relativeTarget(target: string, options: ResolvedOptions): string { + return safeRelativePath(path.relative(options.sandboxRoot, target)); +} + +function validateExistingAncestors(target: string, options: ResolvedOptions): void { + const relative = relativeTarget(target, options); + const sandboxStat = requireDirectory(options.sandboxRoot, options); + let current = options.sandboxRoot; + const segments = relative.split("/").slice(0, -1); + for (const segment of segments) { + current = path.join(current, segment); + let stat: fs.Stats; + try { + stat = fs.lstatSync(current); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return; + fail(`could not inspect transaction path ancestor ${current}`); + } + if (stat.isSymbolicLink() || !stat.isDirectory()) { + fail(`transaction path ancestor is unsafe: ${current}`); + } + if (stat.dev !== sandboxStat.dev) { + fail(`transaction path crosses a nested filesystem mount: ${current}`); + } + } +} + +function agentRoot(agent: ManagedStartupAgent, sandboxRoot: string): string { + switch (agent) { + case "openclaw": + return path.join(sandboxRoot, ".openclaw"); + case "hermes": + return path.join(sandboxRoot, ".hermes"); + case "langchain-deepagents-code": + return path.join(sandboxRoot, ".deepagents"); + } +} + +function resolveUnderAgentRoot(root: string, relativePath: string): string { + const safe = safeRelativePath(relativePath); + const target = path.resolve(root, safe); + if (!target.startsWith(`${root}${path.sep}`)) { + fail(`managed output escapes the agent root: ${relativePath}`); + } + return target; +} + +function renderTarget(root: string, agent: ManagedStartupAgent, target: string): string { + if (agent === "openclaw" && target === "openclaw.json") { + return path.join(root, "openclaw.json"); + } + const prefix = agent === "openclaw" ? "~/.openclaw/" : agent === "hermes" ? "~/.hermes/" : null; + if (!prefix || !target.startsWith(prefix)) { + fail(`unsupported managed messaging render target ${JSON.stringify(target)}`); + } + return resolveUnderAgentRoot(root, target.slice(prefix.length)); +} + +function managedOutputTargets( + profile: ManagedStartupProfile, + options: ResolvedOptions, +): { readonly files: string[]; readonly directories: string[] } { + const root = agentRoot(profile.agent, options.sandboxRoot); + const files = new Set(); + const directories = new Set([root]); + switch (profile.agent) { + case "openclaw": + files.add(path.join(root, "openclaw.json")); + files.add(path.join(root, ".config-hash")); + break; + case "hermes": + files.add(path.join(root, "config.yaml")); + files.add(path.join(root, ".env")); + files.add(path.join(root, ".config-hash")); + break; + case "langchain-deepagents-code": + files.add(path.join(root, "config.toml")); + directories.add(path.join(root, ".state")); + directories.add(path.join(root, "skills")); + break; + } + + if (profile.messaging.plan !== null) { + const plan = parseSandboxMessagingPlan(profile.messaging.plan, { agent: profile.agent }); + if (!plan) fail("managed messaging plan is invalid"); + for (const render of selectEnabledMessagingAgentRender(plan)) { + if (typeof render.target !== "string") continue; + files.add(renderTarget(root, profile.agent, render.target)); + } + for (const step of selectEnabledPostAgentInstallBuildFiles(plan)) { + if (typeof step.value !== "object" || step.value === null) { + continue; + } + const outputPath = (step.value as Record).path; + if (typeof outputPath === "string") { + files.add(resolveUnderAgentRoot(root, outputPath)); + } + } + } + + for (const file of files) { + let parent = path.dirname(file); + while (parent !== options.sandboxRoot && parent.startsWith(`${root}${path.sep}`)) { + directories.add(parent); + if (parent === root) break; + parent = path.dirname(parent); + } + } + return { + files: [...files].sort(), + directories: [...directories].sort( + (left, right) => left.split(path.sep).length - right.split(path.sep).length, + ), + }; +} + +function snapshotFile( + target: string, + index: number, + options: ResolvedOptions, +): { readonly receipt: FileReceipt; readonly bytes: Buffer | null } { + validateExistingAncestors(target, options); + let stat: fs.Stats; + try { + stat = fs.lstatSync(target); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + return { + receipt: { path: relativeTarget(target, options), state: "absent" }, + bytes: null, + }; + } + fail(`could not inspect managed output ${target}`); + } + if (stat.isSymbolicLink() || !stat.isFile() || stat.nlink !== 1) { + fail(`managed output is not a safe regular file: ${target}`); + } + if (stat.dev !== requireDirectory(options.sandboxRoot, options).dev) { + fail(`managed output crosses a nested filesystem mount: ${target}`); + } + const stable = readStableFile(target, MAX_TRANSACTION_FILE_BYTES); + const size = Number(stable.stat.size); + const backup = `${String(index).padStart(3, "0")}.bin`; + return { + receipt: { + path: relativeTarget(target, options), + state: "file", + backup, + sha256: createHash("sha256").update(stable.bytes).digest("hex"), + size, + uid: Number(stable.stat.uid), + gid: Number(stable.stat.gid), + mode: Number(stable.stat.mode & 0o7777n), + }, + bytes: stable.bytes, + }; +} + +function snapshotDirectory(target: string, options: ResolvedOptions): DirectoryReceipt { + validateExistingAncestors(path.join(target, ".receipt"), options); + let stat: fs.Stats; + try { + stat = fs.lstatSync(target); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + return { path: relativeTarget(target, options), state: "absent" }; + } + fail(`could not inspect managed output directory ${target}`); + } + if (stat.isSymbolicLink() || !stat.isDirectory()) { + fail(`managed output directory is unsafe: ${target}`); + } + if (stat.dev !== requireDirectory(options.sandboxRoot, options).dev) { + fail(`managed output directory crosses a nested filesystem mount: ${target}`); + } + return { + path: relativeTarget(target, options), + state: "directory", + uid: stat.uid, + gid: stat.gid, + mode: modeOf(stat), + }; +} + +function atomicWriteTrustedFile( + target: string, + contents: string | Buffer, + mode: number, + uid: number, + gid: number, +): void { + const parent = path.dirname(target); + const temporary = path.join( + parent, + `.${path.basename(target)}.${randomBytes(12).toString("hex")}`, + ); + let descriptor: number | undefined; + try { + descriptor = fs.openSync( + temporary, + fs.constants.O_CREAT | fs.constants.O_EXCL | fs.constants.O_WRONLY | fs.constants.O_NOFOLLOW, + 0o600, + ); + fs.writeFileSync(descriptor, contents); + fs.fchownSync(descriptor, uid, gid); + fs.fchmodSync(descriptor, mode); + fs.fsyncSync(descriptor); + fs.closeSync(descriptor); + descriptor = undefined; + fs.renameSync(temporary, target); + } catch (error) { + if (descriptor !== undefined) fs.closeSync(descriptor); + try { + fs.unlinkSync(temporary); + } catch { + // Preserve the primary failure. + } + fail(`could not atomically write ${target}: ${(error as Error).message}`); + } +} + +function canonicalManifest(manifest: TransactionManifest): string { + return `${JSON.stringify(manifest, null, 2)}\n`; +} + +function requireExactKeys(record: Record, keys: readonly string[]): void { + if (Object.keys(record).sort().join(",") !== [...keys].sort().join(",")) { + fail("transaction manifest contains unexpected fields"); + } +} + +function safeMetadata(value: unknown): value is number { + return Number.isSafeInteger(value) && (value as number) >= 0; +} + +function parseManifest(text: string): TransactionManifest { + let parsed: unknown; + try { + parsed = JSON.parse(text); + } catch { + fail("transaction manifest is not valid JSON"); + } + if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) { + fail("transaction manifest must be an object"); + } + const record = parsed as Record; + requireExactKeys(record, [ + "agent", + "directories", + "files", + "profileFingerprint", + "schemaVersion", + ]); + if ( + record.schemaVersion !== TRANSACTION_SCHEMA_VERSION || + !["openclaw", "hermes", "langchain-deepagents-code"].includes(String(record.agent)) || + typeof record.profileFingerprint !== "string" || + !/^[a-f0-9]{64}$/u.test(record.profileFingerprint) || + !Array.isArray(record.files) || + !Array.isArray(record.directories) || + record.files.length > MAX_TRANSACTION_FILES || + record.directories.length > MAX_TRANSACTION_FILES * 4 + ) { + fail("transaction manifest has an invalid envelope"); + } + const files = record.files.map((value): FileReceipt => { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + return fail("transaction file receipt must be an object"); + } + const receipt = value as Record; + if (typeof receipt.path !== "string") { + return fail("transaction file receipt path must be a string"); + } + const receiptPath = safeRelativePath(receipt.path); + if (receipt.state === "absent") { + requireExactKeys(receipt, ["path", "state"]); + return { path: receiptPath, state: "absent" }; + } + requireExactKeys(receipt, ["backup", "gid", "mode", "path", "sha256", "size", "state", "uid"]); + if ( + receipt.state !== "file" || + typeof receipt.backup !== "string" || + !/^[0-9]{3}\.bin$/u.test(receipt.backup) || + typeof receipt.sha256 !== "string" || + !/^[a-f0-9]{64}$/u.test(receipt.sha256) || + !safeMetadata(receipt.size) || + (receipt.size as number) > MAX_TRANSACTION_FILE_BYTES || + !safeMetadata(receipt.uid) || + !safeMetadata(receipt.gid) || + !safeMetadata(receipt.mode) || + (receipt.mode as number) > 0o7777 + ) { + return fail("transaction file receipt is invalid"); + } + return { + path: receiptPath, + state: "file", + backup: receipt.backup, + sha256: receipt.sha256, + size: receipt.size as number, + uid: receipt.uid as number, + gid: receipt.gid as number, + mode: receipt.mode as number, + }; + }); + const directories = record.directories.map((value): DirectoryReceipt => { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + return fail("transaction directory receipt must be an object"); + } + const receipt = value as Record; + if (typeof receipt.path !== "string") { + return fail("transaction directory receipt path must be a string"); + } + const receiptPath = safeRelativePath(receipt.path); + if (receipt.state === "absent") { + requireExactKeys(receipt, ["path", "state"]); + return { path: receiptPath, state: "absent" }; + } + requireExactKeys(receipt, ["gid", "mode", "path", "state", "uid"]); + if ( + receipt.state !== "directory" || + !safeMetadata(receipt.uid) || + !safeMetadata(receipt.gid) || + !safeMetadata(receipt.mode) || + (receipt.mode as number) > 0o7777 + ) { + return fail("transaction directory receipt is invalid"); + } + return { + path: receiptPath, + state: "directory", + uid: receipt.uid as number, + gid: receipt.gid as number, + mode: receipt.mode as number, + }; + }); + const filePaths = files.map((receipt) => receipt.path); + const directoryPaths = directories.map((receipt) => receipt.path); + const backupNames = files + .filter((receipt): receipt is FilePresentReceipt => receipt.state === "file") + .map((receipt) => receipt.backup); + if ( + new Set(filePaths).size !== filePaths.length || + new Set(directoryPaths).size !== directoryPaths.length || + new Set(backupNames).size !== backupNames.length + ) { + fail("transaction manifest contains duplicate receipts"); + } + const manifest: TransactionManifest = { + schemaVersion: TRANSACTION_SCHEMA_VERSION, + agent: record.agent as ManagedStartupAgent, + profileFingerprint: record.profileFingerprint, + files, + directories, + }; + if (canonicalManifest(manifest) !== text) { + fail("transaction manifest is not canonical"); + } + return manifest; +} + +function requireTrustedTransactionPath( + target: string, + mode: number, + options: ResolvedOptions, +): void { + const stat = fs.lstatSync(target); + if ( + stat.isSymbolicLink() || + (mode === TRANSACTION_DIRECTORY_MODE ? !stat.isDirectory() : !stat.isFile()) || + (!options.readOnlyReceipt && + (stat.uid !== options.trustedUid || stat.gid !== options.trustedGid)) || + modeOf(stat) !== mode + ) { + fail(`transaction artifact has unsafe metadata: ${target}`); + } +} + +function requireReadOnlyReceiptMount(options: ResolvedOptions): void { + if (!options.readOnlyReceipt) return; + const probe = path.join(options.transactionDirectory, ".nemoclaw-write-probe"); + let descriptor: number | undefined; + try { + descriptor = fs.openSync( + probe, + fs.constants.O_CREAT | fs.constants.O_EXCL | fs.constants.O_WRONLY | fs.constants.O_NOFOLLOW, + 0o600, + ); + fs.closeSync(descriptor); + descriptor = undefined; + fs.unlinkSync(probe); + } catch (error) { + if (descriptor !== undefined) fs.closeSync(descriptor); + if ((error as NodeJS.ErrnoException).code === "EROFS") return; + fail("rollback receipt must be mounted on a read-only filesystem"); + } + fail("rollback receipt mount is writable"); +} + +function loadManifest(options: ResolvedOptions): TransactionManifest | null { + requireTransactionBoundaries(options); + if (!pathExistsNoFollow(options.transactionDirectory)) return null; + requireTrustedTransactionPath(options.transactionDirectory, TRANSACTION_DIRECTORY_MODE, options); + requireReadOnlyReceiptMount(options); + requireTrustedTransactionPath(options.backupDirectory, TRANSACTION_DIRECTORY_MODE, options); + requireTrustedTransactionPath(options.manifestFile, TRANSACTION_FILE_MODE, options); + const stable = readStableFile(options.manifestFile, MAX_MANIFEST_BYTES); + if ( + (!options.readOnlyReceipt && + (Number(stable.stat.uid) !== options.trustedUid || + Number(stable.stat.gid) !== options.trustedGid)) || + Number(stable.stat.mode & 0o7777n) !== TRANSACTION_FILE_MODE + ) { + fail("transaction manifest ownership changed while it was read"); + } + return parseManifest(stable.bytes.toString("utf8")); +} + +function verifyBackup(receipt: FilePresentReceipt, options: ResolvedOptions): Buffer { + const backupPath = path.join(options.backupDirectory, receipt.backup); + requireTrustedTransactionPath(backupPath, TRANSACTION_FILE_MODE, options); + const stable = readStableFile(backupPath, MAX_TRANSACTION_FILE_BYTES); + const digest = createHash("sha256").update(stable.bytes).digest("hex"); + if (stable.bytes.length !== receipt.size || digest !== receipt.sha256) { + fail(`transaction backup does not match its receipt: ${receipt.path}`); + } + return stable.bytes; +} + +function verifyAllBackups( + receipts: readonly FileReceipt[], + options: ResolvedOptions, +): ReadonlyMap { + const backups = new Map(); + for (const receipt of receipts) { + if (receipt.state === "file") { + backups.set(receipt.path, verifyBackup(receipt, options)); + } + } + return backups; +} + +function fileMatchesReceipt(target: string, receipt: FilePresentReceipt): boolean { + let stat: fs.Stats; + try { + stat = fs.lstatSync(target); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return false; + fail(`could not inspect managed output ${target}`); + } + if (stat.isSymbolicLink() || !stat.isFile() || stat.nlink !== 1) return false; + const stable = readStableFile(target, MAX_TRANSACTION_FILE_BYTES); + return ( + stable.bytes.length === receipt.size && + createHash("sha256").update(stable.bytes).digest("hex") === receipt.sha256 && + Number(stable.stat.uid) === receipt.uid && + Number(stable.stat.gid) === receipt.gid && + Number(stable.stat.mode & 0o7777n) === receipt.mode + ); +} + +function directoryMatchesReceipt(target: string, receipt: DirectoryPresentReceipt): boolean { + let stat: fs.Stats; + try { + stat = fs.lstatSync(target); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return false; + fail(`could not inspect managed output directory ${target}`); + } + return ( + !stat.isSymbolicLink() && + stat.isDirectory() && + stat.uid === receipt.uid && + stat.gid === receipt.gid && + modeOf(stat) === receipt.mode + ); +} + +function removeTransactionDirectory(options: ResolvedOptions): void { + requireTrustedTransactionPath(options.transactionDirectory, TRANSACTION_DIRECTORY_MODE, options); + fs.rmSync(options.transactionDirectory, { force: false, recursive: true }); + if (pathExistsNoFollow(options.transactionDirectory)) { + fail("transaction directory remained after cleanup"); + } +} + +export function beginManagedStartupSharedStateTransaction( + profile: ManagedStartupProfile, + inputOptions: ManagedStartupSharedTransactionOptions = {}, +): boolean { + const options = resolveOptions(inputOptions); + requireTransactionIdentity(options); + if (options.readOnlyReceipt) { + fail("cannot begin a transaction from a read-only rollback receipt"); + } + requireTransactionBoundaries(options); + const profileFingerprint = fingerprintManagedStartupProfile(profile); + const pending = loadManifest(options); + if (pending) { + if (pending.agent !== profile.agent || pending.profileFingerprint !== profileFingerprint) { + fail("a pending managed startup transaction belongs to a different profile"); + } + verifyAllBackups(pending.files, options); + return false; + } + const targets = managedOutputTargets(profile, options); + if (targets.files.length > MAX_TRANSACTION_FILES) { + fail("managed startup transaction has too many file targets"); + } + const snapshots = targets.files.map((target, index) => snapshotFile(target, index, options)); + const totalBytes = snapshots.reduce((sum, snapshot) => sum + (snapshot.bytes?.length ?? 0), 0); + if (totalBytes > MAX_TRANSACTION_TOTAL_BYTES) { + fail("managed startup transaction backup exceeds the total size limit"); + } + const directories = targets.directories.map((target) => snapshotDirectory(target, options)); + const manifest: TransactionManifest = { + schemaVersion: TRANSACTION_SCHEMA_VERSION, + agent: profile.agent, + profileFingerprint, + files: snapshots.map(({ receipt }) => receipt), + directories, + }; + + let createdTransactionIdentity: + | { readonly dev: bigint; readonly ino: bigint; readonly uid: bigint; readonly gid: bigint } + | undefined; + try { + fs.mkdirSync(options.transactionDirectory, { mode: TRANSACTION_DIRECTORY_MODE }); + const created = fs.lstatSync(options.transactionDirectory, { bigint: true }); + if (!created.isDirectory() || created.isSymbolicLink()) { + fail("new transaction path is not a directory"); + } + createdTransactionIdentity = { + dev: created.dev, + ino: created.ino, + uid: created.uid, + gid: created.gid, + }; + fs.chownSync(options.transactionDirectory, options.trustedUid, options.trustedGid); + fs.chmodSync(options.transactionDirectory, TRANSACTION_DIRECTORY_MODE); + fs.mkdirSync(options.backupDirectory, { mode: TRANSACTION_DIRECTORY_MODE }); + fs.chownSync(options.backupDirectory, options.trustedUid, options.trustedGid); + fs.chmodSync(options.backupDirectory, TRANSACTION_DIRECTORY_MODE); + for (const snapshot of snapshots) { + if (snapshot.receipt.state !== "file" || snapshot.bytes === null) continue; + atomicWriteTrustedFile( + path.join(options.backupDirectory, snapshot.receipt.backup), + snapshot.bytes, + TRANSACTION_FILE_MODE, + options.trustedUid, + options.trustedGid, + ); + } + atomicWriteTrustedFile( + options.manifestFile, + canonicalManifest(manifest), + TRANSACTION_FILE_MODE, + options.trustedUid, + options.trustedGid, + ); + loadManifest(options); + } catch (error) { + try { + if (createdTransactionIdentity && pathExistsNoFollow(options.transactionDirectory)) { + const current = fs.lstatSync(options.transactionDirectory, { bigint: true }); + if ( + !current.isSymbolicLink() && + current.isDirectory() && + current.dev === createdTransactionIdentity.dev && + current.ino === createdTransactionIdentity.ino && + current.uid === createdTransactionIdentity.uid && + current.gid === createdTransactionIdentity.gid + ) { + fs.chmodSync(options.transactionDirectory, TRANSACTION_DIRECTORY_MODE); + fs.chownSync(options.transactionDirectory, options.trustedUid, options.trustedGid); + } + requireTrustedTransactionPath( + options.transactionDirectory, + TRANSACTION_DIRECTORY_MODE, + options, + ); + fs.rmSync(options.transactionDirectory, { force: true, recursive: true }); + } + } catch { + // Preserve the primary transaction preparation failure. + } + throw error; + } + return true; +} + +function ensureOriginalDirectories( + receipts: readonly DirectoryReceipt[], + options: ResolvedOptions, +): void { + for (const receipt of receipts) { + if (receipt.state !== "directory") continue; + const target = absoluteTarget(receipt.path, options); + validateExistingAncestors(path.join(target, ".restore"), options); + let stat: fs.Stats | null = null; + try { + stat = fs.lstatSync(target); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") { + fail(`could not inspect restore directory ${target}`); + } + } + if (stat && (stat.isSymbolicLink() || !stat.isDirectory())) { + fail(`restore directory is unsafe: ${target}`); + } + if (stat && directoryMatchesReceipt(target, receipt)) continue; + if (!stat) fs.mkdirSync(target, { mode: receipt.mode }); + fs.chownSync(target, receipt.uid, receipt.gid); + fs.chmodSync(target, receipt.mode); + } +} + +function restoreFiles( + receipts: readonly FileReceipt[], + backups: ReadonlyMap, + options: ResolvedOptions, +): void { + for (const receipt of receipts) { + const target = absoluteTarget(receipt.path, options); + validateExistingAncestors(target, options); + if (receipt.state === "absent") { + let stat: fs.Stats; + try { + stat = fs.lstatSync(target); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") continue; + fail(`could not inspect new managed output ${target}`); + } + if (stat.isDirectory()) { + fail(`new managed output unexpectedly became a directory: ${target}`); + } + fs.unlinkSync(target); + continue; + } + if (fileMatchesReceipt(target, receipt)) continue; + const bytes = backups.get(receipt.path); + if (!bytes) fail(`verified transaction backup is missing: ${receipt.path}`); + let current: fs.Stats | null = null; + try { + current = fs.lstatSync(target); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") { + fail(`could not inspect managed output before restore: ${target}`); + } + } + if (current?.isDirectory()) { + fail(`managed output unexpectedly became a directory: ${target}`); + } + atomicWriteTrustedFile(target, bytes, receipt.mode, receipt.uid, receipt.gid); + } +} + +function restoreDirectoryMetadata( + receipts: readonly DirectoryReceipt[], + options: ResolvedOptions, +): void { + for (const receipt of [...receipts].reverse()) { + const target = absoluteTarget(receipt.path, options); + if (receipt.state === "absent") { + try { + fs.rmdirSync(target); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") continue; + fail(`could not remove newly created managed directory ${target}`); + } + continue; + } + if (directoryMatchesReceipt(target, receipt)) continue; + const stat = fs.lstatSync(target); + if (stat.isSymbolicLink() || !stat.isDirectory()) { + fail(`managed directory changed type during restore: ${target}`); + } + fs.chownSync(target, receipt.uid, receipt.gid); + fs.chmodSync(target, receipt.mode); + } +} + +function verifyRestoration(manifest: TransactionManifest, options: ResolvedOptions): void { + for (const receipt of manifest.files) { + const target = absoluteTarget(receipt.path, options); + if (receipt.state === "absent") { + if (pathExistsNoFollow(target)) { + fail(`new managed output remained after rollback: ${target}`); + } + continue; + } + const stable = readStableFile(target, MAX_TRANSACTION_FILE_BYTES); + if ( + stable.bytes.length !== receipt.size || + createHash("sha256").update(stable.bytes).digest("hex") !== receipt.sha256 || + Number(stable.stat.uid) !== receipt.uid || + Number(stable.stat.gid) !== receipt.gid || + Number(stable.stat.mode & 0o7777n) !== receipt.mode + ) { + fail(`managed output was not restored exactly: ${target}`); + } + } + for (const receipt of manifest.directories) { + const target = absoluteTarget(receipt.path, options); + if (receipt.state === "absent") { + if (pathExistsNoFollow(target)) { + fail(`new managed directory remained after rollback: ${target}`); + } + continue; + } + const stat = fs.lstatSync(target); + if ( + stat.isSymbolicLink() || + !stat.isDirectory() || + stat.uid !== receipt.uid || + stat.gid !== receipt.gid || + modeOf(stat) !== receipt.mode + ) { + fail(`managed directory metadata was not restored exactly: ${target}`); + } + } +} + +export function rollbackManagedStartupSharedStateTransaction( + expectedAgent: ManagedStartupAgent, + inputOptions: ManagedStartupSharedTransactionOptions = {}, +): boolean { + const options = resolveOptions(inputOptions); + requireTransactionIdentity(options); + const manifest = loadManifest(options); + if (!manifest) return false; + if (manifest.agent !== expectedAgent) { + fail(`pending transaction targets ${manifest.agent}, expected ${expectedAgent}`); + } + const backups = verifyAllBackups(manifest.files, options); + ensureOriginalDirectories(manifest.directories, options); + restoreFiles(manifest.files, backups, options); + restoreDirectoryMetadata(manifest.directories, options); + verifyRestoration(manifest, options); + if (!options.readOnlyReceipt) { + removeTransactionDirectory(options); + } + return true; +} + +export function commitManagedStartupSharedStateTransaction( + expectedAgent: ManagedStartupAgent, + inputOptions: ManagedStartupSharedTransactionOptions = {}, +): boolean { + const options = resolveOptions(inputOptions); + requireTransactionIdentity(options); + if (options.readOnlyReceipt) { + fail("cannot commit a read-only rollback receipt"); + } + const manifest = loadManifest(options); + if (!manifest) return false; + if (manifest.agent !== expectedAgent) { + fail(`pending transaction targets ${manifest.agent}, expected ${expectedAgent}`); + } + removeTransactionDirectory(options); + return true; +} diff --git a/src/lib/onboard/managed-startup/transport.ts b/src/lib/onboard/managed-startup/transport.ts new file mode 100644 index 0000000000..ba37f052c4 --- /dev/null +++ b/src/lib/onboard/managed-startup/transport.ts @@ -0,0 +1,6 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** Minimal host/image transport names shared without loading the image runtime. */ +export const MANAGED_STARTUP_PROFILE_ENV = "NEMOCLAW_STARTUP_PROFILE_B64"; +export const MANAGED_STARTUP_CA_ENV = "NEMOCLAW_CORPORATE_CA_B64"; diff --git a/src/lib/onboard/managed-workload/index.ts b/src/lib/onboard/managed-workload/index.ts new file mode 100644 index 0000000000..4bedf6b6e2 --- /dev/null +++ b/src/lib/onboard/managed-workload/index.ts @@ -0,0 +1,9 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export { getVersion } from "../../core/version"; +export * from "../managed-startup/onboard-profile"; +export * from "../sandbox-create-launch"; +export * from "../workload/preparation"; +export * from "../workload/rebuild"; +export * from "../workload/runtime"; diff --git a/src/lib/onboard/managed-workload/onboard-orchestration.ts b/src/lib/onboard/managed-workload/onboard-orchestration.ts new file mode 100644 index 0000000000..b0942bb1da --- /dev/null +++ b/src/lib/onboard/managed-workload/onboard-orchestration.ts @@ -0,0 +1,413 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { AgentDefinition } from "../../agent/defs"; +import { getVersion } from "../../core/version"; +import type { SandboxMessagingPlan } from "../../messaging/manifest"; +import type { SandboxWorkloadReceipt } from "../../state/registry/types"; +import type { + CreateSandboxBuildContextResult, + PreparedSandboxBuildContext, +} from "../build-context-stage"; +import type { OpenShellComputePlan } from "../compute/plan"; +import { enforceDockerGpuPatchPreserveNetwork } from "../docker-gpu-local-inference"; +import { + initialDockerGpuRoute, + renderSandboxCreateArgsForGpuRoute, + type SelectedDockerGpuRoute, +} from "../docker-gpu-route"; +import type { HermesDashboardOnboardState } from "../hermes-dashboard"; +import type { InitialSandboxPolicy } from "../initial-policy"; +import { + type BuiltManagedStartupOnboardProfile, + buildManagedStartupOnboardProfile, + type ManagedStartupOnboardProfileInput, +} from "../managed-startup/onboard-profile"; +import { getChannelsFromPlan } from "../messaging-plan-session"; +import type { MessagingTokenDef } from "../messaging-prep"; +import { resolveSandboxBuildContext, resolveSandboxBuildPatch } from "../prepared-dcode-rebuild"; +import type { + MaterializeSandboxCreatePlanInput, + SandboxCreateIntent, +} from "../sandbox-create-intent-types"; +import { + prepareSandboxCreateLaunchWithPrebuild, + prepareSandboxCreateManagedImageLaunch, + type SandboxCreateLaunchInput, + type SandboxCreateLaunchWithPrebuild, +} from "../sandbox-create-launch"; +import { getSandboxReadyTimeoutSecs } from "../sandbox-gpu-create"; +import type { SandboxGpuConfig } from "../sandbox-gpu-mode"; +import { + type PreparedSandboxWorkloadSource, + prepareSandboxWorkloadSource, +} from "../workload/preparation"; +import { + type ManagedWorkloadRebuildHandoff, + prepareSandboxWorkloadSourceFromRebuildHandoff, +} from "../workload/rebuild"; +import { resolveSandboxWorkloadRuntimeCapabilities } from "../workload/runtime"; + +type ManagedProfileInput = Omit; +type ResolveBuildPatchInput = Parameters[0]; +type SandboxInferenceConfig = import("../../inference/config").SandboxInferenceConfig; + +export interface ManagedWorkloadOnboardDependencies { + readonly resolveAgentInferenceApi: typeof import("../../inference/config").resolveAgentInferenceApi; + readonly getSandboxInferenceConfig: typeof import("../../inference/config").getSandboxInferenceConfig; +} + +export interface CreateManagedWorkloadOnboardRuntimeInput { + readonly computePlan: OpenShellComputePlan; + readonly managedWorkloadRebuild: ManagedWorkloadRebuildHandoff | null; + readonly agentName: string; + readonly legacyDockerfilePath: string; + readonly customDockerfilePath: string | null; + readonly rootDir: string; + readonly model: string; + readonly provider: string; + readonly preferredInferenceApi: string | null; + readonly endpointUrl: string | null; + readonly startupProfile: ManagedProfileInput; + readonly note: (message: string) => void; + readonly fallbackBuildEstimate: () => string | null; +} + +export interface ManagedWorkloadOnboardRuntime { + ensurePreparedWorkload(): Promise; + ensurePreparedProfile( + workload: PreparedSandboxWorkloadSource, + ): BuiltManagedStartupOnboardProfile | null; +} + +/** + * Own the one-shot managed workload and startup-profile decisions for an + * onboarding create. Both decisions are memoized so pre-delete validation, + * launch, and durable registration consume exactly the same artifacts. + */ +export function createManagedWorkloadOnboardRuntime( + input: CreateManagedWorkloadOnboardRuntimeInput, + dependencies: ManagedWorkloadOnboardDependencies, +): ManagedWorkloadOnboardRuntime { + const runtimeCapabilities = resolveSandboxWorkloadRuntimeCapabilities(input.computePlan); + let preparedWorkloadPromise: Promise | null = null; + let fallbackReported = false; + let preparedProfile: BuiltManagedStartupOnboardProfile | null = null; + + const ensurePreparedWorkload = async (): Promise => { + preparedWorkloadPromise ??= input.managedWorkloadRebuild + ? Promise.resolve( + prepareSandboxWorkloadSourceFromRebuildHandoff( + input.managedWorkloadRebuild, + runtimeCapabilities, + ), + ) + : prepareSandboxWorkloadSource({ + agentName: input.agentName, + legacyDockerfilePath: input.legacyDockerfilePath, + customDockerfilePath: input.customDockerfilePath, + runtime: runtimeCapabilities, + version: getVersion({ rootDir: input.rootDir }), + }); + const prepared = await preparedWorkloadPromise; + if (prepared.fallbackDiagnostic && !fallbackReported) { + fallbackReported = true; + input.note(" Managed image unavailable; using the trusted Dockerfile recipe."); + input.note(` ${prepared.fallbackDiagnostic}`); + const buildEstimate = input.fallbackBuildEstimate(); + if (buildEstimate) input.note(` ${buildEstimate}`); + } + return prepared; + }; + + const ensurePreparedProfile = ( + workload: PreparedSandboxWorkloadSource, + ): BuiltManagedStartupOnboardProfile | null => { + if (workload.source.kind !== "managed-image") return null; + if (input.managedWorkloadRebuild) { + if (workload.source.reference !== input.managedWorkloadRebuild.replacement.source.reference) { + throw new Error("Managed rebuild workload changed before startup profile preparation."); + } + return input.managedWorkloadRebuild.replacementProfile; + } + const inferenceApi = + input.agentName === "langchain-deepagents-code" + ? "openai-completions" + : dependencies.resolveAgentInferenceApi( + input.agentName, + input.provider, + input.preferredInferenceApi, + ); + const inference: SandboxInferenceConfig = dependencies.getSandboxInferenceConfig( + input.model, + input.provider, + inferenceApi, + ); + preparedProfile ??= buildManagedStartupOnboardProfile({ + agentName: input.agentName, + inference: { + routeProvider: inference.providerKey, + upstreamProvider: input.provider.trim() ? input.provider : inference.providerKey, + model: input.model, + routedBaseUrl: inference.inferenceBaseUrl, + upstreamEndpointUrl: + input.agentName === "langchain-deepagents-code" ? input.endpointUrl : null, + api: inference.inferenceApi as + | "openai-completions" + | "openai-responses" + | "anthropic-messages", + primaryModelRef: input.agentName === "openclaw" ? inference.primaryModelRef : null, + compatibility: input.agentName === "openclaw" ? (inference.inferenceCompat ?? {}) : null, + }, + ...input.startupProfile, + }); + return preparedProfile; + }; + + return { ensurePreparedWorkload, ensurePreparedProfile }; +} + +export interface PrepareOnboardSandboxWorkloadLaunchInput { + readonly runtime: ManagedWorkloadOnboardRuntime; + readonly workload: PreparedSandboxWorkloadSource; + readonly legacy: { + readonly preparedBuildContext: PreparedSandboxBuildContext | null; + readonly agent: AgentDefinition | null; + readonly fromDockerfile: string | null; + readonly createAgentSandbox: ( + agent: AgentDefinition, + ) => ReturnType; + readonly patchInput: Omit; + }; + readonly plan: { + readonly intent: SandboxCreateIntent; + readonly rebindMessagingTokenDefs: () => Promise; + readonly runProviderPreDeleteCleanup: () => void; + readonly upsertMessagingProviders: MaterializeSandboxCreatePlanInput["upsertMessagingProviders"]; + readonly getHermesToolGatewayProviderName: (sandboxName: string) => string; + readonly discloseInitialSandboxPolicy: (policy: InitialSandboxPolicy) => void; + }; + readonly launchInput: Omit & { + readonly sandboxName: string; + }; + readonly plannedMessagingPlan: SandboxMessagingPlan | null; + readonly gpu: { + readonly provider: string; + readonly config: SandboxGpuConfig; + readonly dockerDriverGateway: boolean; + readonly gatewayPort: number; + }; + readonly dependencies: { + readonly materializeSandboxCreatePlan: typeof import("../sandbox-create-plan-materialization").materializeSandboxCreatePlan; + readonly prepareSandboxBuildPatchConfig: typeof import("../sandbox-build-patch-config").prepareSandboxBuildPatchConfig; + }; + readonly log?: (message: string) => void; + readonly onExit?: (cleanup: () => void) => void; +} + +export interface PreparedOnboardSandboxWorkloadLaunch { + readonly activeMessagingChannels: string[]; + readonly initialSandboxPolicy: InitialSandboxPolicy; + readonly policyTier: string | null; + readonly messagingProviders: string[]; + readonly gpuRoutePlan: SandboxCreateIntent["gpuRoutePlan"]; + readonly compatibilityPolicyPath: string | null; + readonly initialGpuRoute: SelectedDockerGpuRoute; + readonly sandboxReadyTimeoutSecs: number; + readonly buildId: string; + readonly dashboardRemoteBindPrepared: boolean; + readonly legacyBuildContext: CreateSandboxBuildContextResult | null; + readonly launch: SandboxCreateLaunchWithPrebuild; +} + +function requireLegacyBuildContext( + buildContext: CreateSandboxBuildContextResult | null, +): CreateSandboxBuildContextResult { + if (!buildContext) { + throw new Error("Legacy sandbox workload is missing its staged build context."); + } + return buildContext; +} + +/** + * Materialize the selected workload into one OpenShell create launch. The + * complete-image branch never constructs or patches a Dockerfile context; the + * trusted fallback preserves the existing cleanup and prebuild contract. + */ +export async function prepareOnboardSandboxWorkloadLaunch( + input: PrepareOnboardSandboxWorkloadLaunchInput, +): Promise { + const log = input.log ?? console.log; + const legacyBuildContext = + input.workload.source.kind === "legacy-dockerfile" + ? resolveSandboxBuildContext( + { + preparedBuildContext: input.legacy.preparedBuildContext, + agent: input.legacy.agent, + fromDockerfile: input.legacy.fromDockerfile, + }, + { createAgentSandbox: input.legacy.createAgentSandbox }, + ) + : null; + const fromRef = + input.workload.source.kind === "managed-image" + ? input.workload.source.reference + : `${requireLegacyBuildContext(legacyBuildContext).buildCtx}/Dockerfile`; + const messagingTokenDefs = await input.plan.rebindMessagingTokenDefs(); + const createPlan = input.dependencies.materializeSandboxCreatePlan({ + intent: input.plan.intent, + fromRef, + messagingTokenDefs: [...messagingTokenDefs], + runProviderPreDeleteCleanup: input.plan.runProviderPreDeleteCleanup, + upsertMessagingProviders: input.plan.upsertMessagingProviders, + getHermesToolGatewayProviderName: input.plan.getHermesToolGatewayProviderName, + discloseInitialSandboxPolicy: input.plan.discloseInitialSandboxPolicy, + }); + if (createPlan.initialSandboxPolicy.cleanup) { + (input.onExit ?? ((cleanup) => process.on("exit", cleanup)))( + createPlan.initialSandboxPolicy.cleanup, + ); + } + if (input.plan.intent.sandboxGpuLogMessage) { + log(input.plan.intent.sandboxGpuLogMessage); + } + log( + ` Creating sandbox '${input.launchInput.sandboxName}' (this takes a few minutes on first run)...`, + ); + + const configuredMessagingChannels = + getChannelsFromPlan(input.plannedMessagingPlan) ?? createPlan.activeMessagingChannels; + const initialGpuRoute = initialDockerGpuRoute(createPlan.gpuRoutePlan); + const sandboxReadyTimeoutSecs = getSandboxReadyTimeoutSecs(input.gpu.config); + const launchInput: SandboxCreateLaunchInput & { sandboxName: string } = { + ...input.launchInput, + createArgs: renderSandboxCreateArgsForGpuRoute(createPlan.createArgs, initialGpuRoute, { + compatibilityPolicyPath: createPlan.compatibilityPolicyPath, + }), + }; + + let buildId = String(Date.now()); + let dashboardRemoteBindPrepared = false; + let launch: SandboxCreateLaunchWithPrebuild; + if (input.workload.source.kind === "managed-image") { + await enforceDockerGpuPatchPreserveNetwork(input.gpu.provider, input.gpu.config, { + dockerDriverGateway: input.gpu.dockerDriverGateway, + selectedRoute: initialGpuRoute, + gatewayPort: input.gpu.gatewayPort, + log, + }); + const managedProfile = input.runtime.ensurePreparedProfile(input.workload); + if (!managedProfile) { + throw new Error("Managed sandbox workload is missing its startup profile."); + } + dashboardRemoteBindPrepared = + managedProfile.profile.dashboard.agent === "openclaw" && + managedProfile.profile.dashboard.mode === "remote"; + launch = prepareSandboxCreateManagedImageLaunch({ + ...launchInput, + managedStartupProfile: { + encodedProfile: managedProfile.encodedProfile, + ...(managedProfile.corporateCaB64 === undefined + ? {} + : { corporateCaB64: managedProfile.corporateCaB64 }), + }, + }); + } else { + const buildContext = requireLegacyBuildContext(legacyBuildContext); + input.dependencies.prepareSandboxBuildPatchConfig({ configuredMessagingChannels }); + const patch = await resolveSandboxBuildPatch({ + ...input.legacy.patchInput, + selectedGpuRoute: initialGpuRoute, + stagedDockerfile: buildContext.stagedDockerfile, + }); + buildId = patch.buildId; + dashboardRemoteBindPrepared = patch.dashboardRemoteBindPrepared; + launch = await prepareSandboxCreateLaunchWithPrebuild({ + ...launchInput, + prebuild: { + buildCtx: buildContext.buildCtx, + buildId, + dockerDriverGateway: input.gpu.dockerDriverGateway, + origin: buildContext.origin, + }, + }); + } + + return { + activeMessagingChannels: createPlan.activeMessagingChannels, + initialSandboxPolicy: createPlan.initialSandboxPolicy, + policyTier: createPlan.policyTier, + messagingProviders: createPlan.messagingProviders, + gpuRoutePlan: createPlan.gpuRoutePlan, + compatibilityPolicyPath: createPlan.compatibilityPolicyPath, + initialGpuRoute, + sandboxReadyTimeoutSecs, + buildId, + dashboardRemoteBindPrepared, + legacyBuildContext, + launch, + }; +} + +export interface ResolveOnboardSandboxWorkloadReceiptInput { + readonly runtime: ManagedWorkloadOnboardRuntime; + readonly workload: PreparedSandboxWorkloadSource; + readonly registryImageRef: string | null; + readonly prebuildImageRef: string | null; + readonly firstCreateOutput: string; + readonly createOutput: string; + readonly buildId: string; + readonly extractBuiltImageRef: typeof import("../../build-context").extractBuiltImageRef; + readonly resolveSandboxImageTagFromCreateOutput: typeof import("../../domain/sandbox/image-tag").resolveSandboxImageTagFromCreateOutput; +} + +export function resolveOnboardSandboxWorkloadReceipt( + input: ResolveOnboardSandboxWorkloadReceiptInput, +): { + readonly resolvedImageTag: string; + readonly workloadReceipt: SandboxWorkloadReceipt; +} { + const output = `${input.firstCreateOutput}\n${input.createOutput}`; + const resolvedImageTag = + (input.workload.source.kind === "managed-image" ? input.workload.source.reference : null) ?? + input.registryImageRef ?? + input.prebuildImageRef ?? + input.extractBuiltImageRef(output) ?? + input.resolveSandboxImageTagFromCreateOutput(output, input.buildId); + if (input.workload.source.kind === "legacy-dockerfile") { + return { + resolvedImageTag, + workloadReceipt: { + schemaVersion: 1, + kind: "legacy-dockerfile", + reference: resolvedImageTag, + shared: false, + }, + }; + } + + const managedProfile = input.runtime.ensurePreparedProfile(input.workload); + if (!managedProfile) { + throw new Error("Managed sandbox workload is missing its startup profile."); + } + return { + resolvedImageTag, + workloadReceipt: { + schemaVersion: 1, + kind: "managed-image", + reference: input.workload.source.reference, + release: input.workload.source.contract.source.release, + sourceRevision: input.workload.source.contract.source.revision, + sourceCohort: input.workload.source.contract.source.cohort, + capabilityContractVersion: input.workload.source.contract.capabilityContractVersion, + startupProfileContractVersion: input.workload.source.contract.startupProfileContractVersion, + encodedProfile: managedProfile.encodedProfile, + startupProfileSha256: managedProfile.startupProfileSha256, + credentialProxyReplayRequired: managedProfile.credentialProxyReplayRequired, + ...(managedProfile.corporateCaB64 === undefined + ? {} + : { corporateCaB64: managedProfile.corporateCaB64 }), + shared: true, + }, + }; +} diff --git a/src/lib/onboard/openshell-install.test.ts b/src/lib/onboard/openshell-install.test.ts index 480e295a98..2d9249d681 100644 --- a/src/lib/onboard/openshell-install.test.ts +++ b/src/lib/onboard/openshell-install.test.ts @@ -4,6 +4,7 @@ import { describe, expect, it, vi } from "vitest"; import { + areRequiredDockerDriverBinariesPresent, ensureOpenshellForOnboard, type OpenShellInstallDeps, type OpenShellInstallResult, @@ -48,6 +49,39 @@ function makeDeps(overrides: Partial = {}) { } describe("ensureOpenshellForOnboard", () => { + it("requires only the binaries declared by a pluggable managed-driver profile", () => { + const deps = makeDeps({ + getManagedGatewayBinaryRequirements: () => ({ + driverLabel: "Podman", + gateway: true, + sandbox: false, + }), + resolveOpenShellGatewayBinary: () => "/tmp/openshell-gateway", + resolveOpenShellSandboxBinary: () => null, + }); + + expect(areRequiredDockerDriverBinariesPresent(deps, "linux")).toBe(true); + }); + + it("reinstalls for a missing Podman gateway binary without requiring a host sandbox binary", () => { + const deps = makeDeps({ + getManagedGatewayBinaryRequirements: () => ({ + driverLabel: "Podman", + gateway: true, + sandbox: false, + }), + resolveOpenShellGatewayBinary: () => null, + resolveOpenShellSandboxBinary: () => null, + }); + + ensureOpenshellForOnboard(deps); + + expect(deps.installOpenshell).toHaveBeenCalledOnce(); + expect(deps.log).toHaveBeenCalledWith( + " OpenShell Podman-driver gateway onboarding requires the gateway binaries. Reinstalling...", + ); + }); + it("reinstalls when the installed OpenShell lacks messaging rewrite or MCP L7 support", () => { const hasFeatures = vi.fn().mockReturnValueOnce(false).mockReturnValue(true); const deps = makeDeps({ diff --git a/src/lib/onboard/openshell-install.ts b/src/lib/onboard/openshell-install.ts index 105de2c36e..be343cc540 100644 --- a/src/lib/onboard/openshell-install.ts +++ b/src/lib/onboard/openshell-install.ts @@ -101,11 +101,21 @@ export type DockerDriverBinaryOverrides = { vmDriverBin?: string | null; }; +export type ManagedGatewayBinaryRequirements = { + driverLabel: string; + gateway: boolean; + sandbox: boolean; +}; + export type OpenShellInstallDeps = { isLinuxDockerDriverGatewayEnabled: ( platform?: NodeJS.Platform, arch?: NodeJS.Architecture, ) => boolean; + getManagedGatewayBinaryRequirements?: ( + platform?: NodeJS.Platform, + arch?: NodeJS.Architecture, + ) => ManagedGatewayBinaryRequirements | null; resolveOpenShellGatewayBinary: () => string | null; resolveOpenShellSandboxBinary: () => string | null; isOpenshellInstalled: () => boolean; @@ -130,6 +140,7 @@ export type OpenShellInstallDeps = { export function areRequiredDockerDriverBinariesPresent( deps: Pick< OpenShellInstallDeps, + | "getManagedGatewayBinaryRequirements" | "isLinuxDockerDriverGatewayEnabled" | "resolveOpenShellGatewayBinary" | "resolveOpenShellSandboxBinary" @@ -138,15 +149,24 @@ export function areRequiredDockerDriverBinariesPresent( binaries: DockerDriverBinaryOverrides = {}, arch: NodeJS.Architecture = process.arch, ): boolean { - if (!deps.isLinuxDockerDriverGatewayEnabled(platform, arch)) return true; + const requirements = + deps.getManagedGatewayBinaryRequirements?.(platform, arch) ?? + (deps.isLinuxDockerDriverGatewayEnabled(platform, arch) + ? { + driverLabel: "Docker", + gateway: true, + sandbox: platform === "linux", + } + : null); + if (!requirements) return true; const gatewayBinary = Object.prototype.hasOwnProperty.call(binaries, "gatewayBin") ? binaries.gatewayBin : deps.resolveOpenShellGatewayBinary(); const sandboxBinary = Object.prototype.hasOwnProperty.call(binaries, "sandboxBin") ? binaries.sandboxBin : deps.resolveOpenShellSandboxBinary(); - if (!gatewayBinary) return false; - if (platform === "linux" && !sandboxBinary) return false; + if (requirements.gateway && !gatewayBinary) return false; + if (requirements.sandbox && !sandboxBinary) return false; return true; } @@ -158,6 +178,15 @@ export function ensureOpenshellForOnboard(deps: OpenShellInstallDeps): OpenShell futureShellPathHint: null, }; const minOpenshellVersion = deps.getBlueprintMinOpenshellVersion() ?? "0.0.85"; + const binaryRequirements = + deps.getManagedGatewayBinaryRequirements?.(platform, arch) ?? + (deps.isLinuxDockerDriverGatewayEnabled(platform, arch) + ? { + driverLabel: "Docker", + gateway: true, + sandbox: platform === "linux", + } + : null); if (!deps.isOpenshellInstalled()) { deps.log(" openshell CLI not found. Installing..."); @@ -182,11 +211,11 @@ export function ensureOpenshellForOnboard(deps: OpenShellInstallDeps): OpenShell ignoreError: true, }); const needsDevChannel = - deps.isLinuxDockerDriverGatewayEnabled(platform, arch) && + binaryRequirements !== null && deps.shouldUseOpenshellDevChannel() && !deps.isOpenshellDevVersion(currentVersionOutput); const needsDockerDriverBinaries = - deps.isLinuxDockerDriverGatewayEnabled(platform, arch) && + binaryRequirements !== null && !areRequiredDockerDriverBinariesPresent(deps, platform, {}, arch); const needsMessagingFeatures = !deps.hasRequiredOpenshellMessagingFeatures(); const needsUpgrade = @@ -196,11 +225,13 @@ export function ensureOpenshellForOnboard(deps: OpenShellInstallDeps): OpenShell needsMessagingFeatures; if (needsUpgrade) { if (needsDevChannel) { - deps.log(" OpenShell Docker-driver onboarding requires the dev channel. Upgrading..."); + deps.log( + ` OpenShell ${binaryRequirements?.driverLabel ?? "managed"}-driver onboarding requires the dev channel. Upgrading...`, + ); } else if (needsDockerDriverBinaries) { - const required = platform === "linux" ? "gateway and sandbox" : "gateway"; + const required = binaryRequirements?.sandbox ? "gateway and sandbox" : "gateway"; deps.log( - ` OpenShell Docker-driver gateway onboarding requires the ${required} binaries. Reinstalling...`, + ` OpenShell ${binaryRequirements?.driverLabel ?? "managed"}-driver gateway onboarding requires the ${required} binaries. Reinstalling...`, ); } else if (needsMessagingFeatures) { deps.log( diff --git a/src/lib/onboard/preflight-runtime-selection.test.ts b/src/lib/onboard/preflight-runtime-selection.test.ts new file mode 100644 index 0000000000..1c2abdf045 --- /dev/null +++ b/src/lib/onboard/preflight-runtime-selection.test.ts @@ -0,0 +1,31 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; +import { assessHost } from "./preflight"; + +describe("host runtime probe selection", () => { + it("does not discover or invoke Docker when a qualified non-Docker runtime owns compute", () => { + const calls: string[][] = []; + const result = assessHost({ + platform: "linux", + env: {}, + skipDockerProbe: true, + dockerInfoOutput: JSON.stringify({ + ServerVersion: "29.3.1", + OperatingSystem: "Docker Engine", + }), + commandExistsImpl: () => false, + gpuProbeImpl: () => false, + runCaptureImpl: (command) => { + calls.push([...command]); + return ""; + }, + }); + + expect(result.dockerInstalled).toBe(false); + expect(result.dockerReachable).toBe(false); + expect(result.runtime).toBe("unknown"); + expect(calls.some((command) => command[0] === "docker")).toBe(false); + }); +}); diff --git a/src/lib/onboard/preflight.ts b/src/lib/onboard/preflight.ts index dc3137e62a..654cd8c937 100644 --- a/src/lib/onboard/preflight.ts +++ b/src/lib/onboard/preflight.ts @@ -177,6 +177,8 @@ export interface AssessHostOpts { runCaptureImpl?: RunCaptureFn; commandExistsImpl?: (commandName: string) => boolean; gpuProbeImpl?: () => boolean; + /** Do not discover or invoke Docker when another qualified runtime owns compute. */ + skipDockerProbe?: boolean; } function buildCommandVArgv(commandName: string): readonly string[] { @@ -602,7 +604,8 @@ export function assessHost(opts: AssessHostOpts = {}): HostAssessment { const readFileImpl = opts.readFileImpl ?? fs.readFileSync; const readdirImpl = opts.readdirImpl ?? ((dir: string) => fs.readdirSync(dir)); const dockerInstalled = - opts.commandExistsImpl?.("docker") ?? commandExists("docker", runCaptureImpl); + !opts.skipDockerProbe && + (opts.commandExistsImpl?.("docker") ?? commandExists("docker", runCaptureImpl)); const nodeInstalled = opts.commandExistsImpl?.("node") ?? commandExists("node", runCaptureImpl); const openshellInstalled = opts.commandExistsImpl?.("openshell") ?? commandExists("openshell", runCaptureImpl); @@ -613,7 +616,7 @@ export function assessHost(opts: AssessHostOpts = {}): HostAssessment { const systemctlAvailable = opts.commandExistsImpl?.("systemctl") ?? commandExists("systemctl", runCaptureImpl); - let dockerInfoOutput = opts.dockerInfoOutput; + let dockerInfoOutput = opts.skipDockerProbe ? undefined : opts.dockerInfoOutput; let dockerReachable = false; let dockerRunning = false; if (dockerInstalled && dockerInfoOutput === undefined) { diff --git a/src/lib/onboard/provider-host-state.test.ts b/src/lib/onboard/provider-host-state.test.ts index 82c995869a..de0bbbae9f 100644 --- a/src/lib/onboard/provider-host-state.test.ts +++ b/src/lib/onboard/provider-host-state.test.ts @@ -58,6 +58,37 @@ function detectWithDeps( } describe("detectInferenceProviderHostState", () => { + it("keeps remote-only runtime profiles off every local and Docker provider probe", () => { + const deps = buildDeps(); + + const state = detectInferenceProviderHostState({ + gpu: { nimCapable: true, type: "nvidia", platform: "linux" }, + experimental: true, + localInferenceEnabled: false, + platform: "linux", + env: {}, + log: () => {}, + deps, + }); + + expect(state).toMatchObject({ + hasOllama: false, + ollamaRunning: false, + vllmRunning: false, + hasVllmImage: false, + vllmEntries: [], + gpuNimCapable: false, + ollamaInstallMenu: { entry: null }, + }); + expect(deps.runCapture).not.toHaveBeenCalled(); + expect(deps.dockerCapture).not.toHaveBeenCalled(); + expect(deps.hostCommandExists).not.toHaveBeenCalled(); + expect(deps.findReachableOllamaHost).not.toHaveBeenCalled(); + expect(deps.detectVllmProfile).not.toHaveBeenCalled(); + expect(deps.detectWindowsHostOllama).not.toHaveBeenCalled(); + expect(deps.getContainerRuntime).not.toHaveBeenCalled(); + }); + it("suppresses local endpoint probes when route preflight disallows them (#6315)", () => { const runCapture = vi.fn(() => "{}"); const findReachableOllamaHost = vi.fn(() => "127.0.0.1"); diff --git a/src/lib/onboard/provider-host-state.ts b/src/lib/onboard/provider-host-state.ts index 4a41d5a8f8..6b56d26d35 100644 --- a/src/lib/onboard/provider-host-state.ts +++ b/src/lib/onboard/provider-host-state.ts @@ -63,6 +63,8 @@ export interface InferenceProviderHostState { export interface DetectInferenceProviderHostStateInput { gpu: InferenceProviderHostGpu | null | undefined; experimental: boolean; + /** False for runtime profiles that have not qualified host-local inference. */ + localInferenceEnabled?: boolean; probeOllama?: boolean; probeVllm?: boolean; platform?: NodeJS.Platform; @@ -174,6 +176,26 @@ export function detectInferenceProviderHostState( const deps = buildDeps(input.deps); const log = input.log ?? console.log; const platform = input.platform ?? process.platform; + if (input.localInferenceEnabled === false) { + return { + hasOllama: false, + ollamaHost: null, + ollamaRunning: false, + isWindowsHostOllama: false, + isWsl: false, + hasWindowsOllama: false, + winOllamaInstalledPath: "", + winOllamaLoopbackOnly: false, + windowsOllamaReachable: false, + windowsHostOllamaDockerRequirement: deps.getWindowsHostOllamaDockerRequirement(null), + vllmRunning: false, + vllmProfile: null, + hasVllmImage: false, + vllmEntries: [], + ollamaInstallMenu: { entry: null, hasUpgradableOllama: false }, + gpuNimCapable: false, + }; + } const isWsl = deps.isWsl({ platform, env: input.env }); const hasOllama = deps.hostCommandExists("ollama"); const ollamaHost = input.probeOllama === false ? null : deps.findReachableOllamaHost(); diff --git a/src/lib/onboard/sandbox-create-intent-types.ts b/src/lib/onboard/sandbox-create-intent-types.ts index 5621f6e19c..3e5e35e379 100644 --- a/src/lib/onboard/sandbox-create-intent-types.ts +++ b/src/lib/onboard/sandbox-create-intent-types.ts @@ -87,7 +87,7 @@ export type ResolveSandboxCreateIntentInput = { export type MaterializeSandboxCreatePlanInput = { intent: SandboxCreateIntent; - buildCtx: string; + fromRef: string; messagingTokenDefs: MessagingTokenDef[]; runProviderPreDeleteCleanup(): void; upsertMessagingProviders( diff --git a/src/lib/onboard/sandbox-create-launch.test.ts b/src/lib/onboard/sandbox-create-launch.test.ts index e687681e3f..297a3a8d68 100644 --- a/src/lib/onboard/sandbox-create-launch.test.ts +++ b/src/lib/onboard/sandbox-create-launch.test.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; @@ -10,17 +11,134 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { loadAgent } from "../agent/defs"; import { SANDBOX_BUILD_CONTEXT_PREFIX } from "../sandbox/build-context"; +import { + decodeManagedStartupProfile, + encodeManagedStartupProfile, + type ManagedStartupAgent, + type ManagedStartupProfile, +} from "./managed-startup/profile"; import { createOpenshellCliHelpers } from "./openshell-cli"; import { buildSandboxRuntimeEnvArgs, prepareSandboxCreateLaunch, prepareSandboxCreateLaunchWithPrebuild, + prepareSandboxCreateManagedImageLaunch, + SANDBOX_CREATE_MAX_ARGUMENT_BYTES, } from "./sandbox-create-launch"; const disabledHermesDashboardState = { config: null, enabled: false }; const IMAGE_ID = `sha256:${"a".repeat(64)}`; const temporaryBuildContexts: string[] = []; +function managedProfileForAgent( + agent: ManagedStartupAgent, + bundleSha256: string | null = null, +): ManagedStartupProfile { + const shared = { + schemaVersion: 1 as const, + inference: { + routeProvider: "inference", + upstreamProvider: "nvidia", + model: "nvidia/test-model", + routedBaseUrl: "https://inference.local/v1", + upstreamEndpointUrl: null, + api: "openai-completions" as const, + primaryModelRef: null, + compatibility: null, + inputModalities: null, + }, + proxy: { + managedHost: "host.openshell.internal", + managedPort: 3128, + hostHttpUrl: null, + hostHttpsUrl: null, + hostNoProxy: [], + }, + tools: { disclosure: "progressive" as const, enabledGateways: [] }, + messaging: { plan: null }, + tuning: { + contextWindow: null, + maxTokens: null, + reasoning: null, + reasoningEffort: null, + }, + corporateCa: { bundleSha256 }, + }; + switch (agent) { + case "openclaw": + return { + ...shared, + agent, + agentConfig: { + agent, + webSearch: { enabled: false, provider: "tavily" }, + otel: { + enabled: false, + endpointUrl: "http://host.openshell.internal:4318", + serviceName: "openclaw-gateway", + sampleRate: 1, + }, + agentTimeoutSeconds: 900, + heartbeatEvery: null, + extraAgents: { agents: [], defaults: {}, main: {} }, + deviceAuth: { disabled: true, optOutSource: "managed-onboard" }, + minimalBootstrap: false, + }, + inference: { + ...shared.inference, + primaryModelRef: "inference/nvidia/test-model", + inputModalities: ["text"], + }, + tuning: { + contextWindow: 131_072, + maxTokens: 8192, + reasoning: false, + reasoningEffort: "default", + }, + dashboard: { + agent, + mode: "loopback", + url: "http://127.0.0.1:18789", + port: 18_789, + bindAddress: "127.0.0.1", + wslExposure: false, + }, + }; + case "hermes": + return { + ...shared, + agent, + agentConfig: { + agent, + webSearch: { enabled: false, provider: "tavily" }, + }, + dashboard: { + agent, + mode: "disabled", + url: "http://127.0.0.1:19189", + publicPort: null, + internalPort: null, + tuiEnabled: false, + }, + }; + case "langchain-deepagents-code": + return { + ...shared, + agent, + agentConfig: { + agent, + autoApprovalMode: "disabled", + observabilityEnabled: false, + }, + inference: { + ...shared.inference, + upstreamEndpointUrl: "https://integrate.api.nvidia.com/v1", + }, + dashboard: { agent, mode: "disabled" }, + }; + } +} + function createTrustedBuildContext(): string { const buildCtx = fs.mkdtempSync(path.join(os.tmpdir(), SANDBOX_BUILD_CONTEXT_PREFIX)); temporaryBuildContexts.push(buildCtx); @@ -64,6 +182,185 @@ describe("buildSandboxRuntimeEnvArgs", () => { }); describe("prepareSandboxCreateLaunch", () => { + it.each([ + "openclaw", + "hermes", + "langchain-deepagents-code", + ])("forwards the managed startup handoff only for the explicit %s managed launch", (agentName) => { + const input = { + agent: loadAgent(agentName), + chatUiUrl: "", + createArgs: [ + "--from", + `ghcr.io/nvidia/nemoclaw/${agentName}-sandbox@${IMAGE_ID}`, + "--name", + "demo", + ], + env: {}, + extraPlaceholderKeys: [], + getDashboardForwardPort: () => "0", + hermesDashboardState: disabledHermesDashboardState, + manageDashboard: false, + openshellShellCommand: (args: string[]) => args.join(" "), + buildEnv: () => ({}), + }; + + const legacy = prepareSandboxCreateLaunch(input); + expect(legacy.envArgs.some((arg) => arg.startsWith("NEMOCLAW_STARTUP_PROFILE_B64="))).toBe( + false, + ); + expect(legacy.envArgs.some((arg) => arg.startsWith("NEMOCLAW_CORPORATE_CA_B64="))).toBe(false); + + const encodedProfile = encodeManagedStartupProfile( + managedProfileForAgent(agentName as ManagedStartupAgent), + ); + const managed = prepareSandboxCreateLaunch({ + ...input, + managedStartupProfile: { + encodedProfile, + }, + }); + expect(managed.envArgs.some((arg) => arg.startsWith("NEMOCLAW_STARTUP_PROFILE_B64="))).toBe( + false, + ); + expect(managed.envArgs.some((arg) => arg.startsWith("NEMOCLAW_CORPORATE_CA_B64="))).toBe(false); + expect(managed.managedStartupRootApplyRequest?.encodedProfile).toBe(encodedProfile); + expect(managed.sandboxStartupCommand).toEqual([ + "env", + ...managed.envArgs, + "/usr/local/bin/nemoclaw-managed-startup-hold", + "--agent", + agentName, + "--profile-fingerprint", + managed.managedStartupRootApplyRequest?.profileFingerprint, + ]); + expect(managed.createArgv.join("\n")).not.toContain(encodedProfile); + }); + + it.each([ + "openclaw", + "hermes", + ] as const)("injects exact bounded upper/lower authenticated proxy aliases for managed %s launch", (agentName) => { + const encodedProfile = encodeManagedStartupProfile(managedProfileForAgent(agentName)); + const managed = prepareSandboxCreateManagedImageLaunch({ + agent: loadAgent(agentName), + sandboxName: "demo", + chatUiUrl: "", + createArgs: ["--from", `example.test/${agentName}@${IMAGE_ID}`, "--name", "demo"], + env: { + HTTP_PROXY: "http://upper-http:upper-pass@upper-http.example.test:18080", + HTTPS_PROXY: "http://upper-https:upper-pass@upper-https.example.test:18443", + NO_PROXY: "upper.internal", + http_proxy: "http://lower-http:lower-pass@lower-http.example.test:28080", + https_proxy: "http://lower-https:lower-pass@lower-https.example.test:28443", + no_proxy: "lower.internal", + }, + extraPlaceholderKeys: [], + getDashboardForwardPort: () => "0", + hermesDashboardState: disabledHermesDashboardState, + manageDashboard: false, + openshellShellCommand: (args) => args.join(" "), + managedStartupProfile: { encodedProfile }, + }); + + expect(managed.envArgs).toEqual( + expect.arrayContaining([ + "HTTP_PROXY=http://upper-http:upper-pass@upper-http.example.test:18080", + "HTTPS_PROXY=http://upper-https:upper-pass@upper-https.example.test:18443", + "http_proxy=http://lower-http:lower-pass@lower-http.example.test:28080", + "https_proxy=http://lower-https:lower-pass@lower-https.example.test:28443", + expect.stringMatching(/^NO_PROXY=upper\.internal,localhost,/u), + expect.stringMatching(/^no_proxy=lower\.internal,localhost,/u), + ]), + ); + expect(managed.managedStartupRootApplyRequest?.encodedProfile).toBe(encodedProfile); + expect(managed.createArgv.join("\n")).not.toContain(encodedProfile); + expect(JSON.stringify(decodeManagedStartupProfile(encodedProfile))).not.toContain("upper-pass"); + expect(JSON.stringify(decodeManagedStartupProfile(encodedProfile))).not.toContain("lower-pass"); + }); + + it("rejects malformed managed startup transports before rendering the create command", () => { + const input = { + agent: loadAgent("openclaw"), + chatUiUrl: "", + createArgs: [], + env: {}, + extraPlaceholderKeys: [], + getDashboardForwardPort: () => "0", + hermesDashboardState: disabledHermesDashboardState, + manageDashboard: false, + openshellShellCommand: (args: string[]) => args.join(" "), + buildEnv: () => ({}), + }; + + expect(() => + prepareSandboxCreateLaunch({ + ...input, + managedStartupProfile: { encodedProfile: "not standard base64!" }, + }), + ).toThrow(/Invalid managed startup profile/u); + const encodedProfile = encodeManagedStartupProfile( + managedProfileForAgent("openclaw", "a".repeat(64)), + ); + expect(() => + prepareSandboxCreateLaunch({ + ...input, + managedStartupProfile: { + encodedProfile, + corporateCaB64: "not-base64", + }, + }), + ).toThrow(/corporate CA is not canonical bounded base64/u); + }); + + it("keeps the maximum corporate CA in bounded root stdin instead of create argv", () => { + const acceptedCaBytes = 128 * 1024; + const corporateCa = Buffer.alloc(acceptedCaBytes, 0x41); + const encodedProfile = encodeManagedStartupProfile( + managedProfileForAgent("openclaw", createHash("sha256").update(corporateCa).digest("hex")), + ); + const buildEnv = vi.fn(() => ({})); + const input = { + agent: loadAgent("openclaw"), + chatUiUrl: "", + createArgs: [ + "--from", + `ghcr.io/nvidia/nemoclaw/openclaw-sandbox@${IMAGE_ID}`, + "--name", + "demo", + ], + env: {}, + extraPlaceholderKeys: [], + getDashboardForwardPort: () => "0", + hermesDashboardState: disabledHermesDashboardState, + manageDashboard: false, + openshellShellCommand: (args: string[]) => args.join(" "), + buildEnv, + }; + const corporateCaB64 = corporateCa.toString("base64"); + const accepted = prepareSandboxCreateLaunch({ + ...input, + managedStartupProfile: { + encodedProfile, + corporateCaB64, + }, + }); + expect(accepted.managedStartupRootApplyRequest?.corporateCaB64).toBe(corporateCaB64); + expect(accepted.createArgv.join("\n")).not.toContain(corporateCaB64); + + buildEnv.mockClear(); + expect(() => + prepareSandboxCreateLaunch({ + ...input, + managedStartupProfile: { + encodedProfile, + corporateCaB64: Buffer.alloc(acceptedCaBytes + 3, 0x41).toString("base64"), + }, + }), + ).toThrow(/corporate CA is not canonical bounded base64/u); + expect(buildEnv).not.toHaveBeenCalled(); + }); + it("builds the sandbox create command and runtime env envelope", () => { const openshellShellCommand = vi.fn((args: string[]) => `openshell ${args.join(" ")}`); const result = prepareSandboxCreateLaunch({ @@ -356,6 +653,30 @@ describe("prepareSandboxCreateLaunch", () => { }); describe("prepareSandboxCreateLaunchWithPrebuild", () => { + it("launches an exact managed image without invoking a Dockerfile prebuild", async () => { + const reference = `ghcr.io/nvidia/nemoclaw/hermes-sandbox@${IMAGE_ID}`; + const result = prepareSandboxCreateManagedImageLaunch({ + agent: { name: "hermes" } as any, + chatUiUrl: "", + createArgs: ["--from", reference, "--name", "demo"], + env: {}, + extraPlaceholderKeys: [], + getDashboardForwardPort: () => "0", + hermesDashboardState: disabledHermesDashboardState, + manageDashboard: false, + openshellShellCommand: (args) => args.join(" "), + sandboxName: "demo", + buildEnv: () => ({}), + }); + + expect(result.prebuild).toEqual({ + createArgs: ["--from", reference, "--name", "demo"], + imageRef: null, + imageId: null, + }); + expect(result.createCommand).toContain(`sandbox create --from ${reference} --name demo`); + }); + it("hands the build-qualified image to the canonical launch renderer", async () => { const buildCtx = createTrustedBuildContext(); const dockerfile = path.join(buildCtx, "Dockerfile"); diff --git a/src/lib/onboard/sandbox-create-launch.ts b/src/lib/onboard/sandbox-create-launch.ts index 1e5431bb47..6980d0e1a0 100644 --- a/src/lib/onboard/sandbox-create-launch.ts +++ b/src/lib/onboard/sandbox-create-launch.ts @@ -9,7 +9,16 @@ import { appendExtraPlaceholderKeysEnvArg } from "./extra-placeholder-keys"; import type { HermesDashboardOnboardState } from "./hermes-dashboard"; import { appendHermesDashboardEnvArgs } from "./hermes-dashboard"; import { appendHostProxyEnvArgs } from "./host-proxy-env"; +import { MANAGED_STARTUP_HOLD_EXECUTABLE } from "./managed-startup/hold"; +import { + createManagedStartupRootApplyRequest, + type ManagedStartupRootApplyRequest, +} from "./managed-startup/root-apply"; import { appendOpenClawRuntimeEnvArgs } from "./openclaw-runtime-env"; +import { + assertSandboxCreateArgvWithinTransportLimit, + SANDBOX_CREATE_MAX_ARGUMENT_BYTES, +} from "./sandbox-create/transport"; import { prebuildSandboxImageIfEligible, type SandboxPrebuildInput, @@ -19,6 +28,13 @@ import { type OpenshellShellCommand = (args: string[]) => string; type OpenshellArgv = (args: string[]) => string[]; +export type SandboxWorkloadStartupRequirement = "sandbox-command" | "trusted-image-init"; + +export { + assertSandboxCreateArgvWithinTransportLimit, + SANDBOX_CREATE_MAX_ARGUMENT_BYTES, +} from "./sandbox-create/transport"; + // These non-secret scheduler controls are intentionally forwarded for bounded // live-test and operator tuning. Keep this as an exact allowlist: the host's // broader NEMOCLAW_* environment must not become sandbox runtime input. @@ -55,6 +71,10 @@ export interface SandboxCreateLaunchInput { openshellShellCommand: OpenshellShellCommand; openshellArgv?: OpenshellArgv; buildEnv?(): Record; + managedStartupProfile?: { + encodedProfile: string; + corporateCaB64?: string; + }; } export interface SandboxCreateLaunch { @@ -64,6 +84,8 @@ export interface SandboxCreateLaunch { envArgs: string[]; sandboxEnv: Record; sandboxStartupCommand: string[]; + startupRequirement: SandboxWorkloadStartupRequirement; + managedStartupRootApplyRequest: ManagedStartupRootApplyRequest | null; } export interface SandboxCreateLaunchWithPrebuildInput extends SandboxCreateLaunchInput { @@ -75,6 +97,10 @@ export interface SandboxCreateLaunchWithPrebuild extends SandboxCreateLaunch { prebuild: SandboxPrebuildResult; } +export type SandboxCreateManagedImageLaunchInput = SandboxCreateLaunchInput & { + sandboxName: string; +}; + export function renderSandboxCreateCommand( createArgs: readonly string[], sandboxStartupCommand: readonly string[], @@ -183,18 +209,31 @@ export function prepareSandboxCreateLaunch(input: SandboxCreateLaunchInput): San sandboxName: input.sandboxName, env, }); - - const sandboxEnv = (input.buildEnv ?? buildSubprocessEnv)(); - // Remove host-infrastructure credentials that the generic allowlist - // permits for host-side processes but that must not enter the sandbox. - delete sandboxEnv.KUBECONFIG; - delete sandboxEnv.SSH_AUTH_SOCK; + const managedStartupRootApplyRequest = input.managedStartupProfile + ? createManagedStartupRootApplyRequest({ + agent: (input.agent?.name ?? "openclaw") as ManagedStartupRootApplyRequest["agent"], + encodedProfile: input.managedStartupProfile.encodedProfile, + ...(input.managedStartupProfile.corporateCaB64 === undefined + ? {} + : { corporateCaB64: input.managedStartupProfile.corporateCaB64 }), + }) + : null; // Run without piping through awk; the pipe masked non-zero exit codes // from openshell because bash returns the status of the last pipeline // command (awk, always 0) unless pipefail is set. Removing the pipe // lets the real exit code flow through to run(). - const sandboxStartupCommand = ["env", ...envArgs, "nemoclaw-start"]; + const sandboxStartupCommand = managedStartupRootApplyRequest + ? [ + "env", + ...envArgs, + MANAGED_STARTUP_HOLD_EXECUTABLE, + "--agent", + managedStartupRootApplyRequest.agent, + "--profile-fingerprint", + managedStartupRootApplyRequest.profileFingerprint, + ] + : ["env", ...envArgs, "nemoclaw-start"]; const openshellArgs = ["sandbox", "create", ...input.createArgs, "--", ...sandboxStartupCommand]; const createCommand = renderSandboxCreateCommand( input.createArgs, @@ -204,6 +243,13 @@ export function prepareSandboxCreateLaunch(input: SandboxCreateLaunchInput): San const createArgv = input.openshellArgv ? input.openshellArgv(openshellArgs) : ["bash", "-lc", createCommand]; + assertSandboxCreateArgvWithinTransportLimit(createArgv); + + const sandboxEnv = (input.buildEnv ?? buildSubprocessEnv)(); + // Remove host-infrastructure credentials that the generic allowlist + // permits for host-side processes but that must not enter the sandbox. + delete sandboxEnv.KUBECONFIG; + delete sandboxEnv.SSH_AUTH_SOCK; return { createCommand, @@ -212,6 +258,8 @@ export function prepareSandboxCreateLaunch(input: SandboxCreateLaunchInput): San envArgs, sandboxEnv, sandboxStartupCommand, + startupRequirement: "sandbox-command", + managedStartupRootApplyRequest, }; } @@ -230,3 +278,19 @@ export async function prepareSandboxCreateLaunchWithPrebuild( prebuild, }; } + +/** Launch an immutable complete image without invoking any Dockerfile build path. */ +export function prepareSandboxCreateManagedImageLaunch( + input: SandboxCreateManagedImageLaunchInput, +): SandboxCreateLaunchWithPrebuild { + const prebuild: SandboxPrebuildResult = { + createArgs: [...input.createArgs], + imageRef: null, + imageId: null, + }; + return { + ...prepareSandboxCreateLaunch(input), + startupRequirement: "trusted-image-init", + prebuild, + }; +} diff --git a/src/lib/onboard/sandbox-create-plan-materialization.ts b/src/lib/onboard/sandbox-create-plan-materialization.ts index 44775f3851..46de184614 100644 --- a/src/lib/onboard/sandbox-create-plan-materialization.ts +++ b/src/lib/onboard/sandbox-create-plan-materialization.ts @@ -116,7 +116,7 @@ function filterDisabledMessagingProviders( /** Materialize policy, route metadata, resources, and providers from a secretless intent. */ export function materializeSandboxCreatePlan({ intent, - buildCtx, + fromRef, messagingTokenDefs, runProviderPreDeleteCleanup, upsertMessagingProviders, @@ -149,7 +149,7 @@ export function materializeSandboxCreatePlan({ } const createArgs = [ "--from", - `${buildCtx}/Dockerfile`, + fromRef, "--name", intent.sandboxName, "--policy", diff --git a/src/lib/onboard/sandbox-create-plan.test.ts b/src/lib/onboard/sandbox-create-plan.test.ts index 1b9c294ef5..0d00ba3b1a 100644 --- a/src/lib/onboard/sandbox-create-plan.test.ts +++ b/src/lib/onboard/sandbox-create-plan.test.ts @@ -85,7 +85,7 @@ function expectCredentialBindingFailure({ expect(() => materializeSandboxCreatePlan({ intent, - buildCtx: "/tmp/nemoclaw-build-1", + fromRef: "/tmp/nemoclaw-build-1/Dockerfile", messagingTokenDefs: materializedTokenDefs, prepareInitialSandboxCreatePolicy: preparePolicy, runProviderPreDeleteCleanup: cleanupProviders, @@ -257,7 +257,8 @@ describe("resolveSandboxCreateIntent", () => { const result = materializeSandboxCreatePlan({ intent, - buildCtx: "/tmp/nemoclaw-build-1", + fromRef: + "ghcr.io/nvidia/nemoclaw/hermes-sandbox@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", messagingTokenDefs: tokenDefs, prepareInitialSandboxCreatePolicy: vi.fn(() => { events.push("policy"); @@ -282,7 +283,7 @@ describe("resolveSandboxCreateIntent", () => { expect(events).toEqual(["policy", "disclose", "cleanup", "upsert", "hermes"]); expect(result.createArgs).toEqual([ "--from", - "/tmp/nemoclaw-build-1/Dockerfile", + "ghcr.io/nvidia/nemoclaw/hermes-sandbox@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "--name", "sandbox", "--policy", @@ -328,7 +329,7 @@ describe("resolveSandboxCreateIntent", () => { expect(() => materializeSandboxCreatePlan({ intent, - buildCtx: "/tmp/nemoclaw-build-1", + fromRef: "/tmp/nemoclaw-build-1/Dockerfile", messagingTokenDefs: [], prepareInitialSandboxCreatePolicy: vi.fn(() => ({ policyPath: "/tmp/policy.yaml", diff --git a/src/lib/onboard/sandbox-create-plan.ts b/src/lib/onboard/sandbox-create-plan.ts index b2e7a88d70..959011ec6f 100644 --- a/src/lib/onboard/sandbox-create-plan.ts +++ b/src/lib/onboard/sandbox-create-plan.ts @@ -154,7 +154,7 @@ export function prepareSandboxCreatePlan({ return materializeSandboxCreatePlan({ intent, - buildCtx, + fromRef: `${buildCtx}/Dockerfile`, messagingTokenDefs, runProviderPreDeleteCleanup, upsertMessagingProviders, diff --git a/src/lib/onboard/sandbox-create-runtime/podman.test.ts b/src/lib/onboard/sandbox-create-runtime/podman.test.ts new file mode 100644 index 0000000000..618f0989cf --- /dev/null +++ b/src/lib/onboard/sandbox-create-runtime/podman.test.ts @@ -0,0 +1,277 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import { + createPodmanOpenShellWatcherController, + type PodmanManagedSandboxRecreateTransaction, +} from "../compute/podman/sandbox-recreate"; +import type { PodmanManagedStartupTransaction } from "../managed-startup/podman-root-apply"; +import type { ManagedStartupRootApplyRequest } from "../managed-startup/root-apply"; +import { createPodmanSandboxCreatePatch } from "./podman"; + +const SOCKET_PATH = "/run/user/1000/podman/podman.sock"; +const SOCKET_AUTHORITY = { + directoryChain: [ + { + device: "8", + inode: "7000", + mode: "448", + ownerUid: "1000", + path: "/run/user/1000/podman", + }, + ], + device: "8", + inode: "9001", + ownerUid: "1000", + socketPath: SOCKET_PATH, +} as const; +const CONTAINER_ID = "b".repeat(64); +const IMAGE_ID = `sha256:${"c".repeat(64)}`; + +function watcherController() { + return createPodmanOpenShellWatcherController({ + assertStopped: vi.fn(), + resumeAndProve: vi.fn(), + stopAndProve: vi.fn(() => ({ stopped: true })), + }); +} + +function recreation(): PodmanManagedSandboxRecreateTransaction { + return { + applied: true, + driverName: "podman", + immutableImage: IMAGE_ID, + newContainerId: CONTAINER_ID, + socketAuthority: SOCKET_AUTHORITY, + socketPath: SOCKET_PATH, + } as unknown as PodmanManagedSandboxRecreateTransaction; +} + +function request(agent: ManagedStartupRootApplyRequest["agent"]): ManagedStartupRootApplyRequest { + return { agent } as ManagedStartupRootApplyRequest; +} + +function managedStartup( + agent: ManagedStartupRootApplyRequest["agent"], +): PodmanManagedStartupTransaction { + return { + agent, + containerId: CONTAINER_ID, + image: IMAGE_ID, + runtime: { + fingerprint: "f".repeat(64), + socketAuthority: SOCKET_AUTHORITY, + socketPath: SOCKET_PATH, + }, + }; +} + +function patchHarness(agent: ManagedStartupRootApplyRequest["agent"]) { + const events: string[] = []; + const transaction = recreation(); + const rootTransaction = managedStartup(agent); + const fail = vi.fn(); + const applyRoot = vi.fn(() => { + events.push(`root:${agent}`); + return rootTransaction; + }); + const recreate = vi.fn(() => { + events.push("recreate"); + return transaction; + }); + const waitForSupervisor = vi.fn(() => { + events.push("wait"); + return true; + }); + const finalizeSharedState = vi.fn((input) => { + events.push(`shared:${String(input.supervisorReady)}`); + return { failure: null, supervisorReady: input.supervisorReady }; + }); + const finalizeRecreation = vi.fn((input) => { + events.push(`container:${String(input.replacementReady)}`); + return input.replacementReady + ? { backupRemoved: true, rolledBack: false } + : { backupRemoved: false, rolledBack: true }; + }); + const assertSocketAuthority = vi.fn(); + const patch = createPodmanSandboxCreatePatch({ + managedStartupRootApplyRequest: request(agent), + openshellSandboxCommand: ["/usr/local/bin/node", "/agent/start.js"], + persistStartupCommand: true, + sandboxName: "alpha", + socketAuthority: SOCKET_AUTHORITY, + socketPath: SOCKET_PATH, + timeoutSecs: 60, + watcherController: watcherController(), + deps: { + runCaptureOpenshell: vi.fn(() => ""), + runOpenshell: vi.fn(() => ({ status: 0 })), + sleep: vi.fn(), + assertSocketAuthority, + }, + overrides: { + applyRoot, + fail, + finalizeRecreation, + finalizeSharedState, + findContainerIds: vi.fn(() => ["a".repeat(64)]), + recreate, + waitForSupervisor, + }, + }); + return { + applyRoot, + assertSocketAuthority, + events, + fail, + finalizeRecreation, + finalizeSharedState, + patch, + rootTransaction, + transaction, + waitForSupervisor, + }; +} + +describe("Podman sandbox-create runtime patch", () => { + it("revalidates its exact socket authority at an external mutation edge", () => { + const harness = patchHarness("openclaw"); + harness.patch.revalidateBeforeMutation(); + expect(harness.assertSocketAuthority).toHaveBeenCalledExactlyOnceWith(SOCKET_AUTHORITY); + expect(harness.events).toEqual([]); + }); + + it.each([ + "openclaw", + "hermes", + "langchain-deepagents-code", + ] as const)("commits %s only after recreation, root apply, and reconnect", (agent) => { + const harness = patchHarness(agent); + + harness.patch.maybeApplyDuringCreate(); + harness.patch.exitOnPatchError(); + harness.patch.waitForSupervisorReconnectIfNeeded(); + harness.patch.commitAfterReady(); + + expect(harness.events).toEqual([ + "recreate", + `root:${agent}`, + "wait", + "shared:true", + "container:true", + ]); + expect(harness.finalizeSharedState).toHaveBeenCalledWith( + { + containerRollbackAuthority: harness.transaction, + supervisorReady: true, + transaction: harness.rootTransaction, + }, + expect.objectContaining({ socketAuthority: SOCKET_AUTHORITY }), + ); + expect(harness.fail).not.toHaveBeenCalled(); + }); + + it("rolls shared state back before restoring the original container", () => { + const harness = patchHarness("hermes"); + harness.patch.maybeApplyDuringCreate(); + + harness.patch.rollbackManagedStartupAfterCreateFailure(); + + expect(harness.events).toEqual(["recreate", "root:hermes", "shared:false", "container:false"]); + expect(harness.fail).not.toHaveBeenCalled(); + }); + + it("uses the driver-neutral reconnect budget without Docker policy imports", () => { + const harness = patchHarness("openclaw"); + harness.patch.maybeApplyDuringCreate(); + + harness.patch.waitForSupervisorReconnectIfNeeded(); + + expect(harness.waitForSupervisor).toHaveBeenCalledWith( + "alpha", + 900, + expect.objectContaining({ + runCaptureOpenshell: expect.any(Function), + runOpenshell: expect.any(Function), + sleep: expect.any(Function), + }), + ); + }); + + it("refuses ambiguous discovery without starting a recreation", () => { + const fail = vi.fn(); + const recreate = vi.fn(); + const patch = createPodmanSandboxCreatePatch({ + openshellSandboxCommand: ["node", "agent.js"], + persistStartupCommand: true, + sandboxName: "alpha", + socketAuthority: SOCKET_AUTHORITY, + socketPath: SOCKET_PATH, + timeoutSecs: 60, + watcherController: watcherController(), + deps: { + runCaptureOpenshell: vi.fn(() => ""), + runOpenshell: vi.fn(() => ({ status: 0 })), + sleep: vi.fn(), + assertSocketAuthority: vi.fn(), + }, + overrides: { + fail, + findContainerIds: vi.fn(() => ["a".repeat(64), "b".repeat(64)]), + recreate, + }, + }); + + patch.maybeApplyDuringCreate(); + expect(patch.createFailureMessage()).toContain("Podman managed startup failed"); + patch.exitOnPatchError(); + + expect(recreate).not.toHaveBeenCalled(); + expect(fail).toHaveBeenCalledWith("alpha", expect.objectContaining({ name: "Error" })); + }); + + it("rolls back instead of committing before supervisor reconnect", () => { + const harness = patchHarness("langchain-deepagents-code"); + harness.patch.maybeApplyDuringCreate(); + + harness.patch.commitAfterReady(); + + expect(harness.events).toEqual([ + "recreate", + "root:langchain-deepagents-code", + "shared:false", + "container:false", + ]); + expect(harness.fail).toHaveBeenCalledWith( + "alpha", + expect.objectContaining({ message: expect.stringContaining("before supervisor reconnect") }), + ); + }); + + it("rolls back both transactions when shared-state commit throws", () => { + const harness = patchHarness("hermes"); + harness.finalizeSharedState.mockImplementationOnce((input) => { + harness.events.push(`shared:${String(input.supervisorReady)}`); + throw new Error("shared-state commit crashed"); + }); + harness.patch.maybeApplyDuringCreate(); + harness.patch.waitForSupervisorReconnectIfNeeded(); + + harness.patch.commitAfterReady(); + + expect(harness.events).toEqual([ + "recreate", + "root:hermes", + "wait", + "shared:true", + "shared:false", + "container:false", + ]); + expect(harness.fail).toHaveBeenCalledWith( + "alpha", + expect.objectContaining({ message: "shared-state commit crashed" }), + ); + }); +}); diff --git a/src/lib/onboard/sandbox-create-runtime/podman.ts b/src/lib/onboard/sandbox-create-runtime/podman.ts new file mode 100644 index 0000000000..a908085540 --- /dev/null +++ b/src/lib/onboard/sandbox-create-runtime/podman.ts @@ -0,0 +1,316 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + finalizePodmanManagedSandbox, + findPodmanManagedSandboxContainerIds, + type PodmanManagedSandboxRecreateDeps, + type PodmanManagedSandboxRecreateTransaction, + type PodmanOpenShellWatcherController, + recreatePodmanManagedSandbox, +} from "../compute/podman/sandbox-recreate"; +import { + assertPodmanSocketAuthority, + type PodmanSocketAuthority, +} from "../compute/podman/socket-authority"; +import { + applyPodmanManagedStartupRootRequest, + getPodmanManagedStartupFailureTransaction, + type PodmanManagedStartupTransaction, +} from "../managed-startup/podman-root-apply"; +import { finalizePodmanManagedStartupSharedState } from "../managed-startup/podman-shared-state"; +import type { ManagedStartupRootApplyRequest } from "../managed-startup/root-apply"; +import { + resolveSupervisorReconnectTimeoutSecs, + waitForSupervisorReconnect, +} from "./supervisor-reconnect"; +import type { SandboxCreateRuntimePatch } from "./types"; + +type RunOpenshell = ( + args: string[], + options?: Record, +) => { + readonly status?: number | null; + readonly stderr?: Buffer | string | null; + readonly stdout?: Buffer | string | null; +}; +type RunCaptureOpenshell = (args: string[], options?: { ignoreError?: boolean }) => string; +type WaitForSupervisor = typeof waitForSupervisorReconnect; + +export interface PodmanSandboxCreatePatchOptions { + readonly managedStartupRootApplyRequest?: ManagedStartupRootApplyRequest | null; + readonly openshellSandboxCommand: readonly string[]; + readonly persistStartupCommand: boolean; + readonly requiredUlimits?: + | readonly { + readonly hard: number; + readonly name: string; + readonly soft: number; + }[] + | null; + readonly sandboxName: string; + readonly socketAuthority: PodmanSocketAuthority; + readonly socketPath: string; + readonly timeoutSecs: number; + readonly watcherController: PodmanOpenShellWatcherController; + readonly deps: { + readonly runCaptureOpenshell: RunCaptureOpenshell; + readonly runOpenshell: RunOpenshell; + readonly sleep: (seconds: number) => void; + readonly assertSocketAuthority?: typeof assertPodmanSocketAuthority; + readonly runPodman?: PodmanManagedSandboxRecreateDeps["run"]; + }; + readonly overrides?: { + readonly applyRoot?: typeof applyPodmanManagedStartupRootRequest; + readonly fail?: (sandboxName: string, error: unknown) => void; + readonly finalizeRecreation?: typeof finalizePodmanManagedSandbox; + readonly finalizeSharedState?: typeof finalizePodmanManagedStartupSharedState; + readonly findContainerIds?: typeof findPodmanManagedSandboxContainerIds; + readonly recreate?: typeof recreatePodmanManagedSandbox; + readonly waitForSupervisor?: WaitForSupervisor; + }; +} + +function defaultFailure(sandboxName: string, error: unknown): never { + console.error(""); + console.error( + ` Podman managed-startup cutover failed for sandbox '${sandboxName}': ${ + error instanceof Error ? error.message : String(error) + }`, + ); + process.exit(1); +} + +/** + * Compose the Podman container recreation and managed-startup transactions + * behind the same create-stream lifecycle used by Docker. + */ +export function createPodmanSandboxCreatePatch( + options: PodmanSandboxCreatePatchOptions, +): SandboxCreateRuntimePatch { + const findContainerIds = + options.overrides?.findContainerIds ?? findPodmanManagedSandboxContainerIds; + const recreate = options.overrides?.recreate ?? recreatePodmanManagedSandbox; + const applyRoot = options.overrides?.applyRoot ?? applyPodmanManagedStartupRootRequest; + const finalizeSharedState = + options.overrides?.finalizeSharedState ?? finalizePodmanManagedStartupSharedState; + const finalizeRecreation = options.overrides?.finalizeRecreation ?? finalizePodmanManagedSandbox; + const waitForSupervisor = options.overrides?.waitForSupervisor ?? waitForSupervisorReconnect; + const fail = options.overrides?.fail ?? defaultFailure; + const proveSocketAuthority = options.deps.assertSocketAuthority ?? assertPodmanSocketAuthority; + const podmanDeps = { + assertSocketAuthority: proveSocketAuthority, + ...(options.deps.runPodman ? { run: options.deps.runPodman } : {}), + socketAuthority: options.socketAuthority, + }; + const patchEnabled = + options.persistStartupCommand || options.managedStartupRootApplyRequest != null; + + let recreation: PodmanManagedSandboxRecreateTransaction | null = null; + let managedStartup: PodmanManagedStartupTransaction | null = null; + let applied = false; + let finalized = false; + let needsSupervisorWait = false; + let patchError: unknown = null; + + const rollback = (): Error | null => { + if (finalized || (!managedStartup && !recreation)) return null; + try { + proveSocketAuthority(options.socketAuthority); + if (managedStartup) { + finalizeSharedState( + { + containerRollbackAuthority: recreation, + supervisorReady: false, + transaction: managedStartup, + }, + podmanDeps, + ); + managedStartup = null; + } + if (recreation) { + proveSocketAuthority(options.socketAuthority); + const outcome = finalizeRecreation( + { + replacementReady: false, + transaction: recreation, + watcherController: options.watcherController, + }, + podmanDeps, + ); + if (!outcome.rolledBack) { + throw new Error("Podman sandbox recreation rollback could not be proven."); + } + } + finalized = true; + needsSupervisorWait = false; + return null; + } catch (error) { + return error instanceof Error ? error : new Error(String(error)); + } + }; + + const failWithRollback = (error: unknown): void => { + const rollbackError = rollback(); + const primary = error instanceof Error ? error : new Error(String(error)); + fail( + options.sandboxName, + rollbackError + ? new Error(`${primary.message}; Podman rollback failed: ${rollbackError.message}`) + : primary, + ); + }; + + const apply = (): void => { + proveSocketAuthority(options.socketAuthority); + recreation = recreate( + { + command: options.openshellSandboxCommand, + ...(options.requiredUlimits ? { requiredUlimits: options.requiredUlimits } : {}), + sandboxName: options.sandboxName, + socketAuthority: options.socketAuthority, + socketPath: options.socketPath, + watcherController: options.watcherController, + }, + podmanDeps, + ); + proveSocketAuthority(options.socketAuthority); + needsSupervisorWait = true; + const request = options.managedStartupRootApplyRequest; + if (request) { + proveSocketAuthority(options.socketAuthority); + managedStartup = applyRoot( + { + containerId: recreation.newContainerId, + request, + socketAuthority: options.socketAuthority, + socketPath: options.socketPath, + }, + podmanDeps, + ); + proveSocketAuthority(options.socketAuthority); + } + applied = true; + }; + + const applyOrCaptureError = (): void => { + try { + apply(); + } catch (error) { + managedStartup ??= getPodmanManagedStartupFailureTransaction(error); + patchError = error; + } + }; + + return { + revalidateBeforeMutation() { + proveSocketAuthority(options.socketAuthority); + }, + + maybeApplyDuringCreate() { + if (!patchEnabled || applied || patchError) return; + proveSocketAuthority(options.socketAuthority); + const containerIds = findContainerIds(options.socketPath, options.sandboxName, podmanDeps); + if (containerIds.length === 0) return; + if (containerIds.length !== 1) { + patchError = new Error( + `Podman managed startup observed ${String( + containerIds.length, + )} matching containers and refused an ambiguous cutover.`, + ); + return; + } + applyOrCaptureError(); + }, + + createFailureMessage() { + return patchError + ? "Podman managed startup failed while OpenShell sandbox create was still waiting." + : null; + }, + + exitOnPatchError() { + if (patchError) failWithRollback(patchError); + }, + + rollbackManagedStartupAfterCreateFailure() { + const error = rollback(); + if (error) fail(options.sandboxName, error); + }, + + ensureApplied() { + if (!patchEnabled || applied) return; + applyOrCaptureError(); + if (patchError) failWithRollback(patchError); + }, + + waitForSupervisorReconnectIfNeeded() { + if (!needsSupervisorWait || finalized) return; + proveSocketAuthority(options.socketAuthority); + const timeoutSecs = resolveSupervisorReconnectTimeoutSecs(options.timeoutSecs); + const ready = waitForSupervisor(options.sandboxName, timeoutSecs, { + runOpenshell: options.deps.runOpenshell, + runCaptureOpenshell: options.deps.runCaptureOpenshell, + sleep: options.deps.sleep, + }); + if (ready) { + proveSocketAuthority(options.socketAuthority); + needsSupervisorWait = false; + return; + } + failWithRollback( + new Error("OpenShell supervisor did not reconnect to the recreated Podman container."), + ); + }, + + commitAfterReady() { + if (finalized || (!managedStartup && !recreation)) return; + if (needsSupervisorWait) { + failWithRollback( + new Error("Podman managed startup cannot commit before supervisor reconnect."), + ); + return; + } + if (managedStartup) { + proveSocketAuthority(options.socketAuthority); + let shared: ReturnType; + try { + shared = finalizeSharedState( + { + containerRollbackAuthority: recreation, + supervisorReady: true, + transaction: managedStartup, + }, + podmanDeps, + ); + } catch (error) { + failWithRollback(error); + return; + } + managedStartup = null; + if (!shared.supervisorReady || shared.failure) { + failWithRollback(shared.failure ?? new Error("Podman shared-state commit failed.")); + return; + } + } + if (recreation) { + proveSocketAuthority(options.socketAuthority); + const outcome = finalizeRecreation( + { replacementReady: true, transaction: recreation }, + podmanDeps, + ); + if (!outcome.backupRemoved) { + fail( + options.sandboxName, + new Error( + "Podman managed startup passed Ready, but its rollback backup could not be removed.", + ), + ); + return; + } + } + proveSocketAuthority(options.socketAuthority); + finalized = true; + }, + }; +} diff --git a/src/lib/onboard/sandbox-create-runtime/registry.test.ts b/src/lib/onboard/sandbox-create-runtime/registry.test.ts new file mode 100644 index 0000000000..d189de71d0 --- /dev/null +++ b/src/lib/onboard/sandbox-create-runtime/registry.test.ts @@ -0,0 +1,104 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; +import { CURRENT_OPEN_SHELL_COMPUTE_PLANS } from "../compute/plan"; +import { + CURRENT_SANDBOX_CREATE_RUNTIME_PATCH_ADAPTERS, + createSandboxCreateRuntimePatch, + type SandboxCreateRuntimePatchAdapterRegistry, + type SandboxCreateRuntimePatchRequest, +} from "./registry"; +import type { SandboxCreateRuntimePatch } from "./types"; + +function patch(): SandboxCreateRuntimePatch { + return { + commitAfterReady: vi.fn(), + createFailureMessage: vi.fn(() => null), + ensureApplied: vi.fn(), + exitOnPatchError: vi.fn(), + maybeApplyDuringCreate: vi.fn(), + revalidateBeforeMutation: vi.fn(), + rollbackManagedStartupAfterCreateFailure: vi.fn(), + waitForSupervisorReconnectIfNeeded: vi.fn(), + }; +} + +function request(driverName = "mxc"): SandboxCreateRuntimePatchRequest { + return { + driverName, + lifecycle: { + deps: { + runCaptureOpenshell: vi.fn(() => ""), + runOpenshell: vi.fn(() => ({ status: 0 })), + sleep: vi.fn(), + }, + openshellSandboxCommand: ["nemoclaw-start"], + persistStartupCommand: false, + requiredUlimits: null, + sandboxGpuEnabled: false, + sandboxName: "alpha", + timeoutSecs: 60, + }, + }; +} + +describe("sandbox-create runtime patch registry", () => { + it("covers every current compute driver and preserves Kubernetes direct creation", () => { + expect(Object.keys(CURRENT_SANDBOX_CREATE_RUNTIME_PATCH_ADAPTERS).sort()).toEqual( + Object.values(CURRENT_OPEN_SHELL_COMPUTE_PLANS) + .map(({ driverName }) => driverName) + .sort(), + ); + const direct = createSandboxCreateRuntimePatch(request("kubernetes")); + + direct.maybeApplyDuringCreate(); + direct.ensureApplied(); + direct.waitForSupervisorReconnectIfNeeded(); + direct.commitAfterReady(); + + expect(direct.createFailureMessage()).toBeNull(); + }); + + it("routes a future MXC adapter without changing create orchestration", () => { + const expected = patch(); + const create = vi.fn(() => expected); + const adapters: SandboxCreateRuntimePatchAdapterRegistry = { + mxc: { create, driverName: "mxc" }, + }; + const input = request(); + + expect(createSandboxCreateRuntimePatch(input, adapters)).toBe(expected); + expect(create).toHaveBeenCalledWith(input); + expect(input).not.toHaveProperty("docker"); + }); + + it("rejects managed startup on Kubernetes until a runtime adapter owns it", () => { + const base = request("kubernetes"); + const input = { + ...base, + lifecycle: { + ...base.lifecycle, + managedStartupRootApplyRequest: { agent: "openclaw" } as never, + }, + }; + expect(() => createSandboxCreateRuntimePatch(input)).toThrow( + "has no managed-startup runtime adapter", + ); + }); + + it("fails closed for an unregistered compute driver", () => { + expect(() => createSandboxCreateRuntimePatch(request("mxc"), {})).toThrow( + "has no sandbox-create runtime patch adapter", + ); + }); + + it("rejects a registry key whose adapter claims another driver", () => { + const adapters: SandboxCreateRuntimePatchAdapterRegistry = { + mxc: { create: vi.fn(() => patch()), driverName: "other" }, + }; + expect(() => createSandboxCreateRuntimePatch(request("mxc"), adapters)).toThrow( + "has no sandbox-create runtime patch adapter", + ); + }); +}); diff --git a/src/lib/onboard/sandbox-create-runtime/registry.ts b/src/lib/onboard/sandbox-create-runtime/registry.ts new file mode 100644 index 0000000000..0dcbf4b35e --- /dev/null +++ b/src/lib/onboard/sandbox-create-runtime/registry.ts @@ -0,0 +1,163 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { PodmanSandboxCreateRuntimeAuthority } from "../compute/podman/sandbox-create-authority"; +import type { ManagedStartupRootApplyRequest } from "../managed-startup/root-apply"; +import { createPodmanSandboxCreatePatch, type PodmanSandboxCreatePatchOptions } from "./podman"; +import type { SandboxCreateRuntimePatch } from "./types"; + +type RunOpenshell = ( + args: string[], + options?: Record, +) => { + readonly status?: number | null; + readonly stderr?: Buffer | string | null; + readonly stdout?: Buffer | string | null; +}; + +export interface SandboxCreateRuntimeLifecycleContext { + readonly managedStartupRootApplyRequest?: ManagedStartupRootApplyRequest | null; + readonly openshellSandboxCommand: readonly string[]; + readonly persistStartupCommand: boolean; + readonly requiredUlimits?: + | readonly { + readonly hard: number; + readonly name: string; + readonly soft: number; + }[] + | null; + readonly sandboxGpuEnabled: boolean; + readonly sandboxName: string; + readonly timeoutSecs: number; + readonly deps: { + readonly runCaptureOpenshell: (args: string[], options?: Record) => string; + readonly runOpenshell: RunOpenshell; + readonly sleep: (seconds: number) => void; + }; +} + +/** + * Driver-neutral request dispatched through the sandbox-create registry. + * + * Runtime authority remains opaque to the coordinator. Each registered + * adapter validates and narrows its own authority before mutation. + */ +export interface SandboxCreateRuntimePatchRequest { + readonly driverName: string; + readonly lifecycle: SandboxCreateRuntimeLifecycleContext; + readonly runtimeAuthority?: unknown; +} + +export interface SandboxCreateRuntimePatchAdapter { + readonly driverName: string; + create(request: SandboxCreateRuntimePatchRequest): SandboxCreateRuntimePatch; +} + +export type SandboxCreateRuntimePatchAdapterRegistry = Readonly< + Record +>; + +export function createDirectSandboxCreateRuntimePatch(): SandboxCreateRuntimePatch { + return { + commitAfterReady() {}, + createFailureMessage: () => null, + ensureApplied() {}, + exitOnPatchError() {}, + maybeApplyDuringCreate() {}, + revalidateBeforeMutation() {}, + rollbackManagedStartupAfterCreateFailure() {}, + waitForSupervisorReconnectIfNeeded() {}, + }; +} + +function podmanOptions(request: SandboxCreateRuntimePatchRequest): PodmanSandboxCreatePatchOptions { + const { lifecycle } = request; + if (lifecycle.sandboxGpuEnabled) { + throw new Error("Native Podman managed startup does not support sandbox GPU attachment."); + } + const authority = request.runtimeAuthority as Partial | null; + if ( + !authority || + typeof authority !== "object" || + typeof authority.socketPath !== "string" || + !authority.socketPath.trim() || + !authority.socketAuthority || + !authority.watcherController + ) { + throw new Error("Podman managed startup requires its qualified socket and watcher controller."); + } + const { runCaptureOpenshell, runOpenshell, sleep } = lifecycle.deps; + return { + managedStartupRootApplyRequest: lifecycle.managedStartupRootApplyRequest, + openshellSandboxCommand: lifecycle.openshellSandboxCommand, + persistStartupCommand: lifecycle.persistStartupCommand, + requiredUlimits: lifecycle.requiredUlimits, + sandboxName: lifecycle.sandboxName, + socketAuthority: authority.socketAuthority, + socketPath: authority.socketPath, + timeoutSecs: lifecycle.timeoutSecs, + watcherController: authority.watcherController, + deps: { + runCaptureOpenshell, + runOpenshell, + sleep, + }, + }; +} + +/** + * Build the in-tree registry while keeping the Docker patch owned by its + * caller. Non-Docker selections never construct or accept Docker GPU options. + */ +export function currentSandboxCreateRuntimePatchAdapters( + dockerPatch?: SandboxCreateRuntimePatch, +): SandboxCreateRuntimePatchAdapterRegistry { + return { + docker: { + driverName: "docker", + create: () => { + if (!dockerPatch) { + throw new Error("Docker sandbox-create runtime patch was not composed by its caller."); + } + return dockerPatch; + }, + }, + kubernetes: { + driverName: "kubernetes", + create: (request) => { + if ( + request.lifecycle.managedStartupRootApplyRequest != null || + request.lifecycle.persistStartupCommand || + (request.lifecycle.requiredUlimits?.length ?? 0) > 0 + ) { + throw new Error( + "Kubernetes direct sandbox creation has no managed-startup runtime adapter.", + ); + } + return createDirectSandboxCreateRuntimePatch(); + }, + }, + podman: { + driverName: "podman", + create: (request) => createPodmanSandboxCreatePatch(podmanOptions(request)), + }, + }; +} + +export const CURRENT_SANDBOX_CREATE_RUNTIME_PATCH_ADAPTERS = + currentSandboxCreateRuntimePatchAdapters(); + +export function createSandboxCreateRuntimePatch( + request: SandboxCreateRuntimePatchRequest, + adapters: SandboxCreateRuntimePatchAdapterRegistry = CURRENT_SANDBOX_CREATE_RUNTIME_PATCH_ADAPTERS, +): SandboxCreateRuntimePatch { + const adapter = Object.hasOwn(adapters, request.driverName) + ? adapters[request.driverName] + : undefined; + if (!adapter || adapter.driverName !== request.driverName) { + throw new Error( + `OpenShell compute driver '${request.driverName}' has no sandbox-create runtime patch adapter.`, + ); + } + return adapter.create(request); +} diff --git a/src/lib/onboard/sandbox-create-runtime/supervisor-reconnect.ts b/src/lib/onboard/sandbox-create-runtime/supervisor-reconnect.ts new file mode 100644 index 0000000000..086d301995 --- /dev/null +++ b/src/lib/onboard/sandbox-create-runtime/supervisor-reconnect.ts @@ -0,0 +1,140 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +const SUPERVISOR_RECONNECT_MIN_SECS = 900; +const SUPERVISOR_RECONNECT_COMMAND_TIMEOUT_MS = 30_000; +const SUPERVISOR_RECONNECT_ERROR_PHASE_DEFAULT_DEBOUNCE_POLLS = 60; +const SUPERVISOR_RECONNECT_POLL_INTERVAL_SECS = 2; + +const TERMINAL_SANDBOX_FAILURE_PHASES = new Set(["Error", "Failed", "CrashLoopBackOff"]); +const ANSI_RE = /\x1b\[[0-9;]*m/g; + +type RunResult = { + readonly status?: number | null; + readonly stderr?: Buffer | string | null; + readonly stdout?: Buffer | string | null; +}; + +type RunOpenshell = (args: string[], options?: Record) => RunResult; +type RunCaptureOpenshell = (args: string[], options?: Record) => string; + +export interface SupervisorReconnectDeps { + readonly runOpenshell?: RunOpenshell; + readonly runCaptureOpenshell?: RunCaptureOpenshell; + readonly sleep?: (seconds: number) => void; + readonly errorPhaseDebouncePolls?: number; +} + +export interface SupervisorReconnectPolicy { + readonly commandTimeoutMs?: number; + readonly defaultErrorPhaseDebouncePolls?: number; + readonly pollIntervalSecs?: number; +} + +function defaultSleep(seconds: number): void { + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, Math.max(0, seconds) * 1000); +} + +function positiveRounded(value: number, fallback: number): number { + return Number.isFinite(value) ? Math.max(1, Math.round(value)) : fallback; +} + +function parseSandboxListFailurePhase(output: string, sandboxName: string): string | null { + if (typeof output !== "string" || !output.includes(sandboxName)) return null; + for (const line of output.replace(ANSI_RE, "").split(/\r?\n/)) { + const columns = line.trim().split(/\s+/); + if (columns[0] === sandboxName) { + return columns.find((column) => TERMINAL_SANDBOX_FAILURE_PHASES.has(column)) ?? null; + } + } + return null; +} + +function sandboxListShowsFailurePhase( + sandboxName: string, + runCaptureOpenshell: RunCaptureOpenshell, + commandTimeoutMs: number, +): boolean { + try { + const list = runCaptureOpenshell(["sandbox", "list"], { + ignoreError: true, + suppressOutput: true, + timeout: commandTimeoutMs, + }); + return parseSandboxListFailurePhase(list, sandboxName) !== null; + } catch { + return false; + } +} + +/** + * Resolve the bounded supervisor reconnect window shared by runtime adapters. + * + * Runtime-specific callers may layer their own environment override on top of + * this neutral minimum without exposing that configuration to other drivers. + */ +export function resolveSupervisorReconnectTimeoutSecs( + sandboxReadyTimeoutSecs: number, + minimumSecs = SUPERVISOR_RECONNECT_MIN_SECS, +): number { + return Math.max( + positiveRounded(sandboxReadyTimeoutSecs, 1), + positiveRounded(minimumSecs, SUPERVISOR_RECONNECT_MIN_SECS), + ); +} + +/** + * Poll the OpenShell supervisor through the public sandbox lifecycle. + * + * Container-driver adapters own recreation and rollback. This helper observes + * only driver-neutral OpenShell signals: a successful sandbox exec and the + * transient lifecycle phase reported by `sandbox list`. + */ +export function waitForSupervisorReconnect( + sandboxName: string, + timeoutSecs: number, + deps: SupervisorReconnectDeps, + policy: SupervisorReconnectPolicy = {}, +): boolean { + if (!deps.runOpenshell) return true; + const sleep = deps.sleep ?? defaultSleep; + const commandTimeoutMs = positiveRounded( + policy.commandTimeoutMs ?? SUPERVISOR_RECONNECT_COMMAND_TIMEOUT_MS, + SUPERVISOR_RECONNECT_COMMAND_TIMEOUT_MS, + ); + const defaultErrorPhaseDebouncePolls = positiveRounded( + policy.defaultErrorPhaseDebouncePolls ?? + SUPERVISOR_RECONNECT_ERROR_PHASE_DEFAULT_DEBOUNCE_POLLS, + SUPERVISOR_RECONNECT_ERROR_PHASE_DEFAULT_DEBOUNCE_POLLS, + ); + const errorPhaseDebouncePolls = + deps.errorPhaseDebouncePolls == null || !Number.isFinite(deps.errorPhaseDebouncePolls) + ? defaultErrorPhaseDebouncePolls + : positiveRounded(deps.errorPhaseDebouncePolls, defaultErrorPhaseDebouncePolls); + const pollIntervalSecs = positiveRounded( + policy.pollIntervalSecs ?? SUPERVISOR_RECONNECT_POLL_INTERVAL_SECS, + SUPERVISOR_RECONNECT_POLL_INTERVAL_SECS, + ); + const deadline = Date.now() + positiveRounded(timeoutSecs, 1) * 1000; + let consecutiveFailurePolls = 0; + + while (Date.now() <= deadline) { + const result = deps.runOpenshell(["sandbox", "exec", "-n", sandboxName, "--", "true"], { + ignoreError: true, + suppressOutput: true, + timeout: commandTimeoutMs, + }); + if (result.status === 0) return true; + if ( + deps.runCaptureOpenshell && + sandboxListShowsFailurePhase(sandboxName, deps.runCaptureOpenshell, commandTimeoutMs) + ) { + consecutiveFailurePolls += 1; + if (consecutiveFailurePolls >= errorPhaseDebouncePolls) return false; + } else { + consecutiveFailurePolls = 0; + } + sleep(pollIntervalSecs); + } + return false; +} diff --git a/src/lib/onboard/sandbox-create-runtime/types.ts b/src/lib/onboard/sandbox-create-runtime/types.ts new file mode 100644 index 0000000000..70f0b03ad1 --- /dev/null +++ b/src/lib/onboard/sandbox-create-runtime/types.ts @@ -0,0 +1,21 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * Driver-neutral lifecycle used while OpenShell creates a sandbox. + * + * Docker, Podman, and future driver adapters implement this transaction + * boundary. Runtime-specific capabilities (including Docker GPU patching) are + * composed by their owning adapter/caller instead of leaking into this + * lifecycle contract. + */ +export interface SandboxCreateRuntimePatch { + revalidateBeforeMutation(): void; + maybeApplyDuringCreate(): void; + createFailureMessage(): string | null; + exitOnPatchError(): void; + rollbackManagedStartupAfterCreateFailure(): void; + ensureApplied(): void; + waitForSupervisorReconnectIfNeeded(): void; + commitAfterReady(): void; +} diff --git a/src/lib/onboard/sandbox-create-step.test.ts b/src/lib/onboard/sandbox-create-step.test.ts index ba26917eb0..8eb54c909c 100644 --- a/src/lib/onboard/sandbox-create-step.test.ts +++ b/src/lib/onboard/sandbox-create-step.test.ts @@ -24,6 +24,8 @@ function makeLaunch(overrides: Record = {}) { envArgs: [], sandboxEnv: { FOO: "bar" }, sandboxStartupCommand: ["run", "alpha"], + startupRequirement: "sandbox-command", + managedStartupRootApplyRequest: null, prebuild: { imageRef: "img:tag", createArgs: ["sandbox", "create", "alpha"] }, ...overrides, }; @@ -33,7 +35,13 @@ function makePatch() { return { maybeApplyDuringCreate: vi.fn(), createFailureMessage: vi.fn(() => null), + rollbackManagedStartupAfterCreateFailure: vi.fn(), ensureApplied: vi.fn(), + waitForSupervisorReconnectIfNeeded: vi.fn(), + commitAfterReady: vi.fn(), + selectedMode: vi.fn(() => null), + printReadinessFailureIfEnabled: vi.fn(), + verifyGpuOrExit: vi.fn(), }; } @@ -196,6 +204,48 @@ describe("runSandboxCreateStep", () => { ); }); + it.each([ + "openclaw", + "hermes", + "langchain-deepagents-code", + ] as const)("threads the root-apply request for managed %s Docker launch", async (agent) => { + const managedStartupRootApplyRequest = { + schemaVersion: 1, + agent, + encodedProfile: "profile", + profileFingerprint: "a".repeat(64), + corporateCaB64: null, + } as const; + const launch = makeLaunch({ + sandboxStartupCommand: ["env", "/usr/local/bin/nemoclaw-managed-startup-hold"], + startupRequirement: "trusted-image-init", + managedStartupRootApplyRequest, + }); + const patch = makePatch(); + const deps = makeDeps(launch, patch, { status: 0, output: "created" }); + + await runSandboxCreateStep( + makeContext({ + agent: { name: agent } as SandboxCreateStepContext["agent"], + prebuild: { + buildCtx: "/tmp/ctx", + buildId: "b1", + dockerDriverGateway: true, + origin: "generated", + }, + }), + deps, + ); + + expect(deps.createDockerGpuPatch).toHaveBeenCalledWith( + expect.objectContaining({ + persistStartupCommand: agent !== "openclaw", + managedStartupRootApplyRequest, + openshellSandboxCommand: launch.sandboxStartupCommand, + }), + ); + }); + it("separates readiness detection from GPU patch polling", async () => { const launch = makeLaunch(); const patch = makePatch(); diff --git a/src/lib/onboard/sandbox-create-step.ts b/src/lib/onboard/sandbox-create-step.ts index 04609be02c..7c68ee4896 100644 --- a/src/lib/onboard/sandbox-create-step.ts +++ b/src/lib/onboard/sandbox-create-step.ts @@ -76,6 +76,7 @@ export async function runSandboxCreateStep( createCommand, createArgv, effectiveDashboardPort, + managedStartupRootApplyRequest, prebuild, sandboxEnv, sandboxStartupCommand, @@ -101,6 +102,7 @@ export async function runSandboxCreateStep( const dockerGpuCreatePatch = deps.createDockerGpuPatch({ route: context.useDockerGpuPatch ? "compatibility" : "native", persistStartupCommand: startupCommandPatch.persistStartupCommand, + managedStartupRootApplyRequest, requiredUlimits: startupCommandPatch.requiredUlimits, sandboxName: context.sandboxName, gpuDevice: context.gpuDevice, diff --git a/src/lib/onboard/sandbox-create/transport.ts b/src/lib/onboard/sandbox-create/transport.ts new file mode 100644 index 0000000000..045b675c4b --- /dev/null +++ b/src/lib/onboard/sandbox-create/transport.ts @@ -0,0 +1,19 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * Linux rejects any single execve(2) argument at MAX_ARG_STRLEN (128 KiB on + * the minimum supported 4 KiB page size), independently of aggregate ARG_MAX. + * Keep an explicit 8 KiB reserve for wrappers and the terminating NUL. + */ +export const SANDBOX_CREATE_MAX_ARGUMENT_BYTES = 120 * 1024; + +export function assertSandboxCreateArgvWithinTransportLimit(createArgv: readonly string[]): void { + for (const argument of createArgv) { + if (Buffer.byteLength(argument, "utf8") + 1 > SANDBOX_CREATE_MAX_ARGUMENT_BYTES) { + throw new Error( + "Sandbox create launch exceeds the safe per-argument transport limit; reduce the managed startup profile or corporate CA bundle.", + ); + } + } +} diff --git a/src/lib/onboard/sandbox-gpu-create-flow.test.ts b/src/lib/onboard/sandbox-gpu-create-flow.test.ts index f11e1e2114..009917de9d 100644 --- a/src/lib/onboard/sandbox-gpu-create-flow.test.ts +++ b/src/lib/onboard/sandbox-gpu-create-flow.test.ts @@ -149,6 +149,8 @@ afterEach(resetGpuFlowMocks); describe("runSandboxGpuCreateFlow proof authorization", () => { it("does not retry compatibility when the native proof throws an exec/policy error (#6110)", async () => { const deps = createDeps(); + const patch = createPatch(); + mocks.createDockerGpuSandboxCreatePatch.mockReturnValue(patch); vi.mocked(deps.verifyDirectSandboxGpu).mockImplementation(() => { throw new Error("openshell sandbox exec denied by policy"); }); @@ -161,6 +163,8 @@ describe("runSandboxGpuCreateFlow proof authorization", () => { ["sandbox", "delete", "alpha"], expect.anything(), ); + expect(patch.rollbackManagedStartupAfterCreateFailure).toHaveBeenCalledOnce(); + expect(patch.commitAfterReady).not.toHaveBeenCalled(); }); it("does not let sandbox-controlled CUDA output authorize compatibility fallback (#6110)", async () => { @@ -184,6 +188,11 @@ describe("runSandboxGpuCreateFlow proof authorization", () => { it("retries structured nvidia-smi failure only when host config proves no GPU attachment (#6110)", async () => { const deps = createDeps(); + const nativePatch = createPatch(); + const compatibilityPatch = createPatch(); + mocks.createDockerGpuSandboxCreatePatch + .mockReturnValueOnce(nativePatch) + .mockReturnValueOnce(compatibilityPatch); vi.mocked(deps.verifyDirectSandboxGpu) .mockReturnValueOnce(NVIDIA_SMI_FAILED_PROOF) .mockReturnValue(VERIFIED_PROOF); @@ -199,6 +208,9 @@ describe("runSandboxGpuCreateFlow proof authorization", () => { ["sandbox", "delete", "alpha"], expect.objectContaining({ suppressOutput: true }), ); + expect(nativePatch.rollbackManagedStartupAfterCreateFailure).toHaveBeenCalledOnce(); + expect(nativePatch.commitAfterReady).not.toHaveBeenCalled(); + expect(compatibilityPatch.commitAfterReady).toHaveBeenCalledOnce(); }); it.each([ @@ -307,6 +319,30 @@ describe("runSandboxGpuCreateFlow native failure and readiness", () => { ); }); + it("threads managed root apply on native GPU without forcing recreation", async () => { + const input = createInput(); + input.persistStartupCommand = true; + input.managedStartupRootApplyRequest = { + schemaVersion: 1, + agent: "openclaw", + encodedProfile: "profile", + profileFingerprint: "a".repeat(64), + corporateCaB64: null, + }; + + await expect(runSandboxGpuCreateFlow(input, createDeps())).resolves.toMatchObject({ + route: "native", + }); + + expect(mocks.createDockerGpuSandboxCreatePatch).toHaveBeenCalledWith( + expect.objectContaining({ + route: "native", + persistStartupCommand: false, + managedStartupRootApplyRequest: input.managedStartupRootApplyRequest, + }), + ); + }); + it("applies exact required limits while preserving the native GPU route", async () => { const input = createInput(); input.persistStartupCommand = true; @@ -435,10 +471,59 @@ describe("runSandboxGpuCreateFlow native failure and readiness", () => { ); expect(mocks.enforceDockerGpuPatchPreserveNetwork).not.toHaveBeenCalled(); }); + + it("keeps rollback authority through Ready and GPU proof, then commits once", async () => { + const patch = createPatch(); + mocks.createDockerGpuSandboxCreatePatch.mockReturnValue(patch); + + await expect(runSandboxGpuCreateFlow(createInput(), createDeps())).resolves.toMatchObject({ + route: "native", + }); + + expect(patch.rollbackManagedStartupAfterCreateFailure).not.toHaveBeenCalled(); + expect(patch.commitAfterReady).toHaveBeenCalledOnce(); + expect(patch.ensureApplied.mock.invocationCallOrder[0]).toBeLessThan( + patch.waitForSupervisorReconnectIfNeeded.mock.invocationCallOrder[0] ?? 0, + ); + expect(patch.waitForSupervisorReconnectIfNeeded.mock.invocationCallOrder[0]).toBeLessThan( + mocks.waitForCreatedSandboxReadyWithTrace.mock.invocationCallOrder[0] ?? 0, + ); + expect(mocks.waitForCreatedSandboxReadyWithTrace.mock.invocationCallOrder[0]).toBeLessThan( + mocks.verifyGpuSandboxAccessAfterReady.mock.invocationCallOrder[0] ?? 0, + ); + expect(mocks.verifyGpuSandboxAccessAfterReady.mock.invocationCallOrder[0]).toBeLessThan( + patch.commitAfterReady.mock.invocationCallOrder[0] ?? 0, + ); + }); + + it("rolls back before terminal readiness cleanup and never commits", async () => { + const patch = createPatch(); + mocks.createDockerGpuSandboxCreatePatch.mockReturnValue(patch); + mockReadinessFailure(); + const deps = createDeps(); + mockExit(); + + await expect(runSandboxGpuCreateFlow(createInput(), deps)).rejects.toThrow("process.exit:1"); + + expect(patch.rollbackManagedStartupAfterCreateFailure).toHaveBeenCalledOnce(); + expect(patch.commitAfterReady).not.toHaveBeenCalled(); + const deleteCall = vi + .mocked(deps.runOpenshell) + .mock.calls.findIndex(([args]) => (args as string[])[1] === "delete"); + expect(deleteCall).toBeGreaterThanOrEqual(0); + expect(patch.rollbackManagedStartupAfterCreateFailure.mock.invocationCallOrder[0]).toBeLessThan( + vi.mocked(deps.runOpenshell).mock.invocationCallOrder[deleteCall] ?? 0, + ); + }); }); describe("runSandboxGpuCreateFlow fallback ordering", () => { it("retries readiness only for exact-container host runtime evidence (#6110)", async () => { + const nativePatch = createPatch(); + const compatibilityPatch = createPatch(); + mocks.createDockerGpuSandboxCreatePatch + .mockReturnValueOnce(nativePatch) + .mockReturnValueOnce(compatibilityPatch); mocks.waitForCreatedSandboxReadyWithTrace .mockReturnValueOnce({ ready: false, @@ -456,6 +541,9 @@ describe("runSandboxGpuCreateFlow fallback ordering", () => { }); expect(mocks.streamSandboxCreate).toHaveBeenCalledTimes(2); + expect(nativePatch.rollbackManagedStartupAfterCreateFailure).toHaveBeenCalledOnce(); + expect(nativePatch.commitAfterReady).not.toHaveBeenCalled(); + expect(compatibilityPatch.commitAfterReady).toHaveBeenCalledOnce(); }); it("streams native and compatibility attempts through direct argv without a shell (#6110)", async () => { diff --git a/src/lib/onboard/sandbox-gpu-create-flow.ts b/src/lib/onboard/sandbox-gpu-create-flow.ts index 98574891e0..dfc62f6c03 100644 --- a/src/lib/onboard/sandbox-gpu-create-flow.ts +++ b/src/lib/onboard/sandbox-gpu-create-flow.ts @@ -10,8 +10,10 @@ import type { DockerGpuPatchDeps, DockerUlimit } from "./docker-gpu-patch-types" import type { SelectedDockerGpuRoute } from "./docker-gpu-route"; import { renderCompatibilityFallbackCreateArgs } from "./docker-gpu-route"; import { adaptDockerGpuRouteForPatch } from "./docker-gpu-route-patch-adapter"; -import type { DockerGpuSandboxCreatePatch } from "./docker-gpu-sandbox-create"; +import type { DockerGpuSandboxCreateHooks } from "./docker-gpu-sandbox-create"; +import type { ManagedStartupRootApplyRequest } from "./managed-startup/root-apply"; import { isImmutableDockerImageId } from "./openshell-docker-sandbox-containers"; +import type { SandboxCreateRuntimePatch } from "./sandbox-create-runtime/types"; import * as sandboxGpuCreateAttempt from "./sandbox-gpu-create-attempt"; import { createSandboxGpuCreateAttemptRunner } from "./sandbox-gpu-create-run-attempt"; import type { SandboxGpuConfig } from "./sandbox-gpu-mode"; @@ -25,6 +27,7 @@ type RunCaptureOpenshell = NonNullable; export interface SandboxGpuCreateFlowInput { + computeDriver: string; sandboxName: string; provider: string; sandboxGpuConfig: SandboxGpuConfig; @@ -41,7 +44,9 @@ export interface SandboxGpuCreateFlowInput { restoreBackupPath: string | null; terminalAgent: boolean; persistStartupCommand?: boolean; + managedStartupRootApplyRequest?: ManagedStartupRootApplyRequest | null; requiredUlimits?: readonly DockerUlimit[] | null; + runtimeAuthority?: unknown; } export interface SandboxGpuCreateFlowDeps { @@ -54,7 +59,8 @@ export interface SandboxGpuCreateFlowDeps { export interface SandboxGpuCreateFlowResult { createResult: StreamSandboxCreateResult; - dockerGpuCreatePatch: DockerGpuSandboxCreatePatch; + gpuHooks: DockerGpuSandboxCreateHooks; + runtimePatch: SandboxCreateRuntimePatch; route: SelectedDockerGpuRoute; firstCreateOutput: string; /** Mutable tag/reference retained only for registry and image-GC bookkeeping. */ diff --git a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts index 726758cc0a..e48f57e869 100644 --- a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts +++ b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts @@ -12,13 +12,20 @@ import { cliName } from "./branding"; import { reportSandboxCreateFailure } from "./created-sandbox-failure"; import * as dockerGpuLocalInference from "./docker-gpu-local-inference"; import type { SelectedDockerGpuRoute } from "./docker-gpu-route"; -import { createDockerGpuSandboxCreatePatch } from "./docker-gpu-sandbox-create"; +import { + createDockerGpuSandboxCreatePatch, + type DockerGpuSandboxCreateHooks, +} from "./docker-gpu-sandbox-create"; import { type OpenShellDockerSandboxRuntimeSnapshotQuery, queryOpenShellDockerSandboxContainers, queryOpenShellDockerSandboxRuntimeSnapshot, } from "./openshell-docker-sandbox-containers"; import { printSandboxCreateFailureDiagnostics } from "./sandbox-create-failure"; +import { + createSandboxCreateRuntimePatch, + currentSandboxCreateRuntimePatchAdapters, +} from "./sandbox-create-runtime/registry"; import * as sandboxGpuCreateAttempt from "./sandbox-gpu-create-attempt"; import type { SandboxGpuCreateFlowDeps, @@ -41,6 +48,14 @@ export type SandboxGpuCreateAttemptState = { // Ready row. Require one confirmation poll before advancing to the GPU proof. const COMPATIBILITY_STABLE_READY_POLLS = 2; +function createDirectSandboxGpuHooks(sandboxName: string): DockerGpuSandboxCreateHooks { + return { + printReadinessFailureIfEnabled() {}, + selectedMode: () => null, + verifyGpuOrExit: (verifyDirectSandboxGpu) => verifyDirectSandboxGpu(sandboxName), + }; +} + export function createSandboxGpuCreateAttemptRunner( input: SandboxGpuCreateFlowInput, deps: SandboxGpuCreateFlowDeps, @@ -70,21 +85,49 @@ export function createSandboxGpuCreateAttemptRunner( ); } const hasRequiredUlimits = (input.requiredUlimits?.length ?? 0) > 0; - const dockerGpuCreatePatch = createDockerGpuSandboxCreatePatch({ - route, + const persistStartupCommand = + input.persistStartupCommand === true && // The startup clone preserves native CDI devices, so DCode can apply its // exact required limits without replacing the native GPU envelope. - // Other native routes are not swapped solely to persist a command. - persistStartupCommand: - input.persistStartupCommand === true && (route !== "native" || hasRequiredUlimits), - sandboxName: input.sandboxName, - gpuDevice: input.sandboxGpuConfig.sandboxGpuDevice, - openshellSandboxCommand: input.sandboxStartupCommand, - requiredUlimits: input.requiredUlimits, - timeoutSecs: input.sandboxReadyTimeoutSecs, - backend: input.sandboxGpuConfig.hostGpuPlatform === "jetson" ? "jetson" : "generic", - deps, - }); + // Other native Docker routes are not swapped solely to persist a command. + (input.computeDriver !== "docker" || route !== "native" || hasRequiredUlimits); + const dockerPatch = + input.computeDriver === "docker" + ? createDockerGpuSandboxCreatePatch({ + route, + persistStartupCommand, + managedStartupRootApplyRequest: input.managedStartupRootApplyRequest, + sandboxName: input.sandboxName, + gpuDevice: input.sandboxGpuConfig.sandboxGpuDevice, + openshellSandboxCommand: input.sandboxStartupCommand, + requiredUlimits: input.requiredUlimits, + timeoutSecs: input.sandboxReadyTimeoutSecs, + backend: input.sandboxGpuConfig.hostGpuPlatform === "jetson" ? "jetson" : "generic", + deps, + }) + : undefined; + const runtimePatch = createSandboxCreateRuntimePatch( + { + driverName: input.computeDriver, + lifecycle: { + managedStartupRootApplyRequest: input.managedStartupRootApplyRequest, + openshellSandboxCommand: input.sandboxStartupCommand, + persistStartupCommand, + requiredUlimits: input.requiredUlimits, + sandboxGpuEnabled: input.sandboxGpuConfig.sandboxGpuEnabled, + sandboxName: input.sandboxName, + timeoutSecs: input.sandboxReadyTimeoutSecs, + deps: { + runCaptureOpenshell: deps.runCaptureOpenshell, + runOpenshell: deps.runOpenshell, + sleep: deps.sleep, + }, + }, + runtimeAuthority: input.runtimeAuthority, + }, + currentSandboxCreateRuntimePatchAdapters(dockerPatch), + ); + const gpuHooks = dockerPatch ?? createDirectSandboxGpuHooks(input.sandboxName); const attemptArgv = state.compatibilityArgv ?? input.createArgv; const [createExecutable, ...createExecutableArgs] = attemptArgv; if (!createExecutable) throw new Error("Sandbox create executable is missing."); @@ -97,12 +140,12 @@ export function createSandboxGpuCreateAttemptRunner( const list = deps.runCaptureOpenshell(["sandbox", "list"], { ignoreError: true }); return isSandboxReady(list, input.sandboxName); }, - onPoll: () => dockerGpuCreatePatch.maybeApplyDuringCreate(), + onPoll: () => runtimePatch.maybeApplyDuringCreate(), readyCheckOutputPatterns: getReadyCheckOutputPatternsForAgent( input.terminalAgent, input.sandboxEnv, ), - failureCheck: dockerGpuCreatePatch.createFailureMessage, + failureCheck: runtimePatch.createFailureMessage, traceEvent: addTraceEvent, initialPhase: compatibility && (input.prebuild.imageRef || state.compatibilityArgv) @@ -111,7 +154,7 @@ export function createSandboxGpuCreateAttemptRunner( }, ); if (!state.firstCreateOutput) state.firstCreateOutput = createResult.output; - dockerGpuCreatePatch.exitOnPatchError(); + runtimePatch.exitOnPatchError(); if (createResult.status !== 0) { const failure = classifySandboxCreateFailure(createResult.output); if (failure.kind === "sandbox_create_incomplete") { @@ -144,6 +187,7 @@ export function createSandboxGpuCreateAttemptRunner( return false; })() ) { + runtimePatch.rollbackManagedStartupAfterCreateFailure(); return { ok: false, route, @@ -152,6 +196,7 @@ export function createSandboxGpuCreateAttemptRunner( fallbackEligible: true, } as const; } else { + runtimePatch.rollbackManagedStartupAfterCreateFailure(); reportSandboxCreateFailure( { sandboxName: input.sandboxName, @@ -171,8 +216,8 @@ export function createSandboxGpuCreateAttemptRunner( ); } } - dockerGpuCreatePatch.ensureApplied(); - dockerGpuCreatePatch.waitForSupervisorReconnectIfNeeded(); + runtimePatch.ensureApplied(); + runtimePatch.waitForSupervisorReconnectIfNeeded(); console.log(" Waiting for sandbox to become ready..."); const readiness = sandboxReadinessTracing.waitForCreatedSandboxReadyWithTrace({ sandboxName: input.sandboxName, @@ -204,6 +249,7 @@ export function createSandboxGpuCreateAttemptRunner( }) ) { state.nativeRuntimeSnapshot = runtimeSnapshot; + runtimePatch.rollbackManagedStartupAfterCreateFailure(); return { ok: false, route, @@ -214,10 +260,11 @@ export function createSandboxGpuCreateAttemptRunner( fallbackEligible: true, } as const; } + runtimePatch.rollbackManagedStartupAfterCreateFailure(); printSandboxCreateFailureDiagnostics(input.sandboxName, { backupPath: input.restoreBackupPath, }); - if (compatibility) dockerGpuCreatePatch.printReadinessFailureIfEnabled(); + if (compatibility) gpuHooks.printReadinessFailureIfEnabled(); else { const deletion = deps.runOpenshell(["sandbox", "delete", input.sandboxName], { ignoreError: true, @@ -240,27 +287,29 @@ export function createSandboxGpuCreateAttemptRunner( route === "native" && input.gpuRoutePlan === "native-with-fallback" && nativeFallbackHasCleanBaseline; - const proof: SandboxGpuProofResult = dockerGpuLocalInference.verifyGpuSandboxAccessAfterReady( - input.sandboxGpuConfig, - { + let proof: SandboxGpuProofResult; + try { + proof = dockerGpuLocalInference.verifyGpuSandboxAccessAfterReady(input.sandboxGpuConfig, { sandboxName: input.sandboxName, dockerDriverGateway: input.dockerDriverGateway, selectedRoute: route, verifyDirectSandboxGpu: deps.verifyDirectSandboxGpu, - verifyGpuOrExit: deferNativeProofFailure - ? undefined - : dockerGpuCreatePatch.verifyGpuOrExit, + verifyGpuOrExit: deferNativeProofFailure ? undefined : gpuHooks.verifyGpuOrExit, reportGpuProofFailure: !deferNativeProofFailure, - selectedMode: dockerGpuCreatePatch.selectedMode, + selectedMode: gpuHooks.selectedMode, runCaptureOpenshell: deps.runCaptureOpenshell, log: console.log, - }, - ); + }); + } catch (error) { + runtimePatch.rollbackManagedStartupAfterCreateFailure(); + throw error; + } if (deferNativeProofFailure && proof.status === "failed") { if (sandboxGpuPreflight.isExplicitNvidiaSmiDriverProofFailure(proof)) { const snapshot = inspectNativeRuntime(); if (snapshot.ok && snapshot.nativeGpuAttachmentState === "absent") { state.nativeRuntimeSnapshot = snapshot; + runtimePatch.rollbackManagedStartupAfterCreateFailure(); return { ok: false, route, @@ -272,6 +321,7 @@ export function createSandboxGpuCreateAttemptRunner( } as const; } } + runtimePatch.rollbackManagedStartupAfterCreateFailure(); console.error(""); console.error(" Native sandbox GPU proof failed."); console.error( @@ -283,13 +333,15 @@ export function createSandboxGpuCreateAttemptRunner( process.exit(1); } if (proof.status === "failed") { + runtimePatch.rollbackManagedStartupAfterCreateFailure(); throw new Error("Sandbox GPU proof returned failed status."); } } + runtimePatch.commitAfterReady(); return { ok: true, route, - value: { createResult, dockerGpuCreatePatch }, + value: { createResult, gpuHooks, runtimePatch }, } as const; }; diff --git a/src/lib/onboard/sandbox-registration.ts b/src/lib/onboard/sandbox-registration.ts index 86bfe430a1..a7b11c7d1f 100644 --- a/src/lib/onboard/sandbox-registration.ts +++ b/src/lib/onboard/sandbox-registration.ts @@ -16,6 +16,7 @@ import type { SandboxMessagingState, } from "../state/registry"; import * as registry from "../state/registry"; +import { cloneSandboxWorkloadReceipt } from "../state/registry/workload"; import { DEFAULT_TOOL_DISCLOSURE, type ToolDisclosure } from "../tool-disclosure"; import type { DcodeAutoApprovalMode } from "./dcode-auto-approval"; import { @@ -43,6 +44,7 @@ export interface CreatedSandboxRegistryEntryInput { agent: AgentDefinition | null | undefined; agentVersionKnown: boolean; imageTag: string | null; + workload?: SandboxEntry["workload"]; openclawImagePluginInstalls?: readonly OpenClawImagePluginInstall[]; appliedPolicies: string[]; toolDisclosure?: ToolDisclosure; @@ -173,6 +175,7 @@ export function buildCreatedSandboxRegistryEntry( ...input.runtimeFields, ...getSandboxAgentRegistryFields(input.agent, input.agentVersionKnown), imageTag: input.imageTag, + workload: cloneSandboxWorkloadReceipt(input.workload), ...(input.openclawImagePluginInstalls !== undefined ? { openclawImagePluginInstalls: input.openclawImagePluginInstalls.map((install) => ({ diff --git a/src/lib/onboard/sandbox-registry-metadata.test.ts b/src/lib/onboard/sandbox-registry-metadata.test.ts index 055deb69d8..d6798a13fd 100644 --- a/src/lib/onboard/sandbox-registry-metadata.test.ts +++ b/src/lib/onboard/sandbox-registry-metadata.test.ts @@ -8,34 +8,16 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import type { AgentDefinition } from "../agent/defs"; import type { SandboxGpuConfig } from "./sandbox-gpu-mode"; -const ORIGINAL_PLATFORM = Object.getOwnPropertyDescriptor(process, "platform"); - -/** - * Overrides process.platform for runtime-driver metadata tests. - */ -function setPlatform(platform: NodeJS.Platform): void { - Object.defineProperty(process, "platform", { value: platform, configurable: true }); -} - /** - * Restores the original process.platform descriptor after each platform-specific assertion. + * Loads the compiled metadata helpers with an explicit resolved compute driver. */ -function restorePlatform(): void { - if (ORIGINAL_PLATFORM) { - Object.defineProperty(process, "platform", ORIGINAL_PLATFORM); - } -} - -/** - * Loads the compiled metadata helpers after each test has configured process state. - */ -async function makeHelpers(opts: { dockerDriverEnabled: boolean }) { +async function makeHelpers(driverName: string) { // Import the compiled module: sandbox-registry-metadata.ts pulls in state/registry, // which transitively requires the JS-only `./platform` helper that vitest cannot // resolve from TS source. Same pattern as `vm-dns-monkeypatch.test.ts`. const metadata = await import("./sandbox-registry-metadata"); return metadata.createSandboxRegistryMetadataHelpers({ - isLinuxDockerDriverGatewayEnabled: () => opts.dockerDriverEnabled, + getOpenShellComputeDriverName: () => driverName, getInstalledOpenshellVersion: () => "0.0.42", runCaptureOpenshell: () => null, }); @@ -106,7 +88,7 @@ describe("sandbox registry metadata", () => { }); const helpers = metadata.createSandboxRegistryMetadataHelpers({ - isLinuxDockerDriverGatewayEnabled: () => true, + getOpenShellComputeDriverName: () => "docker", getInstalledOpenshellVersion: () => "0.0.44", runCaptureOpenshell: () => "openshell 0.0.44", }); @@ -155,7 +137,7 @@ describe("sandbox registry metadata", () => { const dashboardPorts = await import("./dashboard-port"); const gatewayRegistry = await import("../state/gateway-registry"); const helpers = metadata.createSandboxRegistryMetadataHelpers({ - isLinuxDockerDriverGatewayEnabled: () => true, + getOpenShellComputeDriverName: () => "docker", getInstalledOpenshellVersion: () => "0.0.44", runCaptureOpenshell: () => "openshell 0.0.44", }); @@ -190,32 +172,44 @@ describe("sandbox registry metadata", () => { }); describe("getSandboxRuntimeRegistryFields openshellDriver", () => { - afterEach(restorePlatform); - - it("records Docker for macOS sandboxes on the Docker-driver gateway path", async () => { - setPlatform("darwin"); - const helpers = await makeHelpers({ dockerDriverEnabled: true }); + it("records the resolved Docker compute driver (#7744)", async () => { + const helpers = await makeHelpers("docker"); const fields = helpers.getSandboxRuntimeRegistryFields(GPU_OFF); expect(fields.openshellDriver).toBe("docker"); }); - it("records Docker for Linux sandboxes on the Docker-driver gateway path", async () => { - setPlatform("linux"); - const helpers = await makeHelpers({ dockerDriverEnabled: true }); + it("records the resolved Kubernetes compute driver (#7744)", async () => { + const helpers = await makeHelpers("kubernetes"); const fields = helpers.getSandboxRuntimeRegistryFields(GPU_OFF); - expect(fields.openshellDriver).toBe("docker"); + expect(fields.openshellDriver).toBe("kubernetes"); }); - it("records Kubernetes for legacy Linux sandboxes when the Docker-driver gateway is disabled", async () => { - setPlatform("linux"); - const helpers = await makeHelpers({ dockerDriverEnabled: false }); + it.each([ + "podman", + "mxc", + ])("passes the resolved %s driver through to registry metadata (#7744)", async (driverName) => { + const helpers = await makeHelpers(driverName); const fields = helpers.getSandboxRuntimeRegistryFields(GPU_OFF); - expect(fields.openshellDriver).toBe("kubernetes"); + expect(fields.openshellDriver).toBe(driverName); + }); + + it("resolves driver identity when metadata is recorded rather than at module load (#7744)", async () => { + const metadata = await import("./sandbox-registry-metadata"); + let driverName = "docker"; + const helpers = metadata.createSandboxRegistryMetadataHelpers({ + getOpenShellComputeDriverName: () => driverName, + getInstalledOpenshellVersion: () => "0.0.42", + runCaptureOpenshell: () => null, + }); + + driverName = "kubernetes"; + + expect(helpers.getSandboxRuntimeRegistryFields(GPU_OFF).openshellDriver).toBe("kubernetes"); }); }); diff --git a/src/lib/onboard/sandbox-registry-metadata.ts b/src/lib/onboard/sandbox-registry-metadata.ts index 9abb49a5ec..57b82000d8 100644 --- a/src/lib/onboard/sandbox-registry-metadata.ts +++ b/src/lib/onboard/sandbox-registry-metadata.ts @@ -8,7 +8,7 @@ import { getSandboxAgentRegistryFields } from "./sandbox-agent"; import type { SandboxGpuConfig } from "./sandbox-gpu-mode"; export interface SandboxRegistryMetadataDeps { - isLinuxDockerDriverGatewayEnabled(): boolean; + getOpenShellComputeDriverName(): string; getInstalledOpenshellVersion(versionOutput?: string | null): string | null; runCaptureOpenshell(args: string[], opts?: Record): string | null; } @@ -55,10 +55,6 @@ export function createSandboxRegistryMetadataHelpers( | "openshellDriver" | "openshellVersion" > { - // OpenShell's Docker-driver gateway always starts with OPENSHELL_DRIVERS=docker, - // including on macOS arm64 (#3454). Recording "vm" for darwin here makes later - // setup misclassify the sandbox and run VM-only DNS monkeypatch / warning paths - // (#3728). return { gpuEnabled: config.sandboxGpuEnabled, hostGpuDetected: config.hostGpuDetected, @@ -68,7 +64,9 @@ export function createSandboxRegistryMetadataHelpers( // Only persist a proof when this run produced one; omit on reuse/update // paths so a prior proof result is preserved rather than nulled out. ...(config.sandboxGpuProof ? { sandboxGpuProof: config.sandboxGpuProof } : {}), - openshellDriver: deps.isLinuxDockerDriverGatewayEnabled() ? "docker" : "kubernetes", + // Driver identity comes from the resolved compute plan, not the host + // gateway launcher; those layers may differ (#7744). + openshellDriver: deps.getOpenShellComputeDriverName(), openshellVersion: deps.getInstalledOpenshellVersion( deps.runCaptureOpenshell(["--version"], { ignoreError: true }), ), diff --git a/src/lib/onboard/sandbox-workload-preparation.test.ts b/src/lib/onboard/sandbox-workload-preparation.test.ts new file mode 100644 index 0000000000..261fee85bb --- /dev/null +++ b/src/lib/onboard/sandbox-workload-preparation.test.ts @@ -0,0 +1,327 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import { + MANAGED_IMAGE_CAPABILITY_CONTRACT_VERSION, + MANAGED_IMAGE_CONTRACT_VERSION, + MANAGED_IMAGE_PLATFORM, + MANAGED_IMAGE_REPOSITORIES, + MANAGED_IMAGE_SOURCE_REPOSITORY, + MANAGED_IMAGE_STARTUP_PROFILE_CONTRACT_VERSION, + type ManagedImageContractCatalog, + type ManagedImageContractV1, + SHIPPED_MANAGED_IMAGE_AGENTS, + type ShippedManagedImageAgent, +} from "./managed-image/contract"; +import { + prepareSandboxWorkloadSource, + SandboxWorkloadPreparationError, +} from "./workload/preparation"; +import { resolveSandboxWorkloadRuntimeCapabilities } from "./workload/runtime"; +import type { SandboxWorkloadRuntimeCapabilities } from "./workload/source"; + +const RELEASE = "v0.0.97"; +const REVISION = "2f03907c37822ea6f1ac9d1bf5c82a4a4568585f"; +const COHORT = "ghrun-7744-2"; + +function contract(agent: ShippedManagedImageAgent, index: number): ManagedImageContractV1 { + const image = MANAGED_IMAGE_REPOSITORIES[agent]; + const digest = `sha256:${String(index + 1).repeat(64)}` as const; + return { + contractVersion: MANAGED_IMAGE_CONTRACT_VERSION, + agent, + platform: MANAGED_IMAGE_PLATFORM, + image, + digest, + reference: `${image}@${digest}`, + source: { + repository: MANAGED_IMAGE_SOURCE_REPOSITORY, + revision: REVISION, + release: RELEASE, + cohort: COHORT, + }, + startupProfileContractVersion: MANAGED_IMAGE_STARTUP_PROFILE_CONTRACT_VERSION, + capabilityContractVersion: MANAGED_IMAGE_CAPABILITY_CONTRACT_VERSION, + }; +} + +const CATALOG: ManagedImageContractCatalog = Object.fromEntries( + SHIPPED_MANAGED_IMAGE_AGENTS.map((agent, index) => [agent, contract(agent, index)]), +); + +function runtime(driverName = "docker"): SandboxWorkloadRuntimeCapabilities { + return { + driverName, + managedImageSelectionPolicy: driverName === "docker" ? "prefer-managed" : "require-managed", + legacyDockerfileBuilds: driverName === "docker", + managedImages: { + exactDigestReferences: true, + platforms: [MANAGED_IMAGE_PLATFORM], + startupProfileContractVersions: [MANAGED_IMAGE_STARTUP_PROFILE_CONTRACT_VERSION], + capabilityContractVersions: [MANAGED_IMAGE_CAPABILITY_CONTRACT_VERSION], + }, + }; +} + +function input(agentName: string) { + return { + agentName, + legacyDockerfilePath: `agents/${agentName}/Dockerfile`, + runtime: runtime(), + version: "0.0.97", + }; +} + +describe("sandbox workload preparation", () => { + it.each( + SHIPPED_MANAGED_IMAGE_AGENTS, + )("resolves the complete release catalog and exact %s image for Podman (#7744)", async (agent) => { + const resolveCatalog = vi.fn(async () => CATALOG); + + const prepared = await prepareSandboxWorkloadSource( + { + ...input(agent), + runtime: resolveSandboxWorkloadRuntimeCapabilities( + { driverName: "podman" }, + undefined, + "x64", + ), + }, + { resolveCatalog }, + ); + + expect(resolveCatalog).toHaveBeenCalledExactlyOnceWith({ release: RELEASE }); + expect(prepared).toEqual({ + source: { + kind: "managed-image", + reference: contract(agent, SHIPPED_MANAGED_IMAGE_AGENTS.indexOf(agent)).reference, + contract: contract(agent, SHIPPED_MANAGED_IMAGE_AGENTS.indexOf(agent)), + }, + release: RELEASE, + fallbackDiagnostic: null, + }); + }); + + it("never resolves a catalog for an explicit custom Dockerfile (#7744)", async () => { + const resolveCatalog = vi.fn(async () => CATALOG); + const prepared = await prepareSandboxWorkloadSource( + { + ...input("openclaw"), + customDockerfilePath: "/workspace/CustomDockerfile", + }, + { resolveCatalog }, + ); + + expect(resolveCatalog).not.toHaveBeenCalled(); + expect(prepared.source).toEqual({ + kind: "legacy-dockerfile", + dockerfilePath: "/workspace/CustomDockerfile", + reason: "custom-dockerfile", + }); + }); + + it("does not fetch for a runtime that has not registered managed-image support (#7744)", async () => { + const resolveCatalog = vi.fn(async () => CATALOG); + const prepared = await prepareSandboxWorkloadSource( + { + ...input("hermes"), + runtime: { + driverName: "kubernetes", + managedImageSelectionPolicy: "prefer-managed", + legacyDockerfileBuilds: true, + managedImages: null, + }, + }, + { resolveCatalog }, + ); + + expect(resolveCatalog).not.toHaveBeenCalled(); + expect(prepared.source).toMatchObject({ + kind: "legacy-dockerfile", + reason: "runtime-unsupported", + }); + }); + + it("retains Dockerfile fallback for Docker on an unshipped host architecture (#7744)", async () => { + const resolveCatalog = vi.fn(async () => CATALOG); + const prepared = await prepareSandboxWorkloadSource( + { + ...input("openclaw"), + runtime: resolveSandboxWorkloadRuntimeCapabilities( + { driverName: "docker" }, + undefined, + "arm64", + ), + }, + { resolveCatalog }, + ); + + expect(resolveCatalog).not.toHaveBeenCalled(); + expect(prepared.source).toMatchObject({ + kind: "legacy-dockerfile", + reason: "runtime-unsupported", + }); + }); + + it.each([ + "podman", + "mxc", + ])("rejects custom Dockerfile preparation for buildless %s before catalog access (#7744)", async (driverName) => { + const resolveCatalog = vi.fn(async () => CATALOG); + const profiles = { + [driverName]: { + support: runtime(driverName).managedImages!, + hostArchitectures: ["amd64"], + managedImageSelectionPolicy: "require-managed" as const, + legacyDockerfileBuilds: false, + }, + }; + + await expect( + prepareSandboxWorkloadSource( + { + ...input("openclaw"), + customDockerfilePath: "/workspace/CustomDockerfile", + runtime: resolveSandboxWorkloadRuntimeCapabilities({ driverName }, profiles, "x64"), + }, + { resolveCatalog }, + ), + ).rejects.toThrow("cannot use the legacy Dockerfile workload for custom-dockerfile"); + expect(resolveCatalog).not.toHaveBeenCalled(); + }); + + it("fails before catalog access when an incapable runtime requires managed images (#7744)", async () => { + const resolveCatalog = vi.fn(async () => CATALOG); + + await expect( + prepareSandboxWorkloadSource( + { + ...input("langchain-deepagents-code"), + runtime: { + driverName: "podman", + managedImageSelectionPolicy: "require-managed", + legacyDockerfileBuilds: false, + managedImages: null, + }, + }, + { resolveCatalog }, + ), + ).rejects.toThrow("does not advertise managed images"); + expect(resolveCatalog).not.toHaveBeenCalled(); + }); + + it("preserves Dockerfile fallback on an unavailable catalog only when policy allows it (#7744)", async () => { + const resolveCatalog = vi.fn(async () => { + throw new Error("registry offline"); + }); + const preferred = await prepareSandboxWorkloadSource(input("openclaw"), { resolveCatalog }); + + expect(preferred.source).toMatchObject({ + kind: "legacy-dockerfile", + reason: "contract-unavailable", + }); + expect(preferred.fallbackDiagnostic).toContain("registry offline"); + + await expect( + prepareSandboxWorkloadSource( + { ...input("openclaw"), policy: "require-managed" }, + { resolveCatalog }, + ), + ).rejects.toThrow(SandboxWorkloadPreparationError); + }); + + it("does not hide a malformed catalog contract behind preferred fallback (#7744)", async () => { + const malformed = { + ...CATALOG, + hermes: { + ...contract("hermes", 1), + reference: `${MANAGED_IMAGE_REPOSITORIES.hermes}:${RELEASE}`, + }, + }; + + await expect( + prepareSandboxWorkloadSource(input("hermes"), { + resolveCatalog: async () => malformed, + }), + ).rejects.toThrow("failed closed validation"); + }); + + it("rejects a catalog that would support only the selected agent (#7744)", async () => { + await expect( + prepareSandboxWorkloadSource(input("openclaw"), { + resolveCatalog: async () => ({ openclaw: contract("openclaw", 0) }), + }), + ).rejects.toThrow("catalog is incomplete; 'hermes' is missing"); + }); + + it("rejects a complete catalog assembled from different revisions (#7744)", async () => { + const mixed = { + ...CATALOG, + hermes: { + ...contract("hermes", 1), + source: { + ...contract("hermes", 1).source, + revision: "3f03907c37822ea6f1ac9d1bf5c82a4a4568585f", + }, + }, + }; + + await expect( + prepareSandboxWorkloadSource(input("hermes"), { + resolveCatalog: async () => mixed, + }), + ).rejects.toThrow("one all-agent source revision"); + }); + + it("rejects a complete catalog assembled from different publication cohorts (#7744)", async () => { + const mixed = { + ...CATALOG, + hermes: { + ...contract("hermes", 1), + source: { + ...contract("hermes", 1).source, + cohort: "ghrun-7744-3", + }, + }, + }; + + await expect( + prepareSandboxWorkloadSource(input("hermes"), { + resolveCatalog: async () => mixed, + }), + ).rejects.toThrow("one all-agent publication cohort"); + }); + + it("rejects a complete catalog whose release identity differs from the requested tag (#7744)", async () => { + const wrongRelease = { + ...CATALOG, + "langchain-deepagents-code": { + ...contract("langchain-deepagents-code", 2), + source: { + ...contract("langchain-deepagents-code", 2).source, + release: "v0.0.98", + }, + }, + }; + + await expect( + prepareSandboxWorkloadSource(input("langchain-deepagents-code"), { + resolveCatalog: async () => wrongRelease, + }), + ).rejects.toThrow("belongs to 'v0.0.98', not 'v0.0.97'"); + }); + + it("supports an independently registered MXC-shaped runtime without branching (#7744)", async () => { + const prepared = await prepareSandboxWorkloadSource( + { ...input("hermes"), runtime: runtime("mxc") }, + { resolveCatalog: async () => CATALOG }, + ); + + expect(prepared.source).toMatchObject({ + kind: "managed-image", + reference: contract("hermes", 1).reference, + }); + }); +}); diff --git a/src/lib/onboard/sandbox-workload-rebuild.test.ts b/src/lib/onboard/sandbox-workload-rebuild.test.ts new file mode 100644 index 0000000000..73fb1c9bc4 --- /dev/null +++ b/src/lib/onboard/sandbox-workload-rebuild.test.ts @@ -0,0 +1,697 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; + +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { + MANAGED_STARTUP_E2E_CORPORATE_CA_PEM, + managedStartupE2eProfile, +} from "../../../scripts/checks/generate-managed-startup-profile-fixture.mts"; +import { loadAgent } from "../agent/defs"; +import type { SandboxMessagingPlan } from "../messaging"; +import type { SandboxEntry, SandboxWorkloadReceipt } from "../state/registry"; +import { + MANAGED_IMAGE_CAPABILITY_CONTRACT_VERSION, + MANAGED_IMAGE_CONTRACT_VERSION, + MANAGED_IMAGE_PLATFORM, + MANAGED_IMAGE_REPOSITORIES, + MANAGED_IMAGE_SOURCE_REPOSITORY, + MANAGED_IMAGE_STARTUP_PROFILE_CONTRACT_VERSION, + type ManagedImageContractV1, + SHIPPED_MANAGED_IMAGE_AGENTS, + type ShippedManagedImageAgent, +} from "./managed-image/contract"; +import { encodeManagedStartupProfile, type ManagedStartupProfile } from "./managed-startup/profile"; +import { + createManagedWorkloadOnboardRuntime, + prepareOnboardSandboxWorkloadLaunch, + resolveOnboardSandboxWorkloadReceipt, +} from "./managed-workload/onboard-orchestration"; +import * as preparedDcodeRebuild from "./prepared-dcode-rebuild"; +import type { MaterializeSandboxCreatePlanInput } from "./sandbox-create-intent-types"; +import * as sandboxCreateLaunch from "./sandbox-create-launch"; +import type { SandboxCreatePlan } from "./sandbox-create-plan-materialization"; +import * as workloadPreparation from "./workload/preparation"; +import { + managedWorkloadRebuildDependencies, + managedWorkloadRebuildHandoffMatchesEntry, + managedWorkloadRebuildProfileEnvironment, + prepareManagedWorkloadRebuildHandoff, + prepareSandboxWorkloadSourceFromRebuildHandoff, + stageManagedWorkloadRebuildProfile, +} from "./workload/rebuild"; +import * as workloadRuntime from "./workload/runtime"; +import type { SandboxWorkloadRuntimeCapabilities } from "./workload/source"; + +const RUNTIME = { + driverName: "docker", + managedImageSelectionPolicy: "prefer-managed", + legacyDockerfileBuilds: true, + managedImages: { + exactDigestReferences: true, + platforms: [MANAGED_IMAGE_PLATFORM], + startupProfileContractVersions: [MANAGED_IMAGE_STARTUP_PROFILE_CONTRACT_VERSION], + capabilityContractVersions: [MANAGED_IMAGE_CAPABILITY_CONTRACT_VERSION], + }, +} as const satisfies SandboxWorkloadRuntimeCapabilities; + +const OLD_RELEASE = "v0.0.97"; +const NEW_RELEASE = "v0.0.98"; +const OLD_REVISION = "1".repeat(40); +const NEW_REVISION = "2".repeat(40); +const OLD_COHORT = "ghrun-123-1"; +const NEW_COHORT = "ghrun-456-2"; + +type DeepMutable = T extends readonly (infer Item)[] + ? DeepMutable[] + : T extends object + ? { -readonly [Key in keyof T]: DeepMutable } + : T; + +function contract( + agent: ShippedManagedImageAgent, + generation: "old" | "new", +): ManagedImageContractV1 { + const image = MANAGED_IMAGE_REPOSITORIES[agent]; + const digest: `sha256:${string}` = `sha256:${(generation === "old" ? "a" : "b").repeat(64)}`; + return { + contractVersion: MANAGED_IMAGE_CONTRACT_VERSION, + agent, + platform: MANAGED_IMAGE_PLATFORM, + image, + digest: digest as `sha256:${string}`, + reference: `${image}@${digest}` as ManagedImageContractV1["reference"], + source: { + repository: MANAGED_IMAGE_SOURCE_REPOSITORY, + revision: generation === "old" ? OLD_REVISION : NEW_REVISION, + release: generation === "old" ? OLD_RELEASE : NEW_RELEASE, + cohort: generation === "old" ? OLD_COHORT : NEW_COHORT, + }, + startupProfileContractVersion: MANAGED_IMAGE_STARTUP_PROFILE_CONTRACT_VERSION, + capabilityContractVersion: MANAGED_IMAGE_CAPABILITY_CONTRACT_VERSION, + }; +} + +function entry( + agent: ShippedManagedImageAgent, + options: { + corporateCa?: boolean; + credentialProxyReplayRequired?: boolean; + profile?: ManagedStartupProfile; + } = {}, +): SandboxEntry { + const profile = + options.profile ?? + managedStartupE2eProfile( + agent, + false, + options.corporateCa === true, + options.credentialProxyReplayRequired === true, + ); + const encodedProfile = encodeManagedStartupProfile(profile); + const old = contract(agent, "old"); + const workload: SandboxWorkloadReceipt = { + schemaVersion: 1, + kind: "managed-image", + reference: old.reference, + release: old.source.release, + sourceRevision: old.source.revision, + sourceCohort: old.source.cohort, + capabilityContractVersion: old.capabilityContractVersion, + startupProfileContractVersion: old.startupProfileContractVersion, + encodedProfile, + startupProfileSha256: createHash("sha256").update(encodedProfile, "utf8").digest("hex"), + credentialProxyReplayRequired: options.credentialProxyReplayRequired === true, + ...(options.corporateCa + ? { + corporateCaB64: Buffer.from(MANAGED_STARTUP_E2E_CORPORATE_CA_PEM, "utf8").toString( + "base64", + ), + } + : {}), + shared: true, + }; + return { + name: "alpha", + agent: agent === "openclaw" ? null : agent, + fromDockerfile: null, + imageTag: old.reference, + workload, + }; +} + +function installReplacement(agent: ShippedManagedImageAgent) { + const replacement = contract(agent, "new"); + const spy = vi + .spyOn(managedWorkloadRebuildDependencies, "prepareSandboxWorkloadSource") + .mockResolvedValue({ + source: { kind: "managed-image", reference: replacement.reference, contract: replacement }, + release: NEW_RELEASE, + fallbackDiagnostic: null, + }); + return { replacement, spy }; +} + +function messagingPlan(agent: "openclaw" | "hermes"): SandboxMessagingPlan { + return { + schemaVersion: 1, + sandboxName: "alpha", + agent, + workflow: "rebuild", + channels: [], + disabledChannels: [], + credentialBindings: [], + networkPolicy: { presets: [], entries: [] }, + agentRender: [], + buildSteps: [], + stateUpdates: [], + healthChecks: [], + }; +} + +function profileInput( + agent: ShippedManagedImageAgent, +): Parameters[1] { + const dashboardEnabled = agent !== "langchain-deepagents-code"; + return { + inference: { + routeProvider: "inference", + upstreamProvider: "nvidia-prod", + model: "nvidia/new-model", + routedBaseUrl: "https://inference.local/v1", + upstreamEndpointUrl: + agent === "langchain-deepagents-code" ? "https://integrate.api.nvidia.com/v1" : null, + api: "openai-completions", + primaryModelRef: agent === "openclaw" ? "inference/nvidia/new-model" : null, + compatibility: agent === "openclaw" ? {} : null, + }, + chatUiUrl: dashboardEnabled ? "http://127.0.0.1:18789" : "", + effectiveDashboardPort: dashboardEnabled ? 18_789 : 0, + manageDashboard: dashboardEnabled, + dashboardBindAddress: undefined, + wslExposure: false, + hermesDashboardState: { config: null, enabled: false }, + webSearch: + agent === "langchain-deepagents-code" + ? null + : { fetchEnabled: true, provider: agent === "hermes" ? "tavily" : "brave" }, + toolDisclosure: "direct", + hermesToolGateways: agent === "hermes" ? ["nous-image"] : [], + messagingPlan: agent === "langchain-deepagents-code" ? null : messagingPlan(agent), + dcodeAutoApprovalMode: agent === "langchain-deepagents-code" ? "thread-opt-in" : "disabled", + observabilityEnabled: agent === "langchain-deepagents-code", + }; +} + +afterEach(() => vi.restoreAllMocks()); + +describe("managed workload rebuild handoff", () => { + it.each([ + "openclaw", + "hermes", + "langchain-deepagents-code", + ] as const)("validates old %s authority and pins the current all-agent release image", async (agent) => { + const oldEntry = entry(agent); + const { replacement, spy } = installReplacement(agent); + + const handoff = await prepareManagedWorkloadRebuildHandoff(oldEntry, { + runtime: RUNTIME, + version: NEW_RELEASE, + }); + + expect(handoff).not.toBeNull(); + expect(handoff?.previousReceipt.reference).toBe(oldEntry.imageTag); + expect(handoff?.replacement.source.reference).toBe(replacement.reference); + expect(handoff?.replacement.source.reference).not.toBe(oldEntry.imageTag); + expect(handoff?.replacement.source.contract.source).toEqual({ + repository: MANAGED_IMAGE_SOURCE_REPOSITORY, + revision: NEW_REVISION, + release: NEW_RELEASE, + cohort: NEW_COHORT, + }); + expect(handoff?.previousProfile.agent).toBe(agent); + expect(spy).toHaveBeenCalledWith( + expect.objectContaining({ + agentName: agent, + runtime: RUNTIME, + version: NEW_RELEASE, + policy: "require-managed", + }), + ); + expect(prepareSandboxWorkloadSourceFromRebuildHandoff(handoff!, RUNTIME).source).toEqual( + handoff?.replacement.source, + ); + expect(managedWorkloadRebuildHandoffMatchesEntry(handoff!, oldEntry)).toBe(true); + }); + + it.each( + SHIPPED_MANAGED_IMAGE_AGENTS, + )("carries the exact staged %s rebuild through managed launch and durable receipt", async (agent) => { + const oldEntry = entry(agent, { corporateCa: agent === "openclaw" }); + const { replacement, spy: catalogSelection } = installReplacement(agent); + const catalogHandoff = (await prepareManagedWorkloadRebuildHandoff(oldEntry, { + runtime: RUNTIME, + version: NEW_RELEASE, + }))!; + const handoff = stageManagedWorkloadRebuildProfile(catalogHandoff, profileInput(agent), {}); + const innerPreparation = vi + .spyOn(workloadPreparation, "prepareSandboxWorkloadSource") + .mockRejectedValue(new Error("inner onboarding attempted catalog or fallback preparation")); + vi.spyOn(workloadRuntime, "resolveSandboxWorkloadRuntimeCapabilities").mockReturnValue(RUNTIME); + const legacyBuildContext = vi + .spyOn(preparedDcodeRebuild, "resolveSandboxBuildContext") + .mockImplementation(() => { + throw new Error("managed rebuild entered legacy build-context preparation"); + }); + const legacyBuildPatch = vi + .spyOn(preparedDcodeRebuild, "resolveSandboxBuildPatch") + .mockImplementation(() => { + throw new Error("managed rebuild entered legacy Dockerfile patching"); + }); + const legacyPrebuild = vi + .spyOn(sandboxCreateLaunch, "prepareSandboxCreateLaunchWithPrebuild") + .mockImplementation(() => { + throw new Error("managed rebuild entered legacy image prebuild"); + }); + const resolveAgentInferenceApi = vi.fn(() => { + throw new Error("managed rebuild regenerated its inference API"); + }); + const getSandboxInferenceConfig = vi.fn(() => { + throw new Error("managed rebuild regenerated its inference profile"); + }); + const note = vi.fn(); + const fallbackBuildEstimate = vi.fn(() => "legacy build estimate"); + const dashboardEnabled = agent !== "langchain-deepagents-code"; + const selectedAgent = agent === "openclaw" ? null : loadAgent(agent); + const runtime = createManagedWorkloadOnboardRuntime( + { + computePlan: { driverName: "docker", gatewayLauncher: "nemoclaw" }, + managedWorkloadRebuild: handoff, + agentName: agent, + legacyDockerfilePath: "/repo/agents/forbidden/Dockerfile", + customDockerfilePath: null, + rootDir: "/repo", + model: "must-not-regenerate", + provider: "nvidia-prod", + preferredInferenceApi: null, + endpointUrl: "https://must-not-regenerate.example.test/v1", + startupProfile: { ...profileInput(agent), environment: {} }, + note, + fallbackBuildEstimate, + }, + { resolveAgentInferenceApi, getSandboxInferenceConfig }, + ); + + const workload = await runtime.ensurePreparedWorkload(); + expect(await runtime.ensurePreparedWorkload()).toBe(workload); + expect(workload.source).toEqual(handoff.replacement.source); + expect(runtime.ensurePreparedProfile(workload)).toBe(handoff.replacementProfile); + + const intent = { + sandboxName: "alpha", + inferenceProvider: "inference", + activeMessagingChannels: [], + messagingProviderRequests: [], + reusableMessagingProviders: [], + extraProviders: [], + staleExtraProviders: [], + hermesToolGateways: [], + policy: { + basePolicyPath: "/repo/policy.yaml", + activeMessagingChannels: [], + options: { + directGpu: false, + additionalPresets: [], + policyTier: null, + baselineExclusions: [], + }, + }, + gpuCreateArgs: [], + resourceCreateArgs: [], + gpuRoutePlan: "none", + sandboxGpuLogMessage: null, + disabledChannelNames: [], + extraPlaceholderKeys: [], + } as const; + const materializeSandboxCreatePlan = vi.fn( + (input: MaterializeSandboxCreatePlanInput): SandboxCreatePlan => ({ + activeMessagingChannels: [], + initialSandboxPolicy: { policyPath: "/tmp/managed-policy.yaml", appliedPresets: [] }, + policyTier: null, + createArgs: [ + "--from", + input.fromRef, + "--name", + input.intent.sandboxName, + "--policy", + "/tmp/managed-policy.yaml", + ], + messagingProviders: [], + gpuRoutePlan: "none", + compatibilityPolicyPath: null, + sandboxGpuLogMessage: null, + }), + ); + const createAgentSandbox = vi.fn(() => { + throw new Error("managed rebuild staged an agent Dockerfile"); + }); + const prepareSandboxBuildPatchConfig = vi.fn(() => { + throw new Error("managed rebuild prepared Dockerfile patch configuration"); + }); + const launch = await prepareOnboardSandboxWorkloadLaunch({ + runtime, + workload, + legacy: { + preparedBuildContext: null, + agent: selectedAgent, + fromDockerfile: null, + createAgentSandbox, + patchInput: {} as never, + }, + plan: { + intent, + rebindMessagingTokenDefs: async () => [], + runProviderPreDeleteCleanup: vi.fn(), + upsertMessagingProviders: vi.fn(() => []), + getHermesToolGatewayProviderName: vi.fn(() => "hermes-tools"), + discloseInitialSandboxPolicy: vi.fn(), + }, + launchInput: { + agent: selectedAgent, + observabilityEnabled: agent === "langchain-deepagents-code", + chatUiUrl: dashboardEnabled ? "http://127.0.0.1:18789" : "", + sandboxName: "alpha", + env: {}, + extraPlaceholderKeys: [], + getDashboardForwardPort: () => (dashboardEnabled ? "18789" : "0"), + hermesDashboardState: { config: null, enabled: false }, + manageDashboard: dashboardEnabled, + openshellShellCommand: (args) => args.join(" "), + openshellArgv: (args) => [...args], + buildEnv: () => ({}), + }, + plannedMessagingPlan: null, + gpu: { + provider: "nvidia-prod", + config: { + mode: "0", + hostGpuDetected: false, + hostGpuPlatform: null, + sandboxGpuEnabled: false, + sandboxGpuDevice: null, + errors: [], + }, + dockerDriverGateway: true, + gatewayPort: 8_080, + }, + dependencies: { + materializeSandboxCreatePlan, + prepareSandboxBuildPatchConfig, + }, + log: vi.fn(), + }); + + const replacementReference = handoff.replacement.source.reference; + const encodedProfile = handoff.replacementProfile.encodedProfile; + const fromIndexes = launch.launch.createArgv.flatMap((argument, index) => + argument === "--from" ? [index] : [], + ); + expect(fromIndexes).toHaveLength(1); + expect(launch.launch.createArgv[fromIndexes[0]! + 1]).toBe(replacementReference); + expect(replacementReference).toBe(`${replacement.image}@${replacement.digest}`); + expect(launch.launch.managedStartupRootApplyRequest?.encodedProfile).toBe(encodedProfile); + expect(launch.launch.sandboxStartupCommand).toContain( + "/usr/local/bin/nemoclaw-managed-startup-hold", + ); + expect(launch.launch.createArgv.join("\n")).not.toContain(encodedProfile); + expect(launch.launch.sandboxStartupCommand.join("\n")).not.toContain(encodedProfile); + expect(launch.launch.startupRequirement).toBe("trusted-image-init"); + expect(launch.launch.createArgv.join("\n")).not.toContain("Dockerfile"); + expect(launch.legacyBuildContext).toBeNull(); + expect(launch.launch.prebuild).toEqual({ + createArgs: expect.arrayContaining(["--from", replacementReference]), + imageRef: null, + imageId: null, + }); + expect(materializeSandboxCreatePlan).toHaveBeenCalledWith( + expect.objectContaining({ fromRef: replacementReference }), + ); + + const extractBuiltImageRef = vi.fn(() => "legacy-built-image"); + const resolveSandboxImageTagFromCreateOutput = vi.fn(() => "legacy-output-image"); + const resolved = resolveOnboardSandboxWorkloadReceipt({ + runtime, + workload, + registryImageRef: "legacy-registry-image", + prebuildImageRef: "legacy-prebuild-image", + firstCreateOutput: "legacy first output", + createOutput: "legacy create output", + buildId: "legacy-build-id", + extractBuiltImageRef, + resolveSandboxImageTagFromCreateOutput, + }); + + expect(resolved.resolvedImageTag).toBe(replacementReference); + expect(resolved.workloadReceipt).toEqual({ + schemaVersion: 1, + kind: "managed-image", + reference: replacementReference, + release: handoff.replacement.source.contract.source.release, + sourceRevision: handoff.replacement.source.contract.source.revision, + sourceCohort: handoff.replacement.source.contract.source.cohort, + capabilityContractVersion: handoff.replacement.source.contract.capabilityContractVersion, + startupProfileContractVersion: + handoff.replacement.source.contract.startupProfileContractVersion, + encodedProfile, + startupProfileSha256: handoff.replacementProfile.startupProfileSha256, + credentialProxyReplayRequired: handoff.replacementProfile.credentialProxyReplayRequired, + ...(handoff.replacementProfile.corporateCaB64 === undefined + ? {} + : { corporateCaB64: handoff.replacementProfile.corporateCaB64 }), + shared: true, + }); + expect(catalogSelection).toHaveBeenCalledOnce(); + expect(innerPreparation).not.toHaveBeenCalled(); + expect(legacyBuildContext).not.toHaveBeenCalled(); + expect(legacyBuildPatch).not.toHaveBeenCalled(); + expect(legacyPrebuild).not.toHaveBeenCalled(); + expect(createAgentSandbox).not.toHaveBeenCalled(); + expect(prepareSandboxBuildPatchConfig).not.toHaveBeenCalled(); + expect(resolveAgentInferenceApi).not.toHaveBeenCalled(); + expect(getSandboxInferenceConfig).not.toHaveBeenCalled(); + expect(note).not.toHaveBeenCalled(); + expect(fallbackBuildEstimate).not.toHaveBeenCalled(); + expect(extractBuiltImageRef).not.toHaveBeenCalled(); + expect(resolveSandboxImageTagFromCreateOutput).not.toHaveBeenCalled(); + }); + + it("retains validated corporate CA bytes while selecting a new OpenClaw image", async () => { + const oldEntry = entry("openclaw", { corporateCa: true }); + installReplacement("openclaw"); + + const handoff = await prepareManagedWorkloadRebuildHandoff(oldEntry, { + runtime: RUNTIME, + version: NEW_RELEASE, + }); + + expect(handoff?.corporateCa?.pem).toBe(MANAGED_STARTUP_E2E_CORPORATE_CA_PEM); + }); + + it("preserves credential-proxy replay while allowing other profile env to change", async () => { + const oldEntry = entry("hermes", { credentialProxyReplayRequired: true }); + installReplacement("hermes"); + const handoff = (await prepareManagedWorkloadRebuildHandoff(oldEntry, { + runtime: RUNTIME, + version: NEW_RELEASE, + }))!; + + expect( + managedWorkloadRebuildProfileEnvironment(handoff, { + HTTPS_PROXY: "http://user:password@proxy.example.test:8443", + NEMOCLAW_CONTEXT_WINDOW: "196608", + }), + ).toEqual({ + HTTPS_PROXY: "http://user:password@proxy.example.test:8443", + NEMOCLAW_CONTEXT_WINDOW: "131072", + NEMOCLAW_PROXY_HOST: "10.200.0.1", + NEMOCLAW_PROXY_PORT: "3128", + }); + }); + + it("renders a complete OpenClaw replacement profile before mutation while preserving receipt-only tuning", async () => { + const previous = structuredClone( + managedStartupE2eProfile("openclaw"), + ) as DeepMutable; + assert(previous.agentConfig.agent === "openclaw", "fixture drift"); + previous.proxy.managedHost = "10.44.0.9"; + previous.proxy.managedPort = 4312; + previous.tuning.contextWindow = 196_608; + previous.tuning.maxTokens = 16_384; + previous.tuning.reasoning = true; + previous.tuning.reasoningEffort = "high"; + previous.inference.inputModalities = ["text", "image"]; + previous.agentConfig.agentTimeoutSeconds = 777; + previous.agentConfig.heartbeatEvery = "45s"; + previous.agentConfig.extraAgents = { + agents: [{ id: "reviewer" }], + defaults: { subagents: {} }, + main: { model: "primary" }, + }; + previous.agentConfig.otel = { + enabled: true, + endpointUrl: "http://otel.example.test:4318", + serviceName: "custom-openclaw", + sampleRate: 0.25, + }; + previous.agentConfig.minimalBootstrap = false; + + installReplacement("openclaw"); + const catalog = (await prepareManagedWorkloadRebuildHandoff( + entry("openclaw", { profile: previous }), + { runtime: RUNTIME, version: NEW_RELEASE }, + ))!; + const handoff = stageManagedWorkloadRebuildProfile(catalog, profileInput("openclaw"), {}); + + expect(handoff.replacementProfile.profile).toMatchObject({ + inference: { + model: "nvidia/new-model", + upstreamProvider: "nvidia-prod", + inputModalities: ["image", "text"], + }, + tools: { disclosure: "direct" }, + proxy: { managedHost: "10.44.0.9", managedPort: 4312 }, + tuning: { + contextWindow: 196_608, + maxTokens: 16_384, + reasoning: true, + reasoningEffort: "high", + }, + agentConfig: { + agent: "openclaw", + agentTimeoutSeconds: 777, + heartbeatEvery: "45s", + minimalBootstrap: false, + otel: { + enabled: true, + endpointUrl: "http://otel.example.test:4318", + serviceName: "custom-openclaw", + sampleRate: 0.25, + }, + webSearch: { enabled: true, provider: "brave" }, + }, + }); + expect(handoff.replacementProfile.profile.messaging.plan).not.toBeNull(); + expect(handoff.replacementProfile.encodedProfile).not.toBe( + catalog.previousReceipt.encodedProfile, + ); + }); + + it("preserves Hermes receipt-only context and proxy while applying current tools and messaging", async () => { + const previous = structuredClone( + managedStartupE2eProfile("hermes"), + ) as DeepMutable; + previous.proxy.managedHost = "10.55.0.7"; + previous.proxy.managedPort = 5312; + previous.tuning.contextWindow = 262_144; + + installReplacement("hermes"); + const catalog = (await prepareManagedWorkloadRebuildHandoff( + entry("hermes", { profile: previous }), + { runtime: RUNTIME, version: NEW_RELEASE }, + ))!; + const handoff = stageManagedWorkloadRebuildProfile(catalog, profileInput("hermes"), {}); + + expect(handoff.replacementProfile.profile).toMatchObject({ + inference: { model: "nvidia/new-model" }, + tools: { disclosure: "direct", enabledGateways: ["nous-image"] }, + proxy: { managedHost: "10.55.0.7", managedPort: 5312 }, + tuning: { + contextWindow: 262_144, + maxTokens: null, + reasoning: null, + reasoningEffort: null, + }, + agentConfig: { + agent: "hermes", + webSearch: { enabled: true, provider: "tavily" }, + }, + }); + expect(handoff.replacementProfile.profile.messaging.plan).not.toBeNull(); + }); + + it("preserves DCode receipt-only proxy while applying current disclosure, approval, and observability", async () => { + const previous = structuredClone( + managedStartupE2eProfile("langchain-deepagents-code"), + ) as DeepMutable; + previous.proxy.managedHost = "10.66.0.5"; + previous.proxy.managedPort = 6312; + + installReplacement("langchain-deepagents-code"); + const catalog = (await prepareManagedWorkloadRebuildHandoff( + entry("langchain-deepagents-code", { profile: previous }), + { runtime: RUNTIME, version: NEW_RELEASE }, + ))!; + const handoff = stageManagedWorkloadRebuildProfile( + catalog, + profileInput("langchain-deepagents-code"), + {}, + ); + + expect(handoff.replacementProfile.profile).toMatchObject({ + inference: { + model: "nvidia/new-model", + upstreamEndpointUrl: "https://integrate.api.nvidia.com/v1", + }, + tools: { disclosure: "direct" }, + proxy: { managedHost: "10.66.0.5", managedPort: 6312 }, + agentConfig: { + agent: "langchain-deepagents-code", + autoApprovalMode: "thread-opt-in", + observabilityEnabled: true, + }, + }); + }); + + it("fails profile rendering before a managed credential proxy can be silently dropped", async () => { + const oldEntry = entry("openclaw", { credentialProxyReplayRequired: true }); + installReplacement("openclaw"); + const catalog = (await prepareManagedWorkloadRebuildHandoff(oldEntry, { + runtime: RUNTIME, + version: NEW_RELEASE, + }))!; + + expect(() => stageManagedWorkloadRebuildProfile(catalog, profileInput("openclaw"), {})).toThrow( + "changed the durable credential-proxy requirement", + ); + }); + + it("fails closed instead of resolving a catalog for a managed image with no receipt", async () => { + const oldEntry = entry("openclaw"); + delete oldEntry.workload; + const { spy } = installReplacement("openclaw"); + + await expect( + prepareManagedWorkloadRebuildHandoff(oldEntry, { + runtime: RUNTIME, + version: NEW_RELEASE, + }), + ).rejects.toThrow("no valid durable workload receipt"); + expect(spy).not.toHaveBeenCalled(); + }); + + it("fails closed before mutation when the current complete catalog is unavailable", async () => { + vi.spyOn(managedWorkloadRebuildDependencies, "prepareSandboxWorkloadSource").mockRejectedValue( + new Error("cohort is torn"), + ); + + await expect( + prepareManagedWorkloadRebuildHandoff(entry("langchain-deepagents-code"), { + runtime: RUNTIME, + version: NEW_RELEASE, + }), + ).rejects.toThrow("complete managed-image catalog is unavailable or invalid"); + }); +}); diff --git a/src/lib/onboard/sandbox-workload-runtime.test.ts b/src/lib/onboard/sandbox-workload-runtime.test.ts new file mode 100644 index 0000000000..de4fb81f3b --- /dev/null +++ b/src/lib/onboard/sandbox-workload-runtime.test.ts @@ -0,0 +1,126 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; +import { + MANAGED_IMAGE_CAPABILITY_CONTRACT_VERSION, + MANAGED_IMAGE_PLATFORM, + MANAGED_IMAGE_STARTUP_PROFILE_CONTRACT_VERSION, +} from "./managed-image/contract"; +import { + CURRENT_MANAGED_IMAGE_RUNTIME_PROFILES, + type ManagedImageRuntimeProfileRegistry, + resolveSandboxWorkloadRuntimeCapabilities, +} from "./workload/runtime"; + +const MANAGED_IMAGE_V1_SUPPORT = { + exactDigestReferences: true, + platforms: [MANAGED_IMAGE_PLATFORM], + startupProfileContractVersions: [MANAGED_IMAGE_STARTUP_PROFILE_CONTRACT_VERSION], + capabilityContractVersions: [MANAGED_IMAGE_CAPABILITY_CONTRACT_VERSION], +} as const; + +describe("sandbox workload runtime capabilities", () => { + it("registers managed-image v1 support for the Docker compute driver (#7744)", () => { + expect( + resolveSandboxWorkloadRuntimeCapabilities({ driverName: "docker" }, undefined, "x64"), + ).toEqual({ + driverName: "docker", + managedImageSelectionPolicy: "prefer-managed", + legacyDockerfileBuilds: true, + managedImages: MANAGED_IMAGE_V1_SUPPORT, + }); + }); + + it("registers Podman as a buildless managed-image v1 runtime on amd64 (#7744)", () => { + expect( + resolveSandboxWorkloadRuntimeCapabilities({ driverName: "podman" }, undefined, "x64"), + ).toEqual({ + driverName: "podman", + managedImageSelectionPolicy: "require-managed", + legacyDockerfileBuilds: false, + managedImages: MANAGED_IMAGE_V1_SUPPORT, + }); + }); + + it.each([ + "arm64", + "s390x", + ])("does not select an amd64-only managed cohort on a %s host (#7744)", (architecture) => { + expect( + resolveSandboxWorkloadRuntimeCapabilities({ driverName: "docker" }, undefined, architecture), + ).toEqual({ + driverName: "docker", + managedImageSelectionPolicy: "prefer-managed", + legacyDockerfileBuilds: true, + managedImages: null, + }); + }); + + it("preserves the registered Kubernetes legacy-build behavior (#7744)", () => { + expect(resolveSandboxWorkloadRuntimeCapabilities({ driverName: "kubernetes" })).toEqual({ + driverName: "kubernetes", + managedImageSelectionPolicy: "prefer-managed", + legacyDockerfileBuilds: true, + managedImages: null, + }); + }); + + it("fails unknown drivers closed instead of inferring Dockerfile support (#7744)", () => { + expect(resolveSandboxWorkloadRuntimeCapabilities({ driverName: "future-runtime" })).toEqual({ + driverName: "future-runtime", + managedImageSelectionPolicy: "require-managed", + legacyDockerfileBuilds: false, + managedImages: null, + }); + }); + + it.each([ + "__proto__", + "constructor", + "toString", + ])("fails inherited-object driver name %s closed (#7744)", (driverName) => { + expect(resolveSandboxWorkloadRuntimeCapabilities({ driverName })).toEqual({ + driverName, + managedImageSelectionPolicy: "require-managed", + legacyDockerfileBuilds: false, + managedImages: null, + }); + }); + + it("lets an MXC-shaped driver inject the same portable contract without Docker coupling (#7744)", () => { + const driverName = "mxc"; + const profiles: ManagedImageRuntimeProfileRegistry = { + ...CURRENT_MANAGED_IMAGE_RUNTIME_PROFILES, + [driverName]: { + support: MANAGED_IMAGE_V1_SUPPORT, + hostArchitectures: ["amd64"], + managedImageSelectionPolicy: "require-managed", + legacyDockerfileBuilds: false, + }, + }; + + expect(resolveSandboxWorkloadRuntimeCapabilities({ driverName }, profiles, "x64")).toEqual({ + driverName, + managedImageSelectionPolicy: "require-managed", + legacyDockerfileBuilds: false, + managedImages: MANAGED_IMAGE_V1_SUPPORT, + }); + }); + + it("returns a defensive copy of registered capability arrays (#7744)", () => { + const first = resolveSandboxWorkloadRuntimeCapabilities( + { driverName: "docker" }, + undefined, + "x64", + ); + const second = resolveSandboxWorkloadRuntimeCapabilities( + { driverName: "docker" }, + undefined, + "x64", + ); + + expect(first.managedImages).not.toBe(second.managedImages); + expect(first.managedImages?.platforms).not.toBe(second.managedImages?.platforms); + }); +}); diff --git a/src/lib/onboard/sandbox-workload-source.test.ts b/src/lib/onboard/sandbox-workload-source.test.ts new file mode 100644 index 0000000000..5851d6657d --- /dev/null +++ b/src/lib/onboard/sandbox-workload-source.test.ts @@ -0,0 +1,244 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; +import { + MANAGED_IMAGE_CAPABILITY_CONTRACT_VERSION, + MANAGED_IMAGE_CONTRACT_VERSION, + MANAGED_IMAGE_PLATFORM, + MANAGED_IMAGE_REPOSITORIES, + MANAGED_IMAGE_SOURCE_REPOSITORY, + MANAGED_IMAGE_STARTUP_PROFILE_CONTRACT_VERSION, + type ManagedImageContractCatalog, + type ManagedImageContractV1, + SHIPPED_MANAGED_IMAGE_AGENTS, + type ShippedManagedImageAgent, +} from "./managed-image/contract"; +import { + resolveSandboxWorkloadSource, + type SandboxWorkloadRuntimeCapabilities, +} from "./workload/source"; + +const SOURCE_REVISION = "2f03907c37822ea6f1ac9d1bf5c82a4a4568585f"; +const SOURCE_RELEASE = "v0.0.89"; +const SOURCE_COHORT = "ghrun-7744-2"; +const DIGESTS = { + openclaw: `sha256:${"4d".repeat(32)}`, + hermes: `sha256:${"5e".repeat(32)}`, + "langchain-deepagents-code": `sha256:${"6f".repeat(32)}`, +} as const satisfies Record; + +function contractFor(agent: ShippedManagedImageAgent): ManagedImageContractV1 { + const image = MANAGED_IMAGE_REPOSITORIES[agent]; + const digest = DIGESTS[agent]; + return { + contractVersion: MANAGED_IMAGE_CONTRACT_VERSION, + agent, + platform: MANAGED_IMAGE_PLATFORM, + image, + digest, + reference: `${image}@${digest}`, + source: { + repository: MANAGED_IMAGE_SOURCE_REPOSITORY, + revision: SOURCE_REVISION, + release: SOURCE_RELEASE, + cohort: SOURCE_COHORT, + }, + startupProfileContractVersion: MANAGED_IMAGE_STARTUP_PROFILE_CONTRACT_VERSION, + capabilityContractVersion: MANAGED_IMAGE_CAPABILITY_CONTRACT_VERSION, + }; +} + +const CATALOG: ManagedImageContractCatalog = Object.fromEntries( + SHIPPED_MANAGED_IMAGE_AGENTS.map((agent) => [agent, contractFor(agent)]), +); + +function managedRuntime(driverName: string): SandboxWorkloadRuntimeCapabilities { + return { + driverName, + managedImageSelectionPolicy: driverName === "docker" ? "prefer-managed" : "require-managed", + legacyDockerfileBuilds: driverName === "docker", + managedImages: { + exactDigestReferences: true, + platforms: [MANAGED_IMAGE_PLATFORM], + startupProfileContractVersions: [MANAGED_IMAGE_STARTUP_PROFILE_CONTRACT_VERSION], + capabilityContractVersions: [MANAGED_IMAGE_CAPABILITY_CONTRACT_VERSION], + }, + }; +} + +describe("sandbox workload source resolution", () => { + it.each( + SHIPPED_MANAGED_IMAGE_AGENTS, + )("selects the published managed image for stock %s on a capable runtime (#7744)", (agent) => { + const source = resolveSandboxWorkloadSource({ + agentName: agent, + legacyDockerfilePath: `agents/${agent}/Dockerfile`, + runtime: managedRuntime("podman"), + catalog: CATALOG, + }); + + expect(source).toEqual({ + kind: "managed-image", + reference: contractFor(agent).reference, + contract: contractFor(agent), + }); + }); + + it("uses the same contract with an MXC-shaped capable driver (#7744)", () => { + const source = resolveSandboxWorkloadSource({ + agentName: "hermes", + legacyDockerfilePath: "agents/hermes/Dockerfile", + runtime: managedRuntime("mxc"), + catalog: CATALOG, + }); + + expect(source).toMatchObject({ + kind: "managed-image", + reference: contractFor("hermes").reference, + }); + }); + + it("preserves an explicit custom Dockerfile on a driver that supports local builds (#7744)", () => { + const source = resolveSandboxWorkloadSource({ + agentName: "openclaw", + legacyDockerfilePath: "Dockerfile", + customDockerfilePath: "/workspace/custom/Dockerfile", + runtime: managedRuntime("docker"), + catalog: CATALOG, + }); + + expect(source).toEqual({ + kind: "legacy-dockerfile", + dockerfilePath: "/workspace/custom/Dockerfile", + reason: "custom-dockerfile", + }); + }); + + it("rejects a custom Dockerfile on a buildless driver before image selection (#7744)", () => { + expect(() => + resolveSandboxWorkloadSource({ + agentName: "openclaw", + legacyDockerfilePath: "Dockerfile", + customDockerfilePath: "/workspace/custom/Dockerfile", + runtime: managedRuntime("podman"), + catalog: CATALOG, + }), + ).toThrow("cannot use the legacy Dockerfile workload for custom-dockerfile"); + }); + + it("preserves the legacy path when the current driver lacks managed-image capabilities (#7744)", () => { + const source = resolveSandboxWorkloadSource({ + agentName: "langchain-deepagents-code", + legacyDockerfilePath: "agents/langchain-deepagents-code/Dockerfile", + runtime: { + driverName: "kubernetes", + managedImageSelectionPolicy: "prefer-managed", + legacyDockerfileBuilds: true, + managedImages: null, + }, + catalog: CATALOG, + }); + + expect(source).toEqual({ + kind: "legacy-dockerfile", + dockerfilePath: "agents/langchain-deepagents-code/Dockerfile", + reason: "runtime-unsupported", + }); + }); + + it("preserves the legacy path for an unshipped custom agent on Docker (#7744)", () => { + const source = resolveSandboxWorkloadSource({ + agentName: "company-agent", + legacyDockerfilePath: "/workspace/company-agent/Dockerfile", + runtime: managedRuntime("docker"), + catalog: CATALOG, + }); + + expect(source).toMatchObject({ + kind: "legacy-dockerfile", + reason: "agent-not-managed", + }); + }); + + it("rejects an unshipped custom agent on a buildless driver (#7744)", () => { + expect(() => + resolveSandboxWorkloadSource({ + agentName: "company-agent", + legacyDockerfilePath: "/workspace/company-agent/Dockerfile", + runtime: managedRuntime("mxc"), + catalog: CATALOG, + }), + ).toThrow("selected agent is not a shipped managed agent"); + }); + + it("fails closed when a supplied stock-agent contract does not match (#7744)", () => { + const mismatchedCatalog = { + ...CATALOG, + openclaw: contractFor("hermes"), + }; + + expect(() => + resolveSandboxWorkloadSource({ + agentName: "openclaw", + legacyDockerfilePath: "Dockerfile", + runtime: managedRuntime("podman"), + catalog: mismatchedCatalog, + policy: "require-managed", + }), + ).toThrow("failed closed validation"); + }); + + it("fails closed when managed selection is required but the catalog has no exact contract (#7744)", () => { + expect(() => + resolveSandboxWorkloadSource({ + agentName: "hermes", + legacyDockerfilePath: "agents/hermes/Dockerfile", + runtime: managedRuntime("podman"), + catalog: {}, + policy: "require-managed", + }), + ).toThrow("catalog has no exact contract"); + }); + + it("fails closed when managed selection is required but the driver cannot apply profile v1 (#7744)", () => { + expect(() => + resolveSandboxWorkloadSource({ + agentName: "langchain-deepagents-code", + legacyDockerfilePath: "agents/langchain-deepagents-code/Dockerfile", + runtime: { + driverName: "podman", + managedImageSelectionPolicy: "require-managed", + legacyDockerfileBuilds: false, + managedImages: { + exactDigestReferences: true, + platforms: [MANAGED_IMAGE_PLATFORM], + startupProfileContractVersions: [], + capabilityContractVersions: [MANAGED_IMAGE_CAPABILITY_CONTRACT_VERSION], + }, + }, + catalog: CATALOG, + policy: "require-managed", + }), + ).toThrow("does not support startup profile contract v1"); + }); + + it("does not hide a malformed supplied contract behind the legacy fallback (#7744)", () => { + const mutableCatalog = { + ...CATALOG, + hermes: { + ...contractFor("hermes"), + reference: `${MANAGED_IMAGE_REPOSITORIES.hermes}:${SOURCE_RELEASE}`, + }, + }; + + expect(() => + resolveSandboxWorkloadSource({ + agentName: "hermes", + legacyDockerfilePath: "agents/hermes/Dockerfile", + runtime: managedRuntime("podman"), + catalog: mutableCatalog, + }), + ).toThrow("failed closed validation"); + }); +}); diff --git a/src/lib/onboard/session-bootstrap.ts b/src/lib/onboard/session-bootstrap.ts index 090b708cb8..dcce937654 100644 --- a/src/lib/onboard/session-bootstrap.ts +++ b/src/lib/onboard/session-bootstrap.ts @@ -24,6 +24,8 @@ export interface OnboardSessionBootstrapInput { requestedToolDisclosure?: ToolDisclosure | null; requestedObservabilityEnabled?: boolean | null; stationExpressIntent?: StationExpressResumeIntent | null; + /** Compute identity resolved before the session is created. */ + openshellDriver?: string | null; } export interface OnboardSessionBootstrapDeps { @@ -293,7 +295,11 @@ function prepareFreshSession( observabilityEnabled: input.requestedObservabilityEnabled === true, observabilityRequestedExplicitly: typeof input.requestedObservabilityEnabled === "boolean", stationExpressIntent: input.stationExpressIntent ?? null, - metadata: { gatewayName: "nemoclaw", fromDockerfile: fromDockerfile || null }, + metadata: { + gatewayName: "nemoclaw", + fromDockerfile: fromDockerfile || null, + openshellDriver: input.openshellDriver ?? null, + }, }), ); return { session, fromDockerfile }; diff --git a/src/lib/onboard/setup-nim-flow.test.ts b/src/lib/onboard/setup-nim-flow.test.ts index b661a5fba5..8b3a592d67 100644 --- a/src/lib/onboard/setup-nim-flow.test.ts +++ b/src/lib/onboard/setup-nim-flow.test.ts @@ -302,6 +302,46 @@ describe("createSetupNim", () => { }); }); + it("lets a compute runtime remove every unqualified local provider before selection", async () => { + const filterProviderOptions = vi.fn>( + (options) => options.filter(({ key }) => Object.hasOwn(REMOTE_PROVIDER_CONFIG, key)), + ); + const handleRemoteProviderSelection = vi.fn( + async ({ selected }, state) => { + expect(selected.key).toBe("build"); + state.model = "nvidia/nemotron"; + state.provider = "nvidia-prod"; + state.endpointUrl = "https://integrate.api.nvidia.com/v1"; + state.credentialEnv = "NVIDIA_INFERENCE_API_KEY"; + return "selected"; + }, + ); + const setupNim = createSetupNim( + makeDeps({ + experimental: true, + filterProviderOptions, + loadRoutedProfile: () => ({ router: { enabled: true } }), + detectInferenceProviderHostState: () => + makeHostState({ + hasOllama: true, + ollamaHost: "127.0.0.1", + ollamaRunning: true, + gpuNimCapable: true, + vllmRunning: true, + vllmEntries: [{ key: "vllm", label: "Local vLLM" }], + }), + handleRemoteProviderSelection, + }), + ); + + const result = await setupNim({ nimCapable: true } as never); + + const offeredKeys = filterProviderOptions.mock.calls[0]?.[0].map(({ key }) => key); + expect(offeredKeys).toEqual(expect.arrayContaining(["nim-local", "ollama", "vllm", "routed"])); + expect(result.provider).toBe("nvidia-prod"); + expect(handleRemoteProviderSelection).toHaveBeenCalledOnce(); + }); + it("re-enters provider selection when a handler requests a retry (#6245)", async () => { vi.stubEnv("NEMOCLAW_PROVIDER", ""); const prompt = vi.fn(async () => ""); diff --git a/src/lib/onboard/setup-nim-flow.ts b/src/lib/onboard/setup-nim-flow.ts index 541293fda5..7476152668 100644 --- a/src/lib/onboard/setup-nim-flow.ts +++ b/src/lib/onboard/setup-nim-flow.ts @@ -81,6 +81,7 @@ export interface SetupNimFlowDeps { probeVllm?: boolean; }): InferenceProviderHostState; getAgentInferenceProviderOptions(agent: AgentDefinition | null | undefined): string[]; + filterProviderOptions?(options: readonly ProviderMenuChoice[]): ProviderMenuChoice[]; loadRoutedProfile(): { router?: { enabled?: boolean } } | null | undefined; readRecordedProvider( sandboxName: string | null | undefined, @@ -185,6 +186,18 @@ function requireSelectedProvider( return selected; } +function availableProviderOptions( + options: readonly ProviderMenuChoice[], + deps: Pick, +): ProviderMenuChoice[] { + const available = deps.filterProviderOptions ? deps.filterProviderOptions(options) : [...options]; + if (available.length === 0) { + deps.error(" No inference providers are available for the selected compute runtime."); + deps.exitProcess(1); + } + return available; +} + function resolveValidationInferenceApi( selectedKey: string, provider: string, @@ -374,7 +387,7 @@ export function createSetupNim( const agentProviderOptions = deps.getAgentInferenceProviderOptions(agent); const blueprintRouterCfg = deps.loadRoutedProfile(); - const { options, hermesProviderAvailable } = buildInferenceProviderMenu({ + const providerMenu = buildInferenceProviderMenu({ remoteProviderConfig: deps.remoteProviderConfig, agentProviderOptions, experimental: deps.experimental, @@ -397,6 +410,8 @@ export function createSetupNim( vllmEntries, routedEnabled: blueprintRouterCfg?.router?.enabled === true, }); + const options = availableProviderOptions(providerMenu.options, deps); + const { hermesProviderAvailable } = providerMenu; function rejectWindowsHostOllama(providerKey: string, windowsHostSelected: boolean): boolean { return deps.rejectWindowsHostOllama( diff --git a/src/lib/onboard/summary.test.ts b/src/lib/onboard/summary.test.ts index cdf42df98d..e234f16a62 100644 --- a/src/lib/onboard/summary.test.ts +++ b/src/lib/onboard/summary.test.ts @@ -16,7 +16,7 @@ describe("onboard summary helpers", () => { webSearchConfig: { fetchEnabled: true }, enabledChannels: ["telegram", "slack"], sandboxName: "my-assistant", - notes: ["Sandbox build typically takes 5–15 minutes on this host."], + notes: ["Local Dockerfile build typically takes 5–15 minutes on this host."], }); assert.ok(summary.includes("Review configuration"), "summary has review heading"); @@ -31,7 +31,9 @@ describe("onboard summary helpers", () => { assert.ok(summary.includes("telegram, slack"), "summary lists enabled channels"); assert.ok(summary.includes("my-assistant"), "summary shows sandbox name"); assert.ok( - summary.includes("Note: Sandbox build typically takes 5–15 minutes on this host."), + summary.includes( + "Note: Local Dockerfile build typically takes 5–15 minutes on this host.", + ), "summary renders notes under sandbox name", ); @@ -81,30 +83,39 @@ describe("onboard summary helpers", () => { assert.ok(tavilySummary.includes("enabled (Tavily Search)")); }); - it("formatSandboxBuildEstimateNote warns when runtime is under-provisioned (#2514)", () => { - const note = formatSandboxBuildEstimateNote({ - isContainerRuntimeUnderProvisioned: true, - dockerCpus: 2, - dockerMemTotalBytes: 2 * 1024 ** 3, - }); + it("formatSandboxBuildEstimateNote warns for an under-provisioned custom build (#2514)", () => { + const note = formatSandboxBuildEstimateNote( + { + isContainerRuntimeUnderProvisioned: true, + dockerCpus: 2, + dockerMemTotalBytes: 2 * 1024 ** 3, + }, + "custom-dockerfile", + ); assert.ok(note != null && note.length > 0, "returns a note"); - assert.match(note as string, /under-provisioned/i, "note flags under-provisioned host"); + assert.match(note as string, /custom Dockerfile build/u); }); - it("formatSandboxBuildEstimateNote returns a tighter range on a generous host (#2514)", () => { - const note = formatSandboxBuildEstimateNote({ - isContainerRuntimeUnderProvisioned: false, - dockerCpus: 12, - dockerMemTotalBytes: 32 * 1024 ** 3, - }); + it("formatSandboxBuildEstimateNote returns a range for a trusted fallback build (#7744)", () => { + const note = formatSandboxBuildEstimateNote( + { + isContainerRuntimeUnderProvisioned: false, + dockerCpus: 12, + dockerMemTotalBytes: 32 * 1024 ** 3, + }, + "trusted-dockerfile-fallback", + ); assert.ok(note != null, "returns a note"); assert.match(note ?? "", /\b3[–-]\d+\s+minutes\b/, "tight range starts at 3 minutes"); }); - it("formatSandboxBuildEstimateNote returns null when no runtime resource signal is available (#2514)", () => { - const note = formatSandboxBuildEstimateNote({ - isContainerRuntimeUnderProvisioned: false, - }); + it("formatSandboxBuildEstimateNote returns null without a resource signal (#2514)", () => { + const note = formatSandboxBuildEstimateNote( + { + isContainerRuntimeUnderProvisioned: false, + }, + "trusted-dockerfile-fallback", + ); assert.strictEqual(note, null); }); }); diff --git a/src/lib/onboard/summary.ts b/src/lib/onboard/summary.ts index 716c509250..0d3479443c 100644 --- a/src/lib/onboard/summary.ts +++ b/src/lib/onboard/summary.ts @@ -22,6 +22,8 @@ export type SandboxBuildEstimateHost = { dockerMemTotalBytes?: number; }; +export type SandboxLocalBuildKind = "custom-dockerfile" | "trusted-dockerfile-fallback"; + export type OnboardConfigSummary = { provider: string | null; model: string | null; @@ -55,10 +57,13 @@ function normalizeHermesAuthMethod(value: string | null | undefined): HermesAuth return null; } -export function formatSandboxBuildEstimateNote(host: SandboxBuildEstimateHost): string | null { +export function formatSandboxBuildEstimateNote( + host: SandboxBuildEstimateHost, + buildKind: SandboxLocalBuildKind, +): string | null { if (host.isContainerRuntimeUnderProvisioned) { return ( - "Container runtime is under-provisioned; the sandbox build may take 30+ minutes " + + `Container runtime is under-provisioned; the ${buildKind === "custom-dockerfile" ? "custom Dockerfile" : "trusted Dockerfile fallback"} build may take 30+ minutes ` + "or stall. See preflight warning above." ); } @@ -67,9 +72,9 @@ export function formatSandboxBuildEstimateNote(host: SandboxBuildEstimateHost): if (typeof cpus === "number" && typeof memBytes === "number") { const memGiB = memBytes / 1024 ** 3; if (cpus >= 8 && memGiB >= 16) { - return "Sandbox build typically takes 3–8 minutes on this host."; + return "Local Dockerfile build typically takes 3–8 minutes on this host."; } - return "Sandbox build typically takes 5–15 minutes on this host."; + return "Local Dockerfile build typically takes 5–15 minutes on this host."; } return null; } diff --git a/src/lib/onboard/types.ts b/src/lib/onboard/types.ts index b60f238fff..83b906869a 100644 --- a/src/lib/onboard/types.ts +++ b/src/lib/onboard/types.ts @@ -83,6 +83,8 @@ export interface SandboxCreateIntent { export type OnboardOptions = { nonInteractive?: boolean; recreateSandbox?: boolean; + /** Public requests and internal persisted driver identities share one pluggable seam. */ + computeDriver?: string; authoritativeResumeConfig?: boolean; /** Internal endpoint provenance preserved across an authoritative rebuild. */ endpointSource?: import("../inference/selection").InferenceEndpointSource | null; @@ -102,6 +104,8 @@ export type OnboardOptions = { providerRecoveryReceipt?: import("./rebuild-route-handoff").ProviderRecoveryReceipt; /** Internal one-shot handoff for the exact image context validated before rebuild deletion. */ preparedImageRebuild?: import("./prepared-dcode-rebuild").PreparedImageRebuildHandoff; + /** Internal immutable managed-image/profile handoff validated before rebuild deletion. */ + managedWorkloadRebuild?: import("./workload/rebuild").ManagedWorkloadRebuildHandoff; /** Internal hint for resolving the sandbox base image without repeating remote discovery. */ baseImageResolutionHint?: | import("../sandbox-base-image").SandboxBaseImageResolutionMetadata diff --git a/src/lib/onboard/workload/preparation.ts b/src/lib/onboard/workload/preparation.ts new file mode 100644 index 0000000000..1e242ae284 --- /dev/null +++ b/src/lib/onboard/workload/preparation.ts @@ -0,0 +1,192 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + normalizeManagedImageRelease, + resolveManagedImageCatalogFromGhcr, +} from "../managed-image/catalog"; +import { + isShippedManagedImageAgent, + type ManagedImageContractCatalog, + parseManagedImageContractV1, + SHIPPED_MANAGED_IMAGE_AGENTS, +} from "../managed-image/contract"; +import { + type ManagedImageSelectionPolicy, + managedImageRuntimeSupportError, + resolveSandboxWorkloadSource, + type SandboxWorkloadRuntimeCapabilities, + type SandboxWorkloadSource, +} from "./source"; + +type ResolveManagedImageCatalog = (options: { + readonly release: string; +}) => Promise; + +export interface PrepareSandboxWorkloadSourceInput { + readonly agentName: string; + readonly legacyDockerfilePath: string; + readonly customDockerfilePath?: string | null; + readonly runtime: SandboxWorkloadRuntimeCapabilities; + readonly version: string; + readonly policy?: ManagedImageSelectionPolicy; +} + +export interface PrepareSandboxWorkloadSourceDependencies { + readonly resolveCatalog?: ResolveManagedImageCatalog; +} + +export interface PreparedSandboxWorkloadSource { + readonly source: SandboxWorkloadSource; + readonly release: string | null; + /** + * A non-secret operator diagnostic for an allowed legacy fallback. Selection + * errors remain exceptions when managed images are required. + */ + readonly fallbackDiagnostic: string | null; +} + +export class SandboxWorkloadPreparationError extends Error { + constructor(message: string, options?: ErrorOptions) { + super(`Sandbox workload preparation failed: ${message}`, options); + this.name = "SandboxWorkloadPreparationError"; + } +} + +function diagnostic(error: unknown): string { + if (error instanceof Error) return error.message; + return "managed image catalog resolution failed"; +} + +function unavailableResult( + input: PrepareSandboxWorkloadSourceInput, + message: string, +): PreparedSandboxWorkloadSource { + if ((input.policy ?? input.runtime.managedImageSelectionPolicy) === "require-managed") { + throw new SandboxWorkloadPreparationError(message); + } + const source = resolveSandboxWorkloadSource({ + agentName: input.agentName, + legacyDockerfilePath: input.legacyDockerfilePath, + customDockerfilePath: input.customDockerfilePath, + runtime: input.runtime, + catalog: {}, + policy: input.policy ?? input.runtime.managedImageSelectionPolicy, + }); + return { + source, + release: null, + fallbackDiagnostic: message, + }; +} + +function requireCompleteManagedImageCatalog( + catalog: ManagedImageContractCatalog, + expectedRelease: string, +): void { + let cohortRevision: string | null = null; + let publicationCohort: string | null = null; + for (const agent of SHIPPED_MANAGED_IMAGE_AGENTS) { + const candidate = catalog[agent]; + if (candidate === undefined) { + throw new SandboxWorkloadPreparationError( + `managed image catalog is incomplete; '${agent}' is missing`, + ); + } + try { + const contract = parseManagedImageContractV1(candidate, agent); + if (contract.source.release !== expectedRelease) { + throw new SandboxWorkloadPreparationError( + `managed image catalog contract for '${agent}' belongs to '${contract.source.release}', not '${expectedRelease}'`, + ); + } + cohortRevision ??= contract.source.revision; + if (contract.source.revision !== cohortRevision) { + throw new SandboxWorkloadPreparationError( + "managed image catalog does not identify one all-agent source revision", + ); + } + publicationCohort ??= contract.source.cohort; + if (contract.source.cohort !== publicationCohort) { + throw new SandboxWorkloadPreparationError( + "managed image catalog does not identify one all-agent publication cohort", + ); + } + } catch (error) { + if (error instanceof SandboxWorkloadPreparationError) throw error; + throw new SandboxWorkloadPreparationError( + `managed image catalog contract for '${agent}' failed closed validation`, + { cause: error }, + ); + } + } +} + +/** + * Resolve a stock workload to an immutable managed image without fetching a + * catalog for custom, unshipped, or incapable runtime paths. + * + * The public catalog is resolved as one all-agent unit. A release cannot + * accidentally advertise buildless support for only OpenClaw while Hermes or + * DCode is missing. + */ +export async function prepareSandboxWorkloadSource( + input: PrepareSandboxWorkloadSourceInput, + dependencies: PrepareSandboxWorkloadSourceDependencies = {}, +): Promise { + const policy = input.policy ?? input.runtime.managedImageSelectionPolicy; + const cannotSelectManaged = + input.customDockerfilePath != null || + !isShippedManagedImageAgent(input.agentName) || + managedImageRuntimeSupportError(input.runtime) !== null; + if (cannotSelectManaged) { + return { + source: resolveSandboxWorkloadSource({ + agentName: input.agentName, + legacyDockerfilePath: input.legacyDockerfilePath, + customDockerfilePath: input.customDockerfilePath, + runtime: input.runtime, + catalog: {}, + policy, + }), + release: null, + fallbackDiagnostic: null, + }; + } + + let release: string; + try { + release = normalizeManagedImageRelease(input.version); + } catch (error) { + return unavailableResult( + input, + `managed image release for CLI version '${input.version}' is unavailable: ${diagnostic(error)}`, + ); + } + + let catalog: ManagedImageContractCatalog; + try { + catalog = await ( + dependencies.resolveCatalog ?? ((options) => resolveManagedImageCatalogFromGhcr(options)) + )({ release }); + } catch (error) { + return unavailableResult( + input, + `managed image catalog '${release}' is unavailable: ${diagnostic(error)}`, + ); + } + requireCompleteManagedImageCatalog(catalog, release); + + return { + source: resolveSandboxWorkloadSource({ + agentName: input.agentName, + legacyDockerfilePath: input.legacyDockerfilePath, + customDockerfilePath: input.customDockerfilePath, + runtime: input.runtime, + catalog, + policy, + }), + release, + fallbackDiagnostic: null, + }; +} diff --git a/src/lib/onboard/workload/rebuild.ts b/src/lib/onboard/workload/rebuild.ts new file mode 100644 index 0000000000..e31bdfdd7e --- /dev/null +++ b/src/lib/onboard/workload/rebuild.ts @@ -0,0 +1,456 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { isDeepStrictEqual } from "node:util"; + +import { getVersion } from "../../core/version"; +import type { SandboxEntry, SandboxWorkloadReceipt } from "../../state/registry/types"; +import { cloneSandboxWorkloadReceipt } from "../../state/registry/workload"; +import type { ResolvedCorporateCa } from "../corporate-ca-types"; +import { + isShippedManagedImageAgent, + MANAGED_IMAGE_CAPABILITY_CONTRACT_VERSION, + MANAGED_IMAGE_CONTRACT_VERSION, + MANAGED_IMAGE_PLATFORM, + MANAGED_IMAGE_REPOSITORIES, + MANAGED_IMAGE_SOURCE_REPOSITORY, + MANAGED_IMAGE_STARTUP_PROFILE_CONTRACT_VERSION, + type ManagedImageContractV1, + parseManagedImageContractV1, + type ShippedManagedImageAgent, +} from "../managed-image/contract"; +import { validateManagedStartupCorporateCaTransport } from "../managed-startup/application"; +import { + type BuiltManagedStartupOnboardProfile, + buildManagedStartupOnboardProfile, + type ManagedStartupOnboardProfileInput, +} from "../managed-startup/onboard-profile"; +import { + decodeManagedStartupProfile, + type ManagedStartupProfile, + type ManagedStartupReasoningEffort, +} from "../managed-startup/profile"; +import { + type PreparedSandboxWorkloadSource, + prepareSandboxWorkloadSource, + SandboxWorkloadPreparationError, +} from "./preparation"; +import { + type ManagedImageWorkloadSource, + resolveSandboxWorkloadSource, + type SandboxWorkloadRuntimeCapabilities, +} from "./source"; + +const MANAGED_REFERENCE_PREFIX = "ghcr.io/nvidia/nemoclaw/"; +const HOST_PROXY_ENV_NAMES = [ + "HTTP_PROXY", + "HTTPS_PROXY", + "NO_PROXY", + "http_proxy", + "https_proxy", + "no_proxy", +] as const; + +type ManagedWorkloadReceipt = Extract; + +export interface ManagedWorkloadRebuildCatalogHandoff { + readonly schemaVersion: 1; + readonly agent: ShippedManagedImageAgent; + /** Pre-delete and rollback authority for the workload that is being replaced. */ + readonly previousReceipt: ManagedWorkloadReceipt; + readonly previousContract: ManagedImageContractV1; + readonly previousProfile: ManagedStartupProfile; + /** Exact current-release image selected from one complete all-agent catalog. */ + readonly replacement: PreparedSandboxWorkloadSource & { + readonly source: ManagedImageWorkloadSource; + }; + /** Validated CA material retained across a profile-only rebuild. */ + readonly corporateCa: ResolvedCorporateCa | null; +} + +export interface ManagedWorkloadRebuildHandoff extends ManagedWorkloadRebuildCatalogHandoff { + /** + * Fully rendered replacement profile. It is prepared before the outer + * rebuild mutates the registry or destroys the old sandbox, then consumed + * verbatim by inner onboarding. + */ + readonly replacementProfile: BuiltManagedStartupOnboardProfile; +} + +export class ManagedWorkloadRebuildError extends Error { + constructor(message: string, options?: ErrorOptions) { + super(`Managed workload rebuild preflight failed: ${message}`, options); + this.name = "ManagedWorkloadRebuildError"; + } +} + +function isManagedImageReference(value: unknown): value is string { + return typeof value === "string" && value.startsWith(MANAGED_REFERENCE_PREFIX); +} + +function exactAgent(value: string): ShippedManagedImageAgent { + if (isShippedManagedImageAgent(value)) return value; + throw new ManagedWorkloadRebuildError(`'${value}' is not a shipped managed-image agent`); +} + +function contractFromReceipt( + receipt: ManagedWorkloadReceipt, + agent: ShippedManagedImageAgent, +): ManagedImageContractV1 { + const image = MANAGED_IMAGE_REPOSITORIES[agent]; + const digest = receipt.reference.slice(`${image}@`.length); + return parseManagedImageContractV1( + { + contractVersion: MANAGED_IMAGE_CONTRACT_VERSION, + agent, + platform: MANAGED_IMAGE_PLATFORM, + image, + digest, + reference: receipt.reference, + source: { + repository: MANAGED_IMAGE_SOURCE_REPOSITORY, + revision: receipt.sourceRevision, + release: receipt.release, + cohort: receipt.sourceCohort, + }, + startupProfileContractVersion: receipt.startupProfileContractVersion, + capabilityContractVersion: receipt.capabilityContractVersion, + }, + agent, + ); +} + +function validateCorporateCaTransport( + receipt: ManagedWorkloadReceipt, + profile: ManagedStartupProfile, +): ResolvedCorporateCa | null { + let bytes: Buffer | null; + try { + bytes = validateManagedStartupCorporateCaTransport(receipt.corporateCaB64, profile); + } catch (error) { + throw new ManagedWorkloadRebuildError( + "the corporate CA transport does not match the recorded startup profile", + { cause: error }, + ); + } + return bytes === null + ? null + : { + pem: bytes.toString("utf8"), + sourcePath: "managed-workload-rebuild-receipt", + sourceEnv: "managed-workload-rebuild-receipt", + }; +} + +/** + * Validate the immutable workload and canonical secret-free startup profile + * recorded for the sandbox being replaced. + */ +function readManagedWorkloadRebuildAuthority( + entry: Pick, +): { + agent: ShippedManagedImageAgent; + receipt: ManagedWorkloadReceipt; + contract: ManagedImageContractV1; + profile: ManagedStartupProfile; + corporateCa: ResolvedCorporateCa | null; +} | null { + const recordedWorkload = entry.workload; + const managedLooking = + isManagedImageReference(entry.imageTag) || + (recordedWorkload !== undefined && recordedWorkload.kind === "managed-image"); + if (!managedLooking) return null; + + const cloned = cloneSandboxWorkloadReceipt(recordedWorkload); + if (cloned?.kind !== "managed-image") { + throw new ManagedWorkloadRebuildError( + "the managed image has no valid durable workload receipt", + ); + } + if (entry.imageTag !== cloned.reference) { + throw new ManagedWorkloadRebuildError( + "the registry image reference does not match the durable workload receipt", + ); + } + if (entry.fromDockerfile) { + throw new ManagedWorkloadRebuildError( + "a managed image receipt cannot be combined with a custom Dockerfile", + ); + } + + const agent = exactAgent(entry.agent?.trim() || "openclaw"); + const contract = contractFromReceipt(cloned, agent); + let profile: ManagedStartupProfile; + try { + profile = decodeManagedStartupProfile(cloned.encodedProfile); + } catch (error) { + throw new ManagedWorkloadRebuildError("the recorded startup profile is invalid", { + cause: error, + }); + } + if (profile.agent !== agent) { + throw new ManagedWorkloadRebuildError( + `the recorded startup profile belongs to '${profile.agent}', not '${agent}'`, + ); + } + const corporateCa = validateCorporateCaTransport(cloned, profile); + + return { + agent, + receipt: cloned, + contract, + profile, + corporateCa, + }; +} + +export const managedWorkloadRebuildDependencies = { + prepareSandboxWorkloadSource, +}; + +/** + * Validate the old receipt, then resolve the current CLI release as a complete + * all-agent catalog before any mutation. Managed rebuild never falls back to a + * Dockerfile and never turns an upgrade into reuse of the stale image. + */ +export async function prepareManagedWorkloadRebuildHandoff( + entry: Pick, + options: { + readonly runtime: SandboxWorkloadRuntimeCapabilities; + readonly version?: string; + }, +): Promise { + const authority = readManagedWorkloadRebuildAuthority(entry); + if (!authority) return null; + + let replacement: PreparedSandboxWorkloadSource; + try { + replacement = await managedWorkloadRebuildDependencies.prepareSandboxWorkloadSource({ + agentName: authority.agent, + legacyDockerfilePath: "managed-rebuild-must-not-stage-this-dockerfile", + runtime: options.runtime, + version: options.version ?? getVersion(), + policy: "require-managed", + }); + } catch (error) { + throw new ManagedWorkloadRebuildError( + "the current release's complete managed-image catalog is unavailable or invalid", + { cause: error }, + ); + } + if (replacement.source.kind !== "managed-image") { + throw new ManagedWorkloadRebuildError( + "the current release did not resolve to an immutable managed image", + ); + } + return Object.freeze({ + schemaVersion: 1 as const, + agent: authority.agent, + previousReceipt: authority.receipt, + previousContract: authority.contract, + previousProfile: authority.profile, + replacement: { + ...replacement, + source: replacement.source, + }, + corporateCa: authority.corporateCa, + }); +} + +/** Revalidate the retained handoff against the live registry row. */ +export function managedWorkloadRebuildHandoffMatchesEntry( + handoff: ManagedWorkloadRebuildCatalogHandoff, + entry: Pick | null, +): boolean { + if (!entry) return false; + try { + const current = readManagedWorkloadRebuildAuthority(entry); + return ( + current !== null && + current.agent === handoff.agent && + isDeepStrictEqual(current.receipt, handoff.previousReceipt) && + isDeepStrictEqual(current.contract, handoff.previousContract) && + isDeepStrictEqual(current.profile, handoff.previousProfile) + ); + } catch { + return false; + } +} + +export interface ManagedWorkloadRebuildProfileOverrides { + readonly openClawContextWindow?: number; + readonly openClawReasoning?: boolean; + readonly openClawReasoningEffort?: ManagedStartupReasoningEffort; +} + +/** + * Keep the source sandbox's proxy contract while allowing every other + * profile-backed rebuild setting to be resolved from current authoritative + * intent. Credential-bearing proxy values remain launch-only and must be + * reacquired separately during preflight. + */ +export function managedWorkloadRebuildProfileEnvironment( + handoff: ManagedWorkloadRebuildCatalogHandoff, + environment: NodeJS.ProcessEnv, + overrides: ManagedWorkloadRebuildProfileOverrides = {}, +): NodeJS.ProcessEnv { + const result: NodeJS.ProcessEnv = { + NEMOCLAW_PROXY_HOST: handoff.previousProfile.proxy.managedHost, + NEMOCLAW_PROXY_PORT: String(handoff.previousProfile.proxy.managedPort), + }; + const previous = handoff.previousProfile; + if (previous.agent === "openclaw" && previous.agentConfig.agent === "openclaw") { + const config = previous.agentConfig; + const contextWindow = overrides.openClawContextWindow ?? previous.tuning.contextWindow; + if (contextWindow !== null) { + result.NEMOCLAW_CONTEXT_WINDOW = String(contextWindow); + } + if (previous.tuning.maxTokens !== null) { + result.NEMOCLAW_MAX_TOKENS = String(previous.tuning.maxTokens); + } + const reasoning = overrides.openClawReasoning ?? previous.tuning.reasoning; + if (reasoning !== null) { + result.NEMOCLAW_REASONING = String(reasoning); + } + const reasoningEffort = overrides.openClawReasoningEffort ?? previous.tuning.reasoningEffort; + if (reasoningEffort !== null) { + result.NEMOCLAW_REASONING_EFFORT = reasoningEffort; + } + if (previous.inference.inputModalities !== null) { + result.NEMOCLAW_INFERENCE_INPUTS = previous.inference.inputModalities.join(","); + } + result.NEMOCLAW_AGENT_TIMEOUT = String(config.agentTimeoutSeconds); + if (config.heartbeatEvery !== null) { + result.NEMOCLAW_AGENT_HEARTBEAT_EVERY = config.heartbeatEvery; + } + result.NEMOCLAW_EXTRA_AGENTS_JSON_B64 = Buffer.from( + JSON.stringify(config.extraAgents), + "utf8", + ).toString("base64"); + result.NEMOCLAW_MINIMAL_BOOTSTRAP = config.minimalBootstrap ? "1" : "0"; + result.NEMOCLAW_OPENCLAW_OTEL = config.otel.enabled ? "1" : "0"; + result.NEMOCLAW_OPENCLAW_OTEL_ENDPOINT = config.otel.endpointUrl; + result.NEMOCLAW_OPENCLAW_OTEL_SERVICE_NAME = config.otel.serviceName; + result.NEMOCLAW_OPENCLAW_OTEL_SAMPLE_RATE = String(config.otel.sampleRate); + } else if (previous.agent === "hermes" && previous.tuning.contextWindow !== null) { + result.NEMOCLAW_CONTEXT_WINDOW = String(previous.tuning.contextWindow); + } + + if (handoff.previousReceipt.credentialProxyReplayRequired) { + for (const name of HOST_PROXY_ENV_NAMES) { + const value = environment[name]; + if (value !== undefined) result[name] = value; + } + return result; + } + const proxy = handoff.previousProfile.proxy; + if (proxy.hostHttpUrl) result.HTTP_PROXY = proxy.hostHttpUrl; + if (proxy.hostHttpsUrl) result.HTTPS_PROXY = proxy.hostHttpsUrl; + if (proxy.hostNoProxy.length > 0) result.NO_PROXY = proxy.hostNoProxy.join(","); + return result; +} + +type ManagedWorkloadRebuildProfileInput = Omit< + ManagedStartupOnboardProfileInput, + "agentName" | "environment" | "corporateCaOverride" +>; + +/** + * Render every fallible replacement-profile input while the old sandbox is + * still intact. Mutable rebuild state is supplied explicitly by the caller; + * receipt-only tuning, managed proxy, host-proxy intent, and CA material are + * reconstructed from the validated previous profile. + */ +export function stageManagedWorkloadRebuildProfile( + handoff: ManagedWorkloadRebuildCatalogHandoff, + input: ManagedWorkloadRebuildProfileInput, + environment: NodeJS.ProcessEnv = process.env, + overrides: ManagedWorkloadRebuildProfileOverrides = {}, +): ManagedWorkloadRebuildHandoff { + let replacementProfile: BuiltManagedStartupOnboardProfile; + try { + replacementProfile = buildManagedStartupOnboardProfile({ + ...input, + agentName: handoff.agent, + environment: managedWorkloadRebuildProfileEnvironment(handoff, environment, overrides), + corporateCaOverride: handoff.corporateCa, + }); + } catch (error) { + throw new ManagedWorkloadRebuildError( + `the replacement startup profile could not be rendered from authoritative rebuild state: ${ + error instanceof Error ? error.message : String(error) + }`, + { cause: error }, + ); + } + if ( + replacementProfile.credentialProxyReplayRequired !== + handoff.previousReceipt.credentialProxyReplayRequired + ) { + throw new ManagedWorkloadRebuildError( + "the replacement startup profile changed the durable credential-proxy requirement", + ); + } + if (replacementProfile.profile.agent !== handoff.agent) { + throw new ManagedWorkloadRebuildError( + "the replacement startup profile does not match the selected managed-image agent", + ); + } + return Object.freeze({ + ...handoff, + replacementProfile, + }); +} + +/** + * Bind the receipt-derived contract to the selected driver capability. This + * uses the same managed workload source resolver as fresh onboarding and never + * consults a mutable release pointer. + */ +export function prepareSandboxWorkloadSourceFromRebuildHandoff( + handoff: ManagedWorkloadRebuildCatalogHandoff, + runtime: SandboxWorkloadRuntimeCapabilities, +): PreparedSandboxWorkloadSource { + let source; + try { + source = resolveSandboxWorkloadSource({ + agentName: handoff.agent, + legacyDockerfilePath: "", + runtime, + catalog: { [handoff.agent]: handoff.replacement.source.contract }, + policy: "require-managed", + }); + } catch (error) { + throw new SandboxWorkloadPreparationError( + "the recorded managed workload is not supported by the selected runtime", + { cause: error }, + ); + } + if (source.kind !== "managed-image") { + throw new SandboxWorkloadPreparationError( + "the recorded managed workload did not resolve to an immutable image", + ); + } + if ( + source.reference !== handoff.replacement.source.reference || + source.contract.source.cohort !== handoff.replacement.source.contract.source.cohort || + source.contract.source.revision !== handoff.replacement.source.contract.source.revision + ) { + throw new SandboxWorkloadPreparationError( + "the recorded managed workload changed during source resolution", + ); + } + if ( + source.contract.capabilityContractVersion !== MANAGED_IMAGE_CAPABILITY_CONTRACT_VERSION || + source.contract.startupProfileContractVersion !== MANAGED_IMAGE_STARTUP_PROFILE_CONTRACT_VERSION + ) { + throw new SandboxWorkloadPreparationError( + "the recorded managed workload uses an unsupported contract version", + ); + } + return { + source, + release: handoff.replacement.release, + fallbackDiagnostic: null, + }; +} diff --git a/src/lib/onboard/workload/runtime.ts b/src/lib/onboard/workload/runtime.ts new file mode 100644 index 0000000000..d7d87345b9 --- /dev/null +++ b/src/lib/onboard/workload/runtime.ts @@ -0,0 +1,98 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { OpenShellComputePlan } from "../compute/plan"; +import { + MANAGED_IMAGE_CAPABILITY_CONTRACT_VERSION, + MANAGED_IMAGE_PLATFORM, + MANAGED_IMAGE_STARTUP_PROFILE_CONTRACT_VERSION, +} from "../managed-image/contract"; +import type { ManagedImageSelectionPolicy, SandboxWorkloadRuntimeCapabilities } from "./source"; + +export type ManagedImageRuntimeSupport = NonNullable< + SandboxWorkloadRuntimeCapabilities["managedImages"] +>; + +export interface ManagedImageRuntimeProfile { + readonly support: ManagedImageRuntimeSupport | null; + /** OCI host architectures for which the published image cohort is complete. */ + readonly hostArchitectures: readonly string[]; + readonly managedImageSelectionPolicy: ManagedImageSelectionPolicy; + readonly legacyDockerfileBuilds: boolean; +} + +/** + * Runtime support is registered by OpenShell compute-driver identity rather + * than inferred from the gateway launcher. Podman and future MXC support can + * therefore opt into this contract without inheriting Docker lifecycle code. + */ +export type ManagedImageRuntimeProfileRegistry = Readonly< + Record +>; + +const COMPLETE_MANAGED_IMAGE_V1_SUPPORT = { + exactDigestReferences: true, + platforms: [MANAGED_IMAGE_PLATFORM], + startupProfileContractVersions: [MANAGED_IMAGE_STARTUP_PROFILE_CONTRACT_VERSION], + capabilityContractVersions: [MANAGED_IMAGE_CAPABILITY_CONTRACT_VERSION], +} as const satisfies ManagedImageRuntimeSupport; + +/** + * PR #7747 established Docker as the first explicit OpenShell compute-driver + * identity. PR #7744 adds further drivers by registering the same workload + * contract here; selection remains independent from driver lifecycle. + */ +export const CURRENT_MANAGED_IMAGE_RUNTIME_PROFILES = { + docker: { + support: COMPLETE_MANAGED_IMAGE_V1_SUPPORT, + hostArchitectures: ["amd64"], + managedImageSelectionPolicy: "prefer-managed", + legacyDockerfileBuilds: true, + }, + podman: { + support: COMPLETE_MANAGED_IMAGE_V1_SUPPORT, + hostArchitectures: ["amd64"], + managedImageSelectionPolicy: "require-managed", + legacyDockerfileBuilds: false, + }, + kubernetes: { + support: null, + hostArchitectures: [], + managedImageSelectionPolicy: "prefer-managed", + legacyDockerfileBuilds: true, + }, +} as const satisfies ManagedImageRuntimeProfileRegistry; + +function hostOciArchitecture(nodeArchitecture: string): string { + if (nodeArchitecture === "x64") return "amd64"; + return nodeArchitecture; +} + +function cloneRuntimeSupport(support: ManagedImageRuntimeSupport): ManagedImageRuntimeSupport { + return { + exactDigestReferences: support.exactDigestReferences, + platforms: [...support.platforms], + startupProfileContractVersions: [...support.startupProfileContractVersions], + capabilityContractVersions: [...support.capabilityContractVersions], + }; +} + +export function resolveSandboxWorkloadRuntimeCapabilities( + plan: Pick, + profiles: ManagedImageRuntimeProfileRegistry = CURRENT_MANAGED_IMAGE_RUNTIME_PROFILES, + nodeArchitecture: string = process.arch, +): SandboxWorkloadRuntimeCapabilities { + const profile = Object.hasOwn(profiles, plan.driverName) ? profiles[plan.driverName] : undefined; + const supportedHost = + profile?.support !== null && + profile?.hostArchitectures.includes(hostOciArchitecture(nodeArchitecture)) === true; + return { + driverName: plan.driverName, + managedImageSelectionPolicy: profile?.managedImageSelectionPolicy ?? "require-managed", + legacyDockerfileBuilds: profile?.legacyDockerfileBuilds ?? false, + managedImages: + profile === undefined || profile.support === null || !supportedHost + ? null + : cloneRuntimeSupport(profile.support), + }; +} diff --git a/src/lib/onboard/workload/source.ts b/src/lib/onboard/workload/source.ts new file mode 100644 index 0000000000..9f3e023d00 --- /dev/null +++ b/src/lib/onboard/workload/source.ts @@ -0,0 +1,185 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + isShippedManagedImageAgent, + MANAGED_IMAGE_CAPABILITY_CONTRACT_VERSION, + MANAGED_IMAGE_PLATFORM, + MANAGED_IMAGE_STARTUP_PROFILE_CONTRACT_VERSION, + type ManagedImageContractCatalog, + type ManagedImageContractV1, + parseManagedImageContractV1, +} from "../managed-image/contract"; + +export type ManagedImageSelectionPolicy = "prefer-managed" | "require-managed"; + +/** + * Capabilities are advertised by the selected OpenShell compute driver. + * `driverName` is intentionally opaque: Docker, Podman, MXC, and future + * drivers all negotiate the same runtime contract. + */ +export interface SandboxWorkloadRuntimeCapabilities { + readonly driverName: string; + /** + * Driver-owned selection policy. Buildless runtimes require a managed image; + * Docker-compatible runtimes may explicitly retain the trusted recipe path. + */ + readonly managedImageSelectionPolicy: ManagedImageSelectionPolicy; + /** Whether this driver can consume NemoClaw's legacy local Dockerfile build. */ + readonly legacyDockerfileBuilds: boolean; + readonly managedImages: { + readonly exactDigestReferences: boolean; + readonly platforms: readonly string[]; + readonly startupProfileContractVersions: readonly number[]; + readonly capabilityContractVersions: readonly number[]; + } | null; +} + +export type LegacyDockerfileReason = + | "custom-dockerfile" + | "agent-not-managed" + | "runtime-unsupported" + | "contract-unavailable"; + +export interface LegacyDockerfileWorkloadSource { + readonly kind: "legacy-dockerfile"; + readonly dockerfilePath: string; + readonly reason: LegacyDockerfileReason; +} + +export interface ManagedImageWorkloadSource { + readonly kind: "managed-image"; + readonly reference: ManagedImageContractV1["reference"]; + readonly contract: ManagedImageContractV1; +} + +export type SandboxWorkloadSource = LegacyDockerfileWorkloadSource | ManagedImageWorkloadSource; + +export interface ResolveSandboxWorkloadSourceOptions { + readonly agentName: string; + readonly legacyDockerfilePath: string; + readonly customDockerfilePath?: string | null; + readonly runtime: SandboxWorkloadRuntimeCapabilities; + readonly catalog: ManagedImageContractCatalog; + readonly policy?: ManagedImageSelectionPolicy; +} + +export class SandboxWorkloadSourceError extends Error { + constructor(message: string, options?: ErrorOptions) { + super(message, options); + this.name = "SandboxWorkloadSourceError"; + } +} + +function requireDockerfilePath(path: string, reason: LegacyDockerfileReason): string { + if (path.trim() === "") { + throw new SandboxWorkloadSourceError( + `Cannot use the legacy Dockerfile workload for ${reason}: no Dockerfile path was supplied.`, + ); + } + return path; +} + +function legacySource( + options: ResolveSandboxWorkloadSourceOptions, + reason: LegacyDockerfileReason, +): LegacyDockerfileWorkloadSource { + if (!options.runtime.legacyDockerfileBuilds) { + throw new SandboxWorkloadSourceError( + `Driver '${options.runtime.driverName}' cannot use the legacy Dockerfile workload for ${reason}.`, + ); + } + const selectedPath = + reason === "custom-dockerfile" + ? (options.customDockerfilePath ?? "") + : options.legacyDockerfilePath; + return { + kind: "legacy-dockerfile", + dockerfilePath: requireDockerfilePath(selectedPath, reason), + reason, + }; +} + +function unavailableSource( + options: ResolveSandboxWorkloadSourceOptions, + reason: Exclude, + detail: string, +): LegacyDockerfileWorkloadSource { + if ((options.policy ?? options.runtime.managedImageSelectionPolicy) === "require-managed") { + throw new SandboxWorkloadSourceError( + `Managed image workload is required for '${options.agentName}', but ${detail}.`, + ); + } + return legacySource(options, reason); +} + +export function managedImageRuntimeSupportError( + runtime: SandboxWorkloadRuntimeCapabilities, +): string | null { + const capabilities = runtime.managedImages; + if (capabilities === null) + return `driver '${runtime.driverName}' does not advertise managed images`; + if (!capabilities.exactDigestReferences) { + return `driver '${runtime.driverName}' does not support exact-digest image references`; + } + if (!capabilities.platforms.includes(MANAGED_IMAGE_PLATFORM)) { + return `driver '${runtime.driverName}' does not support ${MANAGED_IMAGE_PLATFORM}`; + } + if ( + !capabilities.startupProfileContractVersions.includes( + MANAGED_IMAGE_STARTUP_PROFILE_CONTRACT_VERSION, + ) + ) { + return `driver '${runtime.driverName}' does not support startup profile contract v${MANAGED_IMAGE_STARTUP_PROFILE_CONTRACT_VERSION}`; + } + if ( + !capabilities.capabilityContractVersions.includes(MANAGED_IMAGE_CAPABILITY_CONTRACT_VERSION) + ) { + return `driver '${runtime.driverName}' does not support capability contract v${MANAGED_IMAGE_CAPABILITY_CONTRACT_VERSION}`; + } + return null; +} + +export function resolveSandboxWorkloadSource( + options: ResolveSandboxWorkloadSourceOptions, +): SandboxWorkloadSource { + if (options.customDockerfilePath !== undefined && options.customDockerfilePath !== null) { + return legacySource(options, "custom-dockerfile"); + } + + if (!isShippedManagedImageAgent(options.agentName)) { + return unavailableSource( + options, + "agent-not-managed", + "the selected agent is not a shipped managed agent", + ); + } + + const runtimeSupportError = managedImageRuntimeSupportError(options.runtime); + if (runtimeSupportError !== null) { + return unavailableSource(options, "runtime-unsupported", runtimeSupportError); + } + + const candidate = options.catalog[options.agentName]; + if (candidate === undefined) { + return unavailableSource( + options, + "contract-unavailable", + "the catalog has no exact contract for that agent", + ); + } + + try { + const contract = parseManagedImageContractV1(candidate, options.agentName); + return { + kind: "managed-image", + reference: contract.reference, + contract, + }; + } catch (error) { + throw new SandboxWorkloadSourceError( + `Managed image contract for '${options.agentName}' failed closed validation.`, + { cause: error }, + ); + } +} diff --git a/src/lib/openshell-sandbox-list.ts b/src/lib/openshell-sandbox-list.ts index 76754b586e..2a2d0b04e2 100644 --- a/src/lib/openshell-sandbox-list.ts +++ b/src/lib/openshell-sandbox-list.ts @@ -24,6 +24,7 @@ export type SandboxListRecoveryResult = { }; export type CaptureSandboxListWithGatewayRecoveryOptions = { + computeDriver?: string; gatewayName?: string; }; @@ -46,6 +47,9 @@ export async function captureSandboxListWithGatewayRecovery( const recoveryOptions: Parameters[0] = { recoverableStates: ["missing_named", "named_unhealthy", "named_unreachable", "connected_other"], }; + if (options.computeDriver) { + recoveryOptions.computeDriver = options.computeDriver; + } if (options.gatewayName) { recoveryOptions.gatewayName = options.gatewayName; } diff --git a/src/lib/proxy/local-no-proxy.ts b/src/lib/proxy/local-no-proxy.ts new file mode 100644 index 0000000000..aa65144d8e --- /dev/null +++ b/src/lib/proxy/local-no-proxy.ts @@ -0,0 +1,35 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +const LOCAL_NO_PROXY_HOSTS = [ + "localhost", + "127.0.0.1", + "host.docker.internal", + "host.containers.internal", + "::1", + "0.0.0.0", + "inference.local", +] as const; + +/** + * Keep host loopback, container-host aliases, and OpenShell-managed inference + * off any forwarded host proxy chain. + */ +export function withLocalNoProxy(env: Record): void { + const hasProxy = env.HTTP_PROXY || env.HTTPS_PROXY || env.http_proxy || env.https_proxy; + if (!hasProxy) return; + for (const key of ["NO_PROXY", "no_proxy"] as const) { + const parts = (env[key] ?? "") + .split(",") + .map((value) => value.trim()) + .filter(Boolean); + let changed = false; + for (const host of LOCAL_NO_PROXY_HOSTS) { + if (!parts.includes(host)) { + parts.push(host); + changed = true; + } + } + if (changed) env[key] = parts.join(","); + } +} diff --git a/src/lib/sandbox/build-context.ts b/src/lib/sandbox/build-context.ts index 2bb0abafe0..67b9599f9a 100644 --- a/src/lib/sandbox/build-context.ts +++ b/src/lib/sandbox/build-context.ts @@ -71,6 +71,35 @@ function stageMcpToolDiscoveryRuntime(rootDir: string, buildCtx: string): void { normalizeReadModesForDockerCopy(path.join(buildCtx, "tools")); } +function stageManagedStartupRuntimeSources(rootDir: string, buildCtx: string): void { + const sourceLib = path.join(rootDir, "src", "lib"); + const stagedLib = path.join(buildCtx, "src", "lib"); + for (const directory of ["core", "security", "state"]) { + fs.mkdirSync(path.join(stagedLib, directory), { recursive: true }); + } + for (const fileName of ["json-types.ts", "ports.ts"]) { + fs.copyFileSync(path.join(sourceLib, "core", fileName), path.join(stagedLib, "core", fileName)); + } + fs.cpSync(path.join(sourceLib, "messaging"), path.join(stagedLib, "messaging"), { + recursive: true, + }); + fs.cpSync( + path.join(sourceLib, "onboard", "managed-startup"), + path.join(stagedLib, "onboard", "managed-startup"), + { recursive: true }, + ); + fs.copyFileSync( + path.join(sourceLib, "security", "credential-hash.ts"), + path.join(stagedLib, "security", "credential-hash.ts"), + ); + for (const fileName of ["paths.ts", "state-root.ts"]) { + fs.copyFileSync( + path.join(sourceLib, "state", fileName), + path.join(stagedLib, "state", fileName), + ); + } +} + function stageLegacySandboxBuildContext( rootDir: string, tmpDir: string = os.tmpdir(), @@ -94,11 +123,7 @@ function stageLegacySandboxBuildContext( recursive: true, }); normalizeReadModesForDockerCopy(path.join(buildCtx, "scripts")); - fs.cpSync( - path.join(rootDir, "src", "lib", "messaging"), - path.join(buildCtx, "src", "lib", "messaging"), - { recursive: true }, - ); + stageManagedStartupRuntimeSources(rootDir, buildCtx); fs.copyFileSync( path.join(rootDir, "src", "lib", "tool-disclosure.ts"), path.join(buildCtx, "src", "lib", "tool-disclosure.ts"), @@ -199,6 +224,10 @@ function stageOptimizedSandboxBuildContext( path.join(rootDir, "scripts", "nemoclaw-start.sh"), path.join(stagedScriptsDir, "nemoclaw-start.sh"), ); + fs.copyFileSync( + path.join(rootDir, "scripts", "managed-startup-hold.sh"), + path.join(stagedScriptsDir, "managed-startup-hold.sh"), + ); fs.copyFileSync( path.join(rootDir, "scripts", "gateway-control.sh"), path.join(stagedScriptsDir, "gateway-control.sh"), @@ -241,6 +270,10 @@ function stageOptimizedSandboxBuildContext( path.join(rootDir, "scripts", "lib", "sandbox-rlimits.sh"), path.join(stagedScriptsDir, "lib", "sandbox-rlimits.sh"), ); + fs.copyFileSync( + path.join(rootDir, "scripts", "lib", "entrypoint-env-wrapper.sh"), + path.join(stagedScriptsDir, "lib", "entrypoint-env-wrapper.sh"), + ); fs.copyFileSync( path.join(rootDir, "scripts", "lib", "openclaw_device_approval_policy.py"), path.join(stagedScriptsDir, "lib", "openclaw_device_approval_policy.py"), @@ -254,11 +287,7 @@ function stageOptimizedSandboxBuildContext( path.join(stagedScriptsDir, "lib", "normalize_mutable_config_perms.py"), ); // Build-time messaging applier used by OpenClaw and Hermes Dockerfiles. - fs.cpSync( - path.join(rootDir, "src", "lib", "messaging"), - path.join(buildCtx, "src", "lib", "messaging"), - { recursive: true }, - ); + stageManagedStartupRuntimeSources(rootDir, buildCtx); fs.copyFileSync( path.join(rootDir, "src", "lib", "tool-disclosure.ts"), path.join(buildCtx, "src", "lib", "tool-disclosure.ts"), diff --git a/src/lib/sandbox/create-stream-ready-gate.test.ts b/src/lib/sandbox/create-stream-ready-gate.test.ts index b946a0bb29..2ae581b4e3 100644 --- a/src/lib/sandbox/create-stream-ready-gate.test.ts +++ b/src/lib/sandbox/create-stream-ready-gate.test.ts @@ -4,11 +4,11 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { streamSandboxCreate } from "./create-stream"; -import { dockerEnv, FakeChild, makePollingOptions, vmEnv } from "./create-stream-test-fixtures"; import { getReadyCheckOutputPatterns, getReadyCheckOutputPatternsForAgent, } from "./create-stream-ready-gate"; +import { dockerEnv, FakeChild, makePollingOptions, vmEnv } from "./create-stream-test-fixtures"; describe("sandbox-create-stream ready gate", () => { afterEach(() => { @@ -106,7 +106,7 @@ describe("sandbox-create-stream ready gate", () => { makePollingOptions(new FakeChild(), { readyCheck: () => false, onPoll: () => {} }), ), ).toThrow( - "streamSandboxCreate onPoll requires failureCheck (e.g., dockerGpuCreatePatch.createFailureMessage)", + "streamSandboxCreate onPoll requires failureCheck (e.g., runtimePatch.createFailureMessage)", ); }); diff --git a/src/lib/sandbox/create-stream.ts b/src/lib/sandbox/create-stream.ts index 7096916345..3495f8e65a 100644 --- a/src/lib/sandbox/create-stream.ts +++ b/src/lib/sandbox/create-stream.ts @@ -102,7 +102,7 @@ export function streamSandboxCreate( const options = hasArgs ? maybeOptions : ((envOrOptions ?? {}) as StreamSandboxCreateOptions); if (options.onPoll && !options.failureCheck) { throw new Error( - "streamSandboxCreate onPoll requires failureCheck (e.g., dockerGpuCreatePatch.createFailureMessage)", + "streamSandboxCreate onPoll requires failureCheck (e.g., runtimePatch.createFailureMessage)", ); } const child: StreamableChildProcess = (options.spawnImpl ?? spawn)(spawnCommand, commandArgs, { diff --git a/src/lib/state/onboard-session.test.ts b/src/lib/state/onboard-session.test.ts index 0de6c8cb5f..d8d77b18be 100644 --- a/src/lib/state/onboard-session.test.ts +++ b/src/lib/state/onboard-session.test.ts @@ -486,7 +486,7 @@ describe("onboard session", () => { session.saveSession(session.createSession()); const unsafeProviderUpdate: Parameters[1] & { apiKey: string; - metadata: { gatewayName: string; token: string }; + metadata: { gatewayName: string; openshellDriver: string; token: string }; } = { provider: "nvidia-nim", model: "nvidia/test-model", @@ -500,6 +500,7 @@ describe("onboard session", () => { apiKey: "nvapi-secret", metadata: { gatewayName: "nemoclaw", + openshellDriver: "podman", token: "secret", }, }; @@ -520,9 +521,31 @@ describe("onboard session", () => { ); expect("apiKey" in loaded).toBe(false); expect(loaded.metadata.gatewayName).toBe("nemoclaw"); + expect(loaded.metadata.openshellDriver).toBe("podman"); expect("token" in loaded.metadata).toBe(false); }); + it("round-trips durable compute-driver identity and defaults legacy sessions safely", () => { + const current = session.createSession({ + metadata: { + gatewayName: "nemoclaw", + fromDockerfile: null, + openshellDriver: "podman", + }, + }); + expect(requireLoadedSession(session.normalizeSession(current)).metadata.openshellDriver).toBe( + "podman", + ); + + const legacy = session.createSession() as unknown as { + metadata: Record; + }; + delete legacy.metadata.openshellDriver; + expect( + requireLoadedSession(session.normalizeSession(legacy as never)).metadata.openshellDriver, + ).toBeNull(); + }); + // ── GH #2625: provider switch from remote→local must clear stale fields ── // // Before the fix, filterSafeUpdates only accepted `typeof === "string"` for diff --git a/src/lib/state/onboard-session.ts b/src/lib/state/onboard-session.ts index 4c4a4d271b..ed8f70eb29 100644 --- a/src/lib/state/onboard-session.ts +++ b/src/lib/state/onboard-session.ts @@ -98,6 +98,8 @@ export interface SessionFailure { export interface SessionMetadata { gatewayName: string; fromDockerfile: string | null; + /** Durable OpenShell compute identity selected before gateway or sandbox mutation. */ + openshellDriver?: string | null; } export type SessionRecoveryReceiptReason = @@ -272,7 +274,11 @@ export interface SessionUpdates { gpuPassthrough?: boolean; telegramConfig?: TelegramConfig | null; wechatConfig?: WechatConfig | null; - metadata?: { gatewayName?: string; fromDockerfile?: string | null }; + metadata?: { + gatewayName?: string; + fromDockerfile?: string | null; + openshellDriver?: string | null; + }; /** Ephemeral vLLM checkpoint proof consumed by Station provider binding; never persisted. */ stationExpressModelIdentity?: string; } @@ -470,6 +476,7 @@ function parseSessionMetadata(value: SessionJsonValue | undefined): SessionMetad return { gatewayName: readString(value.gatewayName) ?? "nemoclaw", fromDockerfile: readString(value.fromDockerfile), + openshellDriver: readString(value.openshellDriver), }; } @@ -698,6 +705,7 @@ export function createSession(overrides: Partial = {}): Session { metadata: { gatewayName: overrides.metadata?.gatewayName ?? "nemoclaw", fromDockerfile: overrides.metadata?.fromDockerfile ?? null, + openshellDriver: overrides.metadata?.openshellDriver?.trim() || null, }, machine: parseMachineSnapshot(overrides.machine as SessionJsonValue | undefined, sessionId) ?? @@ -1364,6 +1372,10 @@ export function filterSafeUpdates(updates: SessionUpdates): Partial { typeof updates.metadata.fromDockerfile === "string" ? updates.metadata.fromDockerfile : null, + openshellDriver: + typeof updates.metadata.openshellDriver === "string" + ? updates.metadata.openshellDriver.trim() || null + : null, }; } return safe; diff --git a/src/lib/state/registry.ts b/src/lib/state/registry.ts index f7809ad28e..a4736e443f 100644 --- a/src/lib/state/registry.ts +++ b/src/lib/state/registry.ts @@ -16,6 +16,7 @@ import { } from "./extra-providers"; import { withLock } from "./registry/lock"; import { load, save } from "./registry/persistence"; +import { cloneSandboxWorkloadReceipt } from "./registry/workload"; import { normalizeSandboxMcpState } from "./registry-mcp"; import { normalizeBaselineExclusions, @@ -70,6 +71,7 @@ export type { SandboxGpuProofResult, SandboxGpuProofStatus, SandboxRegistry, + SandboxWorkloadReceipt, } from "./registry/types"; export type { McpBridgeEntry, SandboxMcpState } from "./registry-mcp"; @@ -162,6 +164,7 @@ export function registerSandbox(entry: SandboxEntry): void { ? entry.hermesAuthMethod : null, imageTag: entry.imageTag || null, + workload: cloneSandboxWorkloadReceipt(entry.workload), messaging: cloneSandboxMessagingState(entry.messaging), mcp: normalizeSandboxMcpState(entry.mcp), hermesToolGateways: diff --git a/src/lib/state/registry/persistence.ts b/src/lib/state/registry/persistence.ts index 7566520f81..d836bf3b3a 100644 --- a/src/lib/state/registry/persistence.ts +++ b/src/lib/state/registry/persistence.ts @@ -20,6 +20,7 @@ import { import * as reversibleRemoval from "../registry-reversible-removal"; import { nemoclawStateRoot } from "../state-root"; import type { SandboxEntry, SandboxRegistry } from "./types"; +import { cloneSandboxWorkloadReceipt } from "./workload"; export const REGISTRY_FILE = path.join( nemoclawStateRoot(process.env.HOME || "/tmp", GATEWAY_PORT), @@ -83,6 +84,7 @@ function serializeRegistryForDisk(data: SandboxRegistry): SandboxRegistry { function normalizeSandboxEntryForRuntime(entry: SandboxEntry): SandboxEntry { const messaging = cloneSandboxMessagingState(entry.messaging); + const workload = cloneSandboxWorkloadReceipt(entry.workload); const mcp = normalizeSandboxMcpState(entry.mcp); const baselineExclusions = normalizeBaselineExclusions(entry.baselineExclusions); const baselineExclusionTransition = normalizeBaselineExclusionTransition( @@ -90,6 +92,7 @@ function normalizeSandboxEntryForRuntime(entry: SandboxEntry): SandboxEntry { ); const { messaging: _messaging, + workload: _workload, mcp: _mcp, baselineExclusions: _baselineExclusions, baselineExclusionTransition: _baselineExclusionTransition, @@ -97,6 +100,7 @@ function normalizeSandboxEntryForRuntime(entry: SandboxEntry): SandboxEntry { } = entry; return { ...rest, + ...(workload ? { workload } : {}), ...(messaging ? { messaging } : {}), ...(mcp ? { mcp } : {}), ...(baselineExclusions ? { baselineExclusions } : {}), @@ -125,6 +129,7 @@ function serializeSandboxEntryForDisk(entry: SandboxEntry): SandboxEntry { providerCredentialHashes?: unknown; }; const messaging = serializeSandboxMessagingStateForDisk(durable.messaging); + const workload = cloneSandboxWorkloadReceipt(durable.workload); const mcp = serializeSandboxMcpStateForDisk(durable.mcp); const baselineExclusions = normalizeBaselineExclusions(durable.baselineExclusions); const baselineExclusionTransition = normalizeBaselineExclusionTransition( @@ -132,6 +137,7 @@ function serializeSandboxEntryForDisk(entry: SandboxEntry): SandboxEntry { ); const { messaging: _messaging, + workload: _workload, mcp: _mcp, baselineExclusions: _baselineExclusions, baselineExclusionTransition: _baselineExclusionTransition, @@ -140,6 +146,7 @@ function serializeSandboxEntryForDisk(entry: SandboxEntry): SandboxEntry { return { ...rest, ...(rest.dashboardPort === 0 ? { dashboardPort: null } : {}), + ...(workload ? { workload } : {}), ...(messaging ? { messaging } : {}), ...(mcp ? { mcp } : {}), ...(baselineExclusions ? { baselineExclusions } : {}), diff --git a/src/lib/state/registry/types.ts b/src/lib/state/registry/types.ts index 97b3587198..1309d80f2c 100644 --- a/src/lib/state/registry/types.ts +++ b/src/lib/state/registry/types.ts @@ -119,6 +119,12 @@ export interface SandboxEntry extends Partial { fromDockerfile?: string | null; hermesAuthMethod?: "oauth" | "api_key" | null; imageTag?: string | null; + /** + * Durable source and ownership receipt for the workload behind imageTag. + * Managed images are immutable shared release artifacts and must never flow + * through per-sandbox image deletion. + */ + workload?: SandboxWorkloadReceipt; messaging?: SandboxMessagingState; mcp?: SandboxMcpState; hermesToolGateways?: string[]; @@ -141,6 +147,33 @@ export interface SandboxEntry extends Partial { gatewayPort?: number | null; } +export type SandboxWorkloadReceipt = + | { + readonly schemaVersion: 1; + readonly kind: "managed-image"; + readonly reference: string; + readonly release: string; + readonly sourceRevision: string; + /** Exact all-agent publication cohort that produced the immutable image. */ + readonly sourceCohort: string; + readonly capabilityContractVersion: number; + readonly startupProfileContractVersion: number; + /** Canonical, secret-free base64url profile transport used to start this image. */ + readonly encodedProfile: string; + readonly startupProfileSha256: string; + /** Re-acquire launch-only proxy credentials from the operator environment when cloning. */ + readonly credentialProxyReplayRequired: boolean; + /** Optional canonical standard-base64 public CA bundle bound by the profile digest. */ + readonly corporateCaB64?: string; + readonly shared: true; + } + | { + readonly schemaVersion: 1; + readonly kind: "legacy-dockerfile"; + readonly reference: string | null; + readonly shared: false; + }; + export interface SandboxRegistry { sandboxes: Record; defaultSandbox: string | null; diff --git a/src/lib/state/registry/workload.test.ts b/src/lib/state/registry/workload.test.ts new file mode 100644 index 0000000000..2114911902 --- /dev/null +++ b/src/lib/state/registry/workload.test.ts @@ -0,0 +1,133 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { Buffer } from "node:buffer"; +import { createHash } from "node:crypto"; +import { describe, expect, it } from "vitest"; + +import type { SandboxWorkloadReceipt } from "./types"; +import { cloneSandboxWorkloadReceipt } from "./workload"; + +const ENCODED_PROFILE = Buffer.from('{"schemaVersion":1}', "utf8").toString("base64url"); +const PROFILE_SHA256 = createHash("sha256").update(ENCODED_PROFILE, "utf8").digest("hex"); + +const MANAGED_RECEIPT = { + schemaVersion: 1, + kind: "managed-image", + reference: `ghcr.io/nvidia/nemoclaw/openclaw-sandbox@sha256:${"a".repeat(64)}`, + release: "v0.0.97", + sourceRevision: "b".repeat(40), + sourceCohort: "ghrun-123456-1", + capabilityContractVersion: 1, + startupProfileContractVersion: 1, + encodedProfile: ENCODED_PROFILE, + startupProfileSha256: PROFILE_SHA256, + credentialProxyReplayRequired: false, + shared: true, +} as const satisfies SandboxWorkloadReceipt; + +describe("sandbox workload receipt", () => { + it("clones a complete shared managed-image identity", () => { + const cloned = cloneSandboxWorkloadReceipt(MANAGED_RECEIPT); + + expect(cloned).toEqual(MANAGED_RECEIPT); + expect(cloned).not.toBe(MANAGED_RECEIPT); + }); + + it("drops malformed ownership and profile identity instead of persisting it", () => { + expect( + cloneSandboxWorkloadReceipt({ + ...MANAGED_RECEIPT, + sourceCohort: "run-123456", + } as SandboxWorkloadReceipt), + ).toBeUndefined(); + expect( + cloneSandboxWorkloadReceipt({ + ...MANAGED_RECEIPT, + sourceCohort: `ghrun-${"1".repeat(21)}-1`, + } as SandboxWorkloadReceipt), + ).toBeUndefined(); + expect( + cloneSandboxWorkloadReceipt({ + ...MANAGED_RECEIPT, + sourceCohort: `ghrun-1-${"1".repeat(11)}`, + } as SandboxWorkloadReceipt), + ).toBeUndefined(); + expect( + cloneSandboxWorkloadReceipt({ + ...MANAGED_RECEIPT, + reference: `registry.example.test/openclaw@sha256:${"a".repeat(64)}`, + } as SandboxWorkloadReceipt), + ).toBeUndefined(); + expect( + cloneSandboxWorkloadReceipt({ + ...MANAGED_RECEIPT, + reference: "ghcr.io/nvidia/nemoclaw/openclaw-sandbox:latest", + } as SandboxWorkloadReceipt), + ).toBeUndefined(); + expect( + cloneSandboxWorkloadReceipt({ + ...MANAGED_RECEIPT, + release: "latest", + } as SandboxWorkloadReceipt), + ).toBeUndefined(); + expect( + cloneSandboxWorkloadReceipt({ + ...MANAGED_RECEIPT, + capabilityContractVersion: 2, + } as SandboxWorkloadReceipt), + ).toBeUndefined(); + expect( + cloneSandboxWorkloadReceipt({ + ...MANAGED_RECEIPT, + startupProfileContractVersion: 2, + } as SandboxWorkloadReceipt), + ).toBeUndefined(); + expect( + cloneSandboxWorkloadReceipt({ + ...MANAGED_RECEIPT, + startupProfileSha256: "not-a-digest", + } as SandboxWorkloadReceipt), + ).toBeUndefined(); + expect( + cloneSandboxWorkloadReceipt({ + ...MANAGED_RECEIPT, + encodedProfile: `${ENCODED_PROFILE}=`, + } as SandboxWorkloadReceipt), + ).toBeUndefined(); + expect( + cloneSandboxWorkloadReceipt({ + ...MANAGED_RECEIPT, + encodedProfile: Buffer.from("different", "utf8").toString("base64url"), + } as SandboxWorkloadReceipt), + ).toBeUndefined(); + expect( + cloneSandboxWorkloadReceipt({ + ...MANAGED_RECEIPT, + corporateCaB64: "not canonical base64", + } as SandboxWorkloadReceipt), + ).toBeUndefined(); + expect( + cloneSandboxWorkloadReceipt({ + ...MANAGED_RECEIPT, + shared: false, + } as unknown as SandboxWorkloadReceipt), + ).toBeUndefined(); + }); + + it("retains an owned legacy image receipt independently from managed cohorts", () => { + expect( + cloneSandboxWorkloadReceipt({ + schemaVersion: 1, + kind: "legacy-dockerfile", + reference: "nemoclaw-sandbox-local:build-123", + shared: false, + }), + ).toEqual({ + schemaVersion: 1, + kind: "legacy-dockerfile", + reference: "nemoclaw-sandbox-local:build-123", + shared: false, + }); + }); +}); diff --git a/src/lib/state/registry/workload.ts b/src/lib/state/registry/workload.ts new file mode 100644 index 0000000000..98ea552a04 --- /dev/null +++ b/src/lib/state/registry/workload.ts @@ -0,0 +1,113 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { Buffer } from "node:buffer"; +import { createHash } from "node:crypto"; + +import type { SandboxWorkloadReceipt } from "./types"; + +const SHA256_PATTERN = /^[0-9a-f]{64}$/u; +const REVISION_PATTERN = /^[0-9a-f]{40}$/u; +const COHORT_PATTERN = /^ghrun-[1-9][0-9]{0,19}-[1-9][0-9]{0,9}$/u; +const MANAGED_REFERENCE_PATTERN = + /^ghcr[.]io\/nvidia\/nemoclaw\/(?:openclaw|hermes|langchain-deepagents-code)-sandbox@sha256:[0-9a-f]{64}$/u; +const RELEASE_PATTERN = /^v[0-9]+(?:[.][0-9]+){1,3}(?:[-.][0-9A-Za-z][0-9A-Za-z.-]*)?$/u; +const MAX_COHORT_BYTES = 128; +const BASE64URL_PATTERN = /^[A-Za-z0-9_-]+$/u; +const STANDARD_BASE64_PATTERN = /^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/u; +const MAX_PROFILE_BYTES = 64 * 1024; +const MAX_PROFILE_ENCODED_BYTES = Math.ceil(MAX_PROFILE_BYTES / 3) * 4; +const MAX_CORPORATE_CA_BYTES = 128 * 1024; +const MAX_CORPORATE_CA_ENCODED_BYTES = Math.ceil(MAX_CORPORATE_CA_BYTES / 3) * 4; + +function nonEmptyString(value: unknown): value is string { + return typeof value === "string" && value.trim() !== ""; +} + +function decodeCanonicalBase64Url(value: unknown): Buffer | null { + if ( + typeof value !== "string" || + value.length === 0 || + value.length > MAX_PROFILE_ENCODED_BYTES || + value.length % 4 === 1 || + !BASE64URL_PATTERN.test(value) + ) { + return null; + } + const decoded = Buffer.from(value, "base64url"); + return decoded.length > 0 && + decoded.length <= MAX_PROFILE_BYTES && + decoded.toString("base64url") === value + ? decoded + : null; +} + +function decodeCanonicalStandardBase64(value: unknown): Buffer | null { + if ( + typeof value !== "string" || + value.length === 0 || + value.length > MAX_CORPORATE_CA_ENCODED_BYTES || + !STANDARD_BASE64_PATTERN.test(value) + ) { + return null; + } + const decoded = Buffer.from(value, "base64"); + return decoded.length > 0 && + decoded.length <= MAX_CORPORATE_CA_BYTES && + decoded.toString("base64") === value + ? decoded + : null; +} + +export function cloneSandboxWorkloadReceipt( + value: SandboxWorkloadReceipt | undefined, +): SandboxWorkloadReceipt | undefined { + if (!value || value.schemaVersion !== 1) return undefined; + if (value.kind === "legacy-dockerfile") { + if (value.shared !== false || (value.reference !== null && !nonEmptyString(value.reference))) { + return undefined; + } + return { + schemaVersion: 1, + kind: "legacy-dockerfile", + reference: value.reference, + shared: false, + }; + } + if ( + value.kind !== "managed-image" || + value.shared !== true || + !MANAGED_REFERENCE_PATTERN.test(value.reference) || + !RELEASE_PATTERN.test(value.release) || + !REVISION_PATTERN.test(value.sourceRevision) || + typeof value.sourceCohort !== "string" || + Buffer.byteLength(value.sourceCohort, "utf8") > MAX_COHORT_BYTES || + !COHORT_PATTERN.test(value.sourceCohort) || + value.capabilityContractVersion !== 1 || + value.startupProfileContractVersion !== 1 || + !SHA256_PATTERN.test(value.startupProfileSha256) || + typeof value.credentialProxyReplayRequired !== "boolean" || + decodeCanonicalBase64Url(value.encodedProfile) === null || + createHash("sha256").update(value.encodedProfile, "utf8").digest("hex") !== + value.startupProfileSha256 || + (value.corporateCaB64 !== undefined && + decodeCanonicalStandardBase64(value.corporateCaB64) === null) + ) { + return undefined; + } + return { + schemaVersion: 1, + kind: "managed-image", + reference: value.reference, + release: value.release, + sourceRevision: value.sourceRevision, + sourceCohort: value.sourceCohort, + capabilityContractVersion: value.capabilityContractVersion, + startupProfileContractVersion: value.startupProfileContractVersion, + encodedProfile: value.encodedProfile, + startupProfileSha256: value.startupProfileSha256, + credentialProxyReplayRequired: value.credentialProxyReplayRequired, + ...(value.corporateCaB64 === undefined ? {} : { corporateCaB64: value.corporateCaB64 }), + shared: true, + }; +} diff --git a/src/lib/subprocess-env.ts b/src/lib/subprocess-env.ts index f334e79ef6..45761ec764 100644 --- a/src/lib/subprocess-env.ts +++ b/src/lib/subprocess-env.ts @@ -1,6 +1,8 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { withLocalNoProxy } from "./proxy/local-no-proxy"; + /** * Subprocess environment allowlist. * @@ -90,33 +92,7 @@ export function isSubprocessEnvNameAllowed(name: string): boolean { * consults the caller's NO_PROXY for sandbox-create env decisions, this * augmentation can be dropped. */ -export function withLocalNoProxy(env: Record): void { - const hasProxy = env.HTTP_PROXY || env.HTTPS_PROXY || env.http_proxy || env.https_proxy; - if (!hasProxy) return; - for (const key of ["NO_PROXY", "no_proxy"] as const) { - const current = env[key] ?? ""; - const parts = current - .split(",") - .map((s) => s.trim()) - .filter(Boolean); - let changed = false; - for (const host of [ - "localhost", - "127.0.0.1", - "host.docker.internal", - "host.containers.internal", - "::1", - "0.0.0.0", - "inference.local", - ]) { - if (!parts.includes(host)) { - parts.push(host); - changed = true; - } - } - if (changed) env[key] = parts.join(","); - } -} +export { withLocalNoProxy }; export function buildSubprocessEnv(extra?: Record): Record { const env: Record = {}; diff --git a/src/lib/tunnel/sandbox-gateway-stop.test.ts b/src/lib/tunnel/sandbox-gateway-stop.test.ts index 4178109e63..cbb8404a83 100644 --- a/src/lib/tunnel/sandbox-gateway-stop.test.ts +++ b/src/lib/tunnel/sandbox-gateway-stop.test.ts @@ -115,6 +115,23 @@ describe("stopSandboxChannels", () => { ); }); + it("uses OpenShell directly when the runtime forbids Docker gateway exec", () => { + const h = harness(); + h.deps.allowDockerGatewayExec = false; + + stopSandboxChannels("my-sandbox", h.deps); + + expect(h.runDocker).not.toHaveBeenCalled(); + expect(h.runProcess).toHaveBeenCalledWith( + "/usr/local/bin/openshell", + ["sandbox", "exec", "--name", "my-sandbox", "--gateway", "nemoclaw", "--", "sh", "-s"], + expect.objectContaining({ + input: expect.stringContaining("find_gateway_pids"), + timeout: 20000, + }), + ); + }); + it("selects the exact generated sandbox pod and excludes overlapping names", () => { const h = harness(); h.runDocker diff --git a/src/lib/tunnel/sandbox-gateway-stop.ts b/src/lib/tunnel/sandbox-gateway-stop.ts index 7176783c7b..8050738cdb 100644 --- a/src/lib/tunnel/sandbox-gateway-stop.ts +++ b/src/lib/tunnel/sandbox-gateway-stop.ts @@ -24,6 +24,8 @@ type ProcessRunner = ( ) => SpawnSyncReturns; export type SandboxGatewayStopDeps = { + /** Skip the legacy gateway-container kubectl path for non-Docker runtimes. */ + allowDockerGatewayExec?: boolean; getSandbox?: typeof registry.getSandbox; getRegisteredAgent?: typeof agentRuntime.getRegisteredAgent; getAgentDisplayName?: typeof agentRuntime.getAgentDisplayName; @@ -105,12 +107,15 @@ export function stopSandboxChannels(sandboxName: string, deps: SandboxGatewaySto const gatewayLabel = `${agentDisplayName} gateway`; info(`Stopping in-sandbox ${gatewayLabel} (sandbox: ${validatedSandboxName})...`); - const privilegedResult = stopSandboxChannelsViaKubectl( - validatedSandboxName, - gatewayName, - GATEWAY_STOP_SCRIPT, - deps.runDocker ?? dockerSpawnSync, - ); + const privilegedResult = + deps.allowDockerGatewayExec === false + ? null + : stopSandboxChannelsViaKubectl( + validatedSandboxName, + gatewayName, + GATEWAY_STOP_SCRIPT, + deps.runDocker ?? dockerSpawnSync, + ); if (reportStopResult(privilegedResult, gatewayLabel, info, warn)) return; const openshell = (deps.resolveOpenshell ?? resolveOpenshell)(); diff --git a/test/dcode-start-keepalive.test.ts b/test/dcode-start-keepalive.test.ts index 73d2c92c50..550d38f0d8 100644 --- a/test/dcode-start-keepalive.test.ts +++ b/test/dcode-start-keepalive.test.ts @@ -14,6 +14,13 @@ const START_SCRIPT = path.join( "langchain-deepagents-code", "start.sh", ); +const ENTRYPOINT_ENV_WRAPPER = path.join( + import.meta.dirname, + "..", + "scripts", + "lib", + "entrypoint-env-wrapper.sh", +); // start.sh hardcodes this runtime-env path; clean it up so the test is hermetic. const RUNTIME_ENV_FILE = "/tmp/nemoclaw-proxy-env.sh"; @@ -67,6 +74,7 @@ function makeStartFixture(options: { installRlimitHelper?: RlimitHelperInstaller const portFile = path.join(tempDir, "trusted-proxy-port"); const fixture = fs .readFileSync(START_SCRIPT, "utf8") + .replace("/usr/local/lib/nemoclaw/entrypoint-env-wrapper.sh", ENTRYPOINT_ENV_WRAPPER) .replace("/usr/local/lib/nemoclaw/sandbox-rlimits.sh", rlimitLib) .replace( 'readonly MANAGED_PROXY_HOST_FILE="/usr/local/share/nemoclaw/dcode-proxy-host"', diff --git a/test/e2e/fixtures/availability-env.ts b/test/e2e/fixtures/availability-env.ts index a0918446d5..6290a5615e 100644 --- a/test/e2e/fixtures/availability-env.ts +++ b/test/e2e/fixtures/availability-env.ts @@ -10,6 +10,8 @@ const AVAILABILITY_PROBE_EXTRA_ENV_KEYS = [ "DOCKER_TLS_VERIFY", "DOCKER_CERT_PATH", "DOCKER_API_VERSION", + "OPENSHELL_PODMAN_NETWORK_NAME", + "OPENSHELL_PODMAN_SOCKET", "XDG_CONFIG_HOME", "XDG_RUNTIME_DIR", "NEMOCLAW_OLLAMA_PULL_TIMEOUT", diff --git a/test/e2e/fixtures/phases/environment.ts b/test/e2e/fixtures/phases/environment.ts index e57193aeeb..b1359083c8 100644 --- a/test/e2e/fixtures/phases/environment.ts +++ b/test/e2e/fixtures/phases/environment.ts @@ -1,28 +1,31 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { buildAvailabilityProbeEnv } from "../availability-env.ts"; +import { isAbsolute } from "node:path"; +import type { TargetEnvironment } from "../../registry/types.ts"; import type { ArtifactSink } from "../artifacts.ts"; +import { buildAvailabilityProbeEnv } from "../availability-env.ts"; import { artifactLabel, assertExitZero } from "../clients/command.ts"; import type { HostCliClient } from "../clients/host.ts"; import type { ShellProbeResult } from "../shell-probe.ts"; -import type { TargetEnvironment } from "../../registry/types.ts"; const SUPPORTED_INSTALLS = new Set(["repo-current", "launchable"]); -const DOCKER_RUNTIME_EXPECTATIONS = { - "docker-running": "required", - "gpu-docker-cdi": "required", - "docker-missing": "missing", - "macos-docker-optional": "optional", -} as const; +const DOCKER_ENV_KEYS = [ + "DOCKER_CONFIG", + "DOCKER_CONTEXT", + "DOCKER_HOST", + "DOCKER_TLS_VERIFY", + "DOCKER_CERT_PATH", + "DOCKER_API_VERSION", +] as const; -export type DockerRuntimeExpectation = - (typeof DOCKER_RUNTIME_EXPECTATIONS)[keyof typeof DOCKER_RUNTIME_EXPECTATIONS]; +export type ContainerRuntimeExpectation = "required" | "missing" | "optional"; -export interface DockerRuntimeReady { +export interface ContainerRuntimeReady { id: string; - expectation: DockerRuntimeExpectation; + driverName: string; + expectation: ContainerRuntimeExpectation; available: boolean; result?: ShellProbeResult; probeError?: string; @@ -30,20 +33,70 @@ export interface DockerRuntimeReady { export interface EnvironmentReady extends TargetEnvironment { cliPath: string; - docker: DockerRuntimeReady; + containerRuntime: ContainerRuntimeReady; +} + +interface ContainerRuntimeProbe { + driverName: string; + expectation: ContainerRuntimeExpectation; + command: string; + args: () => string[]; + env: () => NodeJS.ProcessEnv; } function errorMessage(error: unknown): string { return error instanceof Error ? error.message : String(error); } -function supportedRuntime(runtime: string): DockerRuntimeExpectation { - const expectation = - DOCKER_RUNTIME_EXPECTATIONS[runtime as keyof typeof DOCKER_RUNTIME_EXPECTATIONS]; - if (!expectation) { +function withoutDockerEnvironment(environment: NodeJS.ProcessEnv): NodeJS.ProcessEnv { + const sanitized = { ...environment }; + for (const key of DOCKER_ENV_KEYS) { + delete sanitized[key]; + } + return sanitized; +} + +function podmanSocketPath(): string { + const socketPath = process.env.OPENSHELL_PODMAN_SOCKET?.trim(); + if (!socketPath) { + throw new Error("podman-running requires OPENSHELL_PODMAN_SOCKET."); + } + if (!isAbsolute(socketPath)) { + throw new Error("OPENSHELL_PODMAN_SOCKET must be an absolute path."); + } + return socketPath; +} + +function dockerProbe(expectation: ContainerRuntimeExpectation): ContainerRuntimeProbe { + return { + driverName: "docker", + expectation, + command: "docker", + args: () => ["info"], + env: () => buildAvailabilityProbeEnv(), + }; +} + +const CONTAINER_RUNTIME_PROBES: Readonly> = { + "docker-running": dockerProbe("required"), + "gpu-docker-cdi": dockerProbe("required"), + "docker-missing": dockerProbe("missing"), + "macos-docker-optional": dockerProbe("optional"), + "podman-running": { + driverName: "podman", + expectation: "required", + command: "podman", + args: () => ["--url", `unix://${podmanSocketPath()}`, "info", "--format", "json"], + env: () => withoutDockerEnvironment(buildAvailabilityProbeEnv()), + }, +}; + +function supportedRuntime(runtime: string): ContainerRuntimeProbe { + const probe = CONTAINER_RUNTIME_PROBES[runtime]; + if (!probe) { throw new Error(`Unsupported target runtime '${runtime}'.`); } - return expectation; + return probe; } export class EnvironmentPhaseFixture { @@ -55,11 +108,11 @@ export class EnvironmentPhaseFixture { async assertReady(environment: TargetEnvironment): Promise { try { await this.assertInstallReady(environment.install); - const docker = await this.assertRuntimeReady(environment.runtime); + const containerRuntime = await this.assertRuntimeReady(environment.runtime); const result = { ...environment, cliPath: this.host.commandPath, - docker, + containerRuntime, }; await this.writeResult("passed", result); return result; @@ -89,44 +142,46 @@ export class EnvironmentPhaseFixture { return this.host.expectNemoclawAvailable(); } - private async assertRuntimeReady(runtime: string): Promise { - const expectation = supportedRuntime(runtime); - const result = await this.probeDocker(runtime, expectation); + private async assertRuntimeReady(runtime: string): Promise { + const probe = supportedRuntime(runtime); + const result = await this.probeContainerRuntime(runtime, probe); if (!result.result) { return result; } - if (expectation === "required") { - assertExitZero(result.result, `docker runtime ${runtime}`); + if (probe.expectation === "required") { + assertExitZero(result.result, `${probe.driverName} runtime ${runtime}`); } - // Missing-runtime targets simulate Docker failure at the phase that + // Missing-runtime targets simulate runtime failure at the phase that // needs it; this probe records host reality without blocking composition. return result; } - private async probeDocker( + private async probeContainerRuntime( runtime: string, - expectation: DockerRuntimeExpectation, - ): Promise { + probe: ContainerRuntimeProbe, + ): Promise { try { - const result = await this.host.command("docker", ["info"], { - artifactName: `runtime-docker-info-${artifactLabel(runtime)}`, - env: buildAvailabilityProbeEnv(), + const result = await this.host.command(probe.command, probe.args(), { + artifactName: `runtime-${artifactLabel(probe.driverName)}-info-${artifactLabel(runtime)}`, + env: probe.env(), timeoutMs: 30_000, }); return { id: runtime, - expectation, + driverName: probe.driverName, + expectation: probe.expectation, available: result.exitCode === 0, result, }; } catch (error) { - if (expectation === "required") { + if (probe.expectation === "required") { throw error; } return { id: runtime, - expectation, + driverName: probe.driverName, + expectation: probe.expectation, available: false, probeError: errorMessage(error), }; diff --git a/test/e2e/fixtures/phases/index.ts b/test/e2e/fixtures/phases/index.ts index dcb7c319e3..8b45f61a4c 100644 --- a/test/e2e/fixtures/phases/index.ts +++ b/test/e2e/fixtures/phases/index.ts @@ -2,8 +2,8 @@ // SPDX-License-Identifier: Apache-2.0 export { - type DockerRuntimeExpectation, - type DockerRuntimeReady, + type ContainerRuntimeExpectation, + type ContainerRuntimeReady, EnvironmentPhaseFixture, type EnvironmentReady, } from "./environment.ts"; diff --git a/test/e2e/fixtures/phases/onboarding.ts b/test/e2e/fixtures/phases/onboarding.ts index 180a0a4601..d6b107f9fc 100644 --- a/test/e2e/fixtures/phases/onboarding.ts +++ b/test/e2e/fixtures/phases/onboarding.ts @@ -27,7 +27,16 @@ const ONBOARD_ARGS = [ ]; const DEFAULT_TIMEOUT_MS = 15 * 60_000; const OPENCLAW_GATEWAY_URL = "http://127.0.0.1:18789"; +const HERMES_GATEWAY_URL = "http://127.0.0.1:8642"; const NEGATIVE_PREFLIGHT_LOG = "negative-preflight.log"; +const DOCKER_ENV_KEYS = [ + "DOCKER_CONFIG", + "DOCKER_CONTEXT", + "DOCKER_HOST", + "DOCKER_TLS_VERIFY", + "DOCKER_CERT_PATH", + "DOCKER_API_VERSION", +] as const; const DOCKER_MISSING_PATTERNS = [ /Cannot connect to the Docker daemon/i, /Is the docker daemon running\??/i, @@ -126,6 +135,48 @@ function commandEnv(sandboxName: string, extra: NodeJS.ProcessEnv = {}): NodeJS. }; } +function runtimeCommandEnv( + environment: EnvironmentReady, + sandboxName: string, + extra: NodeJS.ProcessEnv = {}, +): NodeJS.ProcessEnv { + const env = commandEnv(sandboxName, extra); + if (environment.containerRuntime.driverName !== "podman") { + return env; + } + for (const key of DOCKER_ENV_KEYS) { + delete env[key]; + } + return env; +} + +function runtimeOnlyEnv(driverName: string): NodeJS.ProcessEnv { + const env = buildAvailabilityProbeEnv(); + if (driverName === "podman") { + for (const key of DOCKER_ENV_KEYS) { + delete env[key]; + } + } + return env; +} + +function onboardingArgs( + environment: EnvironmentReady, + trailingArgs: readonly string[] = [], +): string[] { + const driverArgs = + environment.containerRuntime.driverName === "podman" ? ["--compute-driver", "podman"] : []; + return [...ONBOARD_ARGS, ...driverArgs, ...trailingArgs]; +} + +function requireAvailableRuntime(environment: EnvironmentReady, onboarding: string): void { + if (!environment.containerRuntime.available) { + throw new Error( + `${onboarding} onboarding requires an available ${environment.containerRuntime.driverName} runtime.`, + ); + } +} + function noDockerShim(): string { // Source of truth for the Vitest fixture path: simulate the invalid state // where the Docker client exists but the daemon is unreachable. Keep the @@ -181,6 +232,9 @@ export class OnboardingPhaseFixture { case "cloud-openclaw": result = await this.cloudOpenClaw(environment, options); break; + case "cloud-hermes": + result = await this.cloudHermes(environment, options); + break; case "cloud-openclaw-policy-custom-missing-presets": result = await this.cloudOpenClawPolicyCustomMissingPresets(environment, options); break; @@ -205,15 +259,13 @@ export class OnboardingPhaseFixture { environment: EnvironmentReady, options: OnboardingOptions = {}, ): Promise { - if (!environment.docker.available) { - throw new Error("cloud-openclaw onboarding requires an available Docker runtime."); - } + requireAvailableRuntime(environment, "cloud-openclaw"); const sandboxName = sandboxNameFromOptions(environment.onboarding, options); const apiKey = this.secrets.required("NVIDIA_INFERENCE_API_KEY"); - this.registerSandboxCleanup(sandboxName); - const result = await this.host.nemoclaw(ONBOARD_ARGS, { + this.registerSandboxCleanup(sandboxName, environment.containerRuntime.driverName); + const result = await this.host.nemoclaw(onboardingArgs(environment), { artifactName: "onboard-cloud-openclaw", - env: commandEnv(sandboxName, { NVIDIA_INFERENCE_API_KEY: apiKey }), + env: runtimeCommandEnv(environment, sandboxName, { NVIDIA_INFERENCE_API_KEY: apiKey }), redactionValues: [apiKey], timeoutMs: options.timeoutMs ?? DEFAULT_TIMEOUT_MS, }); @@ -229,21 +281,46 @@ export class OnboardingPhaseFixture { }; } + async cloudHermes( + environment: EnvironmentReady, + options: OnboardingOptions = {}, + ): Promise { + requireAvailableRuntime(environment, "cloud-hermes"); + const sandboxName = sandboxNameFromOptions(environment.onboarding, options); + const apiKey = this.secrets.required("NVIDIA_INFERENCE_API_KEY"); + this.registerSandboxCleanup(sandboxName, environment.containerRuntime.driverName); + const result = await this.host.nemoclaw(onboardingArgs(environment), { + artifactName: "onboard-cloud-hermes", + env: runtimeCommandEnv(environment, sandboxName, { + NEMOCLAW_AGENT: "hermes", + NVIDIA_INFERENCE_API_KEY: apiKey, + }), + redactionValues: [apiKey], + timeoutMs: options.timeoutMs ?? DEFAULT_TIMEOUT_MS, + }); + assertExitZero(result, "cloud-hermes onboarding"); + return { + onboarding: environment.onboarding, + sandboxName, + agent: "hermes", + provider: "nvidia", + providerEnv: "cloud", + gatewayUrl: HERMES_GATEWAY_URL, + result, + }; + } + async cloudLangchainDeepAgentsCode( environment: EnvironmentReady, options: OnboardingOptions = {}, ): Promise { - if (!environment.docker.available) { - throw new Error( - "cloud-langchain-deepagents-code onboarding requires an available Docker runtime.", - ); - } + requireAvailableRuntime(environment, "cloud-langchain-deepagents-code"); const sandboxName = sandboxNameFromOptions(environment.onboarding, options); const apiKey = this.secrets.required("NVIDIA_INFERENCE_API_KEY"); - this.registerSandboxCleanup(sandboxName); - const result = await this.host.nemoclaw([...ONBOARD_ARGS, "--observability"], { + this.registerSandboxCleanup(sandboxName, environment.containerRuntime.driverName); + const result = await this.host.nemoclaw(onboardingArgs(environment, ["--observability"]), { artifactName: "onboard-cloud-langchain-deepagents-code", - env: commandEnv(sandboxName, { + env: runtimeCommandEnv(environment, sandboxName, { NEMOCLAW_AGENT: "langchain-deepagents-code", NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1", NEMOCLAW_PROVIDER: HOSTED_INFERENCE_PROVIDER, @@ -280,14 +357,17 @@ export class OnboardingPhaseFixture { environment: EnvironmentReady, options: OnboardingOptions = {}, ): Promise { - if (environment.docker.expectation !== "missing") { + if ( + environment.containerRuntime.driverName !== "docker" || + environment.containerRuntime.expectation !== "missing" + ) { throw new Error( "cloud-openclaw-no-docker onboarding requires the docker-missing runtime expectation.", ); } const sandboxName = sandboxNameFromOptions(environment.onboarding, options); const apiKey = this.secrets.required("NVIDIA_INFERENCE_API_KEY"); - this.registerSandboxCleanup(sandboxName); + this.registerSandboxCleanup(sandboxName, environment.containerRuntime.driverName); const shimDir = await mkdtemp(join(tmpdir(), "e2e-no-docker-")); const shimPath = join(shimDir, "docker"); try { @@ -332,17 +412,13 @@ export class OnboardingPhaseFixture { environment: EnvironmentReady, options: OnboardingOptions = {}, ): Promise { - if (!environment.docker.available) { - throw new Error( - "cloud-openclaw-policy-custom-missing-presets onboarding requires an available Docker runtime.", - ); - } + requireAvailableRuntime(environment, "cloud-openclaw-policy-custom-missing-presets"); const sandboxName = sandboxNameFromOptions(environment.onboarding, options); const apiKey = this.secrets.required("NVIDIA_INFERENCE_API_KEY"); - this.registerSandboxCleanup(sandboxName); - const result = await this.host.nemoclaw(ONBOARD_ARGS, { + this.registerSandboxCleanup(sandboxName, environment.containerRuntime.driverName); + const result = await this.host.nemoclaw(onboardingArgs(environment), { artifactName: "onboard-cloud-openclaw-policy-custom-missing-presets", - env: commandEnv(sandboxName, { + env: runtimeCommandEnv(environment, sandboxName, { NVIDIA_INFERENCE_API_KEY: apiKey, NEMOCLAW_POLICY_MODE: "custom", NEMOCLAW_POLICY_PRESETS: "", @@ -379,11 +455,15 @@ export class OnboardingPhaseFixture { }; } - async destroySandbox(sandboxName: string, artifactName?: string): Promise { + async destroySandbox( + sandboxName: string, + artifactName?: string, + driverName = "docker", + ): Promise { validateSandboxName(sandboxName); const result = await this.host.nemoclaw([sandboxName, "destroy", "--yes"], { artifactName: artifactName ?? `cleanup-destroy-${artifactLabel(sandboxName)}`, - env: buildAvailabilityProbeEnv(), + env: runtimeOnlyEnv(driverName), timeoutMs: DEFAULT_TIMEOUT_MS, }); if (result.exitCode !== 0 && !hasMissingSandboxDeleteSignature(result)) { @@ -392,10 +472,10 @@ export class OnboardingPhaseFixture { return result; } - private registerSandboxCleanup(sandboxName: string): void { + private registerSandboxCleanup(sandboxName: string, driverName = "docker"): void { if (!this.cleanup) return; this.cleanup.add(`destroy NemoClaw sandbox ${sandboxName}`, async () => { - await this.destroySandbox(sandboxName); + await this.destroySandbox(sandboxName, undefined, driverName); }); } diff --git a/test/e2e/live/full-e2e.test.ts b/test/e2e/live/full-e2e.test.ts index 6c64995e02..b8eaa2f0f3 100644 --- a/test/e2e/live/full-e2e.test.ts +++ b/test/e2e/live/full-e2e.test.ts @@ -5,6 +5,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { containsInteger42Answer } from "../../helpers/e2e-answer-assertions.ts"; +import { runManagedImageBuildlessE2e } from "../../helpers/managed-image-buildless-e2e.ts"; import type { ArtifactSink } from "../fixtures/artifacts.ts"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; import { resultText } from "../fixtures/clients/command.ts"; @@ -295,6 +296,7 @@ test("full e2e: install, onboard, inference, cli operations, and cleanup", { meta: { e2ePhases: [ "check full E2E prerequisites", + "validate all-agent buildless managed-image orchestration", "install and onboard OpenClaw sandbox", "validate CLI sandbox and policy state", "exercise hosted and sandbox inference", @@ -315,6 +317,7 @@ test("full e2e: install, onboard, inference, cli operations, and cleanup", { USE_PREINSTALLED_LAUNCHABLE ? "the baked Launchable completes onboarding without installing from source" : "install.sh --non-interactive completes onboarding", + "source-backed OpenClaw, Hermes, and DCode onboarding selects exact managed images without invoking legacy build hooks", "nemoclaw and openshell are installed and usable", "sandbox appears in list/status and has policy/inference configuration", "direct hosted inference and sandbox inference.local both respond", @@ -325,6 +328,9 @@ test("full e2e: install, onboard, inference, cli operations, and cleanup", { ], }); + progress.phase("validate all-agent buildless managed-image orchestration"); + runManagedImageBuildlessE2e(); + const docker = await host.command("docker", ["info"], { artifactName: "phase-0-docker-info", env: env(), diff --git a/test/e2e/live/podman-all-agents.test.ts b/test/e2e/live/podman-all-agents.test.ts new file mode 100644 index 0000000000..4053af095f --- /dev/null +++ b/test/e2e/live/podman-all-agents.test.ts @@ -0,0 +1,1174 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import path from "node:path"; + +import { resolveManagedGatewayStateDirectory } from "../../../src/lib/onboard/gateway-binding.ts"; +import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import type { HostCliClient } from "../fixtures/clients/host.ts"; +import { assertExitZero, resultText } from "../fixtures/clients/index.ts"; +import type { SandboxClient } from "../fixtures/clients/sandbox.ts"; +import { expect, test } from "../fixtures/e2e-test.ts"; +import { CLI_DIST_ENTRYPOINT, CLI_ENTRYPOINT } from "../fixtures/paths.ts"; +import { readRegistrySandboxEntry } from "../fixtures/phases/index.ts"; +import type { TargetEnvironment } from "../registry/types.ts"; + +const AGENTS = ["openclaw", "hermes", "langchain-deepagents-code"] as const; +type Agent = (typeof AGENTS)[number]; + +const AGENT_BINARIES: Readonly> = { + openclaw: "openclaw", + hermes: "hermes", + "langchain-deepagents-code": "dcode", +}; + +const EXPECTED_STATES: Readonly> = { + openclaw: "cloud-openclaw-ready", + hermes: "cloud-hermes-ready", + "langchain-deepagents-code": "cloud-deepagents-code-ready", +}; + +const ONBOARDING_PROFILES: Readonly> = { + openclaw: "cloud-openclaw", + hermes: "cloud-hermes", + "langchain-deepagents-code": "cloud-langchain-deepagents-code", +}; + +const PHASES = [ + "verify the native Podman lane contract", + "onboard the selected agent without Docker", + "verify the expected sandbox, managed image, and agent runtime receipt", + "snapshot and restore the selected agent into a native Podman clone", + "verify Podman doctor and exact-socket ownership", + "stop and start the exact managed sandbox", + "reject a conflicting shared-gateway Docker request", + "stop the host gateway and recover it from protected state", + "resume and rebuild the selected agent from a fresh shell", + "destroy the Podman sandbox and its final shared gateway", + "verify final Podman and gateway cleanup", + "record native Podman completion evidence", +] as const; +const DOCKER_ENV_KEYS = [ + "DOCKER_CONFIG", + "DOCKER_CONTEXT", + "DOCKER_HOST", + "DOCKER_TLS_VERIFY", + "DOCKER_CERT_PATH", + "DOCKER_API_VERSION", +] as const; + +interface ManagedImageEvidence { + agent: Agent; + release: string; + image: string; + reference: string; + digest: string; + sourceRevision: string; + sourceCohort: string; +} + +interface PodmanManagedContainerEvidence { + containerId: string; + containerName: string; + imageId: string; + imageName: string; + imageDigest: string; + imageRepoDigests: string[]; + startupCommand: string[]; + startupEntrypoint: string[]; +} + +interface ManagedStartupCompletionEvidence { + agent: Agent; + corporateCaMerged: boolean; + profileFingerprint: string; + runtimeEnvironmentSha256: string; + schemaVersion: 1; +} + +interface GatewayProcessReceipt { + pid: number; + startTicks: string; +} + +function requiredEnvironment(name: string): string { + const value = process.env[name]?.trim(); + if (!value) throw new Error(`${name} is required for the native Podman E2E lane.`); + return value; +} + +function selectedAgent(): Agent { + const value = requiredEnvironment("E2E_PODMAN_AGENT"); + if (!AGENTS.includes(value as Agent)) { + throw new Error(`E2E_PODMAN_AGENT must be one of ${AGENTS.join(", ")}; got '${value}'.`); + } + return value as Agent; +} + +function requireRecord(value: unknown, label: string): Record { + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error(`${label} must be an object.`); + } + return value as Record; +} + +function requireStringField(record: Record, key: string, label: string): string { + const value = record[key]; + if (typeof value !== "string" || value.length === 0) { + throw new Error(`${label} must be a non-empty string.`); + } + return value; +} + +function requireStringArrayField( + record: Record, + key: string, + label: string, +): string[] { + const value = record[key]; + if ( + !Array.isArray(value) || + !value.every((entry): entry is string => typeof entry === "string") + ) { + throw new Error(`${label} must be a string array.`); + } + return value; +} + +function readCatalogEvidence(file: string, agent: Agent): ManagedImageEvidence { + const parsed = requireRecord(JSON.parse(fs.readFileSync(file, "utf8")), "catalog evidence"); + const entries = parsed.images; + if (!Array.isArray(entries)) throw new Error("catalog evidence images must be an array."); + const entry = entries.find( + (candidate) => requireRecord(candidate, "catalog image").agent === agent, + ); + const image = requireRecord(entry, `${agent} catalog image`); + const evidence = { + agent, + release: image.release, + image: image.image, + reference: image.reference, + digest: image.digest, + sourceRevision: image.sourceRevision, + sourceCohort: image.sourceCohort, + }; + for (const [key, value] of Object.entries(evidence)) { + if (typeof value !== "string" || value.length === 0) { + throw new Error(`${agent} catalog evidence field '${key}' must be a non-empty string.`); + } + } + expect(evidence.digest).toMatch(/^sha256:[0-9a-f]{64}$/u); + expect(evidence.reference).toBe(`${evidence.image}@${evidence.digest}`); + return evidence as ManagedImageEvidence; +} + +function assertManagedWorkloadReceipt( + entry: Record, + catalog: ManagedImageEvidence, +): string { + expect(entry.openshellDriver).toBe("podman"); + expect(entry.pendingRouteReservation).not.toBe(true); + const workload = requireRecord(entry.workload, "registry workload"); + expect(workload).toMatchObject({ + schemaVersion: 1, + kind: "managed-image", + reference: catalog.reference, + release: catalog.release, + sourceRevision: catalog.sourceRevision, + sourceCohort: catalog.sourceCohort, + capabilityContractVersion: 1, + startupProfileContractVersion: 1, + credentialProxyReplayRequired: catalog.agent !== "langchain-deepagents-code", + shared: true, + }); + expect(workload.reference).toMatch( + /^ghcr[.]io\/nvidia\/nemoclaw\/[a-z0-9-]+-sandbox@sha256:[0-9a-f]{64}$/u, + ); + const startupProfileSha256 = requireStringField( + workload, + "startupProfileSha256", + "registry workload startupProfileSha256", + ); + expect(startupProfileSha256).toMatch(/^[0-9a-f]{64}$/u); + expect(workload.encodedProfile).toMatch(/^[A-Za-z0-9_-]+$/u); + return startupProfileSha256; +} + +function assertDockerGuardClean(file: string): void { + expect(fs.existsSync(file), "workflow must install the Docker invocation guard").toBe(true); + expect(fs.readFileSync(file, "utf8"), "native Podman lane invoked the Docker CLI").toBe(""); +} + +function podmanNativeEnv(extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { + const env = buildAvailabilityProbeEnv(); + for (const key of DOCKER_ENV_KEYS) delete env[key]; + env.OPENSHELL_GATEWAY = process.env.OPENSHELL_GATEWAY ?? "nemoclaw"; + return { ...env, ...extra }; +} + +function freshPodmanEnv(extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { + const env = podmanNativeEnv(extra); + delete env.NEMOCLAW_COMPUTE_DRIVER; + delete env.OPENSHELL_PODMAN_NETWORK_NAME; + delete env.OPENSHELL_PODMAN_SOCKET; + delete env.OPENSHELL_SUPERVISOR_IMAGE; + return env; +} + +function gatewayName(entry: Record): string { + const candidate = entry.gatewayName; + if (typeof candidate === "string" && candidate.trim()) return candidate.trim(); + return process.env.OPENSHELL_GATEWAY?.trim() || "nemoclaw"; +} + +function managedRuntimeBinding(stateDir: string): Record { + return requireRecord( + JSON.parse(fs.readFileSync(path.join(stateDir, "managed-runtime.json"), "utf8")), + "managed runtime binding", + ); +} + +function assertPodmanRuntimeBinding( + binding: Record, + socketPath: string, + networkName: string, +): void { + expect(binding).toMatchObject({ + version: 1, + driverName: "podman", + configSha256: expect.stringMatching(/^[0-9a-f]{64}$/u), + values: { + socket_path: socketPath, + network_name: networkName, + }, + }); +} + +function assertHealthyPodmanDoctor(stdout: string, sandboxName: string): void { + const report = requireRecord(JSON.parse(stdout), "doctor report"); + expect(report.schemaVersion).toBe(1); + expect(report.sandbox).toBe(sandboxName); + expect(report.failed).toBe(0); + expect(report.status === "ok" || report.status === "warn").toBe(true); + const checks = report.checks; + if (!Array.isArray(checks)) throw new Error("doctor report checks must be an array."); + for (const label of ["Podman service", "OpenShell status", "Live sandbox"]) { + const check = checks.find( + (candidate) => requireRecord(candidate, "doctor check").label === label, + ); + expect(requireRecord(check, `doctor check '${label}'`).status).toBe("ok"); + } +} + +function managedStartupCommand( + config: Record, + agent: Agent, + profileFingerprint: string, +): void { + const env = requireStringArrayField(config, "Env", "Podman container Config.Env"); + const commandEntries = env.filter((entry) => entry.startsWith("OPENSHELL_SANDBOX_COMMAND=")); + if (commandEntries.length !== 1) { + throw new Error("managed Podman container must persist one sandbox startup command"); + } + const command = commandEntries[0]?.slice("OPENSHELL_SANDBOX_COMMAND=".length) ?? ""; + const exactImageOwnedSuffix = + `/usr/local/bin/nemoclaw-managed-startup-hold --agent ${agent} ` + + `--profile-fingerprint ${profileFingerprint}`; + if ( + !command.startsWith("env ") || + !command.endsWith(exactImageOwnedSuffix) || + /(?:^|\s)sleep\s+infinity(?:\s|$)/u.test(command) + ) { + throw new Error( + "managed Podman container did not persist the exact image-owned startup command", + ); + } +} + +function normalizedPodmanUlimits( + hostConfig: Record, +): Map { + const raw = hostConfig.Ulimits; + if (!Array.isArray(raw)) throw new Error("Podman HostConfig.Ulimits must be an array."); + const limits = new Map(); + for (const [index, entry] of raw.entries()) { + const limit = requireRecord(entry, `Podman HostConfig.Ulimits[${String(index)}]`); + const name = requireStringField( + limit, + "Name", + `Podman HostConfig.Ulimits[${String(index)}].Name`, + ) + .replace(/^RLIMIT_/iu, "") + .toLowerCase(); + const soft = limit.Soft; + const hard = limit.Hard; + if (!Number.isSafeInteger(soft) || !Number.isSafeInteger(hard) || limits.has(name)) { + throw new Error(`Podman HostConfig.Ulimits contains invalid or repeated '${name}'.`); + } + limits.set(name, { hard: hard as number, soft: soft as number }); + } + return limits; +} + +function assertDcodeContainerUlimits(agent: Agent, hostConfig: Record): void { + if (agent !== "langchain-deepagents-code") return; + const limits = normalizedPodmanUlimits(hostConfig); + expect(limits.get("nproc")).toEqual({ hard: 512, soft: 512 }); + expect(limits.get("nofile")).toEqual({ hard: 65_536, soft: 65_536 }); +} + +function readGatewayProcessReceipt(pidFile: string): GatewayProcessReceipt { + const firstPidText = fs.readFileSync(pidFile, "utf8").trim(); + if (!/^[1-9][0-9]*$/u.test(firstPidText)) { + throw new Error("managed Podman gateway PID receipt is malformed"); + } + const pid = Number(firstPidText); + const stat = fs.readFileSync(`/proc/${String(pid)}/stat`, "utf8"); + const commandEnd = stat.lastIndexOf(")"); + const fields = + commandEnd >= 0 + ? stat + .slice(commandEnd + 2) + .trim() + .split(/\s+/u) + : []; + const startTicks = fields[19]; + if (!startTicks || !/^[1-9][0-9]*$/u.test(startTicks)) { + throw new Error("managed Podman gateway process start identity is unavailable"); + } + const argv = fs + .readFileSync(`/proc/${String(pid)}/cmdline`) + .toString("utf8") + .split("\0") + .filter(Boolean); + if (argv.length === 0 || !argv.join(" ").includes("openshell-gateway")) { + throw new Error("managed Podman gateway PID does not identify the OpenShell gateway"); + } + if (fs.readFileSync(pidFile, "utf8").trim() !== firstPidText) { + throw new Error("managed Podman gateway PID receipt changed during inspection"); + } + return { pid, startTicks }; +} + +async function findPodmanContainerIds( + host: HostCliClient, + socketPath: string, + sandboxName: string, + artifactName: string, +): Promise { + const result = await host.command( + "podman", + [ + "--url", + `unix://${socketPath}`, + "ps", + "--all", + "--no-trunc", + "--filter", + `label=openshell.sandbox-name=${sandboxName}`, + "--filter", + "label=openshell.managed=true", + "--format", + "{{.ID}}", + ], + { + artifactName, + env: podmanNativeEnv(), + timeoutMs: 30_000, + }, + ); + assertExitZero(result, `Podman container lookup for ${sandboxName}`); + return [ + ...new Set( + result.stdout + .split(/\r?\n/gu) + .map((value) => value.trim()) + .filter(Boolean), + ), + ]; +} + +async function assertNoPodmanRecreateArtifacts( + host: HostCliClient, + socketPath: string, + sandboxName: string, + artifactName: string, +): Promise { + const result = await host.command( + "podman", + ["--url", `unix://${socketPath}`, "ps", "--all", "--no-trunc", "--format", "{{.Names}}"], + { + artifactName, + env: podmanNativeEnv(), + timeoutMs: 30_000, + }, + ); + assertExitZero(result, `inspect Podman recreate artifacts for ${sandboxName}`); + const backupPrefix = `openshell-sandbox-${sandboxName}-nemoclaw-backup-`; + const backups = result.stdout + .split(/\r?\n/gu) + .map((value) => value.trim()) + .filter((value) => value.startsWith(backupPrefix)); + expect(backups, `Podman recreate backup remained for ${sandboxName}`).toEqual([]); +} + +async function inspectPodmanManagedContainer( + host: HostCliClient, + options: { + artifactPrefix: string; + catalog: ManagedImageEvidence; + expectedRunning?: boolean; + profileFingerprint: string; + sandboxName: string; + socketPath: string; + }, +): Promise { + const ids = await findPodmanContainerIds( + host, + options.socketPath, + options.sandboxName, + `${options.artifactPrefix}-lookup`, + ); + expect( + ids, + `expected exactly one Podman container for managed sandbox ${options.sandboxName}`, + ).toHaveLength(1); + const containerResult = await host.command( + "podman", + [ + "--url", + `unix://${options.socketPath}`, + "container", + "inspect", + "--format", + "{{json .}}", + ids[0], + ], + { + artifactName: `${options.artifactPrefix}-container-inspect`, + env: podmanNativeEnv(), + timeoutMs: 30_000, + }, + ); + assertExitZero(containerResult, `inspect Podman container for ${options.sandboxName}`); + const container = requireRecord( + JSON.parse(containerResult.stdout), + `Podman container ${options.sandboxName}`, + ); + const containerConfig = requireRecord(container.Config, "Podman container Config"); + const containerHostConfig = requireRecord(container.HostConfig, "Podman container HostConfig"); + const containerLabels = requireRecord(containerConfig.Labels, "Podman container labels"); + const containerState = requireRecord(container.State, "Podman container State"); + const containerId = requireStringField(container, "Id", "Podman container Id"); + const containerName = requireStringField(container, "Name", "Podman container Name"); + const imageId = requireStringField(container, "Image", "Podman container Image"); + const imageName = requireStringField(container, "ImageName", "Podman container ImageName"); + expect(containerId).toBe(ids[0]); + expect(containerName).toBe(`openshell-sandbox-${options.sandboxName}`); + expect(imageName).toBe(options.catalog.reference); + expect(containerLabels["openshell.sandbox-name"]).toBe(options.sandboxName); + expect(containerLabels["openshell.managed"]).toBe("true"); + expect(containerState.Running).toBe(options.expectedRunning ?? true); + managedStartupCommand(containerConfig, options.catalog.agent, options.profileFingerprint); + assertDcodeContainerUlimits(options.catalog.agent, containerHostConfig); + + const imageResult = await host.command( + "podman", + [ + "--url", + `unix://${options.socketPath}`, + "image", + "inspect", + "--format", + "{{json .}}", + options.catalog.reference, + ], + { + artifactName: `${options.artifactPrefix}-image-inspect`, + env: podmanNativeEnv(), + timeoutMs: 30_000, + }, + ); + assertExitZero(imageResult, `inspect managed Podman image ${options.catalog.reference}`); + const image = requireRecord(JSON.parse(imageResult.stdout), "Podman managed image"); + const imageConfig = requireRecord(image.Config, "Podman managed image Config"); + const imageLabels = requireRecord(imageConfig.Labels, "Podman managed image labels"); + const startupEntrypoint = requireStringArrayField( + imageConfig, + "Entrypoint", + "Podman managed image Config.Entrypoint", + ); + const startupCommand = requireStringArrayField( + imageConfig, + "Cmd", + "Podman managed image Config.Cmd", + ); + expect( + requireStringArrayField(containerConfig, "Entrypoint", "Podman container Config.Entrypoint"), + ).toContain("/opt/openshell/bin/openshell-sandbox"); + expect( + requireStringArrayField(containerConfig, "Cmd", "Podman container Config.Cmd"), + ).not.toEqual(["sleep", "infinity"]); + expect(startupEntrypoint).toContain("/usr/local/bin/nemoclaw-start"); + expect(startupCommand).not.toEqual(["sleep", "infinity"]); + const imageDigest = image.Digest; + const imageRepoDigests = image.RepoDigests; + const inspectedImageId = requireStringField(image, "Id", "Podman managed image Id"); + if (typeof imageDigest !== "string") { + throw new Error("Podman managed image Digest must be a string."); + } + if ( + !Array.isArray(imageRepoDigests) || + !imageRepoDigests.every((value): value is string => typeof value === "string") + ) { + throw new Error("Podman managed image RepoDigests must be a string array."); + } + expect(imageId).toBe(inspectedImageId); + expect(imageDigest).toBe(options.catalog.digest); + expect(imageRepoDigests).toContain(options.catalog.reference); + expect(imageLabels).toMatchObject({ + "io.nvidia.nemoclaw.agent": options.catalog.agent, + "io.nvidia.nemoclaw.managed-image.contract": "1", + "io.nvidia.nemoclaw.managed-image.platform": "linux/amd64", + "io.nvidia.nemoclaw.managed-image.startup-profile": "1", + "io.nvidia.nemoclaw.managed-image.capabilities": "1", + "io.nvidia.nemoclaw.managed-image.cohort": options.catalog.sourceCohort, + "org.opencontainers.image.revision": options.catalog.sourceRevision, + "org.opencontainers.image.source": "https://github.com/NVIDIA/NemoClaw", + }); + + return { + containerId, + containerName, + imageId, + imageName, + imageDigest, + imageRepoDigests, + startupCommand, + startupEntrypoint, + }; +} + +async function inspectManagedStartupCompletion( + sandbox: SandboxClient, + options: { + agent: Agent; + artifactName: string; + profileFingerprint: string; + sandboxName: string; + }, +): Promise { + const result = await sandbox.exec( + options.sandboxName, + ["cat", "/run/nemoclaw/managed-startup-complete.json"], + { + artifactName: options.artifactName, + env: freshPodmanEnv(), + timeoutMs: 30_000, + }, + ); + assertExitZero(result, `${options.agent} managed-startup completion marker`); + const marker = requireRecord(JSON.parse(result.stdout), "managed-startup completion marker"); + const evidence = { + agent: marker.agent, + corporateCaMerged: marker.corporateCaMerged, + profileFingerprint: marker.profileFingerprint, + runtimeEnvironmentSha256: marker.runtimeEnvironmentSha256, + schemaVersion: marker.schemaVersion, + }; + expect(evidence).toMatchObject({ + agent: options.agent, + profileFingerprint: options.profileFingerprint, + runtimeEnvironmentSha256: expect.stringMatching(/^[0-9a-f]{64}$/u), + schemaVersion: 1, + }); + expect(typeof evidence.corporateCaMerged).toBe("boolean"); + return evidence as ManagedStartupCompletionEvidence; +} + +async function assertDcodeLiveUlimits( + sandbox: SandboxClient, + agent: Agent, + sandboxName: string, + artifactName: string, +): Promise { + if (agent !== "langchain-deepagents-code") return; + const result = await sandbox.exec( + sandboxName, + [ + "bash", + "-lc", + 'printf "%s:%s:%s:%s\\n" "$(ulimit -Su)" "$(ulimit -Hu)" "$(ulimit -Sn)" "$(ulimit -Hn)"', + ], + { + artifactName, + env: freshPodmanEnv(), + timeoutMs: 30_000, + }, + ); + assertExitZero(result, "DCode live nproc/nofile contract"); + expect(result.stdout.trim()).toBe("512:512:65536:65536"); +} + +process.env.NEMOCLAW_CLI_BIN ??= CLI_ENTRYPOINT; + +test("native Podman: all supported agents use managed OCI images without Docker", { + timeout: 75 * 60_000, + meta: { e2ePhases: PHASES }, +}, async ({ + artifacts, + cleanup, + environment, + host, + onboard, + progress, + sandbox, + stateValidation, +}) => { + const agent = selectedAgent(); + const apiKey = requiredEnvironment("NVIDIA_INFERENCE_API_KEY"); + const release = requiredEnvironment("E2E_PODMAN_MANAGED_IMAGE_RELEASE"); + const socketPath = requiredEnvironment("OPENSHELL_PODMAN_SOCKET"); + const networkName = requiredEnvironment("OPENSHELL_PODMAN_NETWORK_NAME"); + const catalogEvidencePath = requiredEnvironment("E2E_PODMAN_CATALOG_EVIDENCE"); + const dockerGuardLog = requiredEnvironment("E2E_DOCKER_GUARD_LOG"); + const sandboxName = `e2e-podman-${agent.replaceAll(/[^a-z0-9]/gu, "-")}`; + const snapshotCloneName = `${sandboxName}-snapshot`; + const catalog = readCatalogEvidence(catalogEvidencePath, agent); + + expect(fs.existsSync(CLI_DIST_ENTRYPOINT), "run `npm run build:cli` before live targets").toBe( + true, + ); + expect(path.isAbsolute(socketPath)).toBe(true); + expect(fs.statSync(socketPath).isSocket()).toBe(true); + expect(process.env.DOCKER_HOST ?? "").toBe(""); + expect(catalog.release).toBe(release); + assertDockerGuardClean(dockerGuardLog); + + await artifacts.target.declare({ + id: "podman-all-agents", + agent, + sandboxName, + contracts: [ + "the CLI selects the native Podman compute driver through an exact rootless API socket", + "OpenClaw, Hermes, and DCode consume immutable managed OCI images", + "managed startup, snapshot clone, and rebuild preserve the agent-specific Podman contract", + "the native Podman lane never invokes Docker or uses DOCKER_HOST", + ], + }); + + const targetEnvironment: TargetEnvironment = { + platform: "ubuntu-local", + install: "repo-current", + runtime: "podman-running", + onboarding: ONBOARDING_PROFILES[agent], + }; + const ready = await environment.assertReady(targetEnvironment); + + progress.phase("onboard the selected agent without Docker"); + const instance = await onboard.from(ready, { sandboxName }); + + progress.phase("verify the expected sandbox, managed image, and agent runtime receipt"); + const state = await stateValidation.from(EXPECTED_STATES[agent], instance); + + const registryEntry = readRegistrySandboxEntry(sandboxName); + const initialProfileFingerprint = assertManagedWorkloadReceipt(registryEntry, catalog); + const runtimeGatewayName = gatewayName(registryEntry); + const gatewayStateDir = resolveManagedGatewayStateDirectory(runtimeGatewayName, { + env: podmanNativeEnv(), + }); + const gatewayPidFile = path.join(gatewayStateDir, "openshell-gateway.pid"); + const runtimeBindingBeforeRecovery = managedRuntimeBinding(gatewayStateDir); + assertPodmanRuntimeBinding(runtimeBindingBeforeRecovery, socketPath, networkName); + const initialContainer = await inspectPodmanManagedContainer(host, { + artifactPrefix: `podman-${agent}-initial`, + catalog, + profileFingerprint: initialProfileFingerprint, + sandboxName, + socketPath, + }); + const initialCompletion = await inspectManagedStartupCompletion(sandbox, { + agent, + artifactName: `podman-${agent}-managed-startup-completion`, + profileFingerprint: initialProfileFingerprint, + sandboxName, + }); + await assertDcodeLiveUlimits(sandbox, agent, sandboxName, `podman-${agent}-live-ulimits`); + await assertNoPodmanRecreateArtifacts( + host, + socketPath, + sandboxName, + `podman-${agent}-recreate-artifacts-initial`, + ); + const initialGatewayProcess = readGatewayProcessReceipt(gatewayPidFile); + const version = await sandbox.exec(sandboxName, [AGENT_BINARIES[agent], "--version"], { + artifactName: `podman-${agent}-version`, + env: podmanNativeEnv(), + timeoutMs: 30_000, + }); + assertExitZero(version, `${agent} managed-image binary`); + expect(resultText(version).trim()).not.toBe(""); + assertDockerGuardClean(dockerGuardLog); + + progress.phase("snapshot and restore the selected agent into a native Podman clone"); + const snapshotCreate = await host.nemoclaw( + [sandboxName, "snapshot", "create", "--name", "podman-runtime"], + { + artifactName: `podman-${agent}-snapshot-create`, + env: freshPodmanEnv(), + timeoutMs: 180_000, + }, + ); + assertExitZero(snapshotCreate, `create Podman snapshot for ${sandboxName}`); + expect(resultText(snapshotCreate)).toMatch(/Snapshot v\d+.*created/u); + + let snapshotCloneDestroyed = false; + cleanup.trackDisposable(`destroy Podman snapshot clone ${snapshotCloneName}`, async () => { + if (snapshotCloneDestroyed) return; + await host.nemoclaw([snapshotCloneName, "destroy", "--yes"], { + artifactName: `podman-${agent}-snapshot-clone-cleanup`, + env: freshPodmanEnv(), + timeoutMs: 180_000, + }); + }); + const snapshotRestore = await host.nemoclaw( + [sandboxName, "snapshot", "restore", "podman-runtime", "--to", snapshotCloneName, "--yes"], + { + artifactName: `podman-${agent}-snapshot-restore-clone`, + env: freshPodmanEnv({ NVIDIA_INFERENCE_API_KEY: apiKey }), + redactionValues: [apiKey], + timeoutMs: 10 * 60_000, + }, + ); + assertExitZero(snapshotRestore, `restore Podman snapshot into ${snapshotCloneName}`); + const snapshotCloneEntry = readRegistrySandboxEntry(snapshotCloneName); + const snapshotCloneProfileFingerprint = assertManagedWorkloadReceipt(snapshotCloneEntry, catalog); + const snapshotCloneContainer = await inspectPodmanManagedContainer(host, { + artifactPrefix: `podman-${agent}-snapshot-clone`, + catalog, + profileFingerprint: snapshotCloneProfileFingerprint, + sandboxName: snapshotCloneName, + socketPath, + }); + expect(snapshotCloneContainer.containerId).not.toBe(initialContainer.containerId); + const snapshotCloneCompletion = await inspectManagedStartupCompletion(sandbox, { + agent, + artifactName: `podman-${agent}-snapshot-clone-completion`, + profileFingerprint: snapshotCloneProfileFingerprint, + sandboxName: snapshotCloneName, + }); + await assertDcodeLiveUlimits( + sandbox, + agent, + snapshotCloneName, + `podman-${agent}-snapshot-clone-live-ulimits`, + ); + await assertNoPodmanRecreateArtifacts( + host, + socketPath, + snapshotCloneName, + `podman-${agent}-snapshot-clone-recreate-artifacts`, + ); + const snapshotCloneVersion = await sandbox.exec( + snapshotCloneName, + [AGENT_BINARIES[agent], "--version"], + { + artifactName: `podman-${agent}-snapshot-clone-version`, + env: freshPodmanEnv(), + timeoutMs: 30_000, + }, + ); + assertExitZero(snapshotCloneVersion, `${agent} managed-image binary in snapshot clone`); + expect(resultText(snapshotCloneVersion).trim()).not.toBe(""); + const destroySnapshotClone = await host.nemoclaw([snapshotCloneName, "destroy", "--yes"], { + artifactName: `podman-${agent}-snapshot-clone-destroy`, + env: freshPodmanEnv(), + timeoutMs: 180_000, + }); + assertExitZero(destroySnapshotClone, `destroy Podman snapshot clone ${snapshotCloneName}`); + snapshotCloneDestroyed = true; + expect( + await findPodmanContainerIds( + host, + socketPath, + snapshotCloneName, + `podman-${agent}-snapshot-clone-after-destroy`, + ), + ).toHaveLength(0); + await assertNoPodmanRecreateArtifacts( + host, + socketPath, + snapshotCloneName, + `podman-${agent}-snapshot-clone-artifacts-after-destroy`, + ); + const snapshotGatewayProcess = readGatewayProcessReceipt(gatewayPidFile); + expect( + `${snapshotGatewayProcess.pid}:${snapshotGatewayProcess.startTicks}`, + "snapshot clone cutover must resume a new exact standalone gateway process", + ).not.toBe(`${initialGatewayProcess.pid}:${initialGatewayProcess.startTicks}`); + assertDockerGuardClean(dockerGuardLog); + + progress.phase("verify Podman doctor and exact-socket ownership"); + const doctor = await host.nemoclaw([sandboxName, "doctor", "--json"], { + artifactName: `podman-${agent}-doctor-initial`, + env: podmanNativeEnv(), + redactionValues: [apiKey], + timeoutMs: 120_000, + }); + assertExitZero(doctor, `doctor the Podman sandbox ${sandboxName}`); + assertHealthyPodmanDoctor(doctor.stdout, sandboxName); + assertPodmanRuntimeBinding(managedRuntimeBinding(gatewayStateDir), socketPath, networkName); + assertDockerGuardClean(dockerGuardLog); + + progress.phase("stop and start the exact managed sandbox"); + const stoppedSandbox = await host.nemoclaw([sandboxName, "stop"], { + artifactName: `podman-${agent}-sandbox-stop`, + env: freshPodmanEnv(), + timeoutMs: 120_000, + }); + assertExitZero(stoppedSandbox, `stop the exact managed Podman sandbox ${sandboxName}`); + const stoppedContainer = await inspectPodmanManagedContainer(host, { + artifactPrefix: `podman-${agent}-after-stop`, + catalog, + expectedRunning: false, + profileFingerprint: initialProfileFingerprint, + sandboxName, + socketPath, + }); + expect(stoppedContainer).toEqual(initialContainer); + assertDockerGuardClean(dockerGuardLog); + + const startedSandbox = await host.nemoclaw([sandboxName, "start"], { + artifactName: `podman-${agent}-sandbox-start`, + env: freshPodmanEnv(), + timeoutMs: 180_000, + }); + assertExitZero(startedSandbox, `start the exact managed Podman sandbox ${sandboxName}`); + const restartedStatus = await host.expectStatus(sandboxName, { + artifactName: `podman-${agent}-status-after-sandbox-start`, + env: freshPodmanEnv(), + timeoutMs: 120_000, + }); + assertExitZero(restartedStatus, `prove readiness after starting Podman sandbox ${sandboxName}`); + const restartedContainer = await inspectPodmanManagedContainer(host, { + artifactPrefix: `podman-${agent}-after-start`, + catalog, + profileFingerprint: initialProfileFingerprint, + sandboxName, + socketPath, + }); + expect( + restartedContainer, + "stop/start must preserve the exact managed Podman container and immutable image identity", + ).toEqual(initialContainer); + const restartedVersion = await sandbox.exec(sandboxName, [AGENT_BINARIES[agent], "--version"], { + artifactName: `podman-${agent}-version-after-sandbox-start`, + env: freshPodmanEnv(), + timeoutMs: 30_000, + }); + assertExitZero(restartedVersion, `${agent} managed-image binary after stop/start`); + expect(resultText(restartedVersion).trim()).not.toBe(""); + assertDockerGuardClean(dockerGuardLog); + + progress.phase("reject a conflicting shared-gateway Docker request"); + const conflictName = `${sandboxName}-docker-conflict`; + const conflictingDriver = await host.nemoclaw( + [ + "onboard", + "--non-interactive", + "--yes", + "--yes-i-accept-third-party-software", + "--compute-driver", + "docker", + "--name", + conflictName, + ], + { + artifactName: `podman-${agent}-reject-docker-conflict`, + env: podmanNativeEnv({ + NVIDIA_INFERENCE_API_KEY: apiKey, + NEMOCLAW_AGENT: agent, + NEMOCLAW_PROVIDER: "cloud", + NEMOCLAW_SANDBOX_NAME: conflictName, + }), + redactionValues: [apiKey], + timeoutMs: 120_000, + }, + ); + expect(conflictingDriver.exitCode).not.toBe(0); + expect(resultText(conflictingDriver)).toMatch( + /Requested OpenShell compute driver 'docker' does not match existing sandbox driver 'podman'|Conflicting persisted OpenShell compute drivers/u, + ); + const afterConflict = await host.expectListed(sandboxName, { + artifactName: `podman-${agent}-listed-after-driver-conflict`, + env: podmanNativeEnv(), + timeoutMs: 120_000, + }); + assertExitZero(afterConflict, `list original Podman sandbox after conflicting driver request`); + expect(resultText(afterConflict)).not.toContain(conflictName); + const afterConflictContainer = await inspectPodmanManagedContainer(host, { + artifactPrefix: `podman-${agent}-after-conflict`, + catalog, + profileFingerprint: initialProfileFingerprint, + sandboxName, + socketPath, + }); + expect(afterConflictContainer.containerId).toBe(initialContainer.containerId); + assertDockerGuardClean(dockerGuardLog); + + progress.phase("stop the host gateway and recover it from protected state"); + const stoppedGateway = await host.command( + "bash", + [ + "-lc", + ` +set -euo pipefail +pid_file="$1" +test -f "$pid_file" +pid="$(tr -d '[:space:]' <"$pid_file")" +[[ "$pid" =~ ^[1-9][0-9]*$ ]] +cmdline="$(tr '\\0' ' ' <"/proc/$pid/cmdline")" +[[ "$cmdline" == *openshell-gateway* ]] +kill -TERM "$pid" +for _ in $(seq 1 30); do + if ! kill -0 "$pid" 2>/dev/null; then exit 0; fi + state="$(ps -o stat= -p "$pid" 2>/dev/null | tr -d '[:space:]' || true)" + [[ "$state" == Z* ]] && exit 0 + sleep 1 +done +echo "managed OpenShell gateway pid $pid did not stop" >&2 +exit 1 +`, + "stop-managed-podman-gateway", + gatewayPidFile, + ], + { + artifactName: `podman-${agent}-stop-managed-gateway`, + env: podmanNativeEnv(), + timeoutMs: 45_000, + }, + ); + assertExitZero(stoppedGateway, `stop the managed Podman gateway for ${sandboxName}`); + expect(fs.existsSync(path.join(gatewayStateDir, "managed-runtime.json"))).toBe(true); + assertDockerGuardClean(dockerGuardLog); + + const recoveredStatus = await host.expectStatus(sandboxName, { + artifactName: `podman-${agent}-fresh-shell-recovery-status`, + env: freshPodmanEnv(), + timeoutMs: 180_000, + }); + assertExitZero(recoveredStatus, `recover the Podman gateway from protected runtime state`); + const runtimeBindingAfterRecovery = managedRuntimeBinding(gatewayStateDir); + assertPodmanRuntimeBinding(runtimeBindingAfterRecovery, socketPath, networkName); + expect(runtimeBindingAfterRecovery).toEqual(runtimeBindingBeforeRecovery); + const recoveredDoctor = await host.nemoclaw([sandboxName, "doctor", "--json"], { + artifactName: `podman-${agent}-doctor-after-recovery`, + env: freshPodmanEnv(), + redactionValues: [apiKey], + timeoutMs: 120_000, + }); + assertExitZero(recoveredDoctor, `doctor recovered Podman sandbox ${sandboxName}`); + assertHealthyPodmanDoctor(recoveredDoctor.stdout, sandboxName); + const recoveredGatewayProcess = readGatewayProcessReceipt(gatewayPidFile); + expect( + `${recoveredGatewayProcess.pid}:${recoveredGatewayProcess.startTicks}`, + "fresh-shell recovery must replace the exact standalone gateway process", + ).not.toBe(`${snapshotGatewayProcess.pid}:${snapshotGatewayProcess.startTicks}`); + assertDockerGuardClean(dockerGuardLog); + + progress.phase("resume and rebuild the selected agent from a fresh shell"); + const resume = await host.nemoclaw( + ["onboard", "--resume", "--non-interactive", "--yes", "--yes-i-accept-third-party-software"], + { + artifactName: `podman-${agent}-fresh-shell-resume`, + env: freshPodmanEnv({ + NVIDIA_INFERENCE_API_KEY: apiKey, + NEMOCLAW_AGENT: agent, + NEMOCLAW_PROVIDER: "cloud", + }), + redactionValues: [apiKey], + timeoutMs: 15 * 60_000, + }, + ); + assertExitZero(resume, `resume Podman onboarding for ${sandboxName}`); + const runtimeBindingAfterResume = managedRuntimeBinding(gatewayStateDir); + assertPodmanRuntimeBinding(runtimeBindingAfterResume, socketPath, networkName); + expect(runtimeBindingAfterResume).toEqual(runtimeBindingBeforeRecovery); + assertDockerGuardClean(dockerGuardLog); + + const rebuilt = await host.nemoclaw([sandboxName, "rebuild", "--yes"], { + artifactName: `podman-${agent}-fresh-shell-rebuild`, + env: freshPodmanEnv({ + NVIDIA_INFERENCE_API_KEY: apiKey, + }), + redactionValues: [apiKey], + timeoutMs: 15 * 60_000, + }); + assertExitZero(rebuilt, `rebuild Podman sandbox ${sandboxName}`); + const rebuiltRegistryEntry = readRegistrySandboxEntry(sandboxName); + const rebuiltProfileFingerprint = assertManagedWorkloadReceipt(rebuiltRegistryEntry, catalog); + const rebuiltContainer = await inspectPodmanManagedContainer(host, { + artifactPrefix: `podman-${agent}-after-rebuild`, + catalog, + profileFingerprint: rebuiltProfileFingerprint, + sandboxName, + socketPath, + }); + expect( + rebuiltContainer.containerId, + "rebuild must replace the managed Podman sandbox container", + ).not.toBe(initialContainer.containerId); + const runtimeBindingAfterRebuild = managedRuntimeBinding(gatewayStateDir); + assertPodmanRuntimeBinding(runtimeBindingAfterRebuild, socketPath, networkName); + expect(runtimeBindingAfterRebuild).toEqual(runtimeBindingBeforeRecovery); + const rebuiltCompletion = await inspectManagedStartupCompletion(sandbox, { + agent, + artifactName: `podman-${agent}-managed-startup-completion-after-rebuild`, + profileFingerprint: rebuiltProfileFingerprint, + sandboxName, + }); + await assertDcodeLiveUlimits( + sandbox, + agent, + sandboxName, + `podman-${agent}-live-ulimits-after-rebuild`, + ); + await assertNoPodmanRecreateArtifacts( + host, + socketPath, + sandboxName, + `podman-${agent}-recreate-artifacts-after-rebuild`, + ); + const rebuiltGatewayProcess = readGatewayProcessReceipt(gatewayPidFile); + expect( + `${rebuiltGatewayProcess.pid}:${rebuiltGatewayProcess.startTicks}`, + "rebuild cutover must resume a new exact standalone gateway process", + ).not.toBe(`${recoveredGatewayProcess.pid}:${recoveredGatewayProcess.startTicks}`); + const rebuiltVersion = await sandbox.exec(sandboxName, [AGENT_BINARIES[agent], "--version"], { + artifactName: `podman-${agent}-version-after-rebuild`, + env: freshPodmanEnv(), + timeoutMs: 30_000, + }); + assertExitZero(rebuiltVersion, `${agent} managed-image binary after rebuild`); + expect(resultText(rebuiltVersion).trim()).not.toBe(""); + assertDockerGuardClean(dockerGuardLog); + + progress.phase("destroy the Podman sandbox and its final shared gateway"); + const destroyed = await host.nemoclaw([sandboxName, "destroy", "--yes", "--cleanup-gateway"], { + artifactName: `podman-${agent}-final-destroy`, + env: freshPodmanEnv(), + timeoutMs: 15 * 60_000, + }); + assertExitZero(destroyed, `destroy Podman sandbox and final gateway ${sandboxName}`); + assertDockerGuardClean(dockerGuardLog); + + progress.phase("verify final Podman and gateway cleanup"); + expect( + await findPodmanContainerIds( + host, + socketPath, + sandboxName, + `podman-${agent}-container-after-final-destroy`, + ), + ).toHaveLength(0); + await assertNoPodmanRecreateArtifacts( + host, + socketPath, + sandboxName, + `podman-${agent}-recreate-artifacts-after-final-destroy`, + ); + const podmanAfterCleanup = await host.command( + "podman", + ["--url", `unix://${socketPath}`, "info", "--format", "json"], + { + artifactName: `podman-${agent}-info-after-final-destroy`, + env: freshPodmanEnv(), + timeoutMs: 30_000, + }, + ); + assertExitZero(podmanAfterCleanup, "prove Podman remained reachable for final cleanup"); + const networkAfterCleanup = await host.command( + "podman", + ["--url", `unix://${socketPath}`, "network", "exists", networkName], + { + artifactName: `podman-${agent}-network-after-final-destroy`, + env: freshPodmanEnv(), + timeoutMs: 30_000, + }, + ); + expect(networkAfterCleanup.timedOut, resultText(networkAfterCleanup)).toBe(false); + expect( + networkAfterCleanup.exitCode, + `managed Podman network '${networkName}' still exists or could not be checked: ${resultText( + networkAfterCleanup, + )}`, + ).toBe(1); + expect(fs.existsSync(gatewayPidFile), "final cleanup must remove the managed gateway PID").toBe( + false, + ); + expect( + fs.existsSync(path.join(gatewayStateDir, "managed-runtime.json")), + "final cleanup must remove the protected runtime binding", + ).toBe(false); + assertDockerGuardClean(dockerGuardLog); + + progress.phase("record native Podman completion evidence"); + await artifacts.writeJson("podman-runtime-evidence.json", { + agent, + candidateRevision: process.env.GITHUB_SHA ?? null, + driverName: registryEntry.openshellDriver, + release: catalog.release, + reference: catalog.reference, + digest: catalog.digest, + sourceRevision: catalog.sourceRevision, + sourceCohort: catalog.sourceCohort, + socketPath, + networkName, + gatewayName: runtimeGatewayName, + gatewayStateDir, + initialContainer, + initialCompletion, + snapshotClone: { + container: snapshotCloneContainer, + completion: snapshotCloneCompletion, + destroyed: true, + sandboxName: snapshotCloneName, + }, + restartedContainer, + rebuiltContainer, + rebuiltCompletion, + gatewayProcessIdentities: { + initial: { + pid: initialGatewayProcess.pid, + startTicks: initialGatewayProcess.startTicks, + }, + recovered: { + pid: recoveredGatewayProcess.pid, + startTicks: recoveredGatewayProcess.startTicks, + }, + snapshotClone: { + pid: snapshotGatewayProcess.pid, + startTicks: snapshotGatewayProcess.startTicks, + }, + rebuilt: { + pid: rebuiltGatewayProcess.pid, + startTicks: rebuiltGatewayProcess.startTicks, + }, + }, + recoveredFromProtectedBinding: true, + snapshotCloneRestored: true, + stoppedAndStarted: true, + rebuilt: true, + finalGatewayCleanup: true, + expectedStateId: state.state.id, + dockerGuardLogBytes: fs.statSync(dockerGuardLog).size, + }); + await artifacts.target.complete({ + id: "podman-all-agents", + agent, + release: catalog.release, + reference: catalog.reference, + expectedStateId: state.state.id, + }); +}); diff --git a/test/e2e/registry/runtime-support.ts b/test/e2e/registry/runtime-support.ts index 40b304207d..717ff75058 100644 --- a/test/e2e/registry/runtime-support.ts +++ b/test/e2e/registry/runtime-support.ts @@ -8,6 +8,7 @@ const SUPPORTED_INSTALLS = new Set(["repo-current"]); const SUPPORTED_RUNTIMES = new Set(["docker-running"]); const SUPPORTED_ONBOARDING = new Set([ "cloud-openclaw", + "cloud-hermes", "cloud-openclaw-policy-custom-missing-presets", "cloud-langchain-deepagents-code", ]); diff --git a/test/e2e/support/base-image-publication.test.ts b/test/e2e/support/base-image-publication.test.ts index 7bd925d003..67230f143d 100644 --- a/test/e2e/support/base-image-publication.test.ts +++ b/test/e2e/support/base-image-publication.test.ts @@ -10,6 +10,7 @@ import { describe, expect, it } from "vitest"; import { collectPaginated, + expandBaseImagePushPaths, type FirstParentHistory, githubRequest, type PublicationRun, @@ -144,8 +145,14 @@ function publisherJob( function successfulJobs(overrides: { runAttempt?: number } = {}): Record[] { const runAttempt = overrides.runAttempt ?? 1; return [ - publisherJob("Build and push OpenClaw base image", { id: 1, run_attempt: runAttempt }), - publisherJob("Build and push Hermes base image", { id: 2, run_attempt: runAttempt }), + publisherJob("Build and push OpenClaw base image", { + id: 1, + run_attempt: runAttempt, + }), + publisherJob("Build and push Hermes base image", { + id: 2, + run_attempt: runAttempt, + }), publisherJob("Build and push Deep Agents Code base image", { id: 3, run_attempt: runAttempt, @@ -154,7 +161,7 @@ function successfulJobs(overrides: { runAttempt?: number } = {}): Record { - it("extracts the checked-in literal publisher paths without runtime dependencies (#7372)", () => { + it("extracts literal paths and the reviewed managed-image input families (#7372)", () => { const source = fs.readFileSync( path.resolve(import.meta.dirname, "../../../.github/workflows/base-image.yaml"), "utf8", @@ -163,9 +170,20 @@ describe("base-image publication evidence", () => { expect(parseBaseImagePushPaths(source)).toEqual( expect.arrayContaining([ ".github/workflows/base-image.yaml", + "Dockerfile", "Dockerfile.base", + "agents/**", "agents/hermes/Dockerfile.base", "agents/langchain-deepagents-code/Dockerfile.base", + "nemoclaw/**", + "nemoclaw-blueprint/**", + "scripts/**", + "src/lib/actions/sandbox/openshell-child-visible-credentials.v*.json", + "src/lib/messaging/**", + "src/lib/onboard/managed-startup/**", + "src/lib/tool-disclosure.ts", + "tools/mcp-tool-discovery-runtime/**", + "tsconfig.runtime-preloads.json", ]), ); }); @@ -216,6 +234,86 @@ describe("base-image publication evidence", () => { expect(() => parseBaseImagePushPaths(source)).toThrow(expected); }); + it("expands only reviewed glob families against first-parent Git history (#7744)", () => { + const calls: string[][] = []; + const expanded = expandBaseImagePushPaths( + EXPECTED_SHA, + ["Dockerfile", "agents/**", "src/lib/messaging/**", "src/lib/onboard/managed-startup/**"], + (args) => { + calls.push(args); + const pathspec = required(args.at(-1), "glob expansion call is missing a pathspec"); + return required( + new Map([ + [ + ":(glob)agents/**", + "agents/hermes/Dockerfile\nagents/openclaw/manifest.yaml\nagents/hermes/Dockerfile", + ], + [ + ":(glob)src/lib/messaging/**", + "src/lib/messaging/channels/slack.ts\nsrc/lib/messaging/types.ts", + ], + [ + ":(glob)src/lib/onboard/managed-startup/**", + "src/lib/onboard/managed-startup/image-runtime.ts\nsrc/lib/onboard/managed-startup/profile.ts", + ], + ]).get(pathspec), + `unexpected glob expansion: ${args.join(" ")}`, + ); + }, + ); + + expect(expanded).toEqual([ + "Dockerfile", + "agents/hermes/Dockerfile", + "agents/openclaw/manifest.yaml", + "src/lib/messaging/channels/slack.ts", + "src/lib/messaging/types.ts", + "src/lib/onboard/managed-startup/image-runtime.ts", + "src/lib/onboard/managed-startup/profile.ts", + ]); + expect(calls).toEqual([ + [ + "log", + "--first-parent", + "--diff-merges=first-parent", + "--format=", + "--name-only", + EXPECTED_SHA, + "--", + ":(glob)agents/**", + ], + [ + "log", + "--first-parent", + "--diff-merges=first-parent", + "--format=", + "--name-only", + EXPECTED_SHA, + "--", + ":(glob)src/lib/messaging/**", + ], + [ + "log", + "--first-parent", + "--diff-merges=first-parent", + "--format=", + "--name-only", + EXPECTED_SHA, + "--", + ":(glob)src/lib/onboard/managed-startup/**", + ], + ]); + }); + + it("fails closed when a reviewed glob is empty or Git returns an out-of-family path (#7744)", () => { + expect(() => expandBaseImagePushPaths(EXPECTED_SHA, ["agents/**"], () => "")).toThrow( + /did not match Git history/u, + ); + expect(() => + expandBaseImagePushPaths(EXPECTED_SHA, ["agents/**"], () => "scripts/escaped.sh"), + ).toThrow(/outside reviewed/u); + }); + it("binds the applicable commit to the checked-out first-parent chain (#7372)", () => { const calls: string[][] = []; const resolved = resolveFirstParentHistory(EXPECTED_SHA, ["Dockerfile.base"], (args) => { @@ -312,7 +410,9 @@ describe("base-image publication evidence", () => { }); it("collects page-two evidence and rejects duplicate or truncated pagination (#7372)", async () => { - const entries = Array.from({ length: 101 }, (_, index) => ({ id: index + 1 })); + const entries = Array.from({ length: 101 }, (_, index) => ({ + id: index + 1, + })); const pages = [ { total_count: entries.length, workflow_runs: entries.slice(0, 100) }, { total_count: entries.length, workflow_runs: entries.slice(100) }, @@ -355,7 +455,10 @@ describe("base-image publication evidence", () => { WORKFLOW_ID, ); - expect(selection).toMatchObject({ state: "ready", run: { headSha: EXPECTED_SHA } }); + expect(selection).toMatchObject({ + state: "ready", + run: { headSha: EXPECTED_SHA }, + }); }); it("prefers the graph-newest trusted run without relying on API order (#7372)", () => { @@ -376,7 +479,10 @@ describe("base-image publication evidence", () => { WORKFLOW_ID, ); - expect(selection).toMatchObject({ state: "ready", run: { id: 11, headSha: DESCENDANT_SHA } }); + expect(selection).toMatchObject({ + state: "ready", + run: { id: 11, headSha: DESCENDANT_SHA }, + }); }); it("ignores pre-rename workflow metadata outside the eligible history (#7372)", () => { @@ -434,7 +540,10 @@ describe("base-image publication evidence", () => { selectPublicationRun( runsPayload([ workflowRun(), - workflowRun({ id: RUN_ID + 1, html_url: `${RUN_URL_ROOT}/${RUN_ID + 1}` }), + workflowRun({ + id: RUN_ID + 1, + html_url: `${RUN_URL_ROOT}/${RUN_ID + 1}`, + }), ]), history(), WORKFLOW_ID, @@ -465,7 +574,10 @@ describe("base-image publication evidence", () => { run_attempt: 1, conclusion: "failure", }), - publisherJob("Build and push Hermes base image", { id: 5, run_attempt: 2 }), + publisherJob("Build and push Hermes base image", { + id: 5, + run_attempt: 2, + }), ].filter((job, index) => index !== 1); expect(() => validatePublisherJobs({ total_count: jobs.length, jobs }, run)).not.toThrow(); @@ -582,7 +694,10 @@ describe("base-image publication evidence", () => { it("retries bounded transient and rate-limited GitHub responses (#7372)", async () => { const transientResponses: Array = [ new Error("network unavailable"), - new Response("unavailable", { status: 503, headers: { "retry-after": "2" } }), + new Response("unavailable", { + status: 503, + headers: { "retry-after": "2" }, + }), new Response(JSON.stringify({ ok: true }), { status: 200 }), ]; const transientSleeps: number[] = []; diff --git a/test/e2e/support/dockerhub-auth-workflow-boundary.test.ts b/test/e2e/support/dockerhub-auth-workflow-boundary.test.ts index a364403179..cc3ba9a4d0 100644 --- a/test/e2e/support/dockerhub-auth-workflow-boundary.test.ts +++ b/test/e2e/support/dockerhub-auth-workflow-boundary.test.ts @@ -16,7 +16,7 @@ import { } from "../../../tools/e2e/workflow-boundary.mts"; import { readWorkflow } from "../../helpers/e2e-workflow-contract"; -const NO_IMAGE_E2E_JOBS = ["staging-brev-launchable", "shared-e2e"] as const; +const NO_IMAGE_E2E_JOBS = ["podman-all-agents", "staging-brev-launchable", "shared-e2e"] as const; const AUTH_STEP_NAME = "Authenticate to Docker Hub"; const CLEANUP_STEP_NAME = "Clean up Docker auth"; const CLEANUP_HELPER_RUN = "bash .github/scripts/docker-auth-cleanup.sh"; diff --git a/test/e2e/support/e2e-phase-environment.test.ts b/test/e2e/support/e2e-phase-environment.test.ts index 043bde360d..7e697e1590 100644 --- a/test/e2e/support/e2e-phase-environment.test.ts +++ b/test/e2e/support/e2e-phase-environment.test.ts @@ -10,7 +10,7 @@ import { describe, expect, expectTypeOf, it } from "vitest"; import { ArtifactSink } from "../fixtures/artifacts.ts"; import { type CommandRunner, HostCliClient } from "../fixtures/clients/index.ts"; import type { E2ETargetFixtures } from "../fixtures/e2e-test.ts"; -import { type DockerRuntimeReady, EnvironmentPhaseFixture } from "../fixtures/phases/index.ts"; +import { type ContainerRuntimeReady, EnvironmentPhaseFixture } from "../fixtures/phases/index.ts"; import type { ShellProbeResult, ShellProbeRunOptions, @@ -89,11 +89,12 @@ describe("environment phase fixture", () => { runtime: "docker-running", onboarding: "cloud-openclaw", cliPath: "./bin/nemoclaw.js", - docker: { + containerRuntime: { id: "docker-running", + driverName: "docker", expectation: "required", available: true, - } satisfies Partial, + } satisfies Partial, }); expect(runner.calls).toEqual([ { @@ -143,8 +144,9 @@ describe("environment phase fixture", () => { onboarding: "cloud-openclaw-no-docker", }); - expect(ready.docker).toMatchObject({ + expect(ready.containerRuntime).toMatchObject({ id: "docker-missing", + driverName: "docker", expectation: "missing", available: false, }); @@ -162,8 +164,9 @@ describe("environment phase fixture", () => { onboarding: "cloud-openclaw-no-docker", }); - expect(ready.docker).toMatchObject({ + expect(ready.containerRuntime).toMatchObject({ id: "docker-missing", + driverName: "docker", expectation: "missing", available: true, }); @@ -181,8 +184,9 @@ describe("environment phase fixture", () => { runtime: "macos-docker-optional", }); - expect(ready.docker).toMatchObject({ + expect(ready.containerRuntime).toMatchObject({ id: "macos-docker-optional", + driverName: "docker", expectation: "optional", available: false, probeError: "spawn docker ENOENT", @@ -201,8 +205,9 @@ describe("environment phase fixture", () => { runtime: "macos-docker-optional", }); - expect(ready.docker).toMatchObject({ + expect(ready.containerRuntime).toMatchObject({ id: "macos-docker-optional", + driverName: "docker", expectation: "optional", available: true, }); @@ -286,8 +291,9 @@ describe("environment phase fixture", () => { runtime: "gpu-docker-cdi", }); - expect(ready.docker).toMatchObject({ + expect(ready.containerRuntime).toMatchObject({ id: "gpu-docker-cdi", + driverName: "docker", expectation: "required", available: true, }); @@ -312,8 +318,79 @@ describe("environment phase fixture", () => { runner.enqueue(shellResult(0, "nemoclaw v0.0.0\n")); await expect( - environment.assertReady({ ...cloudOpenClawEnvironment, runtime: "podman-running" }), - ).rejects.toThrow(/Unsupported target runtime 'podman-running'/); + environment.assertReady({ ...cloudOpenClawEnvironment, runtime: "mxc-running" }), + ).rejects.toThrow(/Unsupported target runtime 'mxc-running'/); + }); + + it("probes Podman through the exact native socket without Docker environment", async () => { + const previousSocket = process.env.OPENSHELL_PODMAN_SOCKET; + const previousDockerHost = process.env.DOCKER_HOST; + const previousDockerConfig = process.env.DOCKER_CONFIG; + process.env.OPENSHELL_PODMAN_SOCKET = "/run/user/1001/podman/podman.sock"; + process.env.DOCKER_HOST = "unix:///var/run/docker.sock"; + process.env.DOCKER_CONFIG = "/tmp/docker-config"; + try { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "nemoclaw v0.0.0\n")); + runner.enqueue(shellResult(0, '{"host":{"security":{"rootless":true}}}\n')); + const environment = new EnvironmentPhaseFixture(new HostCliClient(runner)); + + const ready = await environment.assertReady({ + ...cloudOpenClawEnvironment, + runtime: "podman-running", + }); + + expect(ready.containerRuntime).toMatchObject({ + id: "podman-running", + driverName: "podman", + expectation: "required", + available: true, + }); + expect(runner.calls[1]).toMatchObject({ + command: "podman", + args: ["--url", "unix:///run/user/1001/podman/podman.sock", "info", "--format", "json"], + options: { + artifactName: "runtime-podman-info-podman-running", + timeoutMs: 30_000, + }, + }); + expect(runner.calls[1]?.options?.env).toMatchObject({ + OPENSHELL_PODMAN_SOCKET: "/run/user/1001/podman/podman.sock", + }); + expect(runner.calls[1]?.options?.env).not.toHaveProperty("DOCKER_HOST"); + expect(runner.calls[1]?.options?.env).not.toHaveProperty("DOCKER_CONFIG"); + } finally { + if (previousSocket === undefined) delete process.env.OPENSHELL_PODMAN_SOCKET; + else process.env.OPENSHELL_PODMAN_SOCKET = previousSocket; + if (previousDockerHost === undefined) delete process.env.DOCKER_HOST; + else process.env.DOCKER_HOST = previousDockerHost; + if (previousDockerConfig === undefined) delete process.env.DOCKER_CONFIG; + else process.env.DOCKER_CONFIG = previousDockerConfig; + } + }); + + it("requires an explicit absolute Podman socket", async () => { + const previousSocket = process.env.OPENSHELL_PODMAN_SOCKET; + try { + delete process.env.OPENSHELL_PODMAN_SOCKET; + const missingRunner = new FakeRunner(); + missingRunner.enqueue(shellResult(0, "nemoclaw v0.0.0\n")); + const missing = new EnvironmentPhaseFixture(new HostCliClient(missingRunner)); + await expect( + missing.assertReady({ ...cloudOpenClawEnvironment, runtime: "podman-running" }), + ).rejects.toThrow(/requires OPENSHELL_PODMAN_SOCKET/); + + process.env.OPENSHELL_PODMAN_SOCKET = "relative/podman.sock"; + const relativeRunner = new FakeRunner(); + relativeRunner.enqueue(shellResult(0, "nemoclaw v0.0.0\n")); + const relative = new EnvironmentPhaseFixture(new HostCliClient(relativeRunner)); + await expect( + relative.assertReady({ ...cloudOpenClawEnvironment, runtime: "podman-running" }), + ).rejects.toThrow(/must be an absolute path/); + } finally { + if (previousSocket === undefined) delete process.env.OPENSHELL_PODMAN_SOCKET; + else process.env.OPENSHELL_PODMAN_SOCKET = previousSocket; + } }); it("writes an environment phase result artifact on success", async () => { diff --git a/test/e2e/support/e2e-phase-onboarding.test.ts b/test/e2e/support/e2e-phase-onboarding.test.ts index 8c245a6406..e12b748fcb 100644 --- a/test/e2e/support/e2e-phase-onboarding.test.ts +++ b/test/e2e/support/e2e-phase-onboarding.test.ts @@ -106,8 +106,9 @@ function ready(overrides: Partial = {}): EnvironmentReady { runtime: "docker-running", onboarding: "cloud-openclaw", cliPath: "nemoclaw", - docker: { + containerRuntime: { id: "docker-running", + driverName: "docker", expectation: "required", available: true, result: shellResult(0), @@ -189,6 +190,102 @@ describe("onboarding phase fixture", () => { }); }); + it("runs canonical Hermes onboarding through the shared runtime fixture", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "onboarded\n")); + const secrets = new FakeSecrets({ NVIDIA_INFERENCE_API_KEY: "secret-token" }); + const onboard = new OnboardingPhaseFixture(new HostCliClient(runner), secrets); + + const instance = await onboard.from(ready({ onboarding: "cloud-hermes" }), { + sandboxName: "e2e-ubuntu-repo-cloud-hermes", + }); + + expect(instance).toMatchObject({ + agent: "hermes", + sandboxName: "e2e-ubuntu-repo-cloud-hermes", + gatewayUrl: "http://127.0.0.1:8642", + }); + expect(runner.calls[0]).toMatchObject({ + command: "nemoclaw", + args: ["onboard", "--non-interactive", "--yes", "--yes-i-accept-third-party-software"], + options: { + artifactName: "onboard-cloud-hermes", + env: expect.objectContaining({ + NEMOCLAW_AGENT: "hermes", + NVIDIA_INFERENCE_API_KEY: "secret-token", + }), + redactionValues: ["secret-token"], + timeoutMs: 900_000, + }, + }); + }); + + it("selects native Podman and strips Docker discovery from onboard and cleanup", async () => { + const previousSocket = process.env.OPENSHELL_PODMAN_SOCKET; + const previousDockerHost = process.env.DOCKER_HOST; + const previousDockerConfig = process.env.DOCKER_CONFIG; + process.env.OPENSHELL_PODMAN_SOCKET = "/run/user/1001/podman/podman.sock"; + process.env.DOCKER_HOST = "unix:///var/run/docker.sock"; + process.env.DOCKER_CONFIG = "/tmp/docker-config"; + try { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "onboarded\n")); + const cleanup = new FakeCleanup(); + const onboard = new OnboardingPhaseFixture( + new HostCliClient(runner), + new FakeSecrets({ NVIDIA_INFERENCE_API_KEY: "secret-token" }), + cleanup, + ); + const podmanReady = ready({ + runtime: "podman-running", + containerRuntime: { + id: "podman-running", + driverName: "podman", + expectation: "required", + available: true, + result: shellResult(0), + }, + }); + + await onboard.from(podmanReady, { sandboxName: "e2e-podman-openclaw" }); + + expect(runner.calls[0]).toMatchObject({ + command: "nemoclaw", + args: [ + "onboard", + "--non-interactive", + "--yes", + "--yes-i-accept-third-party-software", + "--compute-driver", + "podman", + ], + options: { + env: expect.objectContaining({ + OPENSHELL_PODMAN_SOCKET: "/run/user/1001/podman/podman.sock", + }), + }, + }); + expect(runner.calls[0]?.options?.env).not.toHaveProperty("DOCKER_HOST"); + expect(runner.calls[0]?.options?.env).not.toHaveProperty("DOCKER_CONFIG"); + + runner.enqueue(shellResult(0, "destroyed\n")); + await cleanup.calls[0]?.run(); + expect(runner.calls[1]).toMatchObject({ + command: "nemoclaw", + args: ["e2e-podman-openclaw", "destroy", "--yes"], + }); + expect(runner.calls[1]?.options?.env).not.toHaveProperty("DOCKER_HOST"); + expect(runner.calls[1]?.options?.env).not.toHaveProperty("DOCKER_CONFIG"); + } finally { + if (previousSocket === undefined) delete process.env.OPENSHELL_PODMAN_SOCKET; + else process.env.OPENSHELL_PODMAN_SOCKET = previousSocket; + if (previousDockerHost === undefined) delete process.env.DOCKER_HOST; + else process.env.DOCKER_HOST = previousDockerHost; + if (previousDockerConfig === undefined) delete process.env.DOCKER_CONFIG; + else process.env.DOCKER_CONFIG = previousDockerConfig; + } + }); + it("fails cloud OpenClaw onboarding on non-zero exit", async () => { const runner = new FakeRunner(); runner.enqueue(shellResult(42, "provider rejected credential")); @@ -240,7 +337,7 @@ describe("onboarding phase fixture", () => { expect(runner.calls).toEqual([]); }); - it("requires Docker for cloud OpenClaw onboarding", async () => { + it("requires the selected runtime for cloud OpenClaw onboarding", async () => { const onboard = new OnboardingPhaseFixture( new HostCliClient(new FakeRunner()), new FakeSecrets({ NVIDIA_INFERENCE_API_KEY: "secret" }), @@ -249,10 +346,15 @@ describe("onboarding phase fixture", () => { await expect( onboard.from( ready({ - docker: { id: "docker-running", expectation: "required", available: false }, + containerRuntime: { + id: "docker-running", + driverName: "docker", + expectation: "required", + available: false, + }, }), ), - ).rejects.toThrow(/requires an available Docker runtime/); + ).rejects.toThrow(/requires an available docker runtime/); }); it("rejects invalid sandbox names before cloud OpenClaw side effects", async () => { @@ -310,7 +412,12 @@ describe("onboarding phase fixture", () => { ready({ runtime: "docker-missing", onboarding: "cloud-openclaw-no-docker", - docker: { id: "docker-missing", expectation: "missing", available: true }, + containerRuntime: { + id: "docker-missing", + driverName: "docker", + expectation: "missing", + available: true, + }, }), { sandboxName: "e2e-no-docker" }, ); @@ -432,7 +539,12 @@ describe("onboarding phase fixture", () => { ready({ runtime: "docker-missing", onboarding: "cloud-openclaw-no-docker", - docker: { id: "docker-missing", expectation: "missing", available: true }, + containerRuntime: { + id: "docker-missing", + driverName: "docker", + expectation: "missing", + available: true, + }, }), ); @@ -462,7 +574,12 @@ describe("onboarding phase fixture", () => { ready({ runtime: "docker-missing", onboarding: "cloud-openclaw-no-docker", - docker: { id: "docker-missing", expectation: "missing", available: false }, + containerRuntime: { + id: "docker-missing", + driverName: "docker", + expectation: "missing", + available: false, + }, }), ); @@ -502,7 +619,12 @@ describe("onboarding phase fixture", () => { ready({ runtime: "docker-missing", onboarding: "cloud-openclaw-no-docker", - docker: { id: "docker-missing", expectation: "missing", available: false }, + containerRuntime: { + id: "docker-missing", + driverName: "docker", + expectation: "missing", + available: false, + }, }), ); @@ -538,7 +660,12 @@ describe("onboarding phase fixture", () => { ready({ runtime: "docker-missing", onboarding: "cloud-openclaw-no-docker", - docker: { id: "docker-missing", expectation: "missing", available: false }, + containerRuntime: { + id: "docker-missing", + driverName: "docker", + expectation: "missing", + available: false, + }, }), { sandboxName: "e2e-no-docker-success" }, ), @@ -571,7 +698,12 @@ describe("onboarding phase fixture", () => { ready({ runtime: "docker-missing", onboarding: "cloud-openclaw-no-docker", - docker: { id: "docker-missing", expectation: "missing", available: false }, + containerRuntime: { + id: "docker-missing", + driverName: "docker", + expectation: "missing", + available: false, + }, }), ), ).rejects.toThrow(/without Docker-missing preflight signature/); @@ -583,8 +715,8 @@ describe("onboarding phase fixture", () => { new FakeSecrets(), ); - await expect(onboard.from(ready({ onboarding: "cloud-hermes" }))).rejects.toThrow( - /Unsupported onboarding profile 'cloud-hermes'/, + await expect(onboard.from(ready({ onboarding: "cloud-unknown" }))).rejects.toThrow( + /Unsupported onboarding profile 'cloud-unknown'/, ); }); @@ -629,16 +761,21 @@ describe("onboarding phase fixture", () => { await expect( onboard.from( ready({ - docker: { id: "docker-running", expectation: "required", available: false }, + containerRuntime: { + id: "docker-running", + driverName: "docker", + expectation: "required", + available: false, + }, }), ), - ).rejects.toThrow(/requires an available Docker runtime/); + ).rejects.toThrow(/requires an available docker runtime/); expect(readJson(path.join(tmp, "onboarding.result.json"))).toMatchObject({ phase: "onboarding", status: "failed", onboarding: "cloud-openclaw", - error: "cloud-openclaw onboarding requires an available Docker runtime.", + error: "cloud-openclaw onboarding requires an available docker runtime.", }); } finally { fs.rmSync(tmp, { recursive: true, force: true }); diff --git a/test/e2e/support/podman-all-agents-workflow-boundary.test.ts b/test/e2e/support/podman-all-agents-workflow-boundary.test.ts new file mode 100644 index 0000000000..49649cedd8 --- /dev/null +++ b/test/e2e/support/podman-all-agents-workflow-boundary.test.ts @@ -0,0 +1,199 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { readFileSync } from "node:fs"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { readWorkflow } from "../../helpers/e2e-workflow-contract"; + +type WorkflowStep = { + env?: Record; + if?: string; + name?: string; + run?: string; + uses?: string; +}; + +type WorkflowJob = { + env?: Record; + if?: string; + needs?: string; + "runs-on"?: string; + steps?: WorkflowStep[]; + strategy?: { matrix?: { agent?: string[] } }; + "timeout-minutes"?: number; +}; + +type Workflow = { + on: { + workflow_dispatch: { + inputs: Record; + }; + }; + jobs: Record; +}; + +function workflow(): Workflow { + return readWorkflow() as Workflow; +} + +function namedStep(job: WorkflowJob, name: string): WorkflowStep { + const step = job.steps?.find((candidate) => candidate.name === name); + if (!step) throw new Error(`missing Podman workflow step '${name}'`); + return step; +} + +function liveTestSource(): string { + return readFileSync( + path.join(import.meta.dirname, "..", "live", "podman-all-agents.test.ts"), + "utf8", + ); +} + +describe("native Podman all-agent workflow boundary", () => { + it("runs all supported agents on the native rootless Podman runner contract", () => { + const parsed = workflow(); + const job = parsed.jobs["podman-all-agents"]; + + expect(job).toBeDefined(); + expect(job).toMatchObject({ + needs: "generate-matrix", + "runs-on": "ubuntu-26.04", + "timeout-minutes": 90, + strategy: { + matrix: { + agent: ["openclaw", "hermes", "langchain-deepagents-code"], + }, + }, + env: { + E2E_JOB: "1", + E2E_DEFAULT_ENABLED: "0", + E2E_TARGET_ID: "podman-all-agents", + NEMOCLAW_SANDBOX_GPU: "0", + }, + }); + expect(job?.if).toContain("inputs.jobs"); + expect(job?.if).toContain("inputs.targets"); + expect(job?.if).toContain("podman-all-agents"); + expect(liveTestSource()).toContain("timeout: 75 * 60_000"); + + const start = namedStep(job!, "Start exact rootless Podman API socket").run ?? ""; + expect(start).toContain('socket_path="$runtime_dir/podman/podman.sock"'); + expect(start).toContain('podman system service --time=0 "unix://$socket_path"'); + expect(start).toContain('podman --url "unix://$socket_path" info --format json'); + expect(start).toContain("(.host.security.rootless // .Host.Security.Rootless) == true"); + expect(start).toContain("OPENSHELL_PODMAN_SOCKET"); + expect(start).toContain("OPENSHELL_PODMAN_NETWORK_NAME"); + }); + + it("gates on a complete published three-agent managed-image release and records digests", () => { + const parsed = workflow(); + const input = parsed.on.workflow_dispatch.inputs.podman_managed_image_release; + const job = parsed.jobs["podman-all-agents"]!; + const catalog = namedStep(job, "Resolve complete published managed-image release"); + const run = catalog.run ?? ""; + + expect(input).toMatchObject({ default: "", type: "string" }); + expect(input.description?.toLowerCase()).toContain("published complete managed-image release"); + expect(input.description?.toLowerCase()).toContain("fails visibly"); + expect(catalog.env).toEqual({ + REQUESTED_RELEASE: "${{ inputs.podman_managed_image_release }}", + }); + expect(run).toContain('status:"gated"'); + expect(run).toContain("reporting this lane as not passed"); + expect(run).toMatch(/if \[ -z "\$release" \]; then[\s\S]*?exit 1/u); + expect(run).toContain("skopeo inspect --override-os linux --override-arch amd64"); + expect(run).toContain("openclaw-sandbox"); + expect(run).toContain("hermes-sandbox"); + expect(run).toContain("langchain-deepagents-code-sandbox"); + expect(run).toContain("cohort-$source_cohort"); + expect(run).toContain("org.opencontainers.image.revision"); + expect(run).toContain('reference: ($image + "@" + $digest)'); + expect(run).toContain("git merge-base --is-ancestor"); + + const bind = namedStep(job, "Bind the candidate CLI to the verified release identity"); + expect(bind.run).toContain('git tag --force "$MANAGED_IMAGE_RELEASE" HEAD'); + expect(bind.run).not.toContain("git push"); + }); + + it("installs a fail-closed Docker guard and runs only the native Podman live test", () => { + const parsed = workflow(); + const job = parsed.jobs["podman-all-agents"]!; + const names = job.steps?.map((step) => step.name).filter(Boolean); + const scripts = job.steps?.map((step) => step.run ?? "").join("\n") ?? ""; + + expect(names).not.toContain("Authenticate to Docker Hub"); + expect(names).not.toContain("Clean up Docker auth"); + expect(names?.indexOf("Install Docker invocation guard")).toBeLessThan( + names?.indexOf("Disable the Docker daemon for native Podman proof") ?? -1, + ); + const disableDocker = namedStep(job, "Disable the Docker daemon for native Podman proof").run; + expect(disableDocker).toContain("systemctl stop docker.service docker.socket"); + expect(disableDocker).toContain("systemctl mask --runtime docker.service docker.socket"); + expect(disableDocker).toContain("[ -S /var/run/docker.sock ]"); + expect(disableDocker).toContain('docker_candidate="$(command -v docker || true)"'); + expect(disableDocker).toContain('"$docker_candidate" != "$E2E_DOCKER_GUARD_BIN"'); + expect(disableDocker).toContain("docker-absence-boundary.json"); + expect(disableDocker).not.toMatch(/\bdocker\s+info\b/u); + expect(namedStep(job, "Install Docker invocation guard").run).toContain("exit 97"); + expect(namedStep(job, "Install Docker invocation guard").run).toContain("DOCKER_HOST="); + expect(namedStep(job, "Verify the lane never invoked Docker").run).toContain( + 'if [ -s "$E2E_DOCKER_GUARD_LOG" ]', + ); + expect(scripts).not.toMatch(/\bdocker\s+(?:login|pull|build|run|info)\b/u); + expect(scripts).not.toContain("podman-docker"); + + const live = namedStep(job, "Run native Podman all-agent live test"); + expect(live.env).toEqual({ + NVIDIA_INFERENCE_API_KEY: "${{ secrets.NVIDIA_INFERENCE_API_KEY }}", + }); + expect(live.run).toContain( + "tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/podman-all-agents.test.ts", + ); + + const reportNeeds = parsed.jobs["report-to-pr"]?.needs; + expect(Array.isArray(reportNeeds) ? reportNeeds : []).toContain("podman-all-agents"); + }); + + it("proves the actual digest-bound Podman container across stop/start, recovery, rebuild, and cleanup", () => { + const source = liveTestSource(); + + expect(source).toContain('"openshell.sandbox-name"'); + expect(source).toContain('"openshell.managed"'); + expect(source).toContain("expect(imageName).toBe(options.catalog.reference)"); + expect(source).toContain("expect(imageDigest).toBe(options.catalog.digest)"); + expect(source).toContain("expect(imageRepoDigests).toContain(options.catalog.reference)"); + expect(source).toContain("OPENSHELL_SANDBOX_COMMAND="); + expect(source).toContain("/usr/local/bin/nemoclaw-managed-startup-hold"); + expect(source).toContain("managed-startup-complete.json"); + expect(source).toContain("profileFingerprint: options.profileFingerprint"); + expect(source).toContain('expect(limits.get("nproc")).toEqual({ hard: 512, soft: 512 })'); + expect(source).toContain( + 'expect(limits.get("nofile")).toEqual({ hard: 65_536, soft: 65_536 })', + ); + expect(source).toContain('"512:512:65536:65536"'); + expect(source).toContain("nemoclaw-backup-"); + expect(source).toContain('"io.nvidia.nemoclaw.managed-image.cohort"'); + expect(source).toContain("runtimeBindingBeforeRecovery"); + expect(source).toContain("socket_path: socketPath"); + expect(source).toContain("network_name: networkName"); + expect(source).toContain('host.nemoclaw([sandboxName, "stop"]'); + expect(source).toContain("expectedRunning: false"); + expect(source).toContain("expect(stoppedContainer).toEqual(initialContainer)"); + expect(source).toContain('host.nemoclaw([sandboxName, "start"]'); + expect(source).toContain("restartedContainer"); + expect(source).toContain("stoppedAndStarted: true"); + expect(source).toContain("host.expectStatus(sandboxName"); + expect(source).not.toMatch(/"--resume"[\s\S]{0,180}"--compute-driver"[\s\S]{0,40}"podman"/u); + expect(source).toContain("rebuild must replace the managed Podman sandbox container"); + expect(source).toContain('"snapshot", "create", "--name", "podman-runtime"'); + expect(source).toContain("snapshotCloneRestored: true"); + expect(source).toContain( + "snapshot clone cutover must resume a new exact standalone gateway process", + ); + expect(source).toContain('"network", "exists", networkName'); + expect(source).toContain("networkAfterCleanup.exitCode"); + }); +}); diff --git a/test/entrypoint-env-wrapper.test.ts b/test/entrypoint-env-wrapper.test.ts new file mode 100644 index 0000000000..a5fc918dea --- /dev/null +++ b/test/entrypoint-env-wrapper.test.ts @@ -0,0 +1,212 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; +import { sliceBlock } from "./helpers/corporate-ca-support"; + +const HELPER = path.join(import.meta.dirname, "..", "scripts", "lib", "entrypoint-env-wrapper.sh"); +const OPENCLAW_START = path.join(import.meta.dirname, "..", "scripts", "nemoclaw-start.sh"); + +function runNormalizer(argv: readonly string[]) { + const harness = [ + "set -euo pipefail", + 'helper="$1"', + "shift", + 'source "$helper"', + 'nemoclaw_normalize_entrypoint_env_wrapper "$@"', + 'if [ "$NEMOCLAW_ENTRYPOINT_NORMALIZED_ARGC" -eq 0 ]; then', + " set --", + "else", + ' set -- "${NEMOCLAW_ENTRYPOINT_NORMALIZED_ARGV[@]}"', + "fi", + `printf 'UID=%s\\n' "$(id -u)"`, + "printf 'PROFILE=%s\\n' \"${NEMOCLAW_STARTUP_PROFILE_B64-__UNSET__}\"", + "printf 'CA=%s\\n' \"${NEMOCLAW_CORPORATE_CA_B64-__UNSET__}\"", + "printf 'HTTP_PROXY=%s\\n' \"${HTTP_PROXY-__UNSET__}\"", + "printf 'NO_PROXY=%s\\n' \"${NO_PROXY-__UNSET__}\"", + `printf 'ARG=%s\\n' "$@"`, + ].join("\n"); + return spawnSync("/bin/bash", ["-c", harness, "entrypoint-env-wrapper-test", HELPER, ...argv], { + encoding: "utf8", + env: { + PATH: "/usr/local/bin:/usr/bin:/bin", + }, + }); +} + +describe("OCI entrypoint env-wrapper normalization", () => { + it("promotes the exact managed handoff before preserving the command tail", () => { + const uid = String(process.getuid?.() ?? ""); + const result = runNormalizer([ + "env", + "NEMOCLAW_STARTUP_PROFILE_B64=eyJzY2hlbWFWZXJzaW9uIjoxfQ", + "NEMOCLAW_CORPORATE_CA_B64=Y2E=", + "HTTP_PROXY=http://user:pass@proxy.example.test:18080", + "NO_PROXY=localhost,127.0.0.1", + "nemoclaw-start", + "/bin/sh", + "-c", + "printf managed command", + ]); + + expect(result.status).toBe(0); + expect(result.stdout).toContain(`UID=${uid}`); + expect(result.stdout).toContain("PROFILE=eyJzY2hlbWFWZXJzaW9uIjoxfQ"); + expect(result.stdout).toContain("CA=Y2E="); + expect(result.stdout).toContain("HTTP_PROXY=http://user:pass@proxy.example.test:18080"); + expect(result.stdout).toContain("NO_PROXY=localhost,127.0.0.1"); + expect(result.stdout).toContain("ARG=/bin/sh\nARG=-c\nARG=printf managed command\n"); + }); + + it("strips a direct self invocation without interpreting its command arguments", () => { + const result = runNormalizer(["/usr/local/bin/nemoclaw-start", "env", "FOO=bar", "printenv"]); + + expect(result.status).toBe(0); + expect(result.stdout).toContain("PROFILE=__UNSET__"); + expect(result.stdout).toContain("ARG=env\nARG=FOO=bar\nARG=printenv\n"); + }); + + it("leaves an unrelated explicit env command untouched", () => { + const result = runNormalizer(["env", "FOO=bar", "printenv", "FOO"]); + + expect(result.status).toBe(0); + expect(result.stdout).toContain("ARG=env\nARG=FOO=bar\nARG=printenv\nARG=FOO\n"); + }); + + it.each([ + { + argv: ["env", "NODE_OPTIONS=--require=/sandbox/untrusted.cjs", "nemoclaw-start"], + message: "unsupported variable 'NODE_OPTIONS'", + }, + { + argv: [ + "env", + "NEMOCLAW_STARTUP_PROFILE_B64=first", + "NEMOCLAW_STARTUP_PROFILE_B64=second", + "nemoclaw-start", + ], + message: "repeats variable 'NEMOCLAW_STARTUP_PROFILE_B64'", + }, + { + argv: ["env", "NEMOCLAW_STARTUP_PROFILE_B64=profile", "/usr/bin/true"], + message: "Malformed managed startup env wrapper", + }, + { + argv: ["env", "NEMOCLAW_CORPORATE_CA_B64=Y2E=", "not-an-assignment", "nemoclaw-start"], + message: "Malformed managed startup env wrapper", + }, + ])("fails closed for malformed or unsafe root handoff: $message", ({ argv, message }) => { + const result = runNormalizer(argv); + + expect(result.status).toBe(1); + expect(result.stderr).toContain(message); + expect(result.stdout).toBe(""); + }); + + it("unwraps the sandbox-create env self-wrapper and applies dashboard port defaults", () => { + const normalizer = fs.readFileSync( + path.join(import.meta.dirname, "..", "scripts", "lib", "entrypoint-env-wrapper.sh"), + "utf-8", + ); + const openClawPortBlock = sliceBlock( + OPENCLAW_START, + 'NEMOCLAW_CMD=("$@")', + "# ── Config integrity check", + ); + const snippet = [ + normalizer, + 'nemoclaw_normalize_entrypoint_env_wrapper "$@"', + 'if [ "$NEMOCLAW_ENTRYPOINT_NORMALIZED_ARGC" -eq 0 ]; then set --; ' + + 'else set -- "${NEMOCLAW_ENTRYPOINT_NORMALIZED_ARGV[@]}"; fi', + openClawPortBlock, + ].join("\n"); + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-env-wrapper-")); + const fakeBin = path.join(tmpDir, "bin"); + const scriptPath = path.join(tmpDir, "run.sh"); + + function runScenario(setArgs: string, extraEnv: Record = {}) { + const script = [ + "#!/usr/bin/env bash", + "set -euo pipefail", + setArgs, + snippet, + 'printf "CHAT_UI_URL=%s\\n" "$CHAT_UI_URL"', + 'printf "PUBLIC_PORT=%s\\n" "$PUBLIC_PORT"', + 'printf "OPENCLAW_GATEWAY_PORT=%s\\n" "$OPENCLAW_GATEWAY_PORT"', + 'printf "OPENCLAW_GATEWAY_URL=%s\\n" "$OPENCLAW_GATEWAY_URL"', + 'printf "SANDBOX_HOME=%s\\n" "$_SANDBOX_HOME"', + 'printf "OPENCLAW_HOME=%s\\n" "$OPENCLAW_HOME"', + 'printf "OPENCLAW_STATE_DIR=%s\\n" "$OPENCLAW_STATE_DIR"', + 'printf "OPENCLAW_CONFIG_PATH=%s\\n" "$OPENCLAW_CONFIG_PATH"', + 'printf "OPENCLAW_OAUTH_DIR=%s\\n" "$OPENCLAW_OAUTH_DIR"', + 'printf "CMD=%s\\n" "${NEMOCLAW_CMD[*]}"', + ].join("\n"); + fs.writeFileSync(scriptPath, script, { mode: 0o700 }); + return spawnSync("bash", [scriptPath], { + encoding: "utf-8", + timeout: 5000, + env: { ...process.env, PATH: `${fakeBin}:${process.env.PATH || ""}`, ...extraEnv }, + }); + } + + try { + fs.mkdirSync(fakeBin); + fs.writeFileSync(path.join(fakeBin, "openclaw"), "#!/usr/bin/env bash\nexit 0\n", { + mode: 0o755, + }); + + const injected = runScenario( + "set -- env CHAT_UI_URL=https://chat.example.test NEMOCLAW_DASHBOARD_PORT=19000 nemoclaw-start openclaw agent --agent main", + ); + expect(injected.status).toBe(0); + expect(injected.stdout).toContain("CHAT_UI_URL=http://127.0.0.1:19000"); + expect(injected.stdout).toContain("PUBLIC_PORT=19000"); + expect(injected.stdout).toContain("OPENCLAW_GATEWAY_PORT=19000"); + expect(injected.stdout).toContain("OPENCLAW_GATEWAY_URL=ws://127.0.0.1:19000"); + expect(injected.stdout).toContain("SANDBOX_HOME=/sandbox"); + expect(injected.stdout).toContain("OPENCLAW_HOME=/sandbox"); + expect(injected.stdout).toContain("OPENCLAW_STATE_DIR=/sandbox/.openclaw"); + expect(injected.stdout).toContain("OPENCLAW_CONFIG_PATH=/sandbox/.openclaw/openclaw.json"); + expect(injected.stdout).toContain("OPENCLAW_OAUTH_DIR=/sandbox/.openclaw/credentials"); + expect(injected.stdout).toContain("CMD=openclaw agent --agent main"); + + const bakedCustomPort = runScenario("set -- nemoclaw-start openclaw agent", { + CHAT_UI_URL: "http://127.0.0.1:18790", + }); + expect(bakedCustomPort.status).toBe(0); + expect(bakedCustomPort.stdout).toContain("CHAT_UI_URL=http://127.0.0.1:18790"); + expect(bakedCustomPort.stdout).toContain("PUBLIC_PORT=18790"); + expect(bakedCustomPort.stdout).toContain("OPENCLAW_GATEWAY_PORT=18790"); + expect(bakedCustomPort.stdout).toContain("OPENCLAW_GATEWAY_URL=ws://127.0.0.1:18790"); + expect(bakedCustomPort.stdout).toContain("OPENCLAW_STATE_DIR=/sandbox/.openclaw"); + expect(bakedCustomPort.stdout).toContain("OPENCLAW_OAUTH_DIR=/sandbox/.openclaw/credentials"); + expect(bakedCustomPort.stdout).toContain("CMD=openclaw agent"); + + const baked = runScenario("set -- nemoclaw-start openclaw agent", { + CHAT_UI_URL: "https://baked.example.test/ui", + }); + expect(baked.status).toBe(0); + expect(baked.stdout).toContain("CHAT_UI_URL=https://baked.example.test/ui"); + expect(baked.stdout).toContain("PUBLIC_PORT=18789"); + expect(baked.stdout).toContain("OPENCLAW_GATEWAY_PORT=18789"); + expect(baked.stdout).toContain("OPENCLAW_GATEWAY_URL=ws://127.0.0.1:18789"); + expect(baked.stdout).toContain("SANDBOX_HOME=/sandbox"); + expect(baked.stdout).toContain("OPENCLAW_STATE_DIR=/sandbox/.openclaw"); + expect(baked.stdout).toContain("CMD=openclaw agent"); + + const invalidHighPort = runScenario("set -- nemoclaw-start openclaw agent", { + NEMOCLAW_DASHBOARD_PORT: "70000", + }); + expect(invalidHighPort.status).toBe(1); + expect(invalidHighPort.stderr).toContain("Invalid NEMOCLAW_DASHBOARD_PORT='70000'"); + expect(invalidHighPort.stderr).toContain("must be an integer between 1024 and 65535"); + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }); +}); diff --git a/test/generate-hermes-config.test.ts b/test/generate-hermes-config.test.ts index 949e59ad6e..93c9cc308a 100644 --- a/test/generate-hermes-config.test.ts +++ b/test/generate-hermes-config.test.ts @@ -997,6 +997,27 @@ describe("agents/hermes/generate-config.ts", () => { expect(Object.keys(config.platforms)).toEqual(["api_server"]); }); + it("keeps every managed-image messaging platform explicitly disabled before first start (#7744)", () => { + const { config, envFile } = generateBaseConfig({ + NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION: "1", + }); + + for (const platform of ["telegram", "discord", "weixin", "slack", "whatsapp", "teams"]) { + expect(config.platforms[platform], platform).toEqual({ enabled: false }); + expect(config.platform_toolsets[platform], platform).toBeUndefined(); + } + for (const credential of [ + "TELEGRAM_BOT_TOKEN", + "DISCORD_BOT_TOKEN", + "WEIXIN_TOKEN", + "SLACK_BOT_TOKEN", + "WHATSAPP_ENABLED", + "TEAMS_CLIENT_SECRET", + ]) { + expect(envFile, credential).not.toContain(`${credential}=`); + } + }); + it("enables Slack under platforms even when the slack token allowlist is empty", async () => { const { config } = await runConfigScriptWithMessaging({ NEMOCLAW_MESSAGING_CHANNELS_B64: encodeJson(["slack"]), diff --git a/test/generate-openclaw-config-plugin-entries.test.ts b/test/generate-openclaw-config-plugin-entries.test.ts index bfa654ef0d..464855f87b 100644 --- a/test/generate-openclaw-config-plugin-entries.test.ts +++ b/test/generate-openclaw-config-plugin-entries.test.ts @@ -6,9 +6,12 @@ // scripts/generate-openclaw-config.mts. Split out of generate-openclaw-config // .test.ts to keep that file within its size budget. +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; import { describe, expect, it } from "vitest"; -import { buildConfig } from "../scripts/generate-openclaw-config.mts"; +import { buildConfig, main } from "../scripts/generate-openclaw-config.mts"; const BASE_ENV: Record = { NEMOCLAW_MODEL: "test-model", @@ -39,4 +42,70 @@ describe("generate-openclaw-config.mts: default plugin entries", () => { const config = buildConfig({ ...BASE_ENV }); expect(config.plugins.entries.qqbot).toBeUndefined(); }); + + it("keeps every managed-image plugin and channel explicitly inert before first start (#7744)", () => { + const config = buildConfig({ + ...BASE_ENV, + NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION: "1", + }); + + for (const pluginId of [ + "telegram", + "discord", + "openclaw-weixin", + "slack", + "whatsapp", + "msteams", + "diagnostics-otel", + "brave", + "tavily", + ]) { + expect(config.plugins.entries[pluginId], pluginId).toEqual({ enabled: false }); + } + for (const channelId of [ + "telegram", + "discord", + "openclaw-weixin", + "slack", + "whatsapp", + "msteams", + ]) { + expect(config.channels[channelId], channelId).toEqual({ enabled: false }); + } + expect(config.tools.web.search).toEqual({ enabled: false }); + }); + + it("retains managed-image install metadata while explicitly disabling the plugin (#7744)", () => { + const tempDirectory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-managed-union-")); + const originalEnvironment = { ...process.env }; + const configPath = path.join(tempDirectory, ".openclaw", "openclaw.json"); + const installEntry = { + source: "npm", + spec: "@tencent-weixin/openclaw-weixin@2.4.3", + installPath: "/sandbox/.openclaw/extensions/openclaw-weixin", + }; + try { + fs.mkdirSync(path.dirname(configPath), { recursive: true }); + fs.writeFileSync( + configPath, + JSON.stringify({ plugins: { installs: { "openclaw-weixin": installEntry } } }), + ); + for (const name of Object.keys(process.env)) delete process.env[name]; + Object.assign(process.env, BASE_ENV, { + HOME: tempDirectory, + NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION: "1", + }); + + main(); + + const config = JSON.parse(fs.readFileSync(configPath, "utf8")); + expect(config.plugins?.installs?.["openclaw-weixin"]).toEqual(installEntry); + expect(config.plugins?.entries?.["openclaw-weixin"]).toEqual({ enabled: false }); + expect(config.channels?.["openclaw-weixin"]).toEqual({ enabled: false }); + } finally { + for (const name of Object.keys(process.env)) delete process.env[name]; + Object.assign(process.env, originalEnvironment); + fs.rmSync(tempDirectory, { force: true, recursive: true }); + } + }); }); diff --git a/test/generate-platform-docs.test.ts b/test/generate-platform-docs.test.ts index 3fc5164e8f..8bfceb63be 100644 --- a/test/generate-platform-docs.test.ts +++ b/test/generate-platform-docs.test.ts @@ -437,6 +437,56 @@ print(block) expect(langchainRow.notes).not.toMatch(/Only OpenClaw and Hermes are integrated\.?\s*$/); }); + it("publishes the qualified Podman contract for every shipped agent without weakening Docker defaults", () => { + const matrixPath = path.join(import.meta.dirname, "..", "ci", "platform-matrix.json"); + const matrix = JSON.parse(readFileSync(matrixPath, "utf-8")); + const podmanPlatform = (matrix.platforms ?? []).find((row: { runtimes?: string[] }) => + row.runtimes?.some((runtime) => runtime.toLowerCase().includes("podman")), + ); + + expect(podmanPlatform).toMatchObject({ + status: "caveated", + ci_tested: false, + }); + const claim = `${podmanPlatform.name} ${podmanPlatform.runtimes.join(" ")} ${ + podmanPlatform.notes + }`; + for (const required of [ + "OpenClaw", + "Hermes", + "LangChain Deep Agents Code", + "Linux amd64", + "rootless Podman 5", + "cgroup v2", + "subordinate UID and GID", + "--compute-driver podman", + "NEMOCLAW_COMPUTE_DRIVER=podman", + "immutable managed OCI image", + "Docker remains the default", + "before this row may claim dedicated CI qualification", + ]) { + expect(claim).toContain(required); + } + expect(claim).toMatch(/CPU sandboxes/i); + expect(claim).toMatch(/does not fall back to .*Docker daemon/i); + expect( + (matrix.out_of_scope ?? []).some((row: { name: string }) => /podman/i.test(row.name)), + ).toBe(false); + + const prerequisites = readFileSync( + path.join(import.meta.dirname, "..", "docs", "get-started", "prerequisites.mdx"), + "utf8", + ); + const troubleshooting = readFileSync( + path.join(import.meta.dirname, "..", "docs", "reference", "troubleshooting.mdx"), + "utf8", + ); + expect(prerequisites).not.toContain("NemoClaw needs Docker access."); + expect(prerequisites).toContain("The default Docker driver needs Docker access"); + expect(troubleshooting).toContain("When you use the default Docker driver"); + expect(troubleshooting).toContain("The rootless Podman path instead"); + }); + it("every `path:line` citation embedded in matrix notes resolves to a non-empty line in the repo", () => { const repoRoot = path.join(import.meta.dirname, ".."); const matrix = JSON.parse( diff --git a/test/helpers/destroy-flow-test-harness.ts b/test/helpers/destroy-flow-test-harness.ts index 1d16cc11b0..ef2648abeb 100644 --- a/test/helpers/destroy-flow-test-harness.ts +++ b/test/helpers/destroy-flow-test-harness.ts @@ -50,6 +50,7 @@ type DestroyHarnessOptions = { liveListOutput?: string; mcpAddState?: "prepared"; mcpServers?: string[]; + openshellDriver?: string | null; promptResponses?: string[]; registeredSandboxCount?: number; restoreMcpError?: string; @@ -137,6 +138,7 @@ export function createDestroyHarness(options: DestroyHarnessOptions = {}): Destr vi.spyOn(registry, "getSandbox").mockReturnValue({ ...sandboxEntry, agent: options.agent ?? sandboxEntry.agent, + ...(options.openshellDriver === undefined ? {} : { openshellDriver: options.openshellDriver }), ...(options.endpointUrl ? { endpointUrl: options.endpointUrl } : {}), ...(options.mcpServers?.length ? { diff --git a/test/helpers/langchain-deepagents-code-headless.ts b/test/helpers/langchain-deepagents-code-headless.ts index 4c1b4428f7..0fa34542eb 100644 --- a/test/helpers/langchain-deepagents-code-headless.ts +++ b/test/helpers/langchain-deepagents-code-headless.ts @@ -51,6 +51,7 @@ export function makeStartScriptFixture( const envFile = path.join(tempDir, "proxy-env.sh"); const scriptPath = path.join(tempDir, "start.sh"); const rlimitLib = path.join(tempDir, "sandbox-rlimits.sh"); + const entrypointEnvWrapper = path.join(repoRoot, "scripts", "lib", "entrypoint-env-wrapper.sh"); const hostFile = path.join(tempDir, "trusted-proxy-host"); const portFile = path.join(tempDir, "trusted-proxy-port"); const caFile = path.join(tempDir, "trusted-ca-bundle.pem"); @@ -59,7 +60,9 @@ export function makeStartScriptFixture( expect(original).toContain('tmp="$(mktemp /tmp/nemoclaw-proxy-env.XXXXXX)"'); expect(original).toContain("local marker_dir=/sandbox/.deepagents"); const fixture = original + .replace("/usr/local/lib/nemoclaw/entrypoint-env-wrapper.sh", entrypointEnvWrapper) .replace("/usr/local/lib/nemoclaw/sandbox-rlimits.sh", rlimitLib) + .replaceAll("/run/nemoclaw/managed-startup-ca-bundle.pem", caFile) .replace( 'readonly MANAGED_PROXY_HOST_FILE="/usr/local/share/nemoclaw/dcode-proxy-host"', `readonly MANAGED_PROXY_HOST_FILE="${hostFile}"`, diff --git a/test/helpers/managed-image-buildless-e2e.ts b/test/helpers/managed-image-buildless-e2e.ts new file mode 100644 index 0000000000..abd7327eb6 --- /dev/null +++ b/test/helpers/managed-image-buildless-e2e.ts @@ -0,0 +1,654 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { expect } from "vitest"; + +import { + MANAGED_IMAGE_CAPABILITY_CONTRACT_VERSION, + MANAGED_IMAGE_CONTRACT_VERSION, + MANAGED_IMAGE_PLATFORM, + MANAGED_IMAGE_REPOSITORIES, + MANAGED_IMAGE_SOURCE_REPOSITORY, + MANAGED_IMAGE_STARTUP_PROFILE_CONTRACT_VERSION, + type ManagedImageContractCatalog, + type ManagedImageContractV1, + SHIPPED_MANAGED_IMAGE_AGENTS, + type ShippedManagedImageAgent, +} from "../../src/lib/onboard/managed-image/contract"; +import { + decodeManagedStartupProfile, + encodeManagedStartupProfile, +} from "../../src/lib/onboard/managed-startup/profile"; +import { nodeOptionsWithoutSourceLoader, SOURCE_REQUIRE_HOOK } from "./source-loader-options"; + +const REPO_ROOT = path.resolve(import.meta.dirname, "../.."); +const MODEL = "nvidia/test-managed-model"; +const PROVIDER = "nvidia-prod"; +const SOURCE_REVISION = "2f03907c3a7ec151d7f5d4bb2a73abafc2849f83"; +const CATALOG_RELEASE = "v0.0.97"; +const SECRET_CANARY = "managed-onboard-secret-canary-must-not-cross"; +const AUTHENTICATED_PROXY_ENVIRONMENT = { + HTTP_PROXY: "http://upper-http:upper-secret@upper-http.example.test:18080", + HTTPS_PROXY: "http://upper-https:upper-secret@upper-https.example.test:18443", + NO_PROXY: "upper.internal", + http_proxy: "http://lower-http:lower-secret@lower-http.example.test:28080", + https_proxy: "http://lower-https:lower-secret@lower-https.example.test:28443", + no_proxy: "lower.internal", +} as const; +const DIGESTS = { + openclaw: `sha256:${"1".repeat(64)}`, + hermes: `sha256:${"2".repeat(64)}`, + "langchain-deepagents-code": `sha256:${"3".repeat(64)}`, +} as const satisfies Record; + +interface CatalogCall { + release: string; + references: Record; +} + +interface SpawnCall { + command: string; + args: string[]; +} + +interface ChildPayload { + agent: ShippedManagedImageAgent; + catalogCalls: CatalogCall[]; + forbiddenCalls: string[]; + managedStartupRootExecCalls: Array<{ + args: string[]; + input: string; + }>; + managedStartupPatchCalls: Array<{ + agent?: string; + encodedProfile?: string; + profileFingerprint?: string; + schemaVersion?: number; + sandboxName?: string; + }>; + registerCalls: Array<{ + agent?: string; + imageTag?: string | null; + name?: string; + workload?: { + schemaVersion?: number; + kind?: string; + reference?: string; + release?: string; + sourceRevision?: string; + capabilityContractVersion?: number; + startupProfileContractVersion?: number; + startupProfileSha256?: string; + credentialProxyReplayRequired?: boolean; + shared?: boolean; + }; + }>; + runnerCommands: string[]; + spawnCalls: SpawnCall[]; +} + +function contractFor(agent: ShippedManagedImageAgent): ManagedImageContractV1 { + const image = MANAGED_IMAGE_REPOSITORIES[agent]; + const digest = DIGESTS[agent]; + return { + contractVersion: MANAGED_IMAGE_CONTRACT_VERSION, + agent, + platform: MANAGED_IMAGE_PLATFORM, + image, + digest, + reference: `${image}@${digest}`, + source: { + repository: MANAGED_IMAGE_SOURCE_REPOSITORY, + revision: SOURCE_REVISION, + release: CATALOG_RELEASE, + cohort: "ghrun-7744-2", + }, + startupProfileContractVersion: MANAGED_IMAGE_STARTUP_PROFILE_CONTRACT_VERSION, + capabilityContractVersion: MANAGED_IMAGE_CAPABILITY_CONTRACT_VERSION, + }; +} + +function completeCatalog(): ManagedImageContractCatalog { + return Object.fromEntries( + SHIPPED_MANAGED_IMAGE_AGENTS.map((agent) => [agent, contractFor(agent)]), + ); +} + +function childSource( + agent: ShippedManagedImageAgent, + sandboxName: string, + catalog: ManagedImageContractCatalog, +): string { + const source = (relativePath: string) => JSON.stringify(path.join(REPO_ROOT, relativePath)); + return String.raw` +const { EventEmitter } = require("node:events"); +const Module = require("node:module"); +const path = require("node:path"); + +const agentName = ${JSON.stringify(agent)}; +const sandboxName = ${JSON.stringify(sandboxName)}; +const catalogTemplate = ${JSON.stringify(catalog)}; +const model = ${JSON.stringify(MODEL)}; +const provider = ${JSON.stringify(PROVIDER)}; +const catalogCalls = []; +const forbiddenCalls = []; +const managedStartupRootExecCalls = []; +const managedStartupPatchCalls = []; +const registerCalls = []; +const runnerCommands = []; +const spawnCalls = []; + +// The protected live-E2E job intentionally runs source without build:cli. +// Route the root CLI's generated shared-boundary import back to its canonical +// .cts source so this test cannot pass only because a local dist tree exists. +const canonicalSandboxNameSource = + ${source("nemoclaw/src/shared/sandbox-name.cts")}; +const generatedSandboxName = + ${source("nemoclaw/dist/shared/sandbox-name.cjs")}; +const resolveFilename = Module._resolveFilename; +Module._extensions[".cts"] = Module._extensions[".ts"]; +Module._resolveFilename = function(request, parent, isMain, options) { + const requestedPath = + request.startsWith(".") && parent && parent.filename + ? path.resolve(path.dirname(parent.filename), request) + : request; + if (requestedPath === generatedSandboxName) return canonicalSandboxNameSource; + return resolveFilename.call(this, request, parent, isMain, options); +}; + +const normalize = (command) => + (Array.isArray(command) ? command.map(String).join(" ") : String(command)).replace(/'/g, ""); +const poison = (name) => { + forbiddenCalls.push(name); + throw new Error("managed onboarding entered forbidden legacy path: " + name); +}; +const replace = (target, name, value) => { + target[name] = value; + if (target[name] !== value) throw new Error("could not install test boundary for " + name); +}; +const childProcess = require("node:child_process"); +const nativeSpawnSync = childProcess.spawnSync; +childProcess.spawnSync = (command, args = [], options = {}) => { + const argv = Array.isArray(args) ? args.map(String) : []; + if ( + path.basename(String(command)) === "docker" && + argv.includes("--apply-root-stdin") + ) { + managedStartupRootExecCalls.push({ + args: argv, + input: String(options.input ?? ""), + }); + } + return nativeSpawnSync(command, args, options); +}; + +const catalogResolver = require(${source("src/lib/onboard/managed-image/catalog.ts")}); +replace(catalogResolver, "resolveManagedImageCatalogFromGhcr", async ({ release }) => { + const catalog = Object.fromEntries( + Object.entries(catalogTemplate).map(([name, contract]) => [ + name, + { ...contract, source: { ...contract.source, release } }, + ]), + ); + catalogCalls.push({ + release, + references: Object.fromEntries( + Object.entries(catalog).map(([name, contract]) => [name, contract.reference]), + ), + }); + return catalog; +}); +const workloadRuntime = require(${source("src/lib/onboard/workload/runtime.ts")}); +const resolveRuntimeCapabilities = workloadRuntime.resolveSandboxWorkloadRuntimeCapabilities; +replace(workloadRuntime, "resolveSandboxWorkloadRuntimeCapabilities", (plan, profiles) => + resolveRuntimeCapabilities(plan, profiles, "x64"), +); + +const agentOnboard = require(${source("src/lib/agent/onboard.ts")}); +replace(agentOnboard, "createAgentSandbox", () => poison("agentOnboard.createAgentSandbox")); +const buildContextStage = require(${source("src/lib/onboard/build-context-stage.ts")}); +replace(buildContextStage, "stageCreateSandboxBuildContext", () => + poison("stageCreateSandboxBuildContext"), +); +const sandboxBuildContext = require(${source("src/lib/sandbox/build-context.ts")}); +replace(sandboxBuildContext, "stageOptimizedSandboxBuildContext", () => + poison("stageOptimizedSandboxBuildContext"), +); +const preparedBuild = require(${source("src/lib/onboard/prepared-dcode-rebuild.ts")}); +replace(preparedBuild, "resolveSandboxBuildContext", () => + poison("resolveSandboxBuildContext"), +); +replace(preparedBuild, "resolveSandboxBuildPatch", () => + poison("resolveSandboxBuildPatch"), +); +const dockerfilePatch = require(${source("src/lib/onboard/sandbox-dockerfile-patch-flow.ts")}); +replace(dockerfilePatch, "prepareSandboxDockerfilePatch", () => + poison("prepareSandboxDockerfilePatch"), +); +const sandboxPrebuild = require(${source("src/lib/onboard/sandbox-prebuild.ts")}); +replace(sandboxPrebuild, "prebuildSandboxImageIfEligible", () => + poison("prebuildSandboxImageIfEligible"), +); +const baseImage = require(${source("src/lib/onboard/base-image.ts")}); +replace(baseImage, "pullAndResolveBaseImageDigest", () => + poison("pullAndResolveBaseImageDigest"), +); +// Keep the compute plan on Docker so managed-image capability negotiation is +// real, while excluding the separate Docker-container restart compatibility +// shim. That shim is covered by its own suites and is not part of workload +// source selection or the sandbox-create transport asserted here. +const dockerDriverPlatform = require(${source("src/lib/onboard/docker-driver-platform.ts")}); +replace(dockerDriverPlatform, "isLinuxDockerDriverGatewayEnabled", () => false); +const dockerSandboxCreatePatch = require( + ${source("src/lib/onboard/docker-gpu-sandbox-create.ts")}, +); +const createDockerGpuSandboxCreatePatch = + dockerSandboxCreatePatch.createDockerGpuSandboxCreatePatch; +replace(dockerSandboxCreatePatch, "createDockerGpuSandboxCreatePatch", (options) => { + const containerId = "a".repeat(64); + const imageId = "b".repeat(64); + managedStartupPatchCalls.push({ + agent: options.managedStartupRootApplyRequest?.agent, + encodedProfile: options.managedStartupRootApplyRequest?.encodedProfile, + profileFingerprint: options.managedStartupRootApplyRequest?.profileFingerprint, + schemaVersion: options.managedStartupRootApplyRequest?.schemaVersion, + sandboxName: options.sandboxName, + }); + return createDockerGpuSandboxCreatePatch({ + ...options, + deps: { + ...options.deps, + dockerCapture(args, captureOptions) { + if (args[0] === "inspect") { + return JSON.stringify([ + { + Id: containerId, + Image: "sha256:" + imageId, + State: { + Running: true, + Paused: false, + Restarting: false, + Dead: false, + }, + }, + ]); + } + return options.deps.dockerCapture?.(args, captureOptions) ?? ""; + }, + }, + overrides: { + ...options.overrides, + findContainerIds: () => [containerId], + finalizeManagedStartupSharedState: ({ supervisorReady }) => ({ + supervisorReady, + failure: null, + }), + }, + }); +}); + +const runner = require(${source("src/lib/runner.ts")}); +runner.run = (command, options = {}) => { + const normalized = normalize(command); + runnerCommands.push(normalized); + if (/(?:^|\s)docker(?:\s+buildx)?\s+build(?:\s|$)/u.test(normalized)) { + return poison("docker build"); + } + return { status: 0, stdout: "", stderr: "" }; +}; +runner.runFile = (file, args = []) => runner.run([file, ...args]); +runner.runCapture = (command) => { + const normalized = normalize(command); + runnerCommands.push(normalized); + if (normalized.includes("sandbox get " + sandboxName)) return ""; + if (normalized.includes("sandbox list")) return sandboxName + " Ready"; + if (normalized.includes("forward list")) { + return sandboxName + " 127.0.0.1 18789 23189 running"; + } + if (normalized.includes("dcode identity")) { + const { getExpectedDcodeInferenceIdentity } = + require(${source("src/lib/onboard/dcode-selection-drift.ts")}); + const identity = getExpectedDcodeInferenceIdentity(provider, model, "openai-completions"); + return [ + "Route: " + identity.route, + "Provider: " + identity.provider, + "Model: " + identity.model, + "Endpoint: " + identity.endpoint, + ].join("\n"); + } + const mocked = require(${source("test/helpers/onboard-script-mocks.cjs")}) + .mockOnboardRunCapture(command); + return mocked === null ? "" : mocked; +}; +runner.runCaptureEx = (command) => ({ + status: 0, + stdout: runner.runCapture(command), + stderr: "", +}); + +const registry = require(${source("src/lib/state/registry.ts")}); +registry.getSandbox = () => null; +registry.getDefault = () => null; +registry.listExtraProviders = () => []; +registry.registerSandbox = (entry) => { + registerCalls.push(entry); + return true; +}; +registry.updateSandbox = () => true; +registry.setDefault = () => true; +registry.removeSandbox = () => true; + +const preflight = require(${source("src/lib/onboard/preflight.ts")}); +preflight.checkPortAvailable = async () => ({ ok: true }); +const credentials = require(${source("src/lib/credentials/store.ts")}); +credentials.prompt = async () => ""; + +childProcess.spawn = (command, args = [], options = {}) => { + const argv = Array.isArray(args) ? args.map(String) : []; + const normalized = normalize([command, ...argv]); + if (/(?:^|\s)docker(?:\s+buildx)?\s+build(?:\s|$)/u.test(normalized)) { + return poison("docker build"); + } + spawnCalls.push({ command: String(command), args: argv }); + const child = new EventEmitter(); + child.stdout = new EventEmitter(); + child.stderr = new EventEmitter(); + child.kill = () => true; + child.unref = () => {}; + child.pid = 7744; + process.nextTick(() => { + child.stdout.emit("data", Buffer.from("Created sandbox: " + sandboxName + "\n")); + child.emit("close", 0); + }); + return child; +}; + +const { loadAgent } = require(${source("src/lib/agent/defs.ts")}); +const { createSandbox } = require(${source("src/lib/onboard.ts")}); + +(async () => { + process.env.OPENSHELL_GATEWAY = "nemoclaw"; + await createSandbox( + null, + model, + provider, + "openai-completions", + sandboxName, + null, + [], + null, + loadAgent(agentName), + ); + console.log(JSON.stringify({ + agent: agentName, + catalogCalls, + forbiddenCalls, + managedStartupRootExecCalls, + managedStartupPatchCalls, + registerCalls, + runnerCommands, + spawnCalls, + })); +})().catch((error) => { + console.error(error && error.stack ? error.stack : error); + process.exit(1); +}); +`; +} + +function writeRuntimeStubs(fakeBin: string, dockerLog: string): void { + fs.writeFileSync( + path.join(fakeBin, "openshell"), + [ + "#!/usr/bin/env bash", + 'if [ "${1:-}" = "--version" ] || [ "${1:-}" = "-V" ]; then', + ' printf "%s\\n" "openshell 0.0.96"', + "fi", + "exit 0", + "", + ].join("\n"), + { mode: 0o755 }, + ); + fs.writeFileSync( + path.join(fakeBin, "docker"), + [ + "#!/usr/bin/env bash", + 'printf "%s\\n" "$*" >> "$NEMOCLAW_TEST_DOCKER_LOG"', + 'if [ "${1:-}" = "build" ] || { [ "${1:-}" = "buildx" ] && [ "${2:-}" = "build" ]; }; then', + ' printf "%s\\n" "forbidden docker build" >&2', + " exit 97", + "fi", + 'if [ "${1:-}" = "info" ]; then printf "%s\\n" "{}"; fi', + "exit 0", + "", + ].join("\n"), + { mode: 0o755 }, + ); + fs.writeFileSync(dockerLog, ""); +} + +function parsePayload(stdout: string): ChildPayload { + const payload = stdout + .trim() + .split(/\r?\n/u) + .reverse() + .find((line) => line.startsWith("{") && line.endsWith("}")); + expect(payload, `managed onboard child did not emit evidence:\n${stdout}`).toBeDefined(); + return JSON.parse(payload as string) as ChildPayload; +} + +function runManagedOnboard( + root: string, + agent: ShippedManagedImageAgent, + catalog: ManagedImageContractCatalog, +): { dockerCommands: string[]; payload: ChildPayload } { + const fixture = path.join(root, agent); + const fakeBin = path.join(fixture, "bin"); + const home = path.join(fixture, "home"); + const script = path.join(fixture, "managed-onboard.cjs"); + const dockerLog = path.join(fixture, "docker.log"); + const sandboxName = `managed-${agent.replaceAll(/[^a-z0-9]/gu, "-")}`; + fs.mkdirSync(fakeBin, { recursive: true }); + fs.mkdirSync(home, { recursive: true }); + writeRuntimeStubs(fakeBin, dockerLog); + fs.writeFileSync(script, childSource(agent, sandboxName, catalog)); + + const result = spawnSync(process.execPath, ["--require", SOURCE_REQUIRE_HOOK, script], { + cwd: REPO_ROOT, + encoding: "utf8", + timeout: 60_000, + killSignal: "SIGKILL", + env: { + HOME: home, + NEMOCLAW_HOME: path.join(home, ".nemoclaw"), + NEMOCLAW_NON_INTERACTIVE: "1", + NEMOCLAW_TEST_SECRET_CANARY: SECRET_CANARY, + NEMOCLAW_TEST_DOCKER_LOG: dockerLog, + NEMOCLAW_TEST_NO_SLEEP: "1", + NODE_OPTIONS: nodeOptionsWithoutSourceLoader(process.env.NODE_OPTIONS), + PATH: `${fakeBin}${path.delimiter}${process.env.PATH ?? ""}`, + TMPDIR: process.env.TMPDIR ?? os.tmpdir(), + ...AUTHENTICATED_PROXY_ENVIRONMENT, + }, + }); + expect( + result.error, + `${agent} managed onboard child failed to complete: ${result.error?.message}`, + ).toBeUndefined(); + expect( + result.signal, + `${agent} managed onboard child was terminated:\n${result.stderr}\n${result.stdout}`, + ).toBeNull(); + expect( + result.status, + `${agent} managed onboard child failed:\n${result.stderr}\n${result.stdout}`, + ).toBe(0); + + const dockerCommands = fs.readFileSync(dockerLog, "utf8").trim().split(/\r?\n/u).filter(Boolean); + return { dockerCommands, payload: parsePayload(result.stdout) }; +} + +function assertManagedLaunch( + result: ReturnType, + agent: ShippedManagedImageAgent, +): void { + const expectedContract = contractFor(agent); + expect(result.payload.agent).toBe(agent); + expect(result.payload.forbiddenCalls).toEqual([]); + expect(result.payload.managedStartupPatchCalls).toHaveLength(1); + expect(result.payload.managedStartupRootExecCalls).toHaveLength(1); + expect(result.payload.managedStartupPatchCalls[0]).toMatchObject({ + agent, + schemaVersion: 1, + sandboxName: expect.stringMatching(/^managed-/u), + profileFingerprint: expect.stringMatching(/^[a-f0-9]{64}$/u), + }); + const rootExec = result.payload.managedStartupRootExecCalls[0]; + expect(rootExec?.args).toEqual([ + "exec", + "--interactive", + "--user", + "0:0", + "--workdir", + "/", + "a".repeat(64), + "/usr/bin/env", + "-i", + "HOME=/root", + "LANG=C.UTF-8", + "LC_ALL=C.UTF-8", + "NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION=1", + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + "/usr/local/bin/node", + "/usr/local/lib/nemoclaw/managed-startup-image-runtime.cjs", + "--apply-root-stdin", + "--agent", + agent, + ]); + expect(rootExec?.input.endsWith("\n")).toBe(true); + expect(JSON.parse(rootExec?.input ?? "{}")).toMatchObject({ + agent, + encodedProfile: result.payload.managedStartupPatchCalls[0]?.encodedProfile, + profileFingerprint: result.payload.managedStartupPatchCalls[0]?.profileFingerprint, + schemaVersion: 1, + }); + expect(result.payload.catalogCalls).toHaveLength(1); + expect(result.payload.catalogCalls[0]?.release).toMatch(/^v[0-9]/u); + expect(result.payload.catalogCalls[0]?.references).toEqual( + Object.fromEntries( + SHIPPED_MANAGED_IMAGE_AGENTS.map((catalogAgent) => [ + catalogAgent, + contractFor(catalogAgent).reference, + ]), + ), + ); + + const createCalls = result.payload.spawnCalls.filter( + ({ args }) => args[0] === "sandbox" && args[1] === "create", + ); + expect(createCalls).toHaveLength(1); + const createArgs = createCalls[0]?.args ?? []; + expect(createArgs.filter((arg) => arg === "--from")).toHaveLength(1); + const fromIndex = createArgs.indexOf("--from"); + expect(createArgs[fromIndex + 1]).toBe(expectedContract.reference); + expect(createArgs.join(" ")).not.toContain("Dockerfile"); + + expect(createArgs.filter((arg) => arg.startsWith("NEMOCLAW_STARTUP_PROFILE_B64="))).toEqual([]); + const encodedProfile = result.payload.managedStartupPatchCalls[0]?.encodedProfile ?? ""; + const profile = decodeManagedStartupProfile(encodedProfile); + expect(encodeManagedStartupProfile(profile)).toBe(encodedProfile); + expect(profile).toMatchObject({ + schemaVersion: MANAGED_IMAGE_STARTUP_PROFILE_CONTRACT_VERSION, + agent, + agentConfig: { agent }, + inference: { + model: MODEL, + upstreamProvider: PROVIDER, + }, + dashboard: { agent }, + }); + expect(createArgs.filter((arg) => arg.startsWith("NEMOCLAW_CORPORATE_CA_B64="))).toEqual([]); + expect(JSON.stringify(profile)).not.toContain(SECRET_CANARY); + expect(profile.proxy).toMatchObject({ + hostHttpUrl: null, + hostHttpsUrl: null, + hostNoProxy: [], + }); + + const serializedCreate = createArgs.join("\n"); + expect(serializedCreate.includes("upper-secret")).toBe(agent !== "langchain-deepagents-code"); + expect(serializedCreate.includes("lower-secret")).toBe(agent !== "langchain-deepagents-code"); + const expectedForwardedProxyEntries = + agent === "langchain-deepagents-code" ? [] : Object.entries(AUTHENTICATED_PROXY_ENVIRONMENT); + for (const [name, value] of expectedForwardedProxyEntries) { + const forwarded = createArgs.find((argument) => argument.startsWith(`${name}=`)); + const expected = + name === "NO_PROXY" || name === "no_proxy" + ? expect.stringContaining(`${name}=${value},localhost,`) + : `${name}=${value}`; + expect(forwarded).toEqual(expected); + } + + const registration = result.payload.registerCalls.find( + (entry) => + entry.imageTag === expectedContract.reference && entry.name?.startsWith("managed-") === true, + ); + expect( + registration, + `${agent} registration did not retain the managed image: ${JSON.stringify( + result.payload.registerCalls, + )}`, + ).toBeDefined(); + expect(registration?.agent ?? "openclaw").toBe(agent); + expect(registration?.workload).toEqual({ + schemaVersion: 1, + kind: "managed-image", + reference: expectedContract.reference, + release: result.payload.catalogCalls[0]?.release, + sourceRevision: SOURCE_REVISION, + sourceCohort: expectedContract.source.cohort, + capabilityContractVersion: MANAGED_IMAGE_CAPABILITY_CONTRACT_VERSION, + startupProfileContractVersion: MANAGED_IMAGE_STARTUP_PROFILE_CONTRACT_VERSION, + encodedProfile, + startupProfileSha256: createHash("sha256").update(encodedProfile, "utf8").digest("hex"), + credentialProxyReplayRequired: agent !== "langchain-deepagents-code", + shared: true, + }); + const serializedReceipt = JSON.stringify(registration?.workload); + expect(serializedReceipt).not.toContain("upper-secret"); + expect(serializedReceipt).not.toContain("lower-secret"); + expect( + result.payload.runnerCommands.some((command) => + /(?:^|\s)docker(?:\s+buildx)?\s+build(?:\s|$)/u.test(command), + ), + ).toBe(false); + expect( + result.dockerCommands.some((command) => /^(?:build|buildx build)(?:\s|$)/u.test(command)), + ).toBe(false); +} + +export function runManagedImageBuildlessE2e(): void { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-managed-onboard-e2e-")); + const catalog = completeCatalog(); + expect(Object.keys(catalog).sort()).toEqual([...SHIPPED_MANAGED_IMAGE_AGENTS].sort()); + + try { + const openclaw = runManagedOnboard(root, "openclaw", catalog); + + const hermes = runManagedOnboard(root, "hermes", catalog); + + const dcode = runManagedOnboard(root, "langchain-deepagents-code", catalog); + + assertManagedLaunch(openclaw, "openclaw"); + assertManagedLaunch(hermes, "hermes"); + assertManagedLaunch(dcode, "langchain-deepagents-code"); + } finally { + fs.rmSync(root, { force: true, recursive: true }); + } +} diff --git a/test/helpers/rebuild-flow-harness.ts b/test/helpers/rebuild-flow-harness.ts index ef2846b26c..16efb8d3d5 100644 --- a/test/helpers/rebuild-flow-harness.ts +++ b/test/helpers/rebuild-flow-harness.ts @@ -51,7 +51,11 @@ const rebuildCustomImagePreflight = requireDist("./rebuild-custom-image-prefligh const rebuildInference = requireDist("./rebuild-inference-preflight.js"); const rebuildFlowHelpers = requireDist("./rebuild-flow-helpers.js"); const rebuildManagedImage = requireDist("./rebuild-managed-image-preflight.js"); +const rebuildManagedWorkloadProfile = requireDist("./agents/managed-workload-rebuild-profile.js"); const rebuildMessagingConflict = requireDist("./rebuild-messaging-conflict-preflight.js"); +const computePlan = requireDist("../../onboard/compute/plan.js"); +const managedWorkloadRebuild = requireDist("../../onboard/workload/rebuild.js"); +const workloadRuntime = requireDist("../../onboard/workload/runtime.js"); const shields = requireDist("../../shields/index.js"); type RebuildFlowStep = { @@ -137,6 +141,9 @@ export type RebuildFlowOverrides = { openShieldsWindow?: () => { relocked: boolean; wasLocked: boolean } | null; preflightMessagingConflicts?: () => Promise | void; preflightAuthoritativeRebuildTarget?: (options: Record) => Promise | void; + managedWorkloadReplacement?: Record; + managedContextWindow?: number | null; + prepareManagedRebuildProfileHandoff?: (input: Record) => Record; mcpPreparation?: { entries: Array>; detachedProviderEntries: Array>; @@ -150,6 +157,7 @@ export type RebuildFlowHarness = { applyPresetContentSpy: MockInstance; backupSandboxStateSpy: MockInstance; disposePreparedDcodeRebuildImageSpy: MockInstance; + dockerBuildSpy: MockInstance; errorSpy: MockInstance; ensureAgentBaseImageSpy: MockInstance; pinTrustedAgentBaseImageOverrideForOperationSpy: MockInstance; @@ -162,9 +170,12 @@ export type RebuildFlowHarness = { openShieldsSpy: MockInstance; onboardSpy: MockInstance; preflightAuthoritativeRebuildTargetSpy: MockInstance; + preflightRebuildImageSpy: MockInstance; preflightMessagingConflictsSpy: MockInstance; preflightDcodeRouteSpy: MockInstance; prepareManagedDcodeRebuildImageSpy: MockInstance; + prepareManagedRebuildProfileHandoffSpy: MockInstance; + prepareManagedWorkloadSourceSpy: MockInstance; removePresetSpy: MockInstance; removeSandboxRegistryEntrySpy: MockInstance; registryUpdateSpy: MockInstance; @@ -304,6 +315,9 @@ export function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): expectedVersion: "0.2.0", dockerfileBasePath: "/tmp/Dockerfile.base", runtime: { kind: "terminal" }, + ...(overrides.managedWorkloadReplacement && agentName !== "langchain-deepagents-code" + ? { forwardPort: 8_000 } + : {}), }; vi.spyOn(gatewayDrift, "detectOpenShellStateRpcPreflightIssue").mockReturnValue(null); @@ -312,11 +326,54 @@ export function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): result: { status: 0, output: overrides.sandboxListOutput ?? "alpha Ready" }, }); vi.spyOn(resolve, "resolveOpenshell").mockReturnValue(null); - vi.spyOn(dockerImage, "dockerBuild").mockReturnValue({ status: 0 }); - vi.spyOn(rebuildCustomImagePreflight, "preflightRebuildImage").mockResolvedValue({ - ok: true, - imageTag: null, + const dockerBuildSpy = vi.spyOn(dockerImage, "dockerBuild").mockReturnValue({ status: 0 }); + const preflightRebuildImageSpy = vi + .spyOn(rebuildCustomImagePreflight, "preflightRebuildImage") + .mockResolvedValue({ + ok: true, + imageTag: null, + }); + const prepareManagedWorkloadSourceSpy = vi + .spyOn( + managedWorkloadRebuild.managedWorkloadRebuildDependencies, + "prepareSandboxWorkloadSource", + ) + .mockImplementation(async () => { + if (!overrides.managedWorkloadReplacement) { + throw new Error("managed workload replacement fixture was not configured"); + } + return overrides.managedWorkloadReplacement; + }); + vi.spyOn(computePlan, "resolveCurrentOpenShellComputePlan").mockReturnValue({ + driverName: "docker", + gatewayLauncher: "nemoclaw", }); + vi.spyOn(workloadRuntime, "resolveSandboxWorkloadRuntimeCapabilities").mockReturnValue({ + driverName: "docker", + managedImageSelectionPolicy: "prefer-managed", + legacyDockerfileBuilds: true, + managedImages: { + exactDigestReferences: true, + platforms: ["linux/amd64"], + startupProfileContractVersions: [1], + capabilityContractVersions: [1], + }, + }); + vi.spyOn( + rebuildManagedWorkloadProfile.managedRebuildProfileDependencies, + "resolveContextWindowForModel", + ).mockReturnValue( + overrides.managedContextWindow === undefined ? 131_072 : overrides.managedContextWindow, + ); + const prepareManagedRebuildProfileHandoff = + rebuildManagedWorkloadProfile.prepareManagedRebuildProfileHandoff; + const prepareManagedRebuildProfileHandoffSpy = vi + .spyOn(rebuildManagedWorkloadProfile, "prepareManagedRebuildProfileHandoff") + .mockImplementation((input: unknown) => + overrides.prepareManagedRebuildProfileHandoff + ? overrides.prepareManagedRebuildProfileHandoff((input ?? {}) as Record) + : prepareManagedRebuildProfileHandoff(input), + ); const imageIdsByRef = new Map([ [agentBaseImageRef, agentBaseImageId], [agentBaseImageId, agentBaseImageId], @@ -575,7 +632,9 @@ export function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): const removeSandboxRegistryEntrySpy = vi .spyOn(destroy, "removeSandboxRegistryEntryWithReceipt") .mockReturnValue({ - entry: { name: "alpha", imageTag: "old-image" }, + entry: overrides.managedWorkloadReplacement + ? sandboxEntry + : { name: "alpha", imageTag: "old-image" }, wasDefault: preDeleteDefaultSandbox === "alpha", fallbackDefault: null, postRemovalDefaultSelectionRevision: 1, @@ -731,6 +790,7 @@ export function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): applyPresetContentSpy, backupSandboxStateSpy, disposePreparedDcodeRebuildImageSpy, + dockerBuildSpy, errorSpy, ensureAgentBaseImageSpy, pinTrustedAgentBaseImageOverrideForOperationSpy, @@ -743,9 +803,12 @@ export function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): openShieldsSpy, onboardSpy, preflightAuthoritativeRebuildTargetSpy, + preflightRebuildImageSpy, preflightMessagingConflictsSpy, preflightDcodeRouteSpy, prepareManagedDcodeRebuildImageSpy, + prepareManagedRebuildProfileHandoffSpy, + prepareManagedWorkloadSourceSpy, removePresetSpy, removeSandboxRegistryEntrySpy, registryUpdateSpy, diff --git a/test/hermes-doctor-config-hash.test.ts b/test/hermes-doctor-config-hash.test.ts index ff32a19fab..23cd182df2 100644 --- a/test/hermes-doctor-config-hash.test.ts +++ b/test/hermes-doctor-config-hash.test.ts @@ -80,6 +80,7 @@ describe("Hermes doctor and config hash boundary", () => { libDir, "openshell-child-visible-credentials.v0.0.85.json", ); + const entrypointEnvWrapperPath = path.join(libDir, "entrypoint-env-wrapper.sh"); const nestedDir = path.join(preloadsDir, "nested"); const profileDir = path.join(tmp, "etc-profile.d"); const bashrcPath = path.join(tmp, "bash.bashrc"); @@ -92,9 +93,11 @@ describe("Hermes doctor and config hash boundary", () => { fs.mkdirSync(profileDir, { recursive: true }); for (const relativePath of [ path.join(binDir, "nemoclaw-start"), + path.join(binDir, "nemoclaw-managed-startup-hold"), path.join(binDir, "nemoclaw-gateway-control"), path.join(libDir, "sandbox-init.sh"), path.join(libDir, "gateway-supervisor.sh"), + entrypointEnvWrapperPath, path.join(libDir, "validate-hermes-env-secret-boundary.py"), path.join(libDir, "patch-hermes-session-list-preview.py"), discordRecoveryPatcherPath, @@ -156,6 +159,7 @@ describe("Hermes doctor and config hash boundary", () => { expect(mode(mcpCredentialBoundaryPath)).toBe("444"); expect(mode(buildMcpDigestPath)).toBe("444"); expect(mode(path.join(libDir, "gateway-supervisor.sh"))).toBe("444"); + expect(mode(entrypointEnvWrapperPath)).toBe("444"); expect(mode(path.join(libDir, "state-dir-guard.py"))).toBe("500"); expect(mode(path.join(libDir, "managed-gateway-control.py"))).toBe("500"); expect(mode(preloadsDir)).toBe("755"); diff --git a/test/hermes-final-image-layout.test.ts b/test/hermes-final-image-layout.test.ts index 9b2c99aac1..cf43512474 100644 --- a/test/hermes-final-image-layout.test.ts +++ b/test/hermes-final-image-layout.test.ts @@ -215,9 +215,12 @@ describe("Hermes final image layout", () => { "COPY --from=mcp-tool-discovery-runtime /opt/mcp-tool-discovery-runtime/dist/ /usr/local/lib/nemoclaw/mcp-tool-discovery-runtime/", "COPY nemoclaw-blueprint/ /opt/nemoclaw-blueprint/", "COPY scripts/lib/sandbox-init.sh /usr/local/lib/nemoclaw/sandbox-init.sh", + "COPY scripts/lib/entrypoint-env-wrapper.sh /usr/local/lib/nemoclaw/entrypoint-env-wrapper.sh", "COPY scripts/lib/gateway-supervisor.sh /usr/local/lib/nemoclaw/gateway-supervisor.sh", "COPY scripts/lib/sandbox-rlimits.sh /usr/local/lib/nemoclaw/sandbox-rlimits.sh", "COPY agents/hermes/start.sh /usr/local/bin/nemoclaw-start", + "COPY scripts/managed-startup-hold.sh /usr/local/bin/nemoclaw-managed-startup-hold", + "COPY --from=managed-startup-runtime-builder /out/managed-startup-image-runtime.cjs /usr/local/lib/nemoclaw/managed-startup-image-runtime.cjs", "COPY scripts/gateway-control.sh /usr/local/bin/nemoclaw-gateway-control", "COPY scripts/managed-gateway-control.py /usr/local/lib/nemoclaw/managed-gateway-control.py", "COPY agents/hermes/validate-env-secret-boundary.py /usr/local/lib/nemoclaw/validate-hermes-env-secret-boundary.py", @@ -323,6 +326,7 @@ describe("Hermes final image layout", () => { "/usr/local/lib/nemoclaw/patch-hermes-discord-recovery-permissions.py 'root:root 755'", "/usr/local/lib/nemoclaw/patch-hermes-profile-policy-defaults.py 'root:root 755'", "/usr/local/bin/nemoclaw-gateway-control 'root:root 700'", + "/sandbox/.nemoclaw 'root:root 1755'", "/usr/local/lib/nemoclaw/preloads/sandbox-safety-net.js 'root:root 444'", "/usr/local/lib/nemoclaw/hermes-wrapper.py 'root:root 755'", "/scripts/checks/node-tar-image-scan.mts 'root:root 755'", @@ -334,9 +338,16 @@ describe("Hermes final image layout", () => { expect(doctorLayer).toContain( "HERMES_HOME=/sandbox/.hermes /usr/local/bin/hermes doctor --fix", ); - expect(doctorLayer).toMatch(/generate-config[.]ts\s+&& rm -rf \/sandbox\/[.]cache$/u); + expect(doctorLayer).toContain('if [ "$NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION" = "1" ]; then'); + expect(doctorLayer).toContain('assert m.version("microsoft-teams-apps") == "2.0.13.4"'); + expect(doctorLayer).toContain('assert m.version("aiohttp") == "3.14.1"'); + expect(doctorLayer).toMatch(/generate-config[.]ts\s+&& if /u); + expect(doctorLayer).toMatch(/fi\s+&& rm -rf \/sandbox\/[.]cache$/u); expect(finalStage).toContain("check_absent /opt/hermes/tests \\"); expect(finalStage).toContain("&& check_absent /sandbox/.cache \\"); + expect(finalStage).toContain("RUN chown root:root /sandbox/.nemoclaw \\"); + expect(finalStage).toContain("&& chmod 1755 /sandbox/.nemoclaw \\"); + expect(finalStage).toContain("&& chown sandbox:sandbox /sandbox/.nemoclaw/config.json"); }); // source-shape-contract: security -- Exact source-to-image digests keep the reviewed Hermes runtime entrypoints bound to the files copied into the sandbox image diff --git a/test/hermes-gateway-wrapper.test.ts b/test/hermes-gateway-wrapper.test.ts index ddfce3a232..1f6a738e77 100644 --- a/test/hermes-gateway-wrapper.test.ts +++ b/test/hermes-gateway-wrapper.test.ts @@ -1056,6 +1056,7 @@ describe.skipIf(!canRun)("agents/hermes/hermes-wrapper.py", () => { contextWindow: null, toolDisclosure: "progressive" as const, webSearchProvider: null, + managedImageCapabilityUnion: false, messagingCredentialPlaceholders: [], managedToolGateways: { brokerEnabled: false, presets: [] }, }; @@ -1104,6 +1105,7 @@ describe.skipIf(!canRun)("agents/hermes/hermes-wrapper.py", () => { contextWindow: null, toolDisclosure: "progressive" as const, webSearchProvider: null, + managedImageCapabilityUnion: false, messagingCredentialPlaceholders: [], managedToolGateways: { brokerEnabled: false, presets: [] }, }; diff --git a/test/hermes-mcp-integrity-state.test.ts b/test/hermes-mcp-integrity-state.test.ts index 6506da1256..ade0957745 100644 --- a/test/hermes-mcp-integrity-state.test.ts +++ b/test/hermes-mcp-integrity-state.test.ts @@ -294,12 +294,14 @@ print(json.dumps({ const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-hermes-mcp-parent-")); const helper = path.join(tempDir, "guard-helper.sh"); const parentFile = path.join(tempDir, "guard-parent"); + const stepDownFile = path.join(tempDir, "guard-step-down"); fs.writeFileSync( helper, [ "#!/bin/bash", "set -euo pipefail", 'printf "%s\\n" "$PPID" >"$NEMOCLAW_TEST_GUARD_PARENT_FILE"', + 'printf "%s\\n" "${NEMOCLAW_TEST_STEPPED_DOWN:-0}" >"$NEMOCLAW_TEST_GUARD_STEP_DOWN_FILE"', 'printf "%s\\n" "mcp_state=current"', ].join("\n"), { mode: 0o700 }, @@ -318,20 +320,25 @@ print(json.dumps({ "HERMES_DIR=/test/.hermes", "HERMES_HASH_FILE=/test/hermes.config-hash", `NEMOCLAW_TEST_GUARD_PARENT_FILE=${bashPrintfQ(parentFile)}`, - "export NEMOCLAW_TEST_GUARD_PARENT_FILE", + `NEMOCLAW_TEST_GUARD_STEP_DOWN_FILE=${bashPrintfQ(stepDownFile)}`, + "export NEMOCLAW_TEST_GUARD_PARENT_FILE NEMOCLAW_TEST_GUARD_STEP_DOWN_FILE", + 'id() { if [ "${1:-}" = "-u" ]; then printf "0\\n"; else command id "$@"; fi; }', + "STEP_DOWN_PREFIX_SANDBOX=(env NEMOCLAW_TEST_STEPPED_DOWN=1)", "HERMES_MCP_RECONCILE_PENDING=9", "caller_pid=$$", "inspect_hermes_mcp_integrity", 'IFS= read -r guard_parent <"$NEMOCLAW_TEST_GUARD_PARENT_FILE"', + 'IFS= read -r guard_step_down <"$NEMOCLAW_TEST_GUARD_STEP_DOWN_FILE"', '[ "$guard_parent" = "$caller_pid" ]', - 'printf "pending=%s\\n" "$HERMES_MCP_RECONCILE_PENDING"', + '[ "$guard_step_down" = "1" ]', + 'printf "pending=%s stepped-down=%s\\n" "$HERMES_MCP_RECONCILE_PENDING" "$guard_step_down"', ].join("\n"), ], { encoding: "utf-8", timeout: 5000 }, ); expect(result.status, result.stderr).toBe(0); - expect(result.stdout).toBe("pending=0\n"); + expect(result.stdout).toBe("pending=0 stepped-down=1\n"); } finally { fs.rmSync(tempDir, { recursive: true, force: true }); } diff --git a/test/hermes-start.test.ts b/test/hermes-start.test.ts index d444e06d5e..2bffbd8b39 100644 --- a/test/hermes-start.test.ts +++ b/test/hermes-start.test.ts @@ -438,6 +438,10 @@ function runHermesSandboxInitPreludeWithFakePath() { const end = src.indexOf("\nif [ -d /opt/hermes/hermes_cli/web_dist ];", start); const prelude = src .slice(start, end) + .replaceAll( + "/usr/local/lib/nemoclaw/entrypoint-env-wrapper.sh", + path.join(import.meta.dirname, "..", "scripts", "lib", "entrypoint-env-wrapper.sh"), + ) .replaceAll("/usr/local/lib/nemoclaw/sandbox-init.sh", fakeInit) .replaceAll("/usr/local/lib/nemoclaw/gateway-supervisor.sh", fakeSupervisor); diff --git a/test/image-cleanup.test.ts b/test/image-cleanup.test.ts index 89f0afec44..8926aa8376 100644 --- a/test/image-cleanup.test.ts +++ b/test/image-cleanup.test.ts @@ -4,23 +4,22 @@ // Verify that sandbox lifecycle operations clean up host-side Docker images. // See: https://github.com/NVIDIA/NemoClaw/issues/2086 -import { describe, it, expect, vi } from "vitest"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; - +import { describe, expect, it, vi } from "vitest"; +import { help as renderRootHelp } from "../src/lib/actions/root-help"; import { cleanupShieldsDestroyArtifacts, removeSandboxImage, removeSandboxRegistryEntry, removeShieldsState, } from "../src/lib/actions/sandbox/destroy"; -import { getSandboxDeleteOutcome } from "../src/lib/domain/sandbox/destroy"; -import { normalizeGarbageCollectImagesOptions } from "../src/lib/domain/lifecycle/options"; -import { resolveNemoclawStateDir } from "../src/lib/state/paths"; -import { help as renderRootHelp } from "../src/lib/actions/root-help"; import { COMMANDS, globalCommandTokens } from "../src/lib/cli/command-registry"; import { getRegisteredOclifCommandMetadata } from "../src/lib/cli/oclif-metadata"; +import { normalizeGarbageCollectImagesOptions } from "../src/lib/domain/lifecycle/options"; +import { getSandboxDeleteOutcome } from "../src/lib/domain/sandbox/destroy"; +import { resolveNemoclawStateDir } from "../src/lib/state/paths"; describe("image cleanup: sandbox destroy removes Docker image (#2086)", () => { it("removes sandbox images before deleting the registry entry", () => { @@ -66,6 +65,36 @@ describe("image cleanup: sandbox destroy removes Docker image (#2086)", () => { expect(removedTags).toEqual([]); }); + it("retains shared immutable managed images during per-sandbox destroy", () => { + const dockerRmi = vi.fn(() => ({ status: 0 }) as any); + + removeSandboxImage("alpha", { + getSandbox: () => + ({ + name: "alpha", + imageTag: `ghcr.io/nvidia/nemoclaw/openclaw-sandbox@sha256:${"a".repeat(64)}`, + workload: { + schemaVersion: 1, + kind: "managed-image", + reference: `ghcr.io/nvidia/nemoclaw/openclaw-sandbox@sha256:${"a".repeat(64)}`, + release: "v0.0.97", + sourceRevision: "b".repeat(40), + sourceCohort: "ghrun-123456-1", + capabilityContractVersion: 1, + startupProfileContractVersion: 1, + encodedProfile: "e30", + startupProfileSha256: + "beab987bef9c00dfc301b490ddb45321517e7d6a6bb3d31d259898b7d46393d8", + credentialProxyReplayRequired: false, + shared: true, + }, + }) as any, + dockerRmi, + }); + + expect(dockerRmi).not.toHaveBeenCalled(); + }); + it("treats missing sandbox delete results as already gone", () => { expect( getSandboxDeleteOutcome({ status: 1, stderr: "Error: sandbox alpha not found" }), diff --git a/test/install-openshell-podman-components.test.ts b/test/install-openshell-podman-components.test.ts new file mode 100644 index 0000000000..438f936daa --- /dev/null +++ b/test/install-openshell-podman-components.test.ts @@ -0,0 +1,185 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +const INSTALLER = path.join(import.meta.dirname, "..", "scripts", "install-openshell.sh"); +const REQUIRED_VERSION = "0.0.85"; +const FEATURES = [ + "request-body-credential-rewrite", + "websocket-credential-rewrite", + "allow_all_known_mcp_methods", +].join(" "); + +function executable(file: string, body: string): void { + fs.writeFileSync(file, body, { mode: 0o755 }); +} + +function runOpenShellInstaller(options: { + driver: "auto" | "podman"; + hostArch?: "arm64" | "x86_64"; + hostOs?: "Darwin" | "Linux"; + installedVersion?: string; + missingSandboxOverride?: boolean; + sandbox?: boolean; +}): { downloads: string; dockerCalls: string; output: string; status: number | null } { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-podman-components-")); + const bin = path.join(tmp, "bin"); + const downloads = path.join(tmp, "downloads.log"); + const dockerCalls = path.join(tmp, "docker.log"); + fs.mkdirSync(bin); + const version = options.installedVersion ?? REQUIRED_VERSION; + + executable( + path.join(bin, "uname"), + `#!/usr/bin/env bash +if [ "\${1:-}" = "-m" ]; then + echo ${options.hostArch ?? "x86_64"} +else + echo ${options.hostOs ?? "Linux"} +fi +`, + ); + executable( + path.join(bin, "openshell"), + `#!/usr/bin/env bash +if [ "\${1:-}" = "--version" ]; then echo "openshell ${version}"; exit 0; fi +# ${FEATURES} +exit 0 +`, + ); + executable( + path.join(bin, "openshell-gateway"), + `#!/usr/bin/env bash +if [ "\${1:-}" = "--version" ]; then echo "openshell-gateway ${version}"; exit 0; fi +# ${FEATURES} +exit 0 +`, + ); + if (options.sandbox) { + executable( + path.join(bin, "openshell-sandbox"), + `#!/usr/bin/env bash +if [ "\${1:-}" = "--version" ]; then echo "openshell-sandbox ${version}"; exit 0; fi +# ${FEATURES} +exit 0 +`, + ); + } + executable( + path.join(bin, "docker"), + `#!/usr/bin/env bash\nprintf '%s\\n' "$*" >> ${JSON.stringify(dockerCalls)}\nexit 97\n`, + ); + executable(path.join(bin, "gh"), "#!/usr/bin/env bash\nexit 1\n"); + executable( + path.join(bin, "curl"), + `#!/usr/bin/env bash +printf '%s\\n' "$*" >> ${JSON.stringify(downloads)} +output="" +while [ "$#" -gt 0 ]; do + if [ "$1" = "-o" ]; then shift; output="$1"; fi + shift || true +done +[ -n "$output" ] && : >"$output" +exit 0 +`, + ); + + const result = spawnSync("bash", [INSTALLER], { + encoding: "utf8", + env: { + ...process.env, + HOME: tmp, + NEMOCLAW_COMPUTE_DRIVER: options.driver, + NEMOCLAW_OPENSHELL_CHANNEL: "stable", + NEMOCLAW_OPENSHELL_SANDBOX_BIN: options.missingSandboxOverride + ? path.join(tmp, "missing-openshell-sandbox") + : "", + PATH: `${bin}:/usr/bin:/bin`, + }, + }); + const outcome = { + downloads: fs.existsSync(downloads) ? fs.readFileSync(downloads, "utf8") : "", + dockerCalls: fs.existsSync(dockerCalls) ? fs.readFileSync(dockerCalls, "utf8") : "", + output: `${result.stdout}${result.stderr}`, + status: result.status, + }; + fs.rmSync(tmp, { recursive: true, force: true }); + return outcome; +} + +describe("install-openshell.sh Podman component contract", () => { + it("reuses a coherent CLI and gateway without requiring a Docker sandbox binary", () => { + const result = runOpenShellInstaller({ driver: "podman" }); + + expect(result.status, result.output).toBe(0); + expect(result.output).toContain(`openshell already installed: ${REQUIRED_VERSION}`); + expect(result.output).not.toContain("missing Docker-driver binaries"); + expect(result.downloads).toBe(""); + expect(result.dockerCalls).toBe(""); + }); + + it("ignores an explicit sandbox override only for Podman", () => { + const result = runOpenShellInstaller({ + driver: "podman", + missingSandboxOverride: true, + }); + + expect(result.status, result.output).toBe(0); + expect(result.output).toContain(`openshell already installed: ${REQUIRED_VERSION}`); + expect(result.output).not.toContain("explicit OpenShell sandbox binary"); + }); + + it("keeps validating an explicit sandbox override for macOS auto/Docker", () => { + const result = runOpenShellInstaller({ + driver: "auto", + hostArch: "arm64", + hostOs: "Darwin", + missingSandboxOverride: true, + }); + + expect(result.status).toBe(1); + expect(result.output).toContain("explicit OpenShell sandbox binary"); + expect(result.output).toContain("missing, unreadable, or not executable"); + }); + + it("keeps the default Docker install dependent on its sandbox helper", () => { + const result = runOpenShellInstaller({ driver: "auto" }); + + expect(result.status).not.toBe(0); + expect(result.output).toContain("missing Docker-driver binaries"); + expect(result.output).toContain(`Installing OpenShell from release 'v${REQUIRED_VERSION}'`); + expect(result.dockerCalls).toBe(""); + }); + + it("downloads only CLI and gateway artifacts for a fresh Podman install", () => { + const result = runOpenShellInstaller({ + driver: "podman", + installedVersion: "0.0.1", + }); + + expect(result.status).not.toBe(0); + expect(result.downloads).toContain("openshell-x86_64-unknown-linux-musl.tar.gz"); + expect(result.downloads).toContain("openshell-gateway-x86_64-unknown-linux-gnu.tar.gz"); + expect(result.downloads).not.toContain("openshell-sandbox-"); + expect(result.downloads).not.toContain("openshell-sandbox-checksums-sha256.txt"); + expect(result.dockerCalls).toBe(""); + }); + + it("continues to request the sandbox artifact for a fresh default Docker install", () => { + const result = runOpenShellInstaller({ + driver: "auto", + installedVersion: "0.0.1", + }); + + expect(result.status).not.toBe(0); + expect(result.downloads).toContain("openshell-sandbox-x86_64-unknown-linux-gnu.tar.gz"); + expect(result.downloads).toContain("openshell-sandbox-checksums-sha256.txt"); + expect(result.dockerCalls).toBe(""); + }); +}); diff --git a/test/install-podman-runtime.test.ts b/test/install-podman-runtime.test.ts new file mode 100644 index 0000000000..dbb781b4c0 --- /dev/null +++ b/test/install-podman-runtime.test.ts @@ -0,0 +1,333 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +const INSTALLER = path.join(import.meta.dirname, "..", "scripts", "install.sh"); + +function runSourced( + body: string, + env: Record = {}, +): { output: string; status: number | null } { + const result = spawnSync( + "bash", + [ + "-c", + ` +set -euo pipefail +source "${INSTALLER}" >/dev/null +error() { printf 'ERROR:%s\\n' "$*" >&2; exit 1; } +info() { printf 'INFO:%s\\n' "$*"; } +ok() { printf 'OK:%s\\n' "$*"; } +${body} +`, + ], + { + encoding: "utf8", + env: { ...process.env, ...env }, + }, + ); + return { output: `${result.stdout}${result.stderr}`, status: result.status }; +} + +const PODMAN_STUB = String.raw` +uname() { + if [ "\${1:-}" = "-m" ]; then printf '%s\n' "\${TEST_ARCH:-x86_64}"; else printf '%s\n' "\${TEST_OS:-Linux}"; fi +} +command_exists() { [ "$1" = "podman" ]; } +podman_socket_is_owned_by_current_user() { [ "\${TEST_SOCKET_OK:-1}" = "1" ]; } +docker() { printf 'DOCKER_INVOKED\n' >&2; exit 97; } +podman() { + case "$*" in + "--version") + printf 'podman version %s\n' "\${TEST_PODMAN_VERSION:-5.6.2}" + ;; + "--url "*" info --format json") + printf '{"host":{"arch":"%s","os":"%s","cgroupVersion":"%s","security":{"rootless":%s}}}\n' \ + "\${TEST_SERVICE_ARCH:-amd64}" "\${TEST_SERVICE_OS:-linux}" \ + "\${TEST_CGROUP:-v2}" "\${TEST_ROOTLESS:-true}" + ;; + "unshare cat /proc/self/uid_map") + printf '%b' "\${TEST_UID_MAP:-0 1000 1\\n1 100000 65536\\n}" + ;; + "unshare cat /proc/self/gid_map") + printf '%b' "\${TEST_GID_MAP:-0 1000 1\\n1 100000 65536\\n}" + ;; + *) + printf 'unexpected podman invocation: %s\n' "$*" >&2 + exit 98 + ;; + esac +} +_INSTALLER_CONTAINER_RUNTIME=podman +OPENSHELL_PODMAN_SOCKET=/run/user/1000/podman/podman.sock +`.replaceAll("\\${", "${"); + +function runMainRuntimeBoundaries(options: { + flag?: "auto" | "podman"; + envDriver?: "docker" | "podman" | ""; +}): { output: string; status: number | null } { + const driverArg = options.flag ? `--compute-driver ${JSON.stringify(options.flag)}` : ""; + return runSourced( + ` +events="" +record() { events="\${events}\${events:+,}$1"; } +uname() { + if [ "\${1:-}" = "-m" ]; then printf 'x86_64\\n'; else printf 'Linux\\n'; fi +} +load_station_vllm_conflict_helpers() { record station-conflict; } +consume_station_local_vllm_resume() { record station-resume; return 1; } +preflight_explicit_express_flags() { record station-classification; } +resolve_nemoclaw_gateway_port() { printf '18789\\n'; } +print_banner() { :; } +preflight_usage_notice_prompt() { :; } +maybe_offer_express_install() { record station-express; } +validate_station_pair_selection() { record station-selection; } +ensure_station_express_host() { record station-host; } +ensure_docker() { record docker; } +ensure_native_podman() { + record podman + OPENSHELL_PODMAN_SOCKET=/runtime/podman.sock + export OPENSHELL_PODMAN_SOCKET +} +ensure_openshell_build_deps() { record build-deps; } +bash() { record jetson; } +step() { :; } +install_nodejs() { :; } +ensure_supported_runtime() { :; } +ensure_station_express_pair() { record station-pair; } +fix_npm_permissions() { :; } +preinstall_backup_and_retire_legacy_gateway() { :; } +install_nemoclaw() { :; } +verify_nemoclaw() { _CLI_PATH=/usr/bin/true; } +require_reportable_openshell_version() { :; } +registered_sandbox_count() { printf '1\\n'; } +run_installer_host_preflight() { + installer_uses_native_podman && return 0 + record cdi +} +recover_preexisting_sandboxes_before_onboard() { + _PREEXISTING_SANDBOX_RECOVERY_RAN=true + return 0 +} +station_express_receipt_retirement_pending() { + record station-reconcile + return 1 +} +run_onboard() { record station-onboard; } +print_done() { :; } +clear_station_local_vllm_resume() { record station-local-clear; } +clear_station_resume_after_completed_onboarding() { record station-final-clear; } +command_exists() { return 1; } +_CLI_BIN=nemoclaw-test +_STATION_LOCAL_VLLM_SELECTED=1 +_PREEXISTING_SANDBOX_ORPHANED=false +main --non-interactive --yes-i-accept-third-party-software ${driverArg} +printf 'events=%s request=%s runtime=%s exported=%s\\n' \ + "$events" "$_INSTALLER_COMPUTE_DRIVER_REQUEST" \ + "$_INSTALLER_CONTAINER_RUNTIME" "$NEMOCLAW_COMPUTE_DRIVER" +`, + { NEMOCLAW_COMPUTE_DRIVER: options.envDriver ?? "" }, + ); +} + +describe("install.sh native Podman runtime selection", () => { + it("publishes and parses the installer compute-driver option", () => { + const help = spawnSync("bash", [INSTALLER, "--help"], { + encoding: "utf8", + env: { ...process.env, NO_COLOR: "1" }, + }); + const invalid = spawnSync("bash", [INSTALLER, "--compute-driver", "container"], { + encoding: "utf8", + env: { ...process.env, NO_COLOR: "1" }, + }); + + expect(help.status, help.stderr).toBe(0); + expect(help.stdout).toContain("--compute-driver "); + expect(help.stdout).toContain("NEMOCLAW_COMPUTE_DRIVER"); + expect(invalid.status).toBe(1); + expect(`${invalid.stdout}${invalid.stderr}`).toContain( + "NEMOCLAW_COMPUTE_DRIVER and --compute-driver must be one of: auto, docker, podman", + ); + }); + + it.each([ + ["unset request", {}, "", "auto", "docker"], + ["environment request", { NEMOCLAW_COMPUTE_DRIVER: " PODMAN " }, "", "podman", "podman"], + ["flag precedence", { NEMOCLAW_COMPUTE_DRIVER: "podman" }, "docker", "docker", "docker"], + ])("resolves %s with the onboard flag/environment precedence", (_name, env, flag, request, runtime) => { + const result = runSourced( + ` +INSTALLER_COMPUTE_DRIVER_FLAG=${JSON.stringify(flag)} +resolve_installer_compute_driver +printf 'request=%s runtime=%s exported=%s\\n' \ + "$_INSTALLER_COMPUTE_DRIVER_REQUEST" "$_INSTALLER_CONTAINER_RUNTIME" "$NEMOCLAW_COMPUTE_DRIVER" +`, + env, + ); + + expect(result.status, result.output).toBe(0); + expect(result.output).toContain(`request=${request} runtime=${runtime} exported=${request}`); + }); + + it("rejects unknown runtime requests before host preparation", () => { + const result = runSourced(` +INSTALLER_COMPUTE_DRIVER_FLAG=container +resolve_installer_compute_driver +printf 'MUTATED\\n' +`); + + expect(result.status).toBe(1); + expect(result.output).toContain( + "NEMOCLAW_COMPUTE_DRIVER and --compute-driver must be one of: auto, docker, podman", + ); + expect(result.output).not.toContain("MUTATED"); + }); + + it("qualifies Podman 5, its exact rootless socket, cgroups v2, and subordinate IDs", () => { + const result = runSourced(`${PODMAN_STUB} +ensure_native_podman +printf 'socket=%s\\n' "$OPENSHELL_PODMAN_SOCKET" +`); + + expect(result.status, result.output).toBe(0); + expect(result.output).toContain("Rootless Podman 5.6.2 is ready"); + expect(result.output).toContain("socket=/run/user/1000/podman/podman.sock"); + expect(result.output).not.toContain("DOCKER_INVOKED"); + }); + + it.each([ + [{ TEST_PODMAN_VERSION: "4.9.9" }, "requires Podman 5.0 or newer"], + [{ TEST_ROOTLESS: "false" }, "requires a rootless Podman API service"], + [{ TEST_CGROUP: "v1" }, "requires cgroups v2"], + [{ TEST_SERVICE_OS: "darwin" }, "service is not reporting Linux"], + [{ TEST_SERVICE_ARCH: "arm64" }, "service is not reporting x86_64"], + [{ TEST_UID_MAP: "0 1000 1\\n" }, "subordinate UID range"], + [{ TEST_GID_MAP: "0 1000 1\\n" }, "subordinate GID range"], + [{ TEST_SOCKET_OK: "0" }, "current-user-owned, non-symlink Unix socket"], + ])("fails closed when the Podman contract is incomplete: %s", (env, message) => { + const result = runSourced( + `${PODMAN_STUB} +ensure_native_podman +`, + env, + ); + + expect(result.status).toBe(1); + expect(result.output).toContain(message); + expect(result.output).not.toContain("DOCKER_INVOKED"); + }); + + it("rejects Podman outside Linux x86_64 before invoking a runtime", () => { + const result = runSourced( + `${PODMAN_STUB} +validate_installer_compute_driver_platform +`, + { TEST_ARCH: "arm64" }, + ); + + expect(result.status).toBe(1); + expect(result.output).toContain("requires Linux x86_64; detected Linux arm64"); + expect(result.output).not.toContain("DOCKER_INVOKED"); + }); + + it("skips every Docker/Station preparation boundary for Podman", () => { + const result = runSourced(` +events="" +record() { events="\${events}\${events:+,}$1"; } +ensure_native_podman() { record podman; export OPENSHELL_PODMAN_SOCKET=/runtime/podman.sock; } +ensure_openshell_build_deps() { record build-deps; } +maybe_offer_express_install() { record express; } +validate_station_pair_selection() { record station-selection; } +ensure_station_express_host() { record station-host; } +ensure_docker() { record docker; return 97; } +_INSTALLER_CONTAINER_RUNTIME=podman +prepare_installer_host +run_installer_platform_setup +run_installer_host_preflight +printf 'events=%s\\n' "$events" +`); + + expect(result.status, result.output).toBe(0); + expect(result.output).toContain("events=podman,build-deps"); + expect(result.output).not.toMatch(/events=.*(?:docker|express|station)/); + }); + + it("preserves the existing Docker preparation path for auto/default installs", () => { + const result = runSourced(` +events="" +record() { events="\${events}\${events:+,}$1"; } +maybe_offer_express_install() { record express; } +validate_station_pair_selection() { record station-selection; } +ensure_station_express_host() { record station-host; } +ensure_docker() { record docker; } +ensure_openshell_build_deps() { record build-deps; } +_INSTALLER_CONTAINER_RUNTIME=docker +prepare_installer_host +printf 'events=%s\\n' "$events" +`); + + expect(result.status, result.output).toBe(0); + expect(result.output).toContain( + "events=express,station-selection,station-host,docker,build-deps", + ); + }); + + it.each([ + ["the unset default", undefined, "", "auto"], + ["an explicit auto flag over Podman env", "auto", "podman", "auto"], + ] as const)("keeps every main-level Docker/Station boundary for %s", (_name, flag, envDriver, request) => { + const result = runMainRuntimeBoundaries({ envDriver, flag }); + + expect(result.status, result.output).toBe(0); + expect(result.output).toContain( + "events=station-conflict,station-resume,station-classification,station-express," + + "station-selection,station-host,docker,build-deps,jetson,station-pair,cdi," + + "station-reconcile,station-local-clear,station-final-clear", + ); + expect(result.output).toContain(`request=${request} runtime=docker exported=${request}`); + expect(result.output).not.toContain("events=podman"); + }); + + it("bypasses every main-level Docker/Station boundary for explicit Podman", () => { + const result = runMainRuntimeBoundaries({ + envDriver: "docker", + flag: "podman", + }); + + expect(result.status, result.output).toBe(0); + expect(result.output).toContain( + "events=podman,build-deps request=podman runtime=podman exported=podman", + ); + expect(result.output).not.toMatch( + /events=.*(?:docker|station|jetson|cdi|express|conflict|reconcile)/, + ); + }); + + it("forwards the installer selection to the actual onboard command", () => { + const result = runSourced(` +tmp="$(mktemp -d)" +trap 'rm -rf "$tmp"' EXIT +cli="$tmp/nemoclaw" +argv="$tmp/argv" +printf '#!/usr/bin/env bash\\nprintf \"%%s\\\\n\" \"$@\" > %q\\n' "$argv" >"$cli" +chmod 755 "$cli" +_CLI_BIN=nemoclaw +_CLI_PATH="$cli" +_INSTALLER_COMPUTE_DRIVER_REQUEST=podman +NON_INTERACTIVE=1 +ACCEPT_THIRD_PARTY_SOFTWARE=1 +show_usage_notice() { :; } +command_exists() { return 1; } +nemoclaw_state_dir() { printf '%s' "$tmp"; } +run_onboard +printf '%s\\n' "\$(paste -sd, "$argv")" +`); + + expect(result.status, result.output).toBe(0); + expect(result.output).toContain("onboard,--compute-driver,podman,--non-interactive"); + }); +}); diff --git a/test/installer-hash-check.test.ts b/test/installer-hash-check.test.ts index 8469e8fc7d..01e4c5f480 100644 --- a/test/installer-hash-check.test.ts +++ b/test/installer-hash-check.test.ts @@ -897,7 +897,7 @@ describe("installer hash verification", () => { expect(result.status).toBe(1); expect(result.stdout).toContain("installer operational template is not base-trusted"); expect(result.stdout).toContain( - "actual_sha256=ee10afaeb5dc1477ca4b35a70a654ed32092399dbb290266f9f138d64484f1e2", + "actual_sha256=02944169d7fe943deeabc36da0618fc68b00029de67cda5aa7b3a9a75cb1f698", ); expect(result.stdout).not.toContain("All installer hashes are current"); }); diff --git a/test/langchain-deepagents-code-fetch-proxy.test.ts b/test/langchain-deepagents-code-fetch-proxy.test.ts index 2d7f5424cb..113c5ab0ce 100644 --- a/test/langchain-deepagents-code-fetch-proxy.test.ts +++ b/test/langchain-deepagents-code-fetch-proxy.test.ts @@ -7,7 +7,7 @@ import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import YAML from "yaml"; - +import { PEM } from "../src/lib/onboard/__test-helpers__/corporate-ca-fixtures.ts"; import { prepareInitialSandboxCreatePolicy } from "../src/lib/onboard/initial-policy.ts"; import { addDarwinFcntlSealConstants } from "./helpers/darwin-fcntl-seal-fixture.ts"; @@ -34,6 +34,61 @@ function readAgentFile(name: string): string { afterEach(cleanupPackageFixtures); describe("LangChain Deep Agents Code managed fetch proxy", () => { + it("selects the managed startup corporate-CA merge for the hardened fetch transport", () => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-dcode-managed-ca-")); + try { + const openshellCa = path.join(tempDir, "openshell-ca.pem"); + const corporateCa = path.join(tempDir, "corporate-ca.pem"); + const mergedCa = path.join(tempDir, "managed-startup-ca-bundle.pem"); + const runtimeFile = path.join(tempDir, "managed-dcode-runtime.py"); + fs.writeFileSync(openshellCa, "OpenShell test trust\n"); + fs.writeFileSync(corporateCa, PEM); + fs.writeFileSync(mergedCa, `OpenShell test trust\n${PEM}`); + for (const file of [openshellCa, corporateCa, mergedCa]) fs.chmodSync(file, 0o444); + fs.writeFileSync( + runtimeFile, + addDarwinFcntlSealConstants(readAgentFile("managed-dcode-runtime.py")) + .replace("/run/nemoclaw/managed-startup-ca-bundle.pem", mergedCa) + .replace("/etc/openshell-tls/ca-bundle.pem", openshellCa), + "utf8", + ); + + const result = spawnSync( + "python3", + [ + "-c", + ` +import importlib.util +import os +from pathlib import Path + +spec = importlib.util.spec_from_file_location("nemoclaw_managed_ca_test", ${JSON.stringify(runtimeFile)}) +runtime = importlib.util.module_from_spec(spec) +spec.loader.exec_module(runtime) +runtime._MANAGED_FILE_OWNER_UID = os.getuid() + +expected = Path(${JSON.stringify(mergedCa)}) +assert runtime._MANAGED_FETCH_CA_BUNDLE_FILE == expected +descriptor, descriptor_path = runtime._managed_fetch_ca_bundle() +try: + selected = Path(descriptor_path).read_bytes() +finally: + runtime._close_managed_fetch_ca_bundle(descriptor) +corporate = Path(${JSON.stringify(corporateCa)}).read_bytes() +assert selected.endswith(corporate) +print("managed-corporate-ca-selected") +`, + ], + { encoding: "utf8", env: { PATH: process.env.PATH } }, + ); + + expect(result.status, result.stderr).toBe(0); + expect(result.stdout).toContain("managed-corporate-ca-selected"); + } finally { + fs.rmSync(tempDir, { force: true, recursive: true }); + } + }); + it("persists the root-owned proxy as the explicit fetch_url delegation", () => { const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-dcode-fetch-proxy-")); try { diff --git a/test/langchain-deepagents-code-image.test.ts b/test/langchain-deepagents-code-image.test.ts index a65671eed7..404d324032 100644 --- a/test/langchain-deepagents-code-image.test.ts +++ b/test/langchain-deepagents-code-image.test.ts @@ -159,6 +159,17 @@ describe("LangChain Deep Agents Code image contracts", () => { expect(dockerfile).toContain("ARG BASE_IMAGE\n"); expect(dockerfile).toContain("ARG NEMOCLAW_MODEL=nvidia/nemotron-3-ultra-550b-a55b"); expect(dockerfile).not.toContain("langchain-deepagents-code-sandbox-base:latest"); + expect(dockerfile).toContain( + 'timeout 10 env -i /usr/local/lib/nemoclaw/dcode-wrapper.sh -n ""', + ); + for (const probe of [ + "/usr/local/lib/nemoclaw/dcode-managed-exec /usr/bin/true", + "/usr/local/bin/dcode --version", + "/usr/local/bin/dcode.real --version", + "/usr/local/bin/deepagents-code --version", + ]) { + expect(dockerfile).toContain(`env -i ${probe}`); + } expect(dockerfile).toContain("chown root:root /sandbox/.nemoclaw"); expect(dockerfile).toContain("chmod 1755 /sandbox/.nemoclaw"); expect(dockerfile).toContain("chown -R root:root /sandbox/.nemoclaw/blueprints"); @@ -166,8 +177,11 @@ describe("LangChain Deep Agents Code image contracts", () => { expect(dockerfile.indexOf("cp -r /opt/nemoclaw-blueprint/*")).toBeLessThan( dockerfile.indexOf("chown -R root:root /sandbox/.nemoclaw/blueprints"), ); + expect(dockerfile).toContain("ARG NEMOCLAW_MANAGED_IMAGE_RUNTIME_USER=sandbox"); + expect(dockerfile).toContain("root|sandbox) ;; \\"); + expect(dockerfile).toContain("&& command -v setpriv >/dev/null 2>&1"); expect(dockerfile.trimEnd()).toMatch( - /USER sandbox\nENTRYPOINT \["\/usr\/local\/bin\/nemoclaw-start"\]\nCMD \["\/bin\/bash"\]$/, + /USER \$\{NEMOCLAW_MANAGED_IMAGE_RUNTIME_USER\}\nENTRYPOINT \["\/usr\/local\/bin\/nemoclaw-start"\]\nCMD \["\/bin\/bash"\]$/, ); }); diff --git a/test/langchain-deepagents-code-profile-build-gate.test.ts b/test/langchain-deepagents-code-profile-build-gate.test.ts index d890231c63..67ddd09ae5 100644 --- a/test/langchain-deepagents-code-profile-build-gate.test.ts +++ b/test/langchain-deepagents-code-profile-build-gate.test.ts @@ -107,8 +107,10 @@ describe("LangChain Deep Agents Code profile build gate", () => { it.each([ "NEMOCLAW_CORPORATE_CA_B64", + "NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION", + "NEMOCLAW_MANAGED_IMAGE_RUNTIME_USER", "NEMOCLAW_UPSTREAM_ENDPOINT_URL", - ])("accepts %s as a reviewed source-gate ARG (#6901)", (reviewedArg) => { + ])("accepts %s as a reviewed non-secret source-gate ARG", (reviewedArg) => { const result = runGateWithFakeDocker("expected-failure-with-marker", (fixtureRoot) => fs.appendFileSync(path.join(fixtureRoot, reviewedDockerfiles[0]), `\nARG ${reviewedArg}\n`), ); diff --git a/test/langchain-deepagents-code-proxy-launcher.test.ts b/test/langchain-deepagents-code-proxy-launcher.test.ts index 1061c8c0d6..ffb94eaed5 100644 --- a/test/langchain-deepagents-code-proxy-launcher.test.ts +++ b/test/langchain-deepagents-code-proxy-launcher.test.ts @@ -61,11 +61,19 @@ function replaceManagedProxyFileConstants(source: string, tempDir: string): stri "utf8", ); return source + .replace( + "/usr/local/lib/nemoclaw/entrypoint-env-wrapper.sh", + path.join(process.cwd(), "scripts", "lib", "entrypoint-env-wrapper.sh"), + ) .replace( 'exec /opt/venv/bin/python3 -I "$MANAGED_SESSION_SUPERVISOR" "$MANAGED_DCODE_WRAPPER" "$@"', 'exec "$MANAGED_DCODE_WRAPPER" "$@"', ) .replace("/usr/local/lib/nemoclaw/sandbox-rlimits.sh", rlimitLib) + .replaceAll( + "/run/nemoclaw/managed-startup-ca-bundle.pem", + path.join(tempDir, "trusted-ca-bundle.pem"), + ) .replace( 'readonly MANAGED_PROXY_HOST_FILE="/usr/local/share/nemoclaw/dcode-proxy-host"', `readonly MANAGED_PROXY_HOST_FILE="${path.join(tempDir, "trusted-proxy-host")}"`, diff --git a/test/managed-image-failure-diagnostics.test.ts b/test/managed-image-failure-diagnostics.test.ts new file mode 100644 index 0000000000..bc5e477f8d --- /dev/null +++ b/test/managed-image-failure-diagnostics.test.ts @@ -0,0 +1,183 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; + +import { + exportManagedImageFailureDiagnostics, + MANAGED_IMAGE_DIAGNOSTIC_EXPORT_LIMITS, +} from "../scripts/checks/export-managed-image-failure-diagnostics.ts"; + +const temporaryRoots: string[] = []; + +function fixture(): { outputRoot: string; sourceRoot: string; temporaryRoot: string } { + const temporaryRoot = fs.mkdtempSync( + path.join(os.tmpdir(), "nemoclaw-managed-image-diagnostics-"), + ); + temporaryRoots.push(temporaryRoot); + const sourceRoot = path.join(temporaryRoot, "onboard-failures"); + const outputRoot = path.join(temporaryRoot, "sanitized"); + fs.mkdirSync(sourceRoot); + fs.mkdirSync(outputRoot); + return { outputRoot, sourceRoot, temporaryRoot }; +} + +function bundle(sourceRoot: string, name = "2026-07-29T01-02-03-000Z-agent"): string { + const directory = path.join(sourceRoot, name); + fs.mkdirSync(directory); + return directory; +} + +function outputText(outputRoot: string): string { + return fs + .readdirSync(outputRoot) + .flatMap((directory) => + fs + .readdirSync(path.join(outputRoot, directory)) + .map((name) => fs.readFileSync(path.join(outputRoot, directory, name), "utf8")), + ) + .join("\n"); +} + +afterEach(() => { + for (const temporaryRoot of temporaryRoots.splice(0)) { + fs.rmSync(temporaryRoot, { recursive: true, force: true }); + } +}); + +describe("managed-image failure diagnostic export", () => { + it("fully redacts known patterns and opaque credential values before export", () => { + const { outputRoot, sourceRoot } = fixture(); + const diagnosticBundle = bundle(sourceRoot); + const opaqueCanary = "opaque-managed-image-secret-canary-928374"; + const dockerCanary = "opaque-docker-password-canary-019283"; + const knownCanary = "ghp_known_pattern_canary_abcdef012345"; + fs.writeFileSync( + path.join(diagnosticBundle, "summary.txt"), + [ + `arbitrary opaque output ${opaqueCanary}`, + `another opaque value ${dockerCanary}`, + `known token ${knownCanary}`, + "Authorization: Bearer bearer-known-canary-123456", + ].join("\n"), + ); + fs.writeFileSync( + path.join(diagnosticBundle, "openshell-gateway-relevant.log"), + `gateway reconnect failed for ${opaqueCanary}\n`, + ); + fs.writeFileSync( + path.join(diagnosticBundle, "rootfs-console.log"), + "managed startup exited before the supervisor reconnected\n", + ); + fs.writeFileSync( + path.join(diagnosticBundle, "docker-logs.txt"), + `recreated container exited while using ${opaqueCanary}\n`, + ); + fs.writeFileSync( + path.join(diagnosticBundle, "unrelated.raw"), + "this raw file must never enter the artifact\n", + ); + + const result = exportManagedImageFailureDiagnostics({ + env: { + DOCKERHUB_TOKEN: dockerCanary, + NEMOCLAW_PROVIDER_KEY: opaqueCanary, + }, + outputRoot, + sourceRoot, + }); + + expect(result).toMatchObject({ bundles: 1, files: 4 }); + const exported = outputText(outputRoot); + expect(exported).toContain(""); + expect(exported).toContain("managed startup exited before the supervisor reconnected"); + expect(exported).toContain("recreated container exited while using "); + for (const secret of [opaqueCanary, dockerCanary, knownCanary, "bearer-known-canary-123456"]) { + expect(exported).not.toContain(secret); + } + expect(exported).not.toContain("this raw file must never enter the artifact"); + expect(fs.existsSync(path.join(outputRoot, "bundle-01", "unrelated.raw"))).toBe(false); + }); + + it("fails closed on symlinks without writing a partial artifact", () => { + const { outputRoot, sourceRoot, temporaryRoot } = fixture(); + const diagnosticBundle = bundle(sourceRoot); + const target = path.join(temporaryRoot, "outside.txt"); + fs.writeFileSync(target, "raw secret outside the diagnostic root\n"); + fs.symlinkSync(target, path.join(diagnosticBundle, "summary.txt")); + + expect(() => exportManagedImageFailureDiagnostics({ outputRoot, sourceRoot })).toThrow( + /not a regular file/, + ); + expect(fs.readdirSync(outputRoot)).toEqual([]); + }); + + it("fails closed on non-file diagnostic entries", () => { + const { outputRoot, sourceRoot } = fixture(); + const diagnosticBundle = bundle(sourceRoot); + fs.mkdirSync(path.join(diagnosticBundle, "rootfs-console.log")); + + expect(() => exportManagedImageFailureDiagnostics({ outputRoot, sourceRoot })).toThrow( + /not a regular file/, + ); + expect(fs.readdirSync(outputRoot)).toEqual([]); + }); + + it("bounds bundle count, file count, individual files, and total exported bytes", () => { + const { outputRoot, sourceRoot } = fixture(); + const largeButReadable = "safe reconnect € evidence\n".repeat(2_500); + const tooLarge = `raw-oversized-canary\n${"x".repeat( + MANAGED_IMAGE_DIAGNOSTIC_EXPORT_LIMITS.maxSourceFileBytes, + )}`; + for (let index = 0; index < MANAGED_IMAGE_DIAGNOSTIC_EXPORT_LIMITS.maxBundles + 2; index++) { + const diagnosticBundle = bundle( + sourceRoot, + `2026-07-29T01-02-${String(index).padStart(2, "0")}-000Z-agent`, + ); + for (const name of [ + "openshell-gateway-relevant.log", + "openshell-gateway-tail.log", + "summary.txt", + ]) { + fs.writeFileSync(path.join(diagnosticBundle, name), largeButReadable); + } + fs.writeFileSync(path.join(diagnosticBundle, "rootfs-console.log"), tooLarge); + } + + const result = exportManagedImageFailureDiagnostics({ outputRoot, sourceRoot }); + const outputFiles = fs + .readdirSync(outputRoot) + .flatMap((directory) => + fs + .readdirSync(path.join(outputRoot, directory)) + .map((name) => path.join(outputRoot, directory, name)), + ); + const outputBytes = outputFiles.reduce( + (total, filePath) => total + fs.statSync(filePath).size, + 0, + ); + + expect(fs.readdirSync(outputRoot).length).toBeLessThanOrEqual( + MANAGED_IMAGE_DIAGNOSTIC_EXPORT_LIMITS.maxBundles, + ); + expect(fs.readdirSync(outputRoot)).toHaveLength(result.bundles); + expect(outputFiles.length).toBeLessThanOrEqual(MANAGED_IMAGE_DIAGNOSTIC_EXPORT_LIMITS.maxFiles); + expect(outputFiles.every((filePath) => fs.statSync(filePath).isFile())).toBe(true); + expect( + outputFiles.every( + (filePath) => + fs.statSync(filePath).size <= MANAGED_IMAGE_DIAGNOSTIC_EXPORT_LIMITS.maxOutputFileBytes, + ), + ).toBe(true); + expect(outputBytes).toBeLessThanOrEqual( + MANAGED_IMAGE_DIAGNOSTIC_EXPORT_LIMITS.maxTotalOutputBytes, + ); + expect(result.bytes).toBe(outputBytes); + const exported = outputText(outputRoot); + expect(exported).toContain("[omitted: source exceeded"); + expect(exported).not.toContain("raw-oversized-canary"); + }); +}); diff --git a/test/managed-image-publication-workflow.test.ts b/test/managed-image-publication-workflow.test.ts new file mode 100644 index 0000000000..af83ffa4c0 --- /dev/null +++ b/test/managed-image-publication-workflow.test.ts @@ -0,0 +1,1455 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { afterEach, describe, expect, it, vi } from "vitest"; +import YAML from "yaml"; +import { parseManagedImageDirectE2eInputs } from "../scripts/checks/run-managed-image-direct-e2e.ts"; +import { + managedImageOpenShellBasePolicyPath, + managedImageOpenShellCommittedProbe, + managedImageOpenShellProbe, + parseManagedImageOpenShellE2eInputs, +} from "../scripts/checks/run-managed-image-openshell-e2e.ts"; + +type Step = { + env?: Record; + id?: string; + if?: string; + name?: string; + run?: string; + uses?: string; + with?: Record; +}; + +type MatrixEntry = { + agent?: string; + artifact_platform?: string; + base_alias?: string; + base_image?: string; + base_repository?: string; + display_name?: string; + dockerfile?: string; + image?: string; + platform?: string; + required_binary?: string; +}; + +type Job = { + if?: string; + name?: string; + needs?: string | string[]; + permissions?: Record; + "runs-on"?: string; + steps?: Step[]; + strategy?: { + "fail-fast"?: boolean; + matrix?: { include?: MatrixEntry[] }; + }; + "timeout-minutes"?: number; + uses?: string; +}; + +type Workflow = { + concurrency?: { + "cancel-in-progress"?: string | boolean; + group?: string; + }; + env?: Record; + jobs?: Record; + on?: { + pull_request?: { + branches?: string[]; + paths?: string[]; + }; + push?: { + paths?: string[]; + }; + workflow_call?: unknown; + }; + permissions?: Record; +}; + +const repoRoot = path.resolve(import.meta.dirname, ".."); +const fullShaAction = /^[^@]+@[0-9a-f]{40}$/iu; +const AsyncFunction = Object.getPrototypeOf(async () => undefined).constructor as new ( + ...parameters: string[] +) => (...args: unknown[]) => Promise; +const managedArtifactInputPaths = [ + ".dockerignore", + ".github/workflows/managed-images.yaml", + "Dockerfile", + "agents/**", + "ci/npm-audit-exceptions.json", + "nemoclaw/**", + "nemoclaw-blueprint/**", + "scripts/**", + "src/lib/actions/sandbox/openshell-child-visible-credentials.v*.json", + "src/lib/core/json-types.ts", + "src/lib/core/ports.ts", + "src/lib/messaging/**", + "src/lib/onboard/managed-startup/**", + "src/lib/security/credential-hash.ts", + "src/lib/state/paths.ts", + "src/lib/state/state-root.ts", + "src/lib/tool-disclosure.ts", + "tools/mcp-tool-discovery-runtime/**", + "tsconfig.runtime-preloads.json", +] as const; +const managedPrRuntimeInputPaths = [...managedArtifactInputPaths, "src/lib/**"] as const; + +function readWorkflow(file: string): Workflow { + return YAML.parse( + fs.readFileSync(path.join(repoRoot, ".github", "workflows", file), "utf8"), + ) as Workflow; +} + +function required(value: T | undefined, message: string): T { + return ( + value ?? + (() => { + throw new Error(message); + })() + ); +} + +function step(job: Job, name: string): Step { + return required( + job.steps?.find((candidate) => candidate.name === name), + `managed-image workflow is missing '${name}'`, + ); +} + +function managedBuilder(workflow: Workflow): Job { + return required( + workflow.jobs?.["build-and-validate"], + "managed-image workflow is missing its build-and-validate matrix", + ); +} + +function managedPromoter(workflow: Workflow): Job { + return required( + workflow.jobs?.promote, + "managed-image workflow is missing its aggregate promoter", + ); +} + +function managedPrBuilder(workflow: Workflow): Job { + return required( + workflow.jobs?.["pr-build-and-entrypoint"], + "managed-image workflow is missing its pull-request managed-startup matrix", + ); +} + +afterEach(() => { + vi.useRealTimers(); + vi.unstubAllEnvs(); +}); + +function publicationBoundaryErrors(baseWorkflow: Workflow, managedWorkflow: Workflow): string[] { + const triggerPaths = baseWorkflow.on?.push?.paths ?? []; + const caller = required( + baseWorkflow.jobs?.["publish-managed-images"], + "base-image workflow is missing the managed-image publisher", + ); + const builder = managedBuilder(managedWorkflow); + const promoter = managedPromoter(managedWorkflow); + const buildSteps = builder.steps ?? []; + const promoteSteps = promoter.steps ?? []; + const build = step(builder, "Build and push managed image by digest"); + const base = step(builder, "Validate exact base image contract"); + const validate = step(builder, "Validate exact managed image before promotion"); + const candidate = step(builder, "Export validated managed image candidate"); + const barrier = step(promoter, "Validate complete managed image candidate set"); + const promote = step(promoter, "Promote validated managed image aliases"); + const workflowSource = JSON.stringify(managedWorkflow); + const validationMarkers = [ + 'mktemp -d "$RUNNER_TEMP/anonymous-docker-XXXXXX"', + 'DOCKER_CONFIG="$anonymous_config" docker pull --platform "$PLATFORM" "$reference"', + "bootstrap the GHCR package", + "/opt/nemoclaw-blueprint/blueprint.yaml", + "/usr/local/share/nemoclaw/node-tar-inventory.json", + "/usr/local/share/nemoclaw/corporate-ca.pem", + 'entry.status !== "fixed"', + '--entrypoint "$REQUIRED_BINARY"', + "io.nvidia.nemoclaw.managed-image.contract", + "io.nvidia.nemoclaw.managed-image.startup-profile", + "io.nvidia.nemoclaw.managed-image.capabilities", + "io.nvidia.nemoclaw.managed-image.cohort", + "^ghrun-[1-9][0-9]{0,19}-[1-9][0-9]{0,9}$", + "NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION", + "@openclaw/diagnostics-otel", + "@openclaw/brave-plugin", + "@openclaw/discord", + "@tencent-weixin/openclaw-weixin", + "@openclaw/slack", + "@openclaw/whatsapp", + "@openclaw/msteams", + "microsoft-teams-apps", + "config.plugins?.entries?.[id]?.enabled !== false", + 'config["platforms"].get(name) != {"enabled": False}', + "run-managed-image-direct-e2e.ts", + '--agent "$AGENT"', + '--image "$reference"', + '--platform "$PLATFORM"', + ]; + const candidateMarkers = [ + 'phase: "candidate"', + '--arg ref "$GITHUB_REF"', + '--arg release "$release_tag"', + '--arg cohort "$PUBLICATION_COHORT"', + "cohort: $cohort", + "^ghrun-[1-9][0-9]{0,19}-[1-9][0-9]{0,9}$", + 'and (has("aliases") | not)', + ]; + const barrierMarkers = [ + "expected exactly three managed image candidate artifacts", + "managed-image-candidate-openclaw-linux-amd64", + "managed-image-candidate-hermes-linux-amd64", + "managed-image-candidate-langchain-deepagents-code-linux-amd64", + "length == 3", + "([.[].source.repository] | unique) == [$repository]", + "([.[].source.revision] | unique) == [$revision]", + "([.[].source.ref] | unique) == [$ref]", + "([.[].source.cohort] | unique) == [$cohort]", + "^ghrun-[1-9][0-9]{0,19}-[1-9][0-9]{0,9}$", + "([.[].run] | unique) == [{id: $runId, attempt: $runAttempt}]", + 'and .release == (if $release == "" then null else $release end)', + ]; + const promotionMarkers = [ + 'cohort="ghrun-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"', + "^ghrun-[1-9][0-9]{0,19}-[1-9][0-9]{0,9}$", + 'cohort_alias="${image}:cohort-${cohort}"', + 'docker buildx imagetools create --tag "$cohort_alias" "$reference"', + 'docker buildx imagetools inspect "$cohort_alias" --raw', + 'openclaw_candidate="$(jq -ce \'.[] | select(.agent == "openclaw")\' "$CANDIDATE_SET")"', + 'consumer_aliases=("${openclaw_image}:${GITHUB_SHA}")', + 'consumer_aliases+=("${openclaw_image}:${release_tag}")', + 'docker buildx imagetools create "${consumer_tag_args[@]}" "$openclaw_reference"', + 'docker buildx imagetools inspect "$reference" --raw', + 'docker buildx imagetools inspect "$alias" --raw', + 'cmp -s "$exact_raw" "$cohort_raw"', + 'cmp -s "$openclaw_exact_raw" "$alias_raw"', + ]; + const buildIndex = buildSteps.indexOf(build); + const validateIndex = buildSteps.indexOf(validate); + const candidateIndex = buildSteps.indexOf(candidate); + const barrierIndex = promoteSteps.indexOf(barrier); + const promoteIndex = promoteSteps.indexOf(promote); + const promotionSource = promote.run ?? ""; + const stageIndex = promotionSource.indexOf( + "# Stage all three unique cohort aliases before any consumer pointer", + ); + const verifyIndex = promotionSource.indexOf( + "# Verify every staged cohort alias against its exact validated", + ); + const pointerIndex = promotionSource.indexOf( + 'consumer_aliases=("${openclaw_image}:${GITHUB_SHA}")', + ); + + return [ + ...managedArtifactInputPaths + .filter((input) => !triggerPaths.includes(input)) + .map((input) => `managed image trigger is missing ${input}`), + ...(baseWorkflow.concurrency?.group === "base-image-${{ github.ref }}" + ? [] + : ["base image concurrency must be scoped by github.ref"]), + ...(baseWorkflow.concurrency?.["cancel-in-progress"] === + "${{ !startsWith(github.ref, 'refs/tags/v') }}" + ? [] + : ["v* release runs must never be cancelled"]), + ...(caller.if?.includes("inputs.openclaw_version == ''") + ? [] + : ["custom OpenClaw base builds must not publish managed images"]), + ...(build.with?.outputs === + "type=image,name=${{ env.REGISTRY }}/${{ matrix.image }},push-by-digest=true,name-canonical=true,push=true" && + build.with.push === undefined && + build.with.tags === undefined + ? [] + : ["managed images must be pushed by digest without consumer tags"]), + ...(!workflowSource.includes("GITHUB_SHA:0:8") && !workflowSource.includes("format=short") + ? [] + : ["managed image handoff and aliases must not use short source SHAs"]), + ...(base.run?.includes('.reference == (.image + "@" + .digest)') && + base.run.includes(".sourceRevision == $revision") && + base.run.includes(".run == {id: $runId, attempt: $runAttempt}") + ? [] + : ["managed image build must consume the same-run exact base digest contract"]), + ...validationMarkers + .filter((marker) => !validate.run?.includes(marker)) + .map((marker) => `exact managed image validation is missing ${marker}`), + ...candidateMarkers + .filter((marker) => !candidate.run?.includes(marker)) + .map((marker) => `managed image candidate contract is missing ${marker}`), + ...barrierMarkers + .filter((marker) => !barrier.run?.includes(marker)) + .map((marker) => `managed image all-three barrier is missing ${marker}`), + ...promotionMarkers + .filter((marker) => !promote.run?.includes(marker)) + .map((marker) => `managed image promotion is missing ${marker}`), + ...(buildIndex >= 0 && buildIndex < validateIndex && validateIndex < candidateIndex + ? [] + : ["managed image validation must finish before candidate publication"]), + ...(promoter.needs === "build-and-validate" && + barrierIndex >= 0 && + barrierIndex < promoteIndex && + promoter.strategy === undefined + ? [] + : ["all matrix validations must finish before aggregate alias promotion"]), + ...(stageIndex >= 0 && stageIndex < verifyIndex && verifyIndex < pointerIndex + ? [] + : ["all cohort aliases must be staged and verified before the OpenClaw pointer moves"]), + ...(promotionSource.includes('consumer_aliases=("${image}:${GITHUB_SHA}")') || + promotionSource.includes('aliases=("${image}:${GITHUB_SHA}")') + ? ["each agent must not receive an independent source-revision pointer"] + : []), + ]; +} + +describe("complete managed-image publication workflow", () => { + it("starts after exact base contracts with complete main triggers and release-safe concurrency (#7744)", () => { + const baseWorkflow = readWorkflow("base-image.yaml"); + const managedWorkflow = readWorkflow("managed-images.yaml"); + const publisher = required( + baseWorkflow.jobs?.["publish-managed-images"], + "base-image workflow is missing the managed-image publisher", + ); + + expect(publicationBoundaryErrors(baseWorkflow, managedWorkflow)).toEqual([]); + expect(publisher).toMatchObject({ + needs: ["build-and-push", "build-and-push-openclaw"], + permissions: { + contents: "read", + packages: "write", + }, + uses: "./.github/workflows/managed-images.yaml", + }); + expect(publisher.if).toContain("github.repository == 'NVIDIA/NemoClaw'"); + expect(publisher.if).toContain("github.ref == 'refs/heads/main'"); + expect(publisher.if).toContain("startsWith(github.ref, 'refs/tags/v')"); + + const qemuPublisher = required( + baseWorkflow.jobs?.["build-and-push"], + "base-image workflow is missing Hermes and DCode publishers", + ); + expect(step(qemuPublisher, "Build and push").id).toBe("build"); + expect(step(qemuPublisher, "Export managed base image contract").run).toContain( + 'reference="${IMAGE}@${DIGEST}"', + ); + expect(step(qemuPublisher, "Upload managed base image contract").with?.name).toBe( + "managed-base-${{ matrix.agent }}", + ); + + const openClawPublisher = required( + baseWorkflow.jobs?.["build-and-push-openclaw"], + "base-image workflow is missing the OpenClaw manifest publisher", + ); + expect(step(openClawPublisher, "Create and verify multi-platform manifest").id).toBe( + "manifest", + ); + expect(step(openClawPublisher, "Export managed base image contract").env?.DIGEST).toBe( + "${{ steps.manifest.outputs.digest }}", + ); + expect(step(openClawPublisher, "Upload managed base image contract").with?.name).toBe( + "managed-base-openclaw", + ); + }); + + it("publishes every complete agent image for the initial Linux x86_64 contract (#7744)", () => { + const workflow = readWorkflow("managed-images.yaml"); + const builder = managedBuilder(workflow); + const promoter = managedPromoter(workflow); + + expect(Object.keys(workflow.on ?? {}).sort()).toEqual(["pull_request", "workflow_call"]); + expect(workflow.permissions).toEqual({ + contents: "read", + packages: "read", + }); + expect(builder.if).toBe("github.event_name != 'pull_request'"); + expect(builder.permissions).toEqual({ contents: "read", packages: "write" }); + expect(promoter.if).toBe("github.event_name != 'pull_request'"); + expect(promoter.permissions).toEqual({ contents: "read", packages: "write" }); + expect(builder["runs-on"]).toBe("ubuntu-24.04"); + expect(builder["timeout-minutes"]).toBe(120); + expect(builder.strategy?.["fail-fast"]).toBe(false); + expect(builder.strategy?.matrix?.include).toEqual([ + { + agent: "openclaw", + display_name: "OpenClaw", + dockerfile: "Dockerfile", + base_image: "nvidia/nemoclaw/sandbox-base", + image: "nvidia/nemoclaw/openclaw-sandbox", + platform: "linux/amd64", + artifact_platform: "linux-amd64", + required_binary: "/usr/local/bin/openclaw", + }, + { + agent: "hermes", + display_name: "Hermes", + dockerfile: "agents/hermes/Dockerfile", + base_image: "nvidia/nemoclaw/hermes-sandbox-base", + image: "nvidia/nemoclaw/hermes-sandbox", + platform: "linux/amd64", + artifact_platform: "linux-amd64", + required_binary: "/usr/local/bin/hermes", + }, + { + agent: "langchain-deepagents-code", + display_name: "Deep Agents Code", + dockerfile: "agents/langchain-deepagents-code/Dockerfile", + base_image: "nvidia/nemoclaw/langchain-deepagents-code-sandbox-base", + image: "nvidia/nemoclaw/langchain-deepagents-code-sandbox", + platform: "linux/amd64", + artifact_platform: "linux-amd64", + required_binary: "/usr/local/bin/dcode", + }, + ]); + expect(promoter).toMatchObject({ + needs: "build-and-validate", + "runs-on": "ubuntu-24.04", + "timeout-minutes": 30, + }); + expect(promoter.strategy).toBeUndefined(); + }); + + it("rejects lightweight and unverified release tags before managed alias promotion", async () => { + const workflow = readWorkflow("managed-images.yaml"); + const promoter = managedPromoter(workflow); + const promoteSteps = promoter.steps ?? []; + const verify = step(promoter, "Verify release tag before managed image promotion"); + const barrier = step(promoter, "Validate complete managed image candidate set"); + const login = step(promoter, "Log in to GHCR for promotion"); + const promotion = step(promoter, "Promote validated managed image aliases"); + const script = required(verify.with?.script as string | undefined, "tag verifier is missing"); + const releaseTag = "v0.0.98"; + const releaseRevision = "b".repeat(40); + const tagObjectSha = "a".repeat(40); + const runVerify = (getRef: ReturnType, getTag: ReturnType) => + new AsyncFunction("github", "context", "core", script)( + { rest: { git: { getRef, getTag } } }, + { repo: { owner: "NVIDIA", repo: "NemoClaw" } }, + { info: vi.fn() }, + ); + vi.stubEnv("RELEASE_TAG", releaseTag); + vi.stubEnv("RELEASE_REVISION", releaseRevision); + + expect(verify.if).toBe("startsWith(github.ref, 'refs/tags/v')"); + expect(verify.uses).toBe("actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3"); + expect(promoteSteps.indexOf(barrier)).toBeLessThan(promoteSteps.indexOf(verify)); + expect(promoteSteps.indexOf(verify)).toBeLessThan(promoteSteps.indexOf(login)); + expect(promoteSteps.indexOf(verify)).toBeLessThan(promoteSteps.indexOf(promotion)); + + const lightweightGetTag = vi.fn(); + await expect( + runVerify( + vi.fn().mockResolvedValue({ + data: { object: { sha: releaseRevision, type: "commit" } }, + }), + lightweightGetTag, + ), + ).rejects.toThrow(`Release tag ${releaseTag} must be annotated`); + expect(lightweightGetTag).not.toHaveBeenCalled(); + + vi.useFakeTimers(); + const unverifiedGetTag = vi.fn().mockResolvedValue({ + data: { + object: { sha: releaseRevision, type: "commit" }, + tag: releaseTag, + verification: { verified: false, reason: "unsigned" }, + }, + }); + const unverified = expect( + runVerify( + vi.fn().mockResolvedValue({ + data: { object: { sha: tagObjectSha, type: "tag" } }, + }), + unverifiedGetTag, + ), + ).rejects.toThrow(`Release tag ${releaseTag} is not GitHub-Verified (unsigned)`); + await vi.runAllTimersAsync(); + await unverified; + expect(unverifiedGetTag).toHaveBeenCalledTimes(10); + }); + + it("builds all three images and exercises managed startup and OpenShell for every relevant pull request", () => { + const workflow = readWorkflow("managed-images.yaml"); + const prBuilder = managedPrBuilder(workflow); + const steps = prBuilder.steps ?? []; + const build = step(prBuilder, "Build PR managed image locally"); + const contract = step(prBuilder, "Validate exact PR managed image contract"); + const gate = step(prBuilder, "Exercise managed startup root stdin and hold"); + const dependencies = step(prBuilder, "Install managed-image OpenShell harness dependencies"); + const install = step(prBuilder, "Install pinned OpenShell runtime"); + const openshell = step(prBuilder, "Exercise exact PR image through real OpenShell"); + const exportDiagnostics = step( + prBuilder, + "Export sanitized managed-image OpenShell failure diagnostics", + ); + const diagnostics = step(prBuilder, "Upload managed-image OpenShell failure diagnostics"); + const prPaths = workflow.on?.pull_request?.paths ?? []; + + expect(prBuilder).toMatchObject({ + if: "github.event_name == 'pull_request'", + name: "PR build, managed startup, and OpenShell (${{ matrix.display_name }})", + permissions: { contents: "read", packages: "read" }, + "runs-on": "ubuntu-24.04", + "timeout-minutes": 90, + }); + expect(prBuilder.strategy?.["fail-fast"]).toBe(false); + expect(prBuilder.strategy?.matrix?.include).toEqual([ + { + agent: "openclaw", + display_name: "OpenClaw", + dockerfile: "Dockerfile", + base_alias: "ghcr.io/nvidia/nemoclaw/sandbox-base:latest", + base_repository: "ghcr.io/nvidia/nemoclaw/sandbox-base", + image: "nemoclaw-managed-pr/openclaw", + }, + { + agent: "hermes", + display_name: "Hermes", + dockerfile: "agents/hermes/Dockerfile", + base_alias: "ghcr.io/nvidia/nemoclaw/hermes-sandbox-base:latest", + base_repository: "ghcr.io/nvidia/nemoclaw/hermes-sandbox-base", + image: "nemoclaw-managed-pr/hermes", + }, + { + agent: "langchain-deepagents-code", + display_name: "Deep Agents Code", + dockerfile: "agents/langchain-deepagents-code/Dockerfile", + base_alias: "ghcr.io/nvidia/nemoclaw/langchain-deepagents-code-sandbox-base:latest", + base_repository: "ghcr.io/nvidia/nemoclaw/langchain-deepagents-code-sandbox-base", + image: "nemoclaw-managed-pr/langchain-deepagents-code", + }, + ]); + expect(workflow.on?.pull_request?.branches).toBeUndefined(); + expect(managedPrRuntimeInputPaths.filter((path) => !prPaths.includes(path))).toEqual([]); + for (const action of steps.filter((candidate) => candidate.uses)) { + expect(action.uses, action.name).toMatch(fullShaAction); + } + expect(steps.some((candidate) => candidate.name?.includes("Log in"))).toBe(false); + expect(step(prBuilder, "Set up Node.js").with?.["node-version"]).toBe("22.19.0"); + const base = step(prBuilder, "Resolve exact linux/amd64 PR base"); + for (const marker of [ + 'docker buildx imagetools inspect "$BASE_ALIAS" --raw', + '.platform.os == "linux"', + '.platform.architecture == "amd64"', + "if length == 1 then .[0].digest", + 'reference="${BASE_REPOSITORY}@${digest}"', + 'docker buildx imagetools inspect "$reference" --raw', + 'actual="sha256:$(sha256sum "$exact_raw"', + 'printf \'ref=%s\\n\' "$reference" >> "$GITHUB_OUTPUT"', + ]) { + expect(base.run).toContain(marker); + } + expect(build.with).toMatchObject({ + context: ".", + file: "${{ matrix.dockerfile }}", + platforms: "linux/amd64", + load: true, + push: false, + tags: "${{ matrix.image }}:${{ github.sha }}", + provenance: false, + sbom: false, + "build-args": + "BASE_IMAGE=${{ steps.base.outputs.ref }}\nNEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION=1\nNEMOCLAW_MANAGED_IMAGE_RUNTIME_USER=root\n", + }); + expect(build.with?.outputs).toBeUndefined(); + expect(build.with?.["cache-to"]).toBeUndefined(); + expect(steps.indexOf(build)).toBeLessThan(steps.indexOf(contract)); + expect(steps.indexOf(contract)).toBeLessThan(steps.indexOf(dependencies)); + expect(steps.indexOf(dependencies)).toBeLessThan(steps.indexOf(gate)); + expect(steps.indexOf(gate)).toBeLessThan(steps.indexOf(install)); + expect(steps.indexOf(install)).toBeLessThan(steps.indexOf(openshell)); + expect(contract).toMatchObject({ + id: "contract", + env: { + AGENT: "${{ matrix.agent }}", + IMAGE_REFERENCE: "${{ matrix.image }}:${{ github.sha }}", + PLATFORM: "linux/amd64", + PUBLICATION_COHORT: "ghrun-${{ github.run_id }}-${{ github.run_attempt }}", + }, + }); + for (const marker of [ + 'docker image inspect "$IMAGE_REFERENCE"', + "expected one local image identity", + "^sha256:[0-9a-f]{64}$", + "docker image inspect --format '{{.Id}}' \"$image_id\"", + '((.[0].Config.User // "") as $user |', + "io.nvidia.nemoclaw.agent", + "io.nvidia.nemoclaw.managed-image.contract", + "io.nvidia.nemoclaw.managed-image.platform", + "io.nvidia.nemoclaw.managed-image.startup-profile", + "io.nvidia.nemoclaw.managed-image.capabilities", + "io.nvidia.nemoclaw.managed-image.cohort", + "org.opencontainers.image.revision", + '.[0].Config.Labels["org.opencontainers.image.revision"] == $revision', + "printf 'reference=%s\\n' \"$image_id\"", + ]) { + expect(contract.run).toContain(marker); + } + expect(contract.run).not.toContain(".Config.Entrypoint"); + expect(contract.run).not.toContain(".Config.Cmd"); + expect(gate.env?.IMAGE_REFERENCE).toBe("${{ steps.contract.outputs.reference }}"); + expect(gate.env?.IMAGE_REFERENCE).not.toContain("matrix.image"); + for (const marker of [ + "run-managed-image-direct-e2e.ts", + '--agent "$AGENT"', + '--image "$IMAGE_REFERENCE"', + "--platform linux/amd64", + ]) { + expect(gate.run).toContain(marker); + } + expect(gate.run).not.toContain("NEMOCLAW_STARTUP_PROFILE_B64"); + expect(gate.run).not.toContain("NEMOCLAW_CORPORATE_CA_B64"); + expect(gate.run).toContain("npx --no-install tsx"); + + expect(dependencies.run).toContain("npm ci --ignore-scripts"); + expect(dependencies.run).toContain("npm run build:policy-boundary"); + expect(steps.indexOf(dependencies)).toBeLessThan(steps.indexOf(gate)); + expect(install.env).toMatchObject({ + NEMOCLAW_NON_INTERACTIVE: "1", + NEMOCLAW_OPENSHELL_MIN_VERSION: "0.0.85", + NEMOCLAW_OPENSHELL_MAX_VERSION: "0.0.85", + NEMOCLAW_OPENSHELL_PIN_VERSION: "0.0.85", + }); + for (const marker of [ + "env -u DOCKER_CONFIG", + "-u DOCKERHUB_USERNAME", + "-u DOCKERHUB_TOKEN", + "-u NVIDIA_API_KEY", + "-u NVIDIA_INFERENCE_API_KEY", + "-u GITHUB_TOKEN", + "-u GH_TOKEN", + "bash scripts/install-openshell.sh", + ]) { + expect(install.run).toContain(marker); + } + expect(openshell.env).toEqual({ + AGENT: "${{ matrix.agent }}", + IMAGE_REFERENCE: "${{ steps.contract.outputs.reference }}", + SANDBOX_NAME: "nemoclaw-pr-${{ matrix.agent }}", + }); + for (const marker of [ + 'export PATH="$HOME/.local/bin:$PATH"', + "npx tsx scripts/checks/run-managed-image-openshell-e2e.ts", + '--agent "$AGENT"', + '--image "$IMAGE_REFERENCE"', + '--sandbox "$SANDBOX_NAME"', + ]) { + expect(openshell.run).toContain(marker); + } + expect(steps.indexOf(openshell)).toBeLessThan(steps.indexOf(exportDiagnostics)); + expect(exportDiagnostics).toMatchObject({ + id: "managed_image_openshell_diagnostics", + if: "failure()", + run: expect.stringContaining('diagnostics_root="$HOME/.nemoclaw/onboard-failures"'), + }); + expect(exportDiagnostics.run).toContain( + 'sanitized_root="$(mktemp -d "$RUNNER_TEMP/managed-image-openshell-diagnostics.XXXXXX")"', + ); + expect(exportDiagnostics.run).toContain( + "scripts/checks/export-managed-image-failure-diagnostics.ts", + ); + expect(exportDiagnostics.run).toContain('--source "$diagnostics_root"'); + expect(exportDiagnostics.run).toContain('--output "$sanitized_root"'); + expect(exportDiagnostics.run).toContain( + 'printf \'path=%s\\n\' "$sanitized_root" >> "$GITHUB_OUTPUT"', + ); + expect(exportDiagnostics.run).not.toContain( + 'printf \'path=%s\\n\' "$diagnostics_root" >> "$GITHUB_OUTPUT"', + ); + expect(exportDiagnostics.run).toContain("sed -n '1,160p' \"$summary\""); + expect(exportDiagnostics.run).toContain('find "$sanitized_root"'); + expect(exportDiagnostics.run).not.toContain('find "$diagnostics_root"'); + expect(steps.indexOf(openshell)).toBeLessThan(steps.indexOf(diagnostics)); + expect(diagnostics).toMatchObject({ + if: "${{ failure() && steps.managed_image_openshell_diagnostics.outcome == 'success' }}", + uses: "actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a", + with: { + name: "managed-image-openshell-pr-${{ matrix.agent }}-${{ github.run_id }}-${{ github.run_attempt }}", + path: "${{ steps.managed_image_openshell_diagnostics.outputs.path }}", + "include-hidden-files": true, + "if-no-files-found": "ignore", + "retention-days": 14, + }, + }); + expect(String(diagnostics.with?.path)).not.toContain("onboard-failures"); + const harness = fs.readFileSync( + path.join(repoRoot, "scripts", "checks", "run-managed-image-openshell-e2e.ts"), + "utf8", + ); + for (const marker of [ + "onboard.startGatewayForRecovery({", + "await assertGatewayPortAvailable()", + "process.env.XDG_CONFIG_HOME", + "process.env.XDG_STATE_HOME", + "process.env.OPENSHELL_DOCKER_NETWORK_NAME", + "const deadline = Date.now() + 240_000", + "createChild.signalCode", + 'killSignal: "SIGKILL"', + "prepareSandboxCreateManagedImageLaunch", + "prepareInitialSandboxCreatePolicy(", + '"--policy",', + "initialSandboxPolicy.policyPath", + "openshellArgv: onboard.openshellArgv", + '"sandbox",\n "exec"', + 'commandResult(["docker", "inspect", candidate], env)', + "record.Image === input.image", + "label=openshell.ai/managed-by=openshell", + "Object.hasOwn(record.NetworkSettings?.Networks ?? {}, networkName)", + 'child.kill("SIGKILL")', + "resolved.exactIds.length === 1", + '["docker", "container", "inspect", cleanupContainerId]', + '["docker", "network", "inspect", networkName]', + "15_000", + "managedStartupE2eProfile(input.agent, false, true, true)", + "collectDockerGpuPatchDiagnostics(", + "collectSandboxCreateFailureDiagnostics(", + ]) { + expect(harness).toContain(marker); + } + const probeFunctionStart = harness.indexOf("async function waitForSandboxProbe"); + const supervisorReconnect = harness.indexOf( + "startupPatch.waitForSupervisorReconnectIfNeeded();", + probeFunctionStart, + ); + const exactHealthProbe = harness.indexOf( + "const result = runProbe(healthProbe", + supervisorReconnect, + ); + const transactionCommit = harness.indexOf("startupPatch.commitAfterReady();", exactHealthProbe); + const transactionAbsenceProbe = harness.indexOf("committedProbe,", transactionCommit); + const rollback = harness.indexOf( + "startupPatch.rollbackManagedStartupAfterCreateFailure();", + transactionAbsenceProbe, + ); + const failureDiagnosticCapture = harness.indexOf( + "captureFailureDiagnosticsBeforeRollback(onboard, input, createLog, startupPatch, error);", + transactionAbsenceProbe, + ); + expect(probeFunctionStart).toBeGreaterThanOrEqual(0); + expect(supervisorReconnect).toBeGreaterThan(probeFunctionStart); + expect(exactHealthProbe).toBeGreaterThan(supervisorReconnect); + expect(transactionCommit).toBeGreaterThan(exactHealthProbe); + expect(transactionAbsenceProbe).toBeGreaterThan(transactionCommit); + expect(failureDiagnosticCapture).toBeGreaterThan(transactionAbsenceProbe); + expect(failureDiagnosticCapture).toBeLessThan(rollback); + expect(rollback).toBeGreaterThan(transactionAbsenceProbe); + expect(harness).not.toContain("Dockerfile"); + expect(harness).not.toContain("packages: write"); + }); + + it("fails the real OpenShell harness closed on mutable images and probes each shipped agent", () => { + const image = `sha256:${"a".repeat(64)}`; + expect( + parseManagedImageOpenShellE2eInputs([ + "--agent", + "openclaw", + "--image", + image, + "--sandbox", + "nemoclaw-pr-openclaw", + ]), + ).toEqual({ + agent: "openclaw", + image, + sandbox: "nemoclaw-pr-openclaw", + }); + expect(() => + parseManagedImageOpenShellE2eInputs([ + "--agent", + "hermes", + "--image", + "ghcr.io/nvidia/nemoclaw/hermes-sandbox:latest", + "--sandbox", + "nemoclaw-pr-hermes", + ]), + ).toThrow("--image must be an immutable local sha256 image ID"); + expect(() => + parseManagedImageOpenShellE2eInputs([ + "--agent", + "unshipped", + "--image", + image, + "--sandbox", + "nemoclaw-pr-unshipped", + ]), + ).toThrow("--agent must identify a shipped managed-image agent"); + + expect(managedImageOpenShellBasePolicyPath("openclaw")).toBe( + path.join(repoRoot, "nemoclaw-blueprint", "policies", "openclaw-sandbox.yaml"), + ); + expect(managedImageOpenShellBasePolicyPath("hermes")).toBe( + path.join(repoRoot, "agents", "hermes", "policy-additions.yaml"), + ); + expect(managedImageOpenShellBasePolicyPath("langchain-deepagents-code")).toBe( + path.join(repoRoot, "agents", "langchain-deepagents-code", "policy-additions.yaml"), + ); + + const probes = { + openclaw: managedImageOpenShellProbe("openclaw"), + hermes: managedImageOpenShellProbe("hermes"), + "langchain-deepagents-code": managedImageOpenShellProbe("langchain-deepagents-code"), + }; + const committedProbe = managedImageOpenShellCommittedProbe(); + expect(probes.openclaw).toContain("/sandbox/.openclaw/openclaw.json"); + expect(probes.openclaw).toContain("http://127.0.0.1:18789/health"); + expect(probes.hermes).toContain("/sandbox/.hermes/config.yaml"); + expect(probes.hermes).toContain("http://127.0.0.1:8642/health"); + expect(probes["langchain-deepagents-code"]).toContain("/sandbox/.deepagents/config.toml"); + expect(probes["langchain-deepagents-code"]).toContain("/usr/local/bin/dcode --version"); + for (const probe of Object.values(probes)) { + expect(probe).toContain("nvidia/nemotron-3-ultra-550b-a55b"); + expect(probe).toContain("/run/nemoclaw/managed-startup-runtime.env"); + expect(probe).not.toContain("managed-startup-shared-state-transaction-v1"); + expect(probe).toContain("/usr/local/share/nemoclaw/corporate-ca.pem"); + expect(probe).toContain("/run/nemoclaw/managed-startup-ca-bundle.pem"); + } + expect(committedProbe).toContain( + "test ! -e /var/lib/nemoclaw/managed-startup-shared-state-transaction-v1", + ); + }); + + it("drives the direct image gate through exact root stdin and the root-trusted hold", () => { + const localImage = `sha256:${"a".repeat(64)}`; + const digestImage = `ghcr.io/nvidia/nemoclaw/openclaw-sandbox@sha256:${"b".repeat(64)}`; + expect( + parseManagedImageDirectE2eInputs([ + "--agent", + "openclaw", + "--image", + localImage, + "--platform", + "linux/amd64", + ]), + ).toEqual({ agent: "openclaw", image: localImage, platform: "linux/amd64" }); + expect( + parseManagedImageDirectE2eInputs([ + "--platform", + "linux/amd64", + "--image", + digestImage, + "--agent", + "langchain-deepagents-code", + ]), + ).toEqual({ + agent: "langchain-deepagents-code", + image: digestImage, + platform: "linux/amd64", + }); + expect(() => + parseManagedImageDirectE2eInputs([ + "--agent", + "hermes", + "--image", + "ghcr.io/nvidia/nemoclaw/hermes-sandbox:latest", + "--platform", + "linux/amd64", + ]), + ).toThrow("--image must be an immutable image ID or digest reference"); + + const harness = fs.readFileSync( + path.join(repoRoot, "scripts", "checks", "run-managed-image-direct-e2e.ts"), + "utf8", + ); + for (const marker of [ + "/usr/local/bin/nemoclaw-managed-startup-hold", + "serializeManagedStartupRootApplyRequest", + '"--apply-root-stdin"', + '"--commit-shared-state-transaction"', + '"--interactive"', + '"/usr/bin/env"', + '"-i"', + '"HOME=/root"', + '"--user",\n "sandbox"', + "managed hold or legacy entrypoint did not preserve the sandbox command identity", + "docker inspect did not return one exact container", + "managed profile or corporate CA entered Docker argv/env metadata", + "transaction pending", + "/var/lib/nemoclaw/managed-startup-shared-state-transaction-v1", + "was already complete", + "changed profile did not require a fresh sandbox", + "sandbox account bypassed root-only profile application", + "upper-http:upper-secret", + "lower-http:lower-secret", + "/tmp/nemoclaw-managed-command-proxy-env", + "/tmp/nemoclaw-managed-dcode-empty-prompt-status", + "managed DCode launcher/supervisor empty-prompt contract failed", + "/run/nemoclaw/managed-startup-runtime.env", + "/run/nemoclaw/managed-startup-complete.json", + "/usr/local/share/nemoclaw/corporate-ca.pem", + "/run/nemoclaw/managed-startup-ca-bundle.pem", + ]) { + expect(harness).toContain(marker); + } + expect(harness).not.toContain("Dockerfile"); + }); + + it("pins a single linux/amd64 PR base descriptor and fails closed on torn index evidence", () => { + const workflow = readWorkflow("managed-images.yaml"); + const resolver = required( + step(managedPrBuilder(workflow), "Resolve exact linux/amd64 PR base").run, + "PR base resolver script is missing", + ); + const temporaryRoot = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-pr-base-")); + const fakeBin = path.join(temporaryRoot, "bin"); + const aliasRaw = path.join(temporaryRoot, "alias.raw"); + const exactRaw = path.join(temporaryRoot, "exact.raw"); + const output = path.join(temporaryRoot, "output"); + const summary = path.join(temporaryRoot, "summary"); + fs.mkdirSync(fakeBin); + const exactBody = JSON.stringify({ + schemaVersion: 2, + mediaType: "application/vnd.oci.image.manifest.v1+json", + config: { digest: `sha256:${"a".repeat(64)}`, size: 1 }, + layers: [], + }); + const digest = `sha256:${createHash("sha256").update(exactBody).digest("hex")}`; + const descriptor = { + mediaType: "application/vnd.oci.image.manifest.v1+json", + digest, + size: exactBody.length, + platform: { os: "linux", architecture: "amd64" }, + }; + const writeAlias = (manifests: unknown[]) => { + fs.writeFileSync( + aliasRaw, + JSON.stringify({ + schemaVersion: 2, + mediaType: "application/vnd.oci.image.index.v1+json", + manifests, + }), + ); + }; + writeAlias([descriptor]); + fs.writeFileSync(exactRaw, exactBody); + fs.writeFileSync( + path.join(fakeBin, "docker"), + `#!/bin/bash +set -euo pipefail +if [ "\${1:-} \${2:-} \${3:-}" != "buildx imagetools inspect" ]; then + exit 90 +fi +if [[ "\${4:-}" == *":latest" ]]; then + cat "$ALIAS_RAW" +else + cat "$EXACT_RAW" +fi +`, + { mode: 0o755 }, + ); + const runResolver = () => + spawnSync("bash", ["-c", resolver], { + encoding: "utf8", + env: { + ...process.env, + ALIAS_RAW: aliasRaw, + BASE_ALIAS: "ghcr.io/nvidia/nemoclaw/sandbox-base:latest", + BASE_REPOSITORY: "ghcr.io/nvidia/nemoclaw/sandbox-base", + DISPLAY_NAME: "OpenClaw", + EXACT_RAW: exactRaw, + GITHUB_OUTPUT: output, + GITHUB_STEP_SUMMARY: summary, + PATH: `${fakeBin}:${process.env.PATH ?? ""}`, + RUNNER_TEMP: temporaryRoot, + }, + }); + + try { + const accepted = runResolver(); + expect(accepted.status, accepted.stderr).toBe(0); + expect(fs.readFileSync(output, "utf8")).toContain( + `ref=ghcr.io/nvidia/nemoclaw/sandbox-base@${digest}`, + ); + + writeAlias([descriptor, descriptor]); + const duplicate = runResolver(); + expect(duplicate.status).not.toBe(0); + expect(duplicate.stderr).toContain("does not contain exactly one linux/amd64 image"); + + writeAlias([descriptor]); + fs.appendFileSync(exactRaw, " "); + const wrongBody = runResolver(); + expect(wrongBody.status).not.toBe(0); + expect(wrongBody.stderr).toContain( + "exact PR base bytes do not match the selected descriptor digest", + ); + } finally { + fs.rmSync(temporaryRoot, { recursive: true, force: true }); + } + }); + + it("pins actions, validates exact digests, and records the immutable image contract (#7744)", () => { + const workflow = readWorkflow("managed-images.yaml"); + const builder = managedBuilder(workflow); + const promoter = managedPromoter(workflow); + const buildSteps = builder.steps ?? []; + const promoteSteps = promoter.steps ?? []; + + for (const action of [...buildSteps, ...promoteSteps].filter((candidate) => candidate.uses)) { + expect(action.uses, action.name).toMatch(fullShaAction); + } + expect(step(builder, "Checkout").with?.["persist-credentials"]).toBe(false); + expect(step(builder, "Download exact base image contract").with).toMatchObject({ + name: "managed-base-${{ matrix.agent }}", + path: "${{ runner.temp }}/managed-base-contract", + }); + + const guard = step(builder, "Validate production build args"); + const build = step(builder, "Build and push managed image by digest"); + const validate = step(builder, "Validate exact managed image before promotion"); + const dependencies = step(builder, "Install managed-image publication harness dependencies"); + const install = step(builder, "Install pinned OpenShell runtime for publication"); + const openshell = step( + builder, + "Exercise exact candidate through real OpenShell before promotion", + ); + const exportDiagnostics = step( + builder, + "Export sanitized managed-image OpenShell failure diagnostics", + ); + const diagnostics = step(builder, "Upload managed-image OpenShell failure diagnostics"); + expect(buildSteps.indexOf(guard)).toBeLessThan(buildSteps.indexOf(build)); + expect(guard.run).toContain('scripts/check-production-build-args.sh "${build_args[@]}"'); + expect(build.uses).toBe("docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a"); + expect(build.with).toMatchObject({ + context: ".", + file: "${{ matrix.dockerfile }}", + platforms: "${{ matrix.platform }}", + "build-args": + "BASE_IMAGE=${{ steps.base.outputs.ref }}\nNEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION=1\nNEMOCLAW_MANAGED_IMAGE_RUNTIME_USER=root\n", + provenance: "mode=max", + sbom: true, + }); + expect(build.with?.push).toBeUndefined(); + expect(build.with?.tags).toBeUndefined(); + expect(build.with?.labels).toContain("org.opencontainers.image.revision=${{ github.sha }}"); + expect(build.with?.labels).toContain("io.nvidia.nemoclaw.managed-image.contract=1"); + expect(build.with?.labels).toContain("io.nvidia.nemoclaw.managed-image.startup-profile=1"); + expect(build.with?.labels).toContain("io.nvidia.nemoclaw.managed-image.capabilities=1"); + expect(build.with?.labels).toContain( + "io.nvidia.nemoclaw.managed-image.cohort=ghrun-${{ github.run_id }}-${{ github.run_attempt }}", + ); + expect(guard.run).toContain('--build-arg "NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION=1"'); + expect(guard.run).toContain('--build-arg "NEMOCLAW_MANAGED_IMAGE_RUNTIME_USER=root"'); + expect(validate.run).not.toMatch( + /if \[ "\$AGENT" = "langchain-deepagents-code" \]; then\s+expected_http_proxy=/u, + ); + expect(validate.run).not.toContain( + 'expected_http_proxy="http://fixture-http-proxy.example.test:18080"', + ); + + const candidate = step(builder, "Export validated managed image candidate"); + expect(buildSteps.indexOf(dependencies)).toBeLessThan(buildSteps.indexOf(validate)); + expect(buildSteps.indexOf(validate)).toBeLessThan(buildSteps.indexOf(install)); + expect(buildSteps.indexOf(install)).toBeLessThan(buildSteps.indexOf(openshell)); + expect(buildSteps.indexOf(openshell)).toBeLessThan(buildSteps.indexOf(candidate)); + expect(buildSteps.indexOf(openshell)).toBeLessThan(buildSteps.indexOf(exportDiagnostics)); + expect(buildSteps.indexOf(exportDiagnostics)).toBeLessThan(buildSteps.indexOf(diagnostics)); + expect(buildSteps.indexOf(openshell)).toBeLessThan(buildSteps.indexOf(diagnostics)); + expect(buildSteps.indexOf(diagnostics)).toBeLessThan(buildSteps.indexOf(candidate)); + expect(dependencies.run).toContain("npm ci --ignore-scripts"); + expect(dependencies.run).toContain("npm run build:policy-boundary"); + expect(install.env).toMatchObject({ + NEMOCLAW_NON_INTERACTIVE: "1", + NEMOCLAW_OPENSHELL_MIN_VERSION: "0.0.85", + NEMOCLAW_OPENSHELL_MAX_VERSION: "0.0.85", + NEMOCLAW_OPENSHELL_PIN_VERSION: "0.0.85", + }); + expect(install.run).toContain("bash scripts/install-openshell.sh"); + expect(openshell.env).toEqual({ + AGENT: "${{ matrix.agent }}", + IMAGE_REFERENCE: "${{ steps.validate.outputs.local_id }}", + SANDBOX_NAME: "nemoclaw-publish-${{ matrix.agent }}", + }); + for (const marker of [ + "npx tsx scripts/checks/run-managed-image-openshell-e2e.ts", + '--agent "$AGENT"', + '--image "$IMAGE_REFERENCE"', + '--sandbox "$SANDBOX_NAME"', + ]) { + expect(openshell.run).toContain(marker); + } + expect(exportDiagnostics).toMatchObject({ + id: "managed_image_openshell_diagnostics", + if: "failure()", + run: expect.stringContaining('diagnostics_root="$HOME/.nemoclaw/onboard-failures"'), + }); + expect(exportDiagnostics.run).toContain( + 'sanitized_root="$(mktemp -d "$RUNNER_TEMP/managed-image-openshell-diagnostics.XXXXXX")"', + ); + expect(exportDiagnostics.run).toContain( + "scripts/checks/export-managed-image-failure-diagnostics.ts", + ); + expect(exportDiagnostics.run).toContain('--source "$diagnostics_root"'); + expect(exportDiagnostics.run).toContain('--output "$sanitized_root"'); + expect(exportDiagnostics.run).toContain( + 'printf \'path=%s\\n\' "$sanitized_root" >> "$GITHUB_OUTPUT"', + ); + expect(exportDiagnostics.run).not.toContain( + 'printf \'path=%s\\n\' "$diagnostics_root" >> "$GITHUB_OUTPUT"', + ); + expect(exportDiagnostics.run).toContain("sed -n '1,160p' \"$summary\""); + expect(exportDiagnostics.run).toContain('find "$sanitized_root"'); + expect(exportDiagnostics.run).not.toContain('find "$diagnostics_root"'); + expect(diagnostics).toMatchObject({ + if: "${{ failure() && steps.managed_image_openshell_diagnostics.outcome == 'success' }}", + uses: "actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a", + with: { + name: "managed-image-openshell-publish-${{ matrix.agent }}-${{ github.run_id }}-${{ github.run_attempt }}", + path: "${{ steps.managed_image_openshell_diagnostics.outputs.path }}", + "include-hidden-files": true, + "if-no-files-found": "ignore", + "retention-days": 14, + }, + }); + expect(String(diagnostics.with?.path)).not.toContain("onboard-failures"); + for (const marker of [ + "--arg baseReference", + "--arg digest", + "--arg platform", + "--arg ref", + "--arg release", + "--arg revision", + "--arg cohort", + "--argjson runAttempt", + "--argjson runId", + "contractVersion: 1", + 'phase: "candidate"', + '(has("aliases") | not)', + ]) { + expect(candidate.run).toContain(marker); + } + expect(candidate.run).not.toContain("aliases:"); + expect(step(builder, "Upload validated managed image candidate").with).toMatchObject({ + name: "managed-image-candidate-${{ matrix.agent }}-${{ matrix.artifact_platform }}", + path: "${{ runner.temp }}/managed-image-candidate/contract.json", + "if-no-files-found": "error", + "retention-days": 1, + }); + + expect(step(promoter, "Download all validated managed image candidates").with).toEqual({ + pattern: "managed-image-candidate-*", + path: "${{ runner.temp }}/managed-image-candidates", + "merge-multiple": false, + }); + const barrier = step(promoter, "Validate complete managed image candidate set"); + const promotion = step(promoter, "Promote validated managed image aliases"); + expect(promoteSteps.indexOf(barrier)).toBeLessThan(promoteSteps.indexOf(promotion)); + expect(promotion.run).toContain( + "# Stage all three unique cohort aliases before any consumer pointer", + ); + expect(promotion.run).toContain( + "# Verify every staged cohort alias against its exact validated", + ); + expect(promotion.run).not.toContain('aliases=("${image}:${GITHUB_SHA}")'); + + expect(step(promoter, "Upload OpenClaw managed image contract").with).toMatchObject({ + name: "managed-image-openclaw-linux-amd64", + path: "${{ runner.temp }}/managed-image-contracts/openclaw/contract.json", + "retention-days": 90, + }); + expect(step(promoter, "Upload Hermes managed image contract").with).toMatchObject({ + name: "managed-image-hermes-linux-amd64", + path: "${{ runner.temp }}/managed-image-contracts/hermes/contract.json", + "retention-days": 90, + }); + expect(step(promoter, "Upload Deep Agents Code managed image contract").with).toMatchObject({ + name: "managed-image-langchain-deepagents-code-linux-amd64", + path: "${{ runner.temp }}/managed-image-contracts/langchain-deepagents-code/contract.json", + "retention-days": 90, + }); + + const validation = required( + step(builder, "Validate exact managed image before promotion").run, + "managed image validation script is missing", + ); + expect(validation.match(/docker run/g)).toHaveLength(2); + expect(validation).toContain("run-managed-image-direct-e2e.ts"); + expect(validation).toContain("npx --no-install tsx"); + expect(validation).toContain('--image "$reference"'); + expect(validation).toContain("printf 'local_id=%s\\n' \"$image_id\""); + expect(validation).not.toContain("NEMOCLAW_STARTUP_PROFILE_B64"); + expect(validation).not.toContain("NEMOCLAW_CORPORATE_CA_B64"); + expect(validation).not.toContain(".Config.Entrypoint"); + expect(validation).not.toContain(".Config.Cmd"); + expect(buildSteps.indexOf(dependencies)).toBeLessThan( + buildSteps.indexOf(step(builder, "Validate exact managed image before promotion")), + ); + }); + + it("executes the all-three barrier and rejects incomplete or stale sets before promotion", () => { + const workflow = readWorkflow("managed-images.yaml"); + const barrier = required( + step(managedPromoter(workflow), "Validate complete managed image candidate set").run, + "managed image all-three barrier script is missing", + ); + const temporaryRoot = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-managed-candidates-")); + const candidateRoot = path.join(temporaryRoot, "candidates"); + const revision = "a".repeat(40); + const repository = "NVIDIA/NemoClaw"; + const runId = "7744"; + const runAttempt = "2"; + const candidates = [ + { + artifact: "managed-image-candidate-openclaw-linux-amd64", + agent: "openclaw", + image: "ghcr.io/nvidia/nemoclaw/openclaw-sandbox", + }, + { + artifact: "managed-image-candidate-hermes-linux-amd64", + agent: "hermes", + image: "ghcr.io/nvidia/nemoclaw/hermes-sandbox", + }, + { + artifact: "managed-image-candidate-langchain-deepagents-code-linux-amd64", + agent: "langchain-deepagents-code", + image: "ghcr.io/nvidia/nemoclaw/langchain-deepagents-code-sandbox", + }, + ] as const; + + const writeCandidate = ( + candidate: (typeof candidates)[number], + sourceRevision: string, + cohort = `ghrun-${runId}-${runAttempt}`, + ) => { + const artifactRoot = path.join(candidateRoot, candidate.artifact); + fs.mkdirSync(artifactRoot, { recursive: true }); + const digest = `sha256:${candidate.agent.charCodeAt(0).toString(16).padStart(2, "0").repeat(32)}`; + fs.writeFileSync( + path.join(artifactRoot, "contract.json"), + `${JSON.stringify( + { + contractVersion: 1, + phase: "candidate", + agent: candidate.agent, + image: candidate.image, + digest, + reference: `${candidate.image}@${digest}`, + baseReference: `ghcr.io/nvidia/nemoclaw/${candidate.agent}-base@${digest}`, + platform: "linux/amd64", + source: { + repository, + revision: sourceRevision, + ref: "refs/heads/main", + cohort, + }, + run: { + id: Number(runId), + attempt: Number(runAttempt), + }, + release: null, + }, + null, + 2, + )}\n`, + ); + }; + const runBarrier = (outputName: string) => + spawnSync("bash", ["-c", barrier], { + cwd: repoRoot, + encoding: "utf8", + env: { + ...process.env, + CANDIDATE_ROOT: candidateRoot, + GITHUB_OUTPUT: path.join(temporaryRoot, outputName), + GITHUB_REF: "refs/heads/main", + GITHUB_REPOSITORY: repository, + GITHUB_RUN_ATTEMPT: runAttempt, + GITHUB_RUN_ID: runId, + GITHUB_SHA: revision, + RUNNER_TEMP: temporaryRoot, + }, + }); + + try { + for (const candidate of candidates) writeCandidate(candidate, revision); + const accepted = runBarrier("accepted-output"); + expect(accepted.status, accepted.stderr).toBe(0); + expect(fs.readFileSync(path.join(temporaryRoot, "accepted-output"), "utf8")).toContain( + "candidate_set=", + ); + + writeCandidate(candidates[1], "b".repeat(40)); + const rejected = runBarrier("rejected-output"); + expect(rejected.status).not.toBe(0); + expect(rejected.stderr).toContain( + "complete managed image candidate set failed closed validation", + ); + + writeCandidate(candidates[1], revision); + writeCandidate(candidates[2], revision, "ghrun-7744-3"); + const mixedCohort = runBarrier("mixed-cohort-output"); + expect(mixedCohort.status).not.toBe(0); + expect(mixedCohort.stderr).toContain( + "complete managed image candidate set failed closed validation", + ); + + writeCandidate(candidates[2], revision); + fs.rmSync(path.join(candidateRoot, candidates[2].artifact), { + recursive: true, + force: true, + }); + const incomplete = runBarrier("incomplete-output"); + expect(incomplete.status).not.toBe(0); + expect(incomplete.stderr).toContain( + "expected exactly three managed image candidate artifacts", + ); + } finally { + fs.rmSync(temporaryRoot, { recursive: true, force: true }); + } + }); + + it("stages the complete cohort before moving only OpenClaw consumer pointers", () => { + const workflow = readWorkflow("managed-images.yaml"); + const promotion = required( + step(managedPromoter(workflow), "Promote validated managed image aliases").run, + "managed image promotion script is missing", + ); + const temporaryRoot = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-managed-promotion-")); + const fakeBin = path.join(temporaryRoot, "bin"); + const callLog = path.join(temporaryRoot, "docker-calls.log"); + const candidateSet = path.join(temporaryRoot, "candidate-set.json"); + const revision = "a".repeat(40); + const cohort = "ghrun-7744-2"; + fs.mkdirSync(fakeBin); + fs.writeFileSync( + path.join(fakeBin, "docker"), + `#!/bin/bash +set -euo pipefail +printf '%s\\n' "$*" >> "$DOCKER_CALL_LOG" +if [ "\${FAIL_OPENCLAW_COHORT:-0}" = "1" ] && + [[ "$*" == *"imagetools create"* ]] && + [[ "$*" == *"openclaw-sandbox:cohort-${cohort}"* ]]; then + exit 91 +fi +if [[ "$*" == *"imagetools inspect"* ]]; then + printf '%s\\n' "validated-manifest" +fi +`, + { mode: 0o755 }, + ); + const candidates = [ + ["hermes", "ghcr.io/nvidia/nemoclaw/hermes-sandbox", "1"], + [ + "langchain-deepagents-code", + "ghcr.io/nvidia/nemoclaw/langchain-deepagents-code-sandbox", + "2", + ], + ["openclaw", "ghcr.io/nvidia/nemoclaw/openclaw-sandbox", "3"], + ].map(([agent, image, digestSeed]) => { + const digest = `sha256:${digestSeed.repeat(64)}`; + return { + contractVersion: 1, + phase: "candidate", + agent, + image, + digest, + reference: `${image}@${digest}`, + baseReference: `${image}-base@${digest}`, + platform: "linux/amd64", + source: { + repository: "NVIDIA/NemoClaw", + revision, + ref: "refs/tags/v0.0.97", + cohort, + }, + run: { id: 7744, attempt: 2 }, + release: "v0.0.97", + }; + }); + fs.writeFileSync(candidateSet, `${JSON.stringify(candidates)}\n`); + + try { + const result = spawnSync("bash", ["-c", promotion], { + cwd: repoRoot, + encoding: "utf8", + env: { + ...process.env, + CANDIDATE_SET: candidateSet, + DOCKER_CALL_LOG: callLog, + FAIL_OPENCLAW_COHORT: "1", + GITHUB_RUN_ATTEMPT: "2", + GITHUB_RUN_ID: "7744", + GITHUB_SHA: revision, + PATH: `${fakeBin}:${process.env.PATH ?? ""}`, + RUNNER_TEMP: temporaryRoot, + }, + }); + + expect(result.status).toBe(91); + const calls = fs.readFileSync(callLog, "utf8"); + expect(calls).toContain(`hermes-sandbox:cohort-${cohort}`); + expect(calls).toContain(`langchain-deepagents-code-sandbox:cohort-${cohort}`); + expect(calls).toContain(`openclaw-sandbox:cohort-${cohort}`); + expect(calls).not.toContain(`openclaw-sandbox:${revision}`); + expect(calls).not.toContain("openclaw-sandbox:v0.0.97"); + expect(calls).not.toContain(`hermes-sandbox:${revision}`); + expect(calls).not.toContain("hermes-sandbox:v0.0.97"); + expect(calls).not.toContain(`langchain-deepagents-code-sandbox:${revision}`); + expect(calls).not.toContain("langchain-deepagents-code-sandbox:v0.0.97"); + + fs.writeFileSync(callLog, ""); + const accepted = spawnSync("bash", ["-c", promotion], { + cwd: repoRoot, + encoding: "utf8", + env: { + ...process.env, + CANDIDATE_SET: candidateSet, + DOCKER_CALL_LOG: callLog, + GITHUB_RUN_ATTEMPT: "2", + GITHUB_RUN_ID: "7744", + GITHUB_SHA: revision, + PATH: `${fakeBin}:${process.env.PATH ?? ""}`, + RUNNER_TEMP: temporaryRoot, + }, + }); + expect(accepted.status, accepted.stderr).toBe(0); + const acceptedCalls = fs.readFileSync(callLog, "utf8"); + const lastCohortStage = Math.max( + acceptedCalls.indexOf(`hermes-sandbox:cohort-${cohort}`), + acceptedCalls.indexOf(`langchain-deepagents-code-sandbox:cohort-${cohort}`), + acceptedCalls.indexOf(`openclaw-sandbox:cohort-${cohort}`), + ); + const consumerPointer = acceptedCalls.indexOf(`openclaw-sandbox:${revision}`); + expect(lastCohortStage).toBeGreaterThanOrEqual(0); + expect(consumerPointer).toBeGreaterThan(lastCohortStage); + expect(acceptedCalls).toContain("openclaw-sandbox:v0.0.97"); + expect(acceptedCalls).not.toContain(`hermes-sandbox:${revision}`); + expect(acceptedCalls).not.toContain("hermes-sandbox:v0.0.97"); + expect(acceptedCalls).not.toContain(`langchain-deepagents-code-sandbox:${revision}`); + expect(acceptedCalls).not.toContain("langchain-deepagents-code-sandbox:v0.0.97"); + + const contractsRoot = path.join(temporaryRoot, "managed-image-contracts"); + const contractAliases = (agent: string) => + JSON.parse(fs.readFileSync(path.join(contractsRoot, agent, "contract.json"), "utf8")) as { + aliases: string[]; + source: { cohort: string }; + }; + expect(contractAliases("hermes")).toMatchObject({ + aliases: [`ghcr.io/nvidia/nemoclaw/hermes-sandbox:cohort-${cohort}`], + source: expect.objectContaining({ cohort }), + }); + expect(contractAliases("langchain-deepagents-code")).toMatchObject({ + aliases: [`ghcr.io/nvidia/nemoclaw/langchain-deepagents-code-sandbox:cohort-${cohort}`], + source: expect.objectContaining({ cohort }), + }); + expect(contractAliases("openclaw")).toMatchObject({ + aliases: [ + `ghcr.io/nvidia/nemoclaw/openclaw-sandbox:cohort-${cohort}`, + `ghcr.io/nvidia/nemoclaw/openclaw-sandbox:${revision}`, + "ghcr.io/nvidia/nemoclaw/openclaw-sandbox:v0.0.97", + ], + source: expect.objectContaining({ cohort }), + }); + } finally { + fs.rmSync(temporaryRoot, { recursive: true, force: true }); + } + }); +}); diff --git a/test/managed-startup-image-entrypoint.test.ts b/test/managed-startup-image-entrypoint.test.ts new file mode 100644 index 0000000000..6f3e5fe3fb --- /dev/null +++ b/test/managed-startup-image-entrypoint.test.ts @@ -0,0 +1,172 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { execFileSync, spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +const ROOT = path.resolve(import.meta.dirname, ".."); +const HOLD = path.join(ROOT, "scripts", "managed-startup-hold.sh"); + +function executable(target: string, contents: string): void { + fs.writeFileSync(target, contents, { mode: 0o755 }); + fs.chmodSync(target, 0o755); +} + +type FakeIdentity = { + readonly currentUid: number; + readonly currentGid: number; + readonly sandboxUid: number; + readonly sandboxGid: number; +}; + +function fakeIdScript(identity: FakeIdentity): string { + return `#!/bin/sh +case "$*" in + "-u") printf '${String(identity.currentUid)}\\n' ;; + "-g") printf '${String(identity.currentGid)}\\n' ;; + "-u sandbox") printf '${String(identity.sandboxUid)}\\n' ;; + "-g sandbox") printf '${String(identity.sandboxGid)}\\n' ;; + *) exit 1 ;; +esac +`; +} + +function runHoldWithFakeIdentity(identity: FakeIdentity, args: readonly string[]) { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-managed-hold-identity-")); + try { + const script = path.join(directory, "hold.sh"); + executable(path.join(directory, "id"), fakeIdScript(identity)); + fs.writeFileSync( + script, + fs + .readFileSync(HOLD, "utf8") + .replace( + 'export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"', + 'export PATH="$TEST_PATH"', + ), + { mode: 0o755 }, + ); + return spawnSync("/bin/bash", [script, ...args], { + encoding: "utf8", + env: { ...process.env, TEST_PATH: directory }, + }); + } finally { + fs.rmSync(directory, { force: true, recursive: true }); + } +} + +describe("managed startup image hold", () => { + it.each([ + "openclaw", + "hermes", + "langchain-deepagents-code", + ] as const)("enters the %s legacy startup as sandbox after the exact handoff", (agent) => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-managed-hold-")); + try { + const trace = path.join(directory, "trace"); + const runtime = path.join(directory, "runtime.cjs"); + const runtimeEnvironment = path.join(directory, "runtime.env"); + const script = path.join(directory, "hold.sh"); + fs.writeFileSync(runtime, ""); + fs.writeFileSync(runtimeEnvironment, "export NEMOCLAW_MANAGED_STARTUP_APPLIED='1'\n", { + mode: 0o444, + }); + executable( + path.join(directory, "id"), + fakeIdScript({ + currentUid: 1000, + currentGid: 1000, + sandboxUid: 1000, + sandboxGid: 1000, + }), + ); + executable(path.join(directory, "stat"), "#!/bin/sh\nprintf '0:0:444\\n'\n"); + executable(path.join(directory, "node"), `#!/bin/sh\nprintf 'node:%s\\n' "$*" >>"$TRACE"\n`); + executable( + path.join(directory, "nemoclaw-start"), + `#!/bin/sh\nprintf 'start:%s:%s:%s:%s\\n' "$NEMOCLAW_MANAGED_STARTUP_APPLIED" "\${NEMOCLAW_STARTUP_PROFILE_B64-unset}" "\${NEMOCLAW_CORPORATE_CA_B64-unset}" "$*" >>"$TRACE"\n`, + ); + const source = fs + .readFileSync(HOLD, "utf8") + .replace( + 'export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"', + 'export PATH="$TEST_PATH"', + ) + .replace( + '_nemoclaw_runtime="/usr/local/lib/nemoclaw/managed-startup-image-runtime.cjs"', + `_nemoclaw_runtime=${JSON.stringify(runtime)}`, + ) + .replace( + '_nemoclaw_runtime_env="/run/nemoclaw/managed-startup-runtime.env"', + `_nemoclaw_runtime_env=${JSON.stringify(runtimeEnvironment)}`, + ) + .replace("/usr/local/bin/node", path.join(directory, "node")) + .replace("/usr/local/bin/nemoclaw-start", path.join(directory, "nemoclaw-start")); + fs.writeFileSync(script, source, { mode: 0o755 }); + fs.chmodSync(script, 0o755); + const fingerprint = "a".repeat(64); + + execFileSync( + script, + [ + "--agent", + agent, + "--profile-fingerprint", + fingerprint, + "/bin/sh", + "-c", + "exec tail -f /dev/null", + ], + { + env: { + ...process.env, + TRACE: trace, + TEST_PATH: directory, + NEMOCLAW_STARTUP_PROFILE_B64: "must-drop", + NEMOCLAW_CORPORATE_CA_B64: "must-drop", + }, + }, + ); + + expect(fs.readFileSync(trace, "utf8").trim().split("\n")).toEqual([ + `node:${runtime} --wait-for-completion --agent ${agent} --profile-fingerprint ${fingerprint}`, + "start:1:unset:unset:/bin/sh -c exec tail -f /dev/null", + ]); + } finally { + fs.rmSync(directory, { force: true, recursive: true }); + } + }); + + it.each([ + { + label: "root supervisor identity", + identity: { currentUid: 0, currentGid: 0, sandboxUid: 1000, sandboxGid: 1000 }, + }, + { + label: "non-sandbox group", + identity: { currentUid: 1000, currentGid: 1001, sandboxUid: 1000, sandboxGid: 1000 }, + }, + ])("rejects $label before invoking the managed runtime", ({ identity }) => { + const result = runHoldWithFakeIdentity(identity, [ + "--agent", + "openclaw", + "--profile-fingerprint", + "a".repeat(64), + ]); + expect(result.status).not.toBe(0); + expect(String(result.stderr ?? "")).toContain("must run as the sandbox account"); + }); + + it("rejects unsupported agents before invoking the runtime", () => { + const result = runHoldWithFakeIdentity( + { currentUid: 1000, currentGid: 1000, sandboxUid: 1000, sandboxGid: 1000 }, + ["--agent", "unknown", "--profile-fingerprint", "a".repeat(64)], + ); + expect(result.status).not.toBe(0); + expect(String(result.stderr ?? "")).toContain("agent is unsupported"); + }); +}); diff --git a/test/messaging-build-applier.test.ts b/test/messaging-build-applier.test.ts index 52e33cfec3..4e9901c40b 100644 --- a/test/messaging-build-applier.test.ts +++ b/test/messaging-build-applier.test.ts @@ -10,6 +10,8 @@ import path from "node:path"; import { beforeEach, describe, expect, it, vi } from "vitest"; import { applyMessagingBuildPhase, + collectManagedImageHermesUvPackages, + collectManagedImageOpenClawPluginInstallSpecs, describeMessagingBuildPhase, type MessagingBuildPhase, readMessagingBuildPlanFromEnv, @@ -156,8 +158,9 @@ async function buildPlanEnv( function runApplierProcess( env: Record, agent: "hermes" | "openclaw", - phase: MessagingBuildPhase, + phase: MessagingBuildPhase | "managed-image-capability-union", dryRun = false, + managedStartupRuntime = false, ) { return spawnSync( "node", @@ -169,6 +172,7 @@ function runApplierProcess( "--phase", phase, ...(dryRun ? ["--dry-run"] : []), + ...(managedStartupRuntime ? ["--managed-startup-runtime"] : []), ], { encoding: "utf-8", @@ -209,6 +213,63 @@ function thrownMessage(run: () => void): string { } describe("messaging-build-applier.mts: agent-install", () => { + it("derives the complete managed-image package union from trusted manifests", () => { + expect(collectManagedImageOpenClawPluginInstallSpecs({ OPENCLAW_VERSION: "2026.7.1" })).toEqual( + [ + "npm:@openclaw/discord@2026.7.1", + "npm:@tencent-weixin/openclaw-weixin@2.4.3", + "npm:@openclaw/slack@2026.7.1", + "npm:@openclaw/whatsapp@2026.7.1", + "npm:@openclaw/msteams@2026.7.1", + ], + ); + expect(collectManagedImageHermesUvPackages()).toEqual([ + "microsoft-teams-apps==2.0.13.4", + "aiohttp==3.14.1", + ]); + }); + + it("installs the Hermes managed-image union only in explicit neutral-image mode", () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-hermes-union-packages-")); + const tracePath = path.join(tmp, "uv.trace"); + fs.writeFileSync( + path.join(tmp, "uv"), + ["#!/bin/sh", 'printf \'%s\\n\' "$*" >> "$UV_TRACE"', "exit 0", ""].join("\n"), + { mode: 0o755 }, + ); + try { + const env = { + ...process.env, + PATH: `${tmp}:${TEST_PATH}`, + UV_TRACE: tracePath, + NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION: "1", + } as Record; + const result = runApplierProcess(env, "hermes", "managed-image-capability-union"); + + expect(result.status, result.stderr).toBe(0); + expect(fs.readFileSync(tracePath, "utf-8").trim()).toBe( + "pip install --python /opt/hermes/.venv/bin/python --no-cache -- microsoft-teams-apps==2.0.13.4 aiohttp==3.14.1", + ); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + }); + + it("rejects a serialized plan before installing the managed-image union", async () => { + const env = await buildPlanEnv({ + OPENCLAW_VERSION: "2026.7.1", + NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION: "1", + NEMOCLAW_MESSAGING_CHANNELS_B64: channelsB64(["discord"]), + }); + + const result = runApplierProcess(env, "openclaw", "managed-image-capability-union", true); + + expect(result.status).toBe(2); + expect(result.stderr).toContain( + "Managed-image capability union must be built without a serialized messaging plan", + ); + }); + it( "collects selected messaging plugin install specs", async () => { @@ -1209,7 +1270,7 @@ describe("messaging-build-applier.mts: agent-install", () => { } }); - it("reapplies OpenClaw messaging render after doctor rewrites config", async () => { + it("keeps legacy doctor rerendering while managed startup skips the broad doctor", async () => { const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-openclaw-doctor-rewrite-")); const tracePath = path.join(tmp, "openclaw.trace"); const fakeOpenclaw = path.join(tmp, "openclaw"); @@ -1271,6 +1332,29 @@ describe("messaging-build-applier.mts: agent-install", () => { expect(config.plugins?.entries?.slack).toEqual({ enabled: true }); expect(config.channels?.["openclaw-weixin"]?.accounts?.primary).toEqual({ enabled: true }); expect(config.channels?.wechat).toBeUndefined(); + + fs.writeFileSync( + path.join(tmp, ".openclaw", "openclaw.json"), + `${JSON.stringify({ channels: {}, plugins: { entries: {} } }, null, 2)}\n`, + ); + fs.writeFileSync(tracePath, ""); + const managedResult = runApplierProcess(env, "openclaw", "post-agent-install", false, true); + expect(managedResult.status, managedResult.stderr).toBe(0); + expect(fs.readFileSync(tracePath, "utf-8")).toBe(""); + const managedConfig = JSON.parse( + fs.readFileSync(path.join(tmp, ".openclaw", "openclaw.json"), "utf-8"), + ); + expect(managedConfig.channels?.telegram?.accounts?.default).toMatchObject({ + botToken: "openshell:resolve:env:TELEGRAM_BOT_TOKEN", + enabled: true, + }); + expect(managedConfig.channels?.discord?.enabled).toBe(true); + expect(managedConfig.plugins?.entries?.discord).toEqual({ enabled: true }); + expect(managedConfig.channels?.slack?.enabled).toBe(true); + expect(managedConfig.plugins?.entries?.slack).toEqual({ enabled: true }); + expect(managedConfig.channels?.["openclaw-weixin"]?.accounts?.primary).toEqual({ + enabled: true, + }); } finally { fs.rmSync(tmp, { recursive: true, force: true }); } diff --git a/test/nemoclaw-start.test.ts b/test/nemoclaw-start.test.ts index b2fd8fb9a7..9abe9c80d5 100644 --- a/test/nemoclaw-start.test.ts +++ b/test/nemoclaw-start.test.ts @@ -350,100 +350,6 @@ describe("nemoclaw-start non-root fallback", () => { expect(result.stderr).not.toContain("#token="); expect(result.stderr).not.toContain(token); }); - - it("unwraps the sandbox-create env self-wrapper and applies dashboard port defaults", () => { - const src = fs.readFileSync(START_SCRIPT, "utf-8"); - const start = src.indexOf("# Normalize the sandbox-create bootstrap wrapper"); - const end = src.indexOf("# ── Config integrity check", start); - if (start === -1 || end === -1 || end <= start) { - throw new Error("Expected sandbox-create wrapper normalization and port block"); - } - const snippet = src.slice(start, end); - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-env-wrapper-")); - const fakeBin = path.join(tmpDir, "bin"); - const scriptPath = path.join(tmpDir, "run.sh"); - - function runScenario(setArgs: string, extraEnv: Record = {}) { - const script = [ - "#!/usr/bin/env bash", - "set -euo pipefail", - setArgs, - snippet, - 'printf "CHAT_UI_URL=%s\\n" "$CHAT_UI_URL"', - 'printf "PUBLIC_PORT=%s\\n" "$PUBLIC_PORT"', - 'printf "OPENCLAW_GATEWAY_PORT=%s\\n" "$OPENCLAW_GATEWAY_PORT"', - 'printf "OPENCLAW_GATEWAY_URL=%s\\n" "$OPENCLAW_GATEWAY_URL"', - 'printf "SANDBOX_HOME=%s\\n" "$_SANDBOX_HOME"', - 'printf "OPENCLAW_HOME=%s\\n" "$OPENCLAW_HOME"', - 'printf "OPENCLAW_STATE_DIR=%s\\n" "$OPENCLAW_STATE_DIR"', - 'printf "OPENCLAW_CONFIG_PATH=%s\\n" "$OPENCLAW_CONFIG_PATH"', - 'printf "OPENCLAW_OAUTH_DIR=%s\\n" "$OPENCLAW_OAUTH_DIR"', - 'printf "CMD=%s\\n" "${NEMOCLAW_CMD[*]}"', - ].join("\n"); - fs.writeFileSync(scriptPath, script, { mode: 0o700 }); - return spawnSync("bash", [scriptPath], { - encoding: "utf-8", - timeout: 5000, - env: { ...process.env, PATH: `${fakeBin}:${process.env.PATH || ""}`, ...extraEnv }, - }); - } - - try { - fs.mkdirSync(fakeBin); - fs.writeFileSync(path.join(fakeBin, "openclaw"), "#!/usr/bin/env bash\nexit 0\n", { - mode: 0o755, - }); - - const injected = runScenario( - "set -- env CHAT_UI_URL=https://chat.example.test NEMOCLAW_DASHBOARD_PORT=19000 nemoclaw-start openclaw agent --agent main", - ); - expect(injected.status).toBe(0); - expect(injected.stdout).toContain("CHAT_UI_URL=http://127.0.0.1:19000"); - expect(injected.stdout).toContain("PUBLIC_PORT=19000"); - expect(injected.stdout).toContain("OPENCLAW_GATEWAY_PORT=19000"); - expect(injected.stdout).toContain("OPENCLAW_GATEWAY_URL=ws://127.0.0.1:19000"); - expect(injected.stdout).toContain("SANDBOX_HOME=/sandbox"); - expect(injected.stdout).toContain("OPENCLAW_HOME=/sandbox"); - expect(injected.stdout).toContain("OPENCLAW_STATE_DIR=/sandbox/.openclaw"); - expect(injected.stdout).toContain("OPENCLAW_CONFIG_PATH=/sandbox/.openclaw/openclaw.json"); - expect(injected.stdout).toContain("OPENCLAW_OAUTH_DIR=/sandbox/.openclaw/credentials"); - expect(injected.stdout).toContain("CMD=openclaw agent --agent main"); - - const bakedCustomPort = runScenario("set -- nemoclaw-start openclaw agent", { - CHAT_UI_URL: "http://127.0.0.1:18790", - }); - expect(bakedCustomPort.status).toBe(0); - expect(bakedCustomPort.stdout).toContain("CHAT_UI_URL=http://127.0.0.1:18790"); - expect(bakedCustomPort.stdout).toContain("PUBLIC_PORT=18790"); - expect(bakedCustomPort.stdout).toContain("OPENCLAW_GATEWAY_PORT=18790"); - expect(bakedCustomPort.stdout).toContain("OPENCLAW_GATEWAY_URL=ws://127.0.0.1:18790"); - expect(bakedCustomPort.stdout).toContain("OPENCLAW_STATE_DIR=/sandbox/.openclaw"); - expect(bakedCustomPort.stdout).toContain("OPENCLAW_OAUTH_DIR=/sandbox/.openclaw/credentials"); - expect(bakedCustomPort.stdout).toContain("CMD=openclaw agent"); - - const baked = runScenario("set -- nemoclaw-start openclaw agent", { - CHAT_UI_URL: "https://baked.example.test/ui", - }); - expect(baked.status).toBe(0); - expect(baked.stdout).toContain("CHAT_UI_URL=https://baked.example.test/ui"); - expect(baked.stdout).toContain("PUBLIC_PORT=18789"); - expect(baked.stdout).toContain("OPENCLAW_GATEWAY_PORT=18789"); - expect(baked.stdout).toContain("OPENCLAW_GATEWAY_URL=ws://127.0.0.1:18789"); - expect(baked.stdout).toContain("SANDBOX_HOME=/sandbox"); - expect(baked.stdout).toContain("OPENCLAW_STATE_DIR=/sandbox/.openclaw"); - expect(baked.stdout).toContain("CMD=openclaw agent"); - - const invalidHighPort = runScenario("set -- nemoclaw-start openclaw agent", { - NEMOCLAW_DASHBOARD_PORT: "70000", - }); - expect(invalidHighPort.status).toBe(1); - expect(invalidHighPort.stderr).toContain("Invalid NEMOCLAW_DASHBOARD_PORT='70000'"); - expect(invalidHighPort.stderr).toContain("must be an integer between 1024 and 65535"); - } finally { - fs.rmSync(tmpDir, { recursive: true, force: true }); - } - }); - it("runs runtime preloads and scans before explicit non-root commands", () => { const src = fs.readFileSync(START_SCRIPT, "utf-8"); const script = [ diff --git a/test/onboard-managed-image-buildless-e2e.test.ts b/test/onboard-managed-image-buildless-e2e.test.ts new file mode 100644 index 0000000000..0dda898f47 --- /dev/null +++ b/test/onboard-managed-image-buildless-e2e.test.ts @@ -0,0 +1,24 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// @module-tag e2e/credential-free + +import { describe } from "vitest"; + +import { test } from "./e2e/fixtures/workflow-e2e-test.ts"; +import { runManagedImageBuildlessE2e } from "./helpers/managed-image-buildless-e2e"; + +describe("managed image buildless onboarding", () => { + test("launches every shipped agent by immutable image and startup profile without Dockerfile work (#7744)", { + timeout: 180_000, + meta: { + e2ePhases: [ + "validate all-agent buildless managed-image orchestration", + "release managed onboarding fixtures", + ], + }, + }, ({ progress }) => { + progress.phase("validate all-agent buildless managed-image orchestration"); + runManagedImageBuildlessE2e(); + progress.phase("release managed onboarding fixtures"); + }); +}); diff --git a/test/onboard-prepared-build-context.test.ts b/test/onboard-prepared-build-context.test.ts index 0d91d612ae..1eae5b23e4 100644 --- a/test/onboard-prepared-build-context.test.ts +++ b/test/onboard-prepared-build-context.test.ts @@ -17,7 +17,7 @@ type PreparedContextResult = { commands: string[]; errorMessage: string | null; patchCalls: number; - planBuildContexts: string[]; + planFromRefs: string[]; registerCalls: Array<{ imageTag?: string | null }>; resolvedBuildIds: string[]; stageCalls: number; @@ -89,7 +89,7 @@ const buildId = ${JSON.stringify(buildId)}; const sandboxName = "prepared-dcode"; const commands = []; const registerCalls = []; -const planBuildContexts = []; +const planFromRefs = []; const resolvedBuildIds = []; let cleanupCalls = 0; let patchCalls = 0; @@ -99,8 +99,10 @@ dockerGpuSandboxCreate.createDockerGpuSandboxCreatePatch = () => ({ maybeApplyDuringCreate: () => {}, createFailureMessage: () => null, exitOnPatchError: () => {}, + rollbackManagedStartupAfterCreateFailure: () => {}, ensureApplied: () => {}, waitForSupervisorReconnectIfNeeded: () => {}, + commitAfterReady: () => {}, selectedMode: () => null, printReadinessFailureIfEnabled: () => {}, verifyGpuOrExit: (verify) => verify(sandboxName), @@ -117,7 +119,7 @@ dockerfilePatchFlow.prepareSandboxDockerfilePatch = async () => { const materializeSandboxCreatePlan = sandboxCreatePlanMaterialization.materializeSandboxCreatePlan; sandboxCreatePlanMaterialization.materializeSandboxCreatePlan = (input) => { - planBuildContexts.push(input.buildCtx); + planFromRefs.push(input.fromRef); return materializeSandboxCreatePlan(input); }; const resolveSandboxImageTagFromCreateOutput = imageTag.resolveSandboxImageTagFromCreateOutput; @@ -225,7 +227,7 @@ const { createSandbox } = require(${onboardPath}); commands, errorMessage, patchCalls, - planBuildContexts, + planFromRefs, registerCalls, resolvedBuildIds, stageCalls, @@ -268,7 +270,7 @@ describe("onboard prepared DCode build context", () => { assert.equal(result.errorMessage, null); assert.equal(result.stageCalls, 0); assert.equal(result.patchCalls, 0); - assert.deepEqual(result.planBuildContexts, [result.buildCtx]); + assert.deepEqual(result.planFromRefs, [`${result.buildCtx}/Dockerfile`]); assert.deepEqual(result.resolvedBuildIds, [result.buildId]); assert.equal(result.cleanupCalls, 1); assert.ok( @@ -296,7 +298,7 @@ describe("onboard prepared DCode build context", () => { ); assert.equal(result.stageCalls, 0); assert.equal(result.patchCalls, 0); - assert.deepEqual(result.planBuildContexts, []); + assert.deepEqual(result.planFromRefs, []); assert.deepEqual(result.resolvedBuildIds, []); assert.equal(result.cleanupCalls, 0); assert.equal( diff --git a/test/onboard-terminal-dashboard.test.ts b/test/onboard-terminal-dashboard.test.ts index e45a9b5b17..757cd6409a 100644 --- a/test/onboard-terminal-dashboard.test.ts +++ b/test/onboard-terminal-dashboard.test.ts @@ -68,8 +68,10 @@ dockerGpuSandboxCreate.createDockerGpuSandboxCreatePatch = () => ({ maybeApplyDuringCreate: () => {}, createFailureMessage: () => null, exitOnPatchError: () => {}, + rollbackManagedStartupAfterCreateFailure: () => {}, ensureApplied: () => {}, waitForSupervisorReconnectIfNeeded: () => {}, + commitAfterReady: () => {}, selectedMode: () => null, printReadinessFailureIfEnabled: () => {}, verifyGpuOrExit: (verify) => verify(sandboxName), diff --git a/test/onboard.test.ts b/test/onboard.test.ts index b929f1ef72..e804dc0705 100644 --- a/test/onboard.test.ts +++ b/test/onboard.test.ts @@ -7,7 +7,11 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { describe, expect, it } from "vitest"; -import { appendHostProxyEnvArgs } from "../src/lib/onboard/host-proxy-env.js"; +import { + appendHostProxyEnvArgs, + credentialHostProxyReplayEnvArgs, + HostProxyEnvironmentError, +} from "../src/lib/onboard/host-proxy-env.js"; import { isValidInferenceInputsOverride, maybePromptForInferenceInputCapability, @@ -205,6 +209,44 @@ describe("onboard helpers", () => { } }); + it("replays bounded credential proxies without collapsing upper and lower case aliases", () => { + const replay = credentialHostProxyReplayEnvArgs({ + HTTP_PROXY: "http://upper-http:upper-pass@upper-http.example.test:18080", + HTTPS_PROXY: "http://upper-https:upper-pass@upper-https.example.test:18443", + NO_PROXY: "upper.internal", + http_proxy: "http://lower-http:lower-pass@lower-http.example.test:28080", + https_proxy: "http://lower-https:lower-pass@lower-https.example.test:28443", + no_proxy: "lower.internal", + }); + + expect(replay).toEqual([ + "HTTP_PROXY=http://upper-http:upper-pass@upper-http.example.test:18080", + "HTTPS_PROXY=http://upper-https:upper-pass@upper-https.example.test:18443", + expect.stringMatching(/^NO_PROXY=upper\.internal,localhost,/u), + "http_proxy=http://lower-http:lower-pass@lower-http.example.test:28080", + "https_proxy=http://lower-https:lower-pass@lower-https.example.test:28443", + expect.stringMatching(/^no_proxy=lower\.internal,localhost,/u), + ]); + }); + + it("fails closed when credential replay is required but absent or unbounded", () => { + expect(() => + credentialHostProxyReplayEnvArgs({ + HTTP_PROXY: "http://proxy.example.test:8080", + }), + ).toThrow(/requires a credential-bearing proxy/u); + expect(() => + credentialHostProxyReplayEnvArgs({ + HTTP_PROXY: `http://operator:secret@proxy.example.test/${"x".repeat(4096)}`, + }), + ).toThrow(HostProxyEnvironmentError); + expect(() => + credentialHostProxyReplayEnvArgs({ + HTTP_PROXY: "http://operator:secret@proxy.example.test:8080\ninjected", + }), + ).toThrow(HostProxyEnvironmentError); + }); + it("propagates NEMOCLAW_MINIMAL_BOOTSTRAP=1 from host into sandbox env (#2598)", () => { const envArgs: string[] = []; appendHostProxyEnvArgs(envArgs, { NEMOCLAW_MINIMAL_BOOTSTRAP: "1" }); diff --git a/test/openclaw-final-image-layout.test.ts b/test/openclaw-final-image-layout.test.ts index 384c9a5486..2413c89765 100644 --- a/test/openclaw-final-image-layout.test.ts +++ b/test/openclaw-final-image-layout.test.ts @@ -82,6 +82,7 @@ describe("OpenClaw final image layout", () => { stage: "openclaw-runtime-payload", copies: [ "COPY scripts/lib/sandbox-init.sh /usr/local/lib/nemoclaw/sandbox-init.sh", + "COPY scripts/lib/entrypoint-env-wrapper.sh /usr/local/lib/nemoclaw/entrypoint-env-wrapper.sh", "COPY scripts/lib/gateway-supervisor.sh /usr/local/lib/nemoclaw/gateway-supervisor.sh", "COPY scripts/lib/sandbox-rlimits.sh /usr/local/lib/nemoclaw/sandbox-rlimits.sh", "COPY scripts/lib/openclaw_device_approval_policy.py /usr/local/lib/nemoclaw/openclaw_device_approval_policy.py", @@ -91,12 +92,14 @@ describe("OpenClaw final image layout", () => { "COPY scripts/openclaw-config-guard.py /usr/local/lib/nemoclaw/openclaw-config-guard.py", "COPY scripts/managed-gateway-control.py /usr/local/lib/nemoclaw/managed-gateway-control.py", "COPY scripts/nemoclaw-start.sh /usr/local/bin/nemoclaw-start", + "COPY scripts/managed-startup-hold.sh /usr/local/bin/nemoclaw-managed-startup-hold", "COPY scripts/gateway-control.sh /usr/local/bin/nemoclaw-gateway-control", "COPY nemoclaw-blueprint/scripts/*.js /usr/local/lib/nemoclaw/preloads/", "COPY --from=runtime-preload-builder /opt/nemoclaw-root/dist/lib/messaging/channels/ /usr/local/lib/nemoclaw/preloads-compiled-channels/", "COPY scripts/codex-acp-wrapper.sh /usr/local/bin/nemoclaw-codex-acp", "COPY scripts/generate-openclaw-config.mts /scripts/generate-openclaw-config.mts", "COPY scripts/validate-openclaw-tool-search.mts /scripts/validate-openclaw-tool-search.mts", + "COPY --from=managed-startup-runtime-builder /out/managed-startup-image-runtime.cjs /usr/local/lib/nemoclaw/managed-startup-image-runtime.cjs", "COPY src/lib/tool-disclosure.ts /src/lib/tool-disclosure.ts", "COPY src/lib/messaging/ /src/lib/messaging/", "COPY nemoclaw-blueprint/openclaw-plugins/ /usr/local/share/nemoclaw/openclaw-plugins/", @@ -170,7 +173,7 @@ describe("OpenClaw final image layout", () => { finalStage, "RUN mkdir -p /sandbox/.nemoclaw/blueprints/0.1.0", ); - const runtimeChmod = indexOfRequired(finalStage, "RUN chmod 755 /usr/local/bin/nemoclaw-start"); + const runtimeChmod = indexOfRequired(finalStage, "&& chmod 755 /usr/local/bin/nemoclaw-start"); const metadataCheck = indexOfRequired(finalStage, "RUN check_metadata()"); expect(dependency).toBeLessThan(tarPatch); diff --git a/test/sandbox-build-context.test.ts b/test/sandbox-build-context.test.ts index ada24769f7..8570fbadb3 100644 --- a/test/sandbox-build-context.test.ts +++ b/test/sandbox-build-context.test.ts @@ -104,6 +104,7 @@ describe("sandbox build context staging", () => { fs.chmodSync(path.join(sourceRoot, "nemoclaw-blueprint", "model-specific-setup"), 0o700); fs.chmodSync(blueprintManifestDir, 0o700); writeFixture(path.join("scripts", "nemoclaw-start.sh")); + writeFixture(path.join("scripts", "managed-startup-hold.sh")); writeFixture(path.join("scripts", "gateway-control.sh")); writeFixture(path.join("scripts", "managed-gateway-control.py")); writeFixture(path.join("scripts", "state-dir-guard.py")); @@ -118,6 +119,7 @@ describe("sandbox build context staging", () => { writeFixture(path.join("scripts", "lib", "sandbox-init.sh")); writeFixture(path.join("scripts", "lib", "gateway-supervisor.sh")); writeFixture(path.join("scripts", "lib", "sandbox-rlimits.sh")); + writeFixture(path.join("scripts", "lib", "entrypoint-env-wrapper.sh")); writeFixture(path.join("scripts", "lib", "openclaw_device_approval_policy.py")); writeFixture(path.join("scripts", "lib", "clean_runtime_shell_env_shim.py")); writeFixture(path.join("scripts", "lib", "normalize_mutable_config_perms.py")); @@ -127,6 +129,12 @@ describe("sandbox build context staging", () => { writeFixture( path.join("src", "lib", "messaging", "channels", "fixture", "hooks", "example.ts"), ); + writeFixture(path.join("src", "lib", "core", "json-types.ts")); + writeFixture(path.join("src", "lib", "core", "ports.ts")); + writeFixture(path.join("src", "lib", "onboard", "managed-startup", "image-runtime.ts")); + writeFixture(path.join("src", "lib", "security", "credential-hash.ts")); + writeFixture(path.join("src", "lib", "state", "paths.ts")); + writeFixture(path.join("src", "lib", "state", "state-root.ts")); writeFixture(path.join("src", "lib", "tool-disclosure.ts")); writeFixture(path.join("scripts", "patch-openclaw-tool-catalog.mts")); writeFixture(path.join("scripts", "patch-openclaw-chat-send.mts")); @@ -255,6 +263,19 @@ describe("sandbox build context staging", () => { expect(fs.existsSync(path.join(buildCtx, "src", "lib", "tool-disclosure.ts"))).toBe(true); } + function expectStagedManagedStartupRuntimeSources(buildCtx: string) { + for (const relativePath of [ + path.join("core", "json-types.ts"), + path.join("core", "ports.ts"), + path.join("onboard", "managed-startup", "image-runtime.ts"), + path.join("security", "credential-hash.ts"), + path.join("state", "paths.ts"), + path.join("state", "state-root.ts"), + ]) { + expect(fs.existsSync(path.join(buildCtx, "src", "lib", relativePath))).toBe(true); + } + } + function expectStagedScriptModes(buildCtx: string) { const stagedScripts = path.join(buildCtx, "scripts"); const stagedLib = path.join(stagedScripts, "lib"); @@ -334,6 +355,7 @@ describe("sandbox build context staging", () => { expectStagedOpenClawRuntimeGraphs(buildCtx, sourceRoot); expectStagedMcpToolDiscoveryRuntime(buildCtx, sourceRoot); expectStagedToolDisclosureContract(buildCtx); + expectStagedManagedStartupRuntimeSources(buildCtx); } finally { fs.rmSync(sourceRoot, { recursive: true, force: true }); fs.rmSync(tmpDir, { recursive: true, force: true }); @@ -365,6 +387,7 @@ describe("sandbox build context staging", () => { expectStagedOpenClawRuntimeGraphs(buildCtx, sourceRoot); expectStagedMcpToolDiscoveryRuntime(buildCtx, sourceRoot); expectStagedToolDisclosureContract(buildCtx); + expectStagedManagedStartupRuntimeSources(buildCtx); } finally { fs.rmSync(sourceRoot, { recursive: true, force: true }); fs.rmSync(tmpDir, { recursive: true, force: true }); @@ -472,6 +495,7 @@ describe("sandbox build context staging", () => { ); expectStagedOpenClawRuntimeGraphs(buildCtx, repoRoot); expectStagedMcpToolDiscoveryRuntime(buildCtx, repoRoot); + expectStagedManagedStartupRuntimeSources(buildCtx); expect(fs.existsSync(path.join(buildCtx, "nemoclaw-blueprint", ".venv"))).toBe(false); expect(fs.existsSync(path.join(buildCtx, "nemoclaw-blueprint", "blueprint.yaml"))).toBe(true); expect( @@ -584,6 +608,9 @@ describe("sandbox build context staging", () => { true, ); expect(fs.existsSync(path.join(buildCtx, "scripts", "lib", "sandbox-rlimits.sh"))).toBe(true); + expect( + fs.existsSync(path.join(buildCtx, "scripts", "lib", "entrypoint-env-wrapper.sh")), + ).toBe(true); expect(fs.existsSync(path.join(buildCtx, "scripts", "setup.sh"))).toBe(false); } finally { fs.rmSync(tmpDir, { recursive: true, force: true }); diff --git a/test/sandbox-provisioning-helper-permissions.test.ts b/test/sandbox-provisioning-helper-permissions.test.ts index 8b89bdd5c4..7d216a6ba8 100644 --- a/test/sandbox-provisioning-helper-permissions.test.ts +++ b/test/sandbox-provisioning-helper-permissions.test.ts @@ -129,16 +129,19 @@ describe("sandbox provisioning: copied OpenClaw helper permissions (#2861)", () const nestedPluginFile = path.join(nestedPluginDir, "helper.js"); const gatewayControlPath = path.join(localBin, "nemoclaw-gateway-control"); const gatewaySupervisorPath = path.join(localLib, "gateway-supervisor.sh"); + const entrypointEnvWrapperPath = path.join(localLib, "entrypoint-env-wrapper.sh"); const stateDirGuardPath = path.join(localLib, "state-dir-guard.py"); const configGuardPath = path.join(localLib, "openclaw-config-guard.py"); const managedGatewayControlPath = path.join(localLib, "managed-gateway-control.py"); const files = [ path.join(localBin, "nemoclaw-start"), + path.join(localBin, "nemoclaw-managed-startup-hold"), path.join(localBin, "nemoclaw-codex-acp"), gatewayControlPath, path.join(localLib, "sandbox-init.sh"), path.join(localLib, "sandbox-rlimits.sh"), gatewaySupervisorPath, + entrypointEnvWrapperPath, stateDirGuardPath, configGuardPath, managedGatewayControlPath, @@ -200,6 +203,7 @@ describe("sandbox provisioning: copied OpenClaw helper permissions (#2861)", () expect((fs.statSync(nestedPluginFile).mode & 0o777).toString(8)).toBe("644"); expect((fs.statSync(gatewayControlPath).mode & 0o777).toString(8)).toBe("700"); expect((fs.statSync(gatewaySupervisorPath).mode & 0o777).toString(8)).toBe("444"); + expect((fs.statSync(entrypointEnvWrapperPath).mode & 0o777).toString(8)).toBe("444"); expect((fs.statSync(stateDirGuardPath).mode & 0o777).toString(8)).toBe("500"); expect((fs.statSync(configGuardPath).mode & 0o777).toString(8)).toBe("500"); expect((fs.statSync(managedGatewayControlPath).mode & 0o777).toString(8)).toBe("500"); diff --git a/test/sandbox-provisioning.test.ts b/test/sandbox-provisioning.test.ts index 1a0ba6d48a..75caacbb5d 100644 --- a/test/sandbox-provisioning.test.ts +++ b/test/sandbox-provisioning.test.ts @@ -1106,6 +1106,7 @@ describe("Hermes sandbox provisioning", () => { const bashrcPath = path.join(etcDir, "bash.bashrc"); const gatewayControlPath = path.join(localBin, "nemoclaw-gateway-control"); const gatewaySupervisorPath = path.join(localLib, "gateway-supervisor.sh"); + const entrypointEnvWrapperPath = path.join(localLib, "entrypoint-env-wrapper.sh"); const buildMcpDigestPath = path.join(localLib, "build-hermes-mcp-digest.py"); const mcpConfigTransactionPath = path.join(localLib, "hermes-mcp-config-transaction.py"); const langfuseCredentialPatcherPath = path.join( @@ -1117,6 +1118,7 @@ describe("Hermes sandbox provisioning", () => { const managedGatewayControlPath = path.join(localLib, "managed-gateway-control.py"); const files = [ path.join(localBin, "nemoclaw-start"), + path.join(localBin, "nemoclaw-managed-startup-hold"), gatewayControlPath, path.join(localLib, "sandbox-init.sh"), path.join(localLib, "validate-hermes-env-secret-boundary.py"), @@ -1131,6 +1133,7 @@ describe("Hermes sandbox provisioning", () => { mcpConfigTransactionPath, mcpManifest, gatewaySupervisorPath, + entrypointEnvWrapperPath, stateDirGuardPath, managedGatewayControlPath, path.join(localLib, "sandbox-rlimits.sh"), @@ -1164,6 +1167,7 @@ describe("Hermes sandbox provisioning", () => { expect((fs.statSync(mcpManifest).mode & 0o777).toString(8)).toBe("444"); expect((fs.statSync(buildMcpDigestPath).mode & 0o777).toString(8)).toBe("444"); expect((fs.statSync(gatewaySupervisorPath).mode & 0o777).toString(8)).toBe("444"); + expect((fs.statSync(entrypointEnvWrapperPath).mode & 0o777).toString(8)).toBe("444"); expect((fs.statSync(stateDirGuardPath).mode & 0o777).toString(8)).toBe("500"); expect((fs.statSync(managedGatewayControlPath).mode & 0o777).toString(8)).toBe("500"); } finally { diff --git a/test/sandbox-rlimit-hooks.test.ts b/test/sandbox-rlimit-hooks.test.ts index 0b8fbb74e4..bdcea6344b 100644 --- a/test/sandbox-rlimit-hooks.test.ts +++ b/test/sandbox-rlimit-hooks.test.ts @@ -573,9 +573,11 @@ describe("sandbox rlimit system hooks (#2173)", () => { const safetyNet = path.join(preloadDir, "sandbox-safety-net.js"); const ciaoGuard = path.join(preloadDir, "ciao-network-guard.js"); const gatewaySupervisor = path.join(localLib, "gateway-supervisor.sh"); + const entrypointEnvWrapper = path.join(localLib, "entrypoint-env-wrapper.sh"); const stateDirGuard = path.join(localLib, "state-dir-guard.py"); const managedGatewayControl = path.join(localLib, "managed-gateway-control.py"); const startBin = path.join(tmp, "nemoclaw-start"); + const managedStartupHold = path.join(tmp, "nemoclaw-managed-startup-hold"); const gatewayControl = path.join(tmp, "nemoclaw-gateway-control"); const bashrc = path.join(tmp, "bash.bashrc"); const expectedRlimitShim = rlimitShim(rlimitLib); @@ -603,9 +605,11 @@ describe("sandbox rlimit system hooks (#2173)", () => { fs.chmodSync(safetyNet, 0o666); fs.chmodSync(ciaoGuard, 0o666); fs.writeFileSync(gatewaySupervisor, "# gateway supervisor fixture\n"); + fs.writeFileSync(entrypointEnvWrapper, "# entrypoint env wrapper fixture\n"); fs.writeFileSync(stateDirGuard, "# state-dir guard fixture\n"); fs.writeFileSync(managedGatewayControl, "# managed gateway control fixture\n"); fs.writeFileSync(startBin, "#!/usr/bin/env bash\n"); + fs.writeFileSync(managedStartupHold, "#!/usr/bin/env bash\n"); fs.writeFileSync(gatewayControl, "#!/usr/bin/env sh\n"); fs.writeFileSync(bashrc, "# stale hermes bashrc\n"); const fixtureOwner = fs.statSync(startBin); @@ -615,9 +619,11 @@ describe("sandbox rlimit system hooks (#2173)", () => { "# Wrap the hermes CLI", ) .replaceAll("/usr/local/bin/nemoclaw-start", startBin) + .replaceAll("/usr/local/bin/nemoclaw-managed-startup-hold", managedStartupHold) .replaceAll("/usr/local/bin/nemoclaw-gateway-control", gatewayControl) .replaceAll("/usr/local/lib/nemoclaw/sandbox-init.sh", initLib) .replaceAll("/usr/local/lib/nemoclaw/gateway-supervisor.sh", gatewaySupervisor) + .replaceAll("/usr/local/lib/nemoclaw/entrypoint-env-wrapper.sh", entrypointEnvWrapper) .replaceAll("/usr/local/lib/nemoclaw/validate-hermes-env-secret-boundary.py", validator) .replaceAll( "/usr/local/lib/nemoclaw/patch-hermes-session-list-preview.py", @@ -675,6 +681,7 @@ describe("sandbox rlimit system hooks (#2173)", () => { expect(fs.statSync(langfuseCredentialPatcher).mode & 0o777).toBe(0o444); expect(fs.statSync(mcpCredentialBoundary).mode & 0o777).toBe(0o444); expect(fs.statSync(buildMcpDigest).mode & 0o777).toBe(0o444); + expect(fs.statSync(entrypointEnvWrapper).mode & 0o777).toBe(0o444); expect(hardenedDir.uid).toBe(fixtureOwner.uid); expect(hardenedDir.gid).toBe(fixtureOwner.gid); expect(hardenedSafetyNet.uid).toBe(fixtureOwner.uid); diff --git a/test/security-c2-dockerfile-injection.test.ts b/test/security-c2-dockerfile-injection.test.ts index 8c4a56d301..4057752e40 100644 --- a/test/security-c2-dockerfile-injection.test.ts +++ b/test/security-c2-dockerfile-injection.test.ts @@ -14,6 +14,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { spawnSync } from "node:child_process"; +import { dockerfileInstructions } from "../src/lib/onboard/dockerfile-tool-disclosure-contract"; const DOCKERFILE = path.join(import.meta.dirname, "..", "Dockerfile"); @@ -122,27 +123,20 @@ describe("C-2 fix: env var pattern (process.env) is safe", () => { describe("Gateway auth hardening: Dockerfile must not hardcode insecure auth defaults", () => { it("NEMOCLAW_DISABLE_DEVICE_AUTH is promoted to ENV before the config generator RUN layer", () => { const src = fs.readFileSync(DOCKERFILE, "utf-8"); - const lines = src.split("\n"); let promoted = false; - let inEnvBlock = false; - for (let i = 0; i < lines.length; i++) { - const line = lines[i]; - if (/^\s*FROM\b/.test(line)) { + for (const instruction of dockerfileInstructions(src)) { + if (/^FROM\b/.test(instruction.text)) { promoted = false; - inEnvBlock = false; } - if (/^\s*ENV\b/.test(line)) { - inEnvBlock = true; - } - if (inEnvBlock && /NEMOCLAW_DISABLE_DEVICE_AUTH[=\s]/.test(line)) { + if ( + /^ENV\b/.test(instruction.text) && + /NEMOCLAW_DISABLE_DEVICE_AUTH[=\s]/.test(instruction.text) + ) { promoted = true; } - if (inEnvBlock && !/\\\s*$/.test(line)) { - inEnvBlock = false; - } if ( - /^\s*RUN\b.*node\s+--experimental-strip-types\s+\/scripts\/generate-openclaw-config\.mts\b/.test( - line, + /^RUN\s+(?:NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION=0\s+)?(?:NEMOCLAW_OPENCLAW_MANAGED_PROXY=0\s+)?node\s+--experimental-strip-types\s+\/scripts\/generate-openclaw-config\.mts$/.test( + instruction.text, ) ) { expect(promoted).toBeTruthy(); diff --git a/test/service-env.test.ts b/test/service-env.test.ts index cf66abd1c9..572874676d 100644 --- a/test/service-env.test.ts +++ b/test/service-env.test.ts @@ -24,6 +24,10 @@ import { beforeAll, describe, expect, it } from "vitest"; import { resolveOpenshell } from "../src/lib/adapters/openshell/resolve"; const NEMOCLAW_START_SCRIPT = join(import.meta.dirname, "../scripts/nemoclaw-start.sh"); +const ENTRYPOINT_ENV_WRAPPER = join( + import.meta.dirname, + "../scripts/lib/entrypoint-env-wrapper.sh", +); const RC_CLEAN_SCRIPT = join(import.meta.dirname, "../scripts/lib/clean_runtime_shell_env_shim.py"); function rcShimWrapperHeader(): string { @@ -45,14 +49,21 @@ function extractRuntimeShellEnvSnippet() { function extractOpenClawBootstrapEnvSnippet() { const src = readFileSync(NEMOCLAW_START_SCRIPT, "utf-8"); - const start = src.indexOf("# Normalize the sandbox-create bootstrap wrapper"); + const normalizer = readFileSync(ENTRYPOINT_ENV_WRAPPER, "utf-8"); + const start = src.indexOf('NEMOCLAW_CMD=("$@")'); const end = src.indexOf("# Marker file the Docker HEALTHCHECK reads", start); const extractionFailure = "Failed to extract OpenClaw bootstrap environment normalization from " + "scripts/nemoclaw-start.sh"; expect(start, extractionFailure).not.toBe(-1); expect(end, extractionFailure).toBeGreaterThan(start); - return src.slice(start, end).trimEnd(); + return [ + normalizer, + 'nemoclaw_normalize_entrypoint_env_wrapper "$@"', + 'if [ "$NEMOCLAW_ENTRYPOINT_NORMALIZED_ARGC" -eq 0 ]; then set --; ' + + 'else set -- "${NEMOCLAW_ENTRYPOINT_NORMALIZED_ARGV[@]}"; fi', + src.slice(start, end).trimEnd(), + ].join("\n"); } function extractRuntimeShellEnvShimSnippet() { diff --git a/test/support/dcode-start-script-fixture.ts b/test/support/dcode-start-script-fixture.ts index 42e881bd8d..de09adabae 100644 --- a/test/support/dcode-start-script-fixture.ts +++ b/test/support/dcode-start-script-fixture.ts @@ -21,6 +21,14 @@ export function makeStartScriptFixture(tempDir: string): { const envFile = path.join(tempDir, "proxy-env.sh"); const scriptPath = path.join(tempDir, "start.sh"); const rlimitLib = path.join(tempDir, "sandbox-rlimits.sh"); + const entrypointEnvWrapper = path.join( + import.meta.dirname, + "..", + "..", + "scripts", + "lib", + "entrypoint-env-wrapper.sh", + ); const hostFile = path.join(tempDir, "trusted-proxy-host"); const portFile = path.join(tempDir, "trusted-proxy-port"); const caFile = path.join(tempDir, "trusted-ca-bundle.pem"); @@ -28,6 +36,7 @@ export function makeStartScriptFixture(tempDir: string): { assert.ok(original.includes("local target=/tmp/nemoclaw-proxy-env.sh")); assert.ok(original.includes('tmp="$(mktemp /tmp/nemoclaw-proxy-env.XXXXXX)"')); const fixture = original + .replace("/usr/local/lib/nemoclaw/entrypoint-env-wrapper.sh", entrypointEnvWrapper) .replace("/usr/local/lib/nemoclaw/sandbox-rlimits.sh", rlimitLib) .replace( 'readonly MANAGED_PROXY_HOST_FILE="/usr/local/share/nemoclaw/dcode-proxy-host"', diff --git a/test/test-boundary-guards.test.ts b/test/test-boundary-guards.test.ts index 5d7bf23eb4..f98fb5728b 100644 --- a/test/test-boundary-guards.test.ts +++ b/test/test-boundary-guards.test.ts @@ -750,8 +750,10 @@ describe("Vitest project membership boundary", () => { ["test/install-build-dependency-preflight.test.ts", "installer-integration"], ["test/install-clone-ref.test.ts", "installer-integration"], ["test/install-express-prompt.test.ts", "installer-integration"], + ["test/install-openshell-podman-components.test.ts", "installer-integration"], ["test/install-openshell-version-pin.test.ts", "installer-integration"], ["test/install-openshell-version-check.test.ts", "installer-integration"], + ["test/install-podman-runtime.test.ts", "installer-integration"], ["test/install-preflight-docker-bootstrap.test.ts", "installer-integration"], ["test/install-preflight.test.ts", "installer-integration"], ["test/install-station-controller-binding.test.ts", "installer-integration"], diff --git a/tools/e2e/base-image-publication.mts b/tools/e2e/base-image-publication.mts index a10335dee0..40eafdc10c 100644 --- a/tools/e2e/base-image-publication.mts +++ b/tools/e2e/base-image-publication.mts @@ -22,6 +22,19 @@ const REQUEST_TIMEOUT_MS = 20_000; const MAX_RETRY_DELAY_MS = 10_000; const SHA_PATTERN = /^[0-9a-f]{40}$/u; const SAFE_PATH_PATTERN = /^[A-Za-z0-9._/-]+$/u; +const REVIEWED_PATH_GLOBS = new Map([ + ["agents/**", /^agents\/.+$/u], + ["nemoclaw/**", /^nemoclaw\/.+$/u], + ["nemoclaw-blueprint/**", /^nemoclaw-blueprint\/.+$/u], + ["scripts/**", /^scripts\/.+$/u], + [ + "src/lib/actions/sandbox/openshell-child-visible-credentials.v*.json", + /^src\/lib\/actions\/sandbox\/openshell-child-visible-credentials[.]v[^/]*[.]json$/u, + ], + ["src/lib/messaging/**", /^src\/lib\/messaging\/.+$/u], + ["src/lib/onboard/managed-startup/**", /^src\/lib\/onboard\/managed-startup\/.+$/u], + ["tools/mcp-tool-discovery-runtime/**", /^tools\/mcp-tool-discovery-runtime\/.+$/u], +]); const PENDING_RUN_STATUSES = new Set(["requested", "waiting", "pending", "queued", "in_progress"]); const COMPLETED_CONCLUSIONS = new Set([ "action_required", @@ -127,6 +140,7 @@ function parseQuotedPath(raw: string, lineNumber: number): string { if (typeof value !== "string" || value.length === 0 || value.trim() !== value) { throw new Error(`base-image push path on line ${lineNumber} must be a non-empty exact path`); } + if (REVIEWED_PATH_GLOBS.has(value)) return value; if ( !SAFE_PATH_PATTERN.test(value) || value.startsWith("/") || @@ -141,9 +155,9 @@ function parseQuotedPath(raw: string, lineNumber: number): string { } /** - * Read the literal path list that controls the publisher without requiring a - * dependency install in the preflight job. Deliberately reject YAML features - * such as flow lists, aliases, globs, and folded scalars instead of guessing. + * Read the controlled path list without requiring a dependency install in the + * preflight job. Deliberately reject YAML features such as flow lists, aliases, + * unreviewed globs, and folded scalars instead of guessing. */ export function parseBaseImagePushPaths(source: string): string[] { const lines = source.split(/\r?\n/u); @@ -226,6 +240,57 @@ function defaultGit(args: string[]): string { }).trim(); } +export function expandBaseImagePushPaths( + expectedSha: string, + paths: readonly string[], + runGit: (args: string[]) => string = defaultGit, +): string[] { + sha(expectedSha, "expected SHA"); + const expanded = new Set(); + + for (const path of paths) { + const matcher = REVIEWED_PATH_GLOBS.get(path); + if (!matcher) { + expanded.add(path); + continue; + } + + const matches = runGit([ + "log", + "--first-parent", + "--diff-merges=first-parent", + "--format=", + "--name-only", + expectedSha, + "--", + `:(glob)${path}`, + ]) + .split(/\r?\n/u) + .filter((candidate) => candidate.length > 0); + if (matches.length === 0) { + throw new Error(`reviewed base-image push glob did not match Git history: ${path}`); + } + for (const candidate of matches) { + if ( + !SAFE_PATH_PATTERN.test(candidate) || + candidate.startsWith("/") || + candidate.includes("//") || + candidate + .split("/") + .some((segment) => segment === "" || segment === "." || segment === "..") + ) { + throw new Error(`reviewed base-image push glob expanded to an unsafe path: ${candidate}`); + } + if (!matcher.test(candidate)) { + throw new Error(`Git returned a path outside reviewed base-image push glob ${path}`); + } + expanded.add(candidate); + } + } + + return [...expanded].sort(); +} + export function resolveFirstParentHistory( expectedSha: string, paths: readonly string[], @@ -243,6 +308,10 @@ export function resolveFirstParentHistory( if (runGit(["rev-parse", "--is-shallow-repository"]) !== "false") { throw new Error("base-image publication gate requires a complete Git history"); } + const expandedPaths = expandBaseImagePushPaths(expectedSha, paths, runGit); + if (expandedPaths.length === 0) { + throw new Error("base-image push paths did not resolve to any Git paths"); + } const relevantSha = runGit([ "log", @@ -252,7 +321,7 @@ export function resolveFirstParentHistory( "--format=%H", expectedSha, "--", - ...paths, + ...expandedPaths, ]); sha(relevantSha, "latest applicable base-image commit"); @@ -332,7 +401,15 @@ function validateRun(value: unknown, index: number, expectedWorkflowId: number): throw new Error(`workflow run ${index} pending state is invalid`); } } - return { id, attempt, workflowId: expectedWorkflowId, headSha, status, conclusion, url }; + return { + id, + attempt, + workflowId: expectedWorkflowId, + headSha, + status, + conclusion, + url, + }; } export function selectPublicationRun( @@ -417,7 +494,11 @@ export function validatePublisherJobs(payload: unknown, run: PublicationRun): vo if (occurrences.some((occurrence) => occurrence.attempt === attempt)) { throw new Error(`publisher job ${job.name} is duplicated in attempt ${attempt}; ${run.url}`); } - occurrences.push({ attempt, status: job.status, conclusion: job.conclusion }); + occurrences.push({ + attempt, + status: job.status, + conclusion: job.conclusion, + }); jobsByName.set(job.name, occurrences); } diff --git a/tools/e2e/workflow-boundary.mts b/tools/e2e/workflow-boundary.mts index e2035913b5..39a2250a6f 100644 --- a/tools/e2e/workflow-boundary.mts +++ b/tools/e2e/workflow-boundary.mts @@ -153,7 +153,11 @@ const PUBLIC_NVIDIA_ENDPOINT_KEY_JOBS = new Set([ "device-auth-health", "model-router-provider-routed-inference", ]); -const NO_IMAGE_E2E_JOBS = new Set(["staging-brev-launchable", SHARED_E2E_JOB_ID]); +const NO_IMAGE_E2E_JOBS = new Set([ + "podman-all-agents", + "staging-brev-launchable", + SHARED_E2E_JOB_ID, +]); const DOCKER_HUB_AUTH_STEP = "Authenticate to Docker Hub"; const DOCKER_HUB_CLEANUP_STEP = "Clean up Docker auth"; const DOCKER_HUB_CLEANUP_RUN = "bash .github/scripts/docker-auth-cleanup.sh"; diff --git a/vitest.config.ts b/vitest.config.ts index 151ed56ac6..fb0e13aeec 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -144,6 +144,7 @@ export default defineConfig({ "test/install-clone-ref.test.ts", "test/install-preflight.test.ts", "test/install-preflight-docker-bootstrap.test.ts", + "test/install-podman-runtime.test.ts", "test/install-station-controller-binding.test.ts", "test/install-station-pair-preparation.test.ts", "test/install-station-resume-cleanup.test.ts", @@ -152,6 +153,7 @@ export default defineConfig({ "test/install-station-host-preparation.test.ts", "test/install-station-package-state.test.ts", "test/install-station-package-transaction.test.ts", + "test/install-openshell-podman-components.test.ts", "test/install-openshell-version-pin.test.ts", "test/install-openshell-version-check.test.ts", ], @@ -173,6 +175,7 @@ export default defineConfig({ "test/install-clone-ref.test.ts", "test/install-preflight.test.ts", "test/install-preflight-docker-bootstrap.test.ts", + "test/install-podman-runtime.test.ts", "test/install-station-controller-binding.test.ts", "test/install-station-pair-preparation.test.ts", "test/install-station-resume-cleanup.test.ts", @@ -181,6 +184,7 @@ export default defineConfig({ "test/install-station-host-preparation.test.ts", "test/install-station-package-state.test.ts", "test/install-station-package-transaction.test.ts", + "test/install-openshell-podman-components.test.ts", "test/install-openshell-version-pin.test.ts", "test/install-openshell-version-check.test.ts", ],