diff --git a/test/e2e/e2e-cloud-experimental/checks/04-deepagents-code-fresh-reonboard.sh b/test/e2e/e2e-cloud-experimental/checks/04-deepagents-code-fresh-reonboard.sh index 3ddecd348f..2acd849b47 100755 --- a/test/e2e/e2e-cloud-experimental/checks/04-deepagents-code-fresh-reonboard.sh +++ b/test/e2e/e2e-cloud-experimental/checks/04-deepagents-code-fresh-reonboard.sh @@ -17,7 +17,17 @@ CLI="${NEMOCLAW_CLI_BIN:-${REPO}/bin/nemoclaw.js}" PREFIX="04-deepagents-code-fresh-reonboard" PRIMARY_TARGET_MODEL="openai/openai/gpt-5.5" FALLBACK_TARGET_MODEL="nvidia/nvidia/nemotron-3-ultra" -HOSTED_ENDPOINT="${NEMOCLAW_ENDPOINT_URL:-https://inference-api.nvidia.com/v1}" + +# shellcheck source=test/e2e/lib/hermetic-compatible-inference.sh +. "$REPO/test/e2e/lib/hermetic-compatible-inference.sh" + +cleanup() { + local status=$? + nemoclaw_e2e_stop_hermetic_compatible_inference + return "$status" +} + +trap cleanup EXIT fail() { printf '%s: FAIL: %s\n' "$PREFIX" "$1" >&2 @@ -65,6 +75,10 @@ encode_source() { base64 | tr -d '\n' } +shell_quote() { + printf "%s" "$1" | sed "s/'/'\\\\''/g; 1s/^/'/; \$s/\$/'/" +} + seed_config_source() { cat <<'PY' import os @@ -149,7 +163,6 @@ if ! sandbox_exec "test -d /sandbox/.deepagents && command -v dcode >/dev/null 2 exit 0 fi -[ -n "${COMPATIBLE_API_KEY:-}" ] || fail "COMPATIBLE_API_KEY is required" [ -x "$CLI" ] || fail "NemoClaw CLI is not executable at $CLI" if ! identity_before="$(dcode_identity)"; then @@ -172,8 +185,14 @@ fi [ "$model_a" != "$model_b" ] || fail "model A and model B must differ" pass "initial live identity reports model A" +export NEMOCLAW_E2E_COMPATIBLE_MODEL="$model_b" +if ! nemoclaw_e2e_start_hermetic_compatible_inference; then + fail "could not start hermetic compatible inference" +fi +pass "started hermetic compatible inference for re-onboard" + seed_source="$(seed_config_source | encode_source)" -seed_command="printf '%s' ${seed_source@Q} | base64 -d | /opt/venv/bin/python3 -I - ${model_a@Q}" +seed_command="printf '%s' $(shell_quote "$seed_source") | base64 -d | /opt/venv/bin/python3 -I - $(shell_quote "$model_a")" seed_output="$(sandbox_exec "$seed_command")" || fail "could not seed stale DCode config" printf '%s\n' "$seed_output" | grep -Fq "NEMOCLAW_DCODE_STALE_CONFIG_SEEDED" || fail "stale config seed marker is missing" pass "seeded safe preferences and stale managed data" @@ -183,8 +202,7 @@ if ! reonboard_output="$( NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ NEMOCLAW_AGENT=langchain-deepagents-code \ NEMOCLAW_COMPAT_MODEL="$model_b" \ - NEMOCLAW_E2E_USE_HOSTED_INFERENCE=1 \ - NEMOCLAW_ENDPOINT_URL="$HOSTED_ENDPOINT" \ + NEMOCLAW_ENDPOINT_URL="$NEMOCLAW_ENDPOINT_URL" \ NEMOCLAW_MODEL="$model_b" \ NEMOCLAW_NON_INTERACTIVE=1 \ NEMOCLAW_PREFERRED_API=openai-completions \ @@ -198,6 +216,7 @@ if ! reonboard_output="$( fi printf '%s\n' "$reonboard_output" | grep -Fq "Backing up workspace state before recreating sandbox..." || fail "re-onboard did not take the pre-recreate backup path" printf '%s\n' "$reonboard_output" | grep -Fq "Restoring workspace state from pre-recreate backup..." || fail "re-onboard did not take the restore path" +nemoclaw_e2e_assert_hermetic_compatible_inference_used || fail "re-onboard did not use the hermetic compatible endpoint" pass "same-name --fresh re-onboard crossed backup and restore boundaries" sandbox_list="$(openshell sandbox list 2>&1)" || fail "could not list sandbox after re-onboard" @@ -232,7 +251,7 @@ if (!entry || entry.agent !== "langchain-deepagents-code" || pass "status and registry report model B" verify_source="$(verify_config_source | encode_source)" -verify_command="printf '%s' ${verify_source@Q} | base64 -d | /opt/venv/bin/python3 -I - ${model_a@Q} ${model_b@Q}" +verify_command="printf '%s' $(shell_quote "$verify_source") | base64 -d | /opt/venv/bin/python3 -I - $(shell_quote "$model_a") $(shell_quote "$model_b")" verify_output="$(sandbox_exec "$verify_command")" || fail "live DCode config does not preserve the managed restore boundary" printf '%s\n' "$verify_output" | grep -Fq "NEMOCLAW_DCODE_FRESH_CONFIG_VERIFIED" || fail "fresh config verification marker is missing" pass "config keeps model B and only the allowlisted preferences" diff --git a/test/e2e/live/cloud-experimental-checks.ts b/test/e2e/live/cloud-experimental-checks.ts index 688dbf0bcb..52cba65c80 100644 --- a/test/e2e/live/cloud-experimental-checks.ts +++ b/test/e2e/live/cloud-experimental-checks.ts @@ -77,6 +77,10 @@ export function buildCloudExperimentalCommandEnv( }; } +export function cloudExperimentalApiKeyForCheck(scriptPath: string, apiKey: string): string { + return scriptPath === DEEPAGENTS_FRESH_REONBOARD_CHECK ? "" : apiKey; +} + export function assertRequiredCloudExperimentalResult( scriptPath: string, result: ShellProbeResult, @@ -137,11 +141,12 @@ export async function runE2eCloudExperimentalChecks( checkScripts.length > 0 ? assertDeepAgentsRuntimeObserved(sandboxName, context) : undefined, ); for (const scriptPath of checkScripts) { + const scriptApiKey = cloudExperimentalApiKeyForCheck(scriptPath, apiKey); const result = await context.host.command("bash", [path.join(REPO_ROOT, scriptPath)], { artifactName: `cloud-experimental-${path.basename(scriptPath, ".sh")}`, cwd: REPO_ROOT, - env: buildCloudExperimentalCommandEnv(sandboxName, apiKey), - redactionValues: [apiKey], + env: buildCloudExperimentalCommandEnv(sandboxName, scriptApiKey), + redactionValues: scriptApiKey ? [scriptApiKey] : [], timeoutMs: cloudExperimentalCheckTimeoutMs(scriptPath), }); assertRequiredCloudExperimentalResult(scriptPath, result); diff --git a/test/e2e/support/platform-parity-cloud-experimental.test.ts b/test/e2e/support/platform-parity-cloud-experimental.test.ts index 1e86941d1a..0f4132fade 100644 --- a/test/e2e/support/platform-parity-cloud-experimental.test.ts +++ b/test/e2e/support/platform-parity-cloud-experimental.test.ts @@ -15,11 +15,15 @@ vi.mock("../../../src/lib/actions/sandbox/exec", () => ({ import SandboxExecCommand from "../../../src/commands/sandbox/exec.ts"; import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; -import { DEEPAGENTS_CLOUD_EXPERIMENTAL_CHECKS } from "../live/cloud-experimental-check-list.ts"; +import { + DEEPAGENTS_CLOUD_EXPERIMENTAL_CHECKS, + DEEPAGENTS_FRESH_REONBOARD_CHECK, +} from "../live/cloud-experimental-check-list.ts"; import { assertRequiredCloudExperimentalResult, buildCloudExperimentalChecksEvidence, buildCloudExperimentalCommandEnv, + cloudExperimentalApiKeyForCheck, cloudExperimentalCheckTimeoutMs, } from "../live/cloud-experimental-checks.ts"; @@ -27,6 +31,12 @@ const dcodeTavilyCheck = path.join( process.cwd(), "test/e2e/e2e-cloud-experimental/checks/09-deepagents-code-tavily-opt-in.sh", ); +const freshReonboardCheck = path.join(process.cwd(), DEEPAGENTS_FRESH_REONBOARD_CHECK); +const freshReonboardTimeoutMs = 30_000; + +function writeExecutable(filePath: string, lines: string[]): void { + fs.writeFileSync(filePath, `${lines.join("\n")}\n`, { mode: 0o755 }); +} function shellResult(exitCode: number, stdout: string, stderr = ""): ShellProbeResult { return { @@ -49,22 +59,13 @@ describe("P0-E cloud-experimental parity guardrails", () => { const binDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-fake-openshell-")); try { fs.writeFileSync(path.join(binDir, "openshell"), "#!/bin/sh\nexit 1\n", { mode: 0o755 }); - const result = spawnSync( - "bash", - [ - path.join( - process.cwd(), - "test/e2e/e2e-cloud-experimental/checks/04-deepagents-code-fresh-reonboard.sh", - ), - ], - { - encoding: "utf8", - env: { - PATH: `${binDir}:${process.env.PATH ?? "/usr/bin:/bin"}`, - SANDBOX_NAME: "openclaw-sandbox", - }, + const result = spawnSync("bash", [freshReonboardCheck], { + encoding: "utf8", + env: { + PATH: `${binDir}:${process.env.PATH ?? "/usr/bin:/bin"}`, + SANDBOX_NAME: "openclaw-sandbox", }, - ); + }); expect(result.status, result.stderr).toBe(0); expect(result.stdout).toContain( @@ -75,6 +76,134 @@ describe("P0-E cloud-experimental parity guardrails", () => { } }); + it( + "runs fresh re-onboard with a fake endpoint and no hosted inference secret", + () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-fresh-reonboard-")); + try { + const binDir = path.join(tmp, "bin"); + const homeDir = path.join(tmp, "home"); + fs.mkdirSync(binDir); + fs.mkdirSync(homeDir); + const marker = path.join(tmp, "reonboard-done"); + const openshell = path.join(binDir, "openshell"); + const cli = path.join(binDir, "nemoclaw"); + + writeExecutable(openshell, [ + "#!/usr/bin/env bash", + "set -euo pipefail", + 'if [ "${1:-}" = "sandbox" ] && [ "${2:-}" = "exec" ]; then', + " shift 2", + ' [ "${1:-}" = "--name" ] || { echo "missing --name" >&2; exit 2; }', + " shift 2", + ' [ "${1:-}" = "--" ] || { echo "missing command boundary" >&2; exit 2; }', + " shift", + ' if [ "${1:-}" = "bash" ] && [ "${2:-}" = "-c" ]; then', + ' case "${3:-}" in', + ' *"test -d /sandbox/.deepagents"*) exit 0 ;;', + " *)", + ' if [ -f "$FAKE_REONBOARD_DONE" ]; then', + ' printf "%s\\n" "NEMOCLAW_DCODE_FRESH_CONFIG_VERIFIED"', + " else", + ' printf "%s\\n" "NEMOCLAW_DCODE_STALE_CONFIG_SEEDED"', + " fi", + " exit 0", + " ;;", + " esac", + " fi", + ' if [ "${1:-}" = "/usr/local/bin/dcode" ] && [ "${2:-}" = "identity" ]; then', + ' if [ -f "$FAKE_REONBOARD_DONE" ]; then', + ' model="openai/openai/gpt-5.5"', + " else", + ' model="nvidia/nvidia/nemotron-3-ultra"', + " fi", + ' printf "Route: inference\\nProvider: compatible-endpoint\\nModel: openai:%s\\nEndpoint: https://inference.local/v1\\n" "$model"', + " exit 0", + " fi", + "fi", + 'if [ "${1:-}" = "sandbox" ] && [ "${2:-}" = "list" ]; then', + ' printf "%s\\n" "deepagents-sandbox Ready"', + " exit 0", + "fi", + 'printf "unexpected openshell args: %s\\n" "$*" >&2', + "exit 2", + ]); + + writeExecutable(cli, [ + "#!/usr/bin/env bash", + "set -euo pipefail", + "fail() { printf 'fake nemoclaw: %s\\n' \"$1\" >&2; exit 2; }", + 'if [ "${1:-}" = "onboard" ]; then', + ' [ -z "${NVIDIA_INFERENCE_API_KEY:-}" ] || fail "NVIDIA_INFERENCE_API_KEY leaked"', + ' [ "${NEMOCLAW_E2E_USE_HOSTED_INFERENCE:-}" != "1" ] || fail "hosted inference flag leaked"', + ' [ "${COMPATIBLE_API_KEY:-}" = "e2e-compatible-key" ] || fail "missing fake compatible key"', + ' [ "${NEMOCLAW_PROVIDER:-}" = "custom" ] || fail "missing custom provider"', + ' [ "${NEMOCLAW_MODEL:-}" = "openai/openai/gpt-5.5" ] || fail "wrong target model"', + ' case "${NEMOCLAW_ENDPOINT_URL:-}" in', + " http://127.0.0.1:*/v1) ;;", + ' *) fail "unexpected endpoint ${NEMOCLAW_ENDPOINT_URL:-}" ;;', + " esac", + " node --input-type=module <<'NODE'", + "const response = await fetch(`${process.env.NEMOCLAW_ENDPOINT_URL}/chat/completions`, {", + ' method: "POST",', + " headers: {", + ' "content-type": "application/json",', + " authorization: `Bearer ${process.env.COMPATIBLE_API_KEY}`", + " },", + " body: JSON.stringify({", + " model: process.env.NEMOCLAW_MODEL,", + ' messages: [{ role: "user", content: "ping" }]', + " })", + "});", + "if (!response.ok) {", + " console.error(await response.text());", + " process.exit(1);", + "}", + "NODE", + ' mkdir -p "$HOME/.nemoclaw"', + ' printf "%s\\n" \'{"sandboxes":{"deepagents-sandbox":{"agent":"langchain-deepagents-code","model":"openai/openai/gpt-5.5","provider":"compatible-endpoint","credentialEnv":"COMPATIBLE_API_KEY"}}}\' > "$HOME/.nemoclaw/sandboxes.json"', + ' touch "$FAKE_REONBOARD_DONE"', + ' printf "%s\\n" "Backing up workspace state before recreating sandbox..."', + ' printf "%s\\n" "Restoring workspace state from pre-recreate backup..."', + " exit 0", + "fi", + 'if [ "${1:-}" = "deepagents-sandbox" ] && [ "${2:-}" = "status" ] && [ "${3:-}" = "--json" ]; then', + ' printf "%s\\n" \'{"name":"deepagents-sandbox","model":"openai/openai/gpt-5.5","provider":"compatible-endpoint"}\'', + " exit 0", + "fi", + 'printf "unexpected nemoclaw args: %s\\n" "$*" >&2', + "exit 2", + ]); + + const result = spawnSync("bash", [freshReonboardCheck], { + encoding: "utf8", + timeout: freshReonboardTimeoutMs, + env: { + FAKE_OPENAI_PUBLIC_HOST: "127.0.0.1", + FAKE_REONBOARD_DONE: marker, + HOME: homeDir, + NEMOCLAW_CLI_BIN: cli, + NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1", + NEMOCLAW_ENDPOINT_URL: "https://inference-api.nvidia.com/v1", + NVIDIA_INFERENCE_API_KEY: "sk-live-should-not-be-used", + PATH: `${binDir}:${process.env.PATH ?? "/usr/bin:/bin"}`, + REPO: process.cwd(), + SANDBOX_NAME: "deepagents-sandbox", + }, + }); + + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + expect(result.stdout).toContain( + "04-deepagents-code-fresh-reonboard: OK (started hermetic compatible inference for re-onboard)", + ); + expect(result.stdout).toContain("04-deepagents-code-fresh-reonboard: 6 passed, 0 failed"); + } finally { + fs.rmSync(tmp, { force: true, recursive: true }); + } + }, + freshReonboardTimeoutMs, + ); + it("preserves the repeated env-unset pairs from the failed observability invocation", async () => { await SandboxExecCommand.run( [ @@ -405,6 +534,18 @@ describe("P0-E cloud-experimental parity guardrails", () => { ).toBe(35 * 60_000); }); + it("does not pass the hosted inference key into the hermetic fresh re-onboard check", () => { + expect(cloudExperimentalApiKeyForCheck(DEEPAGENTS_FRESH_REONBOARD_CHECK, "secret-key")).toBe( + "", + ); + expect( + cloudExperimentalApiKeyForCheck( + "test/e2e/e2e-cloud-experimental/checks/07-deepagents-code-headless-inference.sh", + "secret-key", + ), + ).toBe("secret-key"); + }); + it("documents Deep Agents check scripts in generated launch/QA evidence", () => { const evidence = buildCloudExperimentalChecksEvidence( "cloud-langchain-deepagents-code",