Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,17 @@ CLI="${NEMOCLAW_CLI_BIN:-${REPO}/bin/nemoclaw.js}"
PREFIX="04-deepagents-code-fresh-reonboard"
PRIMARY_TARGET_MODEL="openai/openai/gpt-5.5"
FALLBACK_TARGET_MODEL="nvidia/nvidia/nemotron-3-ultra"
HOSTED_ENDPOINT="${NEMOCLAW_ENDPOINT_URL:-https://inference-api.nvidia.com/v1}"

# shellcheck source=test/e2e/lib/hermetic-compatible-inference.sh
. "$REPO/test/e2e/lib/hermetic-compatible-inference.sh"

cleanup() {
local status=$?
nemoclaw_e2e_stop_hermetic_compatible_inference
return "$status"
}

trap cleanup EXIT

fail() {
printf '%s: FAIL: %s\n' "$PREFIX" "$1" >&2
Expand Down Expand Up @@ -65,6 +75,10 @@ encode_source() {
base64 | tr -d '\n'
}

shell_quote() {
printf "%s" "$1" | sed "s/'/'\\\\''/g; 1s/^/'/; \$s/\$/'/"
}

seed_config_source() {
cat <<'PY'
import os
Expand Down Expand Up @@ -149,7 +163,6 @@ if ! sandbox_exec "test -d /sandbox/.deepagents && command -v dcode >/dev/null 2
exit 0
fi

[ -n "${COMPATIBLE_API_KEY:-}" ] || fail "COMPATIBLE_API_KEY is required"
[ -x "$CLI" ] || fail "NemoClaw CLI is not executable at $CLI"

if ! identity_before="$(dcode_identity)"; then
Expand All @@ -172,8 +185,14 @@ fi
[ "$model_a" != "$model_b" ] || fail "model A and model B must differ"
pass "initial live identity reports model A"

export NEMOCLAW_E2E_COMPATIBLE_MODEL="$model_b"
if ! nemoclaw_e2e_start_hermetic_compatible_inference; then
fail "could not start hermetic compatible inference"
fi
pass "started hermetic compatible inference for re-onboard"

seed_source="$(seed_config_source | encode_source)"
seed_command="printf '%s' ${seed_source@Q} | base64 -d | /opt/venv/bin/python3 -I - ${model_a@Q}"
seed_command="printf '%s' $(shell_quote "$seed_source") | base64 -d | /opt/venv/bin/python3 -I - $(shell_quote "$model_a")"
seed_output="$(sandbox_exec "$seed_command")" || fail "could not seed stale DCode config"
printf '%s\n' "$seed_output" | grep -Fq "NEMOCLAW_DCODE_STALE_CONFIG_SEEDED" || fail "stale config seed marker is missing"
pass "seeded safe preferences and stale managed data"
Expand All @@ -183,8 +202,7 @@ if ! reonboard_output="$(
NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \
NEMOCLAW_AGENT=langchain-deepagents-code \
NEMOCLAW_COMPAT_MODEL="$model_b" \
NEMOCLAW_E2E_USE_HOSTED_INFERENCE=1 \
NEMOCLAW_ENDPOINT_URL="$HOSTED_ENDPOINT" \
NEMOCLAW_ENDPOINT_URL="$NEMOCLAW_ENDPOINT_URL" \
NEMOCLAW_MODEL="$model_b" \
NEMOCLAW_NON_INTERACTIVE=1 \
NEMOCLAW_PREFERRED_API=openai-completions \
Expand All @@ -198,6 +216,7 @@ if ! reonboard_output="$(
fi
printf '%s\n' "$reonboard_output" | grep -Fq "Backing up workspace state before recreating sandbox..." || fail "re-onboard did not take the pre-recreate backup path"
printf '%s\n' "$reonboard_output" | grep -Fq "Restoring workspace state from pre-recreate backup..." || fail "re-onboard did not take the restore path"
nemoclaw_e2e_assert_hermetic_compatible_inference_used || fail "re-onboard did not use the hermetic compatible endpoint"
pass "same-name --fresh re-onboard crossed backup and restore boundaries"

sandbox_list="$(openshell sandbox list 2>&1)" || fail "could not list sandbox after re-onboard"
Expand Down Expand Up @@ -232,7 +251,7 @@ if (!entry || entry.agent !== "langchain-deepagents-code" ||
pass "status and registry report model B"

verify_source="$(verify_config_source | encode_source)"
verify_command="printf '%s' ${verify_source@Q} | base64 -d | /opt/venv/bin/python3 -I - ${model_a@Q} ${model_b@Q}"
verify_command="printf '%s' $(shell_quote "$verify_source") | base64 -d | /opt/venv/bin/python3 -I - $(shell_quote "$model_a") $(shell_quote "$model_b")"
verify_output="$(sandbox_exec "$verify_command")" || fail "live DCode config does not preserve the managed restore boundary"
printf '%s\n' "$verify_output" | grep -Fq "NEMOCLAW_DCODE_FRESH_CONFIG_VERIFIED" || fail "fresh config verification marker is missing"
pass "config keeps model B and only the allowlisted preferences"
Expand Down
9 changes: 7 additions & 2 deletions test/e2e/live/cloud-experimental-checks.ts
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,10 @@ export function buildCloudExperimentalCommandEnv(
};
}

export function cloudExperimentalApiKeyForCheck(scriptPath: string, apiKey: string): string {
return scriptPath === DEEPAGENTS_FRESH_REONBOARD_CHECK ? "" : apiKey;
}

export function assertRequiredCloudExperimentalResult(
scriptPath: string,
result: ShellProbeResult,
Expand Down Expand Up @@ -137,11 +141,12 @@ export async function runE2eCloudExperimentalChecks(
checkScripts.length > 0 ? assertDeepAgentsRuntimeObserved(sandboxName, context) : undefined,
);
for (const scriptPath of checkScripts) {
const scriptApiKey = cloudExperimentalApiKeyForCheck(scriptPath, apiKey);
const result = await context.host.command("bash", [path.join(REPO_ROOT, scriptPath)], {
artifactName: `cloud-experimental-${path.basename(scriptPath, ".sh")}`,
cwd: REPO_ROOT,
env: buildCloudExperimentalCommandEnv(sandboxName, apiKey),
redactionValues: [apiKey],
env: buildCloudExperimentalCommandEnv(sandboxName, scriptApiKey),
redactionValues: scriptApiKey ? [scriptApiKey] : [],
timeoutMs: cloudExperimentalCheckTimeoutMs(scriptPath),
});
assertRequiredCloudExperimentalResult(scriptPath, result);
Expand Down
173 changes: 157 additions & 16 deletions test/e2e/support/platform-parity-cloud-experimental.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,18 +15,28 @@ vi.mock("../../../src/lib/actions/sandbox/exec", () => ({

import SandboxExecCommand from "../../../src/commands/sandbox/exec.ts";
import type { ShellProbeResult } from "../fixtures/shell-probe.ts";
import { DEEPAGENTS_CLOUD_EXPERIMENTAL_CHECKS } from "../live/cloud-experimental-check-list.ts";
import {
DEEPAGENTS_CLOUD_EXPERIMENTAL_CHECKS,
DEEPAGENTS_FRESH_REONBOARD_CHECK,
} from "../live/cloud-experimental-check-list.ts";
import {
assertRequiredCloudExperimentalResult,
buildCloudExperimentalChecksEvidence,
buildCloudExperimentalCommandEnv,
cloudExperimentalApiKeyForCheck,
cloudExperimentalCheckTimeoutMs,
} from "../live/cloud-experimental-checks.ts";

const dcodeTavilyCheck = path.join(
process.cwd(),
"test/e2e/e2e-cloud-experimental/checks/09-deepagents-code-tavily-opt-in.sh",
);
const freshReonboardCheck = path.join(process.cwd(), DEEPAGENTS_FRESH_REONBOARD_CHECK);
const freshReonboardTimeoutMs = 30_000;

function writeExecutable(filePath: string, lines: string[]): void {
fs.writeFileSync(filePath, `${lines.join("\n")}\n`, { mode: 0o755 });
}

function shellResult(exitCode: number, stdout: string, stderr = ""): ShellProbeResult {
return {
Expand All @@ -49,22 +59,13 @@ describe("P0-E cloud-experimental parity guardrails", () => {
const binDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-fake-openshell-"));
try {
fs.writeFileSync(path.join(binDir, "openshell"), "#!/bin/sh\nexit 1\n", { mode: 0o755 });
const result = spawnSync(
"bash",
[
path.join(
process.cwd(),
"test/e2e/e2e-cloud-experimental/checks/04-deepagents-code-fresh-reonboard.sh",
),
],
{
encoding: "utf8",
env: {
PATH: `${binDir}:${process.env.PATH ?? "/usr/bin:/bin"}`,
SANDBOX_NAME: "openclaw-sandbox",
},
const result = spawnSync("bash", [freshReonboardCheck], {
encoding: "utf8",
env: {
PATH: `${binDir}:${process.env.PATH ?? "/usr/bin:/bin"}`,
SANDBOX_NAME: "openclaw-sandbox",
},
);
});

expect(result.status, result.stderr).toBe(0);
expect(result.stdout).toContain(
Expand All @@ -75,6 +76,134 @@ describe("P0-E cloud-experimental parity guardrails", () => {
}
});

it(
"runs fresh re-onboard with a fake endpoint and no hosted inference secret",
() => {
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-fresh-reonboard-"));
try {
const binDir = path.join(tmp, "bin");
const homeDir = path.join(tmp, "home");
fs.mkdirSync(binDir);
fs.mkdirSync(homeDir);
const marker = path.join(tmp, "reonboard-done");
const openshell = path.join(binDir, "openshell");
const cli = path.join(binDir, "nemoclaw");

writeExecutable(openshell, [
"#!/usr/bin/env bash",
"set -euo pipefail",
'if [ "${1:-}" = "sandbox" ] && [ "${2:-}" = "exec" ]; then',
" shift 2",
' [ "${1:-}" = "--name" ] || { echo "missing --name" >&2; exit 2; }',
" shift 2",
' [ "${1:-}" = "--" ] || { echo "missing command boundary" >&2; exit 2; }',
" shift",
' if [ "${1:-}" = "bash" ] && [ "${2:-}" = "-c" ]; then',
' case "${3:-}" in',
' *"test -d /sandbox/.deepagents"*) exit 0 ;;',
" *)",
' if [ -f "$FAKE_REONBOARD_DONE" ]; then',
' printf "%s\\n" "NEMOCLAW_DCODE_FRESH_CONFIG_VERIFIED"',
" else",
' printf "%s\\n" "NEMOCLAW_DCODE_STALE_CONFIG_SEEDED"',
" fi",
" exit 0",
" ;;",
" esac",
" fi",
' if [ "${1:-}" = "/usr/local/bin/dcode" ] && [ "${2:-}" = "identity" ]; then',
' if [ -f "$FAKE_REONBOARD_DONE" ]; then',
' model="openai/openai/gpt-5.5"',
" else",
' model="nvidia/nvidia/nemotron-3-ultra"',
" fi",
' printf "Route: inference\\nProvider: compatible-endpoint\\nModel: openai:%s\\nEndpoint: https://inference.local/v1\\n" "$model"',
" exit 0",
" fi",
"fi",
'if [ "${1:-}" = "sandbox" ] && [ "${2:-}" = "list" ]; then',
' printf "%s\\n" "deepagents-sandbox Ready"',
" exit 0",
"fi",
'printf "unexpected openshell args: %s\\n" "$*" >&2',
"exit 2",
]);

writeExecutable(cli, [
"#!/usr/bin/env bash",
"set -euo pipefail",
"fail() { printf 'fake nemoclaw: %s\\n' \"$1\" >&2; exit 2; }",
'if [ "${1:-}" = "onboard" ]; then',
' [ -z "${NVIDIA_INFERENCE_API_KEY:-}" ] || fail "NVIDIA_INFERENCE_API_KEY leaked"',
' [ "${NEMOCLAW_E2E_USE_HOSTED_INFERENCE:-}" != "1" ] || fail "hosted inference flag leaked"',
' [ "${COMPATIBLE_API_KEY:-}" = "e2e-compatible-key" ] || fail "missing fake compatible key"',
' [ "${NEMOCLAW_PROVIDER:-}" = "custom" ] || fail "missing custom provider"',
' [ "${NEMOCLAW_MODEL:-}" = "openai/openai/gpt-5.5" ] || fail "wrong target model"',
' case "${NEMOCLAW_ENDPOINT_URL:-}" in',
" http://127.0.0.1:*/v1) ;;",
' *) fail "unexpected endpoint ${NEMOCLAW_ENDPOINT_URL:-<unset>}" ;;',
" esac",
" node --input-type=module <<'NODE'",
"const response = await fetch(`${process.env.NEMOCLAW_ENDPOINT_URL}/chat/completions`, {",
' method: "POST",',
" headers: {",
' "content-type": "application/json",',
" authorization: `Bearer ${process.env.COMPATIBLE_API_KEY}`",
" },",
" body: JSON.stringify({",
" model: process.env.NEMOCLAW_MODEL,",
' messages: [{ role: "user", content: "ping" }]',
" })",
"});",
"if (!response.ok) {",
" console.error(await response.text());",
" process.exit(1);",
"}",
"NODE",
' mkdir -p "$HOME/.nemoclaw"',
' printf "%s\\n" \'{"sandboxes":{"deepagents-sandbox":{"agent":"langchain-deepagents-code","model":"openai/openai/gpt-5.5","provider":"compatible-endpoint","credentialEnv":"COMPATIBLE_API_KEY"}}}\' > "$HOME/.nemoclaw/sandboxes.json"',
' touch "$FAKE_REONBOARD_DONE"',
' printf "%s\\n" "Backing up workspace state before recreating sandbox..."',
' printf "%s\\n" "Restoring workspace state from pre-recreate backup..."',
" exit 0",
"fi",
'if [ "${1:-}" = "deepagents-sandbox" ] && [ "${2:-}" = "status" ] && [ "${3:-}" = "--json" ]; then',
' printf "%s\\n" \'{"name":"deepagents-sandbox","model":"openai/openai/gpt-5.5","provider":"compatible-endpoint"}\'',
" exit 0",
"fi",
'printf "unexpected nemoclaw args: %s\\n" "$*" >&2',
"exit 2",
]);

const result = spawnSync("bash", [freshReonboardCheck], {
encoding: "utf8",
timeout: freshReonboardTimeoutMs,
env: {
FAKE_OPENAI_PUBLIC_HOST: "127.0.0.1",
FAKE_REONBOARD_DONE: marker,
HOME: homeDir,
NEMOCLAW_CLI_BIN: cli,
NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1",
NEMOCLAW_ENDPOINT_URL: "https://inference-api.nvidia.com/v1",
NVIDIA_INFERENCE_API_KEY: "sk-live-should-not-be-used",
PATH: `${binDir}:${process.env.PATH ?? "/usr/bin:/bin"}`,
REPO: process.cwd(),
SANDBOX_NAME: "deepagents-sandbox",
},
});

expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0);
expect(result.stdout).toContain(
"04-deepagents-code-fresh-reonboard: OK (started hermetic compatible inference for re-onboard)",
);
expect(result.stdout).toContain("04-deepagents-code-fresh-reonboard: 6 passed, 0 failed");
} finally {
fs.rmSync(tmp, { force: true, recursive: true });
}
},
freshReonboardTimeoutMs,
);

it("preserves the repeated env-unset pairs from the failed observability invocation", async () => {
await SandboxExecCommand.run(
[
Expand Down Expand Up @@ -405,6 +534,18 @@ describe("P0-E cloud-experimental parity guardrails", () => {
).toBe(35 * 60_000);
});

it("does not pass the hosted inference key into the hermetic fresh re-onboard check", () => {
expect(cloudExperimentalApiKeyForCheck(DEEPAGENTS_FRESH_REONBOARD_CHECK, "secret-key")).toBe(
"",
);
expect(
cloudExperimentalApiKeyForCheck(
"test/e2e/e2e-cloud-experimental/checks/07-deepagents-code-headless-inference.sh",
"secret-key",
),
).toBe("secret-key");
});

it("documents Deep Agents check scripts in generated launch/QA evidence", () => {
const evidence = buildCloudExperimentalChecksEvidence(
"cloud-langchain-deepagents-code",
Expand Down