diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index f0358ab595..e586511e2e 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -356,6 +356,7 @@ jobs: e2e-artifacts/live/${{ matrix.id }}/run-plan.json e2e-artifacts/live/${{ matrix.id }}/target.json e2e-artifacts/live/${{ matrix.id }}/target-result.json + e2e-artifacts/live/${{ matrix.id }}/test-progress.json e2e-artifacts/live/${{ matrix.id }}/environment.result.json e2e-artifacts/live/${{ matrix.id }}/onboarding.result.json e2e-artifacts/live/${{ matrix.id }}/state-validation.result.json @@ -3875,18 +3876,83 @@ jobs: shell: bash run: bash .github/scripts/docker-auth-cleanup.sh - # Scheduled coverage for #6108 / #3513 / #3127. This exercises the complete - # managed v0.0.71 runtime with a custom plugin, then proves restart/rebuild - # persistence and target-side runtime-dependency replacement across devices. + # Scheduled release-baseline coverage for #6108. This runs independently of + # the current lifecycle lane so the exact v0.0.71 build does not extend that + # lane's critical path. + openclaw-plugin-runtime-exdev-release: + needs: generate-matrix + if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',openclaw-plugin-runtime-exdev-release,') || contains(format(',{0},', inputs.targets), ',openclaw-plugin-runtime-exdev-release,') }} + runs-on: ubuntu-latest + permissions: + contents: read + timeout-minutes: 55 + env: + E2E_JOB: "1" + E2E_TARGET_ID: "openclaw-plugin-runtime-exdev-release" + E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/openclaw-plugin-runtime-exdev-release + NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js + NEMOCLAW_RUN_LIVE_E2E: "1" + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + NEMOCLAW_SANDBOX_NAME: "e2e-openclaw-plugin-exdev-release" + OPENSHELL_GATEWAY: "nemoclaw" + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + ref: ${{ inputs.checkout_sha || github.sha }} + persist-credentials: false + + - *dockerhub-auth + + - name: Pre-pull release-matched Docker Hub builder image + shell: bash + run: | + set -euo pipefail + docker pull node:22-trixie-slim@sha256:2d9f5c76c8f4dd36e8f253bee5d828a83a6c09f36188f0b0414325232e0b175d + + - name: Remove Docker auth before release-pinned fixture + if: always() + shell: bash + run: | + set -euo pipefail + bash .github/scripts/docker-auth-cleanup.sh + + - name: Prepare E2E workspace + uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@50281ee84c4a6fc759da95ea28fc0b7d9c378a28 + + - name: Run OpenClaw custom-plugin release baseline live test + run: | + set -euo pipefail + test -n "${DOCKER_CONFIG:-}" + test ! -e "${DOCKER_CONFIG}" + test -z "${DOCKERHUB_USERNAME:-}" + test -z "${DOCKERHUB_TOKEN:-}" + env -u DOCKER_CONFIG -u DOCKERHUB_USERNAME -u DOCKERHUB_TOKEN \ + npx tsx tools/e2e/live-vitest-invocation.mts run \ + --test-path test/e2e/live/openclaw-plugin-runtime-exdev.test.ts \ + --selector release-baseline + + - name: Upload OpenClaw plugin release baseline artifacts + if: always() + uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 + + - name: Clean up Docker auth + if: always() + shell: bash + run: bash .github/scripts/docker-auth-cleanup.sh + + # Scheduled current-lifecycle coverage for #6108 / #3513 / #3127. This + # proves restart/rebuild persistence and target-side runtime-dependency + # replacement across devices without waiting for the release baseline. openclaw-plugin-runtime-exdev: needs: generate-matrix if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',openclaw-plugin-runtime-exdev,') || contains(format(',{0},', inputs.targets), ',openclaw-plugin-runtime-exdev,') }} runs-on: ubuntu-latest permissions: contents: read - # Three bounded 25-minute onboards plus the 20-minute rebuild and 15-minute - # Vitest buffer need 110 minutes; allow 20 more for setup and teardown. - timeout-minutes: 130 + # Two bounded 25-minute onboards plus the 20-minute rebuild and 15-minute + # Vitest buffer need 85 minutes; allow 20 more for setup and teardown. + timeout-minutes: 105 env: E2E_JOB: "1" E2E_TARGET_ID: "openclaw-plugin-runtime-exdev" @@ -3929,7 +3995,9 @@ jobs: test -z "${DOCKERHUB_USERNAME:-}" test -z "${DOCKERHUB_TOKEN:-}" env -u DOCKER_CONFIG -u DOCKERHUB_USERNAME -u DOCKERHUB_TOKEN \ - npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/openclaw-plugin-runtime-exdev.test.ts + npx tsx tools/e2e/live-vitest-invocation.mts run \ + --test-path test/e2e/live/openclaw-plugin-runtime-exdev.test.ts \ + --selector current-lifecycle - name: Upload OpenClaw plugin runtime-deps EXDEV artifacts if: always() @@ -4913,6 +4981,7 @@ jobs: sandbox-survival, diagnostics, snapshot-commands, + openclaw-plugin-runtime-exdev-release, openclaw-plugin-runtime-exdev, openclaw-tui-chat-correlation, gateway-guard-recovery, diff --git a/.github/workflows/regression-e2e.yaml b/.github/workflows/regression-e2e.yaml index 68ab1e43dd..466a917adf 100644 --- a/.github/workflows/regression-e2e.yaml +++ b/.github/workflows/regression-e2e.yaml @@ -288,11 +288,63 @@ jobs: /tmp/nemoclaw-e2e-model-router-response.log if-no-files-found: ignore - # ── OpenClaw custom-plugin lifecycle and runtime-deps EXDEV E2E ─ - # Coverage guard for #6108 / #3513 / #3127. On Ubuntu/OpenShell sandbox layouts - # where /tmp and /sandbox can live on different filesystems, OpenClaw's - # first CLI bootstrap must not fail plugin runtime dependency installation - # with EXDEV cross-device rename errors. + # ── OpenClaw release-baseline custom-plugin E2E ──────────────── + # The exact v0.0.71 baseline runs in parallel with the current lifecycle + # contract so release provenance does not extend the EXDEV critical path. + openclaw-plugin-runtime-exdev-release-e2e: + needs: select_regression_jobs + if: >- + github.repository == 'NVIDIA/NemoClaw' && + needs.select_regression_jobs.outputs.openclaw_plugin_runtime_exdev == 'true' + runs-on: ubuntu-latest + permissions: + contents: read + timeout-minutes: 55 + steps: + - name: Checkout + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + - name: Setup Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0 + with: + node-version: "22" + cache: npm + + - name: Install root dependencies + run: npm ci --ignore-scripts + + - name: Build CLI + run: npm run build:cli + + - name: Run OpenClaw custom-plugin release baseline Vitest test + env: + E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/openclaw-plugin-runtime-exdev-release + E2E_TARGET_ID: openclaw-plugin-runtime-exdev-release + NEMOCLAW_RUN_LIVE_E2E: "1" + NEMOCLAW_SANDBOX_NAME: e2e-openclaw-plugin-exdev-release + run: | + set -euo pipefail + npx vitest run --project e2e-live \ + test/e2e/live/openclaw-plugin-runtime-exdev.test.ts \ + -t release-baseline \ + --silent=false --reporter=default + + - name: Upload OpenClaw plugin release baseline artifacts + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: openclaw-plugin-runtime-exdev-release-artifacts + path: e2e-artifacts/live/openclaw-plugin-runtime-exdev-release/ + include-hidden-files: false + if-no-files-found: ignore + retention-days: 14 + + # ── OpenClaw current lifecycle and runtime-deps EXDEV E2E ───── + # Coverage guard for #6108 / #3513 / #3127. On Ubuntu/OpenShell sandbox + # layouts where /tmp and /sandbox can live on different filesystems, the + # runtime dependency replacement must complete without EXDEV failures. openclaw-plugin-runtime-exdev-e2e: needs: select_regression_jobs if: >- @@ -301,9 +353,9 @@ jobs: runs-on: ubuntu-latest permissions: contents: read - # Three bounded 25-minute onboards plus the 20-minute rebuild and 15-minute - # Vitest buffer need 110 minutes; allow 20 more for setup and teardown. - timeout-minutes: 130 + # Two bounded 25-minute onboards plus the 20-minute rebuild and 15-minute + # Vitest buffer need 85 minutes; allow 20 more for setup and teardown. + timeout-minutes: 105 steps: - name: Checkout uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 @@ -325,11 +377,13 @@ jobs: - name: Run OpenClaw custom-plugin lifecycle and runtime-deps EXDEV Vitest test env: E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/openclaw-plugin-runtime-exdev + E2E_TARGET_ID: openclaw-plugin-runtime-exdev NEMOCLAW_RUN_LIVE_E2E: "1" run: | set -euo pipefail npx vitest run --project e2e-live \ test/e2e/live/openclaw-plugin-runtime-exdev.test.ts \ + -t current-lifecycle \ --silent=false --reporter=default - name: Upload OpenClaw plugin runtime-deps EXDEV artifacts diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6a8fc23ee7..53f991f520 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -206,6 +206,7 @@ These are the primary npm scripts for day-to-day development: | `npm run test:watch` | Watch the CLI, plugin, and E2E-support projects and rerun affected tests | | `npm run test:shuffle` | Shuffle test order in the focused source projects without collecting coverage | | `npm run test:diagnose:leaks` | Report async-resource leaks and diagnose a Vitest process that hangs during shutdown | +| `npm run test:runtime-audit -- [...]` | Rank captured live E2E runs by median, p95, variability, and slowest phase | | `npm run test:integration` | Clean-build the CLI and run root integration and installer tests | | `npm run test:package` | Clean-build CLI/plugin artifacts and run compiled-package contracts | | `npm run test:live-e2e` | Opt into live E2E scenarios (mutates real external state) | diff --git a/package.json b/package.json index a0b9c0068b..469d64b556 100644 --- a/package.json +++ b/package.json @@ -30,6 +30,7 @@ "test:watch": "vitest watch --project cli --project plugin --project e2e-support", "test:shuffle": "vitest run --project cli --project plugin --project e2e-support --sequence.shuffle.tests --coverage=false", "test:diagnose:leaks": "vitest run --project cli --project plugin --project e2e-support --detectAsyncLeaks --coverage=false --reporter=default --reporter=hanging-process", + "test:runtime-audit": "tsx scripts/audit-test-runtime.mts", "test:integration": "npm run clean:cli && npm run build:cli && vitest run --project integration --project installer-integration", "test:package": "npm run clean:cli && npm --prefix nemoclaw run clean && npm run build:cli && npm --prefix nemoclaw run build && vitest run --project package-contract", "test:coverage:cli": "npm run clean:cli && npm run build:cli && tsx scripts/check-dist-sourcemaps.mts dist && vitest run --project cli --project integration --coverage --coverage.reporter=text-summary --coverage.reporter=json-summary --coverage.reportsDirectory=coverage/cli --coverage.include=\"bin/**/*.js\" --coverage.include=\"src/**/*.ts\" --coverage.exclude=\"test/**/*.js\" --coverage.exclude=\"test/**/*.ts\" && tsx scripts/check-coverage-ratchet.mts coverage/cli/coverage-summary.json ci/coverage-threshold-cli.json \"CLI coverage\"", diff --git a/scripts/audit-test-runtime.mts b/scripts/audit-test-runtime.mts new file mode 100644 index 0000000000..fe3647d010 --- /dev/null +++ b/scripts/audit-test-runtime.mts @@ -0,0 +1,152 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; + +import type { ProgressSummary } from "../test/e2e/fixtures/progress.ts"; + +export interface RuntimeAuditRow { + target: string; + scenario: string; + runs: number; + medianMs: number; + p95Ms: number; + maxMs: number; + variabilityMs: number; + slowestPhase: string; + slowestPhaseMs: number; +} + +function isProgressSummary(value: unknown): value is ProgressSummary { + if (!value || typeof value !== "object" || Array.isArray(value)) return false; + const summary = value as Partial; + return ( + summary.version === 1 && + typeof summary.scenario === "string" && + summary.scenario.length > 0 && + (summary.targetId === undefined || typeof summary.targetId === "string") && + (summary.shardId === undefined || typeof summary.shardId === "string") && + typeof summary.durationMs === "number" && + Number.isFinite(summary.durationMs) && + summary.durationMs >= 0 && + Array.isArray(summary.phases) && + summary.phases.every( + (phase) => + phase && + typeof phase.label === "string" && + typeof phase.durationMs === "number" && + Number.isFinite(phase.durationMs) && + phase.durationMs >= 0, + ) + ); +} + +function progressFiles(root: string): string[] { + const result: string[] = []; + const pending = [path.resolve(root)]; + while (pending.length > 0) { + const current = pending.pop(); + if (!current || !fs.existsSync(current)) continue; + const stat = fs.lstatSync(current); + if (stat.isSymbolicLink()) continue; + if (stat.isFile()) { + if (path.basename(current) === "test-progress.json") result.push(current); + continue; + } + if (!stat.isDirectory()) continue; + for (const entry of fs.readdirSync(current, { withFileTypes: true })) { + if (!entry.isSymbolicLink()) pending.push(path.join(current, entry.name)); + } + } + return result.sort(); +} + +function percentile(sorted: readonly number[], fraction: number): number { + const index = Math.max(0, Math.ceil(sorted.length * fraction) - 1); + return sorted[index] ?? 0; +} + +function median(sorted: readonly number[]): number { + const middle = Math.floor(sorted.length / 2); + if (sorted.length % 2 === 0) { + return ((sorted[middle - 1] ?? 0) + (sorted[middle] ?? 0)) / 2; + } + return sorted[middle] ?? 0; +} + +export function auditTestRuntime(roots: readonly string[]): RuntimeAuditRow[] { + const summaries = roots.flatMap(progressFiles).map((file) => { + const parsed: unknown = JSON.parse(fs.readFileSync(file, "utf8")); + if (!isProgressSummary(parsed)) throw new Error(`${file}: invalid test progress summary`); + return parsed; + }); + const grouped = new Map(); + for (const summary of summaries) { + const key = JSON.stringify([ + summary.targetId ?? "unlabeled", + summary.shardId, + summary.scenario, + ]); + const group = grouped.get(key) ?? []; + group.push(summary); + grouped.set(key, group); + } + + return [...grouped.entries()] + .map(([, runs]): RuntimeAuditRow => { + const first = runs[0]; + if (!first) throw new Error("runtime audit group is unexpectedly empty"); + const durations = runs.map((run) => run.durationMs as number).sort((a, b) => a - b); + const phases = runs.flatMap((run) => run.phases); + const slowestPhase = phases.reduce( + (slowest, phase) => (phase.durationMs > slowest.durationMs ? phase : slowest), + { label: "n/a", durationMs: 0 }, + ); + const medianMs = median(durations); + const p95Ms = percentile(durations, 0.95); + return { + target: [first.targetId ?? "unlabeled", first.shardId].filter(Boolean).join("/"), + scenario: first.scenario, + runs: runs.length, + medianMs, + p95Ms, + maxMs: durations.at(-1) ?? 0, + variabilityMs: Math.max(0, p95Ms - medianMs), + slowestPhase: slowestPhase.label, + slowestPhaseMs: slowestPhase.durationMs, + }; + }) + .sort((a, b) => b.p95Ms - a.p95Ms || b.variabilityMs - a.variabilityMs); +} + +function seconds(milliseconds: number): string { + return (milliseconds / 1_000).toFixed(1); +} + +export function formatRuntimeAudit(rows: readonly RuntimeAuditRow[]): string { + const lines = [ + "| Target | Scenario | Runs | Median | p95 | Max | p95 - median | Slowest observed phase |", + "| --- | --- | ---: | ---: | ---: | ---: | ---: | --- |", + ]; + for (const row of rows) { + lines.push( + `| ${row.target.replaceAll("|", "\\|")} | ${row.scenario.replaceAll("|", "\\|")} | ${row.runs} | ${seconds(row.medianMs)}s | ${seconds(row.p95Ms)}s | ${seconds(row.maxMs)}s | ${seconds(row.variabilityMs)}s | ${row.slowestPhase.replaceAll("|", "\\|")} (${seconds(row.slowestPhaseMs)}s) |`, + ); + } + return `${lines.join("\n")}\n`; +} + +function main(argv: readonly string[]): void { + const roots = argv.length > 0 ? argv : [".e2e/live"]; + const rows = auditTestRuntime(roots); + if (rows.length === 0) { + throw new Error(`no test-progress.json files found under: ${roots.join(", ")}`); + } + process.stdout.write(formatRuntimeAudit(rows)); +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main(process.argv.slice(2)); +} diff --git a/test/e2e/README.md b/test/e2e/README.md index 2e5734a9f0..01fc9007c0 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -85,6 +85,28 @@ artifact so baseline aggregation stays stable. Older issue references to Vitest target artifacts under `e2e-artifacts/vitest/` map to this consolidated `e2e-artifacts/live/` registry-target artifact layout. +All live Vitest targets use the automatic progress fixture. CI logs identify +the active test immediately and print a content-free heartbeat every minute +with its current phase, phase elapsed time, last child-output age, and runner +resource snapshot. The heartbeat never includes child output. During fixture +teardown, the fixture writes `test-progress.json` into each test's existing +artifact directory for passing and failing tests. The summary records +`E2E_TARGET_ID` and `NEMOCLAW_E2E_SHARD` when those values are set. Compare +extracted artifacts from multiple runs with: + +```bash +npm run test:runtime-audit -- path/to/run-1 path/to/run-2 +``` + +The audit groups each test by target and optional shard, ranks the groups by +p95 runtime, and reports variability and the slowest observed phase. Add +`progress.phase(...)` calls only around meaningful external lifecycle +boundaries in long scenarios; every shared shell probe already updates +child-output liveness without exposing output contents. +Each shared shell probe also supplies its redacted artifact name as the active +command boundary, so an otherwise uninstrumented test still identifies the +specific command that owns a heartbeat. + ## PR E2E gate The controller, coordination check, and required job deliberately use diff --git a/test/e2e/docs/README.md b/test/e2e/docs/README.md index f24249963b..7a9ba47b4f 100644 --- a/test/e2e/docs/README.md +++ b/test/e2e/docs/README.md @@ -49,6 +49,11 @@ Live execution happens through shared fixtures: - `stateValidation` probes host-observable expected state. - `artifacts`, `secrets`, `cleanup`, and `shellProbe` provide shared fixture services. +- The automatic `progress` fixture prints a start line and one content-free + heartbeat per minute in live runs. It reports the current phase, phase + elapsed time, age of the last child output, memory, workspace capacity, and + host load without forwarding command output or credentials. Every shell + probe feeds that liveness signal automatically. The `test/e2e/fixtures/` path is fixture/support code, not a test harness or runner. Vitest remains the only test harness. @@ -73,6 +78,9 @@ npx vitest run --project e2e-support --silent=false --reporter=default # Opt-in live E2E targets npm run test:live-e2e -- --silent=false --reporter=default + +# Rank one or more downloaded/extracted live artifact directories +npm run test:runtime-audit -- e2e-artifacts/run-1 e2e-artifacts/run-2 ``` The aggregate live command rebuilds the CLI before Vitest starts and runs live @@ -84,6 +92,20 @@ storage-exhaustion, or ownership noise. A target may retry a transient operation only inside its own cleanup boundary. Retry a full target by starting a fresh workflow run and runner. +During fixture teardown, every passing or failing live test writes +`test-progress.json` beside its other target artifacts. The runtime audit +groups those files by target, optional shard, and test name, then reports +median, p95, maximum, p95-minus-median variability, and the slowest observed +phase. The summary reads the matrix identity from `E2E_TARGET_ID` and +`NEMOCLAW_E2E_SHARD` when set. Use several recent workflow artifact directories +to distinguish a consistently expensive test from a variable one. Long +scenarios should call `progress.phase("meaningful phase")` at major external +boundaries; individual commands do not need explicit logging or output +observers. +Shared shell probes automatically use their redacted artifact names as active +command boundaries, so a heartbeat identifies the current command even when a +test has no explicit phase calls. + The retired `--emit-matrix` and `--plan-only` paths must not be reintroduced. When adding or changing a live test, update `test/e2e/mock-parity.json` with diff --git a/test/e2e/fixtures/e2e-test.ts b/test/e2e/fixtures/e2e-test.ts index cb2d00af5a..03010033a8 100644 --- a/test/e2e/fixtures/e2e-test.ts +++ b/test/e2e/fixtures/e2e-test.ts @@ -3,6 +3,12 @@ import { test as base, expect } from "vitest"; +import { + appendResourcePhaseBaseline, + collectResourceSnapshot, +} from "../../../tools/e2e/runner-pressure.mts"; +import { renderSnapshotLine } from "../../../tools/e2e/runner-pressure-core.mts"; + import { type ArtifactSink, createArtifactSink } from "./artifacts.ts"; import { assertCleanupPassed, CleanupRegistry } from "./cleanup.ts"; import { @@ -21,6 +27,7 @@ import { RuntimePhaseFixture, StateValidationPhaseFixture, } from "./phases/index.ts"; +import { startTestProgress, type TestProgress } from "./progress.ts"; import { SecretStore } from "./secrets.ts"; import { ShellProbe } from "./shell-probe.ts"; @@ -41,6 +48,15 @@ export interface E2ETargetFixtures { lifecycle: LifecyclePhaseFixture; runtime: RuntimePhaseFixture; stateValidation: StateValidationPhaseFixture; + progress: TestProgress; +} + +function resourcePhaseLabel(targetId: string, phase: string): string { + const suffix = phase + .toLowerCase() + .replace(/[^a-z0-9]+/gu, "-") + .replace(/^-|-$/gu, ""); + return `${targetId}.${suffix}`; } export const test = base.extend({ @@ -59,6 +75,39 @@ export const test = base.extend({ }); } }, + progress: [ + async ({ artifacts, secrets, task }, use) => { + const targetId = process.env.E2E_TARGET_ID; + const baselinePath = process.env.E2E_RESOURCE_PHASE_BASELINES_FILE; + const progress = startTestProgress(task.name, "test body", { + logLine: + process.env.NEMOCLAW_RUN_LIVE_E2E === "1" + ? (line) => process.stdout.write(`${secrets.redact(line)}\n`) + : () => { + // Keep fixture and support tests quiet; live runs need the heartbeat. + }, + ...(targetId && baselinePath + ? { + sampleResourceEvidence: (phase: string) => + renderSnapshotLine(collectResourceSnapshot(resourcePhaseLabel(targetId, phase))), + recordResourceBaseline: (phase: string) => + appendResourcePhaseBaseline(baselinePath, resourcePhaseLabel(targetId, phase)), + } + : {}), + }); + try { + await use(progress); + } finally { + progress.stop(); + await artifacts.writeJson("test-progress.json", { + ...progress.summary(), + ...(process.env.E2E_TARGET_ID ? { targetId: process.env.E2E_TARGET_ID } : {}), + ...(process.env.NEMOCLAW_E2E_SHARD ? { shardId: process.env.NEMOCLAW_E2E_SHARD } : {}), + }); + } + }, + { auto: true }, + ], docker: async ({ artifacts, secrets, skip }, use) => { const probe = new DockerProbe(artifacts, (text, extra) => secrets.redact(text, extra)); await use(new DockerPrerequisite(probe, skip)); @@ -73,12 +122,14 @@ export const test = base.extend({ assertCleanupPassed(result); } }, - shellProbe: async ({ artifacts, secrets, signal }, use) => { + shellProbe: async ({ artifacts, progress, secrets, signal }, use) => { await use( new ShellProbe({ artifacts, redact: (text, extraValues) => secrets.redact(text, extraValues), signal, + onOutput: progress.onOutput, + onActivity: progress.activity, }), ); }, diff --git a/test/e2e/fixtures/progress.ts b/test/e2e/fixtures/progress.ts new file mode 100644 index 0000000000..9b9e644d87 --- /dev/null +++ b/test/e2e/fixtures/progress.ts @@ -0,0 +1,254 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; + +import { REPO_ROOT } from "./paths.ts"; +import type { ShellProbeOutputEvent } from "./shell-probe.ts"; + +interface ResourceSnapshot { + freeMemoryBytes: number; + processRssBytes: number; + totalMemoryBytes: number; + workspaceFreeBytes: number; + loadAverage1m: number; +} + +interface TimerHandle { + unref?: () => void; +} + +export interface ProgressPhase { + label: string; + startedAtMs: number; + finishedAtMs: number; + durationMs: number; + outputEvents: number; + lastOutputAtMs: number | null; +} + +export interface ProgressSummary { + version: 1; + scenario: string; + targetId?: string; + shardId?: string; + startedAtMs: number; + finishedAtMs: number | null; + durationMs: number | null; + phases: readonly ProgressPhase[]; +} + +export interface TestProgressOptions { + heartbeatIntervalMs?: number; + now?: () => number; + setTimer?: (callback: () => void, intervalMs: number) => TimerHandle; + clearTimer?: (timer: TimerHandle) => void; + logLine?: (line: string) => void; + sampleResources?: () => ResourceSnapshot; + sampleResourceEvidence?: (phase: string) => string; + recordResourceBaseline?: (phase: string) => void; +} + +export interface TestProgressTimeline { + phases: ReadonlyArray<{ label: string; elapsedMs: number }>; + totalMs: number; +} + +export interface TestProgress { + onOutput: (event: ShellProbeOutputEvent) => void; + activity: (label: string) => () => void; + phase: (label: string) => void; + stop: () => void; + summary: () => ProgressSummary; + timeline: () => TestProgressTimeline; +} + +const DEFAULT_HEARTBEAT_INTERVAL_MS = 60_000; + +function formatGiB(bytes: number): string { + return `${(bytes / 1024 ** 3).toFixed(1)} GiB`; +} + +function defaultResourceSnapshot(): ResourceSnapshot { + const workspace = fs.statfsSync(REPO_ROOT); + return { + freeMemoryBytes: os.freemem(), + processRssBytes: process.memoryUsage().rss, + totalMemoryBytes: os.totalmem(), + workspaceFreeBytes: workspace.bavail * workspace.bsize, + loadAverage1m: os.loadavg()[0] ?? 0, + }; +} + +function formatResources(sampleResources: () => ResourceSnapshot): string { + try { + const snapshot = sampleResources(); + return [ + `memory free ${formatGiB(snapshot.freeMemoryBytes)}/${formatGiB(snapshot.totalMemoryBytes)}`, + `test RSS ${formatGiB(snapshot.processRssBytes)}`, + `workspace free ${formatGiB(snapshot.workspaceFreeBytes)}`, + `load 1m ${snapshot.loadAverage1m.toFixed(2)}`, + ].join("; "); + } catch { + return "host resources unavailable"; + } +} + +/** + * Keep a long live scenario visible without forwarding command output, which + * may contain credentials. The timestamp-only observer reports child liveness + * while phase names and resource snapshots identify where a runner stalled. + */ +export function startTestProgress( + scenario: string, + initialPhase = "test body", + options: TestProgressOptions = {}, +): TestProgress { + const now = options.now ?? Date.now; + const setTimer = + options.setTimer ?? ((callback, intervalMs) => setInterval(callback, intervalMs)); + const clearTimer = options.clearTimer ?? ((timer) => clearInterval(timer as NodeJS.Timeout)); + const logLine = options.logLine ?? ((line) => process.stdout.write(`${line}\n`)); + const sampleResources = options.sampleResources ?? defaultResourceSnapshot; + const sampleResourceEvidence = options.sampleResourceEvidence; + const recordResourceBaseline = options.recordResourceBaseline; + const scenarioStartedAt = now(); + const phases: ProgressPhase[] = []; + const semanticPhases: Array<{ label: string; elapsedMs: number }> = []; + let basePhaseLabel = initialPhase; + let semanticPhaseStartedAt = scenarioStartedAt; + let phaseLabel = initialPhase; + let phaseStartedAt = scenarioStartedAt; + let lastOutputAt: number | null = null; + let outputEvents = 0; + let finishedAt: number | null = null; + + const recordBaselineBestEffort = () => { + try { + recordResourceBaseline?.(basePhaseLabel); + } catch { + // Diagnostics must not change the live test result. + } + }; + + const logBestEffort = (state: "started" | "running" | "finished") => { + try { + const current = now(); + const elapsedSeconds = Math.max(0, Math.floor((current - phaseStartedAt) / 1_000)); + const outputAge = + lastOutputAt === null + ? "no child output observed" + : `last child output ${Math.max(0, Math.floor((current - lastOutputAt) / 1_000))}s ago`; + logLine( + `[${scenario}] ${phaseLabel} ${state} (${elapsedSeconds}s elapsed; ${outputAge}; ${formatResources(sampleResources)})`, + ); + const evidence = sampleResourceEvidence?.(basePhaseLabel); + if (evidence) logLine(evidence); + } catch { + // Diagnostics must not change the live test result. + } + }; + + const finishPhase = (atMs: number) => { + phases.push({ + label: phaseLabel, + startedAtMs: phaseStartedAt, + finishedAtMs: atMs, + durationMs: Math.max(0, atMs - phaseStartedAt), + outputEvents, + lastOutputAtMs: lastOutputAt, + }); + }; + + recordBaselineBestEffort(); + logBestEffort("started"); + const timer = setTimer( + () => logBestEffort("running"), + options.heartbeatIntervalMs ?? DEFAULT_HEARTBEAT_INTERVAL_MS, + ); + timer.unref?.(); + + return { + onOutput(event) { + if (finishedAt !== null) return; + lastOutputAt = event.atMs; + outputEvents += 1; + }, + activity(label) { + if (finishedAt !== null) return () => undefined; + const startedAt = now(); + finishPhase(startedAt); + phaseLabel = label; + phaseStartedAt = startedAt; + lastOutputAt = null; + outputEvents = 0; + let activityFinished = false; + return () => { + if (activityFinished || finishedAt !== null) return; + activityFinished = true; + const activityFinishedAt = now(); + finishPhase(activityFinishedAt); + phaseLabel = basePhaseLabel; + phaseStartedAt = activityFinishedAt; + lastOutputAt = null; + outputEvents = 0; + }; + }, + phase(label) { + if (finishedAt !== null) return; + const current = now(); + logBestEffort("finished"); + finishPhase(current); + semanticPhases.push({ + label: basePhaseLabel, + elapsedMs: Math.max(0, current - semanticPhaseStartedAt), + }); + basePhaseLabel = label; + semanticPhaseStartedAt = current; + phaseLabel = basePhaseLabel; + phaseStartedAt = current; + lastOutputAt = null; + outputEvents = 0; + recordBaselineBestEffort(); + logBestEffort("started"); + }, + stop() { + if (finishedAt !== null) return; + finishedAt = now(); + clearTimer(timer); + logBestEffort("finished"); + finishPhase(finishedAt); + semanticPhases.push({ + label: basePhaseLabel, + elapsedMs: Math.max(0, finishedAt - semanticPhaseStartedAt), + }); + }, + summary() { + return { + version: 1, + scenario, + startedAtMs: scenarioStartedAt, + finishedAtMs: finishedAt, + durationMs: finishedAt === null ? null : Math.max(0, finishedAt - scenarioStartedAt), + phases: phases.map((phase) => ({ ...phase })), + }; + }, + timeline() { + const current = now(); + return { + phases: + finishedAt === null + ? [ + ...semanticPhases.map((phase) => ({ ...phase })), + { + label: basePhaseLabel, + elapsedMs: Math.max(0, current - semanticPhaseStartedAt), + }, + ] + : semanticPhases.map((phase) => ({ ...phase })), + totalMs: Math.max(0, (finishedAt ?? current) - scenarioStartedAt), + }; + }, + }; +} diff --git a/test/e2e/fixtures/shell-probe.ts b/test/e2e/fixtures/shell-probe.ts index b57be04a2b..aa17f6d6d3 100644 --- a/test/e2e/fixtures/shell-probe.ts +++ b/test/e2e/fixtures/shell-probe.ts @@ -60,6 +60,10 @@ export interface ShellProbeDeps { artifacts: ArtifactSink; redact: (text: string, extraValues?: string[]) => string; signal: AbortSignal; + /** Default timestamp-only observer applied to every command in this test. */ + onOutput?: (event: ShellProbeOutputEvent) => void; + /** Content-free activity boundary applied to every command in this test. */ + onActivity?: (label: string) => (() => void) | void; } const DEFAULT_TIMEOUT_MS = 60_000; @@ -153,11 +157,15 @@ export class ShellProbe { private readonly artifacts: ArtifactSink; private readonly redact: (text: string, extraValues?: string[]) => string; private readonly signal: AbortSignal; + private readonly onOutput?: (event: ShellProbeOutputEvent) => void; + private readonly onActivity?: (label: string) => (() => void) | void; constructor(deps: ShellProbeDeps) { this.artifacts = deps.artifacts; this.redact = deps.redact; this.signal = deps.signal; + this.onOutput = deps.onOutput; + this.onActivity = deps.onActivity; } async run( @@ -191,7 +199,8 @@ export class ShellProbe { : redacted; }; const redactedCommand = [command, ...args].map(redactProbeText); - const artifactBase = `shell/${safeArtifactBase(redactProbeText(options.artifactName ?? command))}`; + const activityName = safeArtifactBase(redactProbeText(options.artifactName ?? command)); + const artifactBase = `shell/${activityName}`; const writeArtifacts = async ( result: Omit, ): Promise => ({ @@ -203,6 +212,32 @@ export class ShellProbe { const stdout = createTextCapture(options.captureLimitBytes); const stderr = createTextCapture(options.captureLimitBytes); const startedAtMs = Date.now(); + let finishActivity: (() => void) | undefined; + try { + finishActivity = this.onActivity?.(`command: ${activityName}`) ?? undefined; + } catch { + // Test instrumentation must not change command execution. + } + const finishActivityBestEffort = () => { + try { + finishActivity?.(); + } catch { + // Test instrumentation must not change command execution. + } + }; + const observeOutput = (event: ShellProbeOutputEvent) => { + try { + this.onOutput?.(event); + } catch { + // Test instrumentation must not change command execution. + } + if (options.onOutput === this.onOutput) return; + try { + options.onOutput?.(event); + } catch { + // Test instrumentation must not change command execution. + } + }; const child = spawn(command, args, { cwd: options.cwd, detached: true, @@ -215,19 +250,11 @@ export class ShellProbe { signal: this.signal, onStdout: (chunk) => { stdout.append(chunk); - try { - options.onOutput?.({ stream: "stdout", atMs: Date.now() }); - } catch { - // Test instrumentation must not change command execution. - } + observeOutput({ stream: "stdout", atMs: Date.now() }); }, onStderr: (chunk) => { stderr.append(chunk); - try { - options.onOutput?.({ stream: "stderr", atMs: Date.now() }); - } catch { - // Test instrumentation must not change command execution. - } + observeOutput({ stream: "stderr", atMs: Date.now() }); }, }); @@ -246,6 +273,7 @@ export class ShellProbe { stdout: redactedStdout, stderr: stderrWithError, }); + finishActivityBestEffort(); throw redactedError(supervised.spawnError, redactedMessage); } @@ -259,6 +287,7 @@ export class ShellProbe { stderr: redactedStderr, }; const artifacts = await writeArtifacts(result); + finishActivityBestEffort(); return { ...result, artifacts }; } } diff --git a/test/e2e/live/agent-turn-latency-helpers.ts b/test/e2e/live/agent-turn-latency-helpers.ts index dc3fa91886..f88f8dcd06 100644 --- a/test/e2e/live/agent-turn-latency-helpers.ts +++ b/test/e2e/live/agent-turn-latency-helpers.ts @@ -13,6 +13,7 @@ import { } from "../fixtures/clients/sandbox.ts"; import { expect } from "../fixtures/e2e-test.ts"; import { CLI_ENTRYPOINT, REPO_ROOT } from "../fixtures/paths.ts"; +import type { TestProgress } from "../fixtures/progress.ts"; import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; import { isTransientProviderValidationFailure } from "./network-policy-transient-provider.ts"; @@ -252,9 +253,12 @@ export async function installSandbox( agent: "openclaw" | "hermes", apiKey: string, cleanupBeforeRetry?: () => Promise, + progress?: Pick, ): Promise { let install: ShellProbeResult | undefined; + const agentLabel = agent === "openclaw" ? "OpenClaw" : "Hermes"; for (let attempt = 1; attempt <= INSTALL_ATTEMPTS; attempt += 1) { + progress?.phase(`${agentLabel} install attempt ${attempt}`); install = await host.command( "bash", ["install.sh", "--non-interactive", "--fresh", "--yes-i-accept-third-party-software"], @@ -262,6 +266,7 @@ export async function installSandbox( artifactName: `${agent}-install-attempt-${attempt}`, cwd: REPO_ROOT, env: env(sandboxName, agent, apiKey), + onOutput: progress?.onOutput, redactionValues: [apiKey], timeoutMs: 30 * 60_000, }, @@ -271,8 +276,14 @@ export async function installSandbox( isTransientProviderValidationFailure(install) && attempt < INSTALL_ATTEMPTS; install.exitCode === 0 && (attempt = INSTALL_ATTEMPTS + 1); - retry && cleanupBeforeRetry && (await cleanupBeforeRetry()); - retry && (await new Promise((resolve) => setTimeout(resolve, 10_000 * attempt))); + if (retry && cleanupBeforeRetry) { + progress?.phase(`${agentLabel} install retry cleanup`); + await cleanupBeforeRetry(); + } + if (retry) { + progress?.phase(`${agentLabel} install retry backoff`); + await new Promise((resolve) => setTimeout(resolve, 10_000 * attempt)); + } !retry && install.exitCode !== 0 && (attempt = INSTALL_ATTEMPTS + 1); } if (!install) throw new Error(`${agent} install command did not run`); @@ -282,33 +293,41 @@ export async function installSandbox( export async function cleanupTurnSandboxes( host: HostCliClient, sandbox: SandboxClient, + progress?: Pick, ): Promise { for (const [name, agent] of [ [OPENCLAW_SANDBOX, "openclaw"], [HERMES_SANDBOX, "hermes"], ] as const) { + progress?.phase(`preclean destroy ${agent} sandbox`); await bestEffortPreclean(`destroy ${agent} sandbox`, () => - cleanupTurnSandbox(host, name, agent), + cleanupTurnSandbox(host, name, agent, progress), ); + progress?.phase(`preclean delete ${agent} sandbox`); await bestEffortPreclean(`delete ${agent} sandbox`, () => sandbox.openshell(["sandbox", "delete", name], { artifactName: `cleanup-${agent}-delete`, env: env(name, agent), + onOutput: progress?.onOutput, timeoutMs: 60_000, }), ); } + progress?.phase("preclean stop Hermes API forward"); await bestEffortPreclean("stop Hermes API forward", () => sandbox.openshell(["forward", "stop", "8642"], { artifactName: "cleanup-forward-stop-hermes-api", env: buildAvailabilityProbeEnv(), + onOutput: progress?.onOutput, timeoutMs: 30_000, }), ); + progress?.phase("preclean destroy OpenShell gateway"); await bestEffortPreclean("destroy OpenShell gateway", () => sandbox.openshell(["gateway", "destroy", "-g", "nemoclaw"], { artifactName: "cleanup-gateway-destroy-turn-latency", env: buildAvailabilityProbeEnv(), + onOutput: progress?.onOutput, timeoutMs: 60_000, }), ); @@ -318,10 +337,12 @@ export async function cleanupTurnSandbox( host: HostCliClient, name: string, agent: "openclaw" | "hermes", + progress?: Pick, ): Promise { const result = await host.command("node", [CLI, name, "destroy", "--yes"], { artifactName: `cleanup-${agent}-destroy`, env: env(name, agent), + onOutput: progress?.onOutput, timeoutMs: 120_000, }); const output = resultText(result); @@ -339,10 +360,12 @@ export async function route( sandboxName: string, agent: "openclaw" | "hermes", artifactName: string, + progress?: Pick, ): Promise { return await sandbox.openshell(["inference", "get", "-g", "nemoclaw"], { artifactName, env: env(sandboxName, agent), + onOutput: progress?.onOutput, timeoutMs: 30_000, }); } @@ -350,6 +373,7 @@ export async function route( export async function openclawTurn( sandbox: SandboxClient, apiKey: string, + progress?: Pick, ): Promise<{ result: ShellProbeResult; elapsedMs: number }> { const started = process.hrtime.bigint(); const result = await sandbox.execShell( @@ -360,6 +384,7 @@ export async function openclawTurn( { artifactName: "openclaw-agent-turn", env: env(OPENCLAW_SANDBOX, "openclaw"), + onOutput: progress?.onOutput, redactionValues: [apiKey], timeoutMs: (MAX_TURN_SECONDS + 30) * 1000, }, @@ -367,13 +392,21 @@ export async function openclawTurn( return { result, elapsedMs: msSince(started) }; } -export async function waitHermesHealth(sandbox: SandboxClient): Promise { +export async function waitHermesHealth( + sandbox: SandboxClient, + progress?: Pick, +): Promise { return await sandbox.execShell( HERMES_SANDBOX, trustedSandboxShellScript( "for attempt in $(seq 1 10); do body=$(curl -sf --max-time 10 http://localhost:8642/health 2>/dev/null || true); printf '%s' \"$body\" | grep -qi '\"ok\"' && { printf '%s' \"$body\"; exit 0; }; sleep 5; done; printf '%s' \"$body\"; exit 1", ), - { artifactName: "hermes-health", env: env(HERMES_SANDBOX, "hermes"), timeoutMs: 90_000 }, + { + artifactName: "hermes-health", + env: env(HERMES_SANDBOX, "hermes"), + onOutput: progress?.onOutput, + timeoutMs: 90_000, + }, ); } diff --git a/test/e2e/live/agent-turn-latency.test.ts b/test/e2e/live/agent-turn-latency.test.ts index 539a7bfd21..3dd8cd8165 100644 --- a/test/e2e/live/agent-turn-latency.test.ts +++ b/test/e2e/live/agent-turn-latency.test.ts @@ -36,7 +36,7 @@ const TIMEOUT_MS = 90 * 60_000; test("OpenClaw and Hermes complete real hosted inference turns within the latency cap", { timeout: TIMEOUT_MS, -}, async ({ artifacts, cleanup, host, sandbox, secrets }) => { +}, async ({ artifacts, cleanup, host, progress, sandbox, secrets }) => { const apiKey = secrets.required("NVIDIA_INFERENCE_API_KEY"); const results: Record = { model: MODEL, maxTurnSeconds: MAX_TURN_SECONDS }; await artifacts.target.declare({ @@ -45,65 +45,92 @@ test("OpenClaw and Hermes complete real hosted inference turns within the latenc openclawSandbox: OPENCLAW_SANDBOX, hermesSandbox: HERMES_SANDBOX, }); + progress.phase("register cleanup"); - cleanup.trackGateway(host, "nemoclaw", { - artifactName: "cleanup-gateway-destroy-turn-latency", - env: buildAvailabilityProbeEnv(), - timeoutMs: 60_000, + cleanup.trackDisposable("remove gateway nemoclaw", async () => { + progress.phase("cleanup remove OpenShell gateway"); + await host.cleanupGatewayRegistration("nemoclaw", { + artifactName: "cleanup-gateway-destroy-turn-latency", + env: buildAvailabilityProbeEnv(), + onOutput: progress.onOutput, + timeoutMs: 60_000, + }); }); - cleanup.trackForward(host, 8642, { - artifactName: "cleanup-forward-stop-hermes-api", - env: buildAvailabilityProbeEnv(), - timeoutMs: 30_000, + cleanup.trackDisposable("stop forward 8642", async () => { + progress.phase("cleanup stop Hermes API forward"); + await host.cleanupForward(8642, { + artifactName: "cleanup-forward-stop-hermes-api", + env: buildAvailabilityProbeEnv(), + onOutput: progress.onOutput, + timeoutMs: 30_000, + }); }); - cleanup.trackDisposable("delete Hermes OpenShell sandbox", () => - sandbox.cleanupSandbox(HERMES_SANDBOX, { + cleanup.trackDisposable("delete Hermes OpenShell sandbox", async () => { + progress.phase("cleanup delete Hermes OpenShell sandbox"); + await sandbox.cleanupSandbox(HERMES_SANDBOX, { artifactName: "cleanup-hermes-delete", env: env(HERMES_SANDBOX, "hermes"), + onOutput: progress.onOutput, timeoutMs: 60_000, - }), - ); - cleanup.trackDisposable("destroy Hermes sandbox", () => - cleanupTurnSandbox(host, HERMES_SANDBOX, "hermes"), - ); - cleanup.trackDisposable("delete OpenClaw OpenShell sandbox", () => - sandbox.cleanupSandbox(OPENCLAW_SANDBOX, { + }); + }); + cleanup.trackDisposable("destroy Hermes sandbox", async () => { + progress.phase("cleanup destroy Hermes sandbox"); + await cleanupTurnSandbox(host, HERMES_SANDBOX, "hermes", progress); + }); + cleanup.trackDisposable("delete OpenClaw OpenShell sandbox", async () => { + progress.phase("cleanup delete OpenClaw OpenShell sandbox"); + await sandbox.cleanupSandbox(OPENCLAW_SANDBOX, { artifactName: "cleanup-openclaw-delete", env: env(OPENCLAW_SANDBOX, "openclaw"), + onOutput: progress.onOutput, timeoutMs: 60_000, - }), - ); - cleanup.trackDisposable("destroy OpenClaw sandbox", () => - cleanupTurnSandbox(host, OPENCLAW_SANDBOX, "openclaw"), - ); + }); + }); + cleanup.trackDisposable("destroy OpenClaw sandbox", async () => { + progress.phase("cleanup destroy OpenClaw sandbox"); + await cleanupTurnSandbox(host, OPENCLAW_SANDBOX, "openclaw", progress); + }); + progress.phase("docker prerequisite"); const docker = await host.command("docker", ["info"], { artifactName: "docker-info", env: buildAvailabilityProbeEnv(), + onOutput: progress.onOutput, timeoutMs: 30_000, }); expect(docker.exitCode, resultText(docker)).toBe(0); - const cleanBeforeRetry = () => cleanupTurnSandboxes(host, sandbox); - await cleanupTurnSandboxes(host, sandbox); + const cleanBeforeRetry = () => cleanupTurnSandboxes(host, sandbox, progress); + await cleanupTurnSandboxes(host, sandbox, progress); const openclawInstall = await installSandbox( host, OPENCLAW_SANDBOX, "openclaw", apiKey, cleanBeforeRetry, + progress, ); expect(openclawInstall.exitCode, resultText(openclawInstall)).toBe(0); - const openclawRoute = await route(sandbox, OPENCLAW_SANDBOX, "openclaw", "openclaw-route"); + progress.phase("OpenClaw route validation"); + const openclawRoute = await route( + sandbox, + OPENCLAW_SANDBOX, + "openclaw", + "openclaw-route", + progress, + ); expect(openclawRoute.exitCode, resultText(openclawRoute)).toBe(0); expect(resultText(openclawRoute)).toContain(EXPECTED_ROUTE_PROVIDER); expect(resultText(openclawRoute)).toContain(MODEL); + progress.phase("OpenClaw config validation"); const openclawConfig = await sandbox.execShell( OPENCLAW_SANDBOX, trustedSandboxShellScript(openclawConfigCommand()), { artifactName: "openclaw-config", env: env(OPENCLAW_SANDBOX, "openclaw"), + onOutput: progress.onOutput, redactionValues: [apiKey], timeoutMs: 30_000, }, @@ -111,7 +138,8 @@ test("OpenClaw and Hermes complete real hosted inference turns within the latenc expect(openclawConfig.exitCode, resultText(openclawConfig)).toBe(0); assertOpenClawConfig(openclawConfig.stdout, MODEL); - const openclaw = await openclawTurn(sandbox, apiKey); + progress.phase("OpenClaw agent turn"); + const openclaw = await openclawTurn(sandbox, apiKey, progress); expect(openclaw.result.exitCode, resultText(openclaw.result)).toBe(0); assertNoOpenClawTransportErrors(resultText(openclaw.result)); expect( @@ -121,9 +149,11 @@ test("OpenClaw and Hermes complete real hosted inference turns within the latenc expect(openclaw.elapsedMs).toBeLessThanOrEqual(MAX_TURN_SECONDS * 1000); results.openclaw = { elapsedMs: openclaw.elapsedMs }; + progress.phase("destroy OpenClaw before Hermes"); await host.command("node", [CLI, OPENCLAW_SANDBOX, "destroy", "--yes"], { artifactName: "destroy-openclaw-before-hermes", env: env(OPENCLAW_SANDBOX, "openclaw"), + onOutput: progress.onOutput, timeoutMs: 120_000, }); @@ -133,17 +163,22 @@ test("OpenClaw and Hermes complete real hosted inference turns within the latenc "hermes", apiKey, cleanBeforeRetry, + progress, ); expect(hermesInstall.exitCode, resultText(hermesInstall)).toBe(0); - const hermesRoute = await route(sandbox, HERMES_SANDBOX, "hermes", "hermes-route"); + progress.phase("Hermes route validation"); + const hermesRoute = await route(sandbox, HERMES_SANDBOX, "hermes", "hermes-route", progress); expect(hermesRoute.exitCode, resultText(hermesRoute)).toBe(0); expect(resultText(hermesRoute)).toContain(EXPECTED_ROUTE_PROVIDER); expect(resultText(hermesRoute)).toContain(MODEL); - const hermesHealth = await waitHermesHealth(sandbox); + progress.phase("Hermes health validation"); + const hermesHealth = await waitHermesHealth(sandbox, progress); expect(hermesHealth.exitCode, resultText(hermesHealth)).toBe(0); + progress.phase("Hermes config validation"); const hermesConfig = await sandbox.exec(HERMES_SANDBOX, ["cat", "/sandbox/.hermes/config.yaml"], { artifactName: "hermes-config", env: env(HERMES_SANDBOX, "hermes"), + onOutput: progress.onOutput, redactionValues: [apiKey], timeoutMs: 30_000, }); @@ -160,6 +195,7 @@ test("OpenClaw and Hermes complete real hosted inference turns within the latenc ], max_tokens: 64, }); + progress.phase("Hermes API turn"); const hermesStarted = process.hrtime.bigint(); const hermesTurn = await sandbox.execShell( HERMES_SANDBOX, @@ -167,6 +203,7 @@ test("OpenClaw and Hermes complete real hosted inference turns within the latenc { artifactName: "hermes-api-turn", env: env(HERMES_SANDBOX, "hermes"), + onOutput: progress.onOutput, redactionValues: [apiKey], timeoutMs: (MAX_TURN_SECONDS + 30) * 1000, }, diff --git a/test/e2e/live/openclaw-plugin-runtime-exdev.test.ts b/test/e2e/live/openclaw-plugin-runtime-exdev.test.ts index bcc2479e4c..030ac0fea4 100644 --- a/test/e2e/live/openclaw-plugin-runtime-exdev.test.ts +++ b/test/e2e/live/openclaw-plugin-runtime-exdev.test.ts @@ -13,7 +13,9 @@ import { hasRequiredOpenshellMessagingFeatures, REQUIRED_OPENSHELL_MCP_FEATURES, } from "../../../src/lib/onboard/openshell-feature-gate.ts"; +import type { ArtifactSink } from "../fixtures/artifacts.ts"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import type { CleanupRegistry } from "../fixtures/cleanup.ts"; import { resultText, shellQuote } from "../fixtures/clients/command.ts"; import type { HostCliClient } from "../fixtures/clients/host.ts"; import { @@ -110,6 +112,9 @@ const EXDEV_PATTERNS = [ ]; type WeatherFixtureVersion = "v1" | "v2" | "v3"; +const RELEASE_BASELINE_TEST_SELECTOR = "release-baseline"; +const CURRENT_LIFECYCLE_TEST_SELECTOR = "current-lifecycle"; + const GATEWAY_CATALOG_CALL_SOURCE = String.raw` import { Buffer } from "node:buffer"; import { accessSync, constants, realpathSync } from "node:fs"; @@ -840,86 +845,13 @@ NODE`; const runtimeDepsReplacementProbe = trustedSandboxShellScript(runtimeDepsReplacementProbeSource); -test("a custom OpenClaw plugin survives restart, recreation, and rebuild without EXDEV failures (#6108)", { - timeout: ONBOARD_TIMEOUT_MS * 3 + REBUILD_TIMEOUT_MS + 15 * 60_000, -}, async ({ artifacts, cleanup, host, sandbox, skip }) => { - await artifacts.target.declare({ - id: "openclaw-plugin-runtime-exdev", - boundary: "fresh-openclaw-sandbox-exec", - regressionTargets: ["#6108", "#3513", "#3127"], - contract: [ - "the exact NemoClaw v0.0.71 checkout installs, builds, and reports its tagged CLI version", - "the tagged CLI uses OpenShell 0.0.71 with matching source, base image, and OpenClaw runtime", - "the current CLI reinstalls OpenShell 0.0.85 before current lifecycle coverage", - "release-matched peer/dev dependencies prune private OpenClaw and link the host runtime", - "gateway log, runtime inspection, tools.catalog, and tools.invoke prove weather/get_weather", - "custom-plugin v1 survives restart, recreation installs v2, and rebuild installs v3", - "workspace state survives both onboarding recreation and rebuild", - `test-only driver config mounts tmpfs at ${EXDEV_TMPFS_MOUNT} without changing production policies`, - "stock OpenClaw policy source bytes remain unchanged through onboard and rebuild", - `sandbox proves ${EXDEV_TMPFS_SOURCE} and plugin-runtime-deps are distinct devices`, - `legacy source-side staging fails with EXDEV across the same ${EXDEV_TMPFS_SOURCE} to plugin-runtime-deps boundary`, - "OpenClaw-style target-side plugin runtime-deps replacement completes without EXDEV", - ], - nemoclawSourceRelease: NEMOCLAW_RELEASE_TAG, - nemoclawSourceCommit: NEMOCLAW_RELEASE_COMMIT, - taggedOpenshellVersion: NEMOCLAW_RELEASE_OPENSHELL_VERSION, - currentOpenshellVersion: CURRENT_OPENSHELL_VERSION, - sandboxBaseImageRef: SANDBOX_BASE_IMAGE_REF, - openclawVersion: WEATHER_OPENCLAW_VERSION, - }); - - const docker = await host.command("docker", ["info"], { - artifactName: "prereq-docker-info-openclaw-plugin-exdev", - env: liveEnv(), - timeoutMs: 30_000, - }); - if (docker.exitCode !== 0) { - if (process.env.GITHUB_ACTIONS === "true") { - throw new Error( - `Docker is required for the OpenClaw plugin EXDEV live guard: ${resultText(docker)}`, - ); - } - skip("Docker is required for the OpenClaw plugin EXDEV live guard"); - } - - expect( - fs.existsSync(CLI_ENTRYPOINT), - "bin/nemoclaw.js missing — run npm run build:cli before this live target", - ).toBe(true); - - cleanup.trackDisposable(`delete OpenShell sandbox ${SANDBOX_NAME}`, () => - sandbox.cleanupSandbox(SANDBOX_NAME, { - artifactName: "cleanup-openshell-delete-openclaw-plugin-exdev", - env: liveEnv(), - timeoutMs: 60_000, - }), - ); - cleanup.trackSandbox(host, SANDBOX_NAME, { - artifactName: "cleanup-nemoclaw-destroy-openclaw-plugin-exdev", - env: liveEnv(), - timeoutMs: 120_000, - }); - - await ignoreCleanupError(() => - host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "destroy", "--yes"], { - artifactName: "pre-cleanup-nemoclaw-destroy-openclaw-plugin-exdev", - env: liveEnv(), - timeoutMs: 120_000, - }), - ); - await ignoreCleanupError(() => - sandbox.openshell(["sandbox", "delete", SANDBOX_NAME], { - artifactName: "pre-cleanup-openshell-delete-openclaw-plugin-exdev", - env: liveEnv(), - timeoutMs: 60_000, - }), - ); - - const policySourceSnapshot = snapshotPolicySources(); - const customPluginContext = createCustomPluginBuildContext(); +async function prepareCustomPluginSource( + host: HostCliClient, + cleanup: CleanupRegistry, +): Promise { + const context = createCustomPluginBuildContext(); cleanup.add("remove v0.0.71 custom-plugin source worktree", () => - fs.rmSync(customPluginContext.sourceParentDir, { recursive: true, force: true }), + fs.rmSync(context.sourceParentDir, { recursive: true, force: true }), ); const cloneRelease = await host.command( "git", @@ -931,7 +863,7 @@ test("a custom OpenClaw plugin survives restart, recreation, and rebuild without NEMOCLAW_RELEASE_TAG, "--single-branch", NEMOCLAW_SOURCE_REPOSITORY, - customPluginContext.sourceRoot, + context.sourceRoot, ], { artifactName: "clone-nemoclaw-v0-0-71-plugin-source", @@ -940,34 +872,21 @@ test("a custom OpenClaw plugin survives restart, recreation, and rebuild without }, ); expect(cloneRelease.exitCode, resultText(cloneRelease)).toBe(0); - const releaseHead = await host.command( - "git", - ["-C", customPluginContext.sourceRoot, "rev-parse", "HEAD"], - { - artifactName: "verify-nemoclaw-v0-0-71-plugin-source", - env: liveEnv(), - timeoutMs: 30_000, - }, - ); + const releaseHead = await host.command("git", ["-C", context.sourceRoot, "rev-parse", "HEAD"], { + artifactName: "verify-nemoclaw-v0-0-71-plugin-source", + env: liveEnv(), + timeoutMs: 30_000, + }); expect(releaseHead.exitCode, resultText(releaseHead)).toBe(0); expect(releaseHead.stdout.trim()).toBe(NEMOCLAW_RELEASE_COMMIT); - createCustomPluginDockerfile(customPluginContext); - await buildAndVerifyTaggedCli(host, customPluginContext); - - await stopOpenShellGatewayBeforeVersionSwitch(host, "existing"); - const taggedPinnedOpenshell = await installAndResolvePinnedOpenShell( - host, - path.join(customPluginContext.sourceRoot, "scripts", "install-openshell.sh"), - "v0-0-71", - NEMOCLAW_RELEASE_OPENSHELL_VERSION, - ); - // OpenShell 0.0.71 predates the current 0.0.85 MCP capability marker. - // The tagged CLI's own onboarding preflight below owns this compatibility check. - const taggedOpenShellWrapper = createOpenShellTmpfsWrapper(taggedPinnedOpenshell.cli); - cleanup.add("remove v0.0.71 EXDEV OpenShell PATH wrapper", taggedOpenShellWrapper.remove); + createCustomPluginDockerfile(context); + return context; +} - // The OpenShell gateway reaches the fixture from its network namespace, so - // runner loopback is not routable from the readiness probe. +async function startDeploymentFixture( + artifacts: ArtifactSink, + cleanup: CleanupRegistry, +): Promise { const fake = await startFakeOpenAiCompatibleServer({ apiKey: "nemoclaw-exdev-dummy-key", host: "0.0.0.0", @@ -981,7 +900,7 @@ test("a custom OpenClaw plugin survives restart, recreation, and rebuild without await fake.close(); }); - const deploymentEnv = liveEnv({ + return liveEnv({ COMPATIBLE_API_KEY: "nemoclaw-exdev-dummy-key", NEMOCLAW_ENDPOINT_URL: fake.baseUrl, NEMOCLAW_MODEL: "nemoclaw-exdev-probe", @@ -992,6 +911,86 @@ test("a custom OpenClaw plugin survives restart, recreation, and rebuild without NEMOCLAW_PREFERRED_API: "openai-completions", NEMOCLAW_PROVIDER: "custom", }); +} + +async function requireDocker( + host: HostCliClient, + artifactName: string, + reason: string, + skip: (note?: string) => never, +): Promise { + const docker = await host.command("docker", ["info"], { + artifactName, + env: liveEnv(), + timeoutMs: 30_000, + }); + if (docker.exitCode === 0) return; + if (process.env.GITHUB_ACTIONS === "true") { + throw new Error(`${reason}: ${resultText(docker)}`); + } + skip(reason); +} + +test("the release-baseline custom plugin loads with its exact NemoClaw and OpenShell versions (#6108)", { + timeout: ONBOARD_TIMEOUT_MS + 15 * 60_000, +}, async ({ artifacts, cleanup, host, sandbox, skip }) => { + await artifacts.target.declare({ + id: "openclaw-plugin-runtime-exdev-release", + boundary: "fresh-openclaw-sandbox-exec", + regressionTargets: ["#6108"], + contract: [ + "the exact NemoClaw v0.0.71 checkout installs, builds, and reports its tagged CLI version", + "the tagged CLI uses OpenShell 0.0.71 with matching source, base image, and OpenClaw runtime", + "release-matched peer/dev dependencies prune private OpenClaw and link the host runtime", + "the release weather plugin loads from the custom image without an EXDEV bootstrap failure", + ], + selector: RELEASE_BASELINE_TEST_SELECTOR, + nemoclawSourceRelease: NEMOCLAW_RELEASE_TAG, + nemoclawSourceCommit: NEMOCLAW_RELEASE_COMMIT, + taggedOpenshellVersion: NEMOCLAW_RELEASE_OPENSHELL_VERSION, + sandboxBaseImageRef: SANDBOX_BASE_IMAGE_REF, + openclawVersion: WEATHER_OPENCLAW_VERSION, + }); + + await requireDocker( + host, + "prereq-docker-info-openclaw-plugin-exdev-release", + "Docker is required for the OpenClaw plugin release baseline", + skip, + ); + + cleanup.trackDisposable(`delete OpenShell sandbox ${SANDBOX_NAME}`, () => + sandbox.cleanupSandbox(SANDBOX_NAME, { + artifactName: "cleanup-openshell-delete-openclaw-plugin-exdev-release", + env: liveEnv(), + timeoutMs: 60_000, + }), + ); + cleanup.trackSandbox(host, SANDBOX_NAME, { + artifactName: "cleanup-nemoclaw-destroy-openclaw-plugin-exdev-release", + env: liveEnv(), + timeoutMs: 120_000, + }); + await ignoreCleanupError(() => + sandbox.openshell(["sandbox", "delete", SANDBOX_NAME], { + artifactName: "pre-cleanup-openshell-delete-openclaw-plugin-exdev-release", + env: liveEnv(), + timeoutMs: 60_000, + }), + ); + + const customPluginContext = await prepareCustomPluginSource(host, cleanup); + await buildAndVerifyTaggedCli(host, customPluginContext); + await stopOpenShellGatewayBeforeVersionSwitch(host, "existing"); + const taggedPinnedOpenshell = await installAndResolvePinnedOpenShell( + host, + path.join(customPluginContext.sourceRoot, "scripts", "install-openshell.sh"), + "v0-0-71", + NEMOCLAW_RELEASE_OPENSHELL_VERSION, + ); + const taggedOpenShellWrapper = createOpenShellTmpfsWrapper(taggedPinnedOpenshell.cli); + cleanup.add("remove v0.0.71 EXDEV OpenShell PATH wrapper", taggedOpenShellWrapper.remove); + const deploymentEnv = await startDeploymentFixture(artifacts, cleanup); const taggedSandboxEnv = withOpenShellWrapperEnv( deploymentEnv, taggedOpenShellWrapper, @@ -1022,6 +1021,7 @@ test("a custom OpenClaw plugin survives restart, recreation, and rebuild without const taggedOnboardText = resultText(taggedOnboard); expect(taggedOnboard.exitCode, taggedOnboardText).toBe(0); expect(taggedOnboardText).toContain("Deployment verified"); + const taggedRuntimeVersion = await sandbox.exec(SANDBOX_NAME, ["openclaw", "--version"], { artifactName: "v0-0-71-openclaw-version", env: liveEnv(), @@ -1046,6 +1046,7 @@ test("a custom OpenClaw plugin survives restart, recreation, and rebuild without expect(taggedPluginInspect.plugin?.id).toBe("weather"); expect(taggedPluginInspect.plugin?.status).toBe("loaded"); expect(taggedPluginInspect.plugin?.toolNames).toContain("get_weather"); + const taggedDestroy = await host.command( "node", [customPluginContext.cliEntrypoint, SANDBOX_NAME, "destroy", "--yes"], @@ -1057,15 +1058,94 @@ test("a custom OpenClaw plugin survives restart, recreation, and rebuild without }, ); expect(taggedDestroy.exitCode, resultText(taggedDestroy)).toBe(0); + + await artifacts.target.complete({ + id: "openclaw-plugin-runtime-exdev-release", + taggedOnboardExitCode: taggedOnboard.exitCode, + taggedDestroyExitCode: taggedDestroy.exitCode, + assertions: { + taggedReleaseRuntimeMatched: taggedRuntimeVersionText.includes( + EXPECTED_RELEASE_OPENCLAW_VERSION, + ), + taggedReleasePluginLoaded: + taggedPluginInspect.plugin?.id === "weather" && + taggedPluginInspect.plugin?.status === "loaded" && + Array.isArray(taggedPluginInspect.plugin?.toolNames) && + taggedPluginInspect.plugin.toolNames.includes("get_weather"), + }, + }); +}); + +test("the current-lifecycle custom plugin survives restart, recreation, and rebuild without EXDEV failures (#6108)", { + timeout: ONBOARD_TIMEOUT_MS * 3 + REBUILD_TIMEOUT_MS + 15 * 60_000, +}, async ({ artifacts, cleanup, host, sandbox, skip }) => { + await artifacts.target.declare({ + id: "openclaw-plugin-runtime-exdev", + boundary: "fresh-openclaw-sandbox-exec", + regressionTargets: ["#6108", "#3513", "#3127"], + contract: [ + "the current CLI uses OpenShell 0.0.85 for current lifecycle coverage", + "gateway log, runtime inspection, tools.catalog, and tools.invoke prove weather/get_weather", + "custom-plugin v1 survives restart, recreation installs v2, and rebuild installs v3", + "workspace state survives both onboarding recreation and rebuild", + `test-only driver config mounts tmpfs at ${EXDEV_TMPFS_MOUNT} without changing production policies`, + "stock OpenClaw policy source bytes remain unchanged through onboard and rebuild", + `sandbox proves ${EXDEV_TMPFS_SOURCE} and plugin-runtime-deps are distinct devices`, + `legacy source-side staging fails with EXDEV across the same ${EXDEV_TMPFS_SOURCE} to plugin-runtime-deps boundary`, + "OpenClaw-style target-side plugin runtime-deps replacement completes without EXDEV", + ], + selector: CURRENT_LIFECYCLE_TEST_SELECTOR, + nemoclawSourceRelease: NEMOCLAW_RELEASE_TAG, + nemoclawSourceCommit: NEMOCLAW_RELEASE_COMMIT, + currentOpenshellVersion: CURRENT_OPENSHELL_VERSION, + sandboxBaseImageRef: SANDBOX_BASE_IMAGE_REF, + openclawVersion: WEATHER_OPENCLAW_VERSION, + }); + + await requireDocker( + host, + "prereq-docker-info-openclaw-plugin-exdev", + "Docker is required for the OpenClaw plugin EXDEV live guard", + skip, + ); + + expect( + fs.existsSync(CLI_ENTRYPOINT), + "bin/nemoclaw.js missing — run npm run build:cli before this live target", + ).toBe(true); + + cleanup.trackDisposable(`delete OpenShell sandbox ${SANDBOX_NAME}`, () => + sandbox.cleanupSandbox(SANDBOX_NAME, { + artifactName: "cleanup-openshell-delete-openclaw-plugin-exdev", + env: liveEnv(), + timeoutMs: 60_000, + }), + ); + cleanup.trackSandbox(host, SANDBOX_NAME, { + artifactName: "cleanup-nemoclaw-destroy-openclaw-plugin-exdev", + env: liveEnv(), + timeoutMs: 120_000, + }); + + await ignoreCleanupError(() => + host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "destroy", "--yes"], { + artifactName: "pre-cleanup-nemoclaw-destroy-openclaw-plugin-exdev", + env: liveEnv(), + timeoutMs: 120_000, + }), + ); await ignoreCleanupError(() => sandbox.openshell(["sandbox", "delete", SANDBOX_NAME], { - artifactName: "v0-0-71-openclaw-plugin-delete-fallback", - env: taggedSandboxEnv, + artifactName: "pre-cleanup-openshell-delete-openclaw-plugin-exdev", + env: liveEnv(), timeoutMs: 60_000, }), ); - await stopOpenShellGatewayBeforeVersionSwitch(host, "v0-0-71", taggedSandboxEnv); + const policySourceSnapshot = snapshotPolicySources(); + const customPluginContext = await prepareCustomPluginSource(host, cleanup); + const deploymentEnv = await startDeploymentFixture(artifacts, cleanup); + await stopOpenShellGatewayBeforeVersionSwitch(host, "existing"); const pinnedOpenshell = await installAndResolvePinnedOpenShell( host, path.join(REPO_ROOT, "scripts", "install-openshell.sh"), @@ -1215,8 +1295,6 @@ test("a custom OpenClaw plugin survives restart, recreation, and rebuild without await artifacts.target.complete({ id: "openclaw-plugin-runtime-exdev", - taggedOnboardExitCode: taggedOnboard.exitCode, - taggedDestroyExitCode: taggedDestroy.exitCode, onboardExitCode: onboard.exitCode, restartExitCode: restart.exitCode, recreateExitCode: recreate.exitCode, @@ -1225,14 +1303,6 @@ test("a custom OpenClaw plugin survives restart, recreation, and rebuild without runtimeDepsProbeExitCode: probe.exitCode, testOnlyTmpfsSource: EXDEV_TMPFS_SOURCE, assertions: { - taggedReleaseRuntimeMatched: taggedRuntimeVersionText.includes( - EXPECTED_RELEASE_OPENCLAW_VERSION, - ), - taggedReleasePluginLoaded: - taggedPluginInspect.plugin?.id === "weather" && - taggedPluginInspect.plugin?.status === "loaded" && - Array.isArray(taggedPluginInspect.plugin?.toolNames) && - taggedPluginInspect.plugin.toolNames.includes("get_weather"), weatherAfterOnboard: weatherAfterOnboard.inspectLoaded && weatherAfterOnboard.catalogToolIds.includes("get_weather") && diff --git a/test/e2e/live/rebuild-hermes-progress.ts b/test/e2e/live/rebuild-hermes-progress.ts deleted file mode 100644 index 8b8219c1c8..0000000000 --- a/test/e2e/live/rebuild-hermes-progress.ts +++ /dev/null @@ -1,185 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import fs from "node:fs"; -import os from "node:os"; - -import { - appendResourcePhaseBaseline, - collectResourceSnapshot, -} from "../../../tools/e2e/runner-pressure.mts"; -import { renderSnapshotLine } from "../../../tools/e2e/runner-pressure-core.mts"; -import { REPO_ROOT } from "../fixtures/paths.ts"; -import type { ShellProbeOutputEvent } from "../fixtures/shell-probe.ts"; -import type { RebuildHermesTimeline, RebuildHermesTimingPhase } from "./rebuild-hermes-timing.ts"; - -interface RebuildHermesResourceSnapshot { - freeMemoryBytes: number; - processRssBytes: number; - totalMemoryBytes: number; - workspaceFreeBytes: number; - loadAverage1m: number; -} - -interface TimerHandle { - unref?: () => void; -} - -export interface RebuildHermesProgressOptions { - heartbeatIntervalMs?: number; - now?: () => number; - setTimer?: (callback: () => void, intervalMs: number) => TimerHandle; - clearTimer?: (timer: TimerHandle) => void; - logLine?: (line: string) => void; - sampleResources?: () => RebuildHermesResourceSnapshot; - sampleResourceEvidence?: (phase: string) => string; - recordResourceBaseline?: (phase: string) => void; -} - -export interface RebuildHermesProgress { - onOutput: (event: ShellProbeOutputEvent) => void; - phase: (label: string) => void; - stop: () => void; - timeline: () => RebuildHermesTimeline; -} - -const DEFAULT_HEARTBEAT_INTERVAL_MS = 60_000; - -function formatGiB(bytes: number): string { - return `${(bytes / 1024 ** 3).toFixed(1)} GiB`; -} - -function defaultResourceSnapshot(): RebuildHermesResourceSnapshot { - const workspace = fs.statfsSync(REPO_ROOT); - return { - freeMemoryBytes: os.freemem(), - processRssBytes: process.memoryUsage().rss, - totalMemoryBytes: os.totalmem(), - workspaceFreeBytes: workspace.bavail * workspace.bsize, - loadAverage1m: os.loadavg()[0] ?? 0, - }; -} - -function formatResources(sampleResources: () => RebuildHermesResourceSnapshot): string { - try { - const snapshot = sampleResources(); - return [ - `memory free ${formatGiB(snapshot.freeMemoryBytes)}/${formatGiB(snapshot.totalMemoryBytes)}`, - `test RSS ${formatGiB(snapshot.processRssBytes)}`, - `workspace free ${formatGiB(snapshot.workspaceFreeBytes)}`, - `load 1m ${snapshot.loadAverage1m.toFixed(2)}`, - ].join("; "); - } catch { - return "host resources unavailable"; - } -} - -function resourcePhaseLabel(label: string): string { - const suffix = label - .toLowerCase() - .replace(/[^a-z0-9]+/gu, "-") - .replace(/^-|-$/gu, ""); - return `rebuild-hermes.${suffix}`; -} - -/** - * Keep the long Hermes scenario visible without forwarding command output, - * which may contain credentials. The timestamp-only output observer records - * liveness while resource snapshots make hosted-runner loss diagnosable. - */ -export function startRebuildHermesProgress( - initialPhase: string, - options: RebuildHermesProgressOptions = {}, -): RebuildHermesProgress { - const now = options.now ?? Date.now; - const setTimer = - options.setTimer ?? ((callback, intervalMs) => setInterval(callback, intervalMs)); - const clearTimer = options.clearTimer ?? ((timer) => clearInterval(timer as NodeJS.Timeout)); - const logLine = options.logLine ?? ((line) => process.stdout.write(`${line}\n`)); - const sampleResources = options.sampleResources ?? defaultResourceSnapshot; - const sampleResourceEvidence = - options.sampleResourceEvidence ?? - ((phase) => renderSnapshotLine(collectResourceSnapshot(resourcePhaseLabel(phase)))); - const recordResourceBaseline = - options.recordResourceBaseline ?? - ((phase) => { - const baselinePath = process.env.E2E_RESOURCE_PHASE_BASELINES_FILE; - if (baselinePath) appendResourcePhaseBaseline(baselinePath, resourcePhaseLabel(phase)); - }); - const overallStartedAt = now(); - const completedPhases: RebuildHermesTimingPhase[] = []; - let phaseLabel = initialPhase; - let phaseStartedAt = overallStartedAt; - let lastOutputAt: number | null = null; - let stopped = false; - let stoppedAt = overallStartedAt; - - const completeActivePhase = (completedAt = now()) => { - completedPhases.push({ label: phaseLabel, elapsedMs: completedAt - phaseStartedAt }); - }; - - const recordBaselineBestEffort = () => { - try { - recordResourceBaseline(phaseLabel); - } catch { - // Diagnostics must not change the live result. Classification retains - // the immutable pre-test baseline when the phase ledger is unavailable. - } - }; - - const logBestEffort = (state: "started" | "running" | "finished") => { - try { - const current = now(); - const elapsedSeconds = Math.max(0, Math.floor((current - phaseStartedAt) / 1_000)); - const outputAge = - lastOutputAt === null - ? "no child output observed" - : `last child output ${Math.max(0, Math.floor((current - lastOutputAt) / 1_000))}s ago`; - logLine( - `[rebuild-hermes] ${phaseLabel} ${state} (${elapsedSeconds}s elapsed; ${outputAge}; ${formatResources(sampleResources)})`, - ); - logLine(sampleResourceEvidence(phaseLabel)); - } catch { - // Diagnostics must not change the live test result. - } - }; - - recordBaselineBestEffort(); - logBestEffort("started"); - const timer = setTimer( - () => logBestEffort("running"), - options.heartbeatIntervalMs ?? DEFAULT_HEARTBEAT_INTERVAL_MS, - ); - timer.unref?.(); - - return { - onOutput(event) { - lastOutputAt = event.atMs; - }, - phase(label) { - if (stopped) return; - logBestEffort("finished"); - completeActivePhase(); - phaseLabel = label; - phaseStartedAt = now(); - lastOutputAt = null; - recordBaselineBestEffort(); - logBestEffort("started"); - }, - stop() { - if (stopped) return; - stopped = true; - stoppedAt = now(); - completeActivePhase(stoppedAt); - clearTimer(timer); - logBestEffort("finished"); - }, - timeline() { - const current = now(); - const phases = stopped - ? [...completedPhases] - : [...completedPhases, { label: phaseLabel, elapsedMs: current - phaseStartedAt }]; - return { phases, totalMs: (stopped ? stoppedAt : current) - overallStartedAt }; - }, - }; -} diff --git a/test/e2e/live/rebuild-hermes.test.ts b/test/e2e/live/rebuild-hermes.test.ts index 04a37c9636..9dc4cfc1d5 100644 --- a/test/e2e/live/rebuild-hermes.test.ts +++ b/test/e2e/live/rebuild-hermes.test.ts @@ -29,7 +29,6 @@ import { rebuildHermesRegistryImageState, requireRebuildHermesInitialImageTag, } from "./rebuild-hermes-image-state.ts"; -import { startRebuildHermesProgress } from "./rebuild-hermes-progress.ts"; import { buildRebuildHermesTimingSummary, describeRunnerClass } from "./rebuild-hermes-timing.ts"; // The migrated scope is the legacy non-interactive shell regression: install.sh, @@ -436,12 +435,11 @@ test(STALE_BASE_REBUILD ? "rebuild-hermes: stale base cache is refreshed while Hermes state survives rebuild" : "rebuild-hermes: old Hermes sandbox rebuild preserves messaging state and upgrades runtime", { timeout: LIVE_TIMEOUT_MS, -}, async ({ artifacts, cleanup, host, sandbox, secrets, skip }) => { +}, async ({ artifacts, cleanup, host, progress, sandbox, secrets, skip }) => { const apiKey = secrets.required("NVIDIA_INFERENCE_API_KEY"); const redactionValues = [apiKey, DISCORD_FAKE_TOKEN]; const expectedVersion = expectedHermesVersion(); - const progress = startRebuildHermesProgress("setup"); - cleanup.trackDisposable("stop Hermes rebuild progress", progress.stop); + progress.phase("setup"); const registrySnapshot = snapshotFile(REGISTRY_FILE); const sessionSnapshot = snapshotFile(SESSION_FILE); @@ -459,7 +457,8 @@ test(STALE_BASE_REBUILD markerFile: MARKER_FILE, preservedBoundaries: [ "bash install.sh --non-interactive", - "docker build agents/hermes/Dockerfile.base for old/current Hermes base images", + "docker build agents/hermes/Dockerfile.base for the old Hermes base image", + "nemoclaw rebuild owns current Hermes base-image resolution and refresh", "openshell provider create/update and sandbox create/exec/list", "curated local ~/.nemoclaw registry and onboard-session rebuild metadata", "real nemoclaw rebuild --yes --verbose", diff --git a/test/e2e/support/agent-turn-latency-progress.test.ts b/test/e2e/support/agent-turn-latency-progress.test.ts new file mode 100644 index 0000000000..f02fd0f3e7 --- /dev/null +++ b/test/e2e/support/agent-turn-latency-progress.test.ts @@ -0,0 +1,223 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { afterEach, describe, expect, it, vi } from "vitest"; + +import type { HostCliClient } from "../fixtures/clients/host.ts"; +import { startTestProgress, type TestProgressOptions } from "../fixtures/progress.ts"; +import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; +import { installSandbox } from "../live/agent-turn-latency-helpers.ts"; + +function progressHarness() { + const state = { + clearCalls: 0, + clockMs: 1_000, + lines: [] as string[], + timerCallback: null as (() => void) | null, + }; + const options: TestProgressOptions = { + heartbeatIntervalMs: 60_000, + now: () => state.clockMs, + setTimer: (callback) => { + state.timerCallback = callback; + return { unref() {} }; + }, + clearTimer: () => { + state.clearCalls += 1; + }, + logLine: (line) => state.lines.push(line), + sampleResources: () => ({ + freeMemoryBytes: 8 * 1024 ** 3, + processRssBytes: 0.5 * 1024 ** 3, + totalMemoryBytes: 16 * 1024 ** 3, + workspaceFreeBytes: 6 * 1024 ** 3, + loadAverage1m: 2.5, + }), + }; + return { options, state }; +} + +function successfulProbe(): ShellProbeResult { + return { + command: ["bash", "install.sh"], + durationMs: 10, + exitCode: 0, + signal: null, + timedOut: false, + stdout: "", + stderr: "", + artifacts: { stdout: "stdout", stderr: "stderr", result: "result" }, + }; +} + +function failedProbe(stderr: string): ShellProbeResult { + return { + ...successfulProbe(), + exitCode: 1, + stderr, + }; +} + +describe("live test progress", () => { + afterEach(() => { + vi.useRealTimers(); + }); + + it("reports the active phase and timestamp-only child-output age", () => { + const { options, state } = progressHarness(); + const progress = startTestProgress("agent-turn-latency", "OpenClaw install attempt 1", options); + + progress.onOutput({ stream: "stderr", atMs: 21_000 }); + state.clockMs = 61_000; + state.timerCallback?.(); + const finishCommand = progress.activity("command: install-openclaw"); + state.clockMs = 71_000; + state.timerCallback?.(); + finishCommand(); + progress.phase("Hermes install attempt 1"); + progress.stop(); + + expect(state.clearCalls).toBe(1); + expect(state.lines).toEqual([ + "[agent-turn-latency] OpenClaw install attempt 1 started (0s elapsed; no child output observed; memory free 8.0 GiB/16.0 GiB; test RSS 0.5 GiB; workspace free 6.0 GiB; load 1m 2.50)", + "[agent-turn-latency] OpenClaw install attempt 1 running (60s elapsed; last child output 40s ago; memory free 8.0 GiB/16.0 GiB; test RSS 0.5 GiB; workspace free 6.0 GiB; load 1m 2.50)", + "[agent-turn-latency] command: install-openclaw running (10s elapsed; no child output observed; memory free 8.0 GiB/16.0 GiB; test RSS 0.5 GiB; workspace free 6.0 GiB; load 1m 2.50)", + "[agent-turn-latency] OpenClaw install attempt 1 finished (0s elapsed; no child output observed; memory free 8.0 GiB/16.0 GiB; test RSS 0.5 GiB; workspace free 6.0 GiB; load 1m 2.50)", + "[agent-turn-latency] Hermes install attempt 1 started (0s elapsed; no child output observed; memory free 8.0 GiB/16.0 GiB; test RSS 0.5 GiB; workspace free 6.0 GiB; load 1m 2.50)", + "[agent-turn-latency] Hermes install attempt 1 finished (0s elapsed; no child output observed; memory free 8.0 GiB/16.0 GiB; test RSS 0.5 GiB; workspace free 6.0 GiB; load 1m 2.50)", + ]); + expect(progress.summary()).toEqual({ + version: 1, + scenario: "agent-turn-latency", + startedAtMs: 1_000, + finishedAtMs: 71_000, + durationMs: 70_000, + phases: [ + { + label: "OpenClaw install attempt 1", + startedAtMs: 1_000, + finishedAtMs: 61_000, + durationMs: 60_000, + outputEvents: 1, + lastOutputAtMs: 21_000, + }, + { + label: "command: install-openclaw", + startedAtMs: 61_000, + finishedAtMs: 71_000, + durationMs: 10_000, + outputEvents: 0, + lastOutputAtMs: null, + }, + { + label: "OpenClaw install attempt 1", + startedAtMs: 71_000, + finishedAtMs: 71_000, + durationMs: 0, + outputEvents: 0, + lastOutputAtMs: null, + }, + { + label: "Hermes install attempt 1", + startedAtMs: 71_000, + finishedAtMs: 71_000, + durationMs: 0, + outputEvents: 0, + lastOutputAtMs: null, + }, + ], + }); + }); + + it("connects each install attempt to the phase and timestamp-only observer", async () => { + const command = vi.fn(async () => successfulProbe()); + const host = { command } as unknown as HostCliClient; + const progress = { onOutput: vi.fn(), phase: vi.fn() }; + + await installSandbox( + host, + "e2e-openclaw-turn-latency", + "openclaw", + "secret-api-key", + undefined, + progress, + ); + + expect(progress.phase).toHaveBeenCalledWith("OpenClaw install attempt 1"); + expect(command).toHaveBeenCalledOnce(); + expect(command.mock.calls[0]?.[2]).toMatchObject({ + artifactName: "openclaw-install-attempt-1", + onOutput: progress.onOutput, + redactionValues: ["secret-api-key"], + }); + }); + + it("reports transient install retry cleanup, backoff, and both attempts", async () => { + vi.useFakeTimers(); + const command = vi + .fn() + .mockResolvedValueOnce( + failedProbe("Chat Completions API validation failed: request timed out"), + ) + .mockResolvedValueOnce(successfulProbe()); + const cleanupBeforeRetry = vi.fn(async () => undefined); + const progress = { onOutput: vi.fn(), phase: vi.fn() }; + const host = { command } as unknown as HostCliClient; + + const resultPromise = installSandbox( + host, + "e2e-openclaw-turn-latency", + "openclaw", + "secret-api-key", + cleanupBeforeRetry, + progress, + ); + await vi.runAllTimersAsync(); + + await expect(resultPromise).resolves.toMatchObject({ exitCode: 0 }); + expect(cleanupBeforeRetry).toHaveBeenCalledOnce(); + expect(progress.phase.mock.calls.map(([phase]) => phase)).toEqual([ + "OpenClaw install attempt 1", + "OpenClaw install retry cleanup", + "OpenClaw install retry backoff", + "OpenClaw install attempt 2", + ]); + expect(command).toHaveBeenCalledTimes(2); + expect(command.mock.calls.map((call) => call[2])).toEqual([ + expect.objectContaining({ + artifactName: "openclaw-install-attempt-1", + onOutput: progress.onOutput, + }), + expect.objectContaining({ + artifactName: "openclaw-install-attempt-2", + onOutput: progress.onOutput, + }), + ]); + }); + + it("does not report retry phases for a non-transient install failure", async () => { + const command = vi.fn(async () => + failedProbe("endpoint validation failed: invalid NVIDIA_INFERENCE_API_KEY credential"), + ); + const cleanupBeforeRetry = vi.fn(async () => undefined); + const progress = { onOutput: vi.fn(), phase: vi.fn() }; + const host = { command } as unknown as HostCliClient; + + await expect( + installSandbox( + host, + "e2e-openclaw-turn-latency", + "openclaw", + "secret-api-key", + cleanupBeforeRetry, + progress, + ), + ).resolves.toMatchObject({ exitCode: 1 }); + + expect(command).toHaveBeenCalledOnce(); + expect(cleanupBeforeRetry).not.toHaveBeenCalled(); + expect(progress.phase.mock.calls.map(([phase]) => phase)).toEqual([ + "OpenClaw install attempt 1", + ]); + }); +}); diff --git a/test/e2e/support/e2e-fixture-context.test.ts b/test/e2e/support/e2e-fixture-context.test.ts index e8414ed942..9296922343 100644 --- a/test/e2e/support/e2e-fixture-context.test.ts +++ b/test/e2e/support/e2e-fixture-context.test.ts @@ -61,6 +61,7 @@ describe("E2E fixture primitives", () => { "run-plan.json", "target.json", "target-result.json", + "test-progress.json", "environment.result.json", "onboarding.result.json", "state-validation.result.json", @@ -245,6 +246,48 @@ describe("E2E fixture primitives", () => { } }); + it("shell probe reports child liveness to the shared observer without duplicating it", async () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-e2e-shell-observer-")); + try { + const artifacts = new ArtifactSink(tmp); + await artifacts.ensureRoot(); + const controller = new AbortController(); + const events: Array<{ stream: "stdout" | "stderr"; atMs: number }> = []; + const activities: string[] = []; + const onOutput = (event: (typeof events)[number]) => events.push(event); + const probe = new ShellProbe({ + artifacts, + redact: (text) => text, + signal: controller.signal, + onOutput, + onActivity: (label) => { + activities.push(`start ${label}`); + return () => activities.push(`finish ${label}`); + }, + }); + + const result = await probe.run( + trustedShellCommand({ + command: process.execPath, + args: ["-e", "console.log('alive')"], + reason: "verify shared child-output liveness observation", + }), + { artifactName: "shared-output-observer", onOutput, timeoutMs: 5_000 }, + ); + + expect(result.exitCode).toBe(0); + expect(events).toHaveLength(1); + expect(events[0]).toMatchObject({ stream: "stdout" }); + expect(events[0]?.atMs).toBeGreaterThan(0); + expect(activities).toEqual([ + "start command: shared-output-observer", + "finish command: shared-output-observer", + ]); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + }); + it("shell probe scrubs overlapping redactionValues longest-first when the injected redactor ignores extra values", async () => { const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-e2e-shell-probe-overlap-")); try { diff --git a/test/e2e/support/e2e-progress-fixture.test.ts b/test/e2e/support/e2e-progress-fixture.test.ts new file mode 100644 index 0000000000..9180abf56b --- /dev/null +++ b/test/e2e/support/e2e-progress-fixture.test.ts @@ -0,0 +1,44 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { afterAll, expect, vi } from "vitest"; + +import { test } from "../fixtures/e2e-test.ts"; +import type { ProgressSummary } from "../fixtures/progress.ts"; + +const artifactRoot = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-progress-fixture-")); +let progressArtifact = ""; + +vi.stubEnv("E2E_ARTIFACT_DIR", artifactRoot); + +afterAll(() => { + try { + const summary = JSON.parse(fs.readFileSync(progressArtifact, "utf8")) as ProgressSummary; + expect(summary).toMatchObject({ + version: 1, + scenario: "automatic progress fixture writes completed target and shard evidence", + targetId: "fixture-progress-target", + shardId: "fixture-progress-shard", + }); + expect(summary.finishedAtMs).not.toBeNull(); + expect(summary.durationMs).not.toBeNull(); + expect(summary.phases.at(-1)?.label).toBe("final fixture phase"); + } finally { + vi.unstubAllEnvs(); + fs.rmSync(artifactRoot, { force: true, recursive: true }); + } +}); + +test("automatic progress fixture writes completed target and shard evidence", async ({ + artifacts, + progress, +}) => { + progressArtifact = artifacts.pathFor("test-progress.json"); + vi.stubEnv("E2E_TARGET_ID", "fixture-progress-target"); + vi.stubEnv("NEMOCLAW_E2E_SHARD", "fixture-progress-shard"); + progress.phase("final fixture phase"); +}); diff --git a/test/e2e/support/openclaw-plugin-runtime-exdev-workflow-boundary.test.ts b/test/e2e/support/openclaw-plugin-runtime-exdev-workflow-boundary.test.ts index 60f2bb2783..f85d61c660 100644 --- a/test/e2e/support/openclaw-plugin-runtime-exdev-workflow-boundary.test.ts +++ b/test/e2e/support/openclaw-plugin-runtime-exdev-workflow-boundary.test.ts @@ -77,7 +77,7 @@ describe("OpenClaw plugin runtime EXDEV workflow boundary", () => { expect(validateOpenClawPluginRuntimeExdevWorkflow(workflow)).toEqual( expect.arrayContaining([ "openclaw-plugin-runtime-exdev must run on ubuntu-latest", - "openclaw-plugin-runtime-exdev must retain its 130 minute runtime proof budget", + "openclaw-plugin-runtime-exdev must retain its 105 minute runtime proof budget", "openclaw-plugin-runtime-exdev must hold only contents: read", "openclaw-plugin-runtime-exdev must set E2E_ARTIFACT_DIR=${{ github.workspace }}/e2e-artifacts/live/openclaw-plugin-runtime-exdev", "openclaw-plugin-runtime-exdev must remain enabled for scheduled and empty manual runs", diff --git a/test/e2e/support/rebuild-hermes-progress.test.ts b/test/e2e/support/rebuild-hermes-progress.test.ts index ea0f40e88a..1c11b55db6 100644 --- a/test/e2e/support/rebuild-hermes-progress.test.ts +++ b/test/e2e/support/rebuild-hermes-progress.test.ts @@ -2,10 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { describe, expect, it, vi } from "vitest"; -import { - type RebuildHermesProgressOptions, - startRebuildHermesProgress, -} from "../live/rebuild-hermes-progress.ts"; +import { startTestProgress, type TestProgressOptions } from "../fixtures/progress.ts"; function progressHarness() { const state = { @@ -15,7 +12,7 @@ function progressHarness() { lines: [] as string[], timerCallback: null as (() => void) | null, }; - const options: RebuildHermesProgressOptions = { + const options: TestProgressOptions = { heartbeatIntervalMs: 60_000, now: () => state.clockMs, setTimer: (callback) => { @@ -42,7 +39,7 @@ function progressHarness() { describe("Hermes rebuild live progress", () => { it("streams timestamp-only phase and resource heartbeats through cleanup", () => { const { options, state } = progressHarness(); - const progress = startRebuildHermesProgress("phase 6 nemoclaw rebuild", options); + const progress = startTestProgress("rebuild-hermes", "phase 6 nemoclaw rebuild", options); progress.onOutput({ stream: "stderr", atMs: 21_000 }); state.clockMs = 61_000; @@ -80,7 +77,7 @@ describe("Hermes rebuild live progress", () => { }; expect(() => { - const progress = startRebuildHermesProgress("phase 2 old base build", options); + const progress = startTestProgress("rebuild-hermes", "phase 2 old base build", options); progress.phase("cleanup"); progress.stop(); }).not.toThrow(); diff --git a/test/e2e/support/rebuild-hermes-timing.test.ts b/test/e2e/support/rebuild-hermes-timing.test.ts index 8c9df0312a..0d08a9c5ed 100644 --- a/test/e2e/support/rebuild-hermes-timing.test.ts +++ b/test/e2e/support/rebuild-hermes-timing.test.ts @@ -2,10 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { describe, expect, it } from "vitest"; -import { - type RebuildHermesProgressOptions, - startRebuildHermesProgress, -} from "../live/rebuild-hermes-progress.ts"; +import { startTestProgress, type TestProgressOptions } from "../fixtures/progress.ts"; import { buildRebuildHermesTimingSummary, describeRunnerClass, @@ -13,7 +10,7 @@ import { function timelineHarness() { const state = { clockMs: 1_000 }; - const options: RebuildHermesProgressOptions = { + const options: TestProgressOptions = { now: () => state.clockMs, setTimer: () => ({ unref() {} }), clearTimer: () => {}, @@ -34,7 +31,7 @@ function timelineHarness() { describe("Hermes rebuild timing timeline", () => { it("records each completed phase and closes the in-flight phase on snapshot", () => { const { options, state } = timelineHarness(); - const progress = startRebuildHermesProgress("setup", options); + const progress = startTestProgress("rebuild-hermes", "setup", options); state.clockMs = 4_000; progress.phase("phase 6 nemoclaw rebuild"); @@ -51,7 +48,7 @@ describe("Hermes rebuild timing timeline", () => { it("freezes the timeline at stop and ignores post-stop transitions", () => { const { options, state } = timelineHarness(); - const progress = startRebuildHermesProgress("phase 7 verification", options); + const progress = startTestProgress("rebuild-hermes", "phase 7 verification", options); state.clockMs = 6_000; progress.stop(); @@ -72,7 +69,7 @@ describe("Hermes rebuild timing timeline", () => { clockMs += 1_000; return current; }; - const progress = startRebuildHermesProgress("phase 7 verification", options); + const progress = startTestProgress("rebuild-hermes", "phase 7 verification", options); progress.stop(); diff --git a/test/e2e/support/test-runtime-audit.test.ts b/test/e2e/support/test-runtime-audit.test.ts new file mode 100644 index 0000000000..a4b5245631 --- /dev/null +++ b/test/e2e/support/test-runtime-audit.test.ts @@ -0,0 +1,97 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { auditTestRuntime, formatRuntimeAudit } from "../../../scripts/audit-test-runtime.mts"; + +function writeProgress( + root: string, + run: string, + scenario: string, + durationMs: number, + phase: string, + phaseDurationMs: number, +): void { + const directory = path.join(root, run, scenario); + fs.mkdirSync(directory, { recursive: true }); + fs.writeFileSync( + path.join(directory, "test-progress.json"), + `${JSON.stringify({ + version: 1, + targetId: `${scenario}-target`, + scenario, + startedAtMs: 1_000, + finishedAtMs: 1_000 + durationMs, + durationMs, + phases: [ + { + label: phase, + startedAtMs: 1_000, + finishedAtMs: 1_000 + phaseDurationMs, + durationMs: phaseDurationMs, + outputEvents: 1, + lastOutputAtMs: 1_500, + }, + ], + })}\n`, + "utf8", + ); +} + +describe("test runtime audit", () => { + it("ranks repeated artifact summaries by p95 and identifies the slowest phase", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-runtime-audit-")); + try { + writeProgress(root, "run-1", "variable test", 10_000, "install", 8_000); + writeProgress(root, "run-2", "variable test", 50_000, "inference", 40_000); + writeProgress(root, "run-1", "steady test", 20_000, "sandbox", 15_000); + + const rows = auditTestRuntime([root]); + + expect(rows).toEqual([ + { + target: "variable test-target", + scenario: "variable test", + runs: 2, + medianMs: 30_000, + p95Ms: 50_000, + maxMs: 50_000, + variabilityMs: 20_000, + slowestPhase: "inference", + slowestPhaseMs: 40_000, + }, + { + target: "steady test-target", + scenario: "steady test", + runs: 1, + medianMs: 20_000, + p95Ms: 20_000, + maxMs: 20_000, + variabilityMs: 0, + slowestPhase: "sandbox", + slowestPhaseMs: 15_000, + }, + ]); + expect(formatRuntimeAudit(rows)).toContain( + "| variable test-target | variable test | 2 | 30.0s | 50.0s | 50.0s | 20.0s | inference (40.0s) |", + ); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } + }); + + it("rejects malformed progress artifacts instead of producing a misleading ranking", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-runtime-audit-invalid-")); + try { + fs.writeFileSync(path.join(root, "test-progress.json"), '{"version":1}\n', "utf8"); + expect(() => auditTestRuntime([root])).toThrow(/invalid test progress summary/); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/test/regression-e2e-workflow.test.ts b/test/regression-e2e-workflow.test.ts index 1ce8247d9b..fa7fab5abb 100644 --- a/test/regression-e2e-workflow.test.ts +++ b/test/regression-e2e-workflow.test.ts @@ -137,6 +137,7 @@ describe("Regression E2E workflow contract", () => { // source-shape-contract: security -- Keeps the custom-plugin EXDEV regression immutable and free of repository secrets it("runs the OpenClaw custom-plugin lifecycle and EXDEV guard in a secret-free lane", () => { + const releaseJob = workflow.jobs?.["openclaw-plugin-runtime-exdev-release-e2e"]; const job = workflow.jobs?.["openclaw-plugin-runtime-exdev-e2e"]; const steps = job?.steps ?? []; const runText = steps.map((step) => step.run ?? "").join("\n"); @@ -151,7 +152,7 @@ describe("Regression E2E workflow contract", () => { const serializedJob = JSON.stringify(job); expect(job?.permissions).toEqual({ contents: "read" }); - expect(job?.["timeout-minutes"]).toBe(130); + expect(job?.["timeout-minutes"]).toBe(105); expect(checkoutStep?.uses).toMatch(FULL_SHA_ACTION); expect(checkoutStep?.with?.["persist-credentials"]).toBe(false); expect(setupNodeStep?.uses).toMatch(FULL_SHA_ACTION); @@ -166,8 +167,16 @@ describe("Regression E2E workflow contract", () => { } expect(runText).toContain("test/e2e/live/openclaw-plugin-runtime-exdev.test.ts"); + expect(runText).toContain("-t current-lifecycle"); expect(runText).toContain("npx vitest run --project e2e-live"); expect(runText).toContain("npm ci --ignore-scripts"); expect(runText).toContain("npm run build:cli"); + + const releaseRunText = (releaseJob?.steps ?? []).map((step) => step.run ?? "").join("\n"); + expect(releaseJob?.permissions).toEqual({ contents: "read" }); + expect(releaseJob?.["timeout-minutes"]).toBe(55); + expect(JSON.stringify(releaseJob)).not.toContain("${{ secrets."); + expect(releaseRunText).toContain("test/e2e/live/openclaw-plugin-runtime-exdev.test.ts"); + expect(releaseRunText).toContain("-t release-baseline"); }); }); diff --git a/tools/e2e/openclaw-plugin-runtime-exdev-workflow-boundary.mts b/tools/e2e/openclaw-plugin-runtime-exdev-workflow-boundary.mts index 676b8f6bbc..6371204e27 100644 --- a/tools/e2e/openclaw-plugin-runtime-exdev-workflow-boundary.mts +++ b/tools/e2e/openclaw-plugin-runtime-exdev-workflow-boundary.mts @@ -11,11 +11,31 @@ import { UPLOAD_E2E_ARTIFACTS_ACTION } from "./upload-e2e-artifacts-workflow-bou const REPO_ROOT = join(dirname(fileURLToPath(import.meta.url)), "..", ".."); const DEFAULT_WORKFLOW_PATH = join(REPO_ROOT, ".github", "workflows", "e2e.yaml"); -const JOB_NAME = "openclaw-plugin-runtime-exdev"; const FULL_SHA_ACTION = /^[^\s@]+@[0-9a-f]{40}$/u; const RELEASE_BUILDER_IMAGE = "node:22-trixie-slim@sha256:2d9f5c76c8f4dd36e8f253bee5d828a83a6c09f36188f0b0414325232e0b175d"; +const JOB_CONTRACTS = [ + { + jobName: "openclaw-plugin-runtime-exdev-release", + timeoutMinutes: 55, + artifactId: "openclaw-plugin-runtime-exdev-release", + sandboxName: "e2e-openclaw-plugin-exdev-release", + selector: "release-baseline", + runName: "Run OpenClaw custom-plugin release baseline live test", + uploadName: "Upload OpenClaw plugin release baseline artifacts", + }, + { + jobName: "openclaw-plugin-runtime-exdev", + timeoutMinutes: 105, + artifactId: "openclaw-plugin-runtime-exdev", + sandboxName: "e2e-openclaw-plugin-exdev", + selector: "current-lifecycle", + runName: "Run OpenClaw custom-plugin lifecycle and runtime-deps EXDEV live test", + uploadName: "Upload OpenClaw plugin runtime-deps EXDEV artifacts", + }, +] as const; + type WorkflowStep = { env?: Record; if?: string; @@ -51,17 +71,19 @@ function findStep(job: WorkflowJob, name: string): WorkflowStep { function requireRunContains( errors: string[], + jobName: string, step: WorkflowStep, fragment: string, description = step.name ?? "", ): void { if (!step.run?.includes(fragment)) { - errors.push(`${JOB_NAME} step '${description}' must run: ${fragment}`); + errors.push(`${jobName} step '${description}' must run: ${fragment}`); } } function requireStepOrder( errors: string[], + jobName: string, steps: WorkflowStep[], beforeName: string, afterName: string, @@ -69,12 +91,13 @@ function requireStepOrder( const before = steps.findIndex((step) => step.name === beforeName); const after = steps.findIndex((step) => step.name === afterName); if (before < 0 || after < 0 || before >= after) { - errors.push(`${JOB_NAME} step '${beforeName}' must precede '${afterName}'`); + errors.push(`${jobName} step '${beforeName}' must precede '${afterName}'`); } } function requireAdjacentSteps( errors: string[], + jobName: string, steps: WorkflowStep[], beforeName: string, afterName: string, @@ -82,7 +105,7 @@ function requireAdjacentSteps( const before = steps.findIndex((step) => step.name === beforeName); const after = steps.findIndex((step) => step.name === afterName); if (before < 0 || after !== before + 1) { - errors.push(`${JOB_NAME} step '${beforeName}' must immediately precede '${afterName}'`); + errors.push(`${jobName} step '${beforeName}' must immediately precede '${afterName}'`); } } @@ -90,124 +113,128 @@ export function validateOpenClawPluginRuntimeExdevWorkflow( workflow: OpenClawPluginRuntimeExdevWorkflow, ): string[] { const errors: string[] = []; - const job = workflow.jobs[JOB_NAME]; - if (!job) return [`workflow is missing ${JOB_NAME}`]; - - if (job.needs !== "generate-matrix") { - errors.push(`${JOB_NAME} must depend on generate-matrix`); - } - if (job["runs-on"] !== "ubuntu-latest") { - errors.push(`${JOB_NAME} must run on ubuntu-latest`); - } - if (job["timeout-minutes"] !== 130) { - errors.push(`${JOB_NAME} must retain its 130 minute runtime proof budget`); - } - if (job.permissions?.contents !== "read" || Object.keys(job.permissions ?? {}).length !== 1) { - errors.push(`${JOB_NAME} must hold only contents: read`); - } - - const env = job.env ?? {}; - const expectedEnv = { - E2E_ARTIFACT_DIR: "${{ github.workspace }}/e2e-artifacts/live/openclaw-plugin-runtime-exdev", - NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", - NEMOCLAW_CLI_BIN: "${{ github.workspace }}/bin/nemoclaw.js", - NEMOCLAW_NON_INTERACTIVE: "1", - NEMOCLAW_RUN_LIVE_E2E: "1", - NEMOCLAW_SANDBOX_NAME: "e2e-openclaw-plugin-exdev", - OPENSHELL_GATEWAY: "nemoclaw", - }; - for (const [name, value] of Object.entries(expectedEnv)) { - if (env[name] !== value) errors.push(`${JOB_NAME} must set ${name}=${value}`); - } - if (Object.hasOwn(env, "E2E_DEFAULT_ENABLED")) { - errors.push(`${JOB_NAME} must remain enabled for scheduled and empty manual runs`); - } - for (const secret of [ - "DOCKERHUB_USERNAME", - "DOCKERHUB_TOKEN", - "GITHUB_TOKEN", - "NVIDIA_API_KEY", - "NVIDIA_INFERENCE_API_KEY", - ]) { - if (Object.hasOwn(env, secret)) - errors.push(`${JOB_NAME} must not expose ${secret} at job scope`); - } - - const steps = job.steps ?? []; - for (const step of steps.filter((candidate) => candidate.uses)) { - if (!FULL_SHA_ACTION.test(step.uses ?? "")) { - errors.push(`${JOB_NAME} action '${step.name ?? step.uses}' must pin a full SHA`); + for (const contract of JOB_CONTRACTS) { + const { artifactId, jobName, runName, sandboxName, selector, timeoutMinutes, uploadName } = + contract; + const job = workflow.jobs[jobName]; + if (!job) { + errors.push(`workflow is missing ${jobName}`); + continue; } - } - const checkout = steps.find((step) => step.uses?.startsWith("actions/checkout@")) ?? {}; - if (checkout.with?.["persist-credentials"] !== false) { - errors.push(`${JOB_NAME} checkout must disable persisted credentials`); - } + if (job.needs !== "generate-matrix") errors.push(`${jobName} must depend on generate-matrix`); + if (job["runs-on"] !== "ubuntu-latest") errors.push(`${jobName} must run on ubuntu-latest`); + if (job["timeout-minutes"] !== timeoutMinutes) { + errors.push(`${jobName} must retain its ${timeoutMinutes} minute runtime proof budget`); + } + if (job.permissions?.contents !== "read" || Object.keys(job.permissions ?? {}).length !== 1) { + errors.push(`${jobName} must hold only contents: read`); + } - const prepare = findStep(job, "Prepare E2E workspace"); - if (prepare.uses !== PREPARE_E2E_ACTION) { - errors.push(`${JOB_NAME} must use the reviewed prepare-e2e action`); - } + const env = job.env ?? {}; + const expectedEnv = { + E2E_ARTIFACT_DIR: `\${{ github.workspace }}/e2e-artifacts/live/${artifactId}`, + E2E_TARGET_ID: artifactId, + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", + NEMOCLAW_CLI_BIN: "${{ github.workspace }}/bin/nemoclaw.js", + NEMOCLAW_NON_INTERACTIVE: "1", + NEMOCLAW_RUN_LIVE_E2E: "1", + NEMOCLAW_SANDBOX_NAME: sandboxName, + OPENSHELL_GATEWAY: "nemoclaw", + }; + for (const [name, value] of Object.entries(expectedEnv)) { + if (env[name] !== value) errors.push(`${jobName} must set ${name}=${value}`); + } + if (Object.hasOwn(env, "E2E_DEFAULT_ENABLED")) { + errors.push(`${jobName} must remain enabled for scheduled and empty manual runs`); + } + for (const secret of [ + "DOCKERHUB_USERNAME", + "DOCKERHUB_TOKEN", + "GITHUB_TOKEN", + "NVIDIA_API_KEY", + "NVIDIA_INFERENCE_API_KEY", + ]) { + if (Object.hasOwn(env, secret)) + errors.push(`${jobName} must not expose ${secret} at job scope`); + } - const prePull = findStep(job, "Pre-pull release-matched Docker Hub builder image"); - requireRunContains(errors, prePull, `docker pull ${RELEASE_BUILDER_IMAGE}`); + const steps = job.steps ?? []; + for (const step of steps.filter((candidate) => candidate.uses)) { + if (!FULL_SHA_ACTION.test(step.uses ?? "")) { + errors.push(`${jobName} action '${step.name ?? step.uses}' must pin a full SHA`); + } + } + const checkout = steps.find((step) => step.uses?.startsWith("actions/checkout@")) ?? {}; + if (checkout.with?.["persist-credentials"] !== false) { + errors.push(`${jobName} checkout must disable persisted credentials`); + } + const prepare = findStep(job, "Prepare E2E workspace"); + if (prepare.uses !== PREPARE_E2E_ACTION) { + errors.push(`${jobName} must use the reviewed prepare-e2e action`); + } - const revoke = findStep(job, "Remove Docker auth before release-pinned fixture"); - if (revoke.if !== "always()") { - errors.push(`${JOB_NAME} must always revoke Docker auth before the release-pinned fixture`); - } - requireRunContains(errors, revoke, "bash .github/scripts/docker-auth-cleanup.sh"); - - const runName = "Run OpenClaw custom-plugin lifecycle and runtime-deps EXDEV live test"; - const run = findStep(job, runName); - for (const fragment of [ - 'test -n "${DOCKER_CONFIG:-}"', - 'test ! -e "${DOCKER_CONFIG}"', - 'test -z "${DOCKERHUB_USERNAME:-}"', - 'test -z "${DOCKERHUB_TOKEN:-}"', - "env -u DOCKER_CONFIG -u DOCKERHUB_USERNAME -u DOCKERHUB_TOKEN", - "tools/e2e/live-vitest-invocation.mts run --test-path", - "test/e2e/live/openclaw-plugin-runtime-exdev.test.ts", - ]) { - requireRunContains(errors, run, fragment, runName); - } - if (Object.keys(run.env ?? {}).length > 0 || JSON.stringify(run).includes("secrets.")) { - errors.push(`${JOB_NAME} runtime proof must not receive workflow credentials`); - } + const prePull = findStep(job, "Pre-pull release-matched Docker Hub builder image"); + requireRunContains(errors, jobName, prePull, `docker pull ${RELEASE_BUILDER_IMAGE}`); + const revoke = findStep(job, "Remove Docker auth before release-pinned fixture"); + if (revoke.if !== "always()") { + errors.push(`${jobName} must always revoke Docker auth before the release-pinned fixture`); + } + requireRunContains(errors, jobName, revoke, "bash .github/scripts/docker-auth-cleanup.sh"); + + const run = findStep(job, runName); + for (const fragment of [ + 'test -n "${DOCKER_CONFIG:-}"', + 'test ! -e "${DOCKER_CONFIG}"', + 'test -z "${DOCKERHUB_USERNAME:-}"', + 'test -z "${DOCKERHUB_TOKEN:-}"', + "env -u DOCKER_CONFIG -u DOCKERHUB_USERNAME -u DOCKERHUB_TOKEN", + "tools/e2e/live-vitest-invocation.mts run", + "--test-path test/e2e/live/openclaw-plugin-runtime-exdev.test.ts", + `--selector ${selector}`, + ]) { + requireRunContains(errors, jobName, run, fragment, runName); + } + if (Object.keys(run.env ?? {}).length > 0 || JSON.stringify(run).includes("secrets.")) { + errors.push(`${jobName} runtime proof must not receive workflow credentials`); + } + const upload = findStep(job, uploadName); + if (upload.uses !== UPLOAD_E2E_ARTIFACTS_ACTION || upload.if !== "always()") { + errors.push(`${jobName} must always use the reviewed artifact uploader`); + } - const upload = findStep(job, "Upload OpenClaw plugin runtime-deps EXDEV artifacts"); - if (upload.uses !== UPLOAD_E2E_ARTIFACTS_ACTION || upload.if !== "always()") { - errors.push(`${JOB_NAME} must always use the reviewed artifact uploader`); + requireStepOrder( + errors, + jobName, + steps, + "Pre-pull release-matched Docker Hub builder image", + "Remove Docker auth before release-pinned fixture", + ); + requireStepOrder( + errors, + jobName, + steps, + "Remove Docker auth before release-pinned fixture", + "Prepare E2E workspace", + ); + requireStepOrder(errors, jobName, steps, "Prepare E2E workspace", runName); + requireStepOrder(errors, jobName, steps, runName, uploadName); + requireAdjacentSteps( + errors, + jobName, + steps, + "Authenticate to Docker Hub", + "Pre-pull release-matched Docker Hub builder image", + ); + requireAdjacentSteps( + errors, + jobName, + steps, + "Pre-pull release-matched Docker Hub builder image", + "Remove Docker auth before release-pinned fixture", + ); } - requireStepOrder( - errors, - steps, - "Pre-pull release-matched Docker Hub builder image", - "Remove Docker auth before release-pinned fixture", - ); - requireStepOrder( - errors, - steps, - "Remove Docker auth before release-pinned fixture", - "Prepare E2E workspace", - ); - requireStepOrder(errors, steps, "Prepare E2E workspace", runName); - requireStepOrder(errors, steps, runName, "Upload OpenClaw plugin runtime-deps EXDEV artifacts"); - requireAdjacentSteps( - errors, - steps, - "Authenticate to Docker Hub", - "Pre-pull release-matched Docker Hub builder image", - ); - requireAdjacentSteps( - errors, - steps, - "Pre-pull release-matched Docker Hub builder image", - "Remove Docker auth before release-pinned fixture", - ); - return errors; } diff --git a/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts b/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts index b261d2fc75..b651b9bbb1 100644 --- a/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts +++ b/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts @@ -62,6 +62,7 @@ const EXPLICIT_UPLOAD_CONTRACTS = new Map([ "e2e-artifacts/live/${{ matrix.id }}/run-plan.json", "e2e-artifacts/live/${{ matrix.id }}/target.json", "e2e-artifacts/live/${{ matrix.id }}/target-result.json", + "e2e-artifacts/live/${{ matrix.id }}/test-progress.json", "e2e-artifacts/live/${{ matrix.id }}/environment.result.json", "e2e-artifacts/live/${{ matrix.id }}/onboarding.result.json", "e2e-artifacts/live/${{ matrix.id }}/state-validation.result.json", diff --git a/tools/e2e/workflow-boundary.mts b/tools/e2e/workflow-boundary.mts index ae825c3c71..a5bb1420a5 100644 --- a/tools/e2e/workflow-boundary.mts +++ b/tools/e2e/workflow-boundary.mts @@ -4231,6 +4231,11 @@ export function validateE2eWorkflow(workflowValue: unknown): string[] { uploadPath, "e2e-artifacts/live/${{ matrix.id }}/target-result.json", ); + requireUploadPathContains( + errors, + uploadPath, + "e2e-artifacts/live/${{ matrix.id }}/test-progress.json", + ); requireUploadPathContains( errors, uploadPath,