diff --git a/.github/workflows/candidate-compatibility.yaml b/.github/workflows/candidate-compatibility.yaml index f72ae48142..c466099b35 100644 --- a/.github/workflows/candidate-compatibility.yaml +++ b/.github/workflows/candidate-compatibility.yaml @@ -348,6 +348,13 @@ jobs: ' fi + - id: artifact_safety + name: Validate final OpenShell gateway auth contract artifacts + if: ${{ always() }} + env: + E2E_ARTIFACT_DIR: ${{ github.workspace }}/candidate-source/e2e-artifacts/live/openshell-gateway-auth-contract + run: node --experimental-strip-types --no-warnings controller/tools/e2e/openshell-gateway-auth-artifact-safety.mts "$E2E_ARTIFACT_DIR" + - name: Upload live evidence if: ${{ always() }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 @@ -356,7 +363,7 @@ jobs: path: | candidate-results/live-openshell-gateway-auth-contract.json candidate-live-observed.json - candidate-source/e2e-artifacts/live/openshell-gateway-auth-contract/ + ${{ steps.artifact_safety.outcome == 'success' && steps.artifact_safety.outputs.approved_path || '' }} if-no-files-found: error retention-days: 30 diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 530874a81b..91b9c556be 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -521,9 +521,17 @@ jobs: "$OPENSHELL_GATEWAY_BIN" --version npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/openshell-gateway-auth-source-contract.test.ts - - name: Upload OpenShell gateway auth contract artifacts + - id: artifact_safety + name: Validate final OpenShell gateway auth contract artifacts if: always() + run: node --experimental-strip-types --no-warnings tools/e2e/openshell-gateway-auth-artifact-safety.mts "$E2E_ARTIFACT_DIR" + + - name: Upload OpenShell gateway auth contract artifacts + if: ${{ always() && steps.artifact_safety.outcome == 'success' && steps.artifact_safety.outputs.approved_path != '' }} uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 + with: + name: e2e-openshell-gateway-auth-contract + path: ${{ steps.artifact_safety.outputs.approved_path }} - name: Clean up Docker auth if: always() diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7e67b6527f..2d61af2936 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -283,11 +283,11 @@ repos: - repo: local hooks: - id: e2e-semantic-phase-plans - name: Live E2E semantic phase plans + name: E2E semantic phase plans entry: npm run test:e2e-phases:check language: system pass_filenames: false - files: ^(test/e2e/live/.*\.ts|test/e2e/fixtures/(e2e-test|paths|progress)\.ts|tools/e2e/check-semantic-phases\.mts|vitest\.config\.ts|package\.json)$ + files: ^(\.github/workflows/e2e\.yaml|test/(gateway-drift-preflight|vllm-docker-storage)\.test\.ts|test/e2e/live/.*\.ts|test/e2e/fixtures/.*\.ts|test/e2e/risk-signal-reporter\.ts|test/e2e/support/(e2e-semantic-phase-check|workflow-e2e-progress)\.test\.ts|tools/e2e/(check-semantic-phases|credential-free-tests|workflow-boundary|workflow-plan)\.mts|vitest\.config\.ts|package\.json)$ priority: 20 - id: test-cli diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 55dfcafd11..e01b7c7f68 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -211,7 +211,7 @@ These are the primary npm scripts for day-to-day development: | `npm run test:watch` | Watch the CLI, plugin, and E2E-support projects and rerun affected tests | | `npm run test:shuffle` | Shuffle test order in the focused source projects without collecting coverage | | `npm run test:diagnose:leaks` | Report async-resource leaks and diagnose a Vitest process that hangs during shutdown | -| `npm run test:e2e-phases:check` | Validate semantic phase plans for every live E2E test without executing live bodies | +| `npm run test:e2e-phases:check` | Validate semantic phase plans for every live E2E test and workflow-selected credential-free integration test without executing test bodies | | `npm run test:runtime-audit -- [...]` | Rank captured live E2E runs by median, p95, variability, and slowest phase | | `npm run test:integration` | Clean-build the CLI and run root integration and installer tests | | `npm run test:package` | Clean-build CLI/plugin artifacts and run compiled-package contracts | @@ -234,16 +234,29 @@ npx vitest run --project e2e-support This project is fast and does not run live targets. Live E2E remains opt-in through `npm run test:live-e2e` or the applicable GitHub Actions workflow. -Every `e2e-live` test must declare its ordered, behavior-specific phase plan in -`meta.e2ePhases`, call `progress.phase("literal phase label")` at those -boundaries, and reach the final test-declared phase on every passing path. The -harness then appends `release registered E2E resources` so cleanup duration and -failures have their own phase. Run `npm run test:e2e-phases:check` after adding -or changing a live E2E case; collection validates the plans without running -infrastructure-mutating test bodies. See +Every `e2e-live` test, plus every credential-free integration test selected by +the shared E2E workflow planner, must declare its ordered, behavior-specific +phase plan in `meta.e2ePhases`, call +`progress.phase("literal phase label")` at those boundaries, and reach the final +test-declared phase on every passing path. Live tests import the shared +`e2e-test` fixture, which appends `release registered E2E resources` so cleanup +duration and failures have their own phase. Workflow-selected integration tests +import `workflow-e2e-test` and declare their own final release phase. Run +`npm run test:e2e-phases:check` after changing either coverage set or its +workflow selection; collection validates the union without running test bodies. See [`test/e2e/docs/README.md`](test/e2e/docs/README.md) for the logging and artifact contract. +Use the shared `ShellProbe` for E2E child processes. The semantic-phase check +also follows shared E2E helpers and rejects new direct asynchronous process +boundaries unless they are explicitly audited for content-free activity and +timestamp-only output reporting. Synchronous process calls must have a positive +timeout shorter than the first heartbeat and use `killSignal: "SIGKILL"` so the +child cannot ignore that bound; write child contents only through the redacted +artifact sink. Pass the auto fixture's frozen, canonical `progress` capability +through unchanged; custom, copied, or no-op progress adapters are rejected at +audited subprocess boundaries. + ### Test Declarative Behavior Do not read a shipped YAML, JSON, manifest, workflow, or E2E runtime file only to assert its keys, diff --git a/test/candidate-compat.test.ts b/test/candidate-compat.test.ts index 3e398323cf..1cef8def4f 100644 --- a/test/candidate-compat.test.ts +++ b/test/candidate-compat.test.ts @@ -95,6 +95,8 @@ describe("OpenShell candidate compatibility contract", () => { if?: string; name?: string; run?: string; + with?: Record; + "working-directory"?: string; }>; } >; @@ -102,8 +104,18 @@ describe("OpenShell candidate compatibility contract", () => { permissions: Record; }; const evidence = workflow.jobs.evidence; + const liveSteps = workflow.jobs.live?.steps ?? []; const finalize = evidence?.steps?.find((step) => step.name === "Finalize auditable evidence"); const enforce = evidence?.steps?.find((step) => step.name === "Enforce aggregate result"); + const uploadLiveEvidence = workflow.jobs.live?.steps?.find( + (step) => step.name === "Upload live evidence", + ); + const artifactSafety = workflow.jobs.live?.steps?.find( + (step) => step.name === "Validate final OpenShell gateway auth contract artifacts", + ); + const recordLiveResult = workflow.jobs.live?.steps?.find( + (step) => step.name === "Record receipt-bound live result", + ); expect(Object.keys(workflow.on.workflow_dispatch.inputs).sort()).toEqual([ "candidate", "component", @@ -122,6 +134,28 @@ describe("OpenShell candidate compatibility contract", () => { expect(source).toContain("RESOLUTION_ID: ${{ needs.resolve.outputs.resolution_id }}"); expect(source).toContain("verify-invocations"); expect(source).toContain("openshell-gateway-auth-source-contract.test.ts"); + expect(artifactSafety).toMatchObject({ + env: { + E2E_ARTIFACT_DIR: + "${{ github.workspace }}/candidate-source/e2e-artifacts/live/openshell-gateway-auth-contract", + }, + id: "artifact_safety", + if: "${{ always() }}", + run: 'node --experimental-strip-types --no-warnings controller/tools/e2e/openshell-gateway-auth-artifact-safety.mts "$E2E_ARTIFACT_DIR"', + }); + expect(liveSteps.findIndex((step) => step.id === "live_test")).toBeLessThan( + liveSteps.indexOf(recordLiveResult!), + ); + expect(liveSteps.indexOf(recordLiveResult!)).toBeLessThan(liveSteps.indexOf(artifactSafety!)); + expect(liveSteps.indexOf(artifactSafety!)).toBeLessThan(liveSteps.indexOf(uploadLiveEvidence!)); + expect(uploadLiveEvidence?.with?.path).toBe( + [ + "candidate-results/live-openshell-gateway-auth-contract.json", + "candidate-live-observed.json", + "${{ steps.artifact_safety.outcome == 'success' && steps.artifact_safety.outputs.approved_path || '' }}", + "", + ].join("\n"), + ); expect(evidence?.permissions).toEqual({ actions: "read", contents: "read" }); expect(finalize?.id).toBe("finalize"); expect(finalize?.env).toMatchObject({ diff --git a/test/compatible-endpoint-context-probe.test.ts b/test/compatible-endpoint-context-probe.test.ts index 6f69bbd05d..2b4ef5a5f4 100644 --- a/test/compatible-endpoint-context-probe.test.ts +++ b/test/compatible-endpoint-context-probe.test.ts @@ -17,9 +17,20 @@ import { type FakeOpenAiCompatibleServer, startFakeOpenAiCompatibleServer, } from "./e2e/fixtures/fake-openai-compatible"; +import { startTestProgress, type TestProgress } from "./e2e/fixtures/progress.ts"; import { testTimeout } from "./helpers/timeouts"; const MODEL = "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4"; +let progress: TestProgress | null = null; + +function testProgress(): TestProgress { + progress ??= startTestProgress( + "compatible endpoint support", + ["serve compatible endpoint", "verify compatible endpoint"], + { targetId: "compatible-endpoint-context-probe" }, + ); + return progress; +} // The fake server binds to loopback (127.0.0.1). Loopback is an allowed // host-side probe target (a locally-run vLLM/Ollama endpoint), so these @@ -43,16 +54,28 @@ function fetchFromServer(apiKey: string): () => unknown | null { } afterEach(async () => { - await server?.close(); - server = null; + try { + await server?.close(); + } finally { + server = null; + progress?.stop(); + progress = null; + } }); describe("compatible-endpoint context probe against a real server (#6177)", { timeout: testTimeout(60_000), }, () => { it("reads max_model_len from a live /v1/models endpoint into NEMOCLAW_CONTEXT_WINDOW (#6177)", async () => { - server = await startFakeOpenAiCompatibleServer({ model: MODEL, maxModelLen: 65_536 }); + const testRun = testProgress(); + testRun.phase("serve compatible endpoint"); + server = await startFakeOpenAiCompatibleServer({ + model: MODEL, + maxModelLen: 65_536, + progress: testRun, + }); + testRun.phase("verify compatible endpoint"); const models = fetchCompatibleEndpointModels(server.baseUrl, ""); expect(models).toMatchObject({ data: [{ id: MODEL, max_model_len: 65_536 }] }); @@ -66,12 +89,16 @@ describe("compatible-endpoint context probe against a real server (#6177)", { }); it("sends the endpoint credential through curl's --config auth flow (#6177)", async () => { + const testRun = testProgress(); + testRun.phase("serve compatible endpoint"); server = await startFakeOpenAiCompatibleServer({ model: MODEL, maxModelLen: 32_768, apiKey: "secret-key", + progress: testRun, }); + testRun.phase("verify compatible endpoint"); const env: NodeJS.ProcessEnv = {}; await applyCompatibleEndpointContextWindow(PUBLIC_ENDPOINT_URL, MODEL, { env, @@ -89,13 +116,17 @@ describe("compatible-endpoint context probe against a real server (#6177)", { }); it("enforces auth on /v1/models: sets the window with the key, skips it without (#6177)", async () => { + const testRun = testProgress(); + testRun.phase("serve compatible endpoint"); server = await startFakeOpenAiCompatibleServer({ model: MODEL, maxModelLen: 65_536, apiKey: "secret-key", + progress: testRun, requireAuthModels: true, }); + testRun.phase("verify compatible endpoint"); // Wrong/absent credential → the endpoint 401s → no window is set. const noKeyEnv: NodeJS.ProcessEnv = {}; await applyCompatibleEndpointContextWindow(PUBLIC_ENDPOINT_URL, MODEL, { @@ -131,7 +162,14 @@ describe("compatible-endpoint context probe against a real server (#6177)", { // so the source-boundary guard exempts loopback (mirroring the chat probe) // and the real curl fetcher must run and propagate the window. Non-loopback // private targets stay blocked — see the unit-test rejection cases. - server = await startFakeOpenAiCompatibleServer({ model: MODEL, maxModelLen: 65_536 }); + const testRun = testProgress(); + testRun.phase("serve compatible endpoint"); + server = await startFakeOpenAiCompatibleServer({ + model: MODEL, + maxModelLen: 65_536, + progress: testRun, + }); + testRun.phase("verify compatible endpoint"); expect(new URL(server.baseUrl).hostname).toBe("127.0.0.1"); const modelsRequestsBefore = server .requests() @@ -151,8 +189,11 @@ describe("compatible-endpoint context probe against a real server (#6177)", { }); it("keeps the default context window when the endpoint omits max_model_len (#6177)", async () => { - server = await startFakeOpenAiCompatibleServer({ model: MODEL }); + const testRun = testProgress(); + testRun.phase("serve compatible endpoint"); + server = await startFakeOpenAiCompatibleServer({ model: MODEL, progress: testRun }); + testRun.phase("verify compatible endpoint"); const env: NodeJS.ProcessEnv = {}; await applyCompatibleEndpointContextWindow(PUBLIC_ENDPOINT_URL, MODEL, { env, diff --git a/test/e2e/README.md b/test/e2e/README.md index 97210e223a..c5f5faf9cc 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -86,35 +86,43 @@ artifact so baseline aggregation stays stable. Older issue references to Vitest target artifacts under `e2e-artifacts/vitest/` map to this consolidated `e2e-artifacts/live/` registry-target artifact layout. -Every `e2e-live` test declares an ordered semantic phase plan in -`meta.e2ePhases` and uses the automatic progress fixture. Normal live output -shows only the phase number, semantic label, and the completed phase's outcome -and duration. It deliberately does not repeat the test identity, test start or -finish, or total test elapsed time already reported by Vitest and GitHub -Actions. A transition looks like: +Every `e2e-live` test and every credential-free integration test selected by +the shared E2E workflow planner declares an ordered semantic phase plan in +`meta.e2ePhases` and uses its automatic progress fixture. Normal E2E output +identifies the workflow target and test scenario, then shows immediate phase +start and completion lines with both phase and total elapsed time. A transition +looks like: ```text -[e2e phase 2/4] onboard the sandbox — passed in 2m 14s; next 3/4: verify hosted inference +[e2e target="cloud-onboard" scenario="onboards a hosted sandbox"] [phase 2/4] completed: onboard the sandbox — passed in 2m 14s (total 2m 21s) +[e2e target="cloud-onboard" scenario="onboards a hosted sandbox"] [phase 3/4] started: verify hosted inference (total 2m 21s; phase 0s) ``` -The harness appends `release registered E2E resources` after the test-declared -plan, so the displayed phase count includes that terminal phase. Registered -cleanup duration, failures, and stall diagnostics are attributed there. Soft -assertion failures remain attributed to the semantic phase in which they -occurred rather than being reassigned to resource release. +For `e2e-live`, the stateful fixture appends `release registered E2E resources` +after the test-declared plan, so the displayed phase count includes that +terminal phase. Registered cleanup duration, failures, and stall diagnostics +are attributed there. Workflow-selected integration tests instead declare and +enter their own final release phase. Soft assertion failures remain attributed +to the semantic phase in which they occurred rather than being reassigned to +resource release. If one phase remains active for five minutes, a content-free diagnostic adds -the phase duration, age of the last child output, current redacted command -activity, and runner resources. It repeats every ten minutes while that same -phase remains active. Child output contents are never forwarded by progress -logging. +the target/scenario identity, total and phase duration, age of the last child +output, current redacted command or cleanup activity, and runner resources. It +repeats every ten minutes while that same phase remains active. Automatic child +output observation forwards only a timestamp and stream name, never contents. +Operations with bounded retries may emit immediate content-free +`progress.event(...)` lines for a timeout, cleanup, backoff, or retry; event +labels are explicitly logged and must never contain child output, request data, +credentials, or tokens. During fixture teardown, the fixture writes `test-progress.json` into each test's existing artifact directory for passing and failing tests. The summary keeps the test identity and overall timestamps, plus each recorded phase's -timestamps, duration, outcome, output-event count, and last-output timestamp. -It also records `E2E_TARGET_ID` and `NEMOCLAW_E2E_SHARD` when those values are -set. Compare extracted artifacts from multiple runs with: +timestamps, duration, outcome, child-output event count, and last-output timestamp. +It records the target from `E2E_TARGET_ID`, falling back to the Actions +`GITHUB_JOB` identity, and records `NEMOCLAW_E2E_SHARD` when set. Compare +extracted artifacts from multiple runs with: ```bash npm run test:runtime-audit -- path/to/run-1 path/to/run-2 @@ -126,15 +134,28 @@ and outcome. Scheduled and ordinary manual runs include the same table for that run in the GitHub Actions scorecard summary. Keep phase labels specific to test behavior, call `progress.phase("literal phase label")` at the declared boundaries in order, and transition through the final -test-declared phase on every passing path. The fixture rejects a passing live -test that never reaches that phase; the harness enters the resource-release -phase automatically. -Validate phase coverage without executing live test bodies with: +test-declared phase on every passing path. Both fixtures reject a passing test +that never reaches that phase; only the stateful live fixture enters its +resource-release phase automatically. +Validate phase coverage without executing test bodies with: ```bash npm run test:e2e-phases:check ``` +The checker preserves coverage for every file under `test/e2e/live/` and adds +workflow-selected integration files from the authoritative shared-job planner. +Live modules import `fixtures/e2e-test.ts`; selected integration modules import +`fixtures/workflow-e2e-test.ts` and declare their final release phase explicitly. +It also follows shared E2E runtime helpers. Run child processes through +`ShellProbe` or an existing audited progress-aware boundary; new direct async +process boundaries fail the check. Synchronous calls require both a positive +timeout shorter than the first heartbeat and `killSignal: "SIGKILL"`. Keep child +contents in redacted artifacts and report only timestamp-based output activity +to the console. Pass the fixture-provided frozen, canonical `progress` +capability unchanged to an audited subprocess boundary; do not replace it with +a custom, copied, or no-op adapter. + ## PR E2E gate The controller, coordination check, and required job deliberately use diff --git a/test/e2e/docs/README.md b/test/e2e/docs/README.md index a0c49abe32..c2c463a044 100644 --- a/test/e2e/docs/README.md +++ b/test/e2e/docs/README.md @@ -50,12 +50,16 @@ Live execution happens through shared fixtures: - `artifacts`, `secrets`, `cleanup`, and `shellProbe` provide shared fixture services. - The automatic `progress` fixture reports the ordered semantic phase plan for - each `e2e-live` case. Normal output contains phase transitions, outcomes, and - durations. The harness appends `release registered E2E resources` to cover - registered cleanup. After five minutes in one phase, a content-free stall - diagnostic adds child-output age, current redacted command activity, and + each `e2e-live` case. Normal output contains the target/scenario identity, + immediate phase starts and completions, and phase plus total durations. The + harness appends `release registered E2E resources` to cover registered + cleanup. After five minutes in one phase, a content-free stall diagnostic + adds child-output age, current redacted command or cleanup activity, and runner resources; it repeats every ten minutes while the phase remains active. +- Credential-free integration tests selected by the shared E2E planner use the + lightweight `workflow-e2e-test` fixture for the same progress and artifact + contract without depending on the stateful live fixture services. The `test/e2e/fixtures/` path is fixture/support code, not a test harness or runner. Vitest remains the only test harness. @@ -78,7 +82,7 @@ npx tsx test/e2e/registry/run.ts --emit-live-matrix --targets ubuntu-repo-cloud- # Fixture/support tests npx vitest run --project e2e-support --silent=false --reporter=default -# Validate every live test's semantic phase metadata without running live bodies +# Validate every live test and workflow-selected integration test without running bodies npm run test:e2e-phases:check # Opt-in live E2E targets @@ -103,36 +107,45 @@ groups those files by target, optional shard, and test name, then reports median, p95, maximum, p95-minus-median variability, and the slowest observed phase with its duration and outcome. Scheduled and ordinary manual workflows publish the current run's table in the GitHub Actions scorecard summary. The -summary reads the matrix identity from `E2E_TARGET_ID` and -`NEMOCLAW_E2E_SHARD` when set. It retains overall start, finish, and duration, -and records each declared or harness-owned phase's start, finish, duration, -outcome, output-event count, and last-output timestamp. Use several recent -workflow artifact directories to distinguish a consistently expensive test -from a variable one. - -Normal phase output intentionally omits test identity and test-level timing -because Vitest and GitHub Actions already provide them. It reports the current -position and semantic label, then the outcome and duration when that phase -ends: +summary reads the target identity from `E2E_TARGET_ID`, falling back to the +Actions `GITHUB_JOB`, and reads `NEMOCLAW_E2E_SHARD` when set. It retains +overall start, finish, and duration, and records each declared or harness-owned +phase's start, finish, duration, outcome, child-output event count, and +last-output timestamp. Use several recent workflow artifact directories to +distinguish a consistently expensive test from a variable one. + +Normal phase output repeats the workflow target and test scenario because a +long-running Actions step may not expose Vitest's final report yet. It reports +the current position and semantic label, total and phase elapsed time, and the +outcome when that phase ends: ```text -[e2e phase 1/4] provision a clean sandbox -[e2e phase 1/4] provision a clean sandbox — passed in 48s; next 2/4: exercise token rotation -[e2e phase 2/4] still running: exercise token rotation (phase 5m; child output 12s ago; activity command: rotate-token; ...) -[e2e phase 3/4] verify the rotated credential — passed in 9s; next 4/4: release registered E2E resources -[e2e phase 4/4] release registered E2E resources — passed in 6s +[e2e target="token-rotation" scenario="rotates a live sandbox credential"] [phase 1/4] started: provision a clean sandbox (total 0s; phase 0s) +[e2e target="token-rotation" scenario="rotates a live sandbox credential"] [phase 1/4] completed: provision a clean sandbox — passed in 48s (total 48s) +[e2e target="token-rotation" scenario="rotates a live sandbox credential"] [phase 2/4] still running: exercise token rotation (total 5m 48s; phase 5m; child output 12s ago; activity command: credential-rotation; ...) +[e2e target="token-rotation" scenario="rotates a live sandbox credential"] [phase 4/4] event: cleanup started: destroy sandbox e2e-token-rotation (total 6m; phase 0s) +[e2e target="token-rotation" scenario="rotates a live sandbox credential"] [phase 4/4] completed: release registered E2E resources — passed in 6s (total 6m 6s) ``` The `still running` line first appears after five minutes in the same phase and then every ten minutes. Shell probes update child-output liveness and redacted command activity automatically, but that detail remains hidden until the stall -threshold. The progress fixture never forwards child output contents. -The harness-owned final phase captures registered cleanup duration, failures, -and stalls. Soft assertion failures are recorded against the semantic phase -where they occurred, while successful resource release retains its own -`passed` outcome. - -Every `e2e-live` test must declare two to twelve behavior-specific phases and +threshold. Automatic child-output observation forwards only the event timestamp +and stream name, never the output contents. +Use `progress.event("literal content-free status")` only for immediate semantic +events such as an operation timeout, retry cleanup, backoff, or the next +attempt. Event labels are logged, so never include child output, request data, +credentials, or tokens. +For the stateful live fixture, the harness-owned final phase captures registered +cleanup duration, failures, and stalls; each registry entry reports a redacted +start/outcome event and is shown as the active cleanup operation in a stall +heartbeat. Workflow-selected integration tests declare their own final release +phase. Soft assertion failures are recorded against the semantic phase where +they occurred, while successful resource release retains its own `passed` +outcome. + +Every `e2e-live` test and every credential-free integration test selected by +the shared E2E planner must declare two to twelve behavior-specific phases and transition through them in order. For example: ```typescript @@ -168,10 +181,31 @@ attribute it to that case. A helper may own the operational boundary by accepting a callback that performs the transition. Completed phases use `passed`, `failed`, or `skipped` outcomes. A passing path must enter the final declared phase before returning, or fixture teardown fails -the test. Do not declare or enter `release registered E2E resources`; the -harness appends and enters it automatically after the test's phase plan. -`npm run test:e2e-phases:check` collects the `e2e-live` project and rejects -missing or invalid plans without executing live test bodies. +the test. In `e2e-live`, do not declare or enter +`release registered E2E resources`; the stateful harness appends and enters it +automatically after the test's phase plan. Workflow-selected integration tests +own and enter their final release phase. +`npm run test:e2e-phases:check` collects every `e2e-live` module plus the +workflow-selected integration modules from the authoritative shared-job plan. +It rejects missing or invalid plans without executing test bodies. Live modules +must import `fixtures/e2e-test.ts`; selected integration modules must import +`fixtures/workflow-e2e-test.ts` and declare their final release phase explicitly. +The same check audits direct child-process boundaries reachable through shared +E2E helpers. Prefer `ShellProbe`; a long-lived process that cannot use it must +live in an explicitly audited progress-aware boundary, close its activity on +exit, and report child output only as `{ stream, atMs }`. Blocking child-process +calls require a positive timeout shorter than the first heartbeat plus +`killSignal: "SIGKILL"`, so that timeout cannot be ignored. Raw output belongs +only in redacted artifacts. + +Audited subprocess helpers require the fixture-provided frozen, canonical +`progress` capability. Forward that exact object unchanged instead of copying +it or constructing a look-alike or no-op adapter. A module-private brand, +runtime registry, frozen-object check, type system, and semantic checker enforce +this boundary. + +Progress callbacks are diagnostic-only: callback failures must not change +command execution, test outcomes, or registered resource release. The retired `--emit-matrix` and `--plan-only` paths must not be reintroduced. @@ -229,6 +263,12 @@ test/e2e/ live E2E targets and uploads an explicit artifact allowlist with JSON summaries plus action, log, and shell command-evidence directories under 14-day retention. + Final OpenShell gateway-auth artifacts pass a fail-closed safety scan after + cleanup. The scanner copies safe files into a private staging directory, + scans that copy again, and adds a marker bound to the current Actions run ID + and attempt. Unsafe source files are quarantined or deleted. The workflow + uploads only the staged copy, so later changes to the source directory cannot + alter the approved payload. The allowlist includes each target's sanitized onboard timing summary at `e2e-artifacts/live//cloud-onboard-trace-timing-summary.json`. Raw onboard traces stay under the runner temporary directory and are deleted diff --git a/test/e2e/fixtures/cleanup.ts b/test/e2e/fixtures/cleanup.ts index 2125bc2ed9..18c13cee0e 100644 --- a/test/e2e/fixtures/cleanup.ts +++ b/test/e2e/fixtures/cleanup.ts @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import type { TestProgress } from "./progress.ts"; import type { ShellProbeRunOptions } from "./shell-probe.ts"; export interface CleanupFailure { @@ -15,6 +16,8 @@ export interface CleanupResult { type CleanupFn = () => Promise | void; type RedactFn = (text: string) => string; +type CleanupProgress = Pick; +const MAX_PROGRESS_CLEANUP_NAME_LENGTH = 120; export interface CleanupHost { cleanupSandbox(name: string, options?: ShellProbeRunOptions): Promise; @@ -30,9 +33,55 @@ interface CleanupEntry { export class CleanupRegistry { private readonly entries: CleanupEntry[] = []; private readonly redact: RedactFn; + private readonly progress?: CleanupProgress; - constructor(redact: RedactFn = (text) => text) { + constructor(redact: RedactFn = (text) => text, progress?: CleanupProgress) { this.redact = redact; + this.progress = progress; + } + + private safeRedact(text: string): string { + try { + return this.redact(text); + } catch { + return "[cleanup metadata unavailable]"; + } + } + + private progressName(name: string): string { + const normalized = this.safeRedact(name) + .replace(/[\u0000-\u001f\u007f]+/gu, " ") + .replace(/\s+/gu, " ") + .trim(); + if (!normalized) return "unnamed cleanup entry"; + if (normalized.length <= MAX_PROGRESS_CLEANUP_NAME_LENGTH) return normalized; + return `${normalized.slice(0, MAX_PROGRESS_CLEANUP_NAME_LENGTH - 12).trimEnd()} [truncated]`; + } + + private startProgress(name: string): () => void { + const redactedName = this.progressName(name); + let finishActivity: (() => void) | undefined; + try { + finishActivity = this.progress?.activity(`cleanup: ${redactedName}`); + this.progress?.event(`cleanup started: ${redactedName}`); + } catch { + // Cleanup diagnostics must never prevent resource release. + } + return () => { + try { + finishActivity?.(); + } catch { + // Cleanup diagnostics must never prevent resource release. + } + }; + } + + private reportProgress(outcome: "failed" | "passed", name: string): void { + try { + this.progress?.event(`cleanup ${outcome}: ${this.progressName(name)}`); + } catch { + // Cleanup diagnostics must never change the cleanup result. + } } add(name: string, run: CleanupFn): void { @@ -73,14 +122,19 @@ export class CleanupRegistry { async runAll(): Promise { const result: CleanupResult = { passed: [], failures: [] }; for (const entry of [...this.entries].reverse()) { + const finishProgress = this.startProgress(entry.name); try { await entry.run(); - result.passed.push(this.redact(entry.name)); + result.passed.push(this.safeRedact(entry.name)); + this.reportProgress("passed", entry.name); } catch (error) { result.failures.push({ - name: this.redact(entry.name), - message: this.redact(error instanceof Error ? error.message : String(error)), + name: this.safeRedact(entry.name), + message: this.safeRedact(error instanceof Error ? error.message : String(error)), }); + this.reportProgress("failed", entry.name); + } finally { + finishProgress(); } } this.entries.length = 0; diff --git a/test/e2e/fixtures/docker-probe.ts b/test/e2e/fixtures/docker-probe.ts index 66df9b6036..d8ffd00329 100644 --- a/test/e2e/fixtures/docker-probe.ts +++ b/test/e2e/fixtures/docker-probe.ts @@ -1,18 +1,16 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { - type SpawnSyncOptionsWithStringEncoding, - type SpawnSyncReturns, - spawnSync, -} from "node:child_process"; +import { type SpawnSyncOptionsWithStringEncoding, type SpawnSyncReturns } from "node:child_process"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import type { ArtifactSink } from "./artifacts.ts"; +import { type ChildProcessProgress, spawnObservedChild } from "./observed-child-process.ts"; import { buildChildEnv } from "./redaction.ts"; import type { SecretStore } from "./secrets.ts"; +import { superviseChild } from "./shell/supervisor.ts"; export type DockerCommandResult = { command: string[]; @@ -43,6 +41,12 @@ const DOCKER_ENV_ALLOWLIST = [ "DOCKER_CERT_PATH", "XDG_RUNTIME_DIR", ] as const; +const MAX_DOCKER_OUTPUT_BYTES = 10 * 1024 * 1024; + +function appendBoundedOutput(output: Buffer, chunk: string): Buffer | null { + const combined = Buffer.concat([output, Buffer.from(chunk, "utf8")]); + return combined.length <= MAX_DOCKER_OUTPUT_BYTES ? combined : null; +} function safeName(value: string): string { return ( @@ -98,7 +102,9 @@ export class DockerProbe { constructor( private readonly artifacts: ArtifactSink, private readonly redact: SecretStore["redact"], - private readonly runDocker: DockerProbeRunner = spawnSync, + private readonly runDocker?: DockerProbeRunner, + private readonly progress?: ChildProcessProgress, + private readonly signal?: AbortSignal, ) { this.dockerConfigDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-docker-config-")); } @@ -109,21 +115,95 @@ export class DockerProbe { ): Promise { fs.mkdirSync(this.dockerConfigDir, { recursive: true }); const command = ["docker", ...args]; - const result = this.runDocker("docker", args, { + const timeoutMs = options.timeoutMs ?? 30_000; + const spawnOptions: SpawnSyncOptionsWithStringEncoding = { cwd: path.resolve(import.meta.dirname, "../../.."), encoding: "utf8", env: buildDockerProbeEnv(process.env, this.dockerConfigDir), + killSignal: "SIGKILL", maxBuffer: 10 * 1024 * 1024, - timeout: options.timeoutMs ?? 30_000, - }); - const rawCommandResult = { - command, - exitCode: result.status, - signal: result.signal, - stdout: result.stdout ?? "", - stderr: result.stderr ?? "", - error: result.error instanceof Error ? result.error.message : undefined, + timeout: timeoutMs, }; + let rawCommandResult: DockerCommandResult; + if (this.runDocker) { + const result = this.runDocker("docker", args, spawnOptions); + rawCommandResult = { + command, + exitCode: result.status, + signal: result.signal, + stdout: result.stdout ?? "", + stderr: result.stderr ?? "", + error: result.error instanceof Error ? result.error.message : undefined, + }; + } else { + if (!this.progress) { + throw new Error("DockerProbe requires progress hooks when using the real Docker CLI"); + } + let stdout: Buffer = Buffer.alloc(0); + let stderr: Buffer = Buffer.alloc(0); + let outputExceeded = false; + const child = spawnObservedChild("docker", args, { + activityLabel: `command: docker-${safeName(options.artifactName)}`, + progress: this.progress, + spawn: { + cwd: spawnOptions.cwd, + detached: true, + env: spawnOptions.env, + stdio: ["ignore", "pipe", "pipe"], + }, + }); + const stopForOutputLimit = (): void => { + try { + if (child.pid !== undefined) process.kill(-child.pid, "SIGKILL"); + else child.kill("SIGKILL"); + } catch { + child.kill("SIGKILL"); + } + }; + const supervised = await superviseChild(child, { + timeoutMs, + killGraceMs: 1_000, + signal: this.signal, + onStdout: (chunk) => { + if (outputExceeded) return; + const next = appendBoundedOutput(stdout, chunk); + if (next) { + stdout = next; + return; + } + outputExceeded = true; + stdout = Buffer.alloc(0); + stderr = Buffer.alloc(0); + stopForOutputLimit(); + }, + onStderr: (chunk) => { + if (outputExceeded) return; + const next = appendBoundedOutput(stderr, chunk); + if (next) { + stderr = next; + return; + } + outputExceeded = true; + stdout = Buffer.alloc(0); + stderr = Buffer.alloc(0); + stopForOutputLimit(); + }, + }); + rawCommandResult = { + command, + exitCode: supervised.exitCode, + signal: supervised.signal, + stdout: outputExceeded + ? "[docker-probe output exceeded safe capture limit]" + : stdout.toString("utf8"), + stderr: outputExceeded + ? "[docker-probe output exceeded safe capture limit]" + : stderr.toString("utf8"), + error: + supervised.spawnError?.message ?? + (outputExceeded ? "Docker output exceeded the safe capture limit" : undefined), + }; + } const commandResult = redactDockerProbeResult(rawCommandResult, (text) => this.redact(text, options.artifactRedactionValues ?? []), ); diff --git a/test/e2e/fixtures/e2e-test.ts b/test/e2e/fixtures/e2e-test.ts index 8705d59d7a..1edc8caed8 100644 --- a/test/e2e/fixtures/e2e-test.ts +++ b/test/e2e/fixtures/e2e-test.ts @@ -111,7 +111,8 @@ export const test = base.extend({ }, progress: [ async ({ artifacts, onTestFinished, secrets, task }, use) => { - const targetId = process.env.E2E_TARGET_ID; + const targetId = process.env.E2E_TARGET_ID || process.env.GITHUB_JOB; + const shardId = process.env.NEMOCLAW_E2E_SHARD; const baselinePath = process.env.E2E_RESOURCE_PHASE_BASELINES_FILE; const phasePlan = task.meta.e2ePhases; assert.ok( @@ -120,8 +121,8 @@ export const test = base.extend({ ); const declaredPhasePlan = phasePlan ?? SUPPORT_PHASES; const declaredFinalPhase = declaredPhasePlan.at(-1) as string; - let reachedDeclaredFinal = !phasePlan; - const baseProgress = startTestProgress(task.name, declaredPhasePlan, { + const progress = startTestProgress(task.name, declaredPhasePlan, { + targetId, terminalPhase: E2E_TEARDOWN_PHASE, taskStatus: () => ({ errorCount: task.result?.errors?.length ?? 0, @@ -144,13 +145,6 @@ export const test = base.extend({ } : {}), }); - const progress: TestProgress = { - ...baseProgress, - phase(label) { - baseProgress.phase(label); - if (label === declaredFinalPhase) reachedDeclaredFinal = true; - }, - }; const completeSupportPlan = phasePlan ? () => undefined : () => { @@ -162,14 +156,14 @@ export const test = base.extend({ finalized = true; const outcome = outcomeForTaskState(task.result?.state); completeSupportPlan(); - const completedPhasePlan = reachedDeclaredFinal; + const completedPhasePlan = !phasePlan || progress.hasReached(declaredFinalPhase); if (!progress.isComplete()) progress.phase(E2E_TEARDOWN_PHASE); const phaseOutcome = outcome === "passed" && !completedPhasePlan ? "failed" : outcome; progress.stop(phaseOutcome); await artifacts.writeJson("test-progress.json", { ...progress.summary(), - ...(process.env.E2E_TARGET_ID ? { targetId: process.env.E2E_TARGET_ID } : {}), - ...(process.env.NEMOCLAW_E2E_SHARD ? { shardId: process.env.NEMOCLAW_E2E_SHARD } : {}), + ...(targetId ? { targetId } : {}), + ...(shardId ? { shardId } : {}), }); assert.ok( outcome !== "passed" || completedPhasePlan, @@ -180,12 +174,18 @@ export const test = base.extend({ }, { auto: true }, ], - docker: async ({ artifacts, secrets, skip }, use) => { - const probe = new DockerProbe(artifacts, (text, extra) => secrets.redact(text, extra)); + docker: async ({ artifacts, progress, secrets, signal, skip }, use) => { + const probe = new DockerProbe( + artifacts, + (text, extra) => secrets.redact(text, extra), + undefined, + progress, + signal, + ); await use(new DockerPrerequisite(probe, skip)); }, cleanup: async ({ artifacts, progress, secrets }, use) => { - const cleanup = new CleanupRegistry((text) => secrets.redact(text)); + const cleanup = new CleanupRegistry((text) => secrets.redact(text), progress); try { await use(cleanup); } finally { @@ -199,10 +199,9 @@ export const test = base.extend({ await use( new ShellProbe({ artifacts, + progress, redact: (text, extraValues) => secrets.redact(text, extraValues), signal, - onOutput: progress.onOutput, - onActivity: progress.activity, }), ); }, @@ -221,8 +220,8 @@ export const test = base.extend({ provider: async ({ shellProbe }, use) => { await use(new ProviderClient(shellProbe)); }, - inference: async ({ artifacts, provider, secrets }, use) => { - const inference = await createE2EInferenceAdapter({ artifacts, provider, secrets }); + inference: async ({ artifacts, progress, provider, secrets }, use) => { + const inference = await createE2EInferenceAdapter({ artifacts, progress, provider, secrets }); try { await use(inference); } finally { diff --git a/test/e2e/fixtures/fake-openai-compatible.ts b/test/e2e/fixtures/fake-openai-compatible.ts index 74af1d4b49..5e0ec5b5e5 100644 --- a/test/e2e/fixtures/fake-openai-compatible.ts +++ b/test/e2e/fixtures/fake-openai-compatible.ts @@ -1,12 +1,15 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { type ChildProcess, spawn } from "node:child_process"; +import type { ChildProcess } from "node:child_process"; import fs from "node:fs"; import http from "node:http"; import os from "node:os"; import path from "node:path"; +import { spawnObservedChild } from "./observed-child-process.ts"; +import type { TestProgress, TestProgressCapability } from "./progress.ts"; + const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); const SERVER_SCRIPT = path.join(REPO_ROOT, "test/e2e/lib/fake-openai-compatible-api.mts"); @@ -39,6 +42,7 @@ export interface FakeOpenAiCompatibleServerOptions { readonly maxModelLen?: number; readonly model?: string; readonly port?: number; + readonly progress: Pick & TestProgressCapability; readonly publicHost?: string; readonly requireAuth?: boolean; readonly requireAuthModels?: boolean; @@ -54,13 +58,34 @@ function readPort(portFile: string): number | null { } } -function waitForExit(child: ChildProcess): Promise { - if (child.exitCode !== null || child.signalCode !== null) return Promise.resolve(); +function waitForExit(child: ChildProcess, timeoutMs: number): Promise { + if (child.exitCode !== null || child.signalCode !== null) return Promise.resolve(true); return new Promise((resolve) => { - child.once("exit", () => resolve()); + let settled = false; + const finish = (exited: boolean): void => { + if (settled) return; + settled = true; + clearTimeout(timer); + child.off("exit", onExit); + resolve(exited); + }; + const onExit = (): void => finish(true); + const timer = setTimeout(() => finish(false), timeoutMs); + timer.unref(); + child.once("exit", onExit); + if (child.exitCode !== null || child.signalCode !== null) finish(true); }); } +async function terminateChild(child: ChildProcess): Promise { + if (child.exitCode !== null || child.signalCode !== null) return; + child.kill("SIGTERM"); + if (await waitForExit(child, 5_000)) return; + child.kill("SIGKILL"); + if (await waitForExit(child, 5_000)) return; + throw new Error("fake OpenAI-compatible endpoint did not stop after SIGKILL"); +} + function readinessProbeHost(host: string): string { if (host === "0.0.0.0") return "127.0.0.1"; if (host === "::") return "::1"; @@ -132,7 +157,7 @@ function parseEnvironmentKeys(environmentFile: string): string[] { } export async function startFakeOpenAiCompatibleServer( - options: FakeOpenAiCompatibleServerOptions = {}, + options: FakeOpenAiCompatibleServerOptions, ): Promise { const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-fake-openai-")); const environmentFile = path.join(tmpDir, "environment-keys.json"); @@ -140,36 +165,73 @@ export async function startFakeOpenAiCompatibleServer( const logFile = path.join(tmpDir, "server.log"); const requestsFile = path.join(tmpDir, "requests.jsonl"); const host = options.host ?? "127.0.0.1"; - const child = spawn(process.execPath, ["--experimental-strip-types", SERVER_SCRIPT], { - env: { - PATH: process.env.PATH ?? "", - HOME: process.env.HOME ?? "", - NEMOCLAW_FAKE_OPENAI_API_KEY: options.apiKey ?? "", - NEMOCLAW_FAKE_OPENAI_CHAT_CONTENT: options.chatContent ?? "ok", - NEMOCLAW_FAKE_OPENAI_ENVIRONMENT_FILE: environmentFile, - NEMOCLAW_FAKE_OPENAI_FORBIDDEN_MARKERS: JSON.stringify(options.forbiddenMarkers ?? []), - NEMOCLAW_FAKE_OPENAI_HOST: host, - NEMOCLAW_FAKE_OPENAI_LOG_FILE: logFile, - NEMOCLAW_FAKE_OPENAI_MAX_MODEL_LEN: - options.maxModelLen !== undefined ? String(options.maxModelLen) : "", - NEMOCLAW_FAKE_OPENAI_MODEL: options.model ?? "test-model", - NEMOCLAW_FAKE_OPENAI_PORT: String(options.port ?? 0), - NEMOCLAW_FAKE_OPENAI_PORT_FILE: portFile, - NEMOCLAW_FAKE_OPENAI_REQUESTS_FILE: requestsFile, - NEMOCLAW_FAKE_OPENAI_REQUIRE_AUTH: options.requireAuth ? "1" : "0", - NEMOCLAW_FAKE_OPENAI_REQUIRE_AUTH_MODELS: options.requireAuthModels ? "1" : "0", - NEMOCLAW_FAKE_OPENAI_RESPONSE_TEXT: options.responseText ?? options.chatContent ?? "ok", - }, - stdio: "ignore", - }); + let child: ChildProcess; + try { + child = spawnObservedChild(process.execPath, ["--experimental-strip-types", SERVER_SCRIPT], { + activityLabel: "command: fake-openai-compatible-server", + progress: options.progress, + spawn: { + env: { + PATH: process.env.PATH ?? "", + HOME: process.env.HOME ?? "", + NEMOCLAW_FAKE_OPENAI_API_KEY: options.apiKey ?? "", + NEMOCLAW_FAKE_OPENAI_CHAT_CONTENT: options.chatContent ?? "ok", + NEMOCLAW_FAKE_OPENAI_ENVIRONMENT_FILE: environmentFile, + NEMOCLAW_FAKE_OPENAI_FORBIDDEN_MARKERS: JSON.stringify(options.forbiddenMarkers ?? []), + NEMOCLAW_FAKE_OPENAI_HOST: host, + NEMOCLAW_FAKE_OPENAI_LOG_FILE: logFile, + NEMOCLAW_FAKE_OPENAI_MAX_MODEL_LEN: + options.maxModelLen !== undefined ? String(options.maxModelLen) : "", + NEMOCLAW_FAKE_OPENAI_MODEL: options.model ?? "test-model", + NEMOCLAW_FAKE_OPENAI_PORT: String(options.port ?? 0), + NEMOCLAW_FAKE_OPENAI_PORT_FILE: portFile, + NEMOCLAW_FAKE_OPENAI_REQUESTS_FILE: requestsFile, + NEMOCLAW_FAKE_OPENAI_REQUIRE_AUTH: options.requireAuth ? "1" : "0", + NEMOCLAW_FAKE_OPENAI_REQUIRE_AUTH_MODELS: options.requireAuthModels ? "1" : "0", + NEMOCLAW_FAKE_OPENAI_RESPONSE_TEXT: options.responseText ?? options.chatContent ?? "ok", + }, + stdio: "ignore", + }, + }); + } catch (error) { + fs.rmSync(tmpDir, { force: true, recursive: true }); + throw error; + } + try { + options.progress?.event("fake OpenAI-compatible server started"); + } catch { + // Progress diagnostics must never change fake endpoint behavior. + } + + let closePromise: Promise | undefined; + const close = (): Promise => { + closePromise ??= (async () => { + try { + await terminateChild(child); + } finally { + try { + options.progress?.event("fake OpenAI-compatible server stopped"); + } catch { + // Progress diagnostics must never change fake endpoint cleanup. + } + fs.rmSync(tmpDir, { force: true, recursive: true }); + } + })(); + return closePromise; + }; let port: number; try { port = await waitForReady(portFile, child, host); } catch (error) { - child.kill("SIGTERM"); - await waitForExit(child); - fs.rmSync(tmpDir, { force: true, recursive: true }); + try { + await close(); + } catch (closeError) { + throw new AggregateError( + [error, closeError], + "fake OpenAI-compatible server failed to become ready and failed to terminate cleanly", + ); + } throw error; } const publicHost = options.publicHost ?? readinessProbeHost(host); @@ -179,10 +241,6 @@ export async function startFakeOpenAiCompatibleServer( requestsFile, environmentKeys: () => parseEnvironmentKeys(environmentFile), requests: () => parseRequests(requestsFile), - close: async () => { - child.kill("SIGTERM"); - await waitForExit(child); - fs.rmSync(tmpDir, { force: true, recursive: true }); - }, + close, }; } diff --git a/test/e2e/fixtures/inference-adapter.ts b/test/e2e/fixtures/inference-adapter.ts index 79e94c2601..5217f61f18 100644 --- a/test/e2e/fixtures/inference-adapter.ts +++ b/test/e2e/fixtures/inference-adapter.ts @@ -19,6 +19,7 @@ import { type HostedInferenceSecrets, requireHostedInferenceConfig, } from "./hosted-inference.ts"; +import type { TestProgress, TestProgressCapability } from "./progress.ts"; /** * Gives E2E suites one inference contract across three execution modes. @@ -63,6 +64,7 @@ export interface E2EInferenceAdapterOptions { readonly artifacts: ArtifactSink; readonly env?: NodeJS.ProcessEnv; readonly provider: Pick; + readonly progress: Pick & TestProgressCapability; readonly secrets: HostedInferenceSecrets; } @@ -374,6 +376,7 @@ export async function createE2EInferenceAdapter( host: "0.0.0.0", model, publicHost: SANDBOX_HOST_ALIAS, + progress: options.progress, requireAuth: true, responseText: "PONG", }); diff --git a/test/e2e/fixtures/observed-child-process.ts b/test/e2e/fixtures/observed-child-process.ts new file mode 100644 index 0000000000..ee31bc5d56 --- /dev/null +++ b/test/e2e/fixtures/observed-child-process.ts @@ -0,0 +1,72 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { type ChildProcess, type SpawnOptions, spawn } from "node:child_process"; +import { isTestProgressCapability, type TestProgressCapability } from "./progress.ts"; + +export interface ChildProcessProgress extends TestProgressCapability { + activity(label: string): (() => void) | void; + onOutput(event: { stream: "stdout" | "stderr"; atMs: number }): void; +} + +export interface ObservedChildProcessOptions { + activityLabel: string; + progress: ChildProcessProgress; + spawn: SpawnOptions; +} + +/** + * The single direct asynchronous child-process boundary used by live E2E code. + * It owns content-free activity and timestamp-only output observation so a + * caller cannot accidentally start a silent subprocess. + */ +export function spawnObservedChild( + command: string, + args: readonly string[], + options: ObservedChildProcessOptions, +): ChildProcess { + if (!isTestProgressCapability(options.progress)) { + throw new TypeError("observed child processes require the canonical E2E progress capability"); + } + let finishActivity: () => void = () => undefined; + try { + finishActivity = options.progress.activity(options.activityLabel) ?? finishActivity; + } catch { + // Progress diagnostics must never change process execution. + } + + let child: ChildProcess; + try { + child = spawn(command, [...args], options.spawn); + } catch (error) { + try { + finishActivity(); + } catch { + // Progress diagnostics must never replace the spawn failure. + } + throw error; + } + + child.stdout?.on("data", () => { + try { + options.progress.onOutput({ stream: "stdout", atMs: Date.now() }); + } catch { + // Child contents never cross this timestamp-only observer boundary. + } + }); + child.stderr?.on("data", () => { + try { + options.progress.onOutput({ stream: "stderr", atMs: Date.now() }); + } catch { + // Child contents never cross this timestamp-only observer boundary. + } + }); + child.once("close", () => { + try { + finishActivity(); + } catch { + // Progress diagnostics must never change process completion. + } + }); + return child; +} diff --git a/test/e2e/fixtures/progress.ts b/test/e2e/fixtures/progress.ts index a61af119fb..81be19b4ad 100644 --- a/test/e2e/fixtures/progress.ts +++ b/test/e2e/fixtures/progress.ts @@ -41,6 +41,7 @@ export interface ProgressSummary { } export interface TestProgressOptions { + targetId?: string; stallThresholdMs?: number; stallReminderIntervalMs?: number; now?: () => number; @@ -61,20 +62,52 @@ export interface TestProgressTimeline { export type ProgressPhaseOutcome = "passed" | "failed" | "skipped"; -export interface TestProgress { +const TEST_PROGRESS_CAPABILITY: unique symbol = Symbol("nemoclaw.test-progress"); +const TEST_PROGRESS_INSTANCES = new WeakSet(); + +/** + * Unforgeable-by-structure capability proving that subprocess diagnostics are + * backed by the shared E2E progress recorder. + */ +export interface TestProgressCapability { + readonly [TEST_PROGRESS_CAPABILITY]: true; +} + +export interface TestProgress extends TestProgressCapability { onOutput: (event: ShellProbeOutputEvent) => void; activity: (label: string) => () => void; + /** Emit a content-free semantic status event. Never pass child output or request data. */ + event: (label: string) => void; phase: (label: string) => void; + hasReached: (label: string) => boolean; isComplete: () => boolean; stop: (outcome?: ProgressPhaseOutcome) => void; summary: () => ProgressSummary; timeline: () => TestProgressTimeline; } +export function isTestProgressCapability(value: unknown): value is TestProgressCapability { + if (typeof value !== "object" || value === null || !TEST_PROGRESS_INSTANCES.has(value)) { + return false; + } + const descriptor = Object.getOwnPropertyDescriptor(value, TEST_PROGRESS_CAPABILITY); + return ( + Object.isFrozen(value) && + descriptor?.value === true && + descriptor.enumerable === false && + descriptor.configurable === false && + descriptor.writable === false + ); +} + const DEFAULT_STALL_THRESHOLD_MS = 5 * 60_000; const DEFAULT_STALL_REMINDER_INTERVAL_MS = 10 * 60_000; const GENERIC_PHASE_LABEL = /^(?:cleanup|execute|phase(?: \d+)?|run test|setup|teardown|test body|verify)$/iu; +const MAX_LOG_IDENTITY_LENGTH = 160; +const MAX_PHASE_LABEL_LENGTH = 160; +const MAX_ACTIVITY_LABEL_LENGTH = 160; +const MAX_EVENT_LABEL_LENGTH = 160; function formatGiB(bytes: number): string { return `${(bytes / 1024 ** 3).toFixed(1)} GiB`; @@ -88,6 +121,33 @@ function formatElapsed(elapsedMs: number): string { return seconds === 0 ? `${minutes}m` : `${minutes}m ${seconds}s`; } +function logIdentity(value: string, fallback: string): string { + const normalized = value + .replace(/[\u0000-\u001f\u007f]+/gu, " ") + .replace(/\s+/gu, " ") + .trim() + .slice(0, MAX_LOG_IDENTITY_LENGTH); + return JSON.stringify(normalized || fallback); +} + +function validateProgressEventLabel(label: string): void { + if (label !== label.trim() || label.length === 0 || label.length > MAX_EVENT_LABEL_LENGTH) { + throw new Error("invalid live E2E progress event label"); + } + if (/[\u0000-\u001f\u007f]/u.test(label)) { + throw new Error("invalid live E2E progress event label"); + } +} + +function validateProgressActivityLabel(label: string): void { + if (label !== label.trim() || label.length === 0 || label.length > MAX_ACTIVITY_LABEL_LENGTH) { + throw new Error("invalid live E2E progress activity label"); + } + if (/[\u0000-\u001f\u007f]/u.test(label)) { + throw new Error("invalid live E2E progress activity label"); + } +} + function defaultResourceSnapshot(): ResourceSnapshot { const workspace = fs.statfsSync(REPO_ROOT); return { @@ -123,8 +183,13 @@ export function validateE2EPhasePlan(phasePlan: readonly string[]): void { const seen = new Set(); for (const label of phasePlan) { - if (label !== label.trim() || label.length === 0) { - throw new Error(`invalid live E2E phase label: ${JSON.stringify(label)}`); + if ( + label !== label.trim() || + label.length === 0 || + label.length > MAX_PHASE_LABEL_LENGTH || + /[\u0000-\u001f\u007f]/u.test(label) + ) { + throw new Error("invalid live E2E phase label"); } if (GENERIC_PHASE_LABEL.test(label) || label.toLowerCase().startsWith("command:")) { throw new Error(`live E2E phase label must describe test behavior: ${JSON.stringify(label)}`); @@ -137,9 +202,9 @@ export function validateE2EPhasePlan(phasePlan: readonly string[]): void { } /** - * Reports only semantic E2E phase transitions during normal execution. Child - * output and command activity are observed without forwarding their contents; - * they become visible only as content-free evidence after a phase is stalled. + * Reports semantic E2E phase transitions plus explicitly requested, + * content-free status events. Child output is observed only as timestamps; + * current command or cleanup activity becomes visible after a phase stalls. */ export function startTestProgress( scenario: string, @@ -168,7 +233,11 @@ export function startTestProgress( const stallReminderIntervalMs = options.stallReminderIntervalMs ?? DEFAULT_STALL_REMINDER_INTERVAL_MS; const scenarioStartedAt = now(); + const identityPrefix = + `[e2e target=${logIdentity(options.targetId ?? "", "unassigned")} ` + + `scenario=${logIdentity(scenario, "unnamed")}]`; const phases: ProgressPhase[] = []; + const reachedPhases = new Set([runtimePhasePlan[0] as string]); const activities = new Map(); let nextActivityId = 0; let phaseIndex = 0; @@ -197,7 +266,8 @@ export function startTestProgress( let phaseStartErrorCount = readTaskStatus().errorCount; const currentPhase = () => runtimePhasePlan[phaseIndex] as string; - const phasePrefix = () => `[e2e phase ${phaseIndex + 1}/${runtimePhasePlan.length}]`; + const phasePrefix = (index = phaseIndex) => + `${identityPrefix} [phase ${index + 1}/${runtimePhasePlan.length}]`; const recordBaselineBestEffort = () => { try { @@ -207,9 +277,12 @@ export function startTestProgress( } }; - const logTransitionBestEffort = () => { + const logTransitionBestEffort = (atMs: number) => { try { - logLine(`${phasePrefix()} ${currentPhase()}`); + logLine( + `${phasePrefix()} started: ${currentPhase()} (` + + `total ${formatElapsed(atMs - scenarioStartedAt)}; phase 0s)`, + ); } catch { // Diagnostics must not change the live test result. } @@ -220,15 +293,13 @@ export function startTestProgress( completedLabel: string, outcome: ProgressPhaseOutcome, durationMs: number, - next?: { index: number; label: string }, + finishedAtMs: number, ) => { try { - const nextText = next - ? `; next ${next.index + 1}/${runtimePhasePlan.length}: ${next.label}` - : ""; logLine( - `[e2e phase ${completedIndex + 1}/${runtimePhasePlan.length}] ${completedLabel} — ` + - `${outcome} in ${formatElapsed(durationMs)}${nextText}`, + `${phasePrefix(completedIndex)} completed: ${completedLabel} — ` + + `${outcome} in ${formatElapsed(durationMs)} ` + + `(total ${formatElapsed(finishedAtMs - scenarioStartedAt)})`, ); } catch { // Diagnostics must not change the live test result. @@ -254,6 +325,7 @@ export function startTestProgress( logLine( `${phasePrefix()} still running: ${currentPhase()} (` + [ + `total ${formatElapsed(current - scenarioStartedAt)}`, `phase ${formatElapsed(current - phaseStartedAt)}`, outputAge, activityEvidence(), @@ -357,6 +429,7 @@ export function startTestProgress( }); } phaseIndex = nextPhaseIndex; + reachedPhases.add(label); phaseStartedAt = current; lastOutputAt = null; outputEvents = 0; @@ -365,20 +438,19 @@ export function startTestProgress( completed.label, completed.outcome, completed.durationMs, - { - index: phaseIndex, - label: currentPhase(), - }, + current, ); + logTransitionBestEffort(current); recordBaselineBestEffort(); resetStallTimer(); }; recordBaselineBestEffort(); - logTransitionBestEffort(); + logTransitionBestEffort(scenarioStartedAt); scheduleStall(stallThresholdMs); const progress: TestProgress = { + [TEST_PROGRESS_CAPABILITY]: true, onOutput(event) { if (finishedAt !== null) return; lastOutputAt = event.atMs; @@ -386,6 +458,7 @@ export function startTestProgress( }, activity(label) { if (finishedAt !== null) return () => undefined; + validateProgressActivityLabel(label); const activityId = nextActivityId; nextActivityId += 1; activities.set(activityId, label); @@ -396,7 +469,24 @@ export function startTestProgress( activities.delete(activityId); }; }, + event(label) { + if (finishedAt !== null) return; + validateProgressEventLabel(label); + const current = now(); + try { + logLine( + `${phasePrefix()} event: ${label} (` + + `total ${formatElapsed(current - scenarioStartedAt)}; ` + + `phase ${formatElapsed(current - phaseStartedAt)})`, + ); + } catch { + // Diagnostics must not change the live test result. + } + }, phase: selectPhase, + hasReached(label) { + return reachedPhases.has(label); + }, isComplete() { return phaseIndex === runtimePhasePlan.length - 1; }, @@ -405,13 +495,20 @@ export function startTestProgress( finishedAt = now(); clearStallTimer(); const completed = finishPhase(finishedAt, outcomeAtBoundary(outcome)); - logCompletionBestEffort(phaseIndex, completed.label, completed.outcome, completed.durationMs); + logCompletionBestEffort( + phaseIndex, + completed.label, + completed.outcome, + completed.durationMs, + finishedAt, + ); activities.clear(); }, summary() { return { version: 1, scenario, + ...(options.targetId ? { targetId: options.targetId } : {}), startedAtMs: scenarioStartedAt, finishedAtMs: finishedAt, durationMs: finishedAt === null ? null : Math.max(0, finishedAt - scenarioStartedAt), @@ -442,5 +539,12 @@ export function startTestProgress( }, }; - return progress; + Object.defineProperty(progress, TEST_PROGRESS_CAPABILITY, { + configurable: false, + enumerable: false, + value: true, + writable: false, + }); + TEST_PROGRESS_INSTANCES.add(progress); + return Object.freeze(progress); } diff --git a/test/e2e/fixtures/shell-probe.ts b/test/e2e/fixtures/shell-probe.ts index 8e68128a6d..e694d10149 100644 --- a/test/e2e/fixtures/shell-probe.ts +++ b/test/e2e/fixtures/shell-probe.ts @@ -1,9 +1,8 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { spawn } from "node:child_process"; - import type { ArtifactSink } from "./artifacts.ts"; +import { type ChildProcessProgress, spawnObservedChild } from "./observed-child-process.ts"; import { superviseChild } from "./shell/supervisor.ts"; import type { TrustedShellCommand } from "./shell/trusted-command.ts"; @@ -58,12 +57,9 @@ export interface ShellProbeResult { export interface ShellProbeDeps { artifacts: ArtifactSink; + progress: ChildProcessProgress; redact: (text: string, extraValues?: string[]) => string; signal: AbortSignal; - /** Default timestamp-only observer applied to every command in this test. */ - onOutput?: (event: ShellProbeOutputEvent) => void; - /** Content-free activity boundary applied to every command in this test. */ - onActivity?: (label: string) => (() => void) | void; } const DEFAULT_TIMEOUT_MS = 60_000; @@ -155,17 +151,15 @@ function redactTruncatedSecretPrefix(text: string, redactionValues: string[]): s export class ShellProbe { private readonly artifacts: ArtifactSink; + private readonly progress: ChildProcessProgress; private readonly redact: (text: string, extraValues?: string[]) => string; private readonly signal: AbortSignal; - private readonly onOutput?: (event: ShellProbeOutputEvent) => void; - private readonly onActivity?: (label: string) => (() => void) | void; constructor(deps: ShellProbeDeps) { this.artifacts = deps.artifacts; + this.progress = deps.progress; this.redact = deps.redact; this.signal = deps.signal; - this.onOutput = deps.onOutput; - this.onActivity = deps.onActivity; } async run( @@ -212,84 +206,68 @@ export class ShellProbe { const stdout = createTextCapture(options.captureLimitBytes); const stderr = createTextCapture(options.captureLimitBytes); const startedAtMs = Date.now(); - let finishActivity: (() => void) | undefined; - try { - finishActivity = this.onActivity?.(`command: ${activityName}`) ?? undefined; - } catch { - // Test instrumentation must not change command execution. - } - const finishActivityBestEffort = () => { - try { - finishActivity?.(); - } catch { - // Test instrumentation must not change command execution. - } - }; - const observeOutput = (event: ShellProbeOutputEvent) => { - try { - this.onOutput?.(event); - } catch { - // Test instrumentation must not change command execution. - } - if (options.onOutput === this.onOutput) return; - try { - options.onOutput?.(event); - } catch { - // Test instrumentation must not change command execution. - } - }; - try { - const child = spawn(command, args, { + const commandOutputObserver = + options.onOutput === this.progress.onOutput ? undefined : options.onOutput; + const child = spawnObservedChild(command, args, { + activityLabel: `command: ${activityName}`, + progress: this.progress, + spawn: { cwd: options.cwd, detached: true, env: { ...(options.env ?? {}) }, stdio: ["ignore", "pipe", "pipe"], - }); - const supervised = await superviseChild(child, { - timeoutMs, - killGraceMs, - signal: this.signal, - onStdout: (chunk) => { - stdout.append(chunk); - observeOutput({ stream: "stdout", atMs: Date.now() }); - }, - onStderr: (chunk) => { - stderr.append(chunk); - observeOutput({ stream: "stderr", atMs: Date.now() }); - }, - }); - - const redactedStdout = renderCapturedText(stdout); - const redactedStderr = renderCapturedText(stderr); - const durationMs = Date.now() - startedAtMs; - if (supervised.spawnError) { - const redactedMessage = redactProbeText(errorMessage(supervised.spawnError)); - const stderrWithError = [redactedStderr, redactedMessage].filter(Boolean).join("\n"); - await writeArtifacts({ - command: redactedCommand, - durationMs, - exitCode: null, - signal: null, - timedOut: supervised.timedOut, - stdout: redactedStdout, - stderr: stderrWithError, - }); - throw redactedError(supervised.spawnError, redactedMessage); - } + }, + }); + const supervised = await superviseChild(child, { + timeoutMs, + killGraceMs, + signal: this.signal, + onStdout: (chunk) => { + stdout.append(chunk); + try { + commandOutputObserver?.({ stream: "stdout", atMs: Date.now() }); + } catch { + // Test instrumentation must not change command execution. + } + }, + onStderr: (chunk) => { + stderr.append(chunk); + try { + commandOutputObserver?.({ stream: "stderr", atMs: Date.now() }); + } catch { + // Test instrumentation must not change command execution. + } + }, + }); - const result: Omit = { + const redactedStdout = renderCapturedText(stdout); + const redactedStderr = renderCapturedText(stderr); + const durationMs = Date.now() - startedAtMs; + if (supervised.spawnError) { + const redactedMessage = redactProbeText(errorMessage(supervised.spawnError)); + const stderrWithError = [redactedStderr, redactedMessage].filter(Boolean).join("\n"); + await writeArtifacts({ command: redactedCommand, durationMs, - exitCode: supervised.exitCode, - signal: supervised.signal, + exitCode: null, + signal: null, timedOut: supervised.timedOut, stdout: redactedStdout, - stderr: redactedStderr, - }; - const artifacts = await writeArtifacts(result); - return { ...result, artifacts }; - } finally { - finishActivityBestEffort(); + stderr: stderrWithError, + }); + throw redactedError(supervised.spawnError, redactedMessage); } + + const result: Omit = { + command: redactedCommand, + durationMs, + exitCode: supervised.exitCode, + signal: supervised.signal, + timedOut: supervised.timedOut, + stdout: redactedStdout, + stderr: redactedStderr, + }; + const artifacts = await writeArtifacts(result); + return { ...result, artifacts }; } } diff --git a/test/e2e/fixtures/shell/supervisor.ts b/test/e2e/fixtures/shell/supervisor.ts index 492c6e5b62..e7d21da9d8 100644 --- a/test/e2e/fixtures/shell/supervisor.ts +++ b/test/e2e/fixtures/shell/supervisor.ts @@ -9,15 +9,12 @@ import type { ChildProcess } from "node:child_process"; * * Spec ownership: detached process-group cleanup, SIGTERM -> SIGKILL * escalation, timeout enforcement, and AbortSignal handling are - * FIXTURE INFRASTRUCTURE. Every TS spawn site delegates here so the - * cleanup contract stays in one place. Callers keep their own spawn() - * call so per-site argv contracts (literal `bash -c` scripts with - * positional argv, host-CLI argv arrays, trusted-command descriptors) - * and any CodeQL suppression markers stay attached to the literal - * spawn line that actually performs the syscall. + * FIXTURE INFRASTRUCTURE. The shared observed-child boundary owns spawn, + * activity, and timestamp-only output reporting; callers that need bounded + * command completion hand its child to this supervisor immediately. * * Contract: - * - The caller spawns the child with `detached: true` so the + * - The observed-child boundary spawns with `detached: true` so the * supervisor can target the whole process group when delivering * signals. Without it, bash ignores SIGTERM until its current * foreground command (e.g. `sleep`) returns, so timeouts never diff --git a/test/e2e/fixtures/workflow-e2e-test.ts b/test/e2e/fixtures/workflow-e2e-test.ts new file mode 100644 index 0000000000..d10ea26c9e --- /dev/null +++ b/test/e2e/fixtures/workflow-e2e-test.ts @@ -0,0 +1,83 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import assert from "node:assert/strict"; + +import { test as base } from "vitest"; + +import { createArtifactSink } from "./artifacts.ts"; +import { type ProgressPhaseOutcome, startTestProgress, type TestProgress } from "./progress.ts"; +import { SecretStore } from "./secrets.ts"; + +declare module "@vitest/runner" { + interface TaskMeta { + e2ePhases?: readonly string[]; + } +} + +export interface WorkflowE2ETestFixtures { + progress: TestProgress; +} + +function taskOutcomeForState(state: string | undefined): ProgressPhaseOutcome | undefined { + return state === "fail" ? "failed" : state === "skip" ? "skipped" : undefined; +} + +function outcomeForTaskState(state: string | undefined): ProgressPhaseOutcome { + return taskOutcomeForState(state) ?? "passed"; +} + +/** + * Progress fixture for credential-free integration tests selected by the E2E + * workflow's shared job. These tests cannot use the stateful e2e-live fixture, + * but they must publish the same semantic timeline contract when run as E2E. + */ +export const test = base.extend({ + progress: [ + async ({ onTestFinished, skip, task }, use) => { + const phasePlan = task.meta.e2ePhases; + assert.ok(phasePlan, `workflow-selected E2E test is missing semantic phases: ${task.name}`); + const declaredFinalPhase = phasePlan.at(-1) as string; + const secrets = new SecretStore(process.env, skip); + const targetId = process.env.E2E_TARGET_ID || process.env.GITHUB_JOB; + const progress = startTestProgress(task.name, phasePlan, { + targetId, + taskStatus: () => ({ + errorCount: task.result?.errors?.length ?? 0, + ...(taskOutcomeForState(task.result?.state) + ? { outcome: taskOutcomeForState(task.result?.state) } + : {}), + }), + logLine: + process.env.NEMOCLAW_RUN_LIVE_E2E === "1" + ? (line) => process.stdout.write(`${secrets.redact(line)}\n`) + : () => { + // Ordinary integration runs stay quiet; the shared E2E job logs progress. + }, + }); + let finalized = false; + onTestFinished(async () => { + if (finalized) return; + finalized = true; + const outcome = outcomeForTaskState(task.result?.state); + const completedPhasePlan = progress.hasReached(declaredFinalPhase); + progress.stop(outcome === "passed" && !completedPhasePlan ? "failed" : outcome); + if (process.env.NEMOCLAW_RUN_LIVE_E2E === "1") { + const artifacts = createArtifactSink(task.name, process.cwd(), secrets.redactionValues()); + await artifacts.ensureRoot(); + await artifacts.writeJson("test-progress.json", { + ...progress.summary(), + ...(targetId ? { targetId } : {}), + ...(process.env.NEMOCLAW_E2E_SHARD ? { shardId: process.env.NEMOCLAW_E2E_SHARD } : {}), + }); + } + assert.ok( + outcome !== "passed" || completedPhasePlan, + `workflow-selected E2E test did not reach its final semantic phase: ${task.name}`, + ); + }); + await use(progress); + }, + { auto: true }, + ], +}); diff --git a/test/e2e/live/agent-turn-latency-helpers.ts b/test/e2e/live/agent-turn-latency-helpers.ts index 57777edab3..4401c3e9be 100644 --- a/test/e2e/live/agent-turn-latency-helpers.ts +++ b/test/e2e/live/agent-turn-latency-helpers.ts @@ -41,6 +41,9 @@ export const EXPECTED_ROUTE_PROVIDER = (PROVIDER === "custom" ? "compatible-endpoint" : "nvidia-prod"); export const MAX_TURN_SECONDS = positiveInt(process.env.NEMOCLAW_TURN_LATENCY_MAX_SECONDS, 300); const INSTALL_ATTEMPTS = positiveInt(process.env.NEMOCLAW_TURN_LATENCY_INSTALL_ATTEMPTS, 2); +const INSTALL_TIMEOUT_MS = 30 * 60_000; + +type AgentTurnProgress = Pick; function positiveInt(value: string | undefined, fallback: number): number { return value && /^[1-9][0-9]*$/u.test(value) ? Number.parseInt(value, 10) : fallback; @@ -78,13 +81,52 @@ export function env( export async function bestEffortPreclean( label: string, run: () => Promise, -): Promise { +): Promise { try { await run(); - } catch (error) { - console.warn( - `best-effort cleanup failed (${label}): ${error instanceof Error ? error.message : String(error)}`, - ); + return true; + } catch { + console.warn(`best-effort cleanup failed (${label}); see redacted command artifacts`); + return false; + } +} + +function emitProgressEvent(progress: AgentTurnProgress | undefined, label: string): void { + try { + progress?.event(label); + } catch { + // Progress diagnostics must never change the agent-turn result. + } +} + +function startProgressActivity(progress: AgentTurnProgress | undefined, label: string): () => void { + let finish: (() => void) | undefined; + try { + finish = progress?.activity(label); + } catch { + return () => undefined; + } + return () => { + try { + finish?.(); + } catch { + // Progress diagnostics must never change the agent-turn result. + } + }; +} + +async function runBestEffortCleanupStep( + label: string, + run: () => Promise, + progress?: AgentTurnProgress, +): Promise { + emitProgressEvent(progress, `${label} started`); + const finishActivity = startProgressActivity(progress, `cleanup: ${label}`); + try { + const succeeded = await bestEffortPreclean(label, run); + emitProgressEvent(progress, `${label} ${succeeded ? "passed" : "failed"}`); + } finally { + finishActivity(); } } @@ -253,21 +295,40 @@ export async function installSandbox( agent: "openclaw" | "hermes", apiKey: string, cleanupBeforeRetry?: () => Promise, - progress?: Pick, + progress?: AgentTurnProgress, ): Promise { let install: ShellProbeResult | undefined; for (let attempt = 1; attempt <= INSTALL_ATTEMPTS; attempt += 1) { - install = await host.command( - "bash", - ["install.sh", "--non-interactive", "--fresh", "--yes-i-accept-third-party-software"], - { - artifactName: `${agent}-install-attempt-${attempt}`, - cwd: REPO_ROOT, - env: env(sandboxName, agent, apiKey), - onOutput: progress?.onOutput, - redactionValues: [apiKey], - timeoutMs: 30 * 60_000, - }, + const attemptLabel = `${agent} install attempt ${attempt}/${INSTALL_ATTEMPTS}`; + emitProgressEvent(progress, `${attemptLabel} started`); + const finishInstallActivity = startProgressActivity( + progress, + `command: ${agent}-install-attempt-${attempt}`, + ); + try { + install = await host.command( + "bash", + ["install.sh", "--non-interactive", "--fresh", "--yes-i-accept-third-party-software"], + { + artifactName: `${agent}-install-attempt-${attempt}`, + cwd: REPO_ROOT, + env: env(sandboxName, agent, apiKey), + onOutput: progress?.onOutput, + redactionValues: [apiKey], + timeoutMs: INSTALL_TIMEOUT_MS, + }, + ); + } catch (error) { + emitProgressEvent(progress, `${attemptLabel} failed before returning a result`); + throw error; + } finally { + finishInstallActivity(); + } + emitProgressEvent( + progress, + install.timedOut + ? `${attemptLabel} timeout fired at the 30-minute limit` + : `${attemptLabel} ${install.exitCode === 0 ? "passed" : "failed"}`, ); const retry = install.exitCode !== 0 && @@ -275,10 +336,25 @@ export async function installSandbox( attempt < INSTALL_ATTEMPTS; install.exitCode === 0 && (attempt = INSTALL_ATTEMPTS + 1); if (retry && cleanupBeforeRetry) { - await cleanupBeforeRetry(); + emitProgressEvent(progress, `${attemptLabel} starting cleanup before retry`); + const finishCleanupActivity = startProgressActivity( + progress, + `cleanup: ${agent}-install-attempt-${attempt}-retry`, + ); + try { + await cleanupBeforeRetry(); + emitProgressEvent(progress, `${attemptLabel} cleanup before retry passed`); + } catch (error) { + emitProgressEvent(progress, `${attemptLabel} cleanup before retry failed`); + throw error; + } finally { + finishCleanupActivity(); + } } if (retry) { - await new Promise((resolve) => setTimeout(resolve, 10_000 * attempt)); + const backoffSeconds = 10 * attempt; + emitProgressEvent(progress, `${attemptLabel} waiting ${backoffSeconds}s before retry`); + await new Promise((resolve) => setTimeout(resolve, backoffSeconds * 1_000)); } !retry && install.exitCode !== 0 && (attempt = INSTALL_ATTEMPTS + 1); } @@ -289,39 +365,50 @@ export async function installSandbox( export async function cleanupTurnSandboxes( host: HostCliClient, sandbox: SandboxClient, - progress?: Pick, + progress?: AgentTurnProgress, ): Promise { for (const [name, agent] of [ [OPENCLAW_SANDBOX, "openclaw"], [HERMES_SANDBOX, "hermes"], ] as const) { - await bestEffortPreclean(`destroy ${agent} sandbox`, () => - cleanupTurnSandbox(host, name, agent, progress), + await runBestEffortCleanupStep( + `destroy ${agent} sandbox`, + () => cleanupTurnSandbox(host, name, agent, progress), + progress, ); - await bestEffortPreclean(`delete ${agent} sandbox`, () => - sandbox.openshell(["sandbox", "delete", name], { - artifactName: `cleanup-${agent}-delete`, - env: env(name, agent), - onOutput: progress?.onOutput, - timeoutMs: 60_000, - }), + await runBestEffortCleanupStep( + `delete ${agent} sandbox`, + () => + sandbox.openshell(["sandbox", "delete", name], { + artifactName: `cleanup-${agent}-delete`, + env: env(name, agent), + onOutput: progress?.onOutput, + timeoutMs: 60_000, + }), + progress, ); } - await bestEffortPreclean("stop Hermes API forward", () => - sandbox.openshell(["forward", "stop", "8642"], { - artifactName: "cleanup-forward-stop-hermes-api", - env: buildAvailabilityProbeEnv(), - onOutput: progress?.onOutput, - timeoutMs: 30_000, - }), + await runBestEffortCleanupStep( + "stop Hermes API forward", + () => + sandbox.openshell(["forward", "stop", "8642"], { + artifactName: "cleanup-forward-stop-hermes-api", + env: buildAvailabilityProbeEnv(), + onOutput: progress?.onOutput, + timeoutMs: 30_000, + }), + progress, ); - await bestEffortPreclean("destroy OpenShell gateway", () => - sandbox.openshell(["gateway", "destroy", "-g", "nemoclaw"], { - artifactName: "cleanup-gateway-destroy-turn-latency", - env: buildAvailabilityProbeEnv(), - onOutput: progress?.onOutput, - timeoutMs: 60_000, - }), + await runBestEffortCleanupStep( + "destroy OpenShell gateway", + () => + sandbox.openshell(["gateway", "destroy", "-g", "nemoclaw"], { + artifactName: "cleanup-gateway-destroy-turn-latency", + env: buildAvailabilityProbeEnv(), + onOutput: progress?.onOutput, + timeoutMs: 60_000, + }), + progress, ); } diff --git a/test/e2e/live/agent-turn-latency.test.ts b/test/e2e/live/agent-turn-latency.test.ts index 4cb78b6459..f7e797ae75 100644 --- a/test/e2e/live/agent-turn-latency.test.ts +++ b/test/e2e/live/agent-turn-latency.test.ts @@ -45,7 +45,7 @@ test("OpenClaw and Hermes complete real hosted inference turns within the latenc "replace OpenClaw with Hermes sandbox", "validate Hermes inference route", "run Hermes hosted inference turn", - "remove provisioned sandboxes", + "record hosted inference timing evidence", ], }, }, async ({ artifacts, cleanup, host, progress, sandbox, secrets }) => { @@ -219,7 +219,7 @@ test("OpenClaw and Hermes complete real hosted inference turns within the latenc expect(hermesMs).toBeLessThanOrEqual(MAX_TURN_SECONDS * 1000); results.hermes = { elapsedMs: hermesMs }; await artifacts.writeJson("turn-latency-results.json", results); - progress.phase("remove provisioned sandboxes"); + progress.phase("record hosted inference timing evidence"); fs.writeFileSync( artifacts.pathFor("agent-turn-latency-results-legacy-path.json"), `${JSON.stringify(results, null, 2)}\n`, diff --git a/test/e2e/live/bedrock-runtime-compatible-anthropic-leaks.ts b/test/e2e/live/bedrock-runtime-compatible-anthropic-leaks.ts index dd665ab9a6..fa3d11adc6 100644 --- a/test/e2e/live/bedrock-runtime-compatible-anthropic-leaks.ts +++ b/test/e2e/live/bedrock-runtime-compatible-anthropic-leaks.ts @@ -3,7 +3,8 @@ export const SNAPSHOT_PROBE_PID_PREFIX = "@@NEMOCLAW_E2E_PROBE_PID@@ "; export const SNAPSHOT_FILE_PREFIX = "@@NEMOCLAW_E2E_FILE@@ "; -export const SNAPSHOT_DATA_PREFIX = "@@NEMOCLAW_E2E_DATA@@ "; +// Keep this per-line tag compact so null-heavy snapshots stay within the bounded capture guard. +export const SNAPSHOT_DATA_PREFIX = "D "; const PID_PATTERN = /^[1-9][0-9]*$/u; export interface ForbiddenLeakPattern { diff --git a/test/e2e/live/bedrock-runtime-compatible-anthropic-raw-command.ts b/test/e2e/live/bedrock-runtime-compatible-anthropic-raw-command.ts new file mode 100644 index 0000000000..9744715508 --- /dev/null +++ b/test/e2e/live/bedrock-runtime-compatible-anthropic-raw-command.ts @@ -0,0 +1,213 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { ArtifactSink } from "../fixtures/artifacts.ts"; +import { spawnObservedChild } from "../fixtures/observed-child-process.ts"; +import { REPO_ROOT } from "../fixtures/paths.ts"; +import type { TestProgress, TestProgressCapability } from "../fixtures/progress.ts"; +import { redactString } from "../fixtures/redaction.ts"; +import { + projectRawOutputForArtifact, + type RawArtifactOutputMode, +} from "./bedrock-runtime-compatible-anthropic-artifacts.ts"; + +const MAX_RAW_COMMAND_OUTPUT_BYTES = 10 * 1024 * 1024; +const RAW_COMMAND_OUTPUT_LIMIT_MARKER = "[bedrock raw-command output exceeded safe capture limit]"; + +interface BoundedOutputCapture { + readonly buffer: Buffer; + length: number; +} + +function boundedOutputCapture(): BoundedOutputCapture { + return { buffer: Buffer.allocUnsafe(MAX_RAW_COMMAND_OUTPUT_BYTES), length: 0 }; +} + +function appendBoundedOutput(capture: BoundedOutputCapture, chunk: Buffer): boolean { + const copied = chunk.copy( + capture.buffer, + capture.length, + 0, + MAX_RAW_COMMAND_OUTPUT_BYTES - capture.length, + ); + capture.length += copied; + return copied === chunk.length; +} + +function capturedOutput(capture: BoundedOutputCapture): string { + return capture.buffer.subarray(0, capture.length).toString("utf8"); +} + +export interface RawRunResult { + readonly command: readonly string[]; + readonly exitCode: number | null; + readonly signal: NodeJS.Signals | null; + readonly timedOut: boolean; + readonly stdout: string; + readonly stderr: string; + readonly redactedStdout: string; + readonly redactedStderr: string; +} + +export interface RawRunOptions { + readonly artifactName: string; + readonly artifacts: ArtifactSink; + readonly artifactOutputMode?: RawArtifactOutputMode; + readonly cwd?: string; + readonly env?: NodeJS.ProcessEnv; + readonly progress: Pick & TestProgressCapability; + readonly redactionValues?: readonly string[]; + readonly timeoutMs?: number; +} + +function progressCommandName(artifactName: string): string { + return /^[A-Za-z0-9][A-Za-z0-9._-]{0,95}$/u.test(artifactName) + ? artifactName + : "bedrock-raw-command"; +} + +function emitProgressEvent(progress: RawRunOptions["progress"], label: string): void { + try { + progress?.event(label); + } catch { + // Progress diagnostics must never change the Bedrock contract result. + } +} + +function redactedCommand(command: readonly string[], values: readonly string[]): string[] { + return command.map((part) => redactString(part, values)); +} + +export async function runRawCommand( + command: string, + args: readonly string[], + options: RawRunOptions, +): Promise { + const timeoutMs = options.timeoutMs ?? 60_000; + const redactionValues = [...(options.redactionValues ?? [])]; + const progressName = progressCommandName(options.artifactName); + emitProgressEvent(options.progress, `command ${progressName} started`); + let child: ReturnType; + try { + child = spawnObservedChild(command, args, { + activityLabel: `command: ${progressName}`, + progress: options.progress, + spawn: { + cwd: options.cwd ?? REPO_ROOT, + detached: true, + env: options.env, + stdio: ["ignore", "pipe", "pipe"], + }, + }); + } catch (error) { + emitProgressEvent(options.progress, `command ${progressName} failed to start`); + throw error; + } + const fullCommand = [command, ...args]; + const stdoutCapture = boundedOutputCapture(); + const stderrCapture = boundedOutputCapture(); + let captureLimitExceeded = false; + let timedOut = false; + let spawnError: Error | undefined; + + const killProcessGroup = (signal: NodeJS.Signals): void => { + if (child.pid === undefined) return; + try { + process.kill(-child.pid, signal); + } catch { + child.kill(signal); + } + }; + + const captureOutput = (capture: BoundedOutputCapture, chunk: Buffer): void => { + if (captureLimitExceeded || appendBoundedOutput(capture, chunk)) return; + captureLimitExceeded = true; + stdoutCapture.length = 0; + stderrCapture.length = 0; + emitProgressEvent( + options.progress, + `command ${progressName} output exceeded safe capture limit`, + ); + killProcessGroup("SIGKILL"); + }; + + const timeout = setTimeout(() => { + timedOut = true; + emitProgressEvent( + options.progress, + `command ${progressName} timeout fired after ${timeoutMs}ms`, + ); + killProcessGroup("SIGTERM"); + setTimeout(() => killProcessGroup("SIGKILL"), 1_000).unref(); + }, timeoutMs); + timeout.unref(); + + child.stdout?.on("data", (chunk: Buffer) => { + captureOutput(stdoutCapture, chunk); + }); + child.stderr?.on("data", (chunk: Buffer) => { + captureOutput(stderrCapture, chunk); + }); + child.on("error", (error) => { + spawnError = error; + }); + + const { exitCode, signal } = await new Promise<{ + exitCode: number | null; + signal: NodeJS.Signals | null; + }>((resolve) => { + child.on("close", (code, closeSignal) => resolve({ exitCode: code, signal: closeSignal })); + }); + clearTimeout(timeout); + emitProgressEvent( + options.progress, + `command ${progressName} ${timedOut ? "stopped after timeout" : exitCode === 0 ? "passed" : "failed"}`, + ); + + if (spawnError) { + const message = redactString(spawnError.message, redactionValues); + throw new Error(`failed to spawn ${redactString(command, redactionValues)}: ${message}`); + } + + const stdout = captureLimitExceeded + ? RAW_COMMAND_OUTPUT_LIMIT_MARKER + : capturedOutput(stdoutCapture); + const stderr = captureLimitExceeded + ? RAW_COMMAND_OUTPUT_LIMIT_MARKER + : capturedOutput(stderrCapture); + const redactedStdout = redactString(stdout, redactionValues); + const redactedStderr = redactString(stderr, redactionValues); + const artifactOutputMode = options.artifactOutputMode ?? "content"; + const artifactStdout = captureLimitExceeded + ? RAW_COMMAND_OUTPUT_LIMIT_MARKER + : projectRawOutputForArtifact(redactedStdout, "stdout", artifactOutputMode); + const artifactStderr = captureLimitExceeded + ? RAW_COMMAND_OUTPUT_LIMIT_MARKER + : projectRawOutputForArtifact(redactedStderr, "stderr", artifactOutputMode); + await options.artifacts.writeText(`raw-shell/${options.artifactName}.stdout.txt`, artifactStdout); + await options.artifacts.writeText(`raw-shell/${options.artifactName}.stderr.txt`, artifactStderr); + await options.artifacts.writeJson(`raw-shell/${options.artifactName}.result.json`, { + command: redactedCommand(fullCommand, redactionValues), + exitCode, + signal, + timedOut, + captureLimitExceeded, + stdout: artifactStdout, + stderr: artifactStderr, + }); + + if (captureLimitExceeded) { + throw new Error(`command ${progressName} output exceeded safe capture limit`); + } + + return { + command: fullCommand, + exitCode, + signal, + timedOut, + stdout, + stderr, + redactedStdout, + redactedStderr, + }; +} diff --git a/test/e2e/live/bedrock-runtime-compatible-anthropic.test.ts b/test/e2e/live/bedrock-runtime-compatible-anthropic.test.ts index 02cd07d0a8..d61cc18b17 100644 --- a/test/e2e/live/bedrock-runtime-compatible-anthropic.test.ts +++ b/test/e2e/live/bedrock-runtime-compatible-anthropic.test.ts @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { spawn, spawnSync } from "node:child_process"; +import { spawnSync } from "node:child_process"; import { randomUUID } from "node:crypto"; import fs from "node:fs"; import * as http2 from "node:http2"; @@ -30,12 +30,8 @@ import { import { expect, test } from "../fixtures/e2e-test.ts"; import { testHomeEnvironment } from "../fixtures/environment-profiles.ts"; import { CLI_DIST_ENTRYPOINT, CLI_ENTRYPOINT, REPO_ROOT } from "../fixtures/paths.ts"; -import { redactString } from "../fixtures/redaction.ts"; -import { - projectRawOutputForArtifact, - type RawArtifactOutputMode, - summarizeSandboxSnapshot, -} from "./bedrock-runtime-compatible-anthropic-artifacts.ts"; +import type { TestProgress, TestProgressCapability } from "../fixtures/progress.ts"; +import { summarizeSandboxSnapshot } from "./bedrock-runtime-compatible-anthropic-artifacts.ts"; import { type ForbiddenLeakPattern, findForbiddenLeaks, @@ -52,6 +48,10 @@ import { BEDROCK_PRE_CONTRACT_ENDPOINT_VALIDATION_SOURCE_BOUNDARY, isPreContractEndpointValidationRateLimitEvidence, } from "./bedrock-runtime-compatible-anthropic-rate-limit.ts"; +import { + type RawRunResult, + runRawCommand, +} from "./bedrock-runtime-compatible-anthropic-raw-command.ts"; // Keep the same live system boundary: host fake Bedrock Runtime endpoint, // /etc/hosts mapping, source CLI onboard, OpenShell provider route, sandbox @@ -85,27 +85,6 @@ type EventStreamCodecConstructor = new ( fromUtf8: (input: string) => Uint8Array, ) => EventStreamCodec; -interface RawRunResult { - readonly command: readonly string[]; - readonly exitCode: number | null; - readonly signal: NodeJS.Signals | null; - readonly timedOut: boolean; - readonly stdout: string; - readonly stderr: string; - readonly redactedStdout: string; - readonly redactedStderr: string; -} - -interface RawRunOptions { - readonly artifactName: string; - readonly artifacts: ArtifactSink; - readonly artifactOutputMode?: RawArtifactOutputMode; - readonly cwd?: string; - readonly env?: NodeJS.ProcessEnv; - readonly redactionValues?: readonly string[]; - readonly timeoutMs?: number; -} - interface MockBedrockRuntime { readonly port: number; readonly logs: readonly string[]; @@ -184,7 +163,7 @@ function createBedrockTlsFixture(home: string): { cert: Buffer; key: Buffer; cer "-addext", `subjectAltName=DNS:${BEDROCK_HOSTNAME}`, ], - { encoding: "utf8" }, + { encoding: "utf8", killSignal: "SIGKILL", timeout: 30_000 }, ); expect( generated.status, @@ -197,96 +176,6 @@ function createBedrockTlsFixture(home: string): { cert: Buffer; key: Buffer; cer }; } -function redactedCommand(command: readonly string[], values: readonly string[]): string[] { - return command.map((part) => redactString(part, values)); -} - -async function runRawCommand( - command: string, - args: readonly string[], - options: RawRunOptions, -): Promise { - const timeoutMs = options.timeoutMs ?? 60_000; - const redactionValues = [...(options.redactionValues ?? [])]; - const child = spawn(command, [...args], { - cwd: options.cwd ?? REPO_ROOT, - detached: true, - env: options.env, - stdio: ["ignore", "pipe", "pipe"], - }); - const fullCommand = [command, ...args]; - let stdout = ""; - let stderr = ""; - let timedOut = false; - let spawnError: Error | undefined; - - const killProcessGroup = (signal: NodeJS.Signals): void => { - if (child.pid === undefined) return; - try { - process.kill(-child.pid, signal); - } catch { - child.kill(signal); - } - }; - - const timeout = setTimeout(() => { - timedOut = true; - killProcessGroup("SIGTERM"); - setTimeout(() => killProcessGroup("SIGKILL"), 1_000).unref(); - }, timeoutMs); - timeout.unref(); - - child.stdout?.on("data", (chunk: Buffer) => { - stdout += chunk.toString("utf8"); - }); - child.stderr?.on("data", (chunk: Buffer) => { - stderr += chunk.toString("utf8"); - }); - child.on("error", (error) => { - spawnError = error; - }); - - const { exitCode, signal } = await new Promise<{ - exitCode: number | null; - signal: NodeJS.Signals | null; - }>((resolve) => { - child.on("close", (code, closeSignal) => resolve({ exitCode: code, signal: closeSignal })); - }); - clearTimeout(timeout); - - if (spawnError) { - const message = redactString(spawnError.message, redactionValues); - throw new Error(`failed to spawn ${redactString(command, redactionValues)}: ${message}`); - } - - const redactedStdout = redactString(stdout, redactionValues); - const redactedStderr = redactString(stderr, redactionValues); - const artifactOutputMode = options.artifactOutputMode ?? "content"; - const artifactStdout = projectRawOutputForArtifact(redactedStdout, "stdout", artifactOutputMode); - const artifactStderr = projectRawOutputForArtifact(redactedStderr, "stderr", artifactOutputMode); - await options.artifacts.writeText(`raw-shell/${options.artifactName}.stdout.txt`, artifactStdout); - await options.artifacts.writeText(`raw-shell/${options.artifactName}.stderr.txt`, artifactStderr); - await options.artifacts.writeJson(`raw-shell/${options.artifactName}.result.json`, { - command: redactedCommand(fullCommand, redactionValues), - exitCode, - signal, - timedOut, - stdout: artifactStdout, - stderr: artifactStderr, - }); - - return { - command: fullCommand, - exitCode, - signal, - timedOut, - stdout, - stderr, - redactedStdout, - redactedStderr, - }; -} - function loadEventStreamCodec(): EventStreamCodec { const loaded = require("@smithy/core/event-streams") as { EventStreamCodec: EventStreamCodecConstructor; @@ -602,7 +491,9 @@ function isBedrockAdapterProcess(pid: number): boolean { const ps = spawnSync("ps", ["-p", String(pid), "-o", "args="], { encoding: "utf8", + killSignal: "SIGKILL", stdio: ["ignore", "pipe", "ignore"], + timeout: 5_000, }); return ps.status === 0 && BEDROCK_ADAPTER_LAUNCHER_PATTERN.test(ps.stdout); } @@ -1242,6 +1133,7 @@ async function assertNoBedrockLeaks(options: { home: string; mock: MockBedrockRuntime; onboarding: RawRunResult; + progress: Pick & TestProgressCapability; sandbox: SandboxClient; redact: (text: string, extraValues?: string[]) => string; }): Promise { @@ -1265,6 +1157,7 @@ async function assertNoBedrockLeaks(options: { artifacts: options.artifacts, artifactOutputMode: "metadata-only", env: testEnv(options.home), + progress: options.progress, redactionValues: [COMPATIBLE_KEY, adapterToken], timeoutMs: 180_000, }, @@ -1433,6 +1326,7 @@ test("bedrock runtime compatible Anthropic endpoint routes through managed infer artifactName: `onboard-bedrock-runtime-${AGENT}`, artifacts, env: onboardEnv(home, AGENT, tls.certPath), + progress, redactionValues: [COMPATIBLE_KEY], timeoutMs: ONBOARD_TIMEOUT_MS, }, @@ -1480,6 +1374,7 @@ test("bedrock runtime compatible Anthropic endpoint routes through managed infer home, mock, onboarding, + progress, sandbox, redact: (text, extraValues) => secrets.redact(text, extraValues), }); diff --git a/test/e2e/live/channels-add-remove.test.ts b/test/e2e/live/channels-add-remove.test.ts index 1088322825..6d61eb4894 100644 --- a/test/e2e/live/channels-add-remove.test.ts +++ b/test/e2e/live/channels-add-remove.test.ts @@ -392,6 +392,7 @@ test( apiKey: BASELINE_API_KEY, host: "0.0.0.0", model: BASELINE_MODEL, + progress, publicHost: "host.openshell.internal", requireAuth: true, }); diff --git a/test/e2e/live/concurrent-gateway-ports.test.ts b/test/e2e/live/concurrent-gateway-ports.test.ts index a60e7754b9..57ae4c8ad2 100644 --- a/test/e2e/live/concurrent-gateway-ports.test.ts +++ b/test/e2e/live/concurrent-gateway-ports.test.ts @@ -343,6 +343,7 @@ test("concurrent gateway ports: onboards two sandboxes on isolated gateways and const fake = await startFakeOpenAiCompatibleServer({ host: "0.0.0.0", port: Number(process.env.NEMOCLAW_E2E_FAKE_PORT ?? 0), + progress, publicHost: "host.openshell.internal", }); await artifacts.target.declare({ diff --git a/test/e2e/live/device-auth-health.test.ts b/test/e2e/live/device-auth-health.test.ts index 85faff5844..ea5d4ece38 100644 --- a/test/e2e/live/device-auth-health.test.ts +++ b/test/e2e/live/device-auth-health.test.ts @@ -53,6 +53,7 @@ test("device auth health probes treat 401 as live instead of offline (#2342)", { apiKey: INFERENCE_API_KEY, host: "0.0.0.0", model: INFERENCE_MODEL, + progress, publicHost: "host.openshell.internal", requireAuth: true, }); diff --git a/test/e2e/live/diagnostics.test.ts b/test/e2e/live/diagnostics.test.ts index 74f503bc6a..70dc06ff3b 100644 --- a/test/e2e/live/diagnostics.test.ts +++ b/test/e2e/live/diagnostics.test.ts @@ -46,17 +46,13 @@ function redactForAssertion(text: string, apiKey: string): string { .replace(/nvapi-[A-Za-z0-9_-]{10,}/g, ""); } -function runRawNodeCliForLeakAssertion( - args: string[], - env: NodeJS.ProcessEnv, - timeoutMs: number, -): RawCommandResult { +function runRawNodeCliForLeakAssertion(args: string[], env: NodeJS.ProcessEnv): RawCommandResult { const result = spawnSync("node", [CLI_ENTRYPOINT, ...args], { cwd: REPO_ROOT, encoding: "utf8", env, killSignal: "SIGKILL", - timeout: timeoutMs, + timeout: 60_000, }); return { status: result.status, @@ -333,7 +329,7 @@ test("diagnostics CLI creates sanitized archives and validates sandbox/credentia expect(resultText(status)).toMatch(/Model/i); progress.phase("audit and reset gateway credentials"); - const rawCredentialsList = runRawNodeCliForLeakAssertion(["credentials", "list"], env, 60_000); + const rawCredentialsList = runRawNodeCliForLeakAssertion(["credentials", "list"], env); const credentialsOutput = rawResultText(rawCredentialsList); const credentialsStdout = rawCredentialsList.stdout; expect(rawCredentialsList.status, redactForAssertion(credentialsOutput, apiKey)).toBe(0); @@ -402,7 +398,7 @@ test("diagnostics CLI creates sanitized archives and validates sandbox/credentia expect(reset.exitCode, resultText(reset)).toBe(0); expect(reset.stdout, resultText(reset)).toContain(`Removed provider '${hosted.providerName}'`); - const rawPostResetList = runRawNodeCliForLeakAssertion(["credentials", "list"], env, 60_000); + const rawPostResetList = runRawNodeCliForLeakAssertion(["credentials", "list"], env); const postResetOutput = rawResultText(rawPostResetList); expect(rawPostResetList.status, redactForAssertion(postResetOutput, apiKey)).toBe(0); expect( diff --git a/test/e2e/live/double-onboard.test.ts b/test/e2e/live/double-onboard.test.ts index 3b438dd01c..45bc6fe5cb 100644 --- a/test/e2e/live/double-onboard.test.ts +++ b/test/e2e/live/double-onboard.test.ts @@ -474,6 +474,7 @@ test("double-onboard: reuses gateway, preserves sibling sandbox, and recovers st const fake = await startFakeOpenAiCompatibleServer({ host: "0.0.0.0", port: Number(process.env.NEMOCLAW_FAKE_PORT ?? 0), + progress, publicHost: "host.openshell.internal", }); await artifacts.writeJson("fake-openai.json", { baseUrl: fake.baseUrl }); diff --git a/test/e2e/live/hermes-gpu-startup.test.ts b/test/e2e/live/hermes-gpu-startup.test.ts index c38462d45e..2bcf9d0dd7 100644 --- a/test/e2e/live/hermes-gpu-startup.test.ts +++ b/test/e2e/live/hermes-gpu-startup.test.ts @@ -329,6 +329,7 @@ test(`hermes-gpu-startup: ${GPU_STARTUP_SCENARIO} OpenShell GPU route reaches st forbiddenMarkers: [EXTRA_PLACEHOLDER_TOKEN_A, EXTRA_PLACEHOLDER_TOKEN_B], host: "0.0.0.0", model: FAKE_MODEL, + progress, publicHost: hostAddress, requireAuth: true, }); diff --git a/test/e2e/live/hermes-inference-switch.test.ts b/test/e2e/live/hermes-inference-switch.test.ts index f0cf53e75b..c67ab05f04 100644 --- a/test/e2e/live/hermes-inference-switch.test.ts +++ b/test/e2e/live/hermes-inference-switch.test.ts @@ -138,6 +138,7 @@ test("Hermes inference set updates route/config and preserves live runtime", { host: "0.0.0.0", model: MOCK_BASELINE_MODEL, publicHost: "host.openshell.internal", + progress, requireAuth: true, }) : undefined; diff --git a/test/e2e/live/hermes-root-entrypoint-smoke.test.ts b/test/e2e/live/hermes-root-entrypoint-smoke.test.ts index 5d4eeacd27..7856561565 100644 --- a/test/e2e/live/hermes-root-entrypoint-smoke.test.ts +++ b/test/e2e/live/hermes-root-entrypoint-smoke.test.ts @@ -413,9 +413,13 @@ test("hermes root-entrypoint smoke preserves runtime layout and legacy pid migra "validate legacy PID migration", ], }, -}, async ({ artifacts, cleanup, progress, secrets, skip }) => { - const probe = new DockerProbe(artifacts, (text, extraValues) => - secrets.redact(text, extraValues), +}, async ({ artifacts, cleanup, progress, secrets, signal, skip }) => { + const probe = new DockerProbe( + artifacts, + (text, extraValues) => secrets.redact(text, extraValues), + undefined, + progress, + signal, ); const runId = safeTag(`${process.env.GITHUB_RUN_ID ?? "local"}-${process.pid}-${Date.now()}`); const image = diff --git a/test/e2e/live/hermes-sandbox-secret-boundary.test.ts b/test/e2e/live/hermes-sandbox-secret-boundary.test.ts index 0c2c276094..037dff8735 100644 --- a/test/e2e/live/hermes-sandbox-secret-boundary.test.ts +++ b/test/e2e/live/hermes-sandbox-secret-boundary.test.ts @@ -733,9 +733,13 @@ test("hermes sandbox secret boundary keeps raw secrets out of images and startup "reject raw secrets from Hermes process env", ], }, -}, async ({ artifacts, cleanup, progress, secrets, skip }) => { - const probe = new DockerProbe(artifacts, (text, extraValues) => - secrets.redact(text, extraValues), +}, async ({ artifacts, cleanup, progress, secrets, signal, skip }) => { + const probe = new DockerProbe( + artifacts, + (text, extraValues) => secrets.redact(text, extraValues), + undefined, + progress, + signal, ); const runId = safeTag(`${process.env.GITHUB_RUN_ID ?? "local"}-${process.pid}-${Date.now()}`); const baseImageFromEnv = Boolean( diff --git a/test/e2e/live/hermes-shields-config.test.ts b/test/e2e/live/hermes-shields-config.test.ts index 8a27824d15..94ea1ef3ce 100644 --- a/test/e2e/live/hermes-shields-config.test.ts +++ b/test/e2e/live/hermes-shields-config.test.ts @@ -189,6 +189,7 @@ test("hermes-shields-config: fresh non-root Hermes sandbox completes two shields apiKey: COMPATIBLE_API_KEY, host: "0.0.0.0", model: COMPATIBLE_MODEL, + progress, publicHost: "host.openshell.internal", requireAuth: true, }); diff --git a/test/e2e/live/inference-routing-helpers.ts b/test/e2e/live/inference-routing-helpers.ts index f3b78b8741..ff4f6cf46f 100644 --- a/test/e2e/live/inference-routing-helpers.ts +++ b/test/e2e/live/inference-routing-helpers.ts @@ -1,7 +1,6 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { spawn } from "node:child_process"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; @@ -13,7 +12,11 @@ import type { SandboxClient } from "../fixtures/clients/sandbox.ts"; import { validateSandboxName } from "../fixtures/clients/sandbox.ts"; import { expect } from "../fixtures/e2e-test.ts"; import { CLI_DIST_ENTRYPOINT, CLI_ENTRYPOINT, REPO_ROOT } from "../fixtures/paths.ts"; -import { redactString } from "../fixtures/redaction.ts"; +import { + type RawRunOptions, + type RawRunResult, + runRawCommand, +} from "./bedrock-runtime-compatible-anthropic-raw-command.ts"; // live conversion: direct CLI/onboard subprocesses plus OpenShell sandbox // probes, with local helpers only where raw in-memory output is required to @@ -54,26 +57,6 @@ function skipLive(skip: SkipFn, note: string): never { throw new Error(note); } -interface RawRunResult { - readonly command: readonly string[]; - readonly exitCode: number | null; - readonly signal: NodeJS.Signals | null; - readonly timedOut: boolean; - readonly stdout: string; - readonly stderr: string; - readonly redactedStdout: string; - readonly redactedStderr: string; -} - -interface RawRunOptions { - readonly artifactName: string; - readonly artifacts: ArtifactSink; - readonly cwd?: string; - readonly env?: NodeJS.ProcessEnv; - readonly redactionValues?: readonly string[]; - readonly timeoutMs?: number; -} - function redactedResultText( result: Pick, ): string { @@ -120,93 +103,6 @@ process.exit(0); return commandLogPath; } -function redactedCommand(command: readonly string[], values: readonly string[]): string[] { - return command.map((part) => redactString(part, values)); -} - -async function runRawCommand( - command: string, - args: readonly string[], - options: RawRunOptions, -): Promise { - const timeoutMs = options.timeoutMs ?? 60_000; - const redactionValues = [...(options.redactionValues ?? [])]; - const child = spawn(command, [...args], { - cwd: options.cwd ?? REPO_ROOT, - detached: true, - env: options.env, - stdio: ["ignore", "pipe", "pipe"], - }); - const fullCommand = [command, ...args]; - let stdout = ""; - let stderr = ""; - let timedOut = false; - let spawnError: Error | undefined; - - const killProcessGroup = (signal: NodeJS.Signals): void => { - if (child.pid === undefined) return; - try { - process.kill(-child.pid, signal); - } catch { - child.kill(signal); - } - }; - - const timeout = setTimeout(() => { - timedOut = true; - killProcessGroup("SIGTERM"); - setTimeout(() => killProcessGroup("SIGKILL"), 1_000).unref(); - }, timeoutMs); - timeout.unref(); - - child.stdout?.on("data", (chunk: Buffer) => { - stdout += chunk.toString("utf8"); - }); - child.stderr?.on("data", (chunk: Buffer) => { - stderr += chunk.toString("utf8"); - }); - child.on("error", (error) => { - spawnError = error; - }); - - const { exitCode, signal } = await new Promise<{ - exitCode: number | null; - signal: NodeJS.Signals | null; - }>((resolve) => { - child.on("close", (code, closeSignal) => resolve({ exitCode: code, signal: closeSignal })); - }); - clearTimeout(timeout); - - if (spawnError) { - const message = redactString(spawnError.message, redactionValues); - throw new Error(`failed to spawn ${redactString(command, redactionValues)}: ${message}`); - } - - const redactedStdout = redactString(stdout, redactionValues); - const redactedStderr = redactString(stderr, redactionValues); - await options.artifacts.writeText(`raw-shell/${options.artifactName}.stdout.txt`, redactedStdout); - await options.artifacts.writeText(`raw-shell/${options.artifactName}.stderr.txt`, redactedStderr); - await options.artifacts.writeJson(`raw-shell/${options.artifactName}.result.json`, { - command: redactedCommand(fullCommand, redactionValues), - exitCode, - signal, - timedOut, - stdout: redactedStdout, - stderr: redactedStderr, - }); - - return { - command: fullCommand, - exitCode, - signal, - timedOut, - stdout, - stderr, - redactedStdout, - redactedStderr, - }; -} - async function runNemoclawCli( args: readonly string[], options: RawRunOptions, @@ -391,6 +287,7 @@ async function onboardSandbox( extraEnv: NodeJS.ProcessEnv, redactionValues: readonly string[], artifactName: string, + progress: RawRunOptions["progress"], timeoutMs = 10 * 60_000, ): Promise { clearOnboardState(); @@ -402,6 +299,7 @@ async function onboardSandbox( NEMOCLAW_SANDBOX_NAME: sandboxName, ...extraEnv, }), + progress, redactionValues, timeoutMs, }); diff --git a/test/e2e/live/inference-routing-provider-smoke.test.ts b/test/e2e/live/inference-routing-provider-smoke.test.ts index b5f22c477a..2c748e44d6 100644 --- a/test/e2e/live/inference-routing-provider-smoke.test.ts +++ b/test/e2e/live/inference-routing-provider-smoke.test.ts @@ -69,6 +69,7 @@ test("TC-INF-05 real NVIDIA key is isolated from sandbox env, process list, and { NVIDIA_INFERENCE_API_KEY: apiKey }, [apiKey], "tc-inf-05-onboard-credential-isolation", + progress, ); expectOnboardSuccess(onboard, "TC-INF-05 credential-isolation onboard"); cleanup.add(`strict inference-routing credential-isolation cleanup for ${sandboxName}`, () => @@ -80,6 +81,7 @@ test("TC-INF-05 real NVIDIA key is isolated from sandbox env, process list, and artifactName: "tc-inf-05-sandbox-env", artifacts, env: buildAvailabilityProbeEnv(), + progress, redactionValues: [apiKey], timeoutMs: 60_000, }); @@ -101,6 +103,7 @@ test("TC-INF-05 real NVIDIA key is isolated from sandbox env, process list, and artifactName: "tc-inf-05-sandbox-process-list", artifacts, env: buildAvailabilityProbeEnv(), + progress, redactionValues: [apiKey], timeoutMs: 60_000, }, @@ -149,6 +152,7 @@ test("TC-INF-05 real NVIDIA key is isolated from sandbox env, process list, and .update(leakCanary) .digest("hex"), }), + progress, timeoutMs: 90_000, }, ); @@ -180,6 +184,7 @@ test("TC-INF-05 real NVIDIA key is isolated from sandbox env, process list, and .update(apiKey) .digest("hex"), }), + progress, redactionValues: [apiKey], timeoutMs: 90_000, }, @@ -238,6 +243,7 @@ test("TC-INF-02 OpenAI provider responds through inference.local", { { NEMOCLAW_MODEL: model, NEMOCLAW_PROVIDER: "openai", OPENAI_API_KEY: apiKey }, [apiKey], "tc-inf-02-onboard-openai", + progress, ); expectOnboardSuccess(onboard, "TC-INF-02 OpenAI onboard"); cleanup.add(`strict inference-routing OpenAI cleanup for ${sandboxName}`, () => @@ -292,6 +298,7 @@ test("TC-INF-03 Anthropic provider responds through inference.local", { { ANTHROPIC_API_KEY: apiKey, NEMOCLAW_MODEL: model, NEMOCLAW_PROVIDER: "anthropic" }, [apiKey], "tc-inf-03-onboard-anthropic", + progress, ); expectOnboardSuccess(onboard, "TC-INF-03 Anthropic onboard"); cleanup.add(`strict inference-routing Anthropic cleanup for ${sandboxName}`, () => diff --git a/test/e2e/live/inference-routing.test.ts b/test/e2e/live/inference-routing.test.ts index 860bcd1f0a..f8a2259b46 100644 --- a/test/e2e/live/inference-routing.test.ts +++ b/test/e2e/live/inference-routing.test.ts @@ -69,6 +69,7 @@ test("TC-INF-06 invalid API key fails with credential classification and cleanup { NVIDIA_INFERENCE_API_KEY: invalidKey }, [invalidKey], "tc-inf-06-onboard-invalid-api-key", + progress, 120_000, ); const raw = resultText(result); @@ -126,6 +127,7 @@ test("TC-INF-07 unreachable endpoint fails with transport classification and cle }, [nvidiaKey, compatibleKey], "tc-inf-07-onboard-unreachable-endpoint", + progress, 120_000, ); const raw = resultText(result); @@ -217,6 +219,7 @@ await main(["apply"]); PATH: `${fakeBinDir}${path.delimiter}${process.env.PATH ?? ""}`, E2E_API_KEY: "e2e-fake-key", }, + progress, redactionValues: ["e2e-fake-key"], timeoutMs: 60_000, }, @@ -259,6 +262,7 @@ test("TC-INF-09 Deep Agents Code uses a local compatible endpoint through infere host: "0.0.0.0", model, port: 8000, + progress, publicHost: "localhost", requireAuth: true, requireAuthModels: true, @@ -296,6 +300,7 @@ test("TC-INF-09 Deep Agents Code uses a local compatible endpoint through infere }, [apiKey], "tc-inf-09-onboard-compatible-endpoint", + progress, 15 * 60_000, ); expectOnboardSuccess(onboard, "TC-INF-09 compatible-endpoint onboard"); @@ -350,6 +355,7 @@ test("TC-INF-09 Deep Agents Code uses a local compatible endpoint through infere artifactName: "tc-inf-09-dcode-compatible-endpoint", artifacts, env: buildAvailabilityProbeEnv(), + progress, redactionValues: [apiKey], timeoutMs: 3 * 60_000, }, diff --git a/test/e2e/live/issue-4434-tui-unreachable-inference.test.ts b/test/e2e/live/issue-4434-tui-unreachable-inference.test.ts index 1a034f957c..caf3c92cf6 100644 --- a/test/e2e/live/issue-4434-tui-unreachable-inference.test.ts +++ b/test/e2e/live/issue-4434-tui-unreachable-inference.test.ts @@ -321,6 +321,7 @@ runIssue4434LiveTest( const fake = await startFakeOpenAiCompatibleServer({ host: "0.0.0.0", model: hosted.model, + progress, publicHost: "host.openshell.internal", }); let fakeClosePromise: Promise | undefined; diff --git a/test/e2e/live/mcp-bridge-servers.ts b/test/e2e/live/mcp-bridge-servers.ts index 2320a330c8..c643a257e9 100644 --- a/test/e2e/live/mcp-bridge-servers.ts +++ b/test/e2e/live/mcp-bridge-servers.ts @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { type ChildProcess, spawn } from "node:child_process"; +import type { ChildProcess } from "node:child_process"; import fs from "node:fs"; import http from "node:http"; import https from "node:https"; @@ -9,12 +9,14 @@ import type { AddressInfo } from "node:net"; import os from "node:os"; import type { CleanupRegistry } from "../fixtures/cleanup.ts"; +import { spawnObservedChild } from "../fixtures/observed-child-process.ts"; import { closeServer, writeJsonResponse as jsonResponse, listenServer as listenOnRandomPort, readRequestBody, } from "../fixtures/http-protocol.ts"; +import type { TestProgress, TestProgressCapability } from "../fixtures/progress.ts"; type TestServer = http.Server | https.Server; @@ -218,6 +220,7 @@ async function probePublicTunnel(origin: string): Promise<{ export async function startPublicMcpHttpsTunnel(options: { cleanup: TunnelCleanupRegistry; label: string; + progress: Pick & TestProgressCapability; server: StartedHttpServer; cloudflaredBin?: string; }): Promise { @@ -225,6 +228,12 @@ export async function startPublicMcpHttpsTunnel(options: { let lastFailure = "cloudflared did not publish a quick-tunnel URL"; for (let attempt = 1; attempt <= QUICK_TUNNEL_ATTEMPTS; attempt += 1) { + const progressName = `cloudflared quick tunnel attempt ${attempt}`; + try { + options.progress.event(`${progressName} started`); + } catch { + // Progress diagnostics must never change tunnel setup. + } let origin: string | null = null; let consecutiveReadyProbes = 0; let childOutputSeen = false; @@ -238,10 +247,14 @@ export async function startPublicMcpHttpsTunnel(options: { carry = candidate.slice(-QUICK_TUNNEL_DISCOVERY_CARRY_LIMIT); }; }; - const child = spawn(options.cloudflaredBin ?? "cloudflared", args, { - detached: true, - env: buildCloudflaredSubprocessEnv(), - stdio: ["ignore", "pipe", "pipe"], + const child = spawnObservedChild(options.cloudflaredBin ?? "cloudflared", args, { + activityLabel: `command: ${progressName}`, + progress: options.progress, + spawn: { + detached: true, + env: buildCloudflaredSubprocessEnv(), + stdio: ["ignore", "pipe", "pipe"], + }, }); const exited = waitForExit(child); child.stdout?.setEncoding("utf8"); @@ -251,6 +264,13 @@ export async function startPublicMcpHttpsTunnel(options: { child.once("error", (error) => { spawnError = error; }); + child.once("close", () => { + try { + options.progress.event(`${progressName} stopped`); + } catch { + // Progress diagnostics must never change tunnel cleanup. + } + }); let closePromise: Promise | undefined; const close = (): Promise => { diff --git a/test/e2e/live/mcp-bridge.test.ts b/test/e2e/live/mcp-bridge.test.ts index ead4826637..7e8045ae87 100644 --- a/test/e2e/live/mcp-bridge.test.ts +++ b/test/e2e/live/mcp-bridge.test.ts @@ -445,11 +445,8 @@ async function assertConcurrentAddSerialized( artifactName: `${options.artifactPrefix}-mcp-concurrent-add-${attempt}`, env, redactionValues: [HOST_SECRET], - // Hermes may need one host-authenticated managed restart (210s), a - // fresh helper-readiness window (90s), and its acknowledged config - // reload (300s). Keep both concurrent clients alive through that - // bounded recovery; the loser then acquires the lifecycle lock and - // rejects the committed duplicate. + // Keep both clients alive through Hermes' bounded restart and config + // reload; the loser then acquires the lock and rejects the duplicate. timeoutMs: MCP_MUTATION_TIMEOUT_MS[options.expectedAdapter], }), ), @@ -862,6 +859,7 @@ test("mcp-bridge", { const fakeMcpTunnel = await startPublicMcpHttpsTunnel({ cleanup, label: "fake MCP HTTPS server", + progress, server: fakeMcp, }); const decoyMcp = await startFakeMcpHttpsServer({ secret: HOST_SECRET }); @@ -869,6 +867,7 @@ test("mcp-bridge", { const decoyMcpTunnel = await startPublicMcpHttpsTunnel({ cleanup, label: "unconfigured decoy MCP HTTPS server", + progress, server: decoyMcp, }); const hostAddress = await hostAddressForSandbox(host); @@ -1213,6 +1212,7 @@ mcpBridgeShardTest("hermes")( const fakeMcpTunnel = await startPublicMcpHttpsTunnel({ cleanup, label: "fake Hermes MCP HTTPS server", + progress, server: fakeMcp, }); const hostAddress = await hostAddressForSandbox(host); @@ -1378,6 +1378,7 @@ mcpBridgeShardTest("deepagents")( const fakeMcpTunnel = await startPublicMcpHttpsTunnel({ cleanup, label: "fake Deep Agents MCP HTTPS server", + progress, server: fakeMcp, }); const hostAddress = await hostAddressForSandbox(host); diff --git a/test/e2e/live/messaging-compatible-endpoint-helpers.ts b/test/e2e/live/messaging-compatible-endpoint-helpers.ts index 6daebdcc14..c1eaab3cf4 100644 --- a/test/e2e/live/messaging-compatible-endpoint-helpers.ts +++ b/test/e2e/live/messaging-compatible-endpoint-helpers.ts @@ -86,6 +86,8 @@ function readProcessSnapshot(pid: number): { cmdline: string; startTime: string if (process.platform === "darwin") { const line = execFileSync("ps", ["-p", String(pid), "-o", "lstart=", "-o", "command="], { encoding: "utf8", + killSignal: "SIGKILL", + timeout: 5_000, }).trim(); if (line.length <= 24) return null; return { cmdline: line.slice(24).trim(), startTime: line.slice(0, 24) }; diff --git a/test/e2e/live/ollama-auth-proxy.test.ts b/test/e2e/live/ollama-auth-proxy.test.ts index b42e4ed0d9..f00767858f 100644 --- a/test/e2e/live/ollama-auth-proxy.test.ts +++ b/test/e2e/live/ollama-auth-proxy.test.ts @@ -8,17 +8,20 @@ * persisted token, container reachability, and token-divergence repair logic. */ -import { type ChildProcess, spawn } from "node:child_process"; +import type { ChildProcess } from "node:child_process"; import { randomBytes } from "node:crypto"; import fs from "node:fs"; import { mkdtemp, rm, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; - +import { type ChildProcessOwner, ownChildProcess } from "../../helpers/child-process-lifecycle.ts"; +import type { ArtifactSink } from "../fixtures/artifacts.ts"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; import { resultText } from "../fixtures/clients/index.ts"; import { expect, test } from "../fixtures/e2e-test.ts"; +import { spawnObservedChild } from "../fixtures/observed-child-process.ts"; import { REPO_ROOT } from "../fixtures/paths.ts"; +import type { TestProgress, TestProgressCapability } from "../fixtures/progress.ts"; import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; const PROXY_SCRIPT = path.join(REPO_ROOT, "scripts", "ollama-auth-proxy.mts"); @@ -48,35 +51,71 @@ function token(): string { return randomBytes(24).toString("hex"); } +const childOwners = new WeakMap(); +const loggedArtifactWrites = new WeakMap>(); +const CHILD_PROCESS_OWNER_OPTIONS = { + forceTimeoutMs: 3_000, + gracefulTimeoutMs: 3_000, +} as const; + async function terminate(child: ChildProcess | undefined): Promise { - if (!child || child.killed || child.exitCode !== null) return; - child.kill("SIGTERM"); - await new Promise((resolve) => { - const timer = setTimeout(() => { - if (!child.killed && child.exitCode === null) child.kill("SIGKILL"); - resolve(); - }, 3_000); - child.once("exit", () => { - clearTimeout(timer); - resolve(); - }); - }); + if (!child) return; + const owner = childOwners.get(child) ?? ownChildProcess(child, CHILD_PROCESS_OWNER_OPTIONS); + await owner.terminate(); + await loggedArtifactWrites.get(child); } function spawnLogged( command: string, args: string[], - logPath: string, + artifacts: ArtifactSink, + artifactName: string, env: NodeJS.ProcessEnv, + progress: Pick & TestProgressCapability, + activityName: string, ): ChildProcess { - fs.mkdirSync(path.dirname(logPath), { recursive: true }); - const out = fs.openSync(logPath, "a"); - const child = spawn(command, args, { - cwd: REPO_ROOT, - env: { ...process.env, ...env }, - stdio: ["ignore", out, out], + try { + progress.event(`command ${activityName} started`); + } catch { + // Progress diagnostics must never change process execution. + } + const child = spawnObservedChild(command, args, { + activityLabel: `command: ${activityName}`, + progress, + spawn: { + cwd: REPO_ROOT, + env: { ...process.env, ...env }, + stdio: ["ignore", "pipe", "pipe"], + }, + }); + childOwners.set(child, ownChildProcess(child, CHILD_PROCESS_OWNER_OPTIONS)); + const outputLimit = 1024 * 1024; + let stdout = ""; + let stderr = ""; + const append = (current: string, chunk: Buffer): string => + `${current}${chunk.toString("utf8")}`.slice(-outputLimit); + const observe = (stream: "stdout" | "stderr", chunk: Buffer): void => { + if (stream === "stdout") stdout = append(stdout, chunk); + else stderr = append(stderr, chunk); + }; + child.stdout?.on("data", (chunk: Buffer) => observe("stdout", chunk)); + child.stderr?.on("data", (chunk: Buffer) => observe("stderr", chunk)); + const artifactWrite = new Promise((resolve, reject) => { + child.once("close", () => { + void Promise.all([ + artifacts.writeText(`process/${artifactName}.stdout.txt`, stdout), + artifacts.writeText(`process/${artifactName}.stderr.txt`, stderr), + ]).then(() => resolve(), reject); + }); + }); + loggedArtifactWrites.set(child, artifactWrite); + child.once("close", () => { + try { + progress.event(`command ${activityName} stopped`); + } catch { + // Progress diagnostics must never change process cleanup. + } }); - child.once("exit", () => fs.closeSync(out)); return child; } @@ -248,9 +287,15 @@ test("Ollama auth proxy enforces tokens, proxies inference, persists tokens, and }, ); - ollama = spawnLogged("ollama", ["serve"], artifacts.pathFor("ollama.log"), { - OLLAMA_HOST: `127.0.0.1:${OLLAMA_PORT}`, - }); + ollama = spawnLogged( + "ollama", + ["serve"], + artifacts, + "ollama", + { OLLAMA_HOST: `127.0.0.1:${OLLAMA_PORT}` }, + progress, + "ollama-serve", + ); await new Promise((resolve) => setTimeout(resolve, 3_000)); const tagsStatus = await curlStatus(host, `http://127.0.0.1:${OLLAMA_PORT}/api/tags`, { artifactName: "phase-2-ollama-tags-status", @@ -266,13 +311,22 @@ test("Ollama auth proxy enforces tokens, proxies inference, persists tokens, and progress.phase("start the tokenized Ollama auth proxy"); const proxyToken = token(); + artifacts.addRedactionValues([proxyToken]); fs.mkdirSync(path.dirname(tokenFile), { recursive: true }); await writeFile(tokenFile, `${proxyToken}\n`, { mode: 0o600 }); - proxy = spawnLogged("node", [PROXY_SCRIPT], artifacts.pathFor("ollama-auth-proxy.log"), { - OLLAMA_BACKEND_PORT: String(OLLAMA_PORT), - OLLAMA_PROXY_PORT: String(PROXY_PORT), - OLLAMA_PROXY_TOKEN: proxyToken, - }); + proxy = spawnLogged( + "node", + [PROXY_SCRIPT], + artifacts, + "ollama-auth-proxy", + { + OLLAMA_BACKEND_PORT: String(OLLAMA_PORT), + OLLAMA_PROXY_PORT: String(PROXY_PORT), + OLLAMA_PROXY_TOKEN: proxyToken, + }, + progress, + "ollama-auth-proxy", + ); await new Promise((resolve) => setTimeout(resolve, 2_000)); const correctAuth = `Bearer ${proxyToken}`; @@ -355,12 +409,15 @@ test("Ollama auth proxy enforces tokens, proxies inference, persists tokens, and proxy = spawnLogged( "node", [PROXY_SCRIPT], - artifacts.pathFor("ollama-auth-proxy-restarted.log"), + artifacts, + "ollama-auth-proxy-restarted", { OLLAMA_BACKEND_PORT: String(OLLAMA_PORT), OLLAMA_PROXY_PORT: String(PROXY_PORT), OLLAMA_PROXY_TOKEN: persistedToken, }, + progress, + "ollama-auth-proxy-restarted", ); await new Promise((resolve) => setTimeout(resolve, 2_000)); expect( @@ -440,6 +497,7 @@ test("Ollama auth proxy enforces tokens, proxies inference, persists tokens, and progress.phase("repair divergent token state"); const divergentToken = `divergent-${token()}`; + artifacts.addRedactionValues([divergentToken]); await writeFile(tokenFile, `${divergentToken}\n`, { mode: 0o600 }); const oldTokenModels = await curlStatus(host, `http://127.0.0.1:${PROXY_PORT}/v1/models`, { artifactName: "phase-9-old-token-models-status", @@ -456,12 +514,15 @@ test("Ollama auth proxy enforces tokens, proxies inference, persists tokens, and proxy = spawnLogged( "node", [PROXY_SCRIPT], - artifacts.pathFor("ollama-auth-proxy-divergent.log"), + artifacts, + "ollama-auth-proxy-divergent", { OLLAMA_BACKEND_PORT: String(OLLAMA_PORT), OLLAMA_PROXY_PORT: String(PROXY_PORT), OLLAMA_PROXY_TOKEN: divergentToken, }, + progress, + "ollama-auth-proxy-divergent", ); await new Promise((resolve) => setTimeout(resolve, 2_000)); expect( diff --git a/test/e2e/live/onboard-repair.test.ts b/test/e2e/live/onboard-repair.test.ts index 87fed23ab7..1f0d9cec54 100644 --- a/test/e2e/live/onboard-repair.test.ts +++ b/test/e2e/live/onboard-repair.test.ts @@ -177,6 +177,7 @@ test("onboard repair resumes missing sandbox and rejects conflicting resume inpu const fake = await startFakeOpenAiCompatibleServer({ host: "0.0.0.0", + progress, publicHost: "host.openshell.internal", }); cleanupRegistry.trackDisposable("close fake OpenAI-compatible endpoint", async () => diff --git a/test/e2e/live/onboard-resume.test.ts b/test/e2e/live/onboard-resume.test.ts index bb4fe761b8..92a9b14c81 100644 --- a/test/e2e/live/onboard-resume.test.ts +++ b/test/e2e/live/onboard-resume.test.ts @@ -230,6 +230,7 @@ test("onboard-resume: interrupted onboard then --resume can recreate with cached apiKey: FAKE_COMPATIBLE_AUTH_VALUE, host: "0.0.0.0", model: FAKE_COMPATIBLE_MODEL, + progress, publicHost: fakePublicHost, requireAuth: true, requireAuthModels: true, @@ -598,6 +599,7 @@ test("onboard-resume: interrupted onboard then --resume can recreate with cached host: "0.0.0.0", model: FAKE_COMPATIBLE_MODEL, port: fakePort, + progress, publicHost: fakePublicHost, requireAuth: true, requireAuthModels: true, diff --git a/test/e2e/live/openclaw-inference-switch.test.ts b/test/e2e/live/openclaw-inference-switch.test.ts index ca3d717c36..58cadedebc 100644 --- a/test/e2e/live/openclaw-inference-switch.test.ts +++ b/test/e2e/live/openclaw-inference-switch.test.ts @@ -945,6 +945,7 @@ test("openclaw-inference-switch: switches route and preserves live OpenClaw beha host: "0.0.0.0", model: MOCK_BASELINE_MODEL, publicHost: "host.openshell.internal", + progress, requireAuth: true, }) : undefined; diff --git a/test/e2e/live/openclaw-plugin-runtime-exdev.test.ts b/test/e2e/live/openclaw-plugin-runtime-exdev.test.ts index d0643a625d..84b08cfd1a 100644 --- a/test/e2e/live/openclaw-plugin-runtime-exdev.test.ts +++ b/test/e2e/live/openclaw-plugin-runtime-exdev.test.ts @@ -258,7 +258,11 @@ function assertPolicySourcesUnchanged(snapshot: PolicySourceSnapshot, phase: str } function runWrapper(wrapper: string, args: readonly string[]): string[] { - const result = spawnSync(wrapper, args, { encoding: "utf8" }); + const result = spawnSync(wrapper, args, { + encoding: "utf8", + killSignal: "SIGKILL", + timeout: 30_000, + }); expect(result.status, result.stderr).toBe(0); return result.stdout.trimEnd().split("\n"); } @@ -355,7 +359,7 @@ test("OpenShell wrapper injects only the reviewed tmpfs config into sandbox crea const duplicateConfig = spawnSync( wrapper.executable, ["sandbox", "create", "--driver-config-json", "{}"], - { encoding: "utf8" }, + { encoding: "utf8", killSignal: "SIGKILL", timeout: 30_000 }, ); expect(duplicateConfig.status).toBe(64); expect(duplicateConfig.stderr).toContain("refusing duplicate --driver-config-json"); @@ -1008,6 +1012,7 @@ test("a custom OpenClaw plugin survives restart, recreation, and rebuild without apiKey: "nemoclaw-exdev-dummy-key", host: "0.0.0.0", model: "nemoclaw-exdev-probe", + progress, publicHost: "host.openshell.internal", responseText: "ok", }); diff --git a/test/e2e/live/openshell-credential-generation-window.test.ts b/test/e2e/live/openshell-credential-generation-window.test.ts index 06fceac1ab..aadb984816 100644 --- a/test/e2e/live/openshell-credential-generation-window.test.ts +++ b/test/e2e/live/openshell-credential-generation-window.test.ts @@ -366,6 +366,7 @@ test("openshell-credential-generation-window", { const tunnel = await startPublicMcpHttpsTunnel({ cleanup, label: "credential-window MCP endpoint", + progress, server: fakeMcp, }); const hostAddress = await hostAddressForSandbox(host); diff --git a/test/e2e/live/openshell-gateway-auth-source-contract-helpers.ts b/test/e2e/live/openshell-gateway-auth-source-contract-helpers.ts index 2925535c09..0b49fef55c 100644 --- a/test/e2e/live/openshell-gateway-auth-source-contract-helpers.ts +++ b/test/e2e/live/openshell-gateway-auth-source-contract-helpers.ts @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { type ChildProcess, spawn, spawnSync } from "node:child_process"; +import { type ChildProcess, spawnSync } from "node:child_process"; import { createPrivateKey, sign as signPayload } from "node:crypto"; import fs from "node:fs"; import http2 from "node:http2"; @@ -12,33 +12,22 @@ import path from "node:path"; import type { buildDockerDriverGatewayLaunch as buildDockerDriverGatewayLaunchSource } from "../../../src/lib/onboard/docker-driver-gateway-launch"; import type { ensureDockerDriverGatewayLocalTlsBundle as ensureDockerDriverGatewayLocalTlsBundleSource } from "../../../src/lib/onboard/docker-driver-gateway-local-tls"; import { getDockerDriverGatewayLocalTlsBundle } from "../../../src/lib/onboard/docker-driver-gateway-local-tls"; +import { + assertOpenShellGatewayAuthArtifactsSafe, + enforceOpenShellGatewayAuthArtifactSafety, +} from "../../../tools/e2e/openshell-gateway-auth-artifact-safety.mts"; import type { ArtifactSink } from "../fixtures/artifacts.ts"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; import type { CleanupRegistry } from "../fixtures/cleanup.ts"; import type { HostCliClient } from "../fixtures/clients/index.ts"; import { expect } from "../fixtures/e2e-test.ts"; +import { spawnObservedChild } from "../fixtures/observed-child-process.ts"; import type { TestProgress } from "../fixtures/progress.ts"; const SANDBOX_JWT_SUBJECT_PREFIX = "spiffe://openshell/sandbox/"; const DOCKER_GRPC_PROBE_IMAGE = "node:22-trixie-slim@sha256:e6d9a389d34ff9678438af985c9913fbd1eb6ed36e80fea56644f4b4f6dd70ba"; -const FORBIDDEN_AUTH_ARTIFACT_CONTENT: Array<{ label: string; pattern: RegExp }> = [ - { label: "authorization header", pattern: /["']?authorization["']?\s*[:=]/i }, - { - label: "Bearer JWT", - pattern: /\bBearer\s+[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\b/, - }, - { label: "JWT signing-key path", pattern: /(?:^|[/\\])jwt[/\\]signing\.pem\b/i }, - { label: "JWT key-id path", pattern: /(?:^|[/\\])jwt[/\\]kid\b/i }, - { label: "gateway auth config path", pattern: /\bopenshell-gateway\.toml\b/i }, - { - label: "gateway JWT configuration", - pattern: /\[openshell\.gateway\.gateway_jwt\]/i, - }, - { label: "private key", pattern: /-----BEGIN [A-Z ]*PRIVATE KEY-----/ }, -]; - type SkipFn = (message?: string) => void; type ScenarioFixtures = { @@ -86,6 +75,7 @@ function run(command: string, args: string[], env: NodeJS.ProcessEnv = process.e const result = spawnSync(command, args, { encoding: "utf-8", env, + killSignal: "SIGKILL", stdio: ["ignore", "pipe", "pipe"], timeout: 60_000, }); @@ -100,41 +90,7 @@ function commandOutput(result: SpawnResult): string { return [result.stdout, result.stderr].filter(Boolean).join("\n"); } -export function assertOpenShellGatewayAuthArtifactsSafe(rootDir: string): void { - const root = path.resolve(rootDir); - const visit = (dir: string): void => { - for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { - const absolutePath = path.join(dir, entry.name); - const relativePath = path.relative(root, absolutePath).split(path.sep).join("/"); - if (entry.isDirectory()) { - visit(absolutePath); - continue; - } - if (!entry.isFile()) { - throw new Error( - `Unsafe OpenShell auth-contract artifact '${relativePath}': non-regular file`, - ); - } - if ( - /^(?:.*\/)?jwt\/(?:signing\.pem|kid)$|(?:^|\/)openshell-gateway\.toml$/i.test(relativePath) - ) { - throw new Error( - `Unsafe OpenShell auth-contract artifact '${relativePath}': sensitive auth file name`, - ); - } - const content = fs.readFileSync(absolutePath, "utf-8"); - const forbidden = FORBIDDEN_AUTH_ARTIFACT_CONTENT.find(({ pattern }) => - pattern.test(content), - ); - if (forbidden) { - throw new Error( - `Unsafe OpenShell auth-contract artifact '${relativePath}': ${forbidden.label}`, - ); - } - } - }; - visit(root); -} +export { assertOpenShellGatewayAuthArtifactsSafe }; export async function withOpenShellGatewayAuthArtifactSafety( rootDir: string, @@ -143,10 +99,17 @@ export async function withOpenShellGatewayAuthArtifactSafety( try { return await operation(); } finally { - assertOpenShellGatewayAuthArtifactsSafe(rootDir); + enforceOpenShellGatewayAuthArtifactSafety(rootDir); } } +export function registerSandboxJwtArtifactRedaction( + artifacts: ArtifactSink, + sandboxToken: string, +): void { + artifacts.addRedactionValues([sandboxToken]); +} + function resolveGatewayBin(): string | null { for (const candidate of [ process.env.OPENSHELL_GATEWAY_BIN, @@ -313,14 +276,15 @@ function callGrpc(options: { async function waitForGatewayReady(options: { gateway: ChildProcess; - logs: () => string; port: number; stateDir: string; }): Promise { const deadline = Date.now() + 60_000; while (Date.now() < deadline) { if (options.gateway.exitCode !== null) { - throw new Error(`openshell-gateway exited early:\n${options.logs()}`); + throw new Error( + "openshell-gateway exited before readiness; output is available in the redacted gateway artifact", + ); } const health = await callGrpc({ path: "/openshell.v1.OpenShell/Health", @@ -336,7 +300,9 @@ async function waitForGatewayReady(options: { } await delay(500); } - throw new Error(`openshell-gateway did not become ready:\n${options.logs()}`); + throw new Error( + "openshell-gateway did not become ready; output is available in the redacted gateway artifact", + ); } function parseTomlString(toml: string, key: string): string { @@ -722,9 +688,18 @@ async function runOpenShellGatewayAuthSourceContractScenarioUnchecked( }); let gatewayLog = ""; - const gateway = spawn(launch.command, launch.args, { - env: launch.env, - stdio: ["ignore", "pipe", "pipe"], + try { + progress.event("OpenShell auth contract gateway started"); + } catch { + // Progress diagnostics must never change gateway launch. + } + const gateway = spawnObservedChild(launch.command, launch.args, { + activityLabel: "command: openshell-auth-contract-gateway", + progress, + spawn: { + env: launch.env, + stdio: ["ignore", "pipe", "pipe"], + }, }); gateway.stdout?.on("data", (chunk: Buffer) => { gatewayLog += chunk.toString("utf-8"); @@ -732,87 +707,96 @@ async function runOpenShellGatewayAuthSourceContractScenarioUnchecked( gateway.stderr?.on("data", (chunk: Buffer) => { gatewayLog += chunk.toString("utf-8"); }); + gateway.once("close", () => { + try { + progress.event("OpenShell auth contract gateway stopped"); + } catch { + // Progress diagnostics must never change gateway cleanup. + } + }); cleanup.add("stop OpenShell auth contract gateway", () => stopGateway(gateway)); - await waitForGatewayReady({ - gateway, - logs: () => gatewayLog, - port, - stateDir, - }); + try { + await waitForGatewayReady({ + gateway, + port, + stateDir, + }); - progress.phase("probe unauthenticated and mTLS-only access"); - const noToken = noTokenContainerProbe(dockerBin, networkName, port, useHostNetwork); - await artifacts.writeJson("no-token-container-probe.json", noToken); - skipUnavailableProbeImage(noToken, skip); - expect(noTokenProbeWasRejected(noToken), commandOutput(noToken)).toBe(true); - - const configPath = String(launch.env.OPENSHELL_GATEWAY_CONFIG || ""); - expect(configPath).toBe(path.join(stateDir, "openshell-gateway.toml")); - const sandboxId = "sandbox-auth-contract"; - const mtlsOnlyContainerCall = sandboxTokenContainerProbe({ - dockerBin, - networkName, - payload: getSandboxConfigRequest(sandboxId), - port, - stateDir, - useHostNetwork, - }); - await artifacts.writeJson("mtls-only-container-probe.json", mtlsOnlyContainerCall); - skipUnavailableProbeImage(mtlsOnlyContainerCall, skip); - expect( - probeDidNotReturnSandboxConfig(mtlsOnlyContainerCall), - commandOutput(mtlsOnlyContainerCall), - ).toBe(true); - - progress.phase("probe sandbox JWT authorization boundaries"); - const sandboxToken = mintSandboxJwt({ configPath, sandboxId }); - const sandboxCall = await callGrpc({ - authorization: `Bearer ${sandboxToken}`, - path: "/openshell.v1.OpenShell/GetSandboxConfig", - payload: getSandboxConfigRequest(sandboxId), - port, - stateDir, - }); - await artifacts.writeJson("sandbox-jwt-probe.json", sandboxCall); - expect(sandboxCall.httpStatus, JSON.stringify(sandboxCall)).toBe(200); - expect(sandboxCall.grpcStatus, JSON.stringify(sandboxCall)).toBeDefined(); - expect(["7", "16"]).not.toContain(sandboxCall.grpcStatus); - - const sandboxContainerCall = sandboxTokenContainerProbe({ - authorization: `Bearer ${sandboxToken}`, - dockerBin, - networkName, - payload: getSandboxConfigRequest(sandboxId), - port, - stateDir, - useHostNetwork, - }); - await artifacts.writeJson("sandbox-jwt-container-probe.json", sandboxContainerCall); - skipUnavailableProbeImage(sandboxContainerCall, skip); - expect(sandboxContainerCall.status, commandOutput(sandboxContainerCall)).toBe(0); - const sandboxContainerResult = JSON.parse(sandboxContainerCall.stdout.trim()) as GrpcResult; - expect(sandboxContainerResult.httpStatus, JSON.stringify(sandboxContainerResult)).toBe(200); - expect(sandboxContainerResult.grpcStatus, JSON.stringify(sandboxContainerResult)).toBeDefined(); - expect(["7", "16"]).not.toContain(sandboxContainerResult.grpcStatus); - - const crossSandboxContainerCall = sandboxTokenContainerProbe({ - authorization: `Bearer ${sandboxToken}`, - dockerBin, - networkName, - payload: getSandboxConfigRequest("sandbox-auth-contract-other"), - port, - stateDir, - useHostNetwork, - }); - await artifacts.writeJson("cross-sandbox-jwt-container-probe.json", crossSandboxContainerCall); - skipUnavailableProbeImage(crossSandboxContainerCall, skip); - expect( - probeDidNotReturnSandboxConfig(crossSandboxContainerCall), - commandOutput(crossSandboxContainerCall), - ).toBe(true); - - await artifacts.writeText("openshell-gateway.log", gatewayLog); + progress.phase("probe unauthenticated and mTLS-only access"); + const noToken = noTokenContainerProbe(dockerBin, networkName, port, useHostNetwork); + await artifacts.writeJson("no-token-container-probe.json", noToken); + skipUnavailableProbeImage(noToken, skip); + expect(noTokenProbeWasRejected(noToken), commandOutput(noToken)).toBe(true); + + const configPath = String(launch.env.OPENSHELL_GATEWAY_CONFIG || ""); + expect(configPath).toBe(path.join(stateDir, "openshell-gateway.toml")); + const sandboxId = "sandbox-auth-contract"; + const mtlsOnlyContainerCall = sandboxTokenContainerProbe({ + dockerBin, + networkName, + payload: getSandboxConfigRequest(sandboxId), + port, + stateDir, + useHostNetwork, + }); + await artifacts.writeJson("mtls-only-container-probe.json", mtlsOnlyContainerCall); + skipUnavailableProbeImage(mtlsOnlyContainerCall, skip); + expect( + probeDidNotReturnSandboxConfig(mtlsOnlyContainerCall), + commandOutput(mtlsOnlyContainerCall), + ).toBe(true); + + progress.phase("probe sandbox JWT authorization boundaries"); + const sandboxToken = mintSandboxJwt({ configPath, sandboxId }); + registerSandboxJwtArtifactRedaction(artifacts, sandboxToken); + const sandboxCall = await callGrpc({ + authorization: `Bearer ${sandboxToken}`, + path: "/openshell.v1.OpenShell/GetSandboxConfig", + payload: getSandboxConfigRequest(sandboxId), + port, + stateDir, + }); + await artifacts.writeJson("sandbox-jwt-probe.json", sandboxCall); + expect(sandboxCall.httpStatus, JSON.stringify(sandboxCall)).toBe(200); + expect(sandboxCall.grpcStatus, JSON.stringify(sandboxCall)).toBeDefined(); + expect(["7", "16"]).not.toContain(sandboxCall.grpcStatus); + + const sandboxContainerCall = sandboxTokenContainerProbe({ + authorization: `Bearer ${sandboxToken}`, + dockerBin, + networkName, + payload: getSandboxConfigRequest(sandboxId), + port, + stateDir, + useHostNetwork, + }); + await artifacts.writeJson("sandbox-jwt-container-probe.json", sandboxContainerCall); + skipUnavailableProbeImage(sandboxContainerCall, skip); + expect(sandboxContainerCall.status, commandOutput(sandboxContainerCall)).toBe(0); + const sandboxContainerResult = JSON.parse(sandboxContainerCall.stdout.trim()) as GrpcResult; + expect(sandboxContainerResult.httpStatus, JSON.stringify(sandboxContainerResult)).toBe(200); + expect(sandboxContainerResult.grpcStatus, JSON.stringify(sandboxContainerResult)).toBeDefined(); + expect(["7", "16"]).not.toContain(sandboxContainerResult.grpcStatus); + + const crossSandboxContainerCall = sandboxTokenContainerProbe({ + authorization: `Bearer ${sandboxToken}`, + dockerBin, + networkName, + payload: getSandboxConfigRequest("sandbox-auth-contract-other"), + port, + stateDir, + useHostNetwork, + }); + await artifacts.writeJson("cross-sandbox-jwt-container-probe.json", crossSandboxContainerCall); + skipUnavailableProbeImage(crossSandboxContainerCall, skip); + expect( + probeDidNotReturnSandboxConfig(crossSandboxContainerCall), + commandOutput(crossSandboxContainerCall), + ).toBe(true); + } finally { + await artifacts.writeText("openshell-gateway.log", gatewayLog); + } } export async function runOpenShellGatewayAuthSourceContractScenario( diff --git a/test/e2e/live/openshell-gateway-upgrade.test.ts b/test/e2e/live/openshell-gateway-upgrade.test.ts index cd5d31d59d..a8ccfe7a0b 100644 --- a/test/e2e/live/openshell-gateway-upgrade.test.ts +++ b/test/e2e/live/openshell-gateway-upgrade.test.ts @@ -1074,6 +1074,8 @@ function runMacInstallerProbe( PATH: `${fakeBin}:/usr/bin:/bin`, }, encoding: "utf8", + killSignal: "SIGKILL", + timeout: 60_000, }); fs.mkdirSync(artifacts.pathFor(`macos-${name}`), { recursive: true }); fs.writeFileSync(artifacts.pathFor(`macos-${name}/stdout.txt`), result.stdout ?? "", "utf8"); @@ -1184,6 +1186,7 @@ runLinuxOpenShellGatewayUpgrade( apiKey: "dummy", host: "0.0.0.0", model: "test-model", + progress, publicHost: "host.openshell.internal", requireAuth: OPENCLAW_STATE_UPGRADE_PROOF, requireAuthModels: OPENCLAW_STATE_UPGRADE_PROOF, diff --git a/test/e2e/live/openshell-version-pin.test.ts b/test/e2e/live/openshell-version-pin.test.ts index 99f0d61137..6b996e0295 100644 --- a/test/e2e/live/openshell-version-pin.test.ts +++ b/test/e2e/live/openshell-version-pin.test.ts @@ -76,6 +76,8 @@ process.stdout.write(JSON.stringify({ cwd: REPO_ROOT, encoding: "utf8", env: { ...process.env, PATH: `${binDir}:${process.env.PATH ?? ""}` }, + killSignal: "SIGKILL", + timeout: 60_000, }, ); progress.phase("confirm the shipping version wins range selection"); @@ -372,6 +374,8 @@ async function runVersionPinTarget( PATH: `${fakeBin}:/usr/bin:/bin`, }, encoding: "utf8", + killSignal: "SIGKILL", + timeout: 60_000, }); // Persist the install transcript so failures can be diagnosed without @@ -409,6 +413,8 @@ async function runVersionPinTarget( // there and it is writable) was overwritten with the pinned 0.0.85 build. const replacedVersion = spawnSync(path.join(fakeBin, "openshell"), ["--version"], { encoding: "utf8", + killSignal: "SIGKILL", + timeout: 30_000, }); expect(replacedVersion.status).toBe(0); expect(replacedVersion.stdout).toContain("0.0.85"); diff --git a/test/e2e/live/rebuild-hermes-phases.ts b/test/e2e/live/rebuild-hermes-phases.ts index 19fcce952e..b97f2ecc98 100644 --- a/test/e2e/live/rebuild-hermes-phases.ts +++ b/test/e2e/live/rebuild-hermes-phases.ts @@ -4,8 +4,8 @@ export const REBUILD_HERMES_PHASES = [ "confirm Docker and prepare Hermes rebuild resources", "onboard the current Hermes sandbox", - "pull and validate the old Hermes base fixture", - "create the old Hermes sandbox", + "pull and verify the historical Hermes base fixture", + "create the historical Hermes sandbox", "seed persistent Hermes state and registry metadata", "prepare the current-base rebuild condition", "rebuild the Hermes sandbox", diff --git a/test/e2e/live/rebuild-hermes.test.ts b/test/e2e/live/rebuild-hermes.test.ts index 3792f144e5..786f888966 100644 --- a/test/e2e/live/rebuild-hermes.test.ts +++ b/test/e2e/live/rebuild-hermes.test.ts @@ -854,7 +854,7 @@ test(STALE_BASE_REBUILD timeoutMs: OPENSHELL_TIMEOUT_MS, }); - progress.phase("pull and validate the old Hermes base fixture"); + progress.phase("pull and verify the historical Hermes base fixture"); const pullOldBase = await host.command( "docker", ["pull", REBUILD_HERMES_OLD_BASE_FIXTURE.imageRef], @@ -999,7 +999,7 @@ test(STALE_BASE_REBUILD ); expectExitZero(provider, "OpenShell Discord provider create/update"); - progress.phase("create the old Hermes sandbox"); + progress.phase("create the historical Hermes sandbox"); const createOldSandbox = await host.command( "openshell", [ @@ -1077,7 +1077,6 @@ test(STALE_BASE_REBUILD ); expectExitZero(seededKanban, "verify historical Hermes kanban seed before rebuild"); expect(resultText(seededKanban)).toContain(KANBAN_TASK_TITLE); - const writeMarker = await host.command( "openshell", [ diff --git a/test/e2e/live/runtime-overrides.test.ts b/test/e2e/live/runtime-overrides.test.ts index a3e39f875b..aae2ecee09 100644 --- a/test/e2e/live/runtime-overrides.test.ts +++ b/test/e2e/live/runtime-overrides.test.ts @@ -1,11 +1,13 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { spawnSync } from "node:child_process"; import path from "node:path"; import { testTimeoutOptions } from "../../helpers/timeouts"; +import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import type { HostCliClient } from "../fixtures/clients/host.ts"; import { expect, test } from "../fixtures/e2e-test.ts"; import { REPO_ROOT } from "../fixtures/paths.ts"; +import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; // Docker-image/entrypoint boundary: build the NemoClaw sandbox image, start // short-lived containers through the real ENTRYPOINT, then read the patched @@ -14,6 +16,8 @@ import { REPO_ROOT } from "../fixtures/paths.ts"; const TEST_TIMEOUT_MS = 45 * 60 * 1000; const DOCKER_BUFFER_BYTES = 20 * 1024 * 1024; const DOCKER_REQUIRED_MESSAGE = "Docker is required for runtime override coverage"; +const DOCKER_COMMAND_TIMEOUT_MS = 5 * 60_000; +const DOCKER_BUILD_TIMEOUT_MS = 30 * 60_000; type CommandResult = { status: number | null; @@ -52,25 +56,46 @@ const MANAGED_INFERENCE_SAFEGUARD_COMPACTION = { truncateAfterCompaction: true, }; -function commandResult(result: ReturnType): CommandResult { +type ObservableCommandRunner = ( + command: string, + args: string[], + artifactName: string, + timeoutMs?: number, +) => Promise; + +function commandResult(result: ShellProbeResult): CommandResult { return { - status: result.status, - stdout: - typeof result.stdout === "string" ? result.stdout : (result.stdout?.toString("utf8") ?? ""), - stderr: - typeof result.stderr === "string" ? result.stderr : (result.stderr?.toString("utf8") ?? ""), - error: result.error, + status: result.exitCode, + stdout: result.stdout, + stderr: result.stderr, }; } -function run(command: string, args: string[]): CommandResult { - return commandResult( - spawnSync(command, args, { - cwd: REPO_ROOT, - encoding: "utf8", - maxBuffer: DOCKER_BUFFER_BYTES, - }), - ); +async function runObserved( + host: HostCliClient, + command: string, + args: string[], + artifactName: string, + timeoutMs = DOCKER_COMMAND_TIMEOUT_MS, +): Promise { + try { + return commandResult( + await host.command(command, args, { + artifactName: `runtime-overrides-${artifactName}`, + captureLimitBytes: DOCKER_BUFFER_BYTES, + env: buildAvailabilityProbeEnv(), + timeoutMs, + cwd: REPO_ROOT, + }), + ); + } catch (error) { + return { + status: null, + stdout: "", + stderr: "", + error: error instanceof Error ? error : new Error(String(error)), + }; + } } function spawnResultText(result: CommandResult): string { @@ -141,24 +166,26 @@ function dockerRunArgs(image: string, env: Record, script: strin ]; } -function runContainer( +async function runContainer( + run: ObservableCommandRunner, dockerLog: string[], image: string, label: string, env: Record, script: string, -): CommandResult { - const result = run("docker", dockerRunArgs(image, env, script)); +): Promise { + const result = await run("docker", dockerRunArgs(image, env, script), label); dockerLog.push(formatLog(label, result)); return result; } -function captureConfig( +async function captureConfig( + run: ObservableCommandRunner, dockerLog: string[], image: string, label: string, env: Record = {}, -): OpenClawConfig { +): Promise { let lastResult: CommandResult | undefined; let lastError: Error | undefined; // Preserve the former shell test's Docker/ENTRYPOINT stdout tolerance: very short @@ -166,7 +193,8 @@ function captureConfig( // though the JSON is written to fd3. Keep this local retry until the startup // capture path no longer uses tee for container stdout/stderr fanout. for (let attempt = 1; attempt <= 3; attempt += 1) { - const result = runContainer( + const result = await runContainer( + run, dockerLog, image, `${label} config capture attempt ${attempt}`, @@ -188,15 +216,17 @@ function captureConfig( ); } -function runConfigHashCheck( +async function runConfigHashCheck( + run: ObservableCommandRunner, dockerLog: string[], image: string, label: string, env: Record = {}, -): string { +): Promise { // Keep the one-shot container alive long enough for its tiny fd3 marker to // drain through Docker attach; the JSON capture above is naturally larger. - const result = runContainer( + const result = await runContainer( + run, dockerLog, image, `${label} config hash check`, @@ -207,8 +237,13 @@ function runConfigHashCheck( return result.stdout.trim(); } -function assertManagedInferenceCompactionRuntime(dockerLog: string[], image: string): void { - const result = runContainer( +async function assertManagedInferenceCompactionRuntime( + run: ObservableCommandRunner, + dockerLog: string[], + image: string, +): Promise { + const result = await runContainer( + run, dockerLog, image, "managed inference compaction runtime validation", @@ -233,38 +268,48 @@ sleep 0.1`, expect(proof.compaction).toEqual(MANAGED_INFERENCE_SAFEGUARD_COMPACTION); } -function runOverrideStderr( +async function runOverrideStderr( + run: ObservableCommandRunner, dockerLog: string[], image: string, label: string, env: Record, -): string { - const result = runContainer(dockerLog, image, label, env, "true"); +): Promise { + const result = await runContainer(run, dockerLog, image, label, env, "true"); return result.stderr; } -function dockerAvailable(): CommandResult { - return run("docker", ["info"]); +function dockerAvailable(run: ObservableCommandRunner): Promise { + return run("docker", ["info"], "docker-info", 30_000); } -function buildImage(dockerLog: string[], image: string): void { - const inspect = run("docker", ["image", "inspect", image]); +async function buildImage( + run: ObservableCommandRunner, + dockerLog: string[], + image: string, +): Promise { + const inspect = await run("docker", ["image", "inspect", image], `inspect-${image}`, 30_000); dockerLog.push(formatLog(`inspect ${image}`, inspect)); if (inspect.status === 0) return; - const build = run("docker", [ - "build", - "-t", - image, - "-f", - path.join(REPO_ROOT, "Dockerfile"), - "--build-arg", - "NEMOCLAW_DISABLE_DEVICE_AUTH=1", - "--build-arg", - `NEMOCLAW_BUILD_ID=${Date.now()}`, - "--quiet", - REPO_ROOT, - ]); + const build = await run( + "docker", + [ + "build", + "-t", + image, + "-f", + path.join(REPO_ROOT, "Dockerfile"), + "--build-arg", + "NEMOCLAW_DISABLE_DEVICE_AUTH=1", + "--build-arg", + `NEMOCLAW_BUILD_ID=${Date.now()}`, + "--quiet", + REPO_ROOT, + ], + `build-${image}`, + DOCKER_BUILD_TIMEOUT_MS, + ); dockerLog.push(formatLog(`build ${image}`, build)); expect(build.status, spawnResultText(build)).toBe(0); } @@ -286,10 +331,12 @@ test( ], }, }, - async ({ artifacts, progress, secrets, skip }) => { + async ({ artifacts, host, progress, secrets, skip }) => { const dockerLog: string[] = []; const image = process.env.NEMOCLAW_TEST_IMAGE ?? `nemoclaw-runtime-overrides-${process.pid}`; const cleanupImage = process.env.NEMOCLAW_TEST_IMAGE === undefined; + const run: ObservableCommandRunner = (command, args, artifactName, timeoutMs) => + runObserved(host, command, args, artifactName, timeoutMs); try { await artifacts.target.declare({ @@ -306,7 +353,7 @@ test( ], }); - const docker = dockerAvailable(); + const docker = await dockerAvailable(run); dockerLog.push(formatLog("docker info", docker)); if (docker.status !== 0) { await artifacts.target.complete({ @@ -320,67 +367,67 @@ test( skip(DOCKER_REQUIRED_MESSAGE); } - buildImage(dockerLog, image); + await buildImage(run, dockerLog, image); progress.phase("capture the baseline OpenClaw config"); - const baseline = captureConfig(dockerLog, image, "baseline"); + const baseline = await captureConfig(run, dockerLog, image, "baseline"); const baselineModel = primaryModel(baseline); const baselineFirstModel = firstProviderModel(baseline); const baselineContextWindow = baselineFirstModel.contextWindow; const baselineOriginCount = allowedOrigins(baseline).length; - assertManagedInferenceCompactionRuntime(dockerLog, image); - expect(runConfigHashCheck(dockerLog, image, "baseline")).toBe("OK"); + await assertManagedInferenceCompactionRuntime(run, dockerLog, image); + expect(await runConfigHashCheck(run, dockerLog, image, "baseline")).toBe("OK"); progress.phase("apply valid runtime overrides"); const overrideModel = "anthropic/claude-sonnet-4-6"; - const modelOverride = captureConfig(dockerLog, image, "model override", { + const modelOverride = await captureConfig(run, dockerLog, image, "model override", { NEMOCLAW_MODEL_OVERRIDE: overrideModel, }); expect(primaryModel(modelOverride)).toBe(overrideModel); expect( - runConfigHashCheck(dockerLog, image, "model override", { + await runConfigHashCheck(run, dockerLog, image, "model override", { NEMOCLAW_MODEL_OVERRIDE: overrideModel, }), ).toBe("OK"); - const apiOverride = captureConfig(dockerLog, image, "inference API override", { + const apiOverride = await captureConfig(run, dockerLog, image, "inference API override", { NEMOCLAW_INFERENCE_API_OVERRIDE: "anthropic-messages", }); expect(firstProvider(apiOverride).api).toBe("anthropic-messages"); expect( - runConfigHashCheck(dockerLog, image, "inference API override", { + await runConfigHashCheck(run, dockerLog, image, "inference API override", { NEMOCLAW_INFERENCE_API_OVERRIDE: "anthropic-messages", }), ).toBe("OK"); - const contextOverride = captureConfig(dockerLog, image, "context window override", { + const contextOverride = await captureConfig(run, dockerLog, image, "context window override", { NEMOCLAW_MODEL_OVERRIDE: overrideModel, NEMOCLAW_CONTEXT_WINDOW: "32768", }); expect(firstProviderModel(contextOverride).contextWindow).toBe(32768); - const maxTokensOverride = captureConfig(dockerLog, image, "max tokens override", { + const maxTokensOverride = await captureConfig(run, dockerLog, image, "max tokens override", { NEMOCLAW_MODEL_OVERRIDE: overrideModel, NEMOCLAW_MAX_TOKENS: "16384", }); expect(firstProviderModel(maxTokensOverride).maxTokens).toBe(16384); - const reasoningOverride = captureConfig(dockerLog, image, "reasoning override", { + const reasoningOverride = await captureConfig(run, dockerLog, image, "reasoning override", { NEMOCLAW_MODEL_OVERRIDE: overrideModel, NEMOCLAW_REASONING: "true", }); expect(firstProviderModel(reasoningOverride).reasoning).toBe(true); const corsOrigin = "https://custom.example.com:9999"; - const corsOverride = captureConfig(dockerLog, image, "CORS origin override", { + const corsOverride = await captureConfig(run, dockerLog, image, "CORS origin override", { NEMOCLAW_CORS_ORIGIN: corsOrigin, }); expect(allowedOrigins(corsOverride)).toContain(corsOrigin); expect(allowedOrigins(corsOverride).length).toBeGreaterThan(baselineOriginCount); progress.phase("exercise the combined override transaction"); - const combined = captureConfig(dockerLog, image, "combined overrides", { + const combined = await captureConfig(run, dockerLog, image, "combined overrides", { NEMOCLAW_MODEL_OVERRIDE: "nvidia/llama-3.3-nemotron-super-49b-v1.5", NEMOCLAW_CONTEXT_WINDOW: "65536", NEMOCLAW_MAX_TOKENS: "8192", @@ -397,42 +444,42 @@ test( progress.phase("reject invalid override values"); expect( - runOverrideStderr(dockerLog, image, "invalid model override", { + await runOverrideStderr(run, dockerLog, image, "invalid model override", { NEMOCLAW_MODEL_OVERRIDE: "bad\u0001model", }), ).toContain("control characters"); expect( - runOverrideStderr(dockerLog, image, "invalid context window", { + await runOverrideStderr(run, dockerLog, image, "invalid context window", { NEMOCLAW_MODEL_OVERRIDE: "test", NEMOCLAW_CONTEXT_WINDOW: "notanumber", }), ).toContain("must be a positive integer"); expect( - runOverrideStderr(dockerLog, image, "invalid max tokens", { + await runOverrideStderr(run, dockerLog, image, "invalid max tokens", { NEMOCLAW_MODEL_OVERRIDE: "test", NEMOCLAW_MAX_TOKENS: "abc", }), ).toContain("must be a positive integer"); expect( - runOverrideStderr(dockerLog, image, "invalid reasoning", { + await runOverrideStderr(run, dockerLog, image, "invalid reasoning", { NEMOCLAW_MODEL_OVERRIDE: "test", NEMOCLAW_REASONING: "maybe", }), ).toContain('must be "true" or "false"'); expect( - runOverrideStderr(dockerLog, image, "invalid CORS origin", { + await runOverrideStderr(run, dockerLog, image, "invalid CORS origin", { NEMOCLAW_CORS_ORIGIN: "ftp://evil.com", }), ).toContain("must start with http"); expect( - runOverrideStderr(dockerLog, image, "invalid inference API", { + await runOverrideStderr(run, dockerLog, image, "invalid inference API", { NEMOCLAW_MODEL_OVERRIDE: "test", NEMOCLAW_INFERENCE_API_OVERRIDE: "graphql", }), ).toContain("openai-completions"); progress.phase("confirm rejected overrides preserve the baseline"); - const rejected = captureConfig(dockerLog, image, "rejected override", { + const rejected = await captureConfig(run, dockerLog, image, "rejected override", { NEMOCLAW_MODEL_OVERRIDE: "test", NEMOCLAW_CONTEXT_WINDOW: "notanumber", }); @@ -447,7 +494,11 @@ test( }); } finally { if (cleanupImage) { - const cleanup = run("docker", ["image", "rm", "-f", image]); + const cleanup = await run( + "docker", + ["image", "rm", "-f", image], + `cleanup-${image}`, + ); dockerLog.push(formatLog(`cleanup ${image}`, cleanup)); } await artifacts.writeText("docker.log", `${secrets.redact(dockerLog.join("\n\n"))}\n`); diff --git a/test/e2e/live/snapshot-commands.test.ts b/test/e2e/live/snapshot-commands.test.ts index 121dc16b2f..81bc3a36ec 100644 --- a/test/e2e/live/snapshot-commands.test.ts +++ b/test/e2e/live/snapshot-commands.test.ts @@ -163,6 +163,7 @@ test("snapshot commands preserve create/list/latest restore/targeted restore/no- apiKey: INFERENCE_API_KEY, host: "0.0.0.0", model: INFERENCE_MODEL, + progress, publicHost: "host.openshell.internal", requireAuth: true, requireAuthModels: true, diff --git a/test/e2e/live/token-rotation.test.ts b/test/e2e/live/token-rotation.test.ts index 785dbbd113..d2b56685ad 100644 --- a/test/e2e/live/token-rotation.test.ts +++ b/test/e2e/live/token-rotation.test.ts @@ -305,6 +305,7 @@ test( const fakeOpenAI = await startFakeOpenAiCompatibleServer({ chatContent: "OK", host: "0.0.0.0", + progress, publicHost: "host.openshell.internal", responseText: "OK", }); diff --git a/test/e2e/mock-parity.json b/test/e2e/mock-parity.json index 948bb5fd70..0feae97195 100644 --- a/test/e2e/mock-parity.json +++ b/test/e2e/mock-parity.json @@ -40,6 +40,7 @@ "test/e2e/support/e2e-progress-fixture.test.ts", "test/e2e/support/e2e-progress-outcome.test.ts", "test/e2e/support/e2e-semantic-phase-check.test.ts", + "test/e2e/support/workflow-e2e-progress.test.ts", "test/e2e/support/prepare-e2e-workflow-boundary.test.ts", "test/installer-hash-check.test.ts", "test/runner.test.ts" @@ -53,6 +54,7 @@ "test/e2e/support/e2e-progress-fixture.test.ts", "test/e2e/support/e2e-progress-outcome.test.ts", "test/e2e/support/e2e-semantic-phase-check.test.ts", + "test/e2e/support/workflow-e2e-progress.test.ts", "test/e2e/support/prepare-e2e-workflow-boundary.test.ts", "test/installer-hash-check.test.ts", "test/runner.test.ts" @@ -156,6 +158,7 @@ { "live": "test/e2e/live/agent-turn-latency.test.ts", "fast": [ + "test/e2e/support/agent-turn-latency-progress.test.ts", "test/e2e/support/e2e-cleanup-resources.test.ts", "test/e2e/support/e2e-clients.test.ts" ] @@ -163,6 +166,7 @@ { "live": "test/e2e/live/bedrock-runtime-compatible-anthropic.test.ts", "fast": [ + "test/e2e/support/bedrock-runtime-compatible-anthropic-progress.test.ts", "test/e2e/support/e2e-cleanup-resources.test.ts", "test/e2e/support/e2e-clients.test.ts" ] @@ -522,7 +526,8 @@ "fast": [ "test/e2e/support/e2e-progress-fixture.test.ts", "test/e2e/support/e2e-progress-outcome.test.ts", - "test/e2e/support/e2e-semantic-phase-check.test.ts" + "test/e2e/support/e2e-semantic-phase-check.test.ts", + "test/e2e/support/workflow-e2e-progress.test.ts" ] }, { @@ -530,7 +535,8 @@ "fast": [ "test/e2e/support/e2e-progress-fixture.test.ts", "test/e2e/support/e2e-progress-outcome.test.ts", - "test/e2e/support/e2e-semantic-phase-check.test.ts" + "test/e2e/support/e2e-semantic-phase-check.test.ts", + "test/e2e/support/workflow-e2e-progress.test.ts" ] }, { @@ -538,7 +544,8 @@ "fast": [ "test/e2e/support/e2e-progress-fixture.test.ts", "test/e2e/support/e2e-progress-outcome.test.ts", - "test/e2e/support/e2e-semantic-phase-check.test.ts" + "test/e2e/support/e2e-semantic-phase-check.test.ts", + "test/e2e/support/workflow-e2e-progress.test.ts" ] }, { @@ -546,7 +553,8 @@ "fast": [ "test/e2e/support/e2e-progress-fixture.test.ts", "test/e2e/support/e2e-progress-outcome.test.ts", - "test/e2e/support/e2e-semantic-phase-check.test.ts" + "test/e2e/support/e2e-semantic-phase-check.test.ts", + "test/e2e/support/workflow-e2e-progress.test.ts" ] }, { @@ -554,7 +562,8 @@ "fast": [ "test/e2e/support/e2e-progress-fixture.test.ts", "test/e2e/support/e2e-progress-outcome.test.ts", - "test/e2e/support/e2e-semantic-phase-check.test.ts" + "test/e2e/support/e2e-semantic-phase-check.test.ts", + "test/e2e/support/workflow-e2e-progress.test.ts" ] }, { @@ -562,7 +571,8 @@ "fast": [ "test/e2e/support/e2e-progress-fixture.test.ts", "test/e2e/support/e2e-progress-outcome.test.ts", - "test/e2e/support/e2e-semantic-phase-check.test.ts" + "test/e2e/support/e2e-semantic-phase-check.test.ts", + "test/e2e/support/workflow-e2e-progress.test.ts" ] }, { @@ -570,7 +580,8 @@ "fast": [ "test/e2e/support/e2e-progress-fixture.test.ts", "test/e2e/support/e2e-progress-outcome.test.ts", - "test/e2e/support/e2e-semantic-phase-check.test.ts" + "test/e2e/support/e2e-semantic-phase-check.test.ts", + "test/e2e/support/workflow-e2e-progress.test.ts" ] }, { @@ -578,7 +589,8 @@ "fast": [ "test/e2e/support/e2e-progress-fixture.test.ts", "test/e2e/support/e2e-progress-outcome.test.ts", - "test/e2e/support/e2e-semantic-phase-check.test.ts" + "test/e2e/support/e2e-semantic-phase-check.test.ts", + "test/e2e/support/workflow-e2e-progress.test.ts" ] }, { @@ -586,7 +598,8 @@ "fast": [ "test/e2e/support/e2e-progress-fixture.test.ts", "test/e2e/support/e2e-progress-outcome.test.ts", - "test/e2e/support/e2e-semantic-phase-check.test.ts" + "test/e2e/support/e2e-semantic-phase-check.test.ts", + "test/e2e/support/workflow-e2e-progress.test.ts" ] }, { @@ -594,7 +607,8 @@ "fast": [ "test/e2e/support/e2e-progress-fixture.test.ts", "test/e2e/support/e2e-progress-outcome.test.ts", - "test/e2e/support/e2e-semantic-phase-check.test.ts" + "test/e2e/support/e2e-semantic-phase-check.test.ts", + "test/e2e/support/workflow-e2e-progress.test.ts" ] }, { @@ -602,7 +616,8 @@ "fast": [ "test/e2e/support/e2e-progress-fixture.test.ts", "test/e2e/support/e2e-progress-outcome.test.ts", - "test/e2e/support/e2e-semantic-phase-check.test.ts" + "test/e2e/support/e2e-semantic-phase-check.test.ts", + "test/e2e/support/workflow-e2e-progress.test.ts" ] }, { @@ -610,7 +625,8 @@ "fast": [ "test/e2e/support/e2e-progress-fixture.test.ts", "test/e2e/support/e2e-progress-outcome.test.ts", - "test/e2e/support/e2e-semantic-phase-check.test.ts" + "test/e2e/support/e2e-semantic-phase-check.test.ts", + "test/e2e/support/workflow-e2e-progress.test.ts" ] } ] diff --git a/test/e2e/risk-signal-reporter.ts b/test/e2e/risk-signal-reporter.ts index 3465426e0c..c823930a1a 100644 --- a/test/e2e/risk-signal-reporter.ts +++ b/test/e2e/risk-signal-reporter.ts @@ -39,7 +39,9 @@ function checkedOutSha(workspace: string): string { return execFileSync("git", ["rev-parse", "--verify", "HEAD"], { cwd: workspace, encoding: "utf8", + killSignal: "SIGKILL", stdio: ["ignore", "pipe", "pipe"], + timeout: 5_000, }).trim(); } diff --git a/test/e2e/support/agent-turn-latency-progress.test.ts b/test/e2e/support/agent-turn-latency-progress.test.ts index dd891e85f7..9f883f379e 100644 --- a/test/e2e/support/agent-turn-latency-progress.test.ts +++ b/test/e2e/support/agent-turn-latency-progress.test.ts @@ -4,13 +4,18 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import type { HostCliClient } from "../fixtures/clients/host.ts"; +import type { SandboxClient } from "../fixtures/clients/sandbox.ts"; import { startTestProgress, type TestProgressOptions, validateE2EPhasePlan, } from "../fixtures/progress.ts"; import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; -import { installSandbox } from "../live/agent-turn-latency-helpers.ts"; +import { + bestEffortPreclean, + cleanupTurnSandboxes, + installSandbox, +} from "../live/agent-turn-latency-helpers.ts"; function progressHarness() { const state = { @@ -57,11 +62,12 @@ function successfulProbe(): ShellProbeResult { }; } -function failedProbe(stderr: string): ShellProbeResult { +function failedProbe(stderr: string, timedOut = false): ShellProbeResult { return { ...successfulProbe(), exitCode: 1, stderr, + timedOut, }; } @@ -91,10 +97,11 @@ describe("live test progress", () => { expect(state.clearCalls).toBe(2); expect(state.scheduledDelays).toEqual([300_000, 600_000, 300_000]); expect(state.lines).toEqual([ - "[e2e phase 1/2] install OpenClaw sandbox", - "[e2e phase 1/2] still running: install OpenClaw sandbox (phase 5m; child output 4m ago; activity command: install-openclaw; rss 0.5 GiB; memory free 8.0 GiB/16.0 GiB; disk free 6.0 GiB; load 2.50)", - "[e2e phase 1/2] install OpenClaw sandbox — passed in 6m; next 2/2: install Hermes sandbox", - "[e2e phase 2/2] install Hermes sandbox — passed in 0s", + '[e2e target="unassigned" scenario="agent-turn-latency"] [phase 1/2] started: install OpenClaw sandbox (total 0s; phase 0s)', + '[e2e target="unassigned" scenario="agent-turn-latency"] [phase 1/2] still running: install OpenClaw sandbox (total 5m; phase 5m; child output 4m ago; activity command: install-openclaw; rss 0.5 GiB; memory free 8.0 GiB/16.0 GiB; disk free 6.0 GiB; load 2.50)', + '[e2e target="unassigned" scenario="agent-turn-latency"] [phase 1/2] completed: install OpenClaw sandbox — passed in 6m (total 6m)', + '[e2e target="unassigned" scenario="agent-turn-latency"] [phase 2/2] started: install Hermes sandbox (total 6m; phase 0s)', + '[e2e target="unassigned" scenario="agent-turn-latency"] [phase 2/2] completed: install Hermes sandbox — passed in 0s (total 6m)', ]); expect(progress.summary()).toEqual({ version: 1, @@ -125,10 +132,10 @@ describe("live test progress", () => { }); }); - it("records the final phase duration and failure outcome without repeating test identity", () => { + it("records test identity, duration, and the final phase failure outcome", () => { const { options, state } = progressHarness(); const progress = startTestProgress( - "identity-that-must-stay-out-of-live-lines", + "visible-agent-turn-scenario", ["prepare hosted inference", "send OpenClaw agent turn"], options, ); @@ -137,10 +144,10 @@ describe("live test progress", () => { progress.stop("failed"); expect(state.lines).toEqual([ - "[e2e phase 1/2] prepare hosted inference", - "[e2e phase 1/2] prepare hosted inference — failed in 1m", + '[e2e target="unassigned" scenario="visible-agent-turn-scenario"] [phase 1/2] started: prepare hosted inference (total 0s; phase 0s)', + '[e2e target="unassigned" scenario="visible-agent-turn-scenario"] [phase 1/2] completed: prepare hosted inference — failed in 1m (total 1m)', ]); - expect(state.lines.join("\n")).not.toContain("identity-that-must-stay-out-of-live-lines"); + expect(state.lines.join("\n")).toContain("visible-agent-turn-scenario"); expect(progress.summary().phases).toEqual([ expect.objectContaining({ label: "prepare hosted inference", @@ -157,6 +164,12 @@ describe("live test progress", () => { expect(() => validateE2EPhasePlan(["prepare inference endpoint", "prepare inference endpoint"]), ).toThrow("duplicate live E2E phase label"); + expect(() => + validateE2EPhasePlan(["prepare inference endpoint\n::error::forged", "validate response"]), + ).toThrow("invalid live E2E phase label"); + expect(() => validateE2EPhasePlan(["p".repeat(161), "validate response"])).toThrow( + "invalid live E2E phase label", + ); const { options } = progressHarness(); const progress = startTestProgress( @@ -180,7 +193,12 @@ describe("live test progress", () => { it("connects install output to the timestamp-only observer", async () => { const command = vi.fn(async () => successfulProbe()); const host = { command } as unknown as HostCliClient; - const progress = { onOutput: vi.fn() }; + const finishActivity = vi.fn(); + const progress = { + activity: vi.fn(() => finishActivity), + event: vi.fn(), + onOutput: vi.fn(), + }; await installSandbox( host, @@ -197,18 +215,29 @@ describe("live test progress", () => { onOutput: progress.onOutput, redactionValues: ["secret-api-key"], }); + expect(progress.event.mock.calls).toEqual([ + ["openclaw install attempt 1/2 started"], + ["openclaw install attempt 1/2 passed"], + ]); + expect(progress.activity).toHaveBeenCalledWith("command: openclaw-install-attempt-1"); + expect(finishActivity).toHaveBeenCalledOnce(); }); - it("retries transient install failures with cleanup and backoff", async () => { + it("reports timeout, cleanup, and backoff before retrying a transient install", async () => { vi.useFakeTimers(); const command = vi .fn() .mockResolvedValueOnce( - failedProbe("Chat Completions API validation failed: request timed out"), + failedProbe("Chat Completions API validation failed: request timed out", true), ) .mockResolvedValueOnce(successfulProbe()); const cleanupBeforeRetry = vi.fn(async () => undefined); - const progress = { onOutput: vi.fn() }; + const finishActivity = vi.fn(); + const progress = { + activity: vi.fn(() => finishActivity), + event: vi.fn(), + onOutput: vi.fn(), + }; const host = { command } as unknown as HostCliClient; const resultPromise = installSandbox( @@ -234,6 +263,21 @@ describe("live test progress", () => { onOutput: progress.onOutput, }), ]); + expect(progress.event.mock.calls).toEqual([ + ["openclaw install attempt 1/2 started"], + ["openclaw install attempt 1/2 timeout fired at the 30-minute limit"], + ["openclaw install attempt 1/2 starting cleanup before retry"], + ["openclaw install attempt 1/2 cleanup before retry passed"], + ["openclaw install attempt 1/2 waiting 10s before retry"], + ["openclaw install attempt 2/2 started"], + ["openclaw install attempt 2/2 passed"], + ]); + expect(progress.activity.mock.calls).toEqual([ + ["command: openclaw-install-attempt-1"], + ["cleanup: openclaw-install-attempt-1-retry"], + ["command: openclaw-install-attempt-2"], + ]); + expect(finishActivity).toHaveBeenCalledTimes(3); }); it("does not report retry phases for a non-transient install failure", async () => { @@ -241,7 +285,12 @@ describe("live test progress", () => { failedProbe("endpoint validation failed: invalid NVIDIA_INFERENCE_API_KEY credential"), ); const cleanupBeforeRetry = vi.fn(async () => undefined); - const progress = { onOutput: vi.fn() }; + const finishActivity = vi.fn(); + const progress = { + activity: vi.fn(() => finishActivity), + event: vi.fn(), + onOutput: vi.fn(), + }; const host = { command } as unknown as HostCliClient; await expect( @@ -257,5 +306,74 @@ describe("live test progress", () => { expect(command).toHaveBeenCalledOnce(); expect(cleanupBeforeRetry).not.toHaveBeenCalled(); + expect(progress.event.mock.calls).toEqual([ + ["openclaw install attempt 1/2 started"], + ["openclaw install attempt 1/2 failed"], + ]); + expect(finishActivity).toHaveBeenCalledOnce(); + }); + + it("reports each pre-clean boundary and closes its heartbeat activity", async () => { + const command = vi.fn(async () => successfulProbe()); + const openshell = vi.fn(async () => successfulProbe()); + const host = { command } as unknown as HostCliClient; + const sandbox = { openshell } as unknown as SandboxClient; + const activityFinishes: ReturnType[] = []; + const progress = { + activity: vi.fn(() => { + const finish = vi.fn(); + activityFinishes.push(finish); + return finish; + }), + event: vi.fn(), + onOutput: vi.fn(), + }; + + await cleanupTurnSandboxes(host, sandbox, progress); + + expect(command).toHaveBeenCalledTimes(2); + expect(openshell).toHaveBeenCalledTimes(4); + expect(progress.activity.mock.calls).toEqual([ + ["cleanup: destroy openclaw sandbox"], + ["cleanup: delete openclaw sandbox"], + ["cleanup: destroy hermes sandbox"], + ["cleanup: delete hermes sandbox"], + ["cleanup: stop Hermes API forward"], + ["cleanup: destroy OpenShell gateway"], + ]); + expect(progress.event.mock.calls).toEqual([ + ["destroy openclaw sandbox started"], + ["destroy openclaw sandbox passed"], + ["delete openclaw sandbox started"], + ["delete openclaw sandbox passed"], + ["destroy hermes sandbox started"], + ["destroy hermes sandbox passed"], + ["delete hermes sandbox started"], + ["delete hermes sandbox passed"], + ["stop Hermes API forward started"], + ["stop Hermes API forward passed"], + ["destroy OpenShell gateway started"], + ["destroy OpenShell gateway passed"], + ]); + expect(activityFinishes).toHaveLength(6); + for (const finish of activityFinishes) expect(finish).toHaveBeenCalledOnce(); + }); + + it("keeps cleanup exception payloads out of live console diagnostics", async () => { + const secret = "opaque-cleanup-exception-secret"; + const warning = vi.spyOn(console, "warn").mockImplementation(() => undefined); + try { + await expect( + bestEffortPreclean("destroy OpenClaw sandbox", async () => { + throw new Error(secret); + }), + ).resolves.toBe(false); + expect(warning).toHaveBeenCalledWith( + "best-effort cleanup failed (destroy OpenClaw sandbox); see redacted command artifacts", + ); + expect(JSON.stringify(warning.mock.calls)).not.toContain(secret); + } finally { + warning.mockRestore(); + } }); }); diff --git a/test/e2e/support/bedrock-runtime-compatible-anthropic-leaks.test.ts b/test/e2e/support/bedrock-runtime-compatible-anthropic-leaks.test.ts index 4be03ab7a6..f28d78e239 100644 --- a/test/e2e/support/bedrock-runtime-compatible-anthropic-leaks.test.ts +++ b/test/e2e/support/bedrock-runtime-compatible-anthropic-leaks.test.ts @@ -7,6 +7,7 @@ import { type ForbiddenLeakPattern, findForbiddenLeaks, frameSnapshotFile, + SNAPSHOT_DATA_PREFIX, SNAPSHOT_FILE_PREFIX, SNAPSHOT_PROBE_PID_PREFIX, scanForbiddenLeaks, @@ -28,6 +29,19 @@ function file(path: string, ...lines: string[]): string[] { } describe("Bedrock Runtime leak snapshot process identity", () => { + it("keeps per-line snapshot framing compact without weakening control separation (#7101)", () => { + expect(SNAPSHOT_DATA_PREFIX).toBe("D "); + expect( + frameSnapshotFile( + "/sandbox/.openclaw/runtime.env", + `${SNAPSHOT_FILE_PREFIX}/proc/1418/environ`, + ).split("\n"), + ).toEqual([ + `${SNAPSHOT_FILE_PREFIX}/sandbox/.openclaw/runtime.env`, + `${SNAPSHOT_DATA_PREFIX}${SNAPSHOT_FILE_PREFIX}/proc/1418/environ`, + ]); + }); + it("allows the provider placeholder name only in the declared probe environment", () => { const text = snapshot( ...file("/proc/1418/environ", `${ADAPTER_ENV_NAME}=openshell-placeholder`), diff --git a/test/e2e/support/bedrock-runtime-compatible-anthropic-progress.test.ts b/test/e2e/support/bedrock-runtime-compatible-anthropic-progress.test.ts new file mode 100644 index 0000000000..a0dcf2abf3 --- /dev/null +++ b/test/e2e/support/bedrock-runtime-compatible-anthropic-progress.test.ts @@ -0,0 +1,207 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; + +import { afterEach, describe, expect, it, onTestFinished } from "vitest"; + +import { ArtifactSink } from "../fixtures/artifacts.ts"; +import { startTestProgress } from "../fixtures/progress.ts"; + +import { SNAPSHOT_DATA_PREFIX } from "../live/bedrock-runtime-compatible-anthropic-leaks.ts"; +import { runRawCommand } from "../live/bedrock-runtime-compatible-anthropic-raw-command.ts"; + +const temporaryRoots: string[] = []; + +function progressProbe() { + const lines: string[] = []; + const timers: Array<() => void> = []; + const progress = startTestProgress( + "Bedrock command support", + ["run Bedrock command", "verify Bedrock result"], + { + clearTimer: () => undefined, + logLine: (line) => lines.push(line), + setTimer: (callback) => { + timers.push(callback); + return {}; + }, + }, + ); + onTestFinished(() => progress.stop()); + return { lines, progress, timers }; +} + +async function artifactSink(name: string): Promise { + const root = await fs.mkdtemp(path.join(os.tmpdir(), `nemoclaw-${name}-`)); + temporaryRoots.push(root); + const artifacts = new ArtifactSink(root); + await artifacts.ensureRoot(); + return artifacts; +} + +afterEach(async () => { + await Promise.all( + temporaryRoots.splice(0).map((root) => fs.rm(root, { force: true, recursive: true })), + ); +}); + +describe("Bedrock raw-command progress", () => { + it("reports timestamp-only output activity without forwarding child payloads", async () => { + const secret = "opaque-bedrock-progress-secret"; + const artifacts = await artifactSink("bedrock-progress-output"); + const observation = progressProbe(); + const { progress } = observation; + + const result = await runRawCommand( + process.execPath, + [ + "-e", + "process.stdout.write(process.env.BEDROCK_TEST_SECRET); process.stderr.write('stderr-ready')", + ], + { + artifactName: "bedrock-progress-output", + artifacts, + env: { ...process.env, BEDROCK_TEST_SECRET: secret }, + progress, + redactionValues: [secret], + }, + ); + + expect(result.exitCode).toBe(0); + expect(result.stdout).toBe(secret); + observation.timers[0]?.(); + expect(observation.lines.at(-1)).toContain("no active command"); + expect(observation.lines).toEqual( + expect.arrayContaining([ + expect.stringContaining("event: command bedrock-progress-output started"), + expect.stringContaining("event: command bedrock-progress-output passed"), + ]), + ); + progress.stop(); + expect(progress.summary().phases[0]?.outputEvents).toBe(2); + expect(JSON.stringify({ lines: observation.lines, summary: progress.summary() })).not.toContain( + secret, + ); + await expect( + fs.readFile( + path.join(artifacts.rootDir, "raw-shell/bedrock-progress-output.stdout.txt"), + "utf8", + ), + ).resolves.toBe("[REDACTED]"); + }); + + it("emits an immediate content-free timeout event and closes command activity", async () => { + const artifacts = await artifactSink("bedrock-progress-timeout"); + const observation = progressProbe(); + const { progress } = observation; + + const result = await runRawCommand( + process.execPath, + ["-e", "setInterval(() => undefined, 1_000)"], + { + artifactName: "bedrock-progress-timeout", + artifacts, + progress, + timeoutMs: 50, + }, + ); + + expect(result.timedOut).toBe(true); + expect(observation.lines).toEqual( + expect.arrayContaining([ + expect.stringContaining("event: command bedrock-progress-timeout started"), + expect.stringContaining("event: command bedrock-progress-timeout timeout fired after 50ms"), + expect.stringContaining("event: command bedrock-progress-timeout stopped after timeout"), + ]), + ); + observation.timers[0]?.(); + expect(observation.lines.at(-1)).toContain("no active command"); + progress.stop(); + }); + + it("fails closed when command output exceeds the bounded capture limit (#7101)", async () => { + const secret = "opaque-bedrock-capture-limit-secret"; + const artifacts = await artifactSink("bedrock-progress-output-limit"); + const observation = progressProbe(); + + await expect( + runRawCommand( + process.execPath, + ["-e", `process.stdout.write(${JSON.stringify(secret)}.repeat(400_000))`], + { + artifactName: "bedrock-progress-output-limit", + artifacts, + progress: observation.progress, + redactionValues: [secret], + }, + ), + ).rejects.toThrow("output exceeded safe capture limit"); + + expect(observation.lines).toEqual( + expect.arrayContaining([ + expect.stringContaining( + "event: command bedrock-progress-output-limit output exceeded safe capture limit", + ), + ]), + ); + expect(observation.lines.join("\n")).not.toContain(secret); + const marker = "[bedrock raw-command output exceeded safe capture limit]"; + await expect( + fs.readFile( + path.join(artifacts.rootDir, "raw-shell/bedrock-progress-output-limit.stdout.txt"), + "utf8", + ), + ).resolves.toBe(marker); + const resultArtifact = JSON.parse( + await fs.readFile( + path.join(artifacts.rootDir, "raw-shell/bedrock-progress-output-limit.result.json"), + "utf8", + ), + ); + expect(resultArtifact).toMatchObject({ + captureLimitExceeded: true, + stdout: marker, + stderr: marker, + }); + expect(JSON.stringify(resultArtifact)).not.toContain(secret); + }); + + it("captures null-heavy snapshot framing within the bounded output limit (#7101)", async () => { + const artifacts = await artifactSink("bedrock-progress-compact-snapshot"); + const observation = progressProbe(); + const record = `${SNAPSHOT_DATA_PREFIX}\n`; + const recordCount = 500_000; + const expectedBytes = Buffer.byteLength(record) * recordCount; + + const result = await runRawCommand( + process.execPath, + ["-e", `process.stdout.write(${JSON.stringify(record)}.repeat(${recordCount}))`], + { + artifactName: "bedrock-progress-compact-snapshot", + artifactOutputMode: "metadata-only", + artifacts, + progress: observation.progress, + }, + ); + + expect(result.exitCode).toBe(0); + expect(Buffer.byteLength(result.stdout)).toBe(expectedBytes); + expect(expectedBytes).toBeLessThan(10 * 1024 * 1024); + await expect( + fs.readFile( + path.join(artifacts.rootDir, "raw-shell/bedrock-progress-compact-snapshot.stdout.txt"), + "utf8", + ), + ).resolves.toBe( + `${JSON.stringify({ + stream: "stdout", + capturedBytes: expectedBytes, + capturedLines: recordCount + 1, + content: "omitted: inspected in memory only", + })}\n`, + ); + }); +}); diff --git a/test/e2e/support/device-auth-health-helpers.test.ts b/test/e2e/support/device-auth-health-helpers.test.ts index 012124e886..7c859b2b28 100644 --- a/test/e2e/support/device-auth-health-helpers.test.ts +++ b/test/e2e/support/device-auth-health-helpers.test.ts @@ -11,9 +11,16 @@ import { createOpenAiLikeAuthConfig } from "../../../src/lib/adapters/http/auth- import { runCurlProbe } from "../../../src/lib/adapters/http/probe"; import type { HostCliClient } from "../fixtures/clients/host.ts"; import { startFakeOpenAiCompatibleServer } from "../fixtures/fake-openai-compatible.ts"; +import { startTestProgress } from "../fixtures/progress.ts"; import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; import { commandEnv, installDeviceAuthSandbox } from "../live/device-auth-health-helpers.ts"; +const NOOP_PROGRESS = startTestProgress( + "device auth support", + ["serve device auth endpoint", "verify device auth endpoint"], + { logLine: () => undefined }, +); + function okResult(command: string[]): ShellProbeResult { return { artifacts: { result: "", stderr: "", stdout: "" }, @@ -75,6 +82,7 @@ describe("device auth health fixture inference wiring", () => { const fake = await startFakeOpenAiCompatibleServer({ apiKey: inference.apiKey, model: inference.model, + progress: NOOP_PROGRESS, requireAuth: true, }); const authConfig = createOpenAiLikeAuthConfig(inference.apiKey); diff --git a/test/e2e/support/docker-probe.test.ts b/test/e2e/support/docker-probe.test.ts index 8dec3a8708..8319d0b0ff 100644 --- a/test/e2e/support/docker-probe.test.ts +++ b/test/e2e/support/docker-probe.test.ts @@ -1,11 +1,23 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import type { ChildProcess } from "node:child_process"; +import { EventEmitter } from "node:events"; import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; +import { PassThrough } from "node:stream"; -import { describe, expect, it } from "vitest"; +import { describe, expect, it, onTestFinished, vi } from "vitest"; + +const { spawnMock } = vi.hoisted(() => ({ + spawnMock: vi.fn(), +})); + +vi.mock("node:child_process", async (importOriginal) => ({ + ...(await importOriginal()), + spawn: spawnMock, +})); import { ArtifactSink } from "../fixtures/artifacts.ts"; import { @@ -13,6 +25,7 @@ import { DockerProbe, redactDockerProbeResult, } from "../fixtures/docker-probe.ts"; +import { startTestProgress } from "../fixtures/progress.ts"; import { SecretStore } from "../fixtures/secrets.ts"; async function readArtifact(root: string, relativePath: string): Promise { @@ -106,6 +119,89 @@ describe("DockerProbe secret hygiene", () => { } }); + it("kills real-branch Docker output at the capture limit without retaining payload (#7101)", async () => { + const secret = "DOCKER_OUTPUT_LIMIT_SECRET"; + const outputBytes = 10 * 1024 * 1024 + Buffer.byteLength(secret); + const output = secret.repeat(Math.ceil(outputBytes / Buffer.byteLength(secret))); + const artifactsRoot = await fs.mkdtemp(path.join(os.tmpdir(), "docker-probe-output-limit-")); + const artifacts = new ArtifactSink(artifactsRoot); + const stdout = new PassThrough(); + const stderr = new PassThrough(); + const childKill = vi.fn(() => true); + const childPid = 42_424; + const child = Object.assign(new EventEmitter(), { + pid: childPid, + stdout, + stderr, + stdin: null, + kill: childKill, + }) as unknown as ChildProcess; + const progress = startTestProgress( + "DockerProbe real-branch output limit", + ["run noisy Docker command", "verify safe artifacts"], + { + clearTimer: () => undefined, + logLine: () => undefined, + setTimer: () => ({}), + targetId: "docker-probe-output-limit", + }, + ); + const processKill = vi.spyOn(process, "kill").mockImplementation((() => { + queueMicrotask(() => child.emit("close", null, "SIGKILL")); + return true; + }) as typeof process.kill); + spawnMock.mockReset(); + spawnMock.mockImplementationOnce(() => { + queueMicrotask(() => { + stderr.write(`before-limit:${secret}`); + stdout.write(output); + stderr.write(`after-limit:${secret}`); + }); + return child; + }); + onTestFinished(() => { + progress.stop(); + processKill.mockRestore(); + spawnMock.mockReset(); + }); + + const probe = new DockerProbe(artifacts, (text) => text, undefined, progress); + const marker = "[docker-probe output exceeded safe capture limit]"; + const result = await probe.run(["version"], { + artifactName: "output-limit", + timeoutMs: 10_000, + }); + progress.phase("verify safe artifacts"); + + expect(spawnMock).toHaveBeenCalledTimes(1); + expect(processKill).toHaveBeenCalledWith(-childPid, "SIGKILL"); + expect(childKill).not.toHaveBeenCalled(); + expect(result).toMatchObject({ + command: ["docker", "version"], + exitCode: null, + signal: "SIGKILL", + stdout: marker, + stderr: marker, + error: "Docker output exceeded the safe capture limit", + }); + const [stdoutArtifact, stderrArtifact, resultArtifactText] = await Promise.all([ + readArtifact(artifactsRoot, "docker/001-output-limit.stdout.txt"), + readArtifact(artifactsRoot, "docker/001-output-limit.stderr.txt"), + readArtifact(artifactsRoot, "docker/001-output-limit.result.json"), + ]); + expect(stdoutArtifact).toBe(marker); + expect(stderrArtifact).toBe(marker); + expect(JSON.parse(resultArtifactText)).toEqual(result); + for (const published of [ + JSON.stringify(result), + stdoutArtifact, + stderrArtifact, + resultArtifactText, + ]) { + expect(published).not.toContain(secret); + } + }); + it("can return raw Docker output for leak assertions while writing only redacted artifacts", async () => { const leakedSecret = "SENTINEL_RAW_SECRET_VALUE"; const artifactsRoot = await fs.mkdtemp(path.join(os.tmpdir(), "docker-probe-raw-output-")); diff --git a/test/e2e/support/e2e-fixture-context.test.ts b/test/e2e/support/e2e-fixture-context.test.ts index 7772975fd0..88c43d17da 100644 --- a/test/e2e/support/e2e-fixture-context.test.ts +++ b/test/e2e/support/e2e-fixture-context.test.ts @@ -5,11 +5,12 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { describe, expect, expectTypeOf, it } from "vitest"; +import { afterEach, describe, expect, expectTypeOf, it } from "vitest"; import { ArtifactSink, createArtifactSink } from "../fixtures/artifacts.ts"; import { assertCleanupPassed, CleanupRegistry } from "../fixtures/cleanup.ts"; import { test as e2eTest } from "../fixtures/e2e-test.ts"; +import { startTestProgress, type TestProgress } from "../fixtures/progress.ts"; import { SecretStore } from "../fixtures/secrets.ts"; import { ShellProbe, @@ -19,6 +20,23 @@ import { const delay = (ms: number): Promise => new Promise((resolve) => setTimeout(resolve, ms)); +const supportProgressInstances: TestProgress[] = []; + +function supportProgress(): TestProgress { + const progress = startTestProgress( + "ShellProbe support", + ["run support command", "verify support result"], + { logLine: () => undefined }, + ); + supportProgressInstances.push(progress); + return progress; +} + +afterEach(() => { + for (const progress of supportProgressInstances) progress.stop(); + supportProgressInstances.length = 0; +}); + function isProcessAlive(pid: number): boolean { try { process.kill(pid, 0); @@ -87,6 +105,7 @@ describe("E2E fixture primitives", () => { const controller = new AbortController(); const shellProbe = new ShellProbe({ artifacts, + progress: supportProgress(), redact: (text) => text, signal: controller.signal, }); @@ -135,6 +154,73 @@ describe("E2E fixture primitives", () => { expect(result).toEqual({ passed: ["second", "first"], failures: [] }); }); + it("reports the active redacted cleanup entry and its outcome", async () => { + const events: string[] = []; + const secret = "cleanup-secret-value"; + const cleanup = new CleanupRegistry((text) => text.replaceAll(secret, "[REDACTED]"), { + activity(label) { + events.push(`activity started: ${label}`); + return () => events.push(`activity finished: ${label}`); + }, + event(label) { + events.push(`event: ${label}`); + }, + }); + cleanup.add(`release ${secret}\nresource`, () => undefined); + cleanup.add("release failing resource", () => { + throw new Error("expected cleanup failure"); + }); + + const result = await cleanup.runAll(); + + expect(events).toEqual([ + "activity started: cleanup: release failing resource", + "event: cleanup started: release failing resource", + "event: cleanup failed: release failing resource", + "activity finished: cleanup: release failing resource", + "activity started: cleanup: release [REDACTED] resource", + "event: cleanup started: release [REDACTED] resource", + "event: cleanup passed: release [REDACTED] resource", + "activity finished: cleanup: release [REDACTED] resource", + ]); + expect(events.join("\n")).not.toContain(secret); + expect(result).toEqual({ + passed: ["release [REDACTED]\nresource"], + failures: [{ name: "release failing resource", message: "expected cleanup failure" }], + }); + }); + + it("still releases every resource when redaction and progress reporting fail", async () => { + const released: string[] = []; + const cleanup = new CleanupRegistry( + () => { + throw new Error("redactor unavailable"); + }, + { + activity() { + throw new Error("progress activity unavailable"); + }, + event() { + throw new Error("progress event unavailable"); + }, + }, + ); + cleanup.add("first sensitive resource", () => { + released.push("first"); + }); + cleanup.add("second sensitive resource", () => { + released.push("second"); + }); + + const result = await cleanup.runAll(); + + expect(released).toEqual(["second", "first"]); + expect(result).toEqual({ + passed: ["[cleanup metadata unavailable]", "[cleanup metadata unavailable]"], + failures: [], + }); + }); + it("cleanup registry redacts failures, continues, and clears callbacks", async () => { const secret = "cleanup-secret-value"; const cleanup = new CleanupRegistry((text) => text.split(secret).join("[REDACTED]")); @@ -203,6 +289,7 @@ describe("E2E fixture primitives", () => { const controller = new AbortController(); const probe = new ShellProbe({ artifacts, + progress: supportProgress(), redact: (text) => text, signal: controller.signal, }); @@ -253,17 +340,13 @@ describe("E2E fixture primitives", () => { await artifacts.ensureRoot(); const controller = new AbortController(); const events: Array<{ stream: "stdout" | "stderr"; atMs: number }> = []; - const activities: string[] = []; const onOutput = (event: (typeof events)[number]) => events.push(event); + const progress = supportProgress(); const probe = new ShellProbe({ artifacts, + progress, redact: (text) => text, signal: controller.signal, - onOutput, - onActivity: (label) => { - activities.push(`start ${label}`); - return () => activities.push(`finish ${label}`); - }, }); const result = await probe.run( @@ -279,10 +362,8 @@ describe("E2E fixture primitives", () => { expect(events).toHaveLength(1); expect(events[0]).toMatchObject({ stream: "stdout" }); expect(events[0]?.atMs).toBeGreaterThan(0); - expect(activities).toEqual([ - "start command: shared-output-observer", - "finish command: shared-output-observer", - ]); + progress.stop(); + expect(progress.summary().phases[0]?.outputEvents).toBe(1); } finally { fs.rmSync(tmp, { recursive: true, force: true }); } @@ -298,6 +379,7 @@ describe("E2E fixture primitives", () => { const controller = new AbortController(); const probe = new ShellProbe({ artifacts, + progress: supportProgress(), redact: (text) => text, signal: controller.signal, }); @@ -347,7 +429,6 @@ describe("E2E fixture primitives", () => { await artifacts.ensureRoot(); const secret = "spawn-secret-value"; const controller = new AbortController(); - const activities: string[] = []; let abortAdds = 0; let abortRemoves = 0; const addEventListener = controller.signal.addEventListener.bind(controller.signal); @@ -372,18 +453,16 @@ describe("E2E fixture primitives", () => { instrumentedAddEventListener as typeof controller.signal.addEventListener; controller.signal.removeEventListener = instrumentedRemoveEventListener as typeof controller.signal.removeEventListener; + const progress = supportProgress(); const probe = new ShellProbe({ artifacts, + progress, redact: (text, extraValues = []) => [secret, ...extraValues].reduce( (redacted, value) => redacted.split(value).join("[REDACTED]"), text, ), signal: controller.signal, - onActivity: (label) => { - activities.push(`start ${label}`); - return () => activities.push(`finish ${label}`); - }, }); let thrown: unknown; @@ -410,7 +489,8 @@ describe("E2E fixture primitives", () => { expect(message).not.toContain(secret); expect(abortAdds).toBe(1); expect(abortRemoves).toBe(1); - expect(activities).toEqual(["start command: spawn-error", "finish command: spawn-error"]); + progress.stop("failed"); + expect(progress.summary().phases[0]).toMatchObject({ outcome: "failed" }); const spawnArtifact = fs.readFileSync( artifacts.pathFor("shell/spawn-error.result.json"), "utf8", @@ -433,6 +513,7 @@ describe("E2E fixture primitives", () => { const controller = new AbortController(); const probe = new ShellProbe({ artifacts, + progress: supportProgress(), redact: (text) => text, signal: controller.signal, }); @@ -470,6 +551,7 @@ describe("E2E fixture primitives", () => { controller.abort(); const probe = new ShellProbe({ artifacts, + progress: supportProgress(), redact: (text) => text, signal: controller.signal, }); @@ -505,6 +587,7 @@ describe("E2E fixture primitives", () => { const controller = new AbortController(); const probe = new ShellProbe({ artifacts, + progress: supportProgress(), redact: (text) => text, signal: controller.signal, }); diff --git a/test/e2e/support/e2e-progress-fixture.test.ts b/test/e2e/support/e2e-progress-fixture.test.ts index 6eac7492cf..0759408c88 100644 --- a/test/e2e/support/e2e-progress-fixture.test.ts +++ b/test/e2e/support/e2e-progress-fixture.test.ts @@ -1,23 +1,59 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { spawnSync } from "node:child_process"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { afterAll, expect, it, vi } from "vitest"; +import { expect, it } from "vitest"; import { assertPhaseLabel } from "../../../tools/e2e/runner-pressure-core.mts"; -import { E2E_TEARDOWN_PHASE, resourcePhaseLabel, test } from "../fixtures/e2e-test.ts"; +import { E2E_TEARDOWN_PHASE, resourcePhaseLabel } from "../fixtures/e2e-test.ts"; +import { REPO_ROOT } from "../fixtures/paths.ts"; import type { ProgressSummary } from "../fixtures/progress.ts"; -const artifactRoot = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-progress-fixture-")); -let progressArtifact = ""; +const VITEST = path.join(REPO_ROOT, "node_modules", "vitest", "vitest.mjs"); +const FIXTURE = "test/e2e/support/fixtures/e2e-progress.fixture.test.ts"; +const ARTIFACT_SLUG = "automatic-progress-fixture-writes-completed-target-and-shard-evidence"; -vi.stubEnv("E2E_ARTIFACT_DIR", artifactRoot); +it("bounds long resource phase labels without losing deterministic identity", () => { + const target = "openshell-gateway-auth-contract"; + const phase = "confirm gateway and Docker prerequisites"; + const label = resourcePhaseLabel(target, phase); + + expect(label).toHaveLength(64); + expect(label).toMatch(/\.[a-f0-9]{12}$/u); + expect(assertPhaseLabel(label)).toBe(label); + expect(resourcePhaseLabel(target, phase)).toBe(label); + expect(resourcePhaseLabel(target, `${phase} again`)).not.toBe(label); +}); -afterAll(() => { +it("writes completed target and shard evidence through the automatic progress fixture", () => { + const artifactRoot = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-progress-fixture-")); try { - const summary = JSON.parse(fs.readFileSync(progressArtifact, "utf8")) as ProgressSummary; + const result = spawnSync( + process.execPath, + [VITEST, "run", "--project", "e2e-support", FIXTURE, "--reporter=default"], + { + cwd: REPO_ROOT, + encoding: "utf8", + killSignal: "SIGKILL", + timeout: 20_000, + env: { + ...process.env, + E2E_ARTIFACT_DIR: artifactRoot, + E2E_TARGET_ID: "", + GITHUB_JOB: "fixture-progress-target", + NEMOCLAW_E2E_PROGRESS_FIXTURE: "identity", + NEMOCLAW_E2E_SHARD: "fixture-progress-shard", + }, + }, + ); + + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + const summary = JSON.parse( + fs.readFileSync(path.join(artifactRoot, ARTIFACT_SLUG, "test-progress.json"), "utf8"), + ) as ProgressSummary; expect(summary).toMatchObject({ version: 1, scenario: "automatic progress fixture writes completed target and shard evidence", @@ -28,38 +64,12 @@ afterAll(() => { expect(summary.durationMs).not.toBeNull(); expect( summary.phases.find((phase) => phase.label === "record final fixture phase"), - ).toMatchObject({ - outcome: "passed", - }); + ).toMatchObject({ outcome: "passed" }); expect(summary.phases.at(-1)).toMatchObject({ label: E2E_TEARDOWN_PHASE, outcome: "passed", }); } finally { - vi.unstubAllEnvs(); fs.rmSync(artifactRoot, { force: true, recursive: true }); } }); - -it("bounds long resource phase labels without losing deterministic identity", () => { - const target = "openshell-gateway-auth-contract"; - const phase = "confirm gateway and Docker prerequisites"; - const label = resourcePhaseLabel(target, phase); - - expect(label).toHaveLength(64); - expect(label).toMatch(/\.[a-f0-9]{12}$/u); - expect(assertPhaseLabel(label)).toBe(label); - expect(resourcePhaseLabel(target, phase)).toBe(label); - expect(resourcePhaseLabel(target, `${phase} again`)).not.toBe(label); -}); - -test("automatic progress fixture writes completed target and shard evidence", { - meta: { - e2ePhases: ["prepare progress artifact", "record final fixture phase"], - }, -}, async ({ artifacts, progress }) => { - progressArtifact = artifacts.pathFor("test-progress.json"); - vi.stubEnv("E2E_TARGET_ID", "fixture-progress-target"); - vi.stubEnv("NEMOCLAW_E2E_SHARD", "fixture-progress-shard"); - progress.phase("record final fixture phase"); -}); diff --git a/test/e2e/support/e2e-progress-outcome.test.ts b/test/e2e/support/e2e-progress-outcome.test.ts index 1119bca627..10bcbef457 100644 --- a/test/e2e/support/e2e-progress-outcome.test.ts +++ b/test/e2e/support/e2e-progress-outcome.test.ts @@ -15,6 +15,39 @@ const VITEST = path.join(REPO_ROOT, "node_modules", "vitest", "vitest.mjs"); const FIXTURE = "test/e2e/support/fixtures/e2e-progress-outcome.fixture.test.ts"; describe("automatic E2E phase outcomes", () => { + it("redacts target identities and explicit progress events before console output", () => { + const artifactDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-progress-redaction-")); + const secret = "progress-event-secret-value"; + try { + const result = spawnSync( + process.execPath, + [VITEST, "run", "--project", "e2e-support", FIXTURE, "--reporter=default"], + { + cwd: REPO_ROOT, + encoding: "utf8", + killSignal: "SIGKILL", + timeout: 20_000, + env: { + ...process.env, + E2E_ARTIFACT_DIR: artifactDir, + E2E_TARGET_ID: `redaction-target-${secret}`, + NEMOCLAW_E2E_PROGRESS_EVENT_SECRET: secret, + NEMOCLAW_E2E_PROGRESS_OUTCOME_FIXTURE: "redacted-event", + NEMOCLAW_RUN_LIVE_E2E: "1", + }, + }, + ); + + const output = `${result.stdout}\n${result.stderr}`; + expect(result.status, output).toBe(0); + expect(output).not.toContain(secret); + expect(output).toContain('target="redaction-target-[REDACTED]"'); + expect(output).toContain("event: retry cleanup for [REDACTED]"); + } finally { + fs.rmSync(artifactDir, { recursive: true, force: true }); + } + }); + it.each([ [ "failed", @@ -72,6 +105,7 @@ describe("automatic E2E phase outcomes", () => { { cwd: REPO_ROOT, encoding: "utf8", + killSignal: "SIGKILL", timeout: 20_000, env: { ...process.env, diff --git a/test/e2e/support/e2e-redaction-entry.test.ts b/test/e2e/support/e2e-redaction-entry.test.ts index 6e2670e8b8..71de84a277 100644 --- a/test/e2e/support/e2e-redaction-entry.test.ts +++ b/test/e2e/support/e2e-redaction-entry.test.ts @@ -22,10 +22,19 @@ import path from "node:path"; import { describe, expect, it } from "vitest"; import { ArtifactSink } from "../fixtures/artifacts.ts"; +import { startTestProgress } from "../fixtures/progress.ts"; import { buildChildEnv, isValidSecretEnvKey, redactString } from "../fixtures/redaction.ts"; import { SecretStore } from "../fixtures/secrets.ts"; import { ShellProbe, trustedShellCommand } from "../fixtures/shell-probe.ts"; +function supportProgress() { + return startTestProgress( + "ShellProbe redaction support", + ["run redaction probe", "verify redacted evidence"], + { logLine: () => undefined }, + ); +} + describe("fixture redaction entry point", () => { it("recognizes pass env names only at exact or underscore-delimited boundaries", () => { for (const key of ["PASS", "PASSWD", "CUSTOM_PASS", "CUSTOM_PASSWD"]) { @@ -267,6 +276,7 @@ describe("fixture redaction entry point", () => { ); const probe = new ShellProbe({ artifacts, + progress: supportProgress(), redact: (text, extra) => secrets.redact(text, extra), signal: new AbortController().signal, }); @@ -348,6 +358,7 @@ describe("fixture redaction entry point", () => { await artifacts.ensureRoot(); const probe = new ShellProbe({ artifacts, + progress: supportProgress(), redact: (text, extra) => redactString(text, extra), signal: new AbortController().signal, }); @@ -403,6 +414,7 @@ describe("fixture redaction entry point", () => { await artifacts.ensureRoot(); const probe = new ShellProbe({ artifacts, + progress: supportProgress(), redact: (text, extra) => redactString(text, extra), signal: new AbortController().signal, }); diff --git a/test/e2e/support/e2e-resource-phase-label.test.ts b/test/e2e/support/e2e-resource-phase-label.test.ts new file mode 100644 index 0000000000..9b4deb35a4 --- /dev/null +++ b/test/e2e/support/e2e-resource-phase-label.test.ts @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { expect, it } from "vitest"; +import { assertPhaseLabel } from "../../../tools/e2e/runner-pressure-core.mts"; +import { resourcePhaseLabel } from "../fixtures/e2e-test.ts"; + +it("bounds long resource phase labels without losing deterministic identity", () => { + const target = "openshell-gateway-auth-contract"; + const phase = "confirm gateway and Docker prerequisites"; + const label = resourcePhaseLabel(target, phase); + + expect(label).toHaveLength(64); + expect(label).toMatch(/\.[a-f0-9]{12}$/u); + expect(assertPhaseLabel(label)).toBe(label); + expect(resourcePhaseLabel(target, phase)).toBe(label); + expect(resourcePhaseLabel(target, `${phase} again`)).not.toBe(label); +}); diff --git a/test/e2e/support/e2e-semantic-phase-check.test.ts b/test/e2e/support/e2e-semantic-phase-check.test.ts index d5e57a1c62..92967ee5c9 100644 --- a/test/e2e/support/e2e-semantic-phase-check.test.ts +++ b/test/e2e/support/e2e-semantic-phase-check.test.ts @@ -1,12 +1,15 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import fs from "node:fs"; import path from "node:path"; import { describe, expect, test } from "vitest"; import { + scanDirectChildProcessSource, scanLiveSourceGraph, + semanticPhaseCoverageModules, validateCollectedSemanticPhaseModule, validateTestScopedPhaseCalls, } from "../../../tools/e2e/check-semantic-phases.mts"; @@ -16,8 +19,98 @@ const INVALID_SOURCE_FIXTURE = path.join( REPO_ROOT, "test/e2e/support/fixtures/semantic-phase-invalid.fixture.ts", ); +const CHILD_PROCESS_SOURCE_FIXTURE = path.join( + REPO_ROOT, + "test/e2e/support/fixtures/semantic-phase-child-process.fixture.ts", +); +const OBSERVED_CHILD_PROCESS_SOURCE = path.join( + REPO_ROOT, + "test/e2e/fixtures/observed-child-process.ts", +); +const TEST_PROGRESS_SOURCE = path.join(REPO_ROOT, "test/e2e/fixtures/progress.ts"); +const FAKE_OPENAI_SOURCE = path.join(REPO_ROOT, "test/e2e/fixtures/fake-openai-compatible.ts"); describe("semantic E2E phase checker", () => { + test("derives coverage from registry, free-standing, shared, and forwarding workflow paths", () => { + const modules = semanticPhaseCoverageModules( + { + matrix: [{}], + testMatrix: [ + { + id: "docs-validation", + file: "test/e2e/live/docs-validation.test.ts", + project: "e2e-live", + }, + { + id: "gateway-drift-preflight", + file: "test/gateway-drift-preflight.test.ts", + project: "integration", + }, + ], + }, + { + liveTestToJobs: new Map([ + ["test/e2e/live/bootstrap-install-smoke.test.ts", ["bootstrap-install-smoke"]], + ["test/e2e/live/docs-validation.test.ts", ["docs-validation"]], + ["test/gateway-drift-preflight.test.ts", ["gateway-drift-preflight"]], + ]), + }, + [ + "test/e2e/live/bootstrap-install-smoke.test.ts", + "test/e2e/live/docs-validation.test.ts", + "test/e2e/live/unselected-regression.test.ts", + ], + ); + + expect(modules).toEqual([ + { + file: "test/e2e/live/bootstrap-install-smoke.test.ts", + project: "e2e-live", + }, + { file: "test/e2e/live/docs-validation.test.ts", project: "e2e-live" }, + { file: "test/e2e/live/launchable-smoke.test.ts", project: "e2e-live" }, + { file: "test/e2e/live/registry-targets.test.ts", project: "e2e-live" }, + { file: "test/e2e/live/unselected-regression.test.ts", project: "e2e-live" }, + { file: "test/gateway-drift-preflight.test.ts", project: "integration" }, + ]); + }); + + test("requires workflow-selected integration tests to use the progress fixture", () => { + const module = { + relativeModuleId: "test/workflow-selected.test.ts", + project: "integration" as const, + errors: [], + tests: [ + { + fullName: "covers a workflow-selected integration path", + phases: ["prepare integration behavior", "verify integration behavior"], + }, + ], + source: { + importsDirectTest: false, + importsSharedTest: false, + phaseCalls: [ + { + file: "test/workflow-selected.test.ts", + line: 12, + label: "verify integration behavior", + }, + ], + testPhaseBodies: [], + }, + }; + + expect(validateCollectedSemanticPhaseModule(module)).toEqual([ + "test/workflow-selected.test.ts: workflow-selected integration E2E tests must import test from the workflow-e2e-test fixture", + ]); + expect( + validateCollectedSemanticPhaseModule({ + ...module, + source: { ...module.source, importsWorkflowTest: true }, + }), + ).toEqual([]); + }); + test("rejects missing metadata and invalid phase transitions from a collected module", () => { const failures = validateCollectedSemanticPhaseModule({ relativeModuleId: "test/e2e/live/invalid-semantic-phase.test.ts", @@ -64,6 +157,32 @@ describe("semantic E2E phase checker", () => { }, }), ).toEqual([]); + expect( + validateCollectedSemanticPhaseModule({ + ...forwardingModule, + source: { + ...forwardingModule.source, + directChildProcessCalls: [ + { + api: "spawnSync", + boundary: "forwardedSetup", + file: "test/e2e/live/bootstrap-install-smoke.test.ts", + hasBoundedTimeout: false, + hasHardKillSignal: true, + line: 8, + observesOutput: false, + outputIgnored: false, + tracksActivity: false, + }, + ], + forwardedTestModules: ["test/e2e/live/launchable-smoke.test.ts"], + }, + }), + ).toEqual([ + expect.stringMatching( + /blocking child-process call spawnSync must declare a positive timeout/u, + ), + ]); const expectedFailure = "test/e2e/live/bootstrap-install-smoke.test.ts: forwarding module must import exactly test/e2e/live/launchable-smoke.test.ts"; for (const forwardedTestModules of [ @@ -78,6 +197,503 @@ describe("semantic E2E phase checker", () => { }), ).toEqual([expectedFailure]); } + + const forwardingSource = scanLiveSourceGraph( + path.join(REPO_ROOT, "test/e2e/live/bootstrap-install-smoke.test.ts"), + ); + expect(forwardingSource.phaseCalls).toEqual( + expect.arrayContaining([ + expect.objectContaining({ file: "test/e2e/live/launchable-smoke.test.ts" }), + ]), + ); + }); + + test("rejects each direct child-process boundary that can hide a heartbeat", () => { + const source = scanLiveSourceGraph(CHILD_PROCESS_SOURCE_FIXTURE); + const failures = validateCollectedSemanticPhaseModule({ + relativeModuleId: "test/e2e/live/direct-child-process-audit.test.ts", + errors: [], + tests: [ + { + fullName: "audits child processes", + phases: ["prepare child-process audit", "audit child processes"], + }, + ], + source: { + ...source, + importsSharedTest: true, + phaseCalls: [ + { + file: "test/e2e/live/direct-child-process-audit.test.ts", + line: 10, + label: "audit child processes", + }, + ], + }, + }); + + const directCalls = source.directChildProcessCalls ?? []; + expect(directCalls).toEqual( + expect.arrayContaining([ + expect.objectContaining({ api: "spawn", boundary: "reassignedNamespaceAsyncChild" }), + expect.objectContaining({ api: "spawn", boundary: "namespaceDestructuredAsyncChild" }), + expect.objectContaining({ api: "spawn", boundary: "assignmentAliasAsyncChild" }), + expect.objectContaining({ api: "execFile", boundary: "assignmentAliasAsyncChild" }), + expect.objectContaining({ api: "execFile", boundary: "nestedRequiredAsyncChild" }), + expect.objectContaining({ api: "fork", boundary: "directNestedRequiredAsyncChild" }), + expect.objectContaining({ api: "spawn", boundary: "boundAsyncChild" }), + expect.objectContaining({ api: "execFile", boundary: "inlineBoundAsyncChild" }), + expect.objectContaining({ api: "exec", boundary: "promisedAsyncChildren" }), + expect.objectContaining({ api: "execFile", boundary: "promisedAsyncChildren" }), + expect.objectContaining({ api: "spawn", boundary: "aliasedLoaderAsyncChildren" }), + expect.objectContaining({ api: "spawn", boundary: "extractedBindAsyncChild" }), + expect.objectContaining({ + api: "spawnSync", + boundary: "boundedSyncChild", + hasBoundedTimeout: true, + hasHardKillSignal: true, + }), + expect.objectContaining({ + api: "spawnSync", + boundary: "otherBoundedSyncChildren", + hasBoundedTimeout: false, + hasHardKillSignal: false, + }), + expect.objectContaining({ + api: "spawnSync", + boundary: "decoyAndPreappliedSyncOptions", + hasBoundedTimeout: false, + hasHardKillSignal: false, + }), + expect.objectContaining({ + api: "spawnSync", + boundary: "conflictingReassignmentSyncOptions", + hasBoundedTimeout: false, + hasHardKillSignal: false, + }), + expect.objectContaining({ + api: "execFileSync", + boundary: "decoyAndPreappliedSyncOptions", + hasBoundedTimeout: false, + hasHardKillSignal: false, + }), + expect.objectContaining({ + api: "spawnSync", + boundary: "decoyAndPreappliedSyncOptions", + hasBoundedTimeout: false, + hasHardKillSignal: false, + }), + expect.objectContaining({ + api: "spawnSync", + boundary: "spreadSyncOptions", + hasBoundedTimeout: true, + hasHardKillSignal: true, + }), + ]), + ); + expect( + failures.filter((failure) => + /asynchronous child-process call (?:exec|execFile|fork|spawn) must use an audited/u.test( + failure, + ), + ), + ).toHaveLength(21); + expect( + failures.filter((failure) => + /blocking child-process call spawnSync must declare a positive timeout/u.test(failure), + ), + ).toHaveLength(10); + expect( + failures.filter((failure) => + /blocking child-process call spawnSync must declare killSignal: "SIGKILL"/u.test(failure), + ), + ).toHaveLength(10); + expect(source.childProcessAuditFailures).toHaveLength(8); + expect(source.childProcessAuditFailures).toEqual( + expect.arrayContaining([ + expect.stringMatching(/namespace access must use a statically known audited API/u), + expect.stringMatching(/must not be passed to an unaudited higher-order function/u), + expect.stringMatching(/must not be stored in object properties/u), + expect.stringMatching(/must not be spread into object properties/u), + expect.stringMatching(/must not be stored in array elements/u), + expect.stringMatching(/unsupported child-process namespace member/u), + ]), + ); + expect( + directCalls.filter((call) => call.boundary === "decoyAndPreappliedSyncOptions"), + ).toHaveLength(4); + }); + + test("rejects wildcard re-exports and process built-in child-process loaders", () => { + const exportScan = scanDirectChildProcessSource( + path.join(REPO_ROOT, "test/e2e/live/child-process-export.fixture.ts"), + 'export * from "child_process";\nexport * from "node:child_process";\n', + ); + expect(exportScan.calls).toEqual([]); + expect(exportScan.auditFailures).toHaveLength(2); + expect(exportScan.auditFailures).toEqual([ + expect.stringMatching(/child-process APIs must not be re-exported/u), + expect.stringMatching(/child-process APIs must not be re-exported/u), + ]); + + const builtinFile = path.join(REPO_ROOT, "test/e2e/live/process-builtin-child.fixture.ts"); + const builtinScan = scanDirectChildProcessSource( + builtinFile, + ` +const getBuiltinModule = process.getBuiltinModule; +process.getBuiltinModule("node:child_process").spawn("node-prefixed-child", [], { stdio: "ignore" }); +getBuiltinModule("child_process").spawn("bare-child", [], { stdio: "ignore" }); +`, + ); + expect(builtinScan.auditFailures).toEqual([]); + expect(builtinScan.calls).toEqual([ + expect.objectContaining({ api: "spawn", outputIgnored: true }), + expect.objectContaining({ api: "spawn", outputIgnored: true }), + ]); + const failures = validateCollectedSemanticPhaseModule({ + relativeModuleId: "test/e2e/live/process-builtin-child.fixture.ts", + errors: [], + tests: [{ fullName: "audits process built-ins", phases: ["prepare", "verify"] }], + source: { + childProcessAuditFailures: builtinScan.auditFailures, + directChildProcessCalls: builtinScan.calls, + importsDirectTest: false, + importsSharedTest: true, + phaseCalls: [ + { + file: "test/e2e/live/process-builtin-child.fixture.ts", + label: "verify", + line: 1, + }, + ], + testPhaseBodies: [], + }, + }); + expect( + failures.filter((failure) => + /asynchronous child-process call spawn must use an audited/u.test(failure), + ), + ).toHaveLength(2); + }); + + test("tracks createRequire aliases without adding an executable CommonJS test seam", () => { + const sourceText = [ + 'import { createRequire as makeLoader } from "node:module";', + "const load = makeLoader(import.meta.url);", + 'const childProcess = load("node:child_process");', + "export function invoke() {", + ' childProcess.spawn("created-require-child", [], { stdio: "ignore" });', + "}", + ].join("\n"); + const result = scanDirectChildProcessSource( + "test/e2e/support/fixtures/create-require-source.fixture.ts", + sourceText, + ); + + expect(result.auditFailures).toEqual([]); + expect(result.calls).toEqual([expect.objectContaining({ api: "spawn", boundary: "invoke" })]); + }); + + test("ties the audited async boundary to the same child's lifecycle and exact output events", () => { + const canonicalSource = fs.readFileSync(OBSERVED_CHILD_PROCESS_SOURCE, "utf8"); + const canonical = scanDirectChildProcessSource(OBSERVED_CHILD_PROCESS_SOURCE, canonicalSource); + expect(canonical.auditFailures).toEqual([]); + expect(canonical.calls).toEqual([ + expect.objectContaining({ + api: "spawn", + boundary: "spawnObservedChild", + observesOutput: true, + tracksActivity: true, + }), + ]); + + const unrelatedActivity = scanDirectChildProcessSource( + OBSERVED_CHILD_PROCESS_SOURCE, + canonicalSource.replace("options.progress.activity(", "unrelated.activity("), + ); + expect(unrelatedActivity.calls[0]).toEqual(expect.objectContaining({ tracksActivity: false })); + + const deadActivity = scanDirectChildProcessSource( + OBSERVED_CHILD_PROCESS_SOURCE, + canonicalSource.replace( + "options.progress.activity(options.activityLabel)", + "false ? options.progress.activity(options.activityLabel) : undefined", + ), + ); + expect(deadActivity.calls[0]).toEqual(expect.objectContaining({ tracksActivity: false })); + + const siblingLifecycle = scanDirectChildProcessSource( + OBSERVED_CHILD_PROCESS_SOURCE, + canonicalSource.replace('child.once("close"', 'sibling.once("close"'), + ); + expect(siblingLifecycle.calls[0]).toEqual(expect.objectContaining({ tracksActivity: false })); + + const reassignedChild = scanDirectChildProcessSource( + OBSERVED_CHILD_PROCESS_SOURCE, + canonicalSource.replace( + "child = spawn(command, [...args], options.spawn);", + "child = spawn(command, [...args], options.spawn);\n child = sibling;", + ), + ); + expect(reassignedChild.calls[0]).toEqual( + expect.objectContaining({ observesOutput: false, tracksActivity: false }), + ); + + const reassignedActivity = scanDirectChildProcessSource( + OBSERVED_CHILD_PROCESS_SOURCE, + canonicalSource.replace( + "child = spawn(command, [...args], options.spawn);", + "child = spawn(command, [...args], options.spawn);\n finishActivity = () => undefined;", + ), + ); + expect(reassignedActivity.calls[0]).toEqual(expect.objectContaining({ tracksActivity: false })); + + const siblingOutput = scanDirectChildProcessSource( + OBSERVED_CHILD_PROCESS_SOURCE, + canonicalSource.replace('child.stderr?.on("data"', 'sibling.stderr?.on("data"'), + ); + expect(siblingOutput.calls[0]).toEqual(expect.objectContaining({ observesOutput: false })); + + const conditionalLifecycle = scanDirectChildProcessSource( + OBSERVED_CHILD_PROCESS_SOURCE, + canonicalSource.replace( + 'child.once("close", () => {\n try {\n finishActivity();', + 'child.once("close", () => {\n try {\n if (false) finishActivity();', + ), + ); + expect(conditionalLifecycle.calls[0]).toEqual( + expect.objectContaining({ tracksActivity: false }), + ); + + const conditionalOutput = scanDirectChildProcessSource( + OBSERVED_CHILD_PROCESS_SOURCE, + canonicalSource.replace( + 'options.progress.onOutput({ stream: "stdout", atMs: Date.now() });', + 'if (false) options.progress.onOutput({ stream: "stdout", atMs: Date.now() });', + ), + ); + expect(conditionalOutput.calls[0]).toEqual(expect.objectContaining({ observesOutput: false })); + + const payloadBearingOutput = scanDirectChildProcessSource( + OBSERVED_CHILD_PROCESS_SOURCE, + canonicalSource.replace( + '{ stream: "stdout", atMs: Date.now() }', + '{ stream: "stdout", atMs: Date.now(), rawChunk }', + ), + ); + expect(payloadBearingOutput.calls[0]).toEqual( + expect.objectContaining({ observesOutput: false }), + ); + + const payloadBearingSibling = scanDirectChildProcessSource( + OBSERVED_CHILD_PROCESS_SOURCE, + canonicalSource.replace( + 'child.stdout?.on("data", () => {', + 'child.stdout?.on("data", (rawChunk) => {\n console.log(rawChunk);', + ), + ); + expect(payloadBearingSibling.calls[0]).toEqual( + expect.objectContaining({ observesOutput: false }), + ); + + const payloadBearingCatch = scanDirectChildProcessSource( + OBSERVED_CHILD_PROCESS_SOURCE, + canonicalSource + .replace('child.stdout?.on("data", () => {', 'child.stdout?.on("data", (rawChunk) => {') + .replace( + 'options.progress.onOutput({ stream: "stdout", atMs: Date.now() });\n } catch {', + 'options.progress.onOutput({ stream: "stdout", atMs: Date.now() });\n } catch {\n console.log(rawChunk);', + ), + ); + expect(payloadBearingCatch.calls[0]).toEqual( + expect.objectContaining({ observesOutput: false }), + ); + + const extraPayloadListener = scanDirectChildProcessSource( + OBSERVED_CHILD_PROCESS_SOURCE, + canonicalSource.replace( + 'child.stderr?.on("data", () => {', + 'child.stdout?.on("data", (rawChunk) => console.log(rawChunk));\n child.stderr?.on("data", () => {', + ), + ); + expect(extraPayloadListener.calls[0]).toEqual( + expect.objectContaining({ observesOutput: false }), + ); + + const pipedPayload = scanDirectChildProcessSource( + OBSERVED_CHILD_PROCESS_SOURCE, + canonicalSource.replace( + 'child.stderr?.on("data", () => {', + 'child.stdout?.pipe(process.stdout);\n child.stderr?.on("data", () => {', + ), + ); + expect(pipedPayload.calls[0]).toEqual(expect.objectContaining({ observesOutput: false })); + + const extraChild = scanDirectChildProcessSource( + OBSERVED_CHILD_PROCESS_SOURCE, + canonicalSource.replace( + "child = spawn(command, [...args], options.spawn);", + "child = spawn(command, [...args], options.spawn);\n spawn(command, [...args], options.spawn);", + ), + ); + expect(extraChild.calls).toHaveLength(2); + const failures = validateCollectedSemanticPhaseModule({ + relativeModuleId: "test/e2e/live/observed-child-contract.test.ts", + errors: [], + tests: [{ fullName: "audits the observed child", phases: ["prepare", "verify"] }], + source: { + directChildProcessCalls: extraChild.calls, + importsDirectTest: false, + importsSharedTest: true, + phaseCalls: [ + { + file: "test/e2e/live/observed-child-contract.test.ts", + label: "verify", + line: 1, + }, + ], + testPhaseBodies: [], + }, + }); + expect(failures).toEqual( + expect.arrayContaining([ + expect.stringMatching(/audited boundary must contain exactly one direct asynchronous/u), + ]), + ); + }); + + test("rejects unreviewed or structurally forged observed-child progress", () => { + const canonicalSource = fs.readFileSync(FAKE_OPENAI_SOURCE, "utf8"); + expect(scanDirectChildProcessSource(FAKE_OPENAI_SOURCE, canonicalSource).auditFailures).toEqual( + [], + ); + + const forgedProgress = scanDirectChildProcessSource( + FAKE_OPENAI_SOURCE, + canonicalSource.replace( + "progress: options.progress,", + "progress: { activity: () => () => undefined, onOutput: () => undefined },", + ), + ); + expect(forgedProgress.auditFailures).toEqual([ + expect.stringMatching(/progress must retain the reviewed TestProgress capability/u), + ]); + + const unreviewedCallsite = scanDirectChildProcessSource( + FAKE_OPENAI_SOURCE, + `${canonicalSource}\nexport function unreviewed(options: FakeOpenAiCompatibleServerOptions) {\n return spawnObservedChild("x", [], { activityLabel: "x", progress: options.progress, spawn: {} });\n}\n`, + ); + expect(unreviewedCallsite.auditFailures).toEqual([ + expect.stringMatching(/must use a reviewed progress-capability callsite/u), + ]); + + const escapedWrapper = scanDirectChildProcessSource( + FAKE_OPENAI_SOURCE, + `${canonicalSource}\nconst wrapperBag = { spawnObservedChild };\nconst wrapperList = [spawnObservedChild];\nexport { spawnObservedChild };\nvoid [wrapperBag, wrapperList];\n`, + ); + expect(escapedWrapper.auditFailures).toHaveLength(3); + expect(escapedWrapper.auditFailures).toEqual( + expect.arrayContaining([ + expect.stringMatching(/must not be stored in object properties/u), + expect.stringMatching(/must not be stored in array elements/u), + expect.stringMatching(/must not be exported/u), + ]), + ); + + const indirectWrapper = scanDirectChildProcessSource( + FAKE_OPENAI_SOURCE, + `${canonicalSource}\nexport function indirect(options: FakeOpenAiCompatibleServerOptions) {\n return spawnObservedChild.call(null, "x", [], { activityLabel: "x", progress: options.progress, spawn: {} });\n}\n`, + ); + expect(indirectWrapper.auditFailures).toEqual([ + expect.stringMatching(/must be invoked directly without member indirection/u), + ]); + + const namespaceAliasSource = canonicalSource + .replace( + 'import { spawnObservedChild } from "./observed-child-process.ts";', + 'import * as observedChild from "./observed-child-process.ts";\nconst observedAlias = observedChild;', + ) + .replace("child = spawnObservedChild(", "child = observedAlias.spawnObservedChild("); + expect( + scanDirectChildProcessSource(FAKE_OPENAI_SOURCE, namespaceAliasSource).auditFailures, + ).toEqual([]); + expect( + scanDirectChildProcessSource( + FAKE_OPENAI_SOURCE, + namespaceAliasSource.replace( + "progress: options.progress,", + "progress: { activity: () => () => undefined, onOutput: () => undefined },", + ), + ).auditFailures, + ).toEqual([ + expect.stringMatching(/progress must retain the reviewed TestProgress capability/u), + ]); + }); + + test("rejects removal or weakening of the private progress capability", () => { + const progressSource = fs.readFileSync(TEST_PROGRESS_SOURCE, "utf8"); + expect( + scanDirectChildProcessSource(TEST_PROGRESS_SOURCE, progressSource).auditFailures, + ).toEqual([]); + expect( + scanDirectChildProcessSource( + TEST_PROGRESS_SOURCE, + progressSource.replace( + "const TEST_PROGRESS_CAPABILITY: unique symbol", + "export const TEST_PROGRESS_CAPABILITY: unique symbol", + ), + ).auditFailures, + ).toEqual([expect.stringMatching(/module-private unique symbol/u)]); + expect( + scanDirectChildProcessSource( + TEST_PROGRESS_SOURCE, + progressSource.replace("new WeakSet()", "new Set()"), + ).auditFailures, + ).toEqual([expect.stringMatching(/module-private WeakSet/u)]); + expect( + scanDirectChildProcessSource( + TEST_PROGRESS_SOURCE, + progressSource.replace("descriptor.enumerable === false", "descriptor.enumerable === true"), + ).auditFailures, + ).toEqual([expect.stringMatching(/exactly validate the private registry/u)]); + expect( + scanDirectChildProcessSource( + TEST_PROGRESS_SOURCE, + progressSource.replace(" enumerable: false,", " enumerable: true,"), + ).auditFailures, + ).toEqual([expect.stringMatching(/privately brand, register, and freeze/u)]); + expect( + scanDirectChildProcessSource( + TEST_PROGRESS_SOURCE, + progressSource.replace("return Object.freeze(progress);", "return progress;"), + ).auditFailures, + ).toEqual([expect.stringMatching(/privately brand, register, and freeze/u)]); + + const observedSource = fs.readFileSync(OBSERVED_CHILD_PROCESS_SOURCE, "utf8"); + expect( + scanDirectChildProcessSource( + OBSERVED_CHILD_PROCESS_SOURCE, + observedSource.replace( + "export interface ChildProcessProgress extends TestProgressCapability", + "export interface ChildProcessProgress", + ), + ).auditFailures, + ).toEqual([expect.stringMatching(/must require the private TestProgress capability/u)]); + expect( + scanDirectChildProcessSource( + OBSERVED_CHILD_PROCESS_SOURCE, + observedSource.replace(" onOutput(event:", " onOutput?(event:"), + ).auditFailures, + ).toEqual([expect.stringMatching(/must require timestamp-only output observation/u)]); + expect( + scanDirectChildProcessSource( + OBSERVED_CHILD_PROCESS_SOURCE, + observedSource.replace( + "if (!isTestProgressCapability(options.progress))", + "if (false && !isTestProgressCapability(options.progress))", + ), + ).auditFailures, + ).toEqual([expect.stringMatching(/must reject non-canonical progress before spawning/u)]); }); test("rejects phase transitions that belong to sibling tests", () => { diff --git a/test/e2e/support/fixtures/e2e-progress-outcome.fixture.test.ts b/test/e2e/support/fixtures/e2e-progress-outcome.fixture.test.ts index 29fdb2922d..3caf48550c 100644 --- a/test/e2e/support/fixtures/e2e-progress-outcome.fixture.test.ts +++ b/test/e2e/support/fixtures/e2e-progress-outcome.fixture.test.ts @@ -71,3 +71,18 @@ test.runIf(outcome === "incomplete")( }, () => undefined, ); + +test.runIf(outcome === "redacted-event")( + "redacts progress identities and explicit events", + { + meta: { + e2ePhases: ["prepare redacted progress event", "finish redacted progress event"], + }, + }, + ({ expect, progress }) => { + const secret = process.env.NEMOCLAW_E2E_PROGRESS_EVENT_SECRET; + expect(secret, "redacted-event fixture secret is required").toBeTruthy(); + progress.event(`retry cleanup for ${secret}`); + progress.phase("finish redacted progress event"); + }, +); diff --git a/test/e2e/support/fixtures/e2e-progress.fixture.test.ts b/test/e2e/support/fixtures/e2e-progress.fixture.test.ts new file mode 100644 index 0000000000..7bcde92fa1 --- /dev/null +++ b/test/e2e/support/fixtures/e2e-progress.fixture.test.ts @@ -0,0 +1,16 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { test } from "../../fixtures/e2e-test.ts"; + +test.runIf(process.env.NEMOCLAW_E2E_PROGRESS_FIXTURE === "identity")( + "automatic progress fixture writes completed target and shard evidence", + { + meta: { + e2ePhases: ["prepare progress artifact", "record final fixture phase"], + }, + }, + ({ progress }) => { + progress.phase("record final fixture phase"); + }, +); diff --git a/test/e2e/support/fixtures/semantic-phase-child-process.fixture.ts b/test/e2e/support/fixtures/semantic-phase-child-process.fixture.ts new file mode 100644 index 0000000000..46b0305bce --- /dev/null +++ b/test/e2e/support/fixtures/semantic-phase-child-process.fixture.ts @@ -0,0 +1,223 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import * as childProcess from "node:child_process"; +import { exec, execFile, execFileSync, execSync, spawn, spawnSync } from "node:child_process"; +import * as nodeUtil from "node:util"; +import { promisify } from "node:util"; + +declare const require: { + (specifier: "node:child_process"): typeof childProcess; + (specifier: "node:util"): typeof nodeUtil; +}; +const aliasedSpawn = childProcess.spawn; +const requiredChildProcess = require("node:child_process"); +const { spawn: requiredSpawn } = require("node:child_process"); +const reassignedChildProcess = childProcess; +let assignmentChildProcess: typeof childProcess; +assignmentChildProcess = childProcess; +const { spawn: namespaceDestructuredSpawn } = reassignedChildProcess; +let assignmentDestructuredExecFile = childProcess.execFile; +({ execFile: assignmentDestructuredExecFile } = assignmentChildProcess); +const boundSpawn = childProcess.spawn.bind(childProcess); +const boundSpawnSync = childProcess.spawnSync.bind(childProcess); +const extractedSpawnBinder = spawn.bind; +const preappliedSpawnSync = spawnSync.bind(null, "preapplied-child", []); +const conditionalSpawnSync = Math.random() > 0.5 ? spawnSync : childProcess.spawnSync; +let conflictingSyncAlias = spawnSync; +conflictingSyncAlias = execFileSync as unknown as typeof spawnSync; +const promisedExec = promisify(exec); +const promisedExecFile = nodeUtil.promisify(execFile); +const requireAlias = require; +const dynamicallyImportedChildProcess = await import("node:child_process"); + +type ProgressProbe = { + activity(label: string): () => void; + onOutput(event: { stream: "stdout" | "stderr"; atMs: number }): void; +}; + +export function silentAsyncChild(): void { + spawn("silent-child", [], { stdio: ["ignore", "pipe", "pipe"] }); +} + +export function observedAsyncChild(progress: ProgressProbe): void { + const finishActivity = progress.activity("command: observed-child"); + const child = spawn("observed-child", [], { stdio: ["ignore", "pipe", "pipe"] }); + child.stdout?.on("data", () => progress.onOutput({ stream: "stdout", atMs: Date.now() })); + child.stderr?.on("data", () => progress.onOutput({ stream: "stderr", atMs: Date.now() })); + child.once("close", finishActivity); +} + +export function ignoredAsyncChild(progress: ProgressProbe): void { + const finishActivity = progress.activity("command: ignored-child"); + const child = spawn("ignored-child", [], { stdio: "ignore" }); + child.once("close", finishActivity); +} + +export function namespaceAsyncChild(): void { + childProcess.spawn("namespace-child", [], { stdio: "ignore" }); +} + +export function aliasedAsyncChild(): void { + aliasedSpawn("aliased-child", [], { stdio: "ignore" }); +} + +export function requiredNamespaceAsyncChild(): void { + requiredChildProcess.spawn("required-namespace-child", [], { stdio: "ignore" }); +} + +export function requiredDestructuredAsyncChild(): void { + requiredSpawn("required-destructured-child", [], { stdio: "ignore" }); +} + +export function reassignedNamespaceAsyncChild(): void { + reassignedChildProcess.spawn("reassigned-namespace-child", [], { stdio: "ignore" }); +} + +export function namespaceDestructuredAsyncChild(): void { + namespaceDestructuredSpawn("namespace-destructured-child", [], { stdio: "ignore" }); +} + +export function assignmentAliasAsyncChild(): void { + assignmentChildProcess.spawn("assignment-namespace-child", [], { stdio: "ignore" }); + assignmentDestructuredExecFile("assignment-destructured-child", [], () => undefined); +} + +export function nestedRequiredAsyncChild(): void { + const { execFile: nestedExecFile } = require("node:child_process"); + nestedExecFile("nested-required-child", [], () => undefined); +} + +export function directNestedRequiredAsyncChild(): void { + require("node:child_process").fork("direct-required-child"); +} + +export function boundAsyncChild(): void { + boundSpawn("bound-child", [], { stdio: "ignore" }); +} + +export function inlineBoundAsyncChild(): void { + childProcess.execFile.bind(childProcess)("inline-bound-child", [], () => undefined); +} + +export function promisedAsyncChildren(): void { + void promisedExec("promised-exec-child"); + void promisedExecFile("promised-exec-file-child"); + void require("node:util").promisify(require("node:child_process").exec)( + "required-promised-child", + ); +} + +export function aliasedLoaderAsyncChildren(): void { + requireAlias("node:child_process").spawn("aliased-required-child", [], { stdio: "ignore" }); + dynamicallyImportedChildProcess.spawn("dynamically-imported-child", [], { stdio: "ignore" }); +} + +export function extractedBindAsyncChild(): void { + (extractedSpawnBinder as unknown as (receiver: null) => typeof spawn)(null)( + "extracted-bind-child", + [], + { stdio: "ignore" }, + ); +} + +export function computedNamespaceEscape(method: string): void { + const candidate = (childProcess as unknown as Record)[method]; + if (typeof candidate === "function") candidate("computed-child"); +} + +export function passedApiEscape(consume: (value: unknown) => void): void { + consume(childProcess.spawn); +} + +export function storedApiEscapes(): void { + const shorthand = { spawn }; + const property = { runner: execFile }; + const spread = { ...childProcess }; + const array = [spawnSync]; + void [shorthand, property, spread, array]; +} + +export function unsupportedNamespaceMember(): void { + void childProcess.ChildProcess; + const { ChildProcess } = childProcess; + void ChildProcess; +} + +export function unboundedSyncChild(): void { + spawnSync("unbounded-child", []); +} + +export function boundedSyncChild(): void { + spawnSync("bounded-child", [], { killSignal: "SIGKILL", timeout: 1_000 }); +} + +export function zeroTimeoutSyncChild(): void { + spawnSync("zero-timeout-child", [], { killSignal: "SIGKILL", timeout: 0 }); +} + +export function undefinedTimeoutSyncChild(): void { + spawnSync("undefined-timeout-child", [], { killSignal: "SIGKILL", timeout: undefined }); +} + +export function heartbeatLengthTimeoutSyncChild(): void { + spawnSync("heartbeat-timeout-child", [], { killSignal: "SIGKILL", timeout: 5 * 60_000 }); +} + +export function missingHardKillSyncChild(): void { + spawnSync("missing-hard-kill-child", [], { timeout: 1_000 }); +} + +export function softKillSyncChild(): void { + spawnSync("soft-kill-child", [], { killSignal: "SIGTERM", timeout: 1_000 }); +} + +export function undefinedKillSyncChild(): void { + spawnSync("undefined-kill-child", [], { killSignal: undefined, timeout: 1_000 }); +} + +export function otherBoundedSyncChildren(): void { + execSync("bounded-exec", { killSignal: "SIGKILL", timeout: 1_000 }); + execFileSync("bounded-exec-file", [], { killSignal: "SIGKILL", timeout: 1_000 }); + boundSpawnSync("bounded-bound-spawn", [], { killSignal: "SIGKILL", timeout: 1_000 }); +} + +export function decoyAndPreappliedSyncOptions(): void { + (spawnSync as unknown as (...args: unknown[]) => unknown)( + "decoy-child", + [], + { killSignal: "SIGTERM", timeout: 0 }, + { killSignal: "SIGKILL", timeout: 1_000 }, + ); + (execFileSync as unknown as (...args: unknown[]) => unknown)( + "misplaced-options-child", + { killSignal: "SIGTERM", timeout: 0 }, + { killSignal: "SIGKILL", timeout: 1_000 }, + ); + preappliedSpawnSync({ killSignal: "SIGKILL", timeout: 1_000 }); + conditionalSpawnSync("conditional-child", [], { + killSignal: "SIGKILL", + timeout: 1_000, + }); +} + +export function conflictingReassignmentSyncOptions(): void { + conflictingSyncAlias("conflicting-child", [], { + killSignal: "SIGKILL", + timeout: 1_000, + }); +} + +export function spreadSyncOptions(): void { + const overrides: Record = { killSignal: "SIGTERM", timeout: 0 }; + spawnSync("unsafe-spread-child", [], { + killSignal: "SIGKILL", + timeout: 1_000, + ...overrides, + }); + spawnSync("safe-spread-child", [], { + ...overrides, + killSignal: "SIGKILL", + timeout: 1_000, + }); +} diff --git a/test/e2e/support/fixtures/workflow-e2e-progress.fixture.test.ts b/test/e2e/support/fixtures/workflow-e2e-progress.fixture.test.ts new file mode 100644 index 0000000000..6165fe2c70 --- /dev/null +++ b/test/e2e/support/fixtures/workflow-e2e-progress.fixture.test.ts @@ -0,0 +1,16 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { test } from "../../fixtures/workflow-e2e-test.ts"; + +test.runIf(process.env.NEMOCLAW_WORKFLOW_PROGRESS_FIXTURE === "redacted-fallback")( + "records shared-job identity without exposing secrets", + { + meta: { + e2ePhases: ["prepare shared workflow fixture", "release shared workflow fixture"], + }, + }, + ({ progress }) => { + progress.phase("release shared workflow fixture"); + }, +); diff --git a/test/e2e/support/hosted-inference.test.ts b/test/e2e/support/hosted-inference.test.ts index 7eba5c50df..c98a88db43 100644 --- a/test/e2e/support/hosted-inference.test.ts +++ b/test/e2e/support/hosted-inference.test.ts @@ -11,6 +11,7 @@ import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; import { ProviderClient, trustedProviderEndpoint } from "../fixtures/clients/provider.ts"; import { startFakeOpenAiCompatibleServer } from "../fixtures/fake-openai-compatible.ts"; import { requireHostedInferenceConfig } from "../fixtures/hosted-inference.ts"; +import { startTestProgress } from "../fixtures/progress.ts"; import type { ShellProbeResult, ShellProbeRunOptions, @@ -372,11 +373,18 @@ describe("hosted inference E2E config", () => { }); it("serves fake OpenAI-compatible chat and responses contracts", async () => { + const progressLines: string[] = []; + const progress = startTestProgress( + "fake compatible server support", + ["serve compatible API", "verify compatible API"], + { logLine: (line) => progressLines.push(line) }, + ); const fake = await startFakeOpenAiCompatibleServer({ apiKey: "fake-compatible-key", chatContent: "CHAT_OK", forbiddenMarkers: ["FORBIDDEN_REQUEST_MARKER"], model: "nvidia/nvidia/fake-model", + progress, requireAuth: true, responseText: "RESP_OK", }); @@ -454,5 +462,11 @@ describe("hosted inference E2E config", () => { } finally { await fake.close(); } + expect(progressLines).toEqual( + expect.arrayContaining([ + expect.stringContaining("event: fake OpenAI-compatible server started"), + expect.stringContaining("event: fake OpenAI-compatible server stopped"), + ]), + ); }); }); diff --git a/test/e2e/support/inference-adapter.test.ts b/test/e2e/support/inference-adapter.test.ts index d3b71a3d0a..6467943e79 100644 --- a/test/e2e/support/inference-adapter.test.ts +++ b/test/e2e/support/inference-adapter.test.ts @@ -18,8 +18,14 @@ import { type E2EInferenceAdapter, requirePublicNvidiaInferenceKey, } from "../fixtures/inference-adapter.ts"; +import { startTestProgress } from "../fixtures/progress.ts"; const adapters: E2EInferenceAdapter[] = []; +const NOOP_PROGRESS = startTestProgress( + "inference adapter support", + ["serve inference endpoint", "verify inference adapter"], + { logLine: () => undefined }, +); function artifacts(): ArtifactSink { const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-inference-adapter-test-")); @@ -117,6 +123,7 @@ async function createAdapter(options: { artifacts: options.artifacts ?? artifacts(), env: options.env ?? {}, provider: options.provider ?? provider(), + progress: NOOP_PROGRESS, secrets: secrets(options.secrets ?? {}), }); adapters.push(adapter); @@ -173,7 +180,7 @@ describe("E2E inference adapter", () => { }); expect(Object.values(adapter.env())).not.toContain(secretValue); - const fake = await startFakeOpenAiCompatibleServer(); + const fake = await startFakeOpenAiCompatibleServer({ progress: NOOP_PROGRESS }); try { expect(fake.environmentKeys()).toContain("NEMOCLAW_FAKE_OPENAI_API_KEY"); expect(fake.environmentKeys()).not.toContain(secretName); diff --git a/test/e2e/support/observed-child-process.test.ts b/test/e2e/support/observed-child-process.test.ts new file mode 100644 index 0000000000..3ce6736079 --- /dev/null +++ b/test/e2e/support/observed-child-process.test.ts @@ -0,0 +1,109 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { ChildProcess } from "node:child_process"; + +import { describe, expect, test } from "vitest"; + +import { spawnObservedChild } from "../fixtures/observed-child-process.ts"; +import { startTestProgress, type TestProgress } from "../fixtures/progress.ts"; + +function observedProgress(): TestProgress { + return startTestProgress("observed child support", ["run observed child", "verify observation"], { + logLine: () => undefined, + }); +} + +function waitForClose( + child: ChildProcess, +): Promise<{ code: number | null; signal: string | null }> { + return new Promise((resolve, reject) => { + child.once("error", reject); + child.once("close", (code, signal) => resolve({ code, signal })); + }); +} + +describe("observed E2E child process", () => { + test("ties activity completion and timestamp-only output events to the child", async () => { + const secret = "OBSERVED_CHILD_SECRET"; + const lines: string[] = []; + const timers: Array<() => void> = []; + const progress = startTestProgress( + "observed child support", + ["run observed child", "verify observation"], + { + clearTimer: () => undefined, + logLine: (line) => lines.push(line), + setTimer: (callback) => { + timers.push(callback); + return {}; + }, + }, + ); + const child = spawnObservedChild( + process.execPath, + ["-e", `process.stdout.write(${JSON.stringify(secret)}); process.stderr.write("err")`], + { + activityLabel: "command: observed-child-contract", + progress, + spawn: { stdio: ["ignore", "pipe", "pipe"] }, + }, + ); + + await expect(waitForClose(child)).resolves.toEqual({ code: 0, signal: null }); + timers[0]?.(); + expect(lines.at(-1)).toContain("no active command"); + progress.stop(); + const phase = progress.summary().phases[0]; + expect(phase?.outputEvents).toBe(2); + expect(phase?.lastOutputAtMs).toEqual(expect.any(Number)); + expect(JSON.stringify(progress.summary())).not.toContain(secret); + }); + + test("keeps process execution independent from rejected activity diagnostics", async () => { + const child = spawnObservedChild(process.execPath, ["-e", "process.stdout.write('ok')"], { + activityLabel: "invalid\nactivity", + progress: observedProgress(), + spawn: { stdio: ["ignore", "pipe", "pipe"] }, + }); + + await expect(waitForClose(child)).resolves.toEqual({ code: 0, signal: null }); + }); + + test("finishes the activity when spawn throws synchronously", () => { + const lines: string[] = []; + const timers: Array<() => void> = []; + const progress = startTestProgress( + "observed child support", + ["run observed child", "verify observation"], + { + clearTimer: () => undefined, + logLine: (line) => lines.push(line), + setTimer: (callback) => { + timers.push(callback); + return {}; + }, + }, + ); + expect(() => + spawnObservedChild("bad\0command", [], { + activityLabel: "command: invalid-spawn", + progress, + spawn: { stdio: "ignore" }, + }), + ).toThrow(); + timers[0]?.(); + expect(lines.at(-1)).toContain("no active command"); + }); + + test("rejects a derived look-alike that copied the private capability", () => { + const derived = { ...observedProgress() } as unknown as TestProgress; + expect(() => + spawnObservedChild(process.execPath, ["-e", "process.exit(0)"], { + activityLabel: "command: derived-progress", + progress: derived, + spawn: { stdio: "ignore" }, + }), + ).toThrow(/canonical E2E progress capability/); + }); +}); diff --git a/test/e2e/support/openshell-gateway-auth-contract-workflow-boundary.test.ts b/test/e2e/support/openshell-gateway-auth-contract-workflow-boundary.test.ts index 461d6ef3a7..c762e83e4c 100644 --- a/test/e2e/support/openshell-gateway-auth-contract-workflow-boundary.test.ts +++ b/test/e2e/support/openshell-gateway-auth-contract-workflow-boundary.test.ts @@ -46,13 +46,24 @@ describe("OpenShell gateway auth contract workflow boundary", () => { )!; run.env = { GITHUB_TOKEN: "${{ github.token }}" }; run.run = "npx vitest run --project e2e-live test/e2e/live/other.test.ts"; + + const artifactSafety = steps.find( + (step) => step.name === "Validate final OpenShell gateway auth contract artifacts", + )!; + artifactSafety.id = "unsafe_scan"; + artifactSafety.if = "success()"; + artifactSafety.run = "true"; steps.splice(steps.indexOf(prePull), 1); steps.splice(steps.indexOf(run) + 1, 0, prePull); const upload = steps.find( (step) => step.name === "Upload OpenShell gateway auth contract artifacts", )!; - upload.if = "success()"; + upload.uses = "actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a"; + upload.if = "always()"; + upload.with = { path: "e2e-artifacts/live/openshell-gateway-auth-contract/" }; + steps.splice(steps.indexOf(artifactSafety), 1); + steps.splice(steps.indexOf(upload) + 1, 0, artifactSafety); expect(validateOpenShellGatewayAuthContractWorkflow(workflow)).toEqual( expect.arrayContaining([ @@ -70,9 +81,26 @@ describe("OpenShell gateway auth contract workflow boundary", () => { "openshell-gateway-auth-contract step 'Install OpenShell CLI' must run: -u DOCKER_CONFIG", "openshell-gateway-auth-contract step 'Pre-pull pinned gateway auth probe image' must run: docker pull \"$DOCKER_GRPC_PROBE_IMAGE\"", "openshell-gateway-auth-contract live test must not receive workflow credentials", - "openshell-gateway-auth-contract must always use the reviewed artifact uploader", + "openshell-gateway-auth-contract final artifact safety scan must run unconditionally with a stable id", + "openshell-gateway-auth-contract step 'Validate final OpenShell gateway auth contract artifacts' must run exactly: node --experimental-strip-types --no-warnings tools/e2e/openshell-gateway-auth-artifact-safety.mts \"$E2E_ARTIFACT_DIR\"", + "openshell-gateway-auth-contract must use the reviewed artifact uploader", + "openshell-gateway-auth-contract must upload artifacts only after this run attempt passes safety scan", + "openshell-gateway-auth-contract must upload only the immutable approved artifact payload", "openshell-gateway-auth-contract step 'Pre-pull pinned gateway auth probe image' must precede 'Run OpenShell gateway auth contract live test'", + "openshell-gateway-auth-contract step 'Validate final OpenShell gateway auth contract artifacts' must precede 'Upload OpenShell gateway auth contract artifacts'", ]), ); }); + + it("rejects artifact safety commands that can mask scanner failures (#7101)", () => { + const workflow = readOpenShellGatewayAuthContractWorkflow(); + const artifactSafety = workflow.jobs["openshell-gateway-auth-contract"].steps!.find( + (step) => step.name === "Validate final OpenShell gateway auth contract artifacts", + )!; + artifactSafety.run = `${artifactSafety.run} || true`; + + expect(validateOpenShellGatewayAuthContractWorkflow(workflow)).toContain( + "openshell-gateway-auth-contract step 'Validate final OpenShell gateway auth contract artifacts' must run exactly: node --experimental-strip-types --no-warnings tools/e2e/openshell-gateway-auth-artifact-safety.mts \"$E2E_ARTIFACT_DIR\"", + ); + }); }); diff --git a/test/e2e/support/openshell-gateway-auth-source-contract-helpers.test.ts b/test/e2e/support/openshell-gateway-auth-source-contract-helpers.test.ts index dc6fc54af4..d48f1ae500 100644 --- a/test/e2e/support/openshell-gateway-auth-source-contract-helpers.test.ts +++ b/test/e2e/support/openshell-gateway-auth-source-contract-helpers.test.ts @@ -6,10 +6,15 @@ import os from "node:os"; import path from "node:path"; import { describe, expect, it, vi } from "vitest"; - +import { + openShellGatewayAuthArtifactSafetyMarkerName, + scanAndApproveOpenShellGatewayAuthArtifacts, +} from "../../../tools/e2e/openshell-gateway-auth-artifact-safety.mts"; +import { ArtifactSink } from "../fixtures/artifacts.ts"; import { assertOpenShellGatewayAuthArtifactsSafe, buildSandboxTokenContainerProbeDockerArgs, + registerSandboxJwtArtifactRedaction, skipUnavailableProbeImage, withOpenShellGatewayAuthArtifactSafety, } from "../live/openshell-gateway-auth-source-contract-helpers.ts"; @@ -124,6 +129,33 @@ describe("OpenShell gateway auth source contract helpers", () => { }); }); + it("redacts a minted sandbox token before gateway output reaches an artifact (#7101)", async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-auth-artifact-redaction-")); + try { + const artifacts = new ArtifactSink(dir); + const sandboxToken = "opaque-sandbox-token-not-covered-by-canonical-patterns"; + registerSandboxJwtArtifactRedaction(artifacts, sandboxToken); + + await artifacts.writeText("openshell-gateway.log", `gateway echoed ${sandboxToken}\n`); + + const content = fs.readFileSync(path.join(dir, "openshell-gateway.log"), "utf8"); + expect(content).toContain("[REDACTED]"); + expect(content).not.toContain(sandboxToken); + expect(() => assertOpenShellGatewayAuthArtifactsSafe(dir)).not.toThrow(); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); + + it("binds artifact safety approval to the current GitHub run attempt (#7101)", () => { + expect( + openShellGatewayAuthArtifactSafetyMarkerName({ + GITHUB_RUN_ATTEMPT: "4", + GITHUB_RUN_ID: "29897237525", + }), + ).toBe("artifact-safety-29897237525-4.passed"); + }); + it.each([ ["authorization header", '{"authorization":"redacted"}\n'], [ @@ -168,8 +200,10 @@ describe("OpenShell gateway auth source contract helpers", () => { }); }); - it("scans artifacts in the failure path before a workflow upload can run", async () => { - const dir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-auth-artifact-scan-")); + it("removes rejected artifacts before an unconditional workflow upload can run (#7101)", async () => { + const parent = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-auth-artifact-scan-")); + const dir = path.join(parent, "uploadable-auth-artifacts"); + fs.mkdirSync(dir); try { await expect( withOpenShellGatewayAuthArtifactSafety(dir, async () => { @@ -177,8 +211,232 @@ describe("OpenShell gateway auth source contract helpers", () => { throw new Error("scenario failed"); }), ).rejects.toThrow(/failed-probe\.json.*authorization header/); + expect(fs.existsSync(dir)).toBe(false); + expect(fs.readdirSync(parent)).toEqual([]); + } finally { + fs.rmSync(parent, { recursive: true, force: true }); + } + }); + + it("preserves safe diagnostics when the scenario itself fails (#7101)", async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-auth-artifact-safe-failure-")); + try { + await expect( + withOpenShellGatewayAuthArtifactSafety(dir, async () => { + fs.writeFileSync(path.join(dir, "failed-probe.json"), '{"status":"failed"}\n'); + throw new Error("scenario failed"); + }), + ).rejects.toThrow("scenario failed"); + expect(fs.readFileSync(path.join(dir, "failed-probe.json"), "utf8")).toContain( + '"status":"failed"', + ); + const approved = scanAndApproveOpenShellGatewayAuthArtifacts(dir); + try { + expect(fs.readFileSync(path.join(approved, "failed-probe.json"), "utf8")).toContain( + '"status":"failed"', + ); + expect( + fs.existsSync(path.join(approved, openShellGatewayAuthArtifactSafetyMarkerName())), + ).toBe(true); + } finally { + fs.rmSync(approved, { recursive: true, force: true }); + } } finally { fs.rmSync(dir, { recursive: true, force: true }); } }); + + it("withholds safety approval when quarantine and deletion both fail (#7101)", async () => { + const parent = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-auth-artifact-fail-closed-")); + const dir = path.join(parent, "uploadable-auth-artifacts"); + fs.mkdirSync(dir); + fs.writeFileSync(path.join(dir, "failed-probe.json"), '{"authorization":"redacted"}\n'); + vi.stubEnv("GITHUB_RUN_ID", "29897237525"); + vi.stubEnv("GITHUB_RUN_ATTEMPT", "9"); + const originalRmSync = fs.rmSync.bind(fs); + const renameSpy = vi.spyOn(fs, "renameSync").mockImplementation(() => { + throw new Error("simulated quarantine move failure"); + }); + const rmSpy = vi.spyOn(fs, "rmSync").mockImplementation((target, options) => { + expect(path.resolve(String(target)), "simulated artifact deletion failure").not.toBe( + path.resolve(dir), + ); + originalRmSync(target, options); + }); + + try { + await expect( + Promise.resolve().then(() => scanAndApproveOpenShellGatewayAuthArtifacts(dir)), + ).rejects.toThrow(/failed safety approval and quarantine/); + expect(fs.existsSync(dir)).toBe(true); + } finally { + rmSpy.mockRestore(); + renameSpy.mockRestore(); + vi.unstubAllEnvs(); + originalRmSync(parent, { recursive: true, force: true }); + } + }); + + it("deletes rejected artifacts when quarantine cannot cross filesystems (#7101)", () => { + const parent = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-auth-artifact-exdev-")); + const dir = path.join(parent, "uploadable-auth-artifacts"); + fs.mkdirSync(dir); + fs.writeFileSync(path.join(dir, "failed-probe.json"), '{"authorization":"redacted"}\n'); + let quarantineRoot: string | undefined; + const renameSpy = vi.spyOn(fs, "renameSync").mockImplementation((_source, destination) => { + quarantineRoot = path.dirname(String(destination)); + throw Object.assign(new Error("simulated cross-device quarantine move"), { code: "EXDEV" }); + }); + + try { + let rejection: unknown; + try { + scanAndApproveOpenShellGatewayAuthArtifacts(dir); + } catch (error) { + rejection = error; + } + expect(rejection).toBeInstanceOf(Error); + expect((rejection as Error).message).toMatch(/failed-probe\.json.*authorization header/); + expect(rejection).not.toBeInstanceOf(AggregateError); + expect(fs.existsSync(dir)).toBe(false); + expect(quarantineRoot).toBeDefined(); + expect(fs.existsSync(quarantineRoot as string)).toBe(false); + } finally { + renameSpy.mockRestore(); + fs.rmSync(parent, { recursive: true, force: true }); + } + }); + + it("rejects an unsafe artifact written after scenario finalization (#7101)", async () => { + const parent = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-auth-artifact-post-test-")); + const dir = path.join(parent, "uploadable-auth-artifacts"); + fs.mkdirSync(dir); + try { + await withOpenShellGatewayAuthArtifactSafety(dir, async () => { + fs.writeFileSync(path.join(dir, "scenario.json"), '{"status":"passed"}\n'); + }); + expect(fs.existsSync(path.join(dir, openShellGatewayAuthArtifactSafetyMarkerName()))).toBe( + false, + ); + + fs.writeFileSync(path.join(dir, "cleanup.json"), '{"authorization":"leaked"}\n'); + + expect(() => scanAndApproveOpenShellGatewayAuthArtifacts(dir)).toThrow( + /cleanup\.json.*authorization header/, + ); + expect(fs.existsSync(dir)).toBe(false); + } finally { + fs.rmSync(parent, { recursive: true, force: true }); + } + }); + + it("uploads a vetted staging payload that later source mutations cannot change (#7101)", () => { + const parent = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-auth-artifact-staging-")); + const dir = path.join(parent, "uploadable-auth-artifacts"); + fs.mkdirSync(dir); + fs.writeFileSync(path.join(dir, "scenario.json"), '{"status":"passed"}\n'); + const approved = scanAndApproveOpenShellGatewayAuthArtifacts(dir, { + GITHUB_RUN_ATTEMPT: "3", + GITHUB_RUN_ID: "29897237525", + }); + try { + fs.writeFileSync(path.join(dir, "late.json"), '{"authorization":"leaked"}\n'); + + expect(fs.existsSync(path.join(approved, "late.json"))).toBe(false); + expect(fs.readFileSync(path.join(approved, "scenario.json"), "utf8")).toBe( + '{"status":"passed"}\n', + ); + expect(fs.existsSync(path.join(approved, "artifact-safety-29897237525-3.passed"))).toBe(true); + } finally { + fs.rmSync(parent, { recursive: true, force: true }); + fs.rmSync(approved, { recursive: true, force: true }); + } + }); + + it("rejects a scanned nested directory replaced before the approved copy (#7101)", () => { + const parent = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-auth-artifact-dir-swap-")); + const dir = path.join(parent, "uploadable-auth-artifacts"); + const nested = path.join(dir, "nested"); + const originalNested = path.join(parent, "original-nested"); + const outside = path.join(parent, "outside"); + fs.mkdirSync(nested, { recursive: true }); + fs.mkdirSync(outside); + fs.writeFileSync(path.join(nested, "scenario.json"), '{"status":"passed"}\n'); + fs.writeFileSync(path.join(outside, "scenario.json"), '{"status":"external"}\n'); + const originalMkdir = fs.mkdirSync.bind(fs); + let approvedRoot: string | undefined; + let approvedNested: string | undefined; + const mkdirSpy = vi.spyOn(fs, "mkdirSync").mockImplementationOnce((target, options) => { + const result = originalMkdir(target, options); + approvedNested = String(target); + approvedRoot = path.dirname(approvedNested); + fs.renameSync(nested, originalNested); + fs.symlinkSync(outside, nested, "dir"); + return result; + }); + + try { + expect(() => scanAndApproveOpenShellGatewayAuthArtifacts(dir)).toThrow( + /nested.*entry identity changed during safety approval/, + ); + expect(path.basename(approvedNested as string)).toBe("nested"); + expect(path.basename(approvedRoot as string)).toMatch(/^nemoclaw-approved-auth-artifacts-/); + expect(approvedRoot).toBeDefined(); + expect(fs.existsSync(approvedRoot as string)).toBe(false); + expect(fs.existsSync(dir)).toBe(false); + expect(fs.readFileSync(path.join(outside, "scenario.json"), "utf8")).toContain("external"); + } finally { + mkdirSpy.mockRestore(); + fs.rmSync(parent, { recursive: true, force: true }); + } + }); + + it("closes the source when the approved artifact cannot be opened (#7101)", () => { + const parent = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-auth-artifact-open-failure-")); + const dir = path.join(parent, "uploadable-auth-artifacts"); + const artifactPath = path.join(dir, "scenario.json"); + fs.mkdirSync(dir); + fs.writeFileSync(artifactPath, '{"status":"passed"}\n'); + const originalOpen = fs.openSync.bind(fs); + let scanSourceFd: number | undefined; + let copySourceFd: number | undefined; + const openSpy = vi + .spyOn(fs, "openSync") + .mockImplementationOnce((target, flags, mode) => { + scanSourceFd = originalOpen(target, flags, mode); + return scanSourceFd; + }) + .mockImplementationOnce((target, flags, mode) => { + copySourceFd = originalOpen(target, flags, mode); + return copySourceFd; + }) + .mockImplementationOnce(() => { + throw new Error("simulated approved destination open failure"); + }); + try { + expect(() => scanAndApproveOpenShellGatewayAuthArtifacts(dir)).toThrow( + "simulated approved destination open failure", + ); + expect(scanSourceFd).toBeDefined(); + expect(copySourceFd).toBeDefined(); + expect(() => fs.fstatSync(scanSourceFd as number)).toThrowError( + expect.objectContaining({ code: "EBADF" }), + ); + expect(() => fs.fstatSync(copySourceFd as number)).toThrowError( + expect.objectContaining({ code: "EBADF" }), + ); + } finally { + openSpy.mockRestore(); + for (const descriptor of [scanSourceFd, copySourceFd].filter( + (candidate): candidate is number => candidate !== undefined, + )) { + try { + fs.closeSync(descriptor); + } catch { + // The implementation already closed the descriptor. + } + } + fs.rmSync(parent, { recursive: true, force: true }); + } + }); }); diff --git a/test/e2e/support/rebuild-hermes-progress.test.ts b/test/e2e/support/rebuild-hermes-progress.test.ts index 70a681de3e..57631739fa 100644 --- a/test/e2e/support/rebuild-hermes-progress.test.ts +++ b/test/e2e/support/rebuild-hermes-progress.test.ts @@ -62,14 +62,49 @@ describe("Hermes rebuild live progress", () => { ]); expect(state.lines).toHaveLength(linesAfterStop); expect(state.lines).toEqual([ - "[e2e phase 1/2] run authoritative Hermes rebuild", - "[e2e phase 1/2] still running: run authoritative Hermes rebuild (phase 5m; child output 4m ago; no active command; rss 0.5 GiB; memory free 8.0 GiB/16.0 GiB; disk free 6.0 GiB; load 2.50)", + '[e2e target="unassigned" scenario="rebuild-hermes"] [phase 1/2] started: run authoritative Hermes rebuild (total 0s; phase 0s)', + '[e2e target="unassigned" scenario="rebuild-hermes"] [phase 1/2] still running: run authoritative Hermes rebuild (total 5m; phase 5m; child output 4m ago; no active command; rss 0.5 GiB; memory free 8.0 GiB/16.0 GiB; disk free 6.0 GiB; load 2.50)', 'E2E_RESOURCE_SNAPSHOT {"phase":"run authoritative Hermes rebuild"}', - "[e2e phase 1/2] run authoritative Hermes rebuild — passed in 5m; next 2/2: remove rebuilt Hermes resources", - "[e2e phase 2/2] remove rebuilt Hermes resources — passed in 0s", + '[e2e target="unassigned" scenario="rebuild-hermes"] [phase 1/2] completed: run authoritative Hermes rebuild — passed in 5m (total 5m)', + '[e2e target="unassigned" scenario="rebuild-hermes"] [phase 2/2] started: remove rebuilt Hermes resources (total 5m; phase 0s)', + '[e2e target="unassigned" scenario="rebuild-hermes"] [phase 2/2] completed: remove rebuilt Hermes resources — passed in 0s (total 5m)', ]); }); + it("reports a target, scenario, total time, and content-free status events", () => { + const { options, state } = progressHarness(); + options.targetId = "rebuild-hermes-target"; + const progress = startTestProgress( + "rebuild-hermes scenario", + ["pull historical base", "validate rebuilt sandbox"], + options, + ); + + state.clockMs = 61_000; + progress.event("historical base pull timed out; retrying attempt 2"); + progress.stop("failed"); + + expect(state.lines).toEqual([ + '[e2e target="rebuild-hermes-target" scenario="rebuild-hermes scenario"] [phase 1/2] started: pull historical base (total 0s; phase 0s)', + '[e2e target="rebuild-hermes-target" scenario="rebuild-hermes scenario"] [phase 1/2] event: historical base pull timed out; retrying attempt 2 (total 1m; phase 1m)', + '[e2e target="rebuild-hermes-target" scenario="rebuild-hermes scenario"] [phase 1/2] completed: pull historical base — failed in 1m (total 1m)', + ]); + expect(() => progress.event("ignored after stop\nsecret-shaped payload")).not.toThrow(); + + const activeProgress = startTestProgress( + "event-validation", + ["prepare event validation", "finish event validation"], + { ...options, logLine: () => undefined }, + ); + expect(() => activeProgress.event("invalid\nsecret-shaped payload")).toThrowError( + /^invalid live E2E progress event label$/u, + ); + expect(() => activeProgress.activity("invalid\nsecret-shaped payload")).toThrowError( + /^invalid live E2E progress activity label$/u, + ); + activeProgress.stop(); + }); + it("keeps diagnostics best-effort when host sampling and output fail", () => { const { options, state } = progressHarness(); options.logLine = vi.fn(() => { diff --git a/test/e2e/support/upload-e2e-artifacts-workflow-boundary.test.ts b/test/e2e/support/upload-e2e-artifacts-workflow-boundary.test.ts index f33c983509..45277222bd 100644 --- a/test/e2e/support/upload-e2e-artifacts-workflow-boundary.test.ts +++ b/test/e2e/support/upload-e2e-artifacts-workflow-boundary.test.ts @@ -147,6 +147,7 @@ describe("upload-e2e-artifacts workflow boundary", () => { uploadStep(workflow.jobs["hermes-slack"]).with!.path = "e2e-artifacts/live/hermes-slack/"; uploadStep(workflow.jobs["gpu-e2e"]).if = "success()"; uploadStep(workflow.jobs["mcp-bridge"]).if = "always()"; + uploadStep(workflow.jobs["openshell-gateway-auth-contract"]).if = "always()"; uploadStep(workflow.jobs["shared-e2e"]).env = { UNEXPECTED: "1" }; workflow.jobs["shared-e2e"].env!.E2E_EXECUTION_PROFILE = "credential-free"; workflow.jobs["shared-e2e"].env!.E2E_JOB = "1"; @@ -164,6 +165,7 @@ describe("upload-e2e-artifacts workflow boundary", () => { "hermes-slack upload-e2e-artifacts must preserve its explicit name/path contract", "gpu-e2e upload-e2e-artifacts invocation must run with always()", "mcp-bridge upload-e2e-artifacts invocation must remain gated by its reviewed pre-upload checks", + "openshell-gateway-auth-contract upload-e2e-artifacts invocation must remain gated by its reviewed pre-upload checks", "shared-e2e must not declare E2E_EXECUTION_PROFILE", "shared-e2e must not declare E2E_JOB", "shared-e2e upload-e2e-artifacts invocation must not override its contract", diff --git a/test/e2e/support/workflow-e2e-progress.test.ts b/test/e2e/support/workflow-e2e-progress.test.ts new file mode 100644 index 0000000000..b6eff1ebe2 --- /dev/null +++ b/test/e2e/support/workflow-e2e-progress.test.ts @@ -0,0 +1,67 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { REPO_ROOT } from "../fixtures/paths.ts"; +import type { ProgressSummary } from "../fixtures/progress.ts"; + +const VITEST = path.join(REPO_ROOT, "node_modules", "vitest", "vitest.mjs"); +const FIXTURE = "test/e2e/support/fixtures/workflow-e2e-progress.fixture.test.ts"; +const ARTIFACT_SLUG = "records-shared-job-identity-without-exposing-secrets"; + +describe("workflow-selected integration progress", () => { + it("falls back to the job identity and redacts console and artifact output", () => { + const artifactRoot = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-workflow-progress-")); + const sensitiveTarget = "workflow-fixture-sensitive-target"; + try { + const result = spawnSync( + process.execPath, + [VITEST, "run", "--project", "e2e-support", FIXTURE, "--reporter=default"], + { + cwd: REPO_ROOT, + encoding: "utf8", + killSignal: "SIGKILL", + timeout: 20_000, + env: { + ...process.env, + E2E_ARTIFACT_DIR: artifactRoot, + E2E_TARGET_ID: "", + GITHUB_JOB: sensitiveTarget, + NEMOCLAW_RUN_LIVE_E2E: "1", + NEMOCLAW_WORKFLOW_PROGRESS_FIXTURE: "redacted-fallback", + WORKFLOW_FIXTURE_API_KEY: sensitiveTarget, + }, + }, + ); + + const output = `${result.stdout}\n${result.stderr}`; + expect(result.status, output).toBe(0); + expect(output).not.toContain(sensitiveTarget); + expect(output).toContain('target="[REDACTED]"'); + + const artifactText = fs.readFileSync( + path.join(artifactRoot, ARTIFACT_SLUG, "test-progress.json"), + "utf8", + ); + const summary = JSON.parse(artifactText) as ProgressSummary; + expect(artifactText).not.toContain(sensitiveTarget); + expect(summary).toMatchObject({ + scenario: "records shared-job identity without exposing secrets", + targetId: "[REDACTED]", + version: 1, + }); + expect(summary.phases.at(-1)).toMatchObject({ + label: "release shared workflow fixture", + outcome: "passed", + }); + } finally { + fs.rmSync(artifactRoot, { force: true, recursive: true }); + } + }); +}); diff --git a/test/gateway-drift-preflight.test.ts b/test/gateway-drift-preflight.test.ts index df2ab1f450..6f19758142 100644 --- a/test/gateway-drift-preflight.test.ts +++ b/test/gateway-drift-preflight.test.ts @@ -2,29 +2,46 @@ // SPDX-License-Identifier: Apache-2.0 // @module-tag e2e/credential-free -import { spawn, spawnSync } from "node:child_process"; +import { spawnSync } from "node:child_process"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { afterAll, describe, expect, it } from "vitest"; - +import { describe, expect } from "vitest"; +import { spawnObservedChild } from "./e2e/fixtures/observed-child-process.ts"; +import type { TestProgress, TestProgressCapability } from "./e2e/fixtures/progress.ts"; +import { test as it } from "./e2e/fixtures/workflow-e2e-test.ts"; import { nodeOptionsWithoutSourceLoader } from "./helpers/source-loader-options"; import { testTimeoutOptions } from "./helpers/timeouts"; const REPO_ROOT = path.join(import.meta.dirname, ".."); const CLI_ENTRYPOINT = path.join(REPO_ROOT, "bin", "nemoclaw.js"); const ARTIFACT_ROOT = process.env.E2E_ARTIFACT_DIR; -const WORK_ROOT = (() => { +let workRoot: string | null = null; + +function createGatewayDriftWorkRoot(): string { const parent = ARTIFACT_ROOT ?? os.tmpdir(); fs.mkdirSync(parent, { recursive: true }); return fs.mkdtempSync(path.join(parent, "nemoclaw-gateway-drift-preflight-")); -})(); +} + +function gatewayDriftWorkRoot(): string { + workRoot ??= createGatewayDriftWorkRoot(); + return workRoot; +} + const commandTimeoutMs = 45_000; +const GATEWAY_DRIFT_PHASES = [ + "prepare gateway drift fixtures", + "exercise container-backed gateway drift cases", + "exercise host-process gateway drift cases", + "verify stale gateway marker fallback", + "release gateway drift fixtures", +] as const; const liveGatewayPids: number[] = []; -afterAll(() => { +function releaseGatewayDriftFixtures(): void { for (const pid of liveGatewayPids.splice(0)) { try { process.kill(pid, "SIGTERM"); @@ -32,8 +49,11 @@ afterAll(() => { // Already exited. } } - if (!ARTIFACT_ROOT) fs.rmSync(WORK_ROOT, { recursive: true, force: true }); -}); + if (!ARTIFACT_ROOT && workRoot) { + fs.rmSync(workRoot, { recursive: true, force: true }); + workRoot = null; + } +} type CommandResult = { caseDir: string; @@ -212,7 +232,7 @@ function writeHostProcessMarker(home: string, gatewayBin: string, pid = 999999): } function prepareCase(name: string): { binDir: string; caseDir: string; home: string } { - const caseDir = path.join(WORK_ROOT, name); + const caseDir = path.join(gatewayDriftWorkRoot(), name); const home = path.join(caseDir, "home"); const binDir = path.join(caseDir, "bin"); fs.mkdirSync(home, { recursive: true }); @@ -251,6 +271,7 @@ function runCli(caseDir: string, home: string, binDir: string, args: string[]): "openshell-docker-gateway", ), }, + killSignal: "SIGKILL", timeout: commandTimeoutMs, }); return { @@ -270,11 +291,22 @@ function runBackupCase( return runCli(caseDir, home, binDir, ["backup-all"]); } -function runLiveHostProcessCase(name: string): CommandResult { +function runLiveHostProcessCase( + name: string, + progress: Pick & TestProgressCapability, +): CommandResult { const { binDir, caseDir, home } = prepareCase(name); writeFakeDockerNoCluster(binDir); const gatewayBin = writeFakeGatewayBinary(binDir, "0.0.43"); - const child = spawn(gatewayBin, ["serve"], { detached: false, stdio: "ignore" }); + progress.event("fake gateway process started"); + const child = spawnObservedChild(gatewayBin, ["serve"], { + activityLabel: "command: fake-gateway-serve", + progress, + spawn: { detached: false, stdio: "ignore" }, + }); + child.once("close", () => { + progress.event("fake gateway process stopped"); + }); expect(child.pid, "fake gateway process must have a pid").toBeTypeOf("number"); const pid = child.pid as number; liveGatewayPids.push(pid); @@ -326,12 +358,14 @@ function expectSandboxListCalled(result: CommandResult, expected: boolean): void } describe("gateway drift preflight E2E migration", () => { - it( - "fails closed before unsafe sandbox state mutation when gateway schema or binary drift is detected", - testTimeoutOptions(180_000), - () => { + it("fails closed before unsafe sandbox state mutation when gateway schema or binary drift is detected", { + ...testTimeoutOptions(180_000), + meta: { e2ePhases: GATEWAY_DRIFT_PHASES }, + }, ({ progress }) => { + try { expect(fs.existsSync(CLI_ENTRYPOINT), "repo CLI entrypoint must exist").toBe(true); + progress.phase("exercise container-backed gateway drift cases"); const protobuf = runBackupCase("protobuf-mismatch", { gatewayImage: "ghcr.io/nvidia/openshell/cluster:0.0.37", gatewayRunning: "false", @@ -377,7 +411,8 @@ describe("gateway drift preflight E2E migration", () => { ); expectSandboxListCalled(imageDrift, false); - const hostBackup = runLiveHostProcessCase("host-process-backup"); + progress.phase("exercise host-process gateway drift cases"); + const hostBackup = runLiveHostProcessCase("host-process-backup", progress); expect(hostBackup.status, hostBackup.output).not.toBe(0); expectContains( hostBackup, @@ -416,6 +451,7 @@ describe("gateway drift preflight E2E migration", () => { ); expectSandboxListCalled(noMarker, false); + progress.phase("verify stale gateway marker fallback"); const stale = (() => { const { binDir, caseDir, home } = prepareCase("host-process-stale-marker"); const oldInstall = path.join(caseDir, "old-install"); @@ -432,6 +468,9 @@ describe("gateway drift preflight E2E migration", () => { "stale marker binary is not used to fabricate drift", ); expectSandboxListCalled(stale, true); - }, - ); + } finally { + progress.phase("release gateway drift fixtures"); + releaseGatewayDriftFixtures(); + } + }); }); diff --git a/test/historical-openclaw-security-revision-container-e2e.test.ts b/test/historical-openclaw-security-revision-container-e2e.test.ts index 7d33c9598a..e76aa91ff2 100644 --- a/test/historical-openclaw-security-revision-container-e2e.test.ts +++ b/test/historical-openclaw-security-revision-container-e2e.test.ts @@ -545,10 +545,14 @@ describe("Historical OpenClaw security revision container E2E contract (#7272)", realContainerTest( "the historical wrapper remediates reviewed plugins, restores failures, and rejects outside state (#7272)", - async ({ artifacts, cleanup, docker, secrets }) => { + async ({ artifacts, cleanup, docker, progress, secrets, signal }) => { const image = configuredImage as string; - const probe = new DockerProbe(artifacts, (text, extraValues) => - secrets.redact(text, extraValues), + const probe = new DockerProbe( + artifacts, + (text, extraValues) => secrets.redact(text, extraValues), + undefined, + progress, + signal, ); const resourcePrefix = safeDockerName(`nemoclaw-security-e2e-${process.pid}-${randomUUID()}`); const containers: string[] = []; diff --git a/test/mcp-bridge-servers.test.ts b/test/mcp-bridge-servers.test.ts index 59f303b7f6..931ed09668 100644 --- a/test/mcp-bridge-servers.test.ts +++ b/test/mcp-bridge-servers.test.ts @@ -18,8 +18,20 @@ import { startFakeMcpHttpsServer, startPublicMcpHttpsTunnel, } from "./e2e/live/mcp-bridge-servers"; +import { startTestProgress } from "./e2e/fixtures/progress.ts"; const servers: StartedHttpServer[] = []; +function progressProbe() { + const lines: string[] = []; + const progress = startTestProgress( + "MCP tunnel support", + ["start MCP tunnel", "verify MCP tunnel"], + { + logLine: (line) => lines.push(line), + }, + ); + return { lines, progress }; +} type CompatibleToolCallResponse = { choices: Array<{ message: { @@ -121,6 +133,8 @@ describe("authenticated MCP live fixtures", () => { .mockResolvedValue({ body: null, status: 405 } as Response); let cleanupName = ""; let cleanupProcess: (() => Promise) | undefined; + const observation = progressProbe(); + const { progress } = observation; try { const tunnel = await startPublicMcpHttpsTunnel({ @@ -134,6 +148,7 @@ describe("authenticated MCP live fixtures", () => { }, }, label: "unit MCP fixture", + progress, server: { port: 43123, close: async () => {} }, }); @@ -146,6 +161,16 @@ describe("authenticated MCP live fixtures", () => { expect(fetchMock).toHaveBeenCalledTimes(6); expect(cleanupName).toBe("stop unit MCP fixture cloudflared quick tunnel"); expect(cleanupProcess).toBeTypeOf("function"); + expect(observation.lines).toEqual( + expect.arrayContaining([ + expect.stringContaining("event: cloudflared quick tunnel attempt 1 started"), + ]), + ); + progress.stop(); + expect(progress.summary().phases[0]?.outputEvents).toBeGreaterThan(0); + expect(JSON.stringify(progress.summary())).not.toContain( + "fixture-cleanup-123.trycloudflare.com", + ); } finally { await cleanupProcess?.(); fetchMock.mockRestore(); @@ -193,6 +218,7 @@ describe("authenticated MCP live fixtures", () => { cloudflaredBin: cloudflared, cleanup: { add: vi.fn() }, label: "redaction fixture", + progress: progressProbe().progress, server: { port: 43123, close: async () => {} }, }); } catch (error) { @@ -243,6 +269,7 @@ describe("authenticated MCP live fixtures", () => { cloudflaredBin: cloudflared, cleanup: { add: vi.fn() }, label: "chunked redaction fixture", + progress: progressProbe().progress, server: { port: 43123, close: async () => {} }, }); } catch (error) { diff --git a/test/openclaw-security-revision-container-e2e.test.ts b/test/openclaw-security-revision-container-e2e.test.ts index 6dc5a4b811..07f7865a29 100644 --- a/test/openclaw-security-revision-container-e2e.test.ts +++ b/test/openclaw-security-revision-container-e2e.test.ts @@ -547,10 +547,14 @@ describe("OpenClaw current-image security revision contract (#7272)", () => { realContainerTest( "the #7286 image ships only the reviewed OpenClaw dependency graph (#7272)", - async ({ artifacts, docker, secrets }) => { + async ({ artifacts, docker, progress, secrets, signal }) => { const image = configuredImage as string; - const probe = new DockerProbe(artifacts, (text, extraValues) => - secrets.redact(text, extraValues), + const probe = new DockerProbe( + artifacts, + (text, extraValues) => secrets.redact(text, extraValues), + undefined, + progress, + signal, ); const container = `nemoclaw-security-e2e-${process.pid}-${randomUUID()}`.toLowerCase(); diff --git a/test/vllm-docker-storage.test.ts b/test/vllm-docker-storage.test.ts index b4359014f4..ea7ca9f08b 100644 --- a/test/vllm-docker-storage.test.ts +++ b/test/vllm-docker-storage.test.ts @@ -9,14 +9,22 @@ import os from "node:os"; import path from "node:path"; import { pathToFileURL } from "node:url"; -import { expect, test } from "vitest"; +import { expect } from "vitest"; import { detectVllmProfile } from "../src/lib/inference/vllm"; import { imageStorageRequirementBytes } from "../src/lib/inference/vllm-storage"; +import { test } from "./e2e/fixtures/workflow-e2e-test.ts"; const TARGET_ID = "vllm-docker-storage"; const DOCKER_HOST = "unix:///run/docker.sock"; const INSTALL_SUBPROCESS_TIMEOUT_MS = 15_000; +const VLLM_STORAGE_PHASES = [ + "inspect Docker storage prerequisites", + "exercise the managed vLLM storage gate", + "verify Docker and filesystem capacity evidence", + "record release-candidate storage evidence", + "release vLLM storage fixtures", +] as const; const RUN_REAL_DOCKER = process.env.E2E_TARGET_ID === TARGET_ID || process.env.NEMOCLAW_RUN_VLLM_STORAGE_DOCKER_E2E === "1"; @@ -112,7 +120,8 @@ function writeEvidence(evidence: Record): void { realDockerTest( "allows non-interactive express managed vLLM past the real /run/docker.sock storage gate (#7039)", - () => { + { timeout: 30_000, meta: { e2ePhases: VLLM_STORAGE_PHASES } }, + ({ progress }) => { expect(process.platform, "this release acceptance requires a native Linux host").toBe("linux"); expect( fs.statSync("/run/docker.sock").isSocket(), @@ -123,6 +132,7 @@ realDockerTest( const infoResult = spawnSync("docker", ["info", "--format", "{{json .}}"], { encoding: "utf8", env, + killSignal: "SIGKILL", timeout: 15_000, }); expect( @@ -142,6 +152,7 @@ realDockerTest( assert(profile, "managed vLLM has no generic Linux profile"); const requiredAvailableBytes = imageStorageRequirementBytes(profile.imageDownloadSizeBytes); + progress.phase("exercise the managed vLLM storage gate"); const fakeBinDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-vllm-storage-")); const blockedHome = path.join(fakeBinDir, "blocked-home"); const commandLogPath = path.join(fakeBinDir, "docker-commands.jsonl"); @@ -149,6 +160,7 @@ realDockerTest( const dockerPathResult = spawnSync("sh", ["-c", "command -v docker"], { encoding: "utf8", env, + killSignal: "SIGKILL", timeout: 5_000, }); expect( @@ -162,7 +174,7 @@ realDockerTest( const cachedImageResult = spawnSync( realDockerPath, ["image", "inspect", "--format", "{{.Id}}", profile.image], - { encoding: "utf8", env, timeout: 10_000 }, + { encoding: "utf8", env, killSignal: "SIGKILL", timeout: 10_000 }, ); expect( cachedImageResult.error, @@ -237,6 +249,7 @@ realDockerTest( new Set(["container ls", "image inspect", "info --format"]), ); + progress.phase("verify Docker and filesystem capacity evidence"); const statfsLog = fs.readFileSync(statfsLogPath, "utf8").trim(); expect(statfsLog, "production did not consume a filesystem capacity sample").not.toBe(""); productionStatfsSamples = statfsLog @@ -250,64 +263,67 @@ realDockerTest( measuredAvailableBytes = BigInt(dockerRootSample.bavail) * BigInt(dockerRootSample.bsize); expect(measuredAvailableBytes).toBeGreaterThan(0n); expect(measuredAvailableBytes).toBeGreaterThanOrEqual(requiredAvailableBytes); + + progress.phase("record release-candidate storage evidence"); + const checkoutResult = spawnSync("git", ["rev-parse", "HEAD"], { + encoding: "utf8", + killSignal: "SIGKILL", + timeout: 5_000, + }); + expect( + checkoutResult.status, + `could not record the validated checkout: ${ + checkoutResult.error?.message || checkoutResult.stderr || checkoutResult.stdout + }`, + ).toBe(0); + const checkoutSha = checkoutResult.stdout.trim(); + expect(checkoutSha).toMatch(/^[0-9a-f]{40}$/u); + const sourceVersionResult = spawnSync("git", ["describe", "--tags", "--always", "--dirty"], { + encoding: "utf8", + killSignal: "SIGKILL", + timeout: 5_000, + }); + expect( + sourceVersionResult.status, + `could not record the validated source version: ${ + sourceVersionResult.error?.message || + sourceVersionResult.stderr || + sourceVersionResult.stdout + }`, + ).toBe(0); + const releaseCandidateSourceVersion = sourceVersionResult.stdout.trim(); + expect(releaseCandidateSourceVersion).not.toBe(""); + const packageMetadata = JSON.parse( + fs.readFileSync(path.join(process.cwd(), "package.json"), "utf8"), + ) as { version?: unknown }; + expect(typeof packageMetadata.version).toBe("string"); + const packageVersion = String(packageMetadata.version); + const evidence = { + schemaVersion: 1, + checkoutSha, + releaseCandidateSourceVersion, + packageVersion, + platform: process.platform, + architecture: process.arch, + dockerHost: DOCKER_HOST, + dockerServerVersion: info.ServerVersion, + dockerRootDir, + dockerRootAvailableBytes: String(measuredAvailableBytes), + measuredPath, + measuredSource: "Docker root directory", + measuredAvailableBytes: String(measuredAvailableBytes), + productionStatfsSamples, + imageDownloadSizeBytes: String(profile.imageDownloadSizeBytes), + requiredAvailableBytes: String(requiredAvailableBytes), + managedInstallCrossedImageStorageGate: true, + installSubprocessTimeoutMs: INSTALL_SUBPROCESS_TIMEOUT_MS, + installDockerCommands, + }; + writeEvidence(evidence); + console.info(`[${TARGET_ID}] ${JSON.stringify(evidence)}`); } finally { + progress.phase("release vLLM storage fixtures"); fs.rmSync(fakeBinDir, { force: true, recursive: true }); } - - const checkoutResult = spawnSync("git", ["rev-parse", "HEAD"], { - encoding: "utf8", - timeout: 5_000, - }); - expect( - checkoutResult.status, - `could not record the validated checkout: ${ - checkoutResult.error?.message || checkoutResult.stderr || checkoutResult.stdout - }`, - ).toBe(0); - const checkoutSha = checkoutResult.stdout.trim(); - expect(checkoutSha).toMatch(/^[0-9a-f]{40}$/u); - const sourceVersionResult = spawnSync("git", ["describe", "--tags", "--always", "--dirty"], { - encoding: "utf8", - timeout: 5_000, - }); - expect( - sourceVersionResult.status, - `could not record the validated source version: ${ - sourceVersionResult.error?.message || - sourceVersionResult.stderr || - sourceVersionResult.stdout - }`, - ).toBe(0); - const releaseCandidateSourceVersion = sourceVersionResult.stdout.trim(); - expect(releaseCandidateSourceVersion).not.toBe(""); - const packageMetadata = JSON.parse( - fs.readFileSync(path.join(process.cwd(), "package.json"), "utf8"), - ) as { version?: unknown }; - expect(typeof packageMetadata.version).toBe("string"); - const packageVersion = String(packageMetadata.version); - const evidence = { - schemaVersion: 1, - checkoutSha, - releaseCandidateSourceVersion, - packageVersion, - platform: process.platform, - architecture: process.arch, - dockerHost: DOCKER_HOST, - dockerServerVersion: info.ServerVersion, - dockerRootDir, - dockerRootAvailableBytes: String(measuredAvailableBytes), - measuredPath, - measuredSource: "Docker root directory", - measuredAvailableBytes: String(measuredAvailableBytes), - productionStatfsSamples, - imageDownloadSizeBytes: String(profile.imageDownloadSizeBytes), - requiredAvailableBytes: String(requiredAvailableBytes), - managedInstallCrossedImageStorageGate: true, - installSubprocessTimeoutMs: INSTALL_SUBPROCESS_TIMEOUT_MS, - installDockerCommands, - }; - writeEvidence(evidence); - console.info(`[${TARGET_ID}] ${JSON.stringify(evidence)}`); }, - 30_000, ); diff --git a/tools/e2e/check-semantic-phases.mts b/tools/e2e/check-semantic-phases.mts index 171edf3ec2..af319aa6ba 100644 --- a/tools/e2e/check-semantic-phases.mts +++ b/tools/e2e/check-semantic-phases.mts @@ -11,6 +11,15 @@ import { createVitest } from "vitest/node"; import { REPO_ROOT } from "../../test/e2e/fixtures/paths.ts"; import { validateE2EPhasePlan } from "../../test/e2e/fixtures/progress.ts"; +import { + type CredentialFreeTestMatrixRow, + type CredentialFreeTestProject, +} from "./credential-free-tests.mts"; +import { + type FreeStandingJobsInventory, + readFreeStandingJobsInventory, +} from "./workflow-boundary.mts"; +import { buildE2eWorkflowPlan } from "./workflow-plan.mts"; declare module "@vitest/runner" { interface TaskMeta { @@ -36,93 +45,1791 @@ export interface TestPhaseBody { skipped?: boolean; } +export interface DirectChildProcessCall { + api: "exec" | "execFile" | "execFileSync" | "execSync" | "fork" | "spawn" | "spawnSync"; + boundary: string; + file: string; + hasBoundedTimeout: boolean; + hasHardKillSignal: boolean; + line: number; + observesOutput: boolean; + outputIgnored: boolean; + tracksActivity: boolean; +} + export interface ScopedPhasePlan { name: string; phases: readonly string[]; } export interface SemanticPhaseSourceGraph { + childProcessAuditFailures?: string[]; + directChildProcessCalls?: DirectChildProcessCall[]; forwardedTestModules?: string[]; importsDirectTest: boolean; importsSharedTest: boolean; + importsWorkflowTest?: boolean; phaseCalls: PhaseCall[]; testPhaseBodies: TestPhaseBody[]; } -export interface CollectedSemanticPhaseModule { - relativeModuleId: string; - errors: readonly string[]; - tests: readonly { - fullName: string; - phases?: readonly string[]; - }[]; - source: SemanticPhaseSourceGraph; +export interface CollectedSemanticPhaseModule { + relativeModuleId: string; + project?: CredentialFreeTestProject; + errors: readonly string[]; + tests: readonly { + fullName: string; + phases?: readonly string[]; + }[]; + source: SemanticPhaseSourceGraph; +} + +export interface WorkflowSemanticPhaseModule { + file: string; + project: CredentialFreeTestProject; +} + +const LIVE_ROOT = path.join(REPO_ROOT, "test", "e2e", "live"); +const E2E_ROOT = path.join(REPO_ROOT, "test", "e2e"); +const E2E_PROCESS_ROOTS = [E2E_ROOT, path.join(REPO_ROOT, "tools", "e2e")]; +const E2E_RUNTIME_OBSERVABILITY_FILES = [path.join(E2E_ROOT, "risk-signal-reporter.ts")]; +const REGISTRY_TARGET_TEST = "test/e2e/live/registry-targets.test.ts"; +const LIVE_TEST_FORWARDERS = new Map([ + ["test/e2e/live/bootstrap-install-smoke.test.ts", "test/e2e/live/launchable-smoke.test.ts"], +]); +const LIVE_TEST_FIXTURE_SUFFIX = "/fixtures/e2e-test.ts"; +const WORKFLOW_TEST_FIXTURE_SUFFIX = "/e2e/fixtures/workflow-e2e-test.ts"; + +type SemanticPhaseWorkflowPlan = { + matrix: readonly unknown[]; + testMatrix: readonly CredentialFreeTestMatrixRow[]; +}; +type SemanticPhaseWorkflowInventory = Pick; + +function credentialFreeProjectForWorkflowFile( + file: string, + testMatrix: readonly CredentialFreeTestMatrixRow[], +): CredentialFreeTestProject { + if (file.startsWith("test/e2e/live/") && file.endsWith(".test.ts")) return "e2e-live"; + const plannedRow = testMatrix.find((row) => row.file === file); + if (plannedRow) return plannedRow.project; + throw new Error( + `workflow-selected test is outside test/e2e/live and missing from the shared E2E planner: ${file}`, + ); +} + +/** + * Returns the complete semantic-phase coverage ledger. Every e2e-live module + * remains covered, including files not selected by today's default workflow; + * the workflow inventory and shared planner add project-aware integration tests + * that execute through the credential-free shared job. + */ +export function semanticPhaseCoverageModules( + plan: SemanticPhaseWorkflowPlan = buildE2eWorkflowPlan(), + inventory: SemanticPhaseWorkflowInventory = readFreeStandingJobsInventory(), + liveTestFiles: readonly string[] = fs + .globSync("**/*.test.ts", { cwd: LIVE_ROOT }) + .map((file) => path.join("test/e2e/live", file).split(path.sep).join("/")), +): WorkflowSemanticPhaseModule[] { + const selected = new Map(); + + function add(file: string, project: CredentialFreeTestProject): void { + const existing = selected.get(file); + if (existing && existing !== project) { + throw new Error( + `workflow-selected test belongs to conflicting Vitest projects: ${file} (${existing}, ${project})`, + ); + } + selected.set(file, project); + } + + for (const file of liveTestFiles) add(file, "e2e-live"); + for (const row of plan.testMatrix) add(row.file, row.project); + for (const file of inventory.liveTestToJobs.keys()) { + add(file, credentialFreeProjectForWorkflowFile(file, plan.testMatrix)); + } + if (plan.matrix.length > 0) add(REGISTRY_TARGET_TEST, "e2e-live"); + + for (const [forwarder, target] of LIVE_TEST_FORWARDERS) { + if (selected.has(forwarder)) add(target, "e2e-live"); + } + + return [...selected] + .map(([file, project]) => ({ file, project })) + .sort((left, right) => left.file.localeCompare(right.file)); +} + +function resolveImportWithin(root: string, fromFile: string, specifier: string): string | null { + if (!specifier.startsWith(".")) return null; + const resolved = path.resolve(path.dirname(fromFile), specifier); + const candidates = [resolved, `${resolved}.ts`, path.join(resolved, "index.ts")]; + const rootPrefix = `${root}${path.sep}`; + return ( + candidates.find( + (candidate) => + candidate.startsWith(rootPrefix) && + fs.existsSync(candidate) && + fs.statSync(candidate).isFile(), + ) ?? null + ); +} + +function resolveLiveImport(fromFile: string, specifier: string): string | null { + return resolveImportWithin(LIVE_ROOT, fromFile, specifier); +} + +function resolveE2EImport(fromFile: string, specifier: string): string | null { + for (const root of E2E_PROCESS_ROOTS) { + const resolved = resolveImportWithin(root, fromFile, specifier); + if (resolved) return resolved; + } + return null; +} + +function importsTestFrom(sourceFile: ts.SourceFile, moduleSuffix: string): boolean { + return sourceFile.statements.some((statement) => { + if ( + !ts.isImportDeclaration(statement) || + !ts.isStringLiteral(statement.moduleSpecifier) || + !statement.moduleSpecifier.text.endsWith(moduleSuffix) + ) { + return false; + } + if (statement.importClause?.isTypeOnly) return false; + const bindings = statement.importClause?.namedBindings; + return ( + !!bindings && + ts.isNamedImports(bindings) && + bindings.elements.some( + (element) => !element.isTypeOnly && (element.propertyName ?? element.name).text === "test", + ) + ); + }); +} + +function importsSharedE2ETest(sourceFile: ts.SourceFile): boolean { + return importsTestFrom(sourceFile, LIVE_TEST_FIXTURE_SUFFIX); +} + +function importsWorkflowE2ETest(sourceFile: ts.SourceFile): boolean { + return importsTestFrom(sourceFile, WORKFLOW_TEST_FIXTURE_SUFFIX); +} + +function importsDirectVitestTest(sourceFile: ts.SourceFile): boolean { + return sourceFile.statements.some((statement) => { + if ( + !ts.isImportDeclaration(statement) || + !ts.isStringLiteral(statement.moduleSpecifier) || + statement.moduleSpecifier.text !== "vitest" + ) { + return false; + } + const bindings = statement.importClause?.namedBindings; + return ( + !!bindings && + ts.isNamedImports(bindings) && + bindings.elements.some((element) => + ["it", "test"].includes((element.propertyName ?? element.name).text), + ) + ); + }); +} + +const DIRECT_CHILD_PROCESS_APIS = new Set([ + "exec", + "execFile", + "execFileSync", + "execSync", + "fork", + "spawn", + "spawnSync", +]); +const ASYNC_CHILD_PROCESS_APIS = new Set([ + "exec", + "execFile", + "fork", + "spawn", +]); +const MAX_BLOCKING_CHILD_PROCESS_MS = 5 * 60_000; +const AUDITED_ASYNC_CHILD_PROCESS_BOUNDARY = + "test/e2e/fixtures/observed-child-process.ts#spawnObservedChild"; +const OBSERVED_CHILD_PROCESS_MODULE = path.join(E2E_ROOT, "fixtures", "observed-child-process.ts"); +const TEST_PROGRESS_MODULE = path.join(E2E_ROOT, "fixtures", "progress.ts"); + +type ObservedChildProgressPolicy = { kind: "path"; path: string }; + +// This closed-world list is intentionally conservative. A new asynchronous +// process boundary must be reviewed here, and its progress expression must +// retain the unforgeable TestProgress capability enforced by TypeScript. +const OBSERVED_CHILD_PROGRESS_POLICIES = new Map([ + [ + "test/e2e/fixtures/fake-openai-compatible.ts#startFakeOpenAiCompatibleServer", + { kind: "path", path: "options.progress" }, + ], + ["test/e2e/fixtures/shell-probe.ts#run", { kind: "path", path: "this.progress" }], + ["test/e2e/fixtures/docker-probe.ts#run", { kind: "path", path: "this.progress" }], + [ + "test/e2e/live/openshell-gateway-auth-source-contract-helpers.ts#runOpenShellGatewayAuthSourceContractScenarioUnchecked", + { kind: "path", path: "progress" }, + ], + [ + "test/e2e/live/mcp-bridge-servers.ts#startPublicMcpHttpsTunnel", + { kind: "path", path: "options.progress" }, + ], + [ + "test/e2e/live/bedrock-runtime-compatible-anthropic-raw-command.ts#runRawCommand", + { kind: "path", path: "options.progress" }, + ], + ["test/e2e/live/ollama-auth-proxy.test.ts#spawnLogged", { kind: "path", path: "progress" }], + [ + "test/gateway-drift-preflight.test.ts#runLiveHostProcessCase", + { kind: "path", path: "progress" }, + ], +]); + +interface DirectChildProcessBindings { + values: Map; +} + +type ProcessProvenance = + | { + kind: "api"; + api: DirectChildProcessCall["api"]; + invocation: "direct" | "indirect"; + } + | { kind: "binder"; api: DirectChildProcessCall["api"] } + | { kind: "builtin-loader" } + | { kind: "create-require" } + | { kind: "namespace" } + | { kind: "promisify" } + | { kind: "require" } + | { kind: "module-namespace" } + | { kind: "util-namespace" }; + +const CHILD_PROCESS_MODULES = new Set(["child_process", "node:child_process"]); +const MODULE_MODULES = new Set(["module", "node:module"]); +const UTIL_MODULES = new Set(["util", "node:util"]); + +function unwrapExpression(expression: ts.Expression): ts.Expression { + let current = expression; + while ( + ts.isParenthesizedExpression(current) || + ts.isAsExpression(current) || + ts.isTypeAssertionExpression(current) || + ts.isNonNullExpression(current) || + ts.isSatisfiesExpression(current) || + ts.isAwaitExpression(current) + ) { + current = current.expression; + } + return current; +} + +function memberName(expression: ts.Expression): string | null { + const unwrapped = unwrapExpression(expression); + if (ts.isPropertyAccessExpression(unwrapped)) return unwrapped.name.text; + if ( + ts.isElementAccessExpression(unwrapped) && + unwrapped.argumentExpression && + ts.isStringLiteralLike(unwrapped.argumentExpression) + ) { + return unwrapped.argumentExpression.text; + } + return null; +} + +function memberReceiver(expression: ts.Expression): ts.Expression | null { + const unwrapped = unwrapExpression(expression); + if (ts.isPropertyAccessExpression(unwrapped) || ts.isElementAccessExpression(unwrapped)) { + return unwrapped.expression; + } + return null; +} + +function expressionProvenance( + expression: ts.Expression | undefined, + bindings: DirectChildProcessBindings, +): ProcessProvenance | null { + if (!expression) return null; + const unwrapped = unwrapExpression(expression); + if (ts.isIdentifier(unwrapped)) { + return ( + bindings.values.get(unwrapped.text) ?? + (unwrapped.text === "require" ? { kind: "require" } : null) + ); + } + if (ts.isPropertyAccessExpression(unwrapped) || ts.isElementAccessExpression(unwrapped)) { + if ( + memberName(unwrapped) === "getBuiltinModule" && + expressionPath(unwrapped.expression) === "process" + ) { + return { kind: "builtin-loader" }; + } + const receiver = expressionProvenance(unwrapped.expression, bindings); + const name = memberName(unwrapped); + if ( + receiver?.kind === "namespace" && + name && + DIRECT_CHILD_PROCESS_APIS.has(name as DirectChildProcessCall["api"]) + ) { + return { + kind: "api", + api: name as DirectChildProcessCall["api"], + invocation: "direct", + }; + } + if (receiver?.kind === "util-namespace" && name === "promisify") { + return { kind: "promisify" }; + } + if (receiver?.kind === "module-namespace" && name === "createRequire") { + return { kind: "create-require" }; + } + if (receiver?.kind === "api" && name === "bind") { + return { kind: "binder", api: receiver.api }; + } + if (receiver?.kind === "api" && (name === "call" || name === "apply")) { + return { ...receiver, invocation: "indirect" }; + } + return null; + } + if (ts.isCallExpression(unwrapped)) { + if (unwrapped.expression.kind === ts.SyntaxKind.ImportKeyword) { + const specifier = unwrapped.arguments[0]; + if (unwrapped.arguments.length !== 1 || !specifier || !ts.isStringLiteralLike(specifier)) { + return null; + } + if (CHILD_PROCESS_MODULES.has(specifier.text)) return { kind: "namespace" }; + if (UTIL_MODULES.has(specifier.text)) return { kind: "util-namespace" }; + if (MODULE_MODULES.has(specifier.text)) return { kind: "module-namespace" }; + return null; + } + const callee = expressionProvenance(unwrapped.expression, bindings); + if (callee?.kind === "builtin-loader") { + const specifier = unwrapped.arguments[0]; + if (unwrapped.arguments.length !== 1 || !specifier || !ts.isStringLiteralLike(specifier)) { + return null; + } + if (CHILD_PROCESS_MODULES.has(specifier.text)) return { kind: "namespace" }; + if (UTIL_MODULES.has(specifier.text)) return { kind: "util-namespace" }; + if (MODULE_MODULES.has(specifier.text)) return { kind: "module-namespace" }; + return null; + } + if (callee?.kind === "require") { + const specifier = unwrapped.arguments[0]; + if (unwrapped.arguments.length !== 1 || !specifier || !ts.isStringLiteralLike(specifier)) { + return null; + } + if (CHILD_PROCESS_MODULES.has(specifier.text)) return { kind: "namespace" }; + if (UTIL_MODULES.has(specifier.text)) return { kind: "util-namespace" }; + if (MODULE_MODULES.has(specifier.text)) return { kind: "module-namespace" }; + return null; + } + if (callee?.kind === "create-require") return { kind: "require" }; + if (callee?.kind === "binder") { + return { kind: "api", api: callee.api, invocation: "indirect" }; + } + if (callee?.kind === "promisify") { + const wrapped = expressionProvenance(unwrapped.arguments[0], bindings); + if (wrapped?.kind === "api") { + return { ...wrapped, invocation: "indirect" }; + } + } + } + if ( + ts.isBinaryExpression(unwrapped) && + unwrapped.operatorToken.kind === ts.SyntaxKind.CommaToken + ) { + return expressionProvenance(unwrapped.right, bindings); + } + if (ts.isConditionalExpression(unwrapped)) { + const provenance = + expressionProvenance(unwrapped.whenTrue, bindings) ?? + expressionProvenance(unwrapped.whenFalse, bindings); + return provenance?.kind === "api" ? { ...provenance, invocation: "indirect" } : provenance; + } + if ( + ts.isBinaryExpression(unwrapped) && + [ + ts.SyntaxKind.AmpersandAmpersandToken, + ts.SyntaxKind.BarBarToken, + ts.SyntaxKind.QuestionQuestionToken, + ].includes(unwrapped.operatorToken.kind) + ) { + const provenance = + expressionProvenance(unwrapped.left, bindings) ?? + expressionProvenance(unwrapped.right, bindings); + return provenance?.kind === "api" ? { ...provenance, invocation: "indirect" } : provenance; + } + return null; +} + +function directChildProcessBindings(sourceFile: ts.SourceFile): DirectChildProcessBindings { + const bindings: DirectChildProcessBindings = { values: new Map() }; + const bindIdentifier = (name: string, provenance: ProcessProvenance | null): boolean => { + if (!provenance) return false; + const existing = bindings.values.get(name); + if (existing) { + if (JSON.stringify(existing) === JSON.stringify(provenance)) return false; + if (existing.kind === "api" && existing.invocation === "direct") { + bindings.values.set(name, { ...existing, invocation: "indirect" }); + return true; + } + // Provenance only grows. Retaining the first conflicting capability is + // conservative and, unlike overwriting it, guarantees fixed-point + // convergence for variables reassigned between process APIs. + return false; + } + bindings.values.set(name, provenance); + return true; + }; + const bindObjectPattern = ( + pattern: ts.ObjectBindingPattern, + provenance: ProcessProvenance | null, + ): boolean => { + if ( + !provenance || + !["namespace", "util-namespace", "module-namespace"].includes(provenance.kind) + ) { + return false; + } + let changed = false; + for (const element of pattern.elements) { + if (!ts.isIdentifier(element.name)) continue; + const imported = propertyNameText(element.propertyName) ?? element.name.text; + if ( + provenance.kind === "namespace" && + DIRECT_CHILD_PROCESS_APIS.has(imported as DirectChildProcessCall["api"]) + ) { + changed = + bindIdentifier(element.name.text, { + kind: "api", + api: imported as DirectChildProcessCall["api"], + invocation: "direct", + }) || changed; + } else if (provenance.kind === "util-namespace" && imported === "promisify") { + changed = bindIdentifier(element.name.text, { kind: "promisify" }) || changed; + } else if (provenance.kind === "module-namespace" && imported === "createRequire") { + changed = bindIdentifier(element.name.text, { kind: "create-require" }) || changed; + } + } + return changed; + }; + const bindAssignmentPattern = ( + pattern: ts.ObjectLiteralExpression, + provenance: ProcessProvenance | null, + ): boolean => { + if ( + !provenance || + !["namespace", "util-namespace", "module-namespace"].includes(provenance.kind) + ) { + return false; + } + let changed = false; + for (const property of pattern.properties) { + const imported = propertyNameText(property.name); + const target = + ts.isShorthandPropertyAssignment(property) && ts.isIdentifier(property.name) + ? property.name + : ts.isPropertyAssignment(property) && ts.isIdentifier(property.initializer) + ? property.initializer + : null; + if (!imported || !target) continue; + if ( + provenance.kind === "namespace" && + DIRECT_CHILD_PROCESS_APIS.has(imported as DirectChildProcessCall["api"]) + ) { + changed = + bindIdentifier(target.text, { + kind: "api", + api: imported as DirectChildProcessCall["api"], + invocation: "direct", + }) || changed; + } else if (provenance.kind === "util-namespace" && imported === "promisify") { + changed = bindIdentifier(target.text, { kind: "promisify" }) || changed; + } else if (provenance.kind === "module-namespace" && imported === "createRequire") { + changed = bindIdentifier(target.text, { kind: "create-require" }) || changed; + } + } + return changed; + }; + + for (const statement of sourceFile.statements) { + if ( + ts.isImportDeclaration(statement) && + ts.isStringLiteral(statement.moduleSpecifier) && + (CHILD_PROCESS_MODULES.has(statement.moduleSpecifier.text) || + UTIL_MODULES.has(statement.moduleSpecifier.text) || + MODULE_MODULES.has(statement.moduleSpecifier.text)) + ) { + const isChildProcess = CHILD_PROCESS_MODULES.has(statement.moduleSpecifier.text); + const isUtil = UTIL_MODULES.has(statement.moduleSpecifier.text); + const importClause = statement.importClause; + if (importClause?.isTypeOnly) continue; + if (importClause?.name) { + bindIdentifier(importClause.name.text, { + kind: isChildProcess ? "namespace" : isUtil ? "util-namespace" : "module-namespace", + }); + } + const namedBindings = importClause?.namedBindings; + if (namedBindings && ts.isNamespaceImport(namedBindings)) { + bindIdentifier(namedBindings.name.text, { + kind: isChildProcess ? "namespace" : isUtil ? "util-namespace" : "module-namespace", + }); + } + if (namedBindings && ts.isNamedImports(namedBindings)) { + for (const binding of namedBindings.elements) { + if (binding.isTypeOnly) continue; + const imported = (binding.propertyName ?? binding.name).text; + if ( + isChildProcess && + DIRECT_CHILD_PROCESS_APIS.has(imported as DirectChildProcessCall["api"]) + ) { + bindIdentifier(binding.name.text, { + kind: "api", + api: imported as DirectChildProcessCall["api"], + invocation: "direct", + }); + } else if (isUtil && imported === "promisify") { + bindIdentifier(binding.name.text, { kind: "promisify" }); + } else if (!isChildProcess && !isUtil && imported === "createRequire") { + bindIdentifier(binding.name.text, { kind: "create-require" }); + } + } + } + } + } + + let changed = true; + while (changed) { + changed = false; + function inspect(node: ts.Node): void { + if (ts.isVariableDeclaration(node) || ts.isParameter(node)) { + const provenance = expressionProvenance(node.initializer, bindings); + if (ts.isIdentifier(node.name)) { + changed = bindIdentifier(node.name.text, provenance) || changed; + } else if (ts.isObjectBindingPattern(node.name)) { + changed = bindObjectPattern(node.name, provenance) || changed; + } + } else if ( + ts.isBinaryExpression(node) && + node.operatorToken.kind === ts.SyntaxKind.EqualsToken + ) { + const provenance = expressionProvenance(node.right, bindings); + const left = unwrapExpression(node.left); + if (ts.isIdentifier(left)) { + changed = bindIdentifier(left.text, provenance) || changed; + } else if (ts.isObjectLiteralExpression(left)) { + changed = bindAssignmentPattern(left, provenance) || changed; + } + } + ts.forEachChild(node, inspect); + } + inspect(sourceFile); + } + return bindings; +} + +function directChildProcessApi( + expression: ts.LeftHandSideExpression, + bindings: DirectChildProcessBindings, +): Extract | null { + const provenance = expressionProvenance(expression, bindings); + return provenance?.kind === "api" ? provenance : null; +} + +function isChildProcessWrapperCreation( + node: ts.CallExpression, + bindings: DirectChildProcessBindings, +): boolean { + const receiver = memberReceiver(node.expression); + if (memberName(node.expression) === "bind" && receiver) { + return expressionProvenance(receiver, bindings)?.kind === "api"; + } + const callee = expressionProvenance(node.expression, bindings); + return ( + (callee?.kind === "promisify" && + expressionProvenance(node.arguments[0], bindings)?.kind === "api") || + callee?.kind === "binder" + ); +} + +function objectProperty( + object: ts.ObjectLiteralExpression | undefined, + name: string, +): ts.ObjectLiteralElementLike | undefined { + let resolved: ts.ObjectLiteralElementLike | undefined; + for (const property of object?.properties ?? []) { + if ( + ts.isSpreadAssignment(property) || + (ts.isComputedPropertyName(property.name) && + !ts.isStringLiteralLike(property.name.expression)) + ) { + resolved = undefined; + continue; + } + if (propertyNameText(property.name) === name) resolved = property; + } + return resolved; +} + +function childProcessOptions( + node: ts.CallExpression, + provenance: Extract, +): ts.ObjectLiteralExpression | undefined { + // Indirect invocations (`bind`, `call`, `apply`, or `promisify`) can shift or + // pre-apply arguments. They cannot prove where the options object reaches + // Node, even if a later argument happens to look safe. + if (provenance.invocation !== "direct") return undefined; + + const args = [...node.arguments]; + const objectAt = (index: number): ts.ObjectLiteralExpression | undefined => { + const argument = args[index]; + return argument && ts.isObjectLiteralExpression(argument) ? argument : undefined; + }; + switch (provenance.api) { + case "exec": + if (args.length < 1 || args.length > 3) return undefined; + return objectAt(1); + case "execSync": + if (args.length < 1 || args.length > 2) return undefined; + return objectAt(1); + case "execFile": + if (args.length < 1 || args.length > 4) return undefined; + if (args.length >= 3 && objectAt(1)) { + return args.length === 3 && !ts.isObjectLiteralExpression(args[2] as ts.Expression) + ? objectAt(1) + : undefined; + } + if (args.length >= 3 && objectAt(2)) return objectAt(2); + return objectAt(1); + case "execFileSync": + case "fork": + case "spawn": + case "spawnSync": + if (args.length < 1 || args.length > 3) return undefined; + if (args.length === 3 && objectAt(1)) return undefined; + return args.length === 3 ? objectAt(2) : objectAt(1); + } +} + +function numericConstantInitializer(sourceFile: ts.SourceFile, name: string): ts.Expression | null { + for (const statement of sourceFile.statements) { + if (!ts.isVariableStatement(statement)) continue; + if (!(statement.declarationList.flags & ts.NodeFlags.Const)) continue; + for (const declaration of statement.declarationList.declarations) { + if (ts.isIdentifier(declaration.name) && declaration.name.text === name) { + return declaration.initializer ?? null; + } + } + } + return null; +} + +function numericExpressionValue( + expression: ts.Expression, + sourceFile: ts.SourceFile, + seen = new Set(), +): number | null { + if (ts.isNumericLiteral(expression)) return Number(expression.text); + if (ts.isParenthesizedExpression(expression)) { + return numericExpressionValue(expression.expression, sourceFile, seen); + } + if (ts.isPrefixUnaryExpression(expression)) { + const value = numericExpressionValue(expression.operand, sourceFile, seen); + if (value === null) return null; + if (expression.operator === ts.SyntaxKind.PlusToken) return value; + if (expression.operator === ts.SyntaxKind.MinusToken) return -value; + return null; + } + if (ts.isBinaryExpression(expression)) { + const left = numericExpressionValue(expression.left, sourceFile, seen); + const right = numericExpressionValue(expression.right, sourceFile, seen); + if (left === null || right === null) return null; + if (expression.operatorToken.kind === ts.SyntaxKind.PlusToken) return left + right; + if (expression.operatorToken.kind === ts.SyntaxKind.MinusToken) return left - right; + if (expression.operatorToken.kind === ts.SyntaxKind.AsteriskToken) return left * right; + if (expression.operatorToken.kind === ts.SyntaxKind.SlashToken && right !== 0) { + return left / right; + } + return null; + } + if (ts.isConditionalExpression(expression)) { + const whenTrue = numericExpressionValue(expression.whenTrue, sourceFile, seen); + const whenFalse = numericExpressionValue(expression.whenFalse, sourceFile, seen); + return whenTrue === null || whenFalse === null ? null : Math.max(whenTrue, whenFalse); + } + if (ts.isIdentifier(expression) && !seen.has(expression.text)) { + const initializer = numericConstantInitializer(sourceFile, expression.text); + if (!initializer) return null; + const nextSeen = new Set(seen); + nextSeen.add(expression.text); + return numericExpressionValue(initializer, sourceFile, nextSeen); + } + return null; +} + +function hasBoundedChildProcessTimeout( + options: ts.ObjectLiteralExpression | undefined, + sourceFile: ts.SourceFile, +): boolean { + const property = objectProperty(options, "timeout"); + if (!property || !ts.isPropertyAssignment(property)) return false; + const timeoutMs = numericExpressionValue(property.initializer, sourceFile); + return timeoutMs !== null && timeoutMs > 0 && timeoutMs < MAX_BLOCKING_CHILD_PROCESS_MS; +} + +function hasHardChildProcessKillSignal(options: ts.ObjectLiteralExpression | undefined): boolean { + const property = objectProperty(options, "killSignal"); + return ( + !!property && + ts.isPropertyAssignment(property) && + ts.isStringLiteralLike(property.initializer) && + property.initializer.text === "SIGKILL" + ); +} + +function ignoresChildOutput(options: ts.ObjectLiteralExpression | undefined): boolean { + const property = objectProperty(options, "stdio"); + if (!property || !ts.isPropertyAssignment(property)) return false; + const value = property.initializer; + if (ts.isStringLiteralLike(value)) return value.text === "ignore"; + if (!ts.isArrayLiteralExpression(value)) return false; + const stdout = value.elements[1]; + const stderr = value.elements[2]; + return ( + !!stdout && + !!stderr && + ts.isStringLiteralLike(stdout) && + stdout.text === "ignore" && + ts.isStringLiteralLike(stderr) && + stderr.text === "ignore" + ); +} + +function childProcessObservationScope(node: ts.CallExpression, sourceFile: ts.SourceFile): ts.Node { + let current: ts.Node | undefined = node.parent; + while (current && current !== sourceFile) { + if (ts.isFunctionLike(current)) return current; + current = current.parent; + } + return sourceFile; +} + +function expressionPath(expression: ts.Expression): string | null { + const unwrapped = unwrapExpression(expression); + if (unwrapped.kind === ts.SyntaxKind.ThisKeyword) return "this"; + if (ts.isIdentifier(unwrapped)) return unwrapped.text; + if (ts.isPropertyAccessExpression(unwrapped)) { + const receiver = expressionPath(unwrapped.expression); + return receiver ? `${receiver}.${unwrapped.name.text}` : null; + } + return null; +} + +function isUnconditionalBoundaryNode(node: ts.Node, scope: ts.Node): boolean { + let current: ts.Node | undefined = node; + while (current && current !== scope) { + if ( + ts.isIfStatement(current) || + ts.isConditionalExpression(current) || + ts.isSwitchStatement(current) || + ts.isForStatement(current) || + ts.isForInStatement(current) || + ts.isForOfStatement(current) || + ts.isWhileStatement(current) || + ts.isDoStatement(current) + ) { + return false; + } + if (current !== node && ts.isFunctionLike(current)) return false; + current = current.parent; + } + return current === scope; +} + +function assignedIdentifierForCall(node: ts.CallExpression, scope: ts.Node): string | null { + let current: ts.Node = node; + while (current.parent && current.parent !== scope) { + const parent = current.parent; + if (ts.isVariableDeclaration(parent) && parent.initializer === current) { + return ts.isIdentifier(parent.name) ? parent.name.text : null; + } + if ( + ts.isBinaryExpression(parent) && + parent.operatorToken.kind === ts.SyntaxKind.EqualsToken && + parent.right === current + ) { + return ts.isIdentifier(parent.left) ? parent.left.text : null; + } + if (ts.isStatement(parent) || ts.isFunctionLike(parent)) return null; + current = parent; + } + return null; +} + +function childBindingForCall(node: ts.CallExpression, scope: ts.Node): string | null { + const parent = node.parent; + if ( + ts.isVariableDeclaration(parent) && + parent.initializer === node && + ts.isIdentifier(parent.name) + ) { + return parent.name.text; + } + if ( + ts.isBinaryExpression(parent) && + parent.operatorToken.kind === ts.SyntaxKind.EqualsToken && + parent.right === node && + ts.isIdentifier(parent.left) && + isUnconditionalBoundaryNode(node, scope) + ) { + return parent.left.text; + } + return null; +} + +function inlineCallback( + node: ts.Expression | undefined, +): ts.ArrowFunction | ts.FunctionExpression | null { + return node && (ts.isArrowFunction(node) || ts.isFunctionExpression(node)) ? node : null; +} + +function callbackProtectedCall( + callback: ts.ArrowFunction | ts.FunctionExpression, +): ts.CallExpression | null { + if ( + callback.parameters.length !== 0 || + !ts.isBlock(callback.body) || + callback.body.statements.length !== 1 + ) { + return null; + } + const statement = callback.body.statements[0]; + if ( + !statement || + !ts.isTryStatement(statement) || + statement.finallyBlock || + !statement.catchClause || + statement.catchClause.variableDeclaration || + statement.catchClause.block.statements.length !== 0 || + statement.tryBlock.statements.length !== 1 + ) { + return null; + } + const expressionStatement = statement.tryBlock.statements[0]; + if (!expressionStatement || !ts.isExpressionStatement(expressionStatement)) return null; + const expression = unwrapExpression(expressionStatement.expression); + return ts.isCallExpression(expression) ? expression : null; +} + +function callbackInvokesIdentifier( + callback: ts.ArrowFunction | ts.FunctionExpression, + identifiers: ReadonlySet, +): boolean { + const call = callbackProtectedCall(callback); + return ( + !!call && + call.arguments.length === 0 && + ts.isIdentifier(call.expression) && + identifiers.has(call.expression.text) + ); +} + +function childLifecycleListener( + call: ts.CallExpression, + childBinding: string, + scope: ts.Node, +): ts.ArrowFunction | ts.FunctionExpression | null { + if (!isUnconditionalBoundaryNode(call, scope)) return null; + const expression = call.expression; + if (!ts.isPropertyAccessExpression(expression)) return null; + if (!["on", "once", "addListener"].includes(expression.name.text)) return null; + if (expressionPath(expression.expression) !== childBinding) return null; + const event = call.arguments[0]; + if (!event || !ts.isStringLiteralLike(event) || !["close", "exit"].includes(event.text)) { + return null; + } + return inlineCallback(call.arguments[1]); +} + +function exactTimestampOutputCall(node: ts.CallExpression, stream: "stdout" | "stderr"): boolean { + if ( + !ts.isPropertyAccessExpression(node.expression) || + node.expression.name.text !== "onOutput" || + expressionPath(node.expression.expression) !== "options.progress" + ) { + return false; + } + const event = node.arguments[0]; + if (!event || !ts.isObjectLiteralExpression(event) || event.properties.length !== 2) return false; + const streamProperty = objectProperty(event, "stream"); + const atMsProperty = objectProperty(event, "atMs"); + if ( + !streamProperty || + !ts.isPropertyAssignment(streamProperty) || + !ts.isStringLiteralLike(streamProperty.initializer) || + streamProperty.initializer.text !== stream || + !atMsProperty || + !ts.isPropertyAssignment(atMsProperty) || + !ts.isCallExpression(atMsProperty.initializer) || + atMsProperty.initializer.arguments.length !== 0 || + !ts.isPropertyAccessExpression(atMsProperty.initializer.expression) || + expressionPath(atMsProperty.initializer.expression.expression) !== "Date" || + atMsProperty.initializer.expression.name.text !== "now" + ) { + return false; + } + return true; +} + +function callbackReportsExactOutput( + callback: ts.ArrowFunction | ts.FunctionExpression, + stream: "stdout" | "stderr", +): boolean { + const call = callbackProtectedCall(callback); + return !!call && exactTimestampOutputCall(call, stream); +} + +function childOutputListener( + call: ts.CallExpression, + childBinding: string, + stream: "stdout" | "stderr", + scope: ts.Node, +): ts.ArrowFunction | ts.FunctionExpression | null { + if (!isUnconditionalBoundaryNode(call, scope)) return null; + const expression = call.expression; + if ( + !ts.isPropertyAccessExpression(expression) || + !["addListener", "on", "once", "prependListener", "prependOnceListener"].includes( + expression.name.text, + ) + ) { + return null; + } + if (expressionPath(expression.expression) !== `${childBinding}.${stream}`) return null; + const event = call.arguments[0]; + if (!event || !ts.isStringLiteralLike(event) || event.text !== "data") return null; + return inlineCallback(call.arguments[1]); +} + +function observedChildProcessEvidence( + node: ts.CallExpression, + sourceFile: ts.SourceFile, +): { observesOutput: boolean; tracksActivity: boolean } { + const scope = childProcessObservationScope(node, sourceFile); + const childBinding = childBindingForCall(node, scope); + if (!childBinding || !isUnconditionalBoundaryNode(node, scope)) { + return { observesOutput: false, tracksActivity: false }; + } + const observedChildBinding = childBinding; + const activityHandles = new Set(); + let lifecycleListeners = 0; + let validLifecycleListeners = 0; + let childBindingReassigned = false; + let activityHandleReassigned = false; + let unsafeChildOperation = false; + const outputListeners = new Map<"stdout" | "stderr", number>([ + ["stdout", 0], + ["stderr", 0], + ]); + const validOutputListeners = new Map<"stdout" | "stderr", number>([ + ["stdout", 0], + ["stderr", 0], + ]); + let unsafeOutputOperation = false; + + function inspect(candidate: ts.Node): void { + if (candidate !== scope && ts.isFunctionLike(candidate)) return; + if ( + candidate.getStart(sourceFile) > node.getStart(sourceFile) && + ts.isBinaryExpression(candidate) && + candidate.operatorToken.kind === ts.SyntaxKind.EqualsToken && + ts.isIdentifier(candidate.left) + ) { + if (candidate.left.text === observedChildBinding) childBindingReassigned = true; + if (activityHandles.has(candidate.left.text)) activityHandleReassigned = true; + } + if ( + candidate.getStart(sourceFile) > node.getStart(sourceFile) && + ts.isVariableDeclaration(candidate) && + candidate.initializer && + [ + observedChildBinding, + `${observedChildBinding}.stdout`, + `${observedChildBinding}.stderr`, + ].includes(expressionPath(candidate.initializer) ?? "") + ) { + unsafeChildOperation = true; + } + if (ts.isCallExpression(candidate)) { + if ( + candidate.getStart(sourceFile) < node.getStart(sourceFile) && + isUnconditionalBoundaryNode(candidate, scope) && + ts.isPropertyAccessExpression(candidate.expression) && + candidate.expression.name.text === "activity" && + expressionPath(candidate.expression.expression) === "options.progress" + ) { + const handle = assignedIdentifierForCall(candidate, scope); + if (handle) activityHandles.add(handle); + } + if (candidate.getStart(sourceFile) > node.getStart(sourceFile)) { + const lifecycle = childLifecycleListener(candidate, observedChildBinding, scope); + if (lifecycle) { + lifecycleListeners += 1; + if (callbackInvokesIdentifier(lifecycle, activityHandles)) { + validLifecycleListeners += 1; + } + } + for (const stream of ["stdout", "stderr"] as const) { + const callback = childOutputListener(candidate, observedChildBinding, stream, scope); + if (callback) { + outputListeners.set(stream, (outputListeners.get(stream) ?? 0) + 1); + if (callbackReportsExactOutput(callback, stream)) { + validOutputListeners.set(stream, (validOutputListeners.get(stream) ?? 0) + 1); + } + } else { + const callPath = expressionPath(candidate.expression); + if (callPath?.startsWith(`${observedChildBinding}.${stream}.`)) { + unsafeOutputOperation = true; + } + if ( + candidate.arguments.some( + (argument) => expressionPath(argument) === `${observedChildBinding}.${stream}`, + ) + ) { + unsafeOutputOperation = true; + } + } + } + const callPath = expressionPath(candidate.expression); + if ( + callPath?.startsWith(`${observedChildBinding}.`) && + !lifecycle && + !(["stdout", "stderr"] as const).some((stream) => + childOutputListener(candidate, observedChildBinding, stream, scope), + ) + ) { + unsafeChildOperation = true; + } + } + } + ts.forEachChild(candidate, inspect); + } + inspect(scope); + return { + observesOutput: + !childBindingReassigned && + !unsafeChildOperation && + !unsafeOutputOperation && + (["stdout", "stderr"] as const).every( + (stream) => outputListeners.get(stream) === 1 && validOutputListeners.get(stream) === 1, + ), + tracksActivity: + !childBindingReassigned && + !activityHandleReassigned && + !unsafeChildOperation && + activityHandles.size === 1 && + lifecycleListeners === 1 && + validLifecycleListeners === 1, + }; +} + +function childProcessBoundaryName(node: ts.CallExpression, sourceFile: ts.SourceFile): string { + const scope = childProcessObservationScope(node, sourceFile); + if ( + (ts.isFunctionDeclaration(scope) || + ts.isFunctionExpression(scope) || + ts.isMethodDeclaration(scope)) && + scope.name + ) { + return propertyNameText(scope.name) ?? ""; + } + if ( + (ts.isArrowFunction(scope) || ts.isFunctionExpression(scope)) && + ts.isVariableDeclaration(scope.parent) && + ts.isIdentifier(scope.parent.name) + ) { + return scope.parent.name.text; + } + return ""; +} + +interface ObservedChildProcessBindings { + calls: Set; + namespaces: Set; +} + +function observedChildProcessBindings( + file: string, + sourceFile: ts.SourceFile, +): ObservedChildProcessBindings { + const bindings: ObservedChildProcessBindings = { calls: new Set(), namespaces: new Set() }; + for (const statement of sourceFile.statements) { + if ( + !ts.isImportDeclaration(statement) || + !ts.isStringLiteralLike(statement.moduleSpecifier) || + resolveE2EImport(file, statement.moduleSpecifier.text) !== OBSERVED_CHILD_PROCESS_MODULE || + statement.importClause?.isTypeOnly + ) { + continue; + } + const namedBindings = statement.importClause?.namedBindings; + if (namedBindings && ts.isNamespaceImport(namedBindings)) { + bindings.namespaces.add(namedBindings.name.text); + } + if (namedBindings && ts.isNamedImports(namedBindings)) { + for (const element of namedBindings.elements) { + if ( + !element.isTypeOnly && + (element.propertyName ?? element.name).text === "spawnObservedChild" + ) { + bindings.calls.add(element.name.text); + } + } + } + } + + const isObservedNamespaceReference = (expression: ts.Expression | undefined): boolean => { + if (!expression) return false; + const unwrapped = unwrapExpression(expression); + return ts.isIdentifier(unwrapped) && bindings.namespaces.has(unwrapped.text); + }; + const isObservedCallReference = (expression: ts.Expression | undefined): boolean => { + if (!expression) return false; + const unwrapped = unwrapExpression(expression); + if (ts.isIdentifier(unwrapped)) return bindings.calls.has(unwrapped.text); + if (!ts.isPropertyAccessExpression(unwrapped) && !ts.isElementAccessExpression(unwrapped)) { + return false; + } + const receiver = unwrapExpression(unwrapped.expression); + return ( + ts.isIdentifier(receiver) && + bindings.namespaces.has(receiver.text) && + memberName(unwrapped) === "spawnObservedChild" + ); + }; + + let changed = true; + while (changed) { + changed = false; + function inspect(node: ts.Node): void { + if ( + ts.isVariableDeclaration(node) && + ts.isIdentifier(node.name) && + isObservedNamespaceReference(node.initializer) && + !bindings.namespaces.has(node.name.text) + ) { + bindings.namespaces.add(node.name.text); + changed = true; + } + if ( + ts.isVariableDeclaration(node) && + ts.isObjectBindingPattern(node.name) && + isObservedNamespaceReference(node.initializer) + ) { + for (const element of node.name.elements) { + const imported = + propertyNameText(element.propertyName) ?? + (ts.isIdentifier(element.name) ? element.name.text : undefined); + if ( + imported === "spawnObservedChild" && + ts.isIdentifier(element.name) && + !bindings.calls.has(element.name.text) + ) { + bindings.calls.add(element.name.text); + changed = true; + } + } + } + if ( + ts.isVariableDeclaration(node) && + ts.isIdentifier(node.name) && + isObservedCallReference(node.initializer) && + !bindings.calls.has(node.name.text) + ) { + bindings.calls.add(node.name.text); + changed = true; + } + if ( + ts.isBinaryExpression(node) && + node.operatorToken.kind === ts.SyntaxKind.EqualsToken && + ts.isIdentifier(node.left) && + isObservedNamespaceReference(node.right) && + !bindings.namespaces.has(node.left.text) + ) { + bindings.namespaces.add(node.left.text); + changed = true; + } + if ( + ts.isBinaryExpression(node) && + node.operatorToken.kind === ts.SyntaxKind.EqualsToken && + ts.isIdentifier(node.left) && + isObservedCallReference(node.right) && + !bindings.calls.has(node.left.text) + ) { + bindings.calls.add(node.left.text); + changed = true; + } + ts.forEachChild(node, inspect); + } + inspect(sourceFile); + } + return bindings; +} + +function observedChildReference( + expression: ts.Expression, + bindings: ObservedChildProcessBindings, +): boolean { + const unwrapped = unwrapExpression(expression); + if (ts.isIdentifier(unwrapped)) return bindings.calls.has(unwrapped.text); + if (!ts.isPropertyAccessExpression(unwrapped) && !ts.isElementAccessExpression(unwrapped)) { + return false; + } + const receiver = unwrapExpression(unwrapped.expression); + return ( + ts.isIdentifier(receiver) && + bindings.namespaces.has(receiver.text) && + memberName(unwrapped) === "spawnObservedChild" + ); } -const LIVE_ROOT = path.join(REPO_ROOT, "test", "e2e", "live"); -const LIVE_TEST_FORWARDERS = new Map([ - ["test/e2e/live/bootstrap-install-smoke.test.ts", "test/e2e/live/launchable-smoke.test.ts"], -]); +function auditObservedChildProcessCall( + node: ts.CallExpression, + file: string, + sourceFile: ts.SourceFile, + reportUnsupported: (node: ts.Node, message: string) => void, +): void { + const relativeFile = path.relative(REPO_ROOT, file).split(path.sep).join("/"); + const boundary = childProcessBoundaryName(node, sourceFile); + const boundaryId = `${relativeFile}#${boundary}`; + const policy = OBSERVED_CHILD_PROGRESS_POLICIES.get(boundaryId); + if (!policy) { + reportUnsupported( + node, + `spawnObservedChild must use a reviewed progress-capability callsite (found ${boundary})`, + ); + return; + } + if (node.arguments.length !== 3) { + reportUnsupported(node, "spawnObservedChild must use its exact three-argument overload"); + return; + } + const options = node.arguments[2]; + if (!options || !ts.isObjectLiteralExpression(options)) { + reportUnsupported(node, "spawnObservedChild options must be a reviewable object literal"); + return; + } + const progress = objectProperty(options, "progress"); + if (!progress) { + reportUnsupported(node, "spawnObservedChild must receive the reviewed TestProgress capability"); + return; + } + const validProgress = + (ts.isPropertyAssignment(progress) && expressionPath(progress.initializer) === policy.path) || + (ts.isShorthandPropertyAssignment(progress) && progress.name.text === policy.path); + if (!validProgress) { + reportUnsupported( + progress, + "spawnObservedChild progress must retain the reviewed TestProgress capability", + ); + } +} -function resolveLiveImport(fromFile: string, specifier: string): string | null { - if (!specifier.startsWith(".")) return null; - const resolved = path.resolve(path.dirname(fromFile), specifier); - const candidates = [resolved, `${resolved}.ts`, path.join(resolved, "index.ts")]; - const livePrefix = `${LIVE_ROOT}${path.sep}`; - return ( - candidates.find( - (candidate) => - candidate.startsWith(livePrefix) && - fs.existsSync(candidate) && - fs.statSync(candidate).isFile(), - ) ?? null +function interfaceExtends(node: ts.InterfaceDeclaration, name: string): boolean { + return (node.heritageClauses ?? []).some((clause) => + clause.types.some((type) => ts.isIdentifier(type.expression) && type.expression.text === name), ); } -function importsSharedE2ETest(sourceFile: ts.SourceFile): boolean { - return sourceFile.statements.some((statement) => { +function compactSource(node: ts.Node, sourceFile: ts.SourceFile): string { + return node.getText(sourceFile).replace(/\s+/gu, ""); +} + +function auditTestProgressCapabilityContract( + file: string, + sourceFile: ts.SourceFile, + reportUnsupported: (node: ts.Node, message: string) => void, +): void { + if (file === TEST_PROGRESS_MODULE) { + const capabilityStatement = sourceFile.statements.find( + (statement): statement is ts.VariableStatement => + ts.isVariableStatement(statement) && + statement.declarationList.declarations.some( + (declaration) => + ts.isIdentifier(declaration.name) && + declaration.name.text === "TEST_PROGRESS_CAPABILITY", + ), + ); + const capabilityDeclaration = capabilityStatement?.declarationList.declarations.find( + (declaration) => + ts.isIdentifier(declaration.name) && declaration.name.text === "TEST_PROGRESS_CAPABILITY", + ); + const privateUniqueSymbol = + !!capabilityStatement && + !(capabilityStatement.modifiers ?? []).some( + (modifier) => modifier.kind === ts.SyntaxKind.ExportKeyword, + ) && + !!capabilityDeclaration?.type && + ts.isTypeOperatorNode(capabilityDeclaration.type) && + capabilityDeclaration.type.operator === ts.SyntaxKind.UniqueKeyword && + !!capabilityDeclaration.initializer && + ts.isCallExpression(capabilityDeclaration.initializer) && + ts.isIdentifier(capabilityDeclaration.initializer.expression) && + capabilityDeclaration.initializer.expression.text === "Symbol"; + if (!privateUniqueSymbol) { + reportUnsupported( + capabilityStatement ?? sourceFile, + "TestProgress capability must be backed by a module-private unique symbol", + ); + } + + const instancesStatement = sourceFile.statements.find( + (statement): statement is ts.VariableStatement => + ts.isVariableStatement(statement) && + statement.declarationList.declarations.some( + (declaration) => + ts.isIdentifier(declaration.name) && + declaration.name.text === "TEST_PROGRESS_INSTANCES", + ), + ); + const instancesDeclaration = instancesStatement?.declarationList.declarations.find( + (declaration) => + ts.isIdentifier(declaration.name) && declaration.name.text === "TEST_PROGRESS_INSTANCES", + ); if ( - !ts.isImportDeclaration(statement) || - !ts.isStringLiteral(statement.moduleSpecifier) || - !statement.moduleSpecifier.text.endsWith("/fixtures/e2e-test.ts") + !instancesStatement || + (instancesStatement.modifiers ?? []).some( + (modifier) => modifier.kind === ts.SyntaxKind.ExportKeyword, + ) || + !instancesDeclaration?.initializer || + !ts.isNewExpression(instancesDeclaration.initializer) || + !ts.isIdentifier(instancesDeclaration.initializer.expression) || + instancesDeclaration.initializer.expression.text !== "WeakSet" ) { - return false; + reportUnsupported( + instancesStatement ?? sourceFile, + "TestProgress instances must be registered in a module-private WeakSet", + ); } - if (statement.importClause?.isTypeOnly) return false; - const bindings = statement.importClause?.namedBindings; - return ( - !!bindings && - ts.isNamedImports(bindings) && - bindings.elements.some( - (element) => !element.isTypeOnly && (element.propertyName ?? element.name).text === "test", - ) + + const capabilityInterface = sourceFile.statements.find( + (statement): statement is ts.InterfaceDeclaration => + ts.isInterfaceDeclaration(statement) && statement.name.text === "TestProgressCapability", ); - }); -} + const brandMember = capabilityInterface?.members.find( + (member): member is ts.PropertySignature => + ts.isPropertySignature(member) && + !!member.name && + ts.isComputedPropertyName(member.name) && + ts.isIdentifier(member.name.expression) && + member.name.expression.text === "TEST_PROGRESS_CAPABILITY", + ); + if ( + !brandMember || + capabilityInterface?.members.length !== 1 || + !(brandMember.modifiers ?? []).some( + (modifier) => modifier.kind === ts.SyntaxKind.ReadonlyKeyword, + ) || + !brandMember.type || + brandMember.type.kind !== ts.SyntaxKind.LiteralType || + !ts.isLiteralTypeNode(brandMember.type) || + brandMember.type.literal.kind !== ts.SyntaxKind.TrueKeyword + ) { + reportUnsupported( + capabilityInterface ?? sourceFile, + "TestProgressCapability must expose only the private readonly true brand", + ); + } -function importsDirectVitestTest(sourceFile: ts.SourceFile): boolean { - return sourceFile.statements.some((statement) => { + const capabilityValidator = sourceFile.statements.find( + (statement): statement is ts.FunctionDeclaration => + ts.isFunctionDeclaration(statement) && statement.name?.text === "isTestProgressCapability", + ); + const expectedValidatorBody = + '{if(typeofvalue!=="object"||value===null||!TEST_PROGRESS_INSTANCES.has(value)){returnfalse;}constdescriptor=Object.getOwnPropertyDescriptor(value,TEST_PROGRESS_CAPABILITY);return(Object.isFrozen(value)&&descriptor?.value===true&&descriptor.enumerable===false&&descriptor.configurable===false&&descriptor.writable===false);}'; if ( - !ts.isImportDeclaration(statement) || - !ts.isStringLiteral(statement.moduleSpecifier) || - statement.moduleSpecifier.text !== "vitest" + !capabilityValidator?.body || + compactSource(capabilityValidator.body, sourceFile) !== expectedValidatorBody ) { - return false; + reportUnsupported( + capabilityValidator ?? sourceFile, + "isTestProgressCapability must exactly validate the private registry, own brand, and frozen object", + ); } - const bindings = statement.importClause?.namedBindings; - return ( - !!bindings && - ts.isNamedImports(bindings) && - bindings.elements.some((element) => - ["it", "test"].includes((element.propertyName ?? element.name).text), - ) + + const progressInterface = sourceFile.statements.find( + (statement): statement is ts.InterfaceDeclaration => + ts.isInterfaceDeclaration(statement) && statement.name.text === "TestProgress", ); - }); + if (!progressInterface || !interfaceExtends(progressInterface, "TestProgressCapability")) { + reportUnsupported( + progressInterface ?? sourceFile, + "TestProgress must retain the private progress capability", + ); + } + + const factory = sourceFile.statements.find( + (statement): statement is ts.FunctionDeclaration => + ts.isFunctionDeclaration(statement) && statement.name?.text === "startTestProgress", + ); + let initializesBrand = false; + if (factory) { + function inspectFactory(node: ts.Node): void { + if ( + ts.isPropertyAssignment(node) && + ts.isComputedPropertyName(node.name) && + ts.isIdentifier(node.name.expression) && + node.name.expression.text === "TEST_PROGRESS_CAPABILITY" && + node.initializer.kind === ts.SyntaxKind.TrueKeyword + ) { + initializesBrand = true; + } + ts.forEachChild(node, inspectFactory); + } + inspectFactory(factory); + } + const factoryStatements = factory?.body?.statements ?? []; + const factoryTail = factoryStatements + .slice(-3) + .map((statement) => compactSource(statement, sourceFile)); + const expectedFactoryTail = [ + "Object.defineProperty(progress,TEST_PROGRESS_CAPABILITY,{configurable:false,enumerable:false,value:true,writable:false,});", + "TEST_PROGRESS_INSTANCES.add(progress);", + "returnObject.freeze(progress);", + ]; + if ( + !factory || + !initializesBrand || + JSON.stringify(factoryTail) !== JSON.stringify(expectedFactoryTail) + ) { + reportUnsupported( + factory ?? sourceFile, + "startTestProgress must privately brand, register, and freeze the canonical capability", + ); + } + } + + if (file === OBSERVED_CHILD_PROCESS_MODULE) { + const childProgress = sourceFile.statements.find( + (statement): statement is ts.InterfaceDeclaration => + ts.isInterfaceDeclaration(statement) && statement.name.text === "ChildProcessProgress", + ); + const outputMember = childProgress?.members.find( + (member) => + (ts.isMethodSignature(member) || ts.isPropertySignature(member)) && + propertyNameText(member.name) === "onOutput", + ); + if (!childProgress || !interfaceExtends(childProgress, "TestProgressCapability")) { + reportUnsupported( + childProgress ?? sourceFile, + "ChildProcessProgress must require the private TestProgress capability", + ); + } + if (!outputMember || outputMember.questionToken) { + reportUnsupported( + childProgress ?? sourceFile, + "ChildProcessProgress must require timestamp-only output observation", + ); + } + const boundary = sourceFile.statements.find( + (statement): statement is ts.FunctionDeclaration => + ts.isFunctionDeclaration(statement) && statement.name?.text === "spawnObservedChild", + ); + const guard = boundary?.body?.statements[0]; + const validGuard = + !!guard && + ts.isIfStatement(guard) && + ts.isPrefixUnaryExpression(guard.expression) && + guard.expression.operator === ts.SyntaxKind.ExclamationToken && + ts.isCallExpression(guard.expression.operand) && + ts.isIdentifier(guard.expression.operand.expression) && + guard.expression.operand.expression.text === "isTestProgressCapability" && + guard.expression.operand.arguments.length === 1 && + expressionPath(guard.expression.operand.arguments[0] as ts.Expression) === + "options.progress" && + ts.isBlock(guard.thenStatement) && + guard.thenStatement.statements.length === 1 && + ts.isThrowStatement(guard.thenStatement.statements[0]); + if (!validGuard) { + reportUnsupported( + boundary ?? sourceFile, + "spawnObservedChild must reject non-canonical progress before spawning", + ); + } + } +} + +function collectDirectChildProcessCalls( + file: string, + sourceFile: ts.SourceFile, + auditFailures: string[], +): DirectChildProcessCall[] { + const bindings = directChildProcessBindings(sourceFile); + const observedBindings = observedChildProcessBindings(file, sourceFile); + const calls: DirectChildProcessCall[] = []; + const relativeFile = path.relative(REPO_ROOT, file).split(path.sep).join("/"); + const reportUnsupported = (node: ts.Node, message: string): void => { + const location = sourceFile.getLineAndCharacterOfPosition(node.getStart(sourceFile)); + auditFailures.push(`${relativeFile}:${location.line + 1}: ${message}`); + }; + const isEscapingProcessCapability = (provenance: ProcessProvenance | null): boolean => + !!provenance && + [ + "api", + "binder", + "builtin-loader", + "create-require", + "namespace", + "promisify", + "require", + "module-namespace", + ].includes(provenance.kind); + auditTestProgressCapabilityContract(file, sourceFile, reportUnsupported); + + function auditStoredCapability(node: ts.Node): void { + if ( + ts.isPropertyAssignment(node) && + !( + ts.isObjectLiteralExpression(node.parent) && + ts.isBinaryExpression(node.parent.parent) && + node.parent.parent.operatorToken.kind === ts.SyntaxKind.EqualsToken && + node.parent.parent.left === node.parent + ) + ) { + const provenance = expressionProvenance(node.initializer, bindings); + if (isEscapingProcessCapability(provenance)) { + reportUnsupported(node, "child-process APIs must not be stored in object properties"); + } + if (observedChildReference(node.initializer, observedBindings)) { + reportUnsupported(node, "spawnObservedChild must not be stored in object properties"); + } + } + if (ts.isShorthandPropertyAssignment(node)) { + const provenance = expressionProvenance(node.name, bindings); + if (isEscapingProcessCapability(provenance)) { + reportUnsupported(node, "child-process APIs must not be stored in object properties"); + } + if (observedChildReference(node.name, observedBindings)) { + reportUnsupported(node, "spawnObservedChild must not be stored in object properties"); + } + } + if (ts.isSpreadAssignment(node)) { + const provenance = expressionProvenance(node.expression, bindings); + if (isEscapingProcessCapability(provenance)) { + reportUnsupported(node, "child-process APIs must not be spread into object properties"); + } + if (observedChildReference(node.expression, observedBindings)) { + reportUnsupported(node, "spawnObservedChild must not be spread into object properties"); + } + } + if (ts.isArrayLiteralExpression(node)) { + for (const element of node.elements) { + if (!ts.isExpression(element)) continue; + const provenance = expressionProvenance(element, bindings); + if (isEscapingProcessCapability(provenance)) { + reportUnsupported(element, "child-process APIs must not be stored in array elements"); + } + if (observedChildReference(element, observedBindings)) { + reportUnsupported(element, "spawnObservedChild must not be stored in array elements"); + } + } + } + if (ts.isParameter(node)) { + const provenance = expressionProvenance(node.initializer, bindings); + if (isEscapingProcessCapability(provenance)) { + reportUnsupported(node, "child-process APIs must not be stored in parameter defaults"); + } + if (node.initializer && observedChildReference(node.initializer, observedBindings)) { + reportUnsupported(node, "spawnObservedChild must not be stored in parameter defaults"); + } + } + if ( + ts.isBinaryExpression(node) && + node.operatorToken.kind === ts.SyntaxKind.EqualsToken && + (ts.isPropertyAccessExpression(node.left) || ts.isElementAccessExpression(node.left)) + ) { + const provenance = expressionProvenance(node.right, bindings); + if (isEscapingProcessCapability(provenance)) { + reportUnsupported(node, "child-process APIs must not be stored in object properties"); + } + if (observedChildReference(node.right, observedBindings)) { + reportUnsupported(node, "spawnObservedChild must not be stored in object properties"); + } + } + if (ts.isExportDeclaration(node)) { + if (node.moduleSpecifier && ts.isStringLiteralLike(node.moduleSpecifier)) { + if (CHILD_PROCESS_MODULES.has(node.moduleSpecifier.text)) { + reportUnsupported(node, "child-process APIs must not be re-exported"); + } + if (resolveE2EImport(file, node.moduleSpecifier.text) === OBSERVED_CHILD_PROCESS_MODULE) { + reportUnsupported(node, "spawnObservedChild must not be re-exported"); + } + } else if (node.exportClause && ts.isNamedExports(node.exportClause)) { + for (const element of node.exportClause.elements) { + const provenance = expressionProvenance(element.propertyName ?? element.name, bindings); + if (isEscapingProcessCapability(provenance)) { + reportUnsupported(element, "child-process APIs must not be exported"); + } + if (observedChildReference(element.propertyName ?? element.name, observedBindings)) { + reportUnsupported(element, "spawnObservedChild must not be exported"); + } + } + } + } + } + + function inspect(node: ts.Node): void { + if (ts.isCallExpression(node)) { + if (observedChildReference(node.expression, observedBindings)) { + auditObservedChildProcessCall(node, file, sourceFile, reportUnsupported); + } + const apiProvenance = directChildProcessApi(node.expression, bindings); + if (apiProvenance && !isChildProcessWrapperCreation(node, bindings)) { + const options = childProcessOptions(node, apiProvenance); + const boundary = childProcessBoundaryName(node, sourceFile); + const boundaryId = `${relativeFile}#${boundary}`; + const evidence = + boundaryId === AUDITED_ASYNC_CHILD_PROCESS_BOUNDARY + ? observedChildProcessEvidence(node, sourceFile) + : { observesOutput: false, tracksActivity: false }; + const location = sourceFile.getLineAndCharacterOfPosition(node.getStart(sourceFile)); + calls.push({ + api: apiProvenance.api, + boundary, + file: relativeFile, + hasBoundedTimeout: hasBoundedChildProcessTimeout(options, sourceFile), + hasHardKillSignal: hasHardChildProcessKillSignal(options), + line: location.line + 1, + observesOutput: evidence.observesOutput, + outputIgnored: ignoresChildOutput(options), + tracksActivity: evidence.tracksActivity, + }); + } + const wrapperCreation = isChildProcessWrapperCreation(node, bindings); + if (!wrapperCreation) { + for (const argument of node.arguments) { + const provenance = expressionProvenance(argument, bindings); + if (isEscapingProcessCapability(provenance)) { + reportUnsupported( + argument, + "child-process APIs must not be passed to an unaudited higher-order function", + ); + } + if (observedChildReference(argument, observedBindings)) { + reportUnsupported( + argument, + "spawnObservedChild must not be passed to an unaudited higher-order function", + ); + } + } + } + } + if (ts.isNewExpression(node)) { + for (const argument of node.arguments ?? []) { + const provenance = expressionProvenance(argument, bindings); + if (isEscapingProcessCapability(provenance)) { + reportUnsupported( + argument, + "child-process APIs must not be passed to an unaudited constructor", + ); + } + if (observedChildReference(argument, observedBindings)) { + reportUnsupported(argument, "spawnObservedChild must not be passed to a constructor"); + } + } + } + if (ts.isElementAccessExpression(node)) { + const receiver = expressionProvenance(node.expression, bindings); + if (receiver?.kind === "namespace" && memberName(node) === null) { + reportUnsupported( + node, + "child-process namespace access must use a statically known audited API", + ); + } + } + if (ts.isPropertyAccessExpression(node) || ts.isElementAccessExpression(node)) { + const receiver = expressionProvenance(node.expression, bindings); + const name = memberName(node); + if ( + receiver?.kind === "namespace" && + name !== null && + !DIRECT_CHILD_PROCESS_APIS.has(name as DirectChildProcessCall["api"]) + ) { + reportUnsupported( + node, + `unsupported child-process namespace member must not escape the audit: ${name}`, + ); + } + const observedReceiver = memberReceiver(node); + if (observedReceiver && observedChildReference(observedReceiver, observedBindings)) { + reportUnsupported( + node, + "spawnObservedChild must be invoked directly without member indirection", + ); + } + } + if ( + ts.isVariableDeclaration(node) && + ts.isObjectBindingPattern(node.name) && + expressionProvenance(node.initializer, bindings)?.kind === "namespace" + ) { + for (const element of node.name.elements) { + const imported = + propertyNameText(element.propertyName) ?? + (ts.isIdentifier(element.name) ? element.name.text : undefined); + if ( + !imported || + !DIRECT_CHILD_PROCESS_APIS.has(imported as DirectChildProcessCall["api"]) + ) { + reportUnsupported( + element, + "child-process namespace destructuring must use a statically known audited API", + ); + } + } + } + if (ts.isReturnStatement(node) || ts.isExportAssignment(node)) { + const provenance = expressionProvenance(node.expression, bindings); + if (isEscapingProcessCapability(provenance)) { + reportUnsupported(node, "child-process APIs must not escape an audited source module"); + } + if (node.expression && observedChildReference(node.expression, observedBindings)) { + reportUnsupported(node, "spawnObservedChild must not escape an audited source module"); + } + } + auditStoredCapability(node); + ts.forEachChild(node, inspect); + } + inspect(sourceFile); + return calls; +} + +export function scanDirectChildProcessSource( + file: string, + source: string, +): { auditFailures: string[]; calls: DirectChildProcessCall[] } { + const sourceFile = ts.createSourceFile( + file, + source, + ts.ScriptTarget.Latest, + true, + ts.ScriptKind.TS, + ); + const auditFailures: string[] = []; + return { + auditFailures, + calls: collectDirectChildProcessCalls(file, sourceFile, auditFailures), + }; } function propertyNameText(name: ts.PropertyName | undefined): string | undefined { if (!name) return undefined; - return ts.isIdentifier(name) || ts.isStringLiteralLike(name) ? name.text : undefined; + if (ts.isIdentifier(name) || ts.isStringLiteralLike(name)) return name.text; + if (ts.isComputedPropertyName(name) && ts.isStringLiteralLike(name.expression)) { + return name.expression.text; + } + return undefined; } function hasE2EPhaseMetadata(node: ts.CallExpression): boolean { @@ -164,7 +1871,20 @@ function phaseCallFromNode( }; } -function forwardedLiveTestFromNode(node: ts.Node, file: string): string | null { +function dynamicLiveImportFromNode(node: ts.Node, file: string): string | null { + const specifier = ts.isCallExpression(node) ? node.arguments[0] : undefined; + if ( + !ts.isCallExpression(node) || + node.expression.kind !== ts.SyntaxKind.ImportKeyword || + !specifier || + !ts.isStringLiteral(specifier) + ) { + return null; + } + return resolveLiveImport(file, specifier.text); +} + +function dynamicE2EImportFromNode(node: ts.Node, file: string): string | null { const specifier = ts.isCallExpression(node) ? node.arguments[0] : undefined; if ( !ts.isCallExpression(node) || @@ -174,7 +1894,11 @@ function forwardedLiveTestFromNode(node: ts.Node, file: string): string | null { ) { return null; } - const resolved = resolveLiveImport(file, specifier.text); + return resolveE2EImport(file, specifier.text); +} + +function forwardedLiveTestFromNode(node: ts.Node, file: string): string | null { + const resolved = dynamicLiveImportFromNode(node, file); if (!resolved?.endsWith(".test.ts")) return null; return path.relative(REPO_ROOT, resolved).split(path.sep).join("/"); } @@ -250,11 +1974,15 @@ export function validateTestScopedPhaseCalls( export function scanLiveSourceGraph(entryFile: string): SemanticPhaseSourceGraph { const visited = new Set(); + const processVisited = new Set(); const forwardedTestModules: string[] = []; const phaseCalls: PhaseCall[] = []; + const directChildProcessCalls: DirectChildProcessCall[] = []; + const childProcessAuditFailures: string[] = []; let testPhaseBodies: TestPhaseBody[] = []; let importsDirectTest = false; let importsSharedTest = false; + let importsWorkflowTest = false; function visit(file: string): void { if (visited.has(file)) return; @@ -269,9 +1997,9 @@ export function scanLiveSourceGraph(entryFile: string): SemanticPhaseSourceGraph if (file === entryFile) { importsDirectTest = importsDirectVitestTest(sourceFile); importsSharedTest = importsSharedE2ETest(sourceFile); + importsWorkflowTest = importsWorkflowE2ETest(sourceFile); testPhaseBodies = collectTestPhaseBodies(file, sourceFile); } - for (const statement of sourceFile.statements) { if (ts.isImportDeclaration(statement) && ts.isStringLiteral(statement.moduleSpecifier)) { const importedFile = resolveLiveImport(file, statement.moduleSpecifier.text); @@ -280,6 +2008,8 @@ export function scanLiveSourceGraph(entryFile: string): SemanticPhaseSourceGraph } function inspect(node: ts.Node): void { + const dynamicImport = dynamicLiveImportFromNode(node, file); + if (dynamicImport) visit(dynamicImport); if (file === entryFile) { const forwardedTest = forwardedLiveTestFromNode(node, file); if (forwardedTest) forwardedTestModules.push(forwardedTest); @@ -291,16 +2021,100 @@ export function scanLiveSourceGraph(entryFile: string): SemanticPhaseSourceGraph inspect(sourceFile); } + function visitProcessBoundaries(file: string): void { + if (processVisited.has(file)) return; + processVisited.add(file); + const sourceFile = ts.createSourceFile( + file, + fs.readFileSync(file, "utf8"), + ts.ScriptTarget.Latest, + true, + ts.ScriptKind.TS, + ); + directChildProcessCalls.push( + ...collectDirectChildProcessCalls(file, sourceFile, childProcessAuditFailures), + ); + + for (const statement of sourceFile.statements) { + const moduleSpecifier = + (ts.isImportDeclaration(statement) || ts.isExportDeclaration(statement)) && + statement.moduleSpecifier && + ts.isStringLiteral(statement.moduleSpecifier) + ? statement.moduleSpecifier.text + : null; + if (!moduleSpecifier) continue; + const importedFile = resolveE2EImport(file, moduleSpecifier); + if (importedFile) visitProcessBoundaries(importedFile); + } + + function inspect(node: ts.Node): void { + const dynamicImport = dynamicE2EImportFromNode(node, file); + if (dynamicImport) visitProcessBoundaries(dynamicImport); + ts.forEachChild(node, inspect); + } + inspect(sourceFile); + } + visit(entryFile); + visitProcessBoundaries(entryFile); + for (const runtimeFile of E2E_RUNTIME_OBSERVABILITY_FILES) { + visitProcessBoundaries(runtimeFile); + } return { + childProcessAuditFailures, + directChildProcessCalls, forwardedTestModules, importsDirectTest, importsSharedTest, + importsWorkflowTest, phaseCalls, testPhaseBodies, }; } +function validateDirectChildProcessCalls(calls: readonly DirectChildProcessCall[]): string[] { + const failures: string[] = []; + const auditedCalls = calls.filter( + (call) => `${call.file}#${call.boundary}` === AUDITED_ASYNC_CHILD_PROCESS_BOUNDARY, + ); + if (auditedCalls.length > 1) { + failures.push( + `${AUDITED_ASYNC_CHILD_PROCESS_BOUNDARY}: audited boundary must contain exactly one direct asynchronous child-process call (found ${auditedCalls.length})`, + ); + } + for (const call of calls) { + if (call.api.endsWith("Sync") && !call.hasBoundedTimeout) { + failures.push( + `${call.file}:${call.line}: blocking child-process call ${call.api} must declare a positive timeout shorter than the first E2E heartbeat`, + ); + } + if (call.api.endsWith("Sync") && !call.hasHardKillSignal) { + failures.push( + `${call.file}:${call.line}: blocking child-process call ${call.api} must declare killSignal: "SIGKILL" so its timeout cannot be ignored`, + ); + } + if (!ASYNC_CHILD_PROCESS_APIS.has(call.api)) continue; + const boundary = `${call.file}#${call.boundary}`; + if (boundary !== AUDITED_ASYNC_CHILD_PROCESS_BOUNDARY) { + failures.push( + `${call.file}:${call.line}: asynchronous child-process call ${call.api} must use an audited progress-aware boundary (found ${call.boundary})`, + ); + continue; + } + if (!call.tracksActivity) { + failures.push( + `${call.file}:${call.line}: audited child-process boundary must register a content-free progress activity`, + ); + } + if (!call.outputIgnored && !call.observesOutput) { + failures.push( + `${call.file}:${call.line}: audited child-process boundary with child output must forward timestamp-only output observations`, + ); + } + } + return failures; +} + export function validateCollectedSemanticPhaseModule( collectedModule: CollectedSemanticPhaseModule, ): string[] { @@ -311,6 +2125,13 @@ export function validateCollectedSemanticPhaseModule( const scopedPhasePlans: ScopedPhasePlan[] = []; const moduleTests = collectedModule.tests.length; const forwardingTarget = LIVE_TEST_FORWARDERS.get(collectedModule.relativeModuleId); + const project = + collectedModule.project ?? + (collectedModule.relativeModuleId.startsWith("test/e2e/live/") ? "e2e-live" : "integration"); + failures.push( + ...(collectedModule.source.childProcessAuditFailures ?? []), + ...validateDirectChildProcessCalls(collectedModule.source.directChildProcessCalls ?? []), + ); if (forwardingTarget) { if (moduleTests !== 0) { @@ -350,14 +2171,20 @@ export function validateCollectedSemanticPhaseModule( } const { source } = collectedModule; - if (!source.importsSharedTest) { + if (project === "e2e-live") { + if (!source.importsSharedTest) { + failures.push( + `${collectedModule.relativeModuleId}: live E2E tests must import test from the shared e2e-test fixture`, + ); + } + } else if (!source.importsWorkflowTest) { failures.push( - `${collectedModule.relativeModuleId}: live E2E tests must import test from the shared e2e-test fixture`, + `${collectedModule.relativeModuleId}: workflow-selected integration E2E tests must import test from the workflow-e2e-test fixture`, ); } if (source.importsDirectTest) { failures.push( - `${collectedModule.relativeModuleId}: live E2E tests must not import test or it directly from Vitest`, + `${collectedModule.relativeModuleId}: workflow-selected E2E tests must not import test or it directly from Vitest`, ); } const declaredLabels = new Set(phasePlans.flat()); @@ -433,12 +2260,15 @@ function quietStream(): Writable { export async function checkSemanticPhaseCoverage(): Promise { process.env.NEMOCLAW_RUN_LIVE_E2E = "1"; process.env.NEMOCLAW_E2E_PHASE_COLLECTION = "1"; + const workflowModules = semanticPhaseCoverageModules(); + const expectedProjects = [...new Set(workflowModules.map(({ project }) => project))]; + const projectByFile = new Map(workflowModules.map(({ file, project }) => [file, project])); const vitest = await createVitest( "test", { root: REPO_ROOT, config: `${REPO_ROOT}/vitest.config.ts`, - project: ["e2e-live"], + project: expectedProjects, run: true, watch: false, silent: true, @@ -453,29 +2283,30 @@ export async function checkSemanticPhaseCoverage(): Promise file)); const failures: string[] = []; let tests = 0; - const expectedModules = new Set( - fs - .globSync("**/*.test.ts", { cwd: LIVE_ROOT }) - .map((file) => path.join("test/e2e/live", file).split(path.sep).join("/")), - ); + const expectedModules = new Set(workflowModules.map(({ file }) => file)); const collectedModules = new Set( result.testModules.map((module) => module.relativeModuleId.split(path.sep).join("/")), ); - if (expectedModules.size === 0) failures.push("no live E2E test files were discovered"); + if (expectedModules.size === 0) { + failures.push("no semantic E2E phase modules were discovered"); + } for (const expected of expectedModules) { - if (!collectedModules.has(expected)) failures.push(`${expected}: not collected by e2e-live`); + if (!collectedModules.has(expected)) { + failures.push(`${expected}: not collected by its semantic-phase Vitest project`); + } } for (const collected of collectedModules) { if (!expectedModules.has(collected)) - failures.push(`${collected}: unexpected live E2E module`); + failures.push(`${collected}: unexpected semantic E2E phase module`); } for (const error of result.unhandledErrors) failures.push(String(error)); for (const module of result.testModules) { + const relativeModuleId = module.relativeModuleId.split(path.sep).join("/"); const collectedTests = [...module.children.allTests()].map((test) => ({ fullName: test.fullName, phases: test.meta().e2ePhases, @@ -483,7 +2314,8 @@ export async function checkSemanticPhaseCoverage(): Promise error.message), tests: collectedTests, source: scanLiveSourceGraph(path.resolve(REPO_ROOT, module.relativeModuleId)), @@ -491,8 +2323,9 @@ export async function checkSemanticPhaseCoverage(): Promise 0) { - throw new Error(`semantic E2E phase coverage failed:\n${failures.join("\n")}`); + const uniqueFailures = [...new Set(failures)]; + if (uniqueFailures.length > 0) { + throw new Error(`semantic E2E phase coverage failed:\n${uniqueFailures.join("\n")}`); } return { files: result.testModules.length, tests }; } finally { diff --git a/tools/e2e/openshell-gateway-auth-artifact-safety.mts b/tools/e2e/openshell-gateway-auth-artifact-safety.mts new file mode 100644 index 0000000000..1c13378b52 --- /dev/null +++ b/tools/e2e/openshell-gateway-auth-artifact-safety.mts @@ -0,0 +1,459 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; +import fs, { type BigIntStats } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const LOCAL_ARTIFACT_SAFETY_RUN_ID = `local-${process.pid}`; +const NO_FOLLOW = fs.constants.O_NOFOLLOW ?? 0; + +const FORBIDDEN_AUTH_ARTIFACT_CONTENT: Array<{ label: string; pattern: RegExp }> = [ + { label: "authorization header", pattern: /["']?authorization["']?\s*[:=]/i }, + { + label: "Bearer JWT", + pattern: /\bBearer\s+[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\b/, + }, + { label: "JWT signing-key path", pattern: /(?:^|[/\\])jwt[/\\]signing\.pem\b/i }, + { label: "JWT key-id path", pattern: /(?:^|[/\\])jwt[/\\]kid\b/i }, + { label: "gateway auth config path", pattern: /\bopenshell-gateway\.toml\b/i }, + { + label: "gateway JWT configuration", + pattern: /\[openshell\.gateway\.gateway_jwt\]/i, + }, + { label: "private key", pattern: /-----BEGIN [A-Z ]*PRIVATE KEY-----/ }, +]; + +type ArtifactEntryKind = "directory" | "file"; + +type ScannedArtifactEntry = { + children?: string[]; + ctimeNs: bigint; + dev: bigint; + ino: bigint; + kind: ArtifactEntryKind; + mode: bigint; + mtimeNs: bigint; + nlink: bigint; + relativePath: string; + sha256?: string; + size: bigint; +}; + +type ScannedArtifactManifest = Map; + +function displayArtifactPath(relativePath: string): string { + return relativePath || "."; +} + +function artifactEntryKind(stat: BigIntStats): ArtifactEntryKind | null { + if (stat.isDirectory()) return "directory"; + if (stat.isFile()) return "file"; + return null; +} + +function artifactEntryIdentity( + stat: BigIntStats, + kind: ArtifactEntryKind, + relativePath: string, +): ScannedArtifactEntry { + return { + ctimeNs: stat.ctimeNs, + dev: stat.dev, + ino: stat.ino, + kind, + mode: stat.mode, + mtimeNs: stat.mtimeNs, + nlink: stat.nlink, + relativePath, + size: stat.size, + }; +} + +function assertEntryIdentity(stat: BigIntStats, expected: ScannedArtifactEntry): void { + if ( + artifactEntryKind(stat) !== expected.kind || + stat.ctimeNs !== expected.ctimeNs || + stat.dev !== expected.dev || + stat.ino !== expected.ino || + stat.mode !== expected.mode || + stat.mtimeNs !== expected.mtimeNs || + stat.nlink !== expected.nlink || + stat.size !== expected.size + ) { + throw new Error( + `Unsafe OpenShell auth-contract artifact '${displayArtifactPath(expected.relativePath)}': ` + + "entry identity changed during safety approval", + ); + } +} + +function lstatEntry(entryPath: string): BigIntStats { + return fs.lstatSync(entryPath, { bigint: true }); +} + +function fstatEntry(fileDescriptor: number): BigIntStats { + return fs.fstatSync(fileDescriptor, { bigint: true }); +} + +function sha256(content: Buffer): string { + return createHash("sha256").update(content).digest("hex"); +} + +function readDirectoryNames(directoryPath: string): string[] { + return fs + .readdirSync(directoryPath, { withFileTypes: true }) + .map((entry) => entry.name) + .sort(); +} + +function assertDirectoryNames(actual: readonly string[], expected: ScannedArtifactEntry): void { + if ( + !expected.children || + actual.length !== expected.children.length || + actual.some((name, index) => name !== expected.children?.[index]) + ) { + throw new Error( + `Unsafe OpenShell auth-contract artifact '${displayArtifactPath(expected.relativePath)}': ` + + "directory entries changed during safety approval", + ); + } +} + +function scanOpenShellGatewayAuthArtifacts(rootDir: string): ScannedArtifactManifest { + const root = path.resolve(rootDir); + const rootStat = lstatEntry(root); + if (!rootStat.isDirectory()) { + throw new Error("Unsafe OpenShell auth-contract artifact '.': non-directory root"); + } + const scannedRoot = artifactEntryIdentity(rootStat, "directory", ""); + const rootRealPath = fs.realpathSync(root); + assertEntryIdentity(lstatEntry(root), scannedRoot); + const manifest: ScannedArtifactManifest = new Map(); + const assertContained = (absolutePath: string, relativePath: string): void => { + const realPath = fs.realpathSync(absolutePath); + if (realPath !== rootRealPath && !realPath.startsWith(`${rootRealPath}${path.sep}`)) { + throw new Error( + `Unsafe OpenShell auth-contract artifact '${displayArtifactPath(relativePath)}': ` + + "entry resolves outside the artifact root", + ); + } + }; + const visit = (absolutePath: string, relativePath: string): void => { + const before = lstatEntry(absolutePath); + if (!relativePath) { + assertEntryIdentity(before, scannedRoot); + } + const kind = artifactEntryKind(before); + if (!kind) { + throw new Error( + `Unsafe OpenShell auth-contract artifact '${displayArtifactPath(relativePath)}': ` + + "non-regular file", + ); + } + assertContained(absolutePath, relativePath); + const scanned = artifactEntryIdentity(before, kind, relativePath); + manifest.set(relativePath, scanned); + + if (kind === "directory") { + const names = readDirectoryNames(absolutePath); + scanned.children = names; + assertEntryIdentity(lstatEntry(absolutePath), scanned); + assertContained(absolutePath, relativePath); + assertDirectoryNames(readDirectoryNames(absolutePath), scanned); + for (const name of names) { + const childPath = path.join(absolutePath, name); + const childRelativePath = relativePath ? `${relativePath}/${name}` : name; + visit(childPath, childRelativePath); + } + assertEntryIdentity(lstatEntry(absolutePath), scanned); + assertContained(absolutePath, relativePath); + assertDirectoryNames(readDirectoryNames(absolutePath), scanned); + assertEntryIdentity(lstatEntry(absolutePath), scanned); + return; + } + + if ( + /^(?:.*\/)?jwt\/(?:signing\.pem|kid)$|(?:^|\/)openshell-gateway\.toml$/i.test(relativePath) + ) { + throw new Error( + `Unsafe OpenShell auth-contract artifact '${relativePath}': sensitive auth file name`, + ); + } + const source = fs.openSync(absolutePath, fs.constants.O_RDONLY | NO_FOLLOW); + let content: Buffer; + try { + const sourceBeforeRead = fstatEntry(source); + assertEntryIdentity(sourceBeforeRead, scanned); + if (sourceBeforeRead.nlink !== 1n) { + throw new Error( + `Unsafe OpenShell auth-contract artifact '${relativePath}': regular file must have one link`, + ); + } + content = fs.readFileSync(source); + assertEntryIdentity(fstatEntry(source), scanned); + scanned.sha256 = sha256(content); + } finally { + fs.closeSync(source); + } + assertEntryIdentity(lstatEntry(absolutePath), scanned); + assertContained(absolutePath, relativePath); + const decodedContent = content.toString("utf8"); + const forbidden = FORBIDDEN_AUTH_ARTIFACT_CONTENT.find(({ pattern }) => + pattern.test(decodedContent), + ); + if (forbidden) { + throw new Error( + `Unsafe OpenShell auth-contract artifact '${relativePath}': ${forbidden.label}`, + ); + } + }; + visit(root, ""); + return manifest; +} + +export function assertOpenShellGatewayAuthArtifactsSafe(rootDir: string): void { + scanOpenShellGatewayAuthArtifacts(rootDir); +} + +function quarantineUnsafeOpenShellGatewayAuthArtifacts(rootDir: string): void { + const root = path.resolve(rootDir); + if (!fs.existsSync(root)) return; + + let quarantineRoot: string | undefined; + let moved = false; + try { + quarantineRoot = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-unsafe-auth-artifacts-")); + fs.chmodSync(quarantineRoot, 0o700); + fs.renameSync(root, path.join(quarantineRoot, "artifacts")); + moved = true; + } catch { + // Cross-device or restricted temp-directory moves can fail. Deleting the + // upload source still keeps rejected evidence outside the publication path. + } + + if (!moved) { + try { + fs.rmSync(root, { recursive: true, force: true, maxRetries: 3, retryDelay: 50 }); + if (fs.existsSync(root)) { + throw new Error("Unsafe OpenShell auth-contract artifacts could not be deleted"); + } + } finally { + if (quarantineRoot) { + fs.rmSync(quarantineRoot, { + recursive: true, + force: true, + maxRetries: 3, + retryDelay: 50, + }); + } + } + return; + } + + if (!quarantineRoot) { + throw new Error("Unsafe OpenShell auth-contract quarantine path was not created"); + } + try { + fs.rmSync(quarantineRoot, { recursive: true, force: true, maxRetries: 3, retryDelay: 50 }); + } catch (cause) { + throw new Error( + "Unsafe OpenShell auth-contract artifacts were quarantined outside the upload path but could not be deleted", + { cause }, + ); + } +} + +function rejectAndQuarantine(rootDir: string, error: unknown): never { + try { + quarantineUnsafeOpenShellGatewayAuthArtifacts(rootDir); + } catch (quarantineError) { + throw new AggregateError( + [error, quarantineError], + "OpenShell auth-contract artifacts failed safety approval and quarantine", + ); + } + throw error; +} + +export function enforceOpenShellGatewayAuthArtifactSafety(rootDir: string): void { + try { + assertOpenShellGatewayAuthArtifactsSafe(rootDir); + } catch (error) { + rejectAndQuarantine(rootDir, error); + } +} + +export function openShellGatewayAuthArtifactSafetyMarkerName( + env: NodeJS.ProcessEnv = process.env, +): string { + const runId = /^\d+$/.test(env.GITHUB_RUN_ID ?? "") + ? String(env.GITHUB_RUN_ID) + : LOCAL_ARTIFACT_SAFETY_RUN_ID; + const runAttempt = /^\d+$/.test(env.GITHUB_RUN_ATTEMPT ?? "") + ? String(env.GITHUB_RUN_ATTEMPT) + : "1"; + return `artifact-safety-${runId}-${runAttempt}.passed`; +} + +function copyApprovedArtifacts( + sourceRoot: string, + approvedRoot: string, + manifest: ScannedArtifactManifest, +): void { + const manifestEntry = (relativePath: string, kind: ArtifactEntryKind): ScannedArtifactEntry => { + const entry = manifest.get(relativePath); + if (!entry || entry.kind !== kind) { + throw new Error( + `Unsafe OpenShell auth-contract artifact '${displayArtifactPath(relativePath)}': ` + + "scanned entry is unavailable during safety approval", + ); + } + return entry; + }; + const sourcePathFor = (relativePath: string): string => + relativePath ? path.join(sourceRoot, ...relativePath.split("/")) : sourceRoot; + const copyRegularFile = ( + sourcePath: string, + approvedPath: string, + expected: ScannedArtifactEntry, + ): void => { + if (!expected.sha256) { + throw new Error( + `Unsafe OpenShell auth-contract artifact '${displayArtifactPath(expected.relativePath)}': ` + + "scanned content digest is unavailable during safety approval", + ); + } + assertEntryIdentity(lstatEntry(sourcePath), expected); + const source = fs.openSync(sourcePath, fs.constants.O_RDONLY | NO_FOLLOW); + try { + const sourceStat = fstatEntry(source); + assertEntryIdentity(sourceStat, expected); + if (sourceStat.nlink !== 1n) { + throw new Error( + `Unsafe OpenShell auth-contract artifact '${displayArtifactPath(expected.relativePath)}': ` + + "regular file must have one link", + ); + } + const approved = fs.openSync( + approvedPath, + fs.constants.O_WRONLY | fs.constants.O_CREAT | fs.constants.O_EXCL | NO_FOLLOW, + 0o600, + ); + try { + const copiedHash = createHash("sha256"); + const buffer = Buffer.allocUnsafe(64 * 1024); + let count = fs.readSync(source, buffer, 0, buffer.length, null); + while (count > 0) { + copiedHash.update(buffer.subarray(0, count)); + let written = 0; + while (written < count) { + written += fs.writeSync(approved, buffer, written, count - written); + } + count = fs.readSync(source, buffer, 0, buffer.length, null); + } + if (copiedHash.digest("hex") !== expected.sha256) { + throw new Error( + `Unsafe OpenShell auth-contract artifact '${displayArtifactPath(expected.relativePath)}': ` + + "file content changed during safety approval", + ); + } + fs.fchmodSync(approved, 0o600); + fs.fsyncSync(approved); + assertEntryIdentity(fstatEntry(source), expected); + } finally { + fs.closeSync(approved); + } + } finally { + fs.closeSync(source); + } + assertEntryIdentity(lstatEntry(sourcePath), expected); + }; + const copy = (relativePath: string, approvedDir: string): void => { + const expectedDirectory = manifestEntry(relativePath, "directory"); + const sourceDir = sourcePathFor(relativePath); + assertEntryIdentity(lstatEntry(sourceDir), expectedDirectory); + assertDirectoryNames(readDirectoryNames(sourceDir), expectedDirectory); + assertEntryIdentity(lstatEntry(sourceDir), expectedDirectory); + for (const name of expectedDirectory.children ?? []) { + const childRelativePath = relativePath ? `${relativePath}/${name}` : name; + const expectedChild = manifest.get(childRelativePath); + if (!expectedChild) { + throw new Error( + `Unsafe OpenShell auth-contract artifact '${childRelativePath}': ` + + "scanned entry is unavailable during safety approval", + ); + } + const sourcePath = sourcePathFor(childRelativePath); + const approvedPath = path.join(approvedDir, name); + if (expectedChild.kind === "directory") { + assertEntryIdentity(lstatEntry(sourcePath), expectedChild); + fs.mkdirSync(approvedPath, { mode: 0o700 }); + copy(childRelativePath, approvedPath); + continue; + } + copyRegularFile(sourcePath, approvedPath, expectedChild); + } + assertEntryIdentity(lstatEntry(sourceDir), expectedDirectory); + assertDirectoryNames(readDirectoryNames(sourceDir), expectedDirectory); + assertEntryIdentity(lstatEntry(sourceDir), expectedDirectory); + }; + copy("", approvedRoot); +} + +export function scanAndApproveOpenShellGatewayAuthArtifacts( + rootDir: string, + env: NodeJS.ProcessEnv = process.env, +): string { + let approvedRoot: string | undefined; + try { + const manifest = scanOpenShellGatewayAuthArtifacts(rootDir); + approvedRoot = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-approved-auth-artifacts-"), { + encoding: "utf8", + }); + fs.chmodSync(approvedRoot, 0o700); + copyApprovedArtifacts(path.resolve(rootDir), approvedRoot, manifest); + assertOpenShellGatewayAuthArtifactsSafe(approvedRoot); + const safetyMarker = path.join(approvedRoot, openShellGatewayAuthArtifactSafetyMarkerName(env)); + fs.writeFileSync(safetyMarker, "approved\n", { + encoding: "utf8", + flag: "wx", + mode: 0o600, + }); + return approvedRoot; + } catch (error) { + if (approvedRoot) { + fs.rmSync(approvedRoot, { recursive: true, force: true, maxRetries: 3, retryDelay: 50 }); + } + rejectAndQuarantine(rootDir, error); + } +} + +function runCli(): void { + const [rootDir, ...extra] = process.argv.slice(2); + if (!rootDir || extra.length > 0) { + throw new Error( + "Usage: node --experimental-strip-types tools/e2e/openshell-gateway-auth-artifact-safety.mts ", + ); + } + const approvedRoot = scanAndApproveOpenShellGatewayAuthArtifacts(rootDir); + const githubOutput = process.env.GITHUB_OUTPUT; + if (githubOutput) { + fs.appendFileSync(githubOutput, `approved_path=${approvedRoot}\n`, "utf8"); + } + process.stdout.write( + `OpenShell gateway auth artifacts copied to approved staging: ${path.basename(approvedRoot)}\n`, + ); +} + +if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + try { + runCli(); + } catch (error) { + const message = error instanceof Error ? error.message : "artifact safety scan failed"; + process.stderr.write(`${message}\n`); + process.exitCode = 1; + } +} diff --git a/tools/e2e/openshell-gateway-auth-contract-workflow-boundary.mts b/tools/e2e/openshell-gateway-auth-contract-workflow-boundary.mts index 74cf283c2f..1394634367 100644 --- a/tools/e2e/openshell-gateway-auth-contract-workflow-boundary.mts +++ b/tools/e2e/openshell-gateway-auth-contract-workflow-boundary.mts @@ -17,9 +17,15 @@ const EXPLICIT_ONLY_CONDITION = "${{ contains(format(',{0},', inputs.jobs), ',openshell-gateway-auth-contract,') || contains(format(',{0},', inputs.targets), ',openshell-gateway-auth-contract,') }}"; const GATEWAY_PROBE_IMAGE = "node:22-trixie-slim@sha256:e6d9a389d34ff9678438af985c9913fbd1eb6ed36e80fea56644f4b4f6dd70ba"; +const ARTIFACT_SAFETY_GATED_UPLOAD = + "${{ always() && steps.artifact_safety.outcome == 'success' && steps.artifact_safety.outputs.approved_path != '' }}"; +const APPROVED_ARTIFACT_PATH = "${{ steps.artifact_safety.outputs.approved_path }}"; +const ARTIFACT_SAFETY_COMMAND = + 'node --experimental-strip-types --no-warnings tools/e2e/openshell-gateway-auth-artifact-safety.mts "$E2E_ARTIFACT_DIR"'; type WorkflowStep = { env?: Record; + id?: string; if?: string; name?: string; run?: string; @@ -155,9 +161,26 @@ export function validateOpenShellGatewayAuthContractWorkflow( errors.push(`${JOB_NAME} live test must not receive workflow credentials`); } + const artifactSafetyName = "Validate final OpenShell gateway auth contract artifacts"; + const artifactSafety = findStep(job, artifactSafetyName); + if (artifactSafety.id !== "artifact_safety" || artifactSafety.if !== "always()") { + errors.push(`${JOB_NAME} final artifact safety scan must run unconditionally with a stable id`); + } + if (artifactSafety.run?.trim() !== ARTIFACT_SAFETY_COMMAND) { + errors.push( + `${JOB_NAME} step '${artifactSafety.name ?? ""}' must run exactly: ${ARTIFACT_SAFETY_COMMAND}`, + ); + } + const upload = findStep(job, "Upload OpenShell gateway auth contract artifacts"); - if (upload.uses !== UPLOAD_E2E_ARTIFACTS_ACTION || upload.if !== "always()") { - errors.push(`${JOB_NAME} must always use the reviewed artifact uploader`); + if (upload.uses !== UPLOAD_E2E_ARTIFACTS_ACTION) { + errors.push(`${JOB_NAME} must use the reviewed artifact uploader`); + } + if (upload.if !== ARTIFACT_SAFETY_GATED_UPLOAD) { + errors.push(`${JOB_NAME} must upload artifacts only after this run attempt passes safety scan`); + } + if (upload.with?.path !== APPROVED_ARTIFACT_PATH) { + errors.push(`${JOB_NAME} must upload only the immutable approved artifact payload`); } requireStepOrder(errors, steps, "Prepare E2E workspace", "Install OpenShell CLI"); @@ -168,6 +191,13 @@ export function validateOpenShellGatewayAuthContractWorkflow( "Pre-pull pinned gateway auth probe image", ); requireStepOrder(errors, steps, "Pre-pull pinned gateway auth probe image", runName); + requireStepOrder(errors, steps, runName, artifactSafetyName); + requireStepOrder( + errors, + steps, + artifactSafetyName, + "Upload OpenShell gateway auth contract artifacts", + ); requireStepOrder(errors, steps, runName, "Upload OpenShell gateway auth contract artifacts"); return errors; diff --git a/tools/e2e/runner-pressure.mts b/tools/e2e/runner-pressure.mts index cea357cfa9..5f25b0707f 100644 --- a/tools/e2e/runner-pressure.mts +++ b/tools/e2e/runner-pressure.mts @@ -74,6 +74,8 @@ const CONTAINER_NAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/u; const BASELINE_FILE_MAX_BYTES = 4096; const PHASE_BASELINES_FILE_MAX_BYTES = 128 * BASELINE_FILE_MAX_BYTES; const CLASSIFICATION_FILE_MAX_BYTES = 2048; +const DEFAULT_RESOURCE_COMMAND_TIMEOUT_MS = 15_000; +const QUICK_RESOURCE_COMMAND_TIMEOUT_MS = 3_000; function readTextOrNull(path: string): string | null { try { @@ -83,9 +85,20 @@ function readTextOrNull(path: string): string | null { } } -function runOrNull(command: string, args: string[], timeout = 15_000): string | null { +function runOrNull( + command: string, + args: string[], + timeout: 3_000 | 15_000 = DEFAULT_RESOURCE_COMMAND_TIMEOUT_MS, +): string | null { try { - const result = spawnSync(command, args, { encoding: "utf-8", timeout }); + const result = spawnSync(command, args, { + encoding: "utf-8", + killSignal: "SIGKILL", + timeout: + timeout === QUICK_RESOURCE_COMMAND_TIMEOUT_MS + ? QUICK_RESOURCE_COMMAND_TIMEOUT_MS + : DEFAULT_RESOURCE_COMMAND_TIMEOUT_MS, + }); return result.status === 0 ? (result.stdout ?? null) : null; } catch { return null; diff --git a/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts b/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts index fb78f58594..33ac7d5238 100644 --- a/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts +++ b/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts @@ -34,6 +34,8 @@ const INNER_ALWAYS = "${{ always() }}"; const CALLER_ALWAYS = "always()"; const MCP_SCANNED_UPLOAD_CONDITION = "${{ always() && steps.mcp_artifact_secret_scan.outcome == 'success' }}"; +const GATEWAY_AUTH_SCANNED_UPLOAD_CONDITION = + "${{ always() && steps.artifact_safety.outcome == 'success' && steps.artifact_safety.outputs.approved_path != '' }}"; const TARGET_ID_PATTERN = /^[A-Za-z0-9_-]+$/; const SHARED_E2E_JOBS: ReadonlyMap = new Map([ @@ -141,6 +143,13 @@ const EXPLICIT_UPLOAD_CONTRACTS = new Map([ name: "e2e-openshell-gateway-upgrade-${{ matrix.id }}", }, ], + [ + "openshell-gateway-auth-contract", + { + name: "e2e-openshell-gateway-auth-contract", + path: "${{ steps.artifact_safety.outputs.approved_path }}", + }, + ], [ "bedrock-runtime-compatible-anthropic", { @@ -174,6 +183,7 @@ const EXPLICIT_UPLOAD_CONTRACTS = new Map([ const EXPLICIT_CALLER_CONDITIONS = new Map([ ["mcp-bridge", MCP_SCANNED_UPLOAD_CONDITION], ["mcp-bridge-dev", MCP_SCANNED_UPLOAD_CONDITION], + ["openshell-gateway-auth-contract", GATEWAY_AUTH_SCANNED_UPLOAD_CONDITION], ]); const EXPECTED_ACTION_INPUTS = {