From d8ed81573c97a4952cd3f5ddbfe493fb9199bf49 Mon Sep 17 00:00:00 2001 From: Charan Jagwani Date: Thu, 16 Jul 2026 10:05:50 -0700 Subject: [PATCH 01/25] ci(e2e): add exact image qualification handoff Signed-off-by: Charan Jagwani --- .../brev-launchable-qualification.yaml | 203 ++ ci/source-shape-test-budget.json | 5 + .../support/exact-image-manifest-cli.test.ts | 158 ++ .../support/exact-image-manifest-fixture.ts | 61 + test/e2e/support/exact-image-manifest.test.ts | 232 +++ ...act-image-qualification-controller.test.ts | 1250 ++++++++++++ ...exact-image-qualification-workflow.test.ts | 122 ++ test/helpers/vitest-watch-triggers.ts | 4 + test/vitest-watch-triggers.test.ts | 4 + tools/advisors/github.mts | 9 +- tools/e2e/exact-image-manifest.mts | 383 ++++ .../exact-image-qualification-controller.mts | 1672 +++++++++++++++++ tools/e2e/validate-exact-image-manifest.mts | 176 ++ 13 files changed, 4277 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/brev-launchable-qualification.yaml create mode 100644 test/e2e/support/exact-image-manifest-cli.test.ts create mode 100644 test/e2e/support/exact-image-manifest-fixture.ts create mode 100644 test/e2e/support/exact-image-manifest.test.ts create mode 100644 test/exact-image-qualification-controller.test.ts create mode 100644 test/exact-image-qualification-workflow.test.ts create mode 100644 tools/e2e/exact-image-manifest.mts create mode 100755 tools/e2e/exact-image-qualification-controller.mts create mode 100644 tools/e2e/validate-exact-image-manifest.mts diff --git a/.github/workflows/brev-launchable-qualification.yaml b/.github/workflows/brev-launchable-qualification.yaml new file mode 100644 index 0000000000..fbf7b5b4fd --- /dev/null +++ b/.github/workflows/brev-launchable-qualification.yaml @@ -0,0 +1,203 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +name: Draft / Brev Launchable Qualification +run-name: Draft image qualification for ${{ inputs.candidate_sha }} + +on: + workflow_dispatch: + inputs: + candidate_sha: + description: Current lowercase 40-character NemoClaw main commit SHA to qualify. + required: true + type: string + reason: + description: Audit reason for starting this pre-tag qualification image build. + required: true + type: string + +permissions: {} + +concurrency: + group: draft-brev-launchable-qualification-${{ inputs.candidate_sha }} + cancel-in-progress: false + +jobs: + preflight: + name: Validate draft qualification request + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + steps: + - name: Checkout trusted controller + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + ref: ${{ github.workflow_sha }} + persist-credentials: false + + - name: Setup Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0 + with: + node-version: "22" + + - name: Validate candidate and maintainer authority + env: + ACTOR: ${{ github.triggering_actor }} + CANDIDATE_SHA: ${{ inputs.candidate_sha }} + EVENT_NAME: ${{ github.event_name }} + GITHUB_TOKEN: ${{ github.token }} + QUALIFICATION_REASON: ${{ inputs.reason }} + REQUESTER_RUN_ATTEMPT: ${{ github.run_attempt }} + REQUESTER_RUN_ID: ${{ github.run_id }} + REQUEST_REF: ${{ github.ref }} + WORKFLOW_SHA: ${{ github.workflow_sha }} + run: >- + node --experimental-strip-types --no-warnings + tools/e2e/exact-image-qualification-controller.mts + --mode preflight + --actor "$ACTOR" + --candidate-sha "$CANDIDATE_SHA" + --event-name "$EVENT_NAME" + --reason "$QUALIFICATION_REASON" + --ref "$REQUEST_REF" + --requester-run-attempt "$REQUESTER_RUN_ATTEMPT" + --requester-run-id "$REQUESTER_RUN_ID" + --workflow-sha "$WORKFLOW_SHA" + + qualify: + name: Build and verify exact staging image + needs: preflight + runs-on: ubuntu-latest + timeout-minutes: 60 + environment: + name: approve-brev-launchable-qualification + deployment: false + permissions: + contents: read + steps: + - name: Checkout trusted controller + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + ref: ${{ github.workflow_sha }} + persist-credentials: false + + - name: Setup Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0 + with: + node-version: "22" + + - id: workspace + name: Create private evidence workspace + shell: bash + run: | + set -euo pipefail + work_dir="$(mktemp -d "${RUNNER_TEMP}/nemoclaw-image-qualification.XXXXXX")" + chmod 700 "$work_dir" + printf 'work_dir=%s\n' "$work_dir" >> "$GITHUB_OUTPUT" + + - id: start + name: Dispatch exact qualification image build + env: + ACTOR: ${{ github.triggering_actor }} + CANDIDATE_SHA: ${{ inputs.candidate_sha }} + EVENT_NAME: ${{ github.event_name }} + GITHUB_TOKEN: ${{ github.token }} + NEMOCLAW_IMAGE_QUALIFICATION_TOKEN: ${{ secrets.NEMOCLAW_IMAGE_QUALIFICATION_TOKEN }} + QUALIFICATION_REASON: ${{ inputs.reason }} + REQUESTER_RUN_ATTEMPT: ${{ github.run_attempt }} + REQUESTER_RUN_ID: ${{ github.run_id }} + REQUEST_REF: ${{ github.ref }} + WORKFLOW_SHA: ${{ github.workflow_sha }} + run: >- + node --experimental-strip-types --no-warnings + tools/e2e/exact-image-qualification-controller.mts + --mode start + --actor "$ACTOR" + --candidate-sha "$CANDIDATE_SHA" + --event-name "$EVENT_NAME" + --reason "$QUALIFICATION_REASON" + --ref "$REQUEST_REF" + --requester-run-attempt "$REQUESTER_RUN_ATTEMPT" + --requester-run-id "$REQUESTER_RUN_ID" + --workflow-sha "$WORKFLOW_SHA" + --work-dir "${{ steps.workspace.outputs.work_dir }}" + + - name: Wait for the bound producer run + env: + NEMOCLAW_IMAGE_QUALIFICATION_TOKEN: ${{ secrets.NEMOCLAW_IMAGE_QUALIFICATION_TOKEN }} + run: >- + node --experimental-strip-types --no-warnings + tools/e2e/exact-image-qualification-controller.mts + --mode wait + --work-dir "${{ steps.workspace.outputs.work_dir }}" + + - name: Download and verify the bound manifest artifact + env: + NEMOCLAW_IMAGE_QUALIFICATION_TOKEN: ${{ secrets.NEMOCLAW_IMAGE_QUALIFICATION_TOKEN }} + run: >- + node --experimental-strip-types --no-warnings + tools/e2e/exact-image-qualification-controller.mts + --mode download + --work-dir "${{ steps.workspace.outputs.work_dir }}" + + - name: Validate the exact image manifest + env: + CANDIDATE_SHA: ${{ inputs.candidate_sha }} + CORRELATION_ID: ${{ steps.start.outputs.correlation_id }} + IMAGE_REPOSITORY_SHA: ${{ steps.start.outputs.image_repository_sha }} + PRODUCER_RUN_ATTEMPT: ${{ steps.start.outputs.producer_run_attempt }} + PRODUCER_RUN_ID: ${{ steps.start.outputs.producer_run_id }} + REQUESTER_RUN_ATTEMPT: ${{ github.run_attempt }} + REQUESTER_RUN_ID: ${{ github.run_id }} + run: >- + node --experimental-strip-types --no-warnings + tools/e2e/validate-exact-image-manifest.mts + --manifest "${{ steps.workspace.outputs.work_dir }}/nemoclaw-image-manifest.v1.json" + --output "${{ steps.workspace.outputs.work_dir }}/validated-manifest.v1.json" + --nemoclaw-sha "$CANDIDATE_SHA" + --requester-run-id "$REQUESTER_RUN_ID" + --requester-run-attempt "$REQUESTER_RUN_ATTEMPT" + --correlation-id "$CORRELATION_ID" + --image-repository-sha "$IMAGE_REPOSITORY_SHA" + --producer-run-id "$PRODUCER_RUN_ID" + --producer-run-attempt "$PRODUCER_RUN_ATTEMPT" + + - name: Record accepted provenance and hashes + run: >- + node --experimental-strip-types --no-warnings + tools/e2e/exact-image-qualification-controller.mts + --mode finalize + --work-dir "${{ steps.workspace.outputs.work_dir }}" + + - name: Cancel an incomplete producer run + if: ${{ always() && steps.workspace.outputs.work_dir != '' && steps.start.outcome != 'skipped' }} + continue-on-error: true + env: + NEMOCLAW_IMAGE_QUALIFICATION_TOKEN: ${{ secrets.NEMOCLAW_IMAGE_QUALIFICATION_TOKEN }} + run: >- + node --experimental-strip-types --no-warnings + tools/e2e/exact-image-qualification-controller.mts + --mode cancel + --work-dir "${{ steps.workspace.outputs.work_dir }}" + + - name: Upload draft qualification evidence + if: ${{ always() && steps.workspace.outputs.work_dir != '' }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: draft-brev-launchable-qualification-${{ inputs.candidate_sha }}-${{ github.run_id }} + path: | + ${{ steps.workspace.outputs.work_dir }}/dispatch-intent.v1.json + ${{ steps.workspace.outputs.work_dir }}/dispatch-reconciliation.v1.json + ${{ steps.workspace.outputs.work_dir }}/controller-state.json + ${{ steps.workspace.outputs.work_dir }}/controller-state.corrupt-*.json + ${{ steps.workspace.outputs.work_dir }}/nemoclaw-image-handoff.zip + ${{ steps.workspace.outputs.work_dir }}/nemoclaw-image-manifest.v1.json + ${{ steps.workspace.outputs.work_dir }}/validated-manifest.v1.json + ${{ steps.workspace.outputs.work_dir }}/qualification-evidence.v1.json + if-no-files-found: warn + retention-days: 90 + + - name: Remove private evidence workspace + if: ${{ always() && steps.workspace.outputs.work_dir != '' }} + run: rm -rf -- "${{ steps.workspace.outputs.work_dir }}" diff --git a/ci/source-shape-test-budget.json b/ci/source-shape-test-budget.json index 1daed33c77..ca2f8d9754 100644 --- a/ci/source-shape-test-budget.json +++ b/ci/source-shape-test-budget.json @@ -126,6 +126,11 @@ "test": "replaces legacy target_ref dispatches with the validated checkout contract", "category": "security" }, + { + "file": "test/exact-image-qualification-workflow.test.ts", + "test": "keeps draft exact-image qualification manual, protected, and evidence-only", + "category": "security" + }, { "file": "test/e2e/live/hermes-e2e.test.ts", "test": "hermes-e2e: install.sh onboards Hermes and proves health plus live inference", diff --git a/test/e2e/support/exact-image-manifest-cli.test.ts b/test/e2e/support/exact-image-manifest-cli.test.ts new file mode 100644 index 0000000000..fe0ac65037 --- /dev/null +++ b/test/e2e/support/exact-image-manifest-cli.test.ts @@ -0,0 +1,158 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; + +import { normalizedExactImageManifestJson } from "../../../tools/e2e/exact-image-manifest.mts"; +import { + CANDIDATE_SHA, + CORRELATION_ID, + exactImageManifest, + IMAGE_REPOSITORY_SHA, +} from "./exact-image-manifest-fixture.ts"; + +const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); +const CLI = path.join(REPO_ROOT, "tools/e2e/validate-exact-image-manifest.mts"); +const tempDirs: string[] = []; + +afterEach(() => { + for (const directory of tempDirs.splice(0)) { + fs.rmSync(directory, { recursive: true, force: true }); + } +}); + +function tempDir(): string { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-image-manifest-")); + tempDirs.push(directory); + return directory; +} + +function args(manifest: string, output: string, overrides: Record = {}): string[] { + const values = { + "--manifest": manifest, + "--output": output, + "--nemoclaw-sha": CANDIDATE_SHA, + "--requester-run-id": "8001", + "--requester-run-attempt": "1", + "--correlation-id": CORRELATION_ID, + "--image-repository-sha": IMAGE_REPOSITORY_SHA, + "--producer-run-id": "9002", + "--producer-run-attempt": "1", + ...overrides, + }; + return Object.entries(values).flat(); +} + +function runCli(cliArgs: string[]) { + return spawnSync(process.execPath, ["--experimental-strip-types", CLI, ...cliArgs], { + cwd: REPO_ROOT, + encoding: "utf8", + env: { ...process.env, NODE_NO_WARNINGS: "1" }, + }); +} + +function runCliWithTsx(cliArgs: string[]) { + return spawnSync(process.execPath, ["--import", "tsx", CLI, ...cliArgs], { + cwd: REPO_ROOT, + encoding: "utf8", + env: { ...process.env, NODE_NO_WARNINGS: "1" }, + }); +} + +describe("exact staging image manifest CLI", () => { + it("writes only normalized accepted JSON with private permissions", () => { + const directory = tempDir(); + const input = path.join(directory, "nemoclaw-image-manifest.v1.json"); + const output = path.join(directory, "accepted.json"); + const manifest = exactImageManifest(); + fs.writeFileSync(input, JSON.stringify(manifest), "utf8"); + + const result = runCli(args(input, output)); + + expect(result.status).toBe(0); + expect(result.stderr).toBe(""); + expect(fs.readFileSync(output, "utf8")).toBe(normalizedExactImageManifestJson(manifest)); + expect(fs.statSync(output).mode & 0o777).toBe(0o600); + }); + + it("loads through the repository tsx runtime", () => { + const directory = tempDir(); + const input = path.join(directory, "nemoclaw-image-manifest.v1.json"); + const output = path.join(directory, "accepted.json"); + fs.writeFileSync(input, JSON.stringify(exactImageManifest()), "utf8"); + + const result = runCliWithTsx(args(input, output)); + + expect(result.status).toBe(0); + expect(result.stderr).toBe(""); + expect(fs.existsSync(output)).toBe(true); + }); + + it("reports a stable provenance failure code and leaves no accepted output", () => { + const directory = tempDir(); + const input = path.join(directory, "manifest.json"); + const output = path.join(directory, "accepted.json"); + fs.writeFileSync(input, JSON.stringify(exactImageManifest()), "utf8"); + + const result = runCli(args(input, output, { "--producer-run-id": "9003" })); + + expect(result.status).toBe(1); + expect(result.stderr).toMatch(/^PROVENANCE_MISMATCH: workflowRunId/u); + expect(fs.existsSync(output)).toBe(false); + }); + + it("rejects symlinked, oversized, and non-UTF-8 inputs", () => { + const directory = tempDir(); + const valid = path.join(directory, "valid.json"); + fs.writeFileSync(valid, JSON.stringify(exactImageManifest()), "utf8"); + const symlink = path.join(directory, "symlink.json"); + fs.symlinkSync(valid, symlink); + + const oversized = path.join(directory, "oversized.json"); + fs.writeFileSync(oversized, Buffer.alloc(64 * 1024 + 1, 0x20)); + + const invalidUtf8 = path.join(directory, "invalid-utf8.json"); + fs.writeFileSync(invalidUtf8, Buffer.from([0xc3, 0x28])); + + for (const [name, input, message] of [ + ["symlink", symlink, "manifest input could not be opened safely"], + ["oversized", oversized, "manifest input exceeds 65536 bytes"], + ["non-UTF-8", invalidUtf8, "manifest input must be valid UTF-8"], + ]) { + const output = path.join(directory, `${name}-accepted.json`); + const result = runCli(args(input, output)); + expect(result.status, name).toBe(1); + expect(result.stderr, name).toContain(`ARTIFACT_MISSING_OR_INVALID: ${message}`); + expect(fs.existsSync(output), name).toBe(false); + } + }); + + it("refuses to replace a symlinked accepted-output path", () => { + const directory = tempDir(); + const input = path.join(directory, "manifest.json"); + const target = path.join(directory, "target.json"); + const output = path.join(directory, "accepted.json"); + fs.writeFileSync(input, JSON.stringify(exactImageManifest()), "utf8"); + fs.writeFileSync(target, "unchanged\n", "utf8"); + fs.symlinkSync(target, output); + + const result = runCli(args(input, output)); + + expect(result.status).toBe(1); + expect(result.stderr).toBe( + "OUTPUT_WRITE_FAILED: accepted manifest output could not be written safely\n", + ); + expect(fs.readFileSync(target, "utf8")).toBe("unchanged\n"); + }); + + it("reports invalid or incomplete invocation as a request failure", () => { + const result = runCli(["--manifest", "manifest.json"]); + + expect(result.status).toBe(1); + expect(result.stderr).toBe("REQUEST_INVALID: --output is required\n"); + }); +}); diff --git a/test/e2e/support/exact-image-manifest-fixture.ts b/test/e2e/support/exact-image-manifest-fixture.ts new file mode 100644 index 0000000000..5c6ab66f8e --- /dev/null +++ b/test/e2e/support/exact-image-manifest-fixture.ts @@ -0,0 +1,61 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { + ExactImageManifest, + ExactImageManifestExpectations, +} from "../../../tools/e2e/exact-image-manifest.mts"; + +export const CANDIDATE_SHA = "a".repeat(40); +export const IMAGE_REPOSITORY_SHA = "b".repeat(40); +export const CORRELATION_ID = "12345678-1234-4123-8123-123456789abc"; + +export function exactImageManifest( + overrides: Partial = {}, +): ExactImageManifest { + return { + schemaVersion: 1, + kind: "nemoclaw-exact-image-manifest", + correlationId: CORRELATION_ID, + requesterRepository: "NVIDIA/NemoClaw", + requesterWorkflowRunId: "8001", + requesterWorkflowRunAttempt: 1, + nemoclawSha: CANDIDATE_SHA, + imageRepository: "brevdev/nemoclaw-image", + imageRepositorySha: IMAGE_REPOSITORY_SHA, + producerWorkflow: ".github/workflows/build-qualification-image.yml", + workflowRunId: "9002", + workflowRunAttempt: 1, + imageOriginWorkflowRunId: "9002", + imageOriginWorkflowRunAttempt: 1, + imageKind: "compute#image", + project: "brevdevprod", + imageName: "nemoclaw-brev-cpu-v0-1-0-20260716-a-staging-190-1", + imageId: "12345678901234567890", + imageSelfLink: + "https://www.googleapis.com/compute/v1/projects/brevdevprod/global/images/nemoclaw-brev-cpu-v0-1-0-20260716-a-staging-190-1", + status: "READY", + imageCreationTimestamp: "2026-07-16T12:00:00.000Z", + manifestCreatedAt: "2026-07-16T12:01:00.000Z", + channel: "staging", + variant: "cpu", + observedFamily: "nemoclaw-brev-staging-cpu", + result: "built", + ...overrides, + }; +} + +export function exactImageManifestExpectations( + overrides: Partial = {}, +): ExactImageManifestExpectations { + return { + correlationId: CORRELATION_ID, + requesterWorkflowRunId: "8001", + requesterWorkflowRunAttempt: 1, + nemoclawSha: CANDIDATE_SHA, + imageRepositorySha: IMAGE_REPOSITORY_SHA, + workflowRunId: "9002", + workflowRunAttempt: 1, + ...overrides, + }; +} diff --git a/test/e2e/support/exact-image-manifest.test.ts b/test/e2e/support/exact-image-manifest.test.ts new file mode 100644 index 0000000000..c3d4d0eecb --- /dev/null +++ b/test/e2e/support/exact-image-manifest.test.ts @@ -0,0 +1,232 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { + EXACT_IMAGE_MANIFEST_MAX_CLOCK_SKEW_MS, + ExactImageManifestError, + normalizedExactImageManifestJson, + parseAndValidateExactImageManifest, + validateExactImageManifest, +} from "../../../tools/e2e/exact-image-manifest.mts"; +import { + CANDIDATE_SHA, + CORRELATION_ID, + exactImageManifest, + exactImageManifestExpectations, + IMAGE_REPOSITORY_SHA, +} from "./exact-image-manifest-fixture.ts"; + +function expectCode(run: () => unknown, code: ExactImageManifestError["code"]): void { + try { + run(); + } catch (error) { + expect(error).toBeInstanceOf(ExactImageManifestError); + expect((error as ExactImageManifestError).code).toBe(code); + return; + } + throw new Error(`expected ${code}`); +} + +describe("exact staging image manifest consumer", () => { + it("accepts and normalizes one exact built CPU image", () => { + const source = exactImageManifest(); + const accepted = validateExactImageManifest(source, exactImageManifestExpectations()); + + expect(accepted).toEqual(source); + expect(accepted.imageId).toBe("12345678901234567890"); + expect(normalizedExactImageManifestJson(accepted)).toBe(`${JSON.stringify(source, null, 2)}\n`); + }); + + it("accepts a reused image with required origin evidence at the 24-hour boundary", () => { + const reused = exactImageManifest({ + result: "reused", + imageOriginWorkflowRunId: "7000", + imageOriginWorkflowRunAttempt: 3, + imageCreationTimestamp: "2026-07-15T12:01:00.000Z", + }); + + expect(validateExactImageManifest(reused, exactImageManifestExpectations())).toEqual(reused); + }); + + it("requires reused manifests to carry both origin fields", () => { + for (const field of ["imageOriginWorkflowRunId", "imageOriginWorkflowRunAttempt"] as const) { + const reused = { ...exactImageManifest({ result: "reused" }) } as Record; + delete reused[field]; + expectCode( + () => validateExactImageManifest(reused, exactImageManifestExpectations()), + "ARTIFACT_MISSING_OR_INVALID", + ); + } + }); + + it("rejects missing and additional fields", () => { + const missing = { ...exactImageManifest() } as Record; + delete missing.imageId; + expectCode( + () => validateExactImageManifest(missing, exactImageManifestExpectations()), + "ARTIFACT_MISSING_OR_INVALID", + ); + + const additional = { ...exactImageManifest(), mutableImageFamilyFallback: true }; + expectCode( + () => validateExactImageManifest(additional, exactImageManifestExpectations()), + "ARTIFACT_MISSING_OR_INVALID", + ); + }); + + it.each([ + ["uppercase candidate SHA", { nemoclawSha: CANDIDATE_SHA.toUpperCase() }], + ["short image repository SHA", { imageRepositorySha: IMAGE_REPOSITORY_SHA.slice(0, 12) }], + ["uppercase correlation UUID", { correlationId: CORRELATION_ID.toUpperCase() }], + ["zero requester run ID", { requesterWorkflowRunId: "0" }], + ["numeric image ID", { imageId: 123456789 }], + ["zero image ID", { imageId: "0" }], + ["fractional workflow attempt", { workflowRunAttempt: 1.5 }], + ])("rejects an invalid %s", (_name, overrides) => { + expectCode( + () => + validateExactImageManifest( + { ...exactImageManifest(), ...overrides }, + exactImageManifestExpectations(), + ), + "ARTIFACT_MISSING_OR_INVALID", + ); + }); + + it.each([ + ["requester repository", { requesterRepository: "somewhere/NemoClaw" }], + ["image repository", { imageRepository: "brevdev/other-image" }], + ["producer workflow", { producerWorkflow: ".github/workflows/build-image.yml" }], + ["image kind", { imageKind: "compute#family" }], + ["status", { status: "PENDING" }], + ["channel", { channel: "production" }], + ["observed family", { observedFamily: "nemoclaw-brev-production-cpu" }], + ])("rejects the wrong fixed %s", (_name, overrides) => { + expectCode( + () => + validateExactImageManifest( + { ...exactImageManifest(), ...overrides }, + exactImageManifestExpectations(), + ), + "PROVENANCE_MISMATCH", + ); + }); + + it("rejects GPU before any later dispatch can consume it", () => { + expectCode( + () => + validateExactImageManifest( + { + ...exactImageManifest(), + variant: "gpu", + observedFamily: "nemoclaw-brev-staging-gpu", + }, + exactImageManifestExpectations(), + ), + "UNSUPPORTED_VARIANT", + ); + }); + + it("requires an immutable self-link reconstructed from project and image name", () => { + for (const imageSelfLink of [ + "https://www.googleapis.com/compute/v1/projects/brevdevprod/global/images/family/nemoclaw-brev-staging-cpu", + "https://www.googleapis.com/compute/v1/projects/other/global/images/nemoclaw-brev-cpu-v0-1-0-20260716-a-staging-190-1", + ]) { + expectCode( + () => + validateExactImageManifest( + exactImageManifest({ imageSelfLink }), + exactImageManifestExpectations(), + ), + "IMAGE_IDENTITY_MISMATCH", + ); + } + }); + + it("binds the manifest to the trusted request and producer run", () => { + const expectationMismatches = [ + { nemoclawSha: "c".repeat(40) }, + { requesterWorkflowRunId: "8002" }, + { requesterWorkflowRunAttempt: 2 }, + { correlationId: "87654321-4321-4321-8321-cba987654321" }, + { imageRepositorySha: "d".repeat(40) }, + { workflowRunId: "9003" }, + { workflowRunAttempt: 2 }, + ]; + for (const expected of expectationMismatches) { + expectCode( + () => + validateExactImageManifest( + exactImageManifest(), + exactImageManifestExpectations(expected), + ), + "PROVENANCE_MISMATCH", + ); + } + }); + + it("requires built images to originate in the current run", () => { + expectCode( + () => + validateExactImageManifest( + exactImageManifest({ imageOriginWorkflowRunId: "7000" }), + exactImageManifestExpectations(), + ), + "PROVENANCE_MISMATCH", + ); + }); + + it("enforces real timestamps, bounded clock skew, and the reused-image age", () => { + expectCode( + () => + validateExactImageManifest( + exactImageManifest({ imageCreationTimestamp: "2026-02-30T12:00:00Z" }), + exactImageManifestExpectations(), + ), + "ARTIFACT_MISSING_OR_INVALID", + ); + + const beyondSkew = new Date( + Date.parse("2026-07-16T12:01:00.000Z") + EXACT_IMAGE_MANIFEST_MAX_CLOCK_SKEW_MS + 1, + ).toISOString(); + const atSkew = new Date( + Date.parse("2026-07-16T12:01:00.000Z") + EXACT_IMAGE_MANIFEST_MAX_CLOCK_SKEW_MS, + ).toISOString(); + expect( + validateExactImageManifest( + exactImageManifest({ imageCreationTimestamp: atSkew }), + exactImageManifestExpectations(), + ).imageCreationTimestamp, + ).toBe(atSkew); + expectCode( + () => + validateExactImageManifest( + exactImageManifest({ imageCreationTimestamp: beyondSkew }), + exactImageManifestExpectations(), + ), + "PROVENANCE_MISMATCH", + ); + + expectCode( + () => + validateExactImageManifest( + exactImageManifest({ + result: "reused", + imageOriginWorkflowRunId: "7000", + imageCreationTimestamp: "2026-07-15T12:00:59.999Z", + }), + exactImageManifestExpectations(), + ), + "PROVENANCE_MISMATCH", + ); + }); + + it("rejects invalid JSON before semantic validation", () => { + expectCode( + () => parseAndValidateExactImageManifest("{not-json", exactImageManifestExpectations()), + "ARTIFACT_MISSING_OR_INVALID", + ); + }); +}); diff --git a/test/exact-image-qualification-controller.test.ts b/test/exact-image-qualification-controller.test.ts new file mode 100644 index 0000000000..fd6cfcef29 --- /dev/null +++ b/test/exact-image-qualification-controller.test.ts @@ -0,0 +1,1250 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { githubApi } from "../tools/advisors/github.mts"; +import { + ARCHIVE_FILE, + cancelActiveExactImageQualification, + DISPATCH_INTENT_FILE, + DISPATCH_RECONCILIATION_FILE, + downloadExactImageManifest, + EVIDENCE_FILE, + ExactImageQualificationError, + type ExactImageQualificationRequest, + extractExactManifestArchive, + finalizeExactImageQualification, + GITHUB_API_VERSION, + MANIFEST_ARTIFACT_FILE, + PRODUCER_REPOSITORY, + PRODUCER_WORKFLOW_FILE, + PRODUCER_WORKFLOW_PATH, + parseExactImageQualificationCommand, + preflightExactImageQualification, + type QualificationDependencies, + readExactImageQualificationState, + STATE_FILE, + startExactImageQualification, + VALIDATED_MANIFEST_FILE, + validateExactImageQualificationRequest, + validateQualificationArtifactList, + validateQualificationWorkflowRun, + validateWorkflowDispatchDetails, + waitForExactImageQualification, +} from "../tools/e2e/exact-image-qualification-controller.mts"; + +const CANDIDATE_SHA = "a".repeat(40); +const PRODUCER_SHA = "b".repeat(40); +const CORRELATION_ID = "123e4567-e89b-42d3-a456-426614174000"; +const RUN_ID = "24680"; +const WORKFLOW_ID = 13579; +const BASE_TIME = Date.UTC(2026, 0, 1); +const API_RUN_URL = `https://api.github.com/repos/${PRODUCER_REPOSITORY}/actions/runs/${RUN_ID}`; +const HTML_RUN_URL = `https://github.com/${PRODUCER_REPOSITORY}/actions/runs/${RUN_ID}`; + +const REQUEST: ExactImageQualificationRequest = { + actor: "maintainer", + candidateSha: CANDIDATE_SHA, + eventName: "workflow_dispatch", + reason: "Qualify the current daily candidate before tagging", + ref: "refs/heads/main", + requesterRunAttempt: 1, + requesterRunId: "97531", + workflowSha: CANDIDATE_SHA, +}; + +type ApiOptions = { + artifacts?: unknown; + dispatch?: unknown; + permission?: string; + roleName?: string; + producerSha?: string; + requesterSha?: string; + run?: unknown; + runs?: unknown; + workflow?: unknown; +}; + +function mainRef(sha: string) { + return { ref: "refs/heads/main", object: { type: "commit", sha } }; +} + +function dispatchDetails() { + return { + workflow_run_id: Number(RUN_ID), + run_url: API_RUN_URL, + html_url: HTML_RUN_URL, + }; +} + +function workflowRun(overrides: Record = {}) { + return { + id: Number(RUN_ID), + workflow_id: WORKFLOW_ID, + run_attempt: 1, + event: "workflow_dispatch", + head_branch: "main", + head_sha: PRODUCER_SHA, + path: PRODUCER_WORKFLOW_PATH, + display_title: `Qualify NemoClaw ${CANDIDATE_SHA} (${CORRELATION_ID})`, + url: API_RUN_URL, + html_url: HTML_RUN_URL, + repository: { full_name: PRODUCER_REPOSITORY }, + head_repository: { full_name: PRODUCER_REPOSITORY }, + status: "queued", + conclusion: null, + created_at: new Date(BASE_TIME).toISOString(), + ...overrides, + }; +} + +function createApi(options: ApiOptions = {}) { + return vi.fn(async (apiPath: string, _token: string, requestOptions?: unknown) => { + if (apiPath === "repos/NVIDIA/NemoClaw/git/ref/heads/main") { + return mainRef(options.requesterSha ?? CANDIDATE_SHA); + } + if (apiPath.includes("/collaborators/")) { + return { + permission: options.permission ?? "write", + role_name: options.roleName ?? "maintain", + }; + } + if (apiPath === `repos/${PRODUCER_REPOSITORY}/git/ref/heads/main`) { + return mainRef(options.producerSha ?? PRODUCER_SHA); + } + if (apiPath === `repos/${PRODUCER_REPOSITORY}/actions/workflows/${PRODUCER_WORKFLOW_FILE}`) { + return ( + options.workflow ?? { + id: WORKFLOW_ID, + path: PRODUCER_WORKFLOW_PATH, + state: "active", + } + ); + } + if ( + apiPath === + `repos/${PRODUCER_REPOSITORY}/actions/workflows/${PRODUCER_WORKFLOW_FILE}/dispatches` + ) { + return options.dispatch === undefined ? dispatchDetails() : options.dispatch; + } + if (apiPath === `repos/${PRODUCER_REPOSITORY}/actions/runs/${RUN_ID}`) { + return options.run ?? workflowRun(); + } + if (apiPath === `repos/${PRODUCER_REPOSITORY}/actions/runs/${RUN_ID}/cancel`) { + return undefined; + } + if ( + apiPath.startsWith( + `repos/${PRODUCER_REPOSITORY}/actions/workflows/${PRODUCER_WORKFLOW_FILE}/runs?`, + ) + ) { + return options.runs ?? { total_count: 0, workflow_runs: [] }; + } + if (apiPath === `repos/${PRODUCER_REPOSITORY}/actions/runs/${RUN_ID}/artifacts?per_page=100`) { + return options.artifacts; + } + throw new Error(`unexpected API call ${apiPath} ${JSON.stringify(requestOptions)}`); + }); +} + +function dependencies(api: ReturnType, extra: QualificationDependencies = {}) { + return { + now: () => BASE_TIME, + ...extra, + api: api as QualificationDependencies["api"], + randomUuid: () => CORRELATION_ID, + }; +} + +function tempDirectory(): string { + return fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-image-qualification-test-")); +} + +async function startedState(api = createApi()) { + const workDir = tempDirectory(); + const state = await startExactImageQualification( + { + request: REQUEST, + coreToken: "core-token", + producerToken: "producer-token", + workDir, + }, + dependencies(api), + ); + return { api, state, workDir }; +} + +function artifactList(archive: Buffer, overrides: Record = {}) { + const artifactId = 86420; + return { + total_count: 1, + artifacts: [ + { + id: artifactId, + name: `nemoclaw-image-handoff-v1-${RUN_ID}-1`, + expired: false, + digest: `sha256:${createHash("sha256").update(archive).digest("hex")}`, + size_in_bytes: archive.length, + url: `https://api.github.com/repos/${PRODUCER_REPOSITORY}/actions/artifacts/${artifactId}`, + archive_download_url: `https://api.github.com/repos/${PRODUCER_REPOSITORY}/actions/artifacts/${artifactId}/zip`, + workflow_run: { id: Number(RUN_ID), head_sha: PRODUCER_SHA }, + ...overrides, + }, + ], + }; +} + +afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllGlobals(); +}); + +describe("exact image qualification request", () => { + it("accepts only a first-attempt manual current-workflow candidate", () => { + expect(validateExactImageQualificationRequest(REQUEST)).toEqual(REQUEST); + expect(() => + validateExactImageQualificationRequest({ + ...REQUEST, + candidateSha: "c".repeat(40), + }), + ).toThrowError(ExactImageQualificationError); + expect(() => + validateExactImageQualificationRequest({ ...REQUEST, requesterRunAttempt: 2 }), + ).toThrow(/reruns/u); + expect(() => validateExactImageQualificationRequest({ ...REQUEST, reason: " bad " })).toThrow( + /reason/u, + ); + }); + + it("parses the fixed CLI surface and rejects undeclared controls", () => { + expect( + parseExactImageQualificationCommand([ + "--mode", + "preflight", + "--actor", + REQUEST.actor, + "--candidate-sha", + CANDIDATE_SHA, + "--event-name", + "workflow_dispatch", + "--reason", + REQUEST.reason, + "--ref", + "refs/heads/main", + "--requester-run-attempt", + "1", + "--requester-run-id", + REQUEST.requesterRunId, + "--workflow-sha", + CANDIDATE_SHA, + ]), + ).toEqual({ mode: "preflight", request: REQUEST }); + expect(() => + parseExactImageQualificationCommand([ + "--mode", + "wait", + "--work-dir", + "/tmp/work", + "--producer-ref", + "feature", + ]), + ).toThrow(/unknown argument/u); + }); +}); + +describe("exact producer dispatch binding", () => { + it("accepts GitHub's maintain role mapping but rejects ordinary write access", async () => { + const accepted = await startedState(createApi({ permission: "write", roleName: "maintain" })); + fs.rmSync(accepted.workDir, { recursive: true, force: true }); + + const workDir = tempDirectory(); + try { + await expect( + startExactImageQualification( + { + request: REQUEST, + coreToken: "core-token", + producerToken: "producer-token", + workDir, + }, + dependencies(createApi({ permission: "write", roleName: "write" })), + ), + ).rejects.toMatchObject({ code: "DISPATCH_FORBIDDEN" }); + } finally { + fs.rmSync(workDir, { recursive: true, force: true }); + } + }); + + it("uses the 2026 API contract and binds the returned run without listing runs", async () => { + const api = createApi(); + const { state, workDir } = await startedState(api); + try { + const dispatchCall = api.mock.calls.find(([apiPath]) => + String(apiPath).endsWith(`/workflows/${PRODUCER_WORKFLOW_FILE}/dispatches`), + ); + expect(dispatchCall?.[2]).toMatchObject({ + method: "POST", + apiVersion: GITHUB_API_VERSION, + expectedStatus: 200, + body: { + ref: "main", + inputs: { + nemoclaw_sha: CANDIDATE_SHA, + correlation_id: CORRELATION_ID, + requester_workflow_run_id: REQUEST.requesterRunId, + requester_workflow_run_attempt: "1", + }, + return_run_details: true, + }, + signal: expect.any(AbortSignal), + }); + expect(api.mock.calls.some(([apiPath]) => /actions\/runs\?/u.test(String(apiPath)))).toBe( + false, + ); + expect(state.producer.runId).toBe(RUN_ID); + expect(state.producer.repositorySha).toBe(PRODUCER_SHA); + expect(readExactImageQualificationState(workDir)).toEqual(state); + } finally { + fs.rmSync(workDir, { recursive: true, force: true }); + } + }); + + it("requires HTTP 200 and sends the explicit API-version header", async () => { + const fetchMock = vi.fn( + async (_input: string | URL | Request, _init?: RequestInit) => + new Response(null, { status: 204 }), + ); + vi.stubGlobal("fetch", fetchMock); + await expect( + githubApi("repos/example/actions/workflows/build.yml/dispatches", "token", { + method: "POST", + apiVersion: GITHUB_API_VERSION, + expectedStatus: 200, + body: { return_run_details: true }, + }), + ).rejects.toThrow(/204/u); + const headers = new Headers(fetchMock.mock.calls[0]?.[1]?.headers); + expect(headers.get("X-GitHub-Api-Version")).toBe(GITHUB_API_VERSION); + }); + + it.each([ + null, + {}, + { workflow_run_id: Number(RUN_ID) }, + ])("fails closed when dispatch returns no complete run details: %j", async (dispatch) => { + const workDir = tempDirectory(); + try { + const api = createApi({ dispatch }); + let clock = BASE_TIME; + await expect( + startExactImageQualification( + { + request: REQUEST, + coreToken: "core-token", + producerToken: "producer-token", + workDir, + }, + dependencies(api, { + now: () => clock, + sleep: async () => { + clock += 2; + }, + limits: { + dispatchReconciliationTimeoutMs: 1, + reconciliationPollIntervalMs: 1, + }, + }), + ), + ).rejects.toMatchObject({ code: "DISPATCH_AMBIGUOUS" }); + } finally { + fs.rmSync(workDir, { recursive: true, force: true }); + } + }); + + it("records, reconciles, cancels, and rejects a server-accepted dispatch whose response is lost", async () => { + const workDir = tempDirectory(); + const base = createApi(); + let cancelObserved = false; + const api = vi.fn(async (apiPath: string, token: string, requestOptions?: unknown) => { + if (apiPath.endsWith(`/workflows/${PRODUCER_WORKFLOW_FILE}/dispatches`)) { + throw new Error("response connection reset after server acceptance"); + } + if (apiPath.includes(`/workflows/${PRODUCER_WORKFLOW_FILE}/runs?`)) { + return { total_count: 1, workflow_runs: [workflowRun()] }; + } + if (apiPath === `repos/${PRODUCER_REPOSITORY}/actions/runs/${RUN_ID}/cancel`) { + expect(fs.existsSync(path.join(workDir, STATE_FILE))).toBe(true); + cancelObserved = true; + return undefined; + } + if (apiPath === `repos/${PRODUCER_REPOSITORY}/actions/runs/${RUN_ID}` && cancelObserved) { + return workflowRun({ status: "completed", conclusion: "cancelled" }); + } + return base(apiPath, token, requestOptions); + }); + try { + await expect( + startExactImageQualification( + { + request: REQUEST, + coreToken: "core-token", + producerToken: "producer-token", + workDir, + }, + dependencies(api as ReturnType), + ), + ).rejects.toMatchObject({ code: "DISPATCH_AMBIGUOUS" }); + expect( + api.mock.calls.filter(([apiPath]) => + String(apiPath).endsWith(`/workflows/${PRODUCER_WORKFLOW_FILE}/dispatches`), + ), + ).toHaveLength(1); + expect(cancelObserved).toBe(true); + expect(readExactImageQualificationState(workDir).producer.runId).toBe(RUN_ID); + expect(fs.existsSync(path.join(workDir, DISPATCH_INTENT_FILE))).toBe(true); + expect( + JSON.parse(fs.readFileSync(path.join(workDir, DISPATCH_RECONCILIATION_FILE), "utf8")), + ).toMatchObject({ outcome: "recovered-one", runIds: [RUN_ID] }); + } finally { + fs.rmSync(workDir, { recursive: true, force: true }); + } + }); + + it("finds the current correlation in a bounded creation window despite history and producer drift", async () => { + const workDir = tempDirectory(); + const movedSha = "c".repeat(40); + const base = createApi(); + let cancelled = false; + const movedRun = workflowRun({ head_sha: movedSha }); + const historicalRuns = Array.from({ length: 101 }, (_value, index) => + workflowRun({ created_at: new Date(BASE_TIME - (index + 2) * 10 * 60_000).toISOString() }), + ); + const api = vi.fn(async (apiPath: string, token: string, requestOptions?: unknown) => { + if (apiPath.endsWith(`/workflows/${PRODUCER_WORKFLOW_FILE}/dispatches`)) { + throw new Error("lost response after main advanced"); + } + if (apiPath.includes(`/workflows/${PRODUCER_WORKFLOW_FILE}/runs?`)) { + const query = new URLSearchParams(apiPath.split("?", 2)[1]); + expect(query.get("created")).toBe("2025-12-31T23:59:00Z..2026-01-01T00:01:30Z"); + expect(apiPath).not.toContain("head_sha="); + const [earliest, latest] = (query.get("created") ?? "").split("..").map(Date.parse); + const scoped = [...historicalRuns, movedRun].filter(({ created_at }) => { + const createdAt = Date.parse(created_at); + return createdAt >= earliest && createdAt <= latest; + }); + expect(historicalRuns).toHaveLength(101); + expect(scoped).toEqual([movedRun]); + return { total_count: scoped.length, workflow_runs: scoped }; + } + if (apiPath === `repos/${PRODUCER_REPOSITORY}/actions/runs/${RUN_ID}/cancel`) { + cancelled = true; + return undefined; + } + if (apiPath === `repos/${PRODUCER_REPOSITORY}/actions/runs/${RUN_ID}` && cancelled) { + return workflowRun({ + head_sha: movedSha, + status: "completed", + conclusion: "cancelled", + }); + } + return base(apiPath, token, requestOptions); + }); + try { + await expect( + startExactImageQualification( + { + request: REQUEST, + coreToken: "core-token", + producerToken: "producer-token", + workDir, + }, + dependencies(api as ReturnType), + ), + ).rejects.toMatchObject({ code: "DISPATCH_AMBIGUOUS" }); + expect(cancelled).toBe(true); + expect(readExactImageQualificationState(workDir).producer).toMatchObject({ + runId: RUN_ID, + repositorySha: movedSha, + }); + expect( + JSON.parse(fs.readFileSync(path.join(workDir, DISPATCH_RECONCILIATION_FILE), "utf8")), + ).toMatchObject({ + outcome: "recovered-one", + runIds: [RUN_ID], + producerHeadShas: { [RUN_ID]: movedSha }, + }); + } finally { + fs.rmSync(workDir, { recursive: true, force: true }); + } + }); + + it("uses the response-bound run ID for cleanup after strict normal-path provenance fails", async () => { + const workDir = tempDirectory(); + const movedSha = "c".repeat(40); + try { + await expect( + startExactImageQualification( + { + request: REQUEST, + coreToken: "core-token", + producerToken: "producer-token", + workDir, + }, + dependencies(createApi({ run: workflowRun({ head_sha: movedSha }) })), + ), + ).rejects.toMatchObject({ code: "PROVENANCE_MISMATCH" }); + expect(readExactImageQualificationState(workDir).producer.runId).toBe(RUN_ID); + + const base = createApi(); + let cancelled = false; + const cleanupApi = vi.fn(async (apiPath: string, token: string, requestOptions?: unknown) => { + if (apiPath === `repos/${PRODUCER_REPOSITORY}/actions/runs/${RUN_ID}/cancel`) { + cancelled = true; + return undefined; + } + if (apiPath === `repos/${PRODUCER_REPOSITORY}/actions/runs/${RUN_ID}`) { + return workflowRun({ + head_sha: movedSha, + status: cancelled ? "completed" : "queued", + conclusion: cancelled ? "cancelled" : null, + }); + } + return base(apiPath, token, requestOptions); + }); + await expect( + cancelActiveExactImageQualification( + { workDir, producerToken: "producer-token" }, + dependencies(cleanupApi as ReturnType), + ), + ).resolves.toBe(true); + expect(cancelled).toBe(true); + expect(cleanupApi.mock.calls[0]?.[0]).toBe( + `repos/${PRODUCER_REPOSITORY}/actions/runs/${RUN_ID}/cancel`, + ); + } finally { + fs.rmSync(workDir, { recursive: true, force: true }); + } + }); + + it("never accepts a recovered ambiguous dispatch that already completed successfully", async () => { + const workDir = tempDirectory(); + const base = createApi(); + const completed = workflowRun({ status: "completed", conclusion: "success" }); + const api = vi.fn(async (apiPath: string, token: string, requestOptions?: unknown) => { + if (apiPath.endsWith(`/workflows/${PRODUCER_WORKFLOW_FILE}/dispatches`)) { + throw new Error("lost response"); + } + if (apiPath.includes(`/workflows/${PRODUCER_WORKFLOW_FILE}/runs?`)) { + return { total_count: 1, workflow_runs: [completed] }; + } + return base(apiPath, token, requestOptions); + }); + try { + await expect( + startExactImageQualification( + { + request: REQUEST, + coreToken: "core-token", + producerToken: "producer-token", + workDir, + }, + dependencies(api as ReturnType), + ), + ).rejects.toMatchObject({ code: "DISPATCH_AMBIGUOUS" }); + expect( + api.mock.calls.some(([apiPath]) => String(apiPath).endsWith(`/${RUN_ID}/cancel`)), + ).toBe(false); + expect(readExactImageQualificationState(workDir).status).toBe("dispatched"); + } finally { + fs.rmSync(workDir, { recursive: true, force: true }); + } + }); + + it("retains the recovered run identity when cancellation fails", async () => { + const workDir = tempDirectory(); + const base = createApi(); + const api = vi.fn(async (apiPath: string, token: string, requestOptions?: unknown) => { + if (apiPath.endsWith(`/workflows/${PRODUCER_WORKFLOW_FILE}/dispatches`)) { + throw new Error("lost response"); + } + if (apiPath.includes(`/workflows/${PRODUCER_WORKFLOW_FILE}/runs?`)) { + return { total_count: 1, workflow_runs: [workflowRun()] }; + } + if (apiPath === `repos/${PRODUCER_REPOSITORY}/actions/runs/${RUN_ID}/cancel`) { + throw new Error("cancel transport failed"); + } + return base(apiPath, token, requestOptions); + }); + try { + await expect( + startExactImageQualification( + { + request: REQUEST, + coreToken: "core-token", + producerToken: "producer-token", + workDir, + }, + dependencies(api as ReturnType), + ), + ).rejects.toThrow(/cancel transport failed/u); + expect(readExactImageQualificationState(workDir).producer.runId).toBe(RUN_ID); + expect( + JSON.parse(fs.readFileSync(path.join(workDir, DISPATCH_RECONCILIATION_FILE), "utf8")), + ).toMatchObject({ outcome: "recovered-one", runIds: [RUN_ID] }); + } finally { + fs.rmSync(workDir, { recursive: true, force: true }); + } + }); + + it("ignores near-match runs and retains a zero-match reconciliation audit", async () => { + const workDir = tempDirectory(); + const base = createApi(); + let clock = BASE_TIME; + const api = vi.fn(async (apiPath: string, token: string, requestOptions?: unknown) => { + if (apiPath.endsWith(`/workflows/${PRODUCER_WORKFLOW_FILE}/dispatches`)) { + throw new Error("lost response"); + } + if (apiPath.includes(`/workflows/${PRODUCER_WORKFLOW_FILE}/runs?`)) { + const nearMatches = [ + workflowRun({ display_title: "wrong correlation" }), + workflowRun({ workflow_id: WORKFLOW_ID + 1 }), + workflowRun({ run_attempt: 2 }), + workflowRun({ event: "push" }), + workflowRun({ head_branch: "feature" }), + workflowRun({ path: ".github/workflows/other.yml" }), + workflowRun({ repository: { full_name: "other/repository" } }), + workflowRun({ head_repository: { full_name: "other/repository" } }), + workflowRun({ created_at: new Date(BASE_TIME - 2 * 60_000).toISOString() }), + workflowRun({ url: `${API_RUN_URL}/wrong` }), + ]; + return { + total_count: nearMatches.length, + workflow_runs: nearMatches, + }; + } + return base(apiPath, token, requestOptions); + }); + try { + await expect( + startExactImageQualification( + { + request: REQUEST, + coreToken: "core-token", + producerToken: "producer-token", + workDir, + }, + dependencies(api as ReturnType, { + now: () => clock, + sleep: async () => { + clock += 2; + }, + limits: { + dispatchReconciliationTimeoutMs: 1, + reconciliationPollIntervalMs: 1, + }, + }), + ), + ).rejects.toMatchObject({ code: "DISPATCH_AMBIGUOUS" }); + expect( + api.mock.calls.some(([apiPath]) => String(apiPath).endsWith(`/${RUN_ID}/cancel`)), + ).toBe(false); + expect( + JSON.parse(fs.readFileSync(path.join(workDir, DISPATCH_RECONCILIATION_FILE), "utf8")), + ).toMatchObject({ outcome: "none", runIds: [] }); + } finally { + fs.rmSync(workDir, { recursive: true, force: true }); + } + }); + + it("lets always-run cleanup recover and cancel a run that appeared after start reconciliation", async () => { + const workDir = tempDirectory(); + let startClock = BASE_TIME; + try { + await expect( + startExactImageQualification( + { + request: REQUEST, + coreToken: "core-token", + producerToken: "producer-token", + workDir, + }, + dependencies(createApi({ dispatch: null }), { + now: () => startClock, + sleep: async () => { + startClock += 2; + }, + limits: { + dispatchReconciliationTimeoutMs: 1, + reconciliationPollIntervalMs: 1, + }, + }), + ), + ).rejects.toMatchObject({ code: "DISPATCH_AMBIGUOUS" }); + expect(fs.existsSync(path.join(workDir, STATE_FILE))).toBe(false); + fs.writeFileSync(path.join(workDir, STATE_FILE), "", { mode: 0o600 }); + + const base = createApi(); + let cancelled = false; + const cleanupApi = vi.fn(async (apiPath: string, token: string, requestOptions?: unknown) => { + if (apiPath.includes(`/workflows/${PRODUCER_WORKFLOW_FILE}/runs?`)) { + return { total_count: 1, workflow_runs: [workflowRun()] }; + } + if (apiPath === `repos/${PRODUCER_REPOSITORY}/actions/runs/${RUN_ID}/cancel`) { + cancelled = true; + return undefined; + } + if (apiPath === `repos/${PRODUCER_REPOSITORY}/actions/runs/${RUN_ID}` && cancelled) { + return workflowRun({ status: "completed", conclusion: "cancelled" }); + } + return base(apiPath, token, requestOptions); + }); + await expect( + cancelActiveExactImageQualification( + { workDir, producerToken: "producer-token" }, + dependencies(cleanupApi as ReturnType, { + now: () => BASE_TIME + 10_000, + limits: { cleanupTimeoutMs: 10_000 }, + }), + ), + ).resolves.toBe(true); + expect(cancelled).toBe(true); + expect(readExactImageQualificationState(workDir).producer.runId).toBe(RUN_ID); + expect( + fs.readdirSync(workDir).some((name) => name.startsWith("controller-state.corrupt-")), + ).toBe(true); + } finally { + fs.rmSync(workDir, { recursive: true, force: true }); + } + }); + + it("fails closed on multiple strict reconciliation matches and records every exact ID", async () => { + const workDir = tempDirectory(); + const secondId = "24681"; + const base = createApi(); + const second = workflowRun({ + id: Number(secondId), + url: `https://api.github.com/repos/${PRODUCER_REPOSITORY}/actions/runs/${secondId}`, + html_url: `https://github.com/${PRODUCER_REPOSITORY}/actions/runs/${secondId}`, + }); + const api = vi.fn(async (apiPath: string, token: string, requestOptions?: unknown) => { + if (apiPath.endsWith(`/workflows/${PRODUCER_WORKFLOW_FILE}/dispatches`)) { + throw new Error("lost response"); + } + if (apiPath.includes(`/workflows/${PRODUCER_WORKFLOW_FILE}/runs?`)) { + return { total_count: 2, workflow_runs: [workflowRun(), second] }; + } + if (apiPath.endsWith("/cancel")) return undefined; + return base(apiPath, token, requestOptions); + }); + try { + await expect( + startExactImageQualification( + { + request: REQUEST, + coreToken: "core-token", + producerToken: "producer-token", + workDir, + }, + dependencies(api as ReturnType), + ), + ).rejects.toMatchObject({ code: "DISPATCH_AMBIGUOUS" }); + const audit = JSON.parse( + fs.readFileSync(path.join(workDir, DISPATCH_RECONCILIATION_FILE), "utf8"), + ); + expect(audit).toMatchObject({ outcome: "multiple", runIds: [RUN_ID, secondId] }); + expect( + api.mock.calls.filter(([apiPath]) => String(apiPath).endsWith("/cancel")), + ).toHaveLength(2); + } finally { + fs.rmSync(workDir, { recursive: true, force: true }); + } + }); + + it("aborts a GitHub REST call at the configured per-request cap", async () => { + let observedSignal: AbortSignal | undefined; + const api = vi.fn( + async (_apiPath: string, _token: string, requestOptions?: { signal?: AbortSignal }) => + new Promise((_resolve, reject) => { + observedSignal = requestOptions?.signal; + requestOptions?.signal?.addEventListener("abort", () => reject(new Error("aborted")), { + once: true, + }); + }), + ); + await expect( + preflightExactImageQualification(REQUEST, "core-token", { + api: api as QualificationDependencies["api"], + now: Date.now, + limits: { apiRequestTimeoutMs: 5 }, + }), + ).rejects.toMatchObject({ code: "QUALIFICATION_TIMEOUT" }); + expect(observedSignal?.aborted).toBe(true); + }); + + it("validates the exact returned URLs", () => { + expect(validateWorkflowDispatchDetails(dispatchDetails())).toEqual({ + workflowRunId: RUN_ID, + runUrl: API_RUN_URL, + htmlUrl: HTML_RUN_URL, + }); + expect(() => + validateWorkflowDispatchDetails({ + ...dispatchDetails(), + html_url: `${HTML_RUN_URL}/attempts/1`, + }), + ).toThrow(/html_url/u); + }); + + it("rejects producer workflow identity drift", () => { + expect(() => + validateQualificationWorkflowRun(workflowRun({ head_sha: "c".repeat(40) }), { + candidateSha: CANDIDATE_SHA, + correlationId: CORRELATION_ID, + producerSha: PRODUCER_SHA, + runId: RUN_ID, + runUrl: API_RUN_URL, + htmlUrl: HTML_RUN_URL, + }), + ).toThrow(/head SHA/u); + }); +}); + +describe("bound producer polling", () => { + it("accepts success only from the exact dispatched run", async () => { + const { workDir } = await startedState(); + const api = createApi({ run: workflowRun({ status: "completed", conclusion: "success" }) }); + try { + await expect( + waitForExactImageQualification( + { workDir, producerToken: "producer-token" }, + dependencies(api, { now: () => BASE_TIME + 1_000 }), + ), + ).resolves.toMatchObject({ id: RUN_ID, conclusion: "success" }); + expect(readExactImageQualificationState(workDir).status).toBe("completed"); + } finally { + fs.rmSync(workDir, { recursive: true, force: true }); + } + }); + + it("cancels a run that remains queued beyond the queue budget", async () => { + const { workDir } = await startedState(); + const api = createApi({ run: workflowRun({ status: "queued" }) }); + try { + await expect( + waitForExactImageQualification( + { workDir, producerToken: "producer-token" }, + dependencies(api, { + now: () => BASE_TIME + 10 * 60_000 + 1, + limits: { queueTimeoutMs: 10 * 60_000 }, + }), + ), + ).rejects.toMatchObject({ code: "RUN_QUEUE_TIMEOUT" }); + expect( + api.mock.calls.some( + ([apiPath]) => apiPath === `repos/${PRODUCER_REPOSITORY}/actions/runs/${RUN_ID}/cancel`, + ), + ).toBe(true); + } finally { + fs.rmSync(workDir, { recursive: true, force: true }); + } + }); + + it("rejects a successful producer completion observed at the shared deadline", async () => { + const { workDir } = await startedState(); + const api = createApi({ run: workflowRun({ status: "completed", conclusion: "success" }) }); + try { + await expect( + waitForExactImageQualification( + { workDir, producerToken: "producer-token" }, + dependencies(api, { now: () => BASE_TIME + 45 * 60_000 }), + ), + ).rejects.toMatchObject({ code: "QUALIFICATION_TIMEOUT" }); + expect(readExactImageQualificationState(workDir).status).toBe("dispatched"); + } finally { + fs.rmSync(workDir, { recursive: true, force: true }); + } + }); +}); + +describe("qualification artifact integrity", () => { + it("requires one non-expired digest-bound artifact from the exact run and SHA", async () => { + const archive = Buffer.from("archive"); + const { state, workDir } = await startedState(); + try { + expect(validateQualificationArtifactList(artifactList(archive), state)).toMatchObject({ + id: "86420", + digest: `sha256:${createHash("sha256").update(archive).digest("hex")}`, + }); + expect(() => + validateQualificationArtifactList( + artifactList(archive, { workflow_run: { id: Number(RUN_ID), head_sha: "c".repeat(40) } }), + state, + ), + ).toThrow(/head SHA/u); + expect(() => + validateQualificationArtifactList(artifactList(archive, { digest: null }), state), + ).toThrow(/digest/u); + } finally { + fs.rmSync(workDir, { recursive: true, force: true }); + } + }); + + it("checks the archive digest before accepting the single root manifest entry", async () => { + const archive = Buffer.from("deterministic archive bytes"); + const manifest = Buffer.from('{"schemaVersion":1}\n'); + const started = await startedState(); + const completedApi = createApi({ + run: workflowRun({ status: "completed", conclusion: "success" }), + }); + await waitForExactImageQualification( + { workDir: started.workDir, producerToken: "producer-token" }, + dependencies(completedApi, { now: () => BASE_TIME + 1_000 }), + ); + const artifactApi = createApi({ artifacts: artifactList(archive) }); + const runCommand = vi.fn((command: string, args: readonly string[]) => ({ + status: 0, + stdout: + args[0] === "-Z1" + ? Buffer.from(`${MANIFEST_ARTIFACT_FILE}\n`) + : args[0] === "-Zl" + ? Buffer.from(`-rw-r--r-- 3.0 unx 20 tx 20 stor ${MANIFEST_ARTIFACT_FILE}\n`) + : manifest, + stderr: Buffer.alloc(0), + })); + const fetchMock = vi.fn( + async (_input: string | URL | Request, _init?: RequestInit) => + new Response(archive, { status: 200 }), + ); + try { + await downloadExactImageManifest( + { workDir: started.workDir, producerToken: "producer-token" }, + dependencies(artifactApi, { fetch: fetchMock, runCommand }), + ); + const fetchHeaders = new Headers(fetchMock.mock.calls[0]?.[1]?.headers); + expect(fetchMock.mock.calls[0]?.[0]).toBe( + `https://api.github.com/repos/${PRODUCER_REPOSITORY}/actions/artifacts/86420/zip`, + ); + expect(fetchHeaders.get("X-GitHub-Api-Version")).toBe(GITHUB_API_VERSION); + expect(fs.readFileSync(path.join(started.workDir, ARCHIVE_FILE))).toEqual(archive); + expect(fs.readFileSync(path.join(started.workDir, MANIFEST_ARTIFACT_FILE))).toEqual(manifest); + expect(readExactImageQualificationState(started.workDir)).toMatchObject({ + status: "downloaded", + artifact: { + archiveSha256: createHash("sha256").update(archive).digest("hex"), + manifestSha256: createHash("sha256").update(manifest).digest("hex"), + }, + }); + } finally { + fs.rmSync(started.workDir, { recursive: true, force: true }); + } + }); + + it("does not inspect or extract an archive whose digest mismatches GitHub metadata", async () => { + const archive = Buffer.from("archive bytes"); + const started = await startedState(); + await waitForExactImageQualification( + { workDir: started.workDir, producerToken: "producer-token" }, + dependencies( + createApi({ run: workflowRun({ status: "completed", conclusion: "success" }) }), + { now: () => BASE_TIME + 1_000 }, + ), + ); + const runCommand = vi.fn(); + try { + await expect( + downloadExactImageManifest( + { workDir: started.workDir, producerToken: "producer-token" }, + dependencies( + createApi({ artifacts: artifactList(archive, { digest: `sha256:${"0".repeat(64)}` }) }), + { + fetch: async () => new Response(archive, { status: 200 }), + runCommand, + }, + ), + ), + ).rejects.toMatchObject({ code: "ARTIFACT_MISSING_OR_INVALID" }); + expect(runCommand).not.toHaveBeenCalled(); + } finally { + fs.rmSync(started.workDir, { recursive: true, force: true }); + } + }); + + it("caps artifact propagation at the shared qualification deadline", async () => { + const started = await startedState(); + await waitForExactImageQualification( + { workDir: started.workDir, producerToken: "producer-token" }, + dependencies( + createApi({ run: workflowRun({ status: "completed", conclusion: "success" }) }), + { now: () => BASE_TIME + 43 * 60_000 }, + ), + ); + let clock = BASE_TIME + 44 * 60_000; + const api = createApi({ artifacts: { total_count: 0, artifacts: [] } }); + try { + await expect( + downloadExactImageManifest( + { workDir: started.workDir, producerToken: "producer-token" }, + dependencies(api, { + now: () => clock, + sleep: async () => { + clock = BASE_TIME + 45 * 60_000; + }, + }), + ), + ).rejects.toMatchObject({ code: "QUALIFICATION_TIMEOUT" }); + expect( + api.mock.calls.filter(([apiPath]) => String(apiPath).includes("/artifacts?")), + ).toHaveLength(1); + } finally { + fs.rmSync(started.workDir, { recursive: true, force: true }); + } + }); + + it("does not accept an archive whose extraction crosses the shared deadline", async () => { + const archive = Buffer.from("archive"); + const manifest = Buffer.from('{"schemaVersion":1}\n'); + const started = await startedState(); + await waitForExactImageQualification( + { workDir: started.workDir, producerToken: "producer-token" }, + dependencies( + createApi({ run: workflowRun({ status: "completed", conclusion: "success" }) }), + { now: () => BASE_TIME + 43 * 60_000 }, + ), + ); + let clock = BASE_TIME + 44 * 60_000; + try { + await expect( + downloadExactImageManifest( + { workDir: started.workDir, producerToken: "producer-token" }, + dependencies(createApi({ artifacts: artifactList(archive) }), { + now: () => clock, + fetch: async () => new Response(archive, { status: 200 }), + runCommand: (_command, args) => { + if (args[0] === "-Z1") { + return { + status: 0, + stdout: Buffer.from(`${MANIFEST_ARTIFACT_FILE}\n`), + stderr: Buffer.alloc(0), + }; + } + if (args[0] === "-Zl") { + return { + status: 0, + stdout: Buffer.from( + `-rw-r--r-- 3.0 unx 20 tx 20 stor ${MANIFEST_ARTIFACT_FILE}\n`, + ), + stderr: Buffer.alloc(0), + }; + } + clock = BASE_TIME + 45 * 60_000; + return { status: 0, stdout: manifest, stderr: Buffer.alloc(0) }; + }, + }), + ), + ).rejects.toMatchObject({ code: "QUALIFICATION_TIMEOUT" }); + expect(readExactImageQualificationState(started.workDir).status).toBe("completed"); + } finally { + fs.rmSync(started.workDir, { recursive: true, force: true }); + } + }); + + it("rejects duplicate or nested ZIP entry inventories", () => { + const tempDir = tempDirectory(); + const archivePath = path.join(tempDir, ARCHIVE_FILE); + fs.writeFileSync(archivePath, "not inspected by the command seam"); + try { + expect(() => + extractExactManifestArchive( + archivePath, + path.join(tempDir, MANIFEST_ARTIFACT_FILE), + () => ({ + status: 0, + stdout: Buffer.from(`${MANIFEST_ARTIFACT_FILE}\n${MANIFEST_ARTIFACT_FILE}\n`), + stderr: Buffer.alloc(0), + }), + ), + ).toThrow(/exactly/u); + expect(() => + extractExactManifestArchive( + archivePath, + path.join(tempDir, MANIFEST_ARTIFACT_FILE), + () => ({ + status: 0, + stdout: Buffer.from(`nested/${MANIFEST_ARTIFACT_FILE}\n`), + stderr: Buffer.alloc(0), + }), + ), + ).toThrow(/exactly/u); + } finally { + fs.rmSync(tempDir, { recursive: true, force: true }); + } + }); + + it("rejects a ZIP entry marked as a symbolic link before extraction", () => { + const tempDir = tempDirectory(); + const archivePath = path.join(tempDir, ARCHIVE_FILE); + fs.writeFileSync(archivePath, "not inspected by the command seam"); + const runCommand = vi.fn((_command: string, args: readonly string[]) => ({ + status: 0, + stdout: + args[0] === "-Z1" + ? Buffer.from(`${MANIFEST_ARTIFACT_FILE}\n`) + : Buffer.from(`lrwxrwxrwx 3.0 unx 20 tx 20 stor ${MANIFEST_ARTIFACT_FILE}\n`), + stderr: Buffer.alloc(0), + })); + try { + expect(() => + extractExactManifestArchive( + archivePath, + path.join(tempDir, MANIFEST_ARTIFACT_FILE), + runCommand, + ), + ).toThrow(/regular file/u); + expect(runCommand).toHaveBeenCalledTimes(2); + } finally { + fs.rmSync(tempDir, { recursive: true, force: true }); + } + }); +}); + +describe("qualification evidence finalization", () => { + it("records immutable producer, artifact, and accepted-manifest hashes", async () => { + const archive = Buffer.from("archive"); + const manifest = Buffer.from('{"schemaVersion":1}\n'); + const normalized = Buffer.from('{"schemaVersion":1,"accepted":true}\n'); + const started = await startedState(); + await waitForExactImageQualification( + { workDir: started.workDir, producerToken: "producer-token" }, + dependencies( + createApi({ run: workflowRun({ status: "completed", conclusion: "success" }) }), + { now: () => BASE_TIME + 1_000 }, + ), + ); + await downloadExactImageManifest( + { workDir: started.workDir, producerToken: "producer-token" }, + dependencies(createApi({ artifacts: artifactList(archive) }), { + fetch: async () => new Response(archive, { status: 200 }), + runCommand: (_command, args) => ({ + status: 0, + stdout: + args[0] === "-Z1" + ? Buffer.from(`${MANIFEST_ARTIFACT_FILE}\n`) + : args[0] === "-Zl" + ? Buffer.from(`-rw-r--r-- 3.0 unx 20 tx 20 stor ${MANIFEST_ARTIFACT_FILE}\n`) + : manifest, + stderr: Buffer.alloc(0), + }), + }), + ); + fs.writeFileSync(path.join(started.workDir, VALIDATED_MANIFEST_FILE), normalized, { + mode: 0o600, + }); + try { + const finalState = finalizeExactImageQualification(started.workDir, { + now: () => BASE_TIME + 2_000, + }); + expect(finalState).toMatchObject({ + status: "validated", + validation: { + manifestSha256: createHash("sha256").update(manifest).digest("hex"), + normalizedManifestSha256: createHash("sha256").update(normalized).digest("hex"), + }, + }); + const evidence = JSON.parse( + fs.readFileSync(path.join(started.workDir, EVIDENCE_FILE), "utf8"), + ); + expect(evidence).toMatchObject({ + qualificationStatus: "accepted", + producer: { runId: RUN_ID, repositorySha: PRODUCER_SHA }, + artifact: { id: "86420" }, + }); + } finally { + fs.rmSync(started.workDir, { recursive: true, force: true }); + } + }); + + it("refuses accepted evidence at the shared deadline", () => { + const workDir = tempDirectory(); + const manifest = Buffer.from('{"schemaVersion":1}\n'); + const normalized = Buffer.from('{"schemaVersion":1,"accepted":true}\n'); + const manifestSha256 = createHash("sha256").update(manifest).digest("hex"); + const state = { + schemaVersion: 1, + status: "downloaded", + dispatchedAt: new Date(BASE_TIME).toISOString(), + request: { + actor: REQUEST.actor, + candidateSha: CANDIDATE_SHA, + correlationId: CORRELATION_ID, + reason: REQUEST.reason, + requesterRunAttempt: 1, + requesterRunId: REQUEST.requesterRunId, + workflowSha: CANDIDATE_SHA, + }, + producer: { + repository: PRODUCER_REPOSITORY, + repositorySha: PRODUCER_SHA, + ref: "main", + workflowPath: PRODUCER_WORKFLOW_PATH, + runId: RUN_ID, + runAttempt: 1, + workflowId: String(WORKFLOW_ID), + runUrl: API_RUN_URL, + htmlUrl: HTML_RUN_URL, + }, + artifact: { + id: "86420", + name: `nemoclaw-image-handoff-v1-${RUN_ID}-1`, + digest: `sha256:${"0".repeat(64)}`, + sizeInBytes: 7, + apiUrl: `https://api.github.com/repos/${PRODUCER_REPOSITORY}/actions/artifacts/86420`, + archiveDownloadUrl: `https://api.github.com/repos/${PRODUCER_REPOSITORY}/actions/artifacts/86420/zip`, + archiveSha256: "0".repeat(64), + manifestSha256, + }, + }; + fs.writeFileSync(path.join(workDir, STATE_FILE), `${JSON.stringify(state)}\n`, { mode: 0o600 }); + fs.writeFileSync(path.join(workDir, MANIFEST_ARTIFACT_FILE), manifest, { mode: 0o600 }); + fs.writeFileSync(path.join(workDir, VALIDATED_MANIFEST_FILE), normalized, { mode: 0o600 }); + try { + expect(() => + finalizeExactImageQualification(workDir, { + now: () => BASE_TIME + 45 * 60_000, + }), + ).toThrowError(expect.objectContaining({ code: "QUALIFICATION_TIMEOUT" })); + expect(fs.existsSync(path.join(workDir, EVIDENCE_FILE))).toBe(false); + } finally { + fs.rmSync(workDir, { recursive: true, force: true }); + } + }); + + it("loads under the workflow's dependency-free Node strip-types runtime", () => { + const script = path.resolve("tools/e2e/exact-image-qualification-controller.mts"); + const result = spawnSync( + process.execPath, + [ + "--experimental-strip-types", + "--no-warnings", + script, + "--mode", + "cancel", + "--work-dir", + "/does/not/exist", + ], + { + encoding: "utf8", + env: { ...process.env, NEMOCLAW_IMAGE_QUALIFICATION_TOKEN: "test-token" }, + timeout: 10_000, + }, + ); + expect(result.status, result.stderr).toBe(0); + expect(result.stdout).toContain("No active producer run"); + }); +}); diff --git a/test/exact-image-qualification-workflow.test.ts b/test/exact-image-qualification-workflow.test.ts new file mode 100644 index 0000000000..0d78a07817 --- /dev/null +++ b/test/exact-image-qualification-workflow.test.ts @@ -0,0 +1,122 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { expect, it } from "vitest"; + +import { + readRepoText, + readYaml, + type Workflow, + type WorkflowJob, +} from "./helpers/e2e-workflow-contract"; + +const WORKFLOW_PATH = ".github/workflows/brev-launchable-qualification.yaml"; +const CONTROLLER_PATH = "tools/e2e/exact-image-qualification-controller.mts"; + +type DraftQualificationWorkflow = Workflow & { + name: string; + on: Record; + permissions: Record; + concurrency: { group: string; "cancel-in-progress": boolean }; +}; + +function strings(value: unknown): string[] { + return typeof value === "string" + ? [value] + : Array.isArray(value) + ? value.flatMap(strings) + : value && typeof value === "object" + ? Object.values(value).flatMap(strings) + : []; +} + +function job(workflow: DraftQualificationWorkflow, name: string): WorkflowJob { + const value = workflow.jobs[name]; + expect(value, `missing ${name} job`).toBeDefined(); + return value!; +} + +// source-shape-contract: security -- The draft cross-repository image handoff must remain manual, protected, fixed-target, least-privilege, and stop before any Brev deployment +it("keeps draft exact-image qualification manual, protected, and evidence-only", () => { + const workflow = readYaml(WORKFLOW_PATH); + const source = readRepoText(WORKFLOW_PATH); + const controller = readRepoText(CONTROLLER_PATH); + const preflight = job(workflow, "preflight"); + const qualify = job(workflow, "qualify"); + const workflowStrings = strings(workflow); + + expect(workflow.name).toBe("Draft / Brev Launchable Qualification"); + expect(Object.keys(workflow.on)).toEqual(["workflow_dispatch"]); + expect(source).not.toMatch(/^\s+(?:push|schedule|workflow_run|pull_request):/mu); + expect(Object.keys((workflow.on.workflow_dispatch as { inputs: object }).inputs)).toEqual([ + "candidate_sha", + "reason", + ]); + expect(workflow.permissions).toEqual({}); + expect(workflow.concurrency).toEqual({ + group: "draft-brev-launchable-qualification-${{ inputs.candidate_sha }}", + "cancel-in-progress": false, + }); + + expect(preflight.permissions).toEqual({ contents: "read" }); + expect(preflight.environment).toBeUndefined(); + expect(JSON.stringify(preflight)).not.toContain("NEMOCLAW_IMAGE_QUALIFICATION_TOKEN"); + expect(qualify.permissions).toEqual({ contents: "read" }); + expect(qualify.environment).toEqual({ + name: "approve-brev-launchable-qualification", + deployment: false, + }); + expect(JSON.stringify(qualify)).toContain("NEMOCLAW_IMAGE_QUALIFICATION_TOKEN"); + expect(source.match(/secrets\.NEMOCLAW_IMAGE_QUALIFICATION_TOKEN/gu)).toHaveLength(4); + expect(workflowStrings).not.toContain("id-token: write"); + expect(source).not.toMatch(/npm (?:ci|install)/u); + for (const step of [...(preflight.steps ?? []), ...(qualify.steps ?? [])]) { + expect(step.run ?? "").not.toContain("${{ inputs."); + } + + const actionUses = workflowStrings.filter((value) => value.includes("actions/")); + expect(actionUses.length).toBeGreaterThan(0); + for (const use of actionUses) expect(use).toMatch(/^actions\/[a-z-]+@[0-9a-f]{40}$/u); + for (const checkout of qualify.steps?.filter((step) => + step.uses?.startsWith("actions/checkout@"), + ) ?? []) { + expect(checkout.with).toMatchObject({ + ref: "${{ github.workflow_sha }}", + "persist-credentials": false, + }); + } + + expect(controller).toContain('export const PRODUCER_REPOSITORY = "brevdev/nemoclaw-image"'); + expect(controller).toContain( + 'export const PRODUCER_WORKFLOW_FILE = "build-qualification-image.yml"', + ); + expect(controller).toContain('export const PRODUCER_REF = "main"'); + expect(controller).toContain('export const GITHUB_API_VERSION = "2026-03-10"'); + expect(controller).toContain("return_run_details: true"); + expect(controller).toContain("fs.renameSync(temporary, file)"); + expect(controller).not.toMatch(/actions\/runs\?/u); + expect(controller).toContain("actions/workflows/${PRODUCER_WORKFLOW_FILE}/runs"); + + const validate = qualify.steps?.find((step) => step.name === "Validate the exact image manifest"); + expect(validate?.run).toContain("tools/e2e/validate-exact-image-manifest.mts"); + for (const flag of [ + "--nemoclaw-sha", + "--requester-run-id", + "--requester-run-attempt", + "--correlation-id", + "--image-repository-sha", + "--producer-run-id", + "--producer-run-attempt", + ]) { + expect(validate?.run).toContain(flag); + } + + expect(source).toContain("retention-days: 90"); + expect(source).toContain("dispatch-intent.v1.json"); + expect(source).toContain("dispatch-reconciliation.v1.json"); + expect(source).toContain("controller-state.corrupt-*.json"); + expect(source).toContain("--mode finalize"); + expect(source).not.toMatch(/\b(?:brev|gcloud)\s+(?:launch|deploy|set|create|update)\b/iu); + expect(source).not.toContain("launchable_id"); + expect(source).not.toContain("image_family"); +}); diff --git a/test/helpers/vitest-watch-triggers.ts b/test/helpers/vitest-watch-triggers.ts index b4b7fea2db..2c6ba99bf8 100644 --- a/test/helpers/vitest-watch-triggers.ts +++ b/test/helpers/vitest-watch-triggers.ts @@ -91,6 +91,10 @@ export const vitestWatchTriggerPatterns: VitestWatchTriggerPattern[] = [ pattern: /(?:^|\/)\.github\/workflows\/pr-e2e-gate\.yaml$/, testsToRun: runTests("test/pr-e2e-gate-workflow.test.ts", "test/pr-e2e-required.test.ts"), }, + { + pattern: /(?:^|\/)\.github\/workflows\/brev-launchable-qualification\.yaml$/, + testsToRun: runTests("test/exact-image-qualification-workflow.test.ts"), + }, { pattern: /(?:^|\/)(?:\.github\/workflows\/platform-vitest-main\.yaml|ci\/platform-vitest-macos-requirements\.lock)$/, diff --git a/test/vitest-watch-triggers.test.ts b/test/vitest-watch-triggers.test.ts index cc2c0c8557..fac5b90a5c 100644 --- a/test/vitest-watch-triggers.test.ts +++ b/test/vitest-watch-triggers.test.ts @@ -56,6 +56,7 @@ const OPAQUE_INPUTS = [ "test/e2e/docs/parity-inventory.generated.json", ".github/workflows/e2e.yaml", ".github/workflows/pr-e2e-gate.yaml", + ".github/workflows/brev-launchable-qualification.yaml", ".github/workflows/platform-vitest-main.yaml", "ci/platform-vitest-macos-requirements.lock", ] as const; @@ -106,6 +107,9 @@ describe("Vitest opaque-input watch triggers", () => { "test/pr-e2e-gate-workflow.test.ts", "test/pr-e2e-required.test.ts", ]); + expect(triggeredBy(".github/workflows/brev-launchable-qualification.yaml")).toEqual([ + "test/exact-image-qualification-workflow.test.ts", + ]); expect(triggeredBy(".github/workflows/platform-vitest-main.yaml")).toEqual([ "test/platform-vitest-main-workflow.test.ts", ]); diff --git a/tools/advisors/github.mts b/tools/advisors/github.mts index c9a7f58a50..1cbdcb4d04 100644 --- a/tools/advisors/github.mts +++ b/tools/advisors/github.mts @@ -10,6 +10,8 @@ export type GitHubComment = { export type GitHubRequestOptions = { method?: string; body?: unknown; + apiVersion?: string; + expectedStatus?: number; userAgent?: string; signal?: AbortSignal; }; @@ -91,14 +93,17 @@ export async function githubApi( Accept: "application/vnd.github+json", Authorization: `Bearer ${token}`, "Content-Type": "application/json", - "X-GitHub-Api-Version": "2022-11-28", + "X-GitHub-Api-Version": options.apiVersion ?? "2022-11-28", ...(options.userAgent ? { "User-Agent": options.userAgent } : {}), }, body: options.body === undefined ? undefined : JSON.stringify(options.body), signal: options.signal, }); const text = await response.text(); - if (!response.ok) { + if ( + !response.ok || + (options.expectedStatus !== undefined && response.status !== options.expectedStatus) + ) { throw new Error(`GitHub API ${apiPath} failed: ${response.status} ${text}`); } return (text ? JSON.parse(text) : undefined) as T; diff --git a/tools/e2e/exact-image-manifest.mts b/tools/e2e/exact-image-manifest.mts new file mode 100644 index 0000000000..db4a5f5214 --- /dev/null +++ b/tools/e2e/exact-image-manifest.mts @@ -0,0 +1,383 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export const EXACT_IMAGE_MANIFEST_KIND = "nemoclaw-exact-image-manifest"; +export const EXACT_IMAGE_REQUESTER_REPOSITORY = "NVIDIA/NemoClaw"; +export const EXACT_IMAGE_REPOSITORY = "brevdev/nemoclaw-image"; +export const EXACT_IMAGE_PRODUCER_WORKFLOW = ".github/workflows/build-qualification-image.yml"; +// This mutable family is publication evidence only. Consumers accept the +// immutable name, numeric ID, and self-link below as the image identity. +export const EXACT_IMAGE_STAGING_FAMILY = "nemoclaw-brev-staging-cpu"; +export const EXACT_IMAGE_MANIFEST_MAX_CLOCK_SKEW_MS = 5 * 60_000; +export const EXACT_IMAGE_MANIFEST_MAX_REUSED_AGE_MS = 24 * 60 * 60_000; + +export type ExactImageManifestFailureCode = + | "REQUEST_INVALID" + | "ARTIFACT_MISSING_OR_INVALID" + | "UNSUPPORTED_VARIANT" + | "PROVENANCE_MISMATCH" + | "IMAGE_IDENTITY_MISMATCH" + | "OUTPUT_WRITE_FAILED" + | "UNKNOWN"; + +export class ExactImageManifestError extends Error { + readonly code: ExactImageManifestFailureCode; + + constructor(code: ExactImageManifestFailureCode, message: string) { + super(message); + this.name = "ExactImageManifestError"; + this.code = code; + } +} + +export type ExactImageManifest = { + schemaVersion: 1; + kind: typeof EXACT_IMAGE_MANIFEST_KIND; + correlationId: string; + requesterRepository: typeof EXACT_IMAGE_REQUESTER_REPOSITORY; + requesterWorkflowRunId: string; + requesterWorkflowRunAttempt: number; + nemoclawSha: string; + imageRepository: typeof EXACT_IMAGE_REPOSITORY; + imageRepositorySha: string; + producerWorkflow: typeof EXACT_IMAGE_PRODUCER_WORKFLOW; + workflowRunId: string; + workflowRunAttempt: number; + imageOriginWorkflowRunId: string; + imageOriginWorkflowRunAttempt: number; + imageKind: "compute#image"; + project: string; + imageName: string; + imageId: string; + imageSelfLink: string; + status: "READY"; + imageCreationTimestamp: string; + manifestCreatedAt: string; + channel: "staging"; + variant: "cpu"; + observedFamily: typeof EXACT_IMAGE_STAGING_FAMILY; + result: "built" | "reused"; +}; + +export type ExactImageManifestExpectations = { + correlationId: string; + requesterWorkflowRunId: string; + requesterWorkflowRunAttempt: number; + nemoclawSha: string; + imageRepositorySha: string; + workflowRunId: string; + workflowRunAttempt: number; +}; + +const REQUIRED_FIELDS = [ + "schemaVersion", + "kind", + "correlationId", + "requesterRepository", + "requesterWorkflowRunId", + "requesterWorkflowRunAttempt", + "nemoclawSha", + "imageRepository", + "imageRepositorySha", + "producerWorkflow", + "workflowRunId", + "workflowRunAttempt", + "imageOriginWorkflowRunId", + "imageOriginWorkflowRunAttempt", + "imageKind", + "project", + "imageName", + "imageId", + "imageSelfLink", + "status", + "imageCreationTimestamp", + "manifestCreatedAt", + "channel", + "variant", + "observedFamily", + "result", +] as const; + +const REQUIRED_FIELD_SET = new Set(REQUIRED_FIELDS); +const FULL_SHA_PATTERN = /^[0-9a-f]{40}$/u; +const UUID_V4_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/u; +const DECIMAL_ID_PATTERN = /^[1-9][0-9]*$/u; +const IMAGE_NAME_PATTERN = /^[a-z](?:[-a-z0-9]{0,61}[a-z0-9])?$/u; +const IMAGE_SELF_LINK_PATTERN = + /^https:\/\/www[.]googleapis[.]com\/compute\/v1\/projects\/[^/]+\/global\/images\/[^/]+$/u; +const RFC3339_PATTERN = + /^(\d{4})-(\d{2})-(\d{2})T([01]\d|2[0-3]):([0-5]\d):([0-5]\d)(?:[.](\d{1,9}))?(Z|[+-](?:[01]\d|2[0-3]):[0-5]\d)$/u; + +function fail(code: ExactImageManifestFailureCode, message: string): never { + throw new ExactImageManifestError(code, message); +} + +function requireRecord(value: unknown): Record { + if (value === null || typeof value !== "object" || Array.isArray(value)) { + fail("ARTIFACT_MISSING_OR_INVALID", "manifest must be a JSON object"); + } + return value as Record; +} + +function validateExactFields(record: Record): void { + for (const field of REQUIRED_FIELDS) { + if (!Object.hasOwn(record, field)) { + fail("ARTIFACT_MISSING_OR_INVALID", `manifest is missing required field ${field}`); + } + } + const unexpected = Object.keys(record) + .filter((field) => !REQUIRED_FIELD_SET.has(field)) + .sort(); + if (unexpected.length > 0) { + fail("ARTIFACT_MISSING_OR_INVALID", `manifest contains unexpected field ${unexpected[0]}`); + } +} + +function requireString(record: Record, field: string): string { + const value = record[field]; + if (typeof value !== "string") { + fail("ARTIFACT_MISSING_OR_INVALID", `${field} must be a string`); + } + return value; +} + +function requirePositiveInteger(record: Record, field: string): number { + const value = record[field]; + if (!Number.isSafeInteger(value) || (value as number) < 1) { + fail("ARTIFACT_MISSING_OR_INVALID", `${field} must be a positive safe integer`); + } + return value as number; +} + +function requirePattern(value: string, field: string, pattern: RegExp): void { + if (!pattern.test(value)) { + fail("ARTIFACT_MISSING_OR_INVALID", `${field} has an invalid format`); + } +} + +function requireIdentityPattern(value: string, field: string, pattern: RegExp): void { + if (!pattern.test(value)) { + fail("IMAGE_IDENTITY_MISMATCH", `${field} has an invalid format`); + } +} + +function requireConstant( + actual: unknown, + expected: T, + field: string, +): asserts actual is T { + if (actual !== expected) { + fail("PROVENANCE_MISMATCH", `${field} must equal ${JSON.stringify(expected)}`); + } +} + +function daysInMonth(year: number, month: number): number { + if (month === 2) { + const leap = year % 4 === 0 && (year % 100 !== 0 || year % 400 === 0); + return leap ? 29 : 28; + } + return [4, 6, 9, 11].includes(month) ? 30 : 31; +} + +function parseRfc3339(value: string, field: string): number { + const match = RFC3339_PATTERN.exec(value); + if (!match) { + fail("ARTIFACT_MISSING_OR_INVALID", `${field} must be an RFC 3339 date-time`); + } + const year = Number(match[1]); + const month = Number(match[2]); + const day = Number(match[3]); + if (month < 1 || month > 12 || day < 1 || day > daysInMonth(year, month)) { + fail("ARTIFACT_MISSING_OR_INVALID", `${field} must be a real calendar date`); + } + const timestamp = Date.parse(value); + if (!Number.isFinite(timestamp)) { + fail("ARTIFACT_MISSING_OR_INVALID", `${field} must be an RFC 3339 date-time`); + } + return timestamp; +} + +function validateExpectations(expected: ExactImageManifestExpectations): void { + const stringPatterns: Array<[string, string, RegExp]> = [ + [expected.correlationId, "expected correlationId", UUID_V4_PATTERN], + [expected.requesterWorkflowRunId, "expected requesterWorkflowRunId", DECIMAL_ID_PATTERN], + [expected.nemoclawSha, "expected nemoclawSha", FULL_SHA_PATTERN], + [expected.imageRepositorySha, "expected imageRepositorySha", FULL_SHA_PATTERN], + [expected.workflowRunId, "expected workflowRunId", DECIMAL_ID_PATTERN], + ]; + for (const [value, field, pattern] of stringPatterns) { + if (typeof value !== "string" || !pattern.test(value)) { + fail("REQUEST_INVALID", `${field} has an invalid format`); + } + } + for (const [value, field] of [ + [expected.requesterWorkflowRunAttempt, "expected requesterWorkflowRunAttempt"], + [expected.workflowRunAttempt, "expected workflowRunAttempt"], + ] as const) { + if (!Number.isSafeInteger(value) || value < 1) { + fail("REQUEST_INVALID", `${field} must be a positive safe integer`); + } + } +} + +function assertExpected( + manifest: ExactImageManifest, + expected: ExactImageManifestExpectations, +): void { + const comparisons: Array<[unknown, unknown, string]> = [ + [manifest.correlationId, expected.correlationId, "correlationId"], + [manifest.requesterWorkflowRunId, expected.requesterWorkflowRunId, "requesterWorkflowRunId"], + [ + manifest.requesterWorkflowRunAttempt, + expected.requesterWorkflowRunAttempt, + "requesterWorkflowRunAttempt", + ], + [manifest.nemoclawSha, expected.nemoclawSha, "nemoclawSha"], + [manifest.imageRepositorySha, expected.imageRepositorySha, "imageRepositorySha"], + [manifest.workflowRunId, expected.workflowRunId, "workflowRunId"], + [manifest.workflowRunAttempt, expected.workflowRunAttempt, "workflowRunAttempt"], + ]; + for (const [actual, wanted, field] of comparisons) { + if (actual !== wanted) { + fail("PROVENANCE_MISMATCH", `${field} does not match the trusted request`); + } + } +} + +function validateTemporalContract(manifest: ExactImageManifest): void { + const imageCreated = parseRfc3339(manifest.imageCreationTimestamp, "imageCreationTimestamp"); + const manifestCreated = parseRfc3339(manifest.manifestCreatedAt, "manifestCreatedAt"); + if (imageCreated > manifestCreated + EXACT_IMAGE_MANIFEST_MAX_CLOCK_SKEW_MS) { + fail( + "PROVENANCE_MISMATCH", + "imageCreationTimestamp is later than manifestCreatedAt beyond allowed clock skew", + ); + } + if ( + manifest.result === "reused" && + manifestCreated - imageCreated > EXACT_IMAGE_MANIFEST_MAX_REUSED_AGE_MS + ) { + fail("PROVENANCE_MISMATCH", "reused image is older than 24 hours"); + } +} + +function buildManifest(record: Record): ExactImageManifest { + requireConstant(record.schemaVersion, 1, "schemaVersion"); + requireConstant(record.kind, EXACT_IMAGE_MANIFEST_KIND, "kind"); + requireConstant( + record.requesterRepository, + EXACT_IMAGE_REQUESTER_REPOSITORY, + "requesterRepository", + ); + requireConstant(record.imageRepository, EXACT_IMAGE_REPOSITORY, "imageRepository"); + requireConstant(record.producerWorkflow, EXACT_IMAGE_PRODUCER_WORKFLOW, "producerWorkflow"); + requireConstant(record.imageKind, "compute#image", "imageKind"); + requireConstant(record.status, "READY", "status"); + requireConstant(record.channel, "staging", "channel"); + + const variant = requireString(record, "variant"); + if (variant !== "cpu") { + fail("UNSUPPORTED_VARIANT", 'variant must equal "cpu"'); + } + requireConstant(record.observedFamily, EXACT_IMAGE_STAGING_FAMILY, "observedFamily"); + + const result = requireString(record, "result"); + if (result !== "built" && result !== "reused") { + fail("ARTIFACT_MISSING_OR_INVALID", 'result must equal "built" or "reused"'); + } + + const manifest: ExactImageManifest = { + schemaVersion: 1, + kind: EXACT_IMAGE_MANIFEST_KIND, + correlationId: requireString(record, "correlationId"), + requesterRepository: EXACT_IMAGE_REQUESTER_REPOSITORY, + requesterWorkflowRunId: requireString(record, "requesterWorkflowRunId"), + requesterWorkflowRunAttempt: requirePositiveInteger(record, "requesterWorkflowRunAttempt"), + nemoclawSha: requireString(record, "nemoclawSha"), + imageRepository: EXACT_IMAGE_REPOSITORY, + imageRepositorySha: requireString(record, "imageRepositorySha"), + producerWorkflow: EXACT_IMAGE_PRODUCER_WORKFLOW, + workflowRunId: requireString(record, "workflowRunId"), + workflowRunAttempt: requirePositiveInteger(record, "workflowRunAttempt"), + imageOriginWorkflowRunId: requireString(record, "imageOriginWorkflowRunId"), + imageOriginWorkflowRunAttempt: requirePositiveInteger(record, "imageOriginWorkflowRunAttempt"), + imageKind: "compute#image", + project: requireString(record, "project"), + imageName: requireString(record, "imageName"), + imageId: requireString(record, "imageId"), + imageSelfLink: requireString(record, "imageSelfLink"), + status: "READY", + imageCreationTimestamp: requireString(record, "imageCreationTimestamp"), + manifestCreatedAt: requireString(record, "manifestCreatedAt"), + channel: "staging", + variant, + observedFamily: EXACT_IMAGE_STAGING_FAMILY, + result, + }; + + requirePattern(manifest.correlationId, "correlationId", UUID_V4_PATTERN); + requirePattern(manifest.requesterWorkflowRunId, "requesterWorkflowRunId", DECIMAL_ID_PATTERN); + requirePattern(manifest.nemoclawSha, "nemoclawSha", FULL_SHA_PATTERN); + requirePattern(manifest.imageRepositorySha, "imageRepositorySha", FULL_SHA_PATTERN); + requirePattern(manifest.workflowRunId, "workflowRunId", DECIMAL_ID_PATTERN); + requirePattern(manifest.imageOriginWorkflowRunId, "imageOriginWorkflowRunId", DECIMAL_ID_PATTERN); + requirePattern(manifest.imageName, "imageName", IMAGE_NAME_PATTERN); + requirePattern(manifest.imageId, "imageId", DECIMAL_ID_PATTERN); + requireIdentityPattern(manifest.imageSelfLink, "imageSelfLink", IMAGE_SELF_LINK_PATTERN); + if (manifest.project.length === 0) { + fail("ARTIFACT_MISSING_OR_INVALID", "project must not be empty"); + } + return manifest; +} + +export function parseExactImageManifestJson(contents: string): unknown { + try { + return JSON.parse(contents) as unknown; + } catch { + fail("ARTIFACT_MISSING_OR_INVALID", "manifest is not valid JSON"); + } +} + +export function validateExactImageManifest( + value: unknown, + expected: ExactImageManifestExpectations, +): ExactImageManifest { + validateExpectations(expected); + const record = requireRecord(value); + validateExactFields(record); + const manifest = buildManifest(record); + + const expectedSelfLink = `https://www.googleapis.com/compute/v1/projects/${manifest.project}/global/images/${manifest.imageName}`; + if (manifest.imageSelfLink !== expectedSelfLink) { + fail("IMAGE_IDENTITY_MISMATCH", "imageSelfLink does not exactly match project and imageName"); + } + if ( + manifest.result === "built" && + (manifest.imageOriginWorkflowRunId !== manifest.workflowRunId || + manifest.imageOriginWorkflowRunAttempt !== manifest.workflowRunAttempt) + ) { + fail( + "PROVENANCE_MISMATCH", + "built image origin run and attempt must match the current producer run", + ); + } + + validateTemporalContract(manifest); + assertExpected(manifest, expected); + return manifest; +} + +export function parseAndValidateExactImageManifest( + contents: string, + expected: ExactImageManifestExpectations, +): ExactImageManifest { + return validateExactImageManifest(parseExactImageManifestJson(contents), expected); +} + +export function normalizedExactImageManifestJson(manifest: ExactImageManifest): string { + return `${JSON.stringify(manifest, null, 2)}\n`; +} + +export function exactImageManifestFailureCode(error: unknown): ExactImageManifestFailureCode { + return error instanceof ExactImageManifestError ? error.code : "UNKNOWN"; +} diff --git a/tools/e2e/exact-image-qualification-controller.mts b/tools/e2e/exact-image-qualification-controller.mts new file mode 100755 index 0000000000..da75f2be66 --- /dev/null +++ b/tools/e2e/exact-image-qualification-controller.mts @@ -0,0 +1,1672 @@ +#!/usr/bin/env node + +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import { createHash, randomUUID } from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; + +import { type GitHubRequestOptions, githubApi } from "../advisors/github.mts"; +import * as privateFile from "./private-file.ts"; + +// Root .ts files are exposed as CommonJS under tsx and as ESM under Node's +// strip-types runtime. Normalize both forms so the workflow uses the same +// no-follow state-file helpers in either test or production execution. +const privateFileRuntime = ( + "default" in privateFile && privateFile.default ? privateFile.default : privateFile +) as typeof import("./private-file.ts"); + +export const REQUESTER_REPOSITORY = "NVIDIA/NemoClaw"; +export const PRODUCER_REPOSITORY = "brevdev/nemoclaw-image"; +export const PRODUCER_WORKFLOW_FILE = "build-qualification-image.yml"; +export const PRODUCER_WORKFLOW_PATH = `.github/workflows/${PRODUCER_WORKFLOW_FILE}`; +export const PRODUCER_REF = "main"; +export const GITHUB_API_VERSION = "2026-03-10"; +export const MANIFEST_ARTIFACT_FILE = "nemoclaw-image-manifest.v1.json"; +export const ARCHIVE_FILE = "nemoclaw-image-handoff.zip"; +export const VALIDATED_MANIFEST_FILE = "validated-manifest.v1.json"; +export const EVIDENCE_FILE = "qualification-evidence.v1.json"; +export const STATE_FILE = "controller-state.json"; +export const DISPATCH_INTENT_FILE = "dispatch-intent.v1.json"; +export const DISPATCH_RECONCILIATION_FILE = "dispatch-reconciliation.v1.json"; + +const FULL_SHA_PATTERN = /^[0-9a-f]{40}$/u; +const UUID_V4_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/u; +const DECIMAL_ID_PATTERN = /^[1-9][0-9]*$/u; +const ARTIFACT_DIGEST_PATTERN = /^sha256:[0-9a-f]{64}$/u; +const GITHUB_LOGIN_PATTERN = /^(?!-)[A-Za-z0-9-]{1,39}(?( + apiPath: string, + token: string, + options?: GitHubRequestOptions, +) => Promise; + +type CommandResult = { + status: number | null; + signal?: NodeJS.Signals | null; + stdout: Buffer | string; + stderr: Buffer | string; + error?: Error; +}; + +type CommandRunner = (command: string, args: readonly string[]) => CommandResult; + +export type QualificationDependencies = { + api?: GitHubApiClient; + fetch?: typeof fetch; + now?: () => number; + randomUuid?: () => string; + runCommand?: CommandRunner; + sleep?: (milliseconds: number) => Promise; + limits?: Partial<{ + artifactPropagationTimeoutMs: number; + apiRequestTimeoutMs: number; + cancelReserveMs: number; + cleanupTimeoutMs: number; + dispatchReconciliationTimeoutMs: number; + downloadTimeoutMs: number; + pollIntervalMs: number; + qualificationTimeoutMs: number; + queueTimeoutMs: number; + reconciliationPollIntervalMs: number; + warningAfterMs: number; + }>; +}; + +export type ExactImageQualificationRequest = { + actor: string; + candidateSha: string; + eventName: string; + reason: string; + ref: string; + requesterRunAttempt: number; + requesterRunId: string; + workflowSha: string; +}; + +export type DispatchDetails = { + workflowRunId: string; + runUrl: string; + htmlUrl: string; +}; + +export type ExactImageDispatchIntent = { + schemaVersion: 1; + kind: "nemoclaw-exact-image-dispatch-intent"; + requestStartedAt: string; + request: { + actor: string; + candidateSha: string; + correlationId: string; + reason: string; + requesterRunAttempt: number; + requesterRunId: string; + workflowSha: string; + }; + producer: { + repository: typeof PRODUCER_REPOSITORY; + repositorySha: string; + ref: typeof PRODUCER_REF; + workflowId: string; + workflowPath: typeof PRODUCER_WORKFLOW_PATH; + }; +}; + +export type QualificationWorkflowRun = { + id: string; + workflowId: string; + headSha: string; + runAttempt: number; + status: string; + conclusion: string | null; + url: string; + htmlUrl: string; +}; + +export type QualificationArtifact = { + id: string; + name: string; + digest: string; + sizeInBytes: number; + apiUrl: string; + archiveDownloadUrl: string; +}; + +export type ExactImageQualificationState = { + schemaVersion: 1; + status: "dispatched" | "completed" | "downloaded" | "validated"; + dispatchedAt: string; + request: { + actor: string; + candidateSha: string; + correlationId: string; + reason: string; + requesterRunAttempt: number; + requesterRunId: string; + workflowSha: string; + }; + producer: { + repository: typeof PRODUCER_REPOSITORY; + repositorySha: string; + ref: typeof PRODUCER_REF; + workflowPath: typeof PRODUCER_WORKFLOW_PATH; + runId: string; + runAttempt: 1; + workflowId?: string; + runUrl: string; + htmlUrl: string; + completedAt?: string; + }; + artifact?: QualificationArtifact & { + archiveSha256: string; + manifestSha256: string; + }; + validation?: { + acceptedAt: string; + manifestSha256: string; + normalizedManifestSha256: string; + }; +}; + +type ControllerCommand = + | { mode: "preflight"; request: ExactImageQualificationRequest } + | { + mode: "start"; + request: ExactImageQualificationRequest; + workDir: string; + } + | { mode: "wait" | "download" | "finalize" | "cancel"; workDir: string }; + +function fail(code: ExactImageQualificationFailureCode, message: string): never { + throw new ExactImageQualificationError(code, message); +} + +function record(value: unknown, label: string): Record { + if (value === null || typeof value !== "object" || Array.isArray(value)) { + fail("PROVENANCE_MISMATCH", `${label} must be an object`); + } + return value as Record; +} + +function safePositiveId(value: unknown, label: string): string { + if (!Number.isSafeInteger(value) || (value as number) < 1) { + fail("PROVENANCE_MISMATCH", `${label} must be a positive safe integer`); + } + return String(value); +} + +function positiveInteger(value: string, label: string): number { + if (!DECIMAL_ID_PATTERN.test(value)) fail("REQUEST_INVALID", `${label} must be positive`); + const parsed = Number(value); + if (!Number.isSafeInteger(parsed)) fail("REQUEST_INVALID", `${label} is outside the safe range`); + return parsed; +} + +function requireExactString(value: unknown, expected: string, label: string): void { + if (value !== expected) fail("PROVENANCE_MISMATCH", `${label} did not match the trusted request`); +} + +function sha256(bytes: Uint8Array): string { + return createHash("sha256").update(bytes).digest("hex"); +} + +function timestamp(now: () => number): string { + return new Date(now()).toISOString(); +} + +function sleep(milliseconds: number): Promise { + return new Promise((resolve) => setTimeout(resolve, milliseconds)); +} + +function limits(dependencies: QualificationDependencies) { + return { + artifactPropagationTimeoutMs: + dependencies.limits?.artifactPropagationTimeoutMs ?? ARTIFACT_PROPAGATION_TIMEOUT_MS, + apiRequestTimeoutMs: dependencies.limits?.apiRequestTimeoutMs ?? API_REQUEST_TIMEOUT_MS, + cancelReserveMs: dependencies.limits?.cancelReserveMs ?? CANCEL_RESERVE_MS, + cleanupTimeoutMs: dependencies.limits?.cleanupTimeoutMs ?? CLEANUP_TIMEOUT_MS, + dispatchReconciliationTimeoutMs: + dependencies.limits?.dispatchReconciliationTimeoutMs ?? DISPATCH_RECONCILIATION_TIMEOUT_MS, + downloadTimeoutMs: dependencies.limits?.downloadTimeoutMs ?? DOWNLOAD_TIMEOUT_MS, + pollIntervalMs: dependencies.limits?.pollIntervalMs ?? POLL_INTERVAL_MS, + qualificationTimeoutMs: dependencies.limits?.qualificationTimeoutMs ?? QUALIFICATION_TIMEOUT_MS, + queueTimeoutMs: dependencies.limits?.queueTimeoutMs ?? QUEUE_TIMEOUT_MS, + reconciliationPollIntervalMs: + dependencies.limits?.reconciliationPollIntervalMs ?? RECONCILIATION_POLL_INTERVAL_MS, + warningAfterMs: dependencies.limits?.warningAfterMs ?? WARNING_AFTER_MS, + }; +} + +function qualificationDeadline( + state: ExactImageQualificationState, + qualificationTimeoutMs: number, +): number { + const dispatchedAt = Date.parse(state.dispatchedAt); + if (!Number.isFinite(dispatchedAt)) { + fail("PROVENANCE_MISMATCH", "controller state has an invalid dispatch timestamp"); + } + return dispatchedAt + qualificationTimeoutMs; +} + +function requireQualificationTimeRemaining(deadline: number, now: () => number): number { + const remaining = deadline - now(); + if (remaining <= 0) { + fail("QUALIFICATION_TIMEOUT", "qualification exceeded the shared 45-minute budget"); + } + return remaining; +} + +function boundedApiClient( + api: GitHubApiClient, + dependencies: QualificationDependencies, + deadline?: number, +): GitHubApiClient { + const now = dependencies.now ?? Date.now; + const requestCap = limits(dependencies).apiRequestTimeoutMs; + return async (apiPath: string, token: string, options: GitHubRequestOptions = {}) => { + const remaining = deadline === undefined ? requestCap : deadline - now(); + if (remaining <= 0) { + fail("QUALIFICATION_TIMEOUT", "no time remains for the bounded GitHub API request"); + } + const controller = new AbortController(); + const timeout = Math.min(requestCap, remaining); + const timer = setTimeout(() => controller.abort(), timeout); + timer.unref?.(); + const signal = options.signal + ? AbortSignal.any([options.signal, controller.signal]) + : controller.signal; + try { + return await api(apiPath, token, { ...options, signal }); + } catch (error) { + if (controller.signal.aborted) { + fail("QUALIFICATION_TIMEOUT", `GitHub API request exceeded its ${timeout}ms budget`); + } + throw error; + } finally { + clearTimeout(timer); + } + }; +} + +function acceptanceDeadline( + state: ExactImageQualificationState, + dependencies: QualificationDependencies, +): number { + const configured = limits(dependencies); + return ( + qualificationDeadline(state, configured.qualificationTimeoutMs) - configured.cancelReserveMs + ); +} + +export function validateExactImageQualificationRequest( + request: ExactImageQualificationRequest, +): ExactImageQualificationRequest { + if (request.eventName !== "workflow_dispatch") { + fail("REQUEST_INVALID", "qualification must be started by workflow_dispatch"); + } + if (request.ref !== "refs/heads/main") { + fail("REQUEST_INVALID", "qualification must run from refs/heads/main"); + } + if (request.requesterRunAttempt !== 1) { + fail("REQUEST_INVALID", "qualification does not accept GitHub UI reruns"); + } + if (!FULL_SHA_PATTERN.test(request.candidateSha)) { + fail("REQUEST_INVALID", "candidate SHA must be a lowercase full commit SHA"); + } + if (!FULL_SHA_PATTERN.test(request.workflowSha)) { + fail("REQUEST_INVALID", "workflow SHA must be a lowercase full commit SHA"); + } + if (request.candidateSha !== request.workflowSha) { + fail("REQUEST_INVALID", "candidate SHA must equal the trusted workflow SHA"); + } + if (!DECIMAL_ID_PATTERN.test(request.requesterRunId)) { + fail("REQUEST_INVALID", "requester run ID must be a positive decimal string"); + } + if (!GITHUB_LOGIN_PATTERN.test(request.actor)) { + fail("REQUEST_INVALID", "triggering actor has an invalid GitHub login"); + } + if ( + request.reason.length === 0 || + request.reason !== request.reason.trim() || + Buffer.byteLength(request.reason, "utf8") > MAX_REASON_BYTES || + /[\u0000-\u001f\u007f]/u.test(request.reason) + ) { + fail("REQUEST_INVALID", "reason must be trimmed, nonempty, bounded text without controls"); + } + return request; +} + +function validateMainRef(value: unknown, repository: string): string { + const payload = record(value, `${repository} main ref`); + requireExactString(payload.ref, "refs/heads/main", `${repository} ref`); + const object = record(payload.object, `${repository} ref object`); + requireExactString(object.type, "commit", `${repository} ref object type`); + if (typeof object.sha !== "string" || !FULL_SHA_PATTERN.test(object.sha)) { + fail("PROVENANCE_MISMATCH", `${repository} main did not resolve to a full commit SHA`); + } + return object.sha; +} + +function validateProducerWorkflow(value: unknown): string { + const workflow = record(value, "producer workflow"); + const workflowId = safePositiveId(workflow.id, "producer workflow ID"); + requireExactString(workflow.path, PRODUCER_WORKFLOW_PATH, "producer workflow path"); + requireExactString(workflow.state, "active", "producer workflow state"); + return workflowId; +} + +async function authorizeRequest( + request: ExactImageQualificationRequest, + token: string, + api: GitHubApiClient, +): Promise { + validateExactImageQualificationRequest(request); + const main = await api(`repos/${REQUESTER_REPOSITORY}/git/ref/heads/main`, token); + if (validateMainRef(main, REQUESTER_REPOSITORY) !== request.candidateSha) { + fail("DISPATCH_FORBIDDEN", "candidate SHA is no longer the current NemoClaw main commit"); + } + const permission = record( + await api( + `repos/${REQUESTER_REPOSITORY}/collaborators/${encodeURIComponent(request.actor)}/permission`, + token, + ), + "collaborator permission", + ); + if (permission.permission !== "admin" && permission.role_name !== "maintain") { + fail("DISPATCH_FORBIDDEN", "triggering actor must have maintain or admin permission"); + } +} + +export async function preflightExactImageQualification( + request: ExactImageQualificationRequest, + token: string, + dependencies: QualificationDependencies = {}, +): Promise { + await authorizeRequest( + request, + token, + boundedApiClient(dependencies.api ?? githubApi, dependencies), + ); +} + +export function validateWorkflowDispatchDetails(value: unknown): DispatchDetails { + const payload = record(value, "workflow dispatch response"); + const workflowRunId = safePositiveId(payload.workflow_run_id, "workflow_run_id"); + const runUrl = `https://api.github.com/repos/${PRODUCER_REPOSITORY}/actions/runs/${workflowRunId}`; + const htmlUrl = `https://github.com/${PRODUCER_REPOSITORY}/actions/runs/${workflowRunId}`; + requireExactString(payload.run_url, runUrl, "dispatch run_url"); + requireExactString(payload.html_url, htmlUrl, "dispatch html_url"); + return { workflowRunId, runUrl, htmlUrl }; +} + +export function validateQualificationWorkflowRun( + value: unknown, + expected: { + candidateSha: string; + correlationId: string; + producerSha: string; + runId: string; + runUrl: string; + htmlUrl: string; + workflowId?: string; + }, +): QualificationWorkflowRun { + const run = record(value, "producer workflow run"); + const id = safePositiveId(run.id, "producer run ID"); + if (id !== expected.runId) fail("PROVENANCE_MISMATCH", "producer run ID changed"); + const workflowId = safePositiveId(run.workflow_id, "producer workflow ID"); + if (expected.workflowId !== undefined && workflowId !== expected.workflowId) { + fail("PROVENANCE_MISMATCH", "producer workflow ID did not match the dispatch intent"); + } + if (run.run_attempt !== 1) fail("PROVENANCE_MISMATCH", "producer run attempt must equal 1"); + requireExactString(run.event, "workflow_dispatch", "producer event"); + requireExactString(run.head_branch, PRODUCER_REF, "producer head branch"); + requireExactString(run.head_sha, expected.producerSha, "producer head SHA"); + requireExactString(run.path, PRODUCER_WORKFLOW_PATH, "producer workflow path"); + requireExactString( + run.display_title, + `Qualify NemoClaw ${expected.candidateSha} (${expected.correlationId})`, + "producer display title", + ); + requireExactString(run.url, expected.runUrl, "producer run URL"); + requireExactString(run.html_url, expected.htmlUrl, "producer HTML URL"); + requireExactString( + record(run.repository, "producer repository").full_name, + PRODUCER_REPOSITORY, + "producer repository", + ); + requireExactString( + record(run.head_repository, "producer head repository").full_name, + PRODUCER_REPOSITORY, + "producer head repository", + ); + const allowedStatuses = new Set([ + "queued", + "in_progress", + "completed", + "waiting", + "requested", + "pending", + ]); + if (typeof run.status !== "string" || !allowedStatuses.has(run.status)) { + fail("PROVENANCE_MISMATCH", "producer run has an unsupported status"); + } + if (run.conclusion !== null && typeof run.conclusion !== "string") { + fail("PROVENANCE_MISMATCH", "producer conclusion must be a string or null"); + } + return { + id, + workflowId, + headSha: expected.producerSha, + runAttempt: 1, + status: run.status, + conclusion: run.conclusion as string | null, + url: expected.runUrl, + htmlUrl: expected.htmlUrl, + }; +} + +function validateCancellationWorkflowRun( + value: unknown, + runId: string, +): { status: string; conclusion: string | null } { + const run = record(value, "producer cancellation workflow run"); + if (safePositiveId(run.id, "producer cancellation run ID") !== runId) { + fail("PROVENANCE_MISMATCH", "producer cancellation run ID changed"); + } + const details = canonicalDispatchDetails(runId); + requireExactString(run.url, details.runUrl, "producer cancellation run URL"); + requireExactString(run.html_url, details.htmlUrl, "producer cancellation HTML URL"); + const allowedStatuses = new Set([ + "queued", + "in_progress", + "completed", + "waiting", + "requested", + "pending", + ]); + if (typeof run.status !== "string" || !allowedStatuses.has(run.status)) { + fail("PROVENANCE_MISMATCH", "producer cancellation run has an unsupported status"); + } + if (run.conclusion !== null && typeof run.conclusion !== "string") { + fail("PROVENANCE_MISMATCH", "producer cancellation conclusion must be a string or null"); + } + return { status: run.status, conclusion: run.conclusion as string | null }; +} + +function statePath(workDir: string): string { + return path.join(workDir, STATE_FILE); +} + +function intentPath(workDir: string): string { + return path.join(workDir, DISPATCH_INTENT_FILE); +} + +function writeDispatchIntent(workDir: string, intent: ExactImageDispatchIntent): void { + try { + privateFileRuntime.writePrivateRegularFile( + intentPath(workDir), + `${JSON.stringify(intent, null, 2)}\n`, + ); + } catch { + fail("OUTPUT_WRITE_FAILED", "dispatch intent could not be written safely"); + } +} + +function readDispatchIntent(workDir: string): ExactImageDispatchIntent | null { + let contents: string | null; + try { + contents = privateFileRuntime.readPrivateRegularFile(intentPath(workDir), { + maxBytes: MAX_STATE_BYTES, + }); + } catch { + fail("PROVENANCE_MISMATCH", "dispatch intent could not be read safely"); + } + if (contents === null) return null; + try { + const intent = JSON.parse(contents) as ExactImageDispatchIntent; + if ( + intent.schemaVersion !== 1 || + intent.kind !== "nemoclaw-exact-image-dispatch-intent" || + !GITHUB_LOGIN_PATTERN.test(intent.request?.actor ?? "") || + !FULL_SHA_PATTERN.test(intent.request?.candidateSha ?? "") || + !UUID_V4_PATTERN.test(intent.request?.correlationId ?? "") || + !FULL_SHA_PATTERN.test(intent.request?.workflowSha ?? "") || + intent.request?.candidateSha !== intent.request?.workflowSha || + intent.request?.requesterRunAttempt !== 1 || + !DECIMAL_ID_PATTERN.test(intent.request?.requesterRunId ?? "") || + typeof intent.request?.reason !== "string" || + intent.request.reason.length === 0 || + intent.request.reason !== intent.request.reason.trim() || + Buffer.byteLength(intent.request.reason, "utf8") > MAX_REASON_BYTES || + /[\u0000-\u001f\u007f]/u.test(intent.request.reason) || + intent.producer?.repository !== PRODUCER_REPOSITORY || + !FULL_SHA_PATTERN.test(intent.producer?.repositorySha ?? "") || + intent.producer?.ref !== PRODUCER_REF || + intent.producer?.workflowPath !== PRODUCER_WORKFLOW_PATH || + !DECIMAL_ID_PATTERN.test(intent.producer?.workflowId ?? "") || + !Number.isFinite(Date.parse(intent.requestStartedAt)) + ) { + fail("PROVENANCE_MISMATCH", "dispatch intent is invalid"); + } + return intent; + } catch (error) { + if (error instanceof ExactImageQualificationError) throw error; + fail("PROVENANCE_MISMATCH", "dispatch intent is not valid JSON"); + } +} + +function writeDispatchReconciliation( + workDir: string, + intent: ExactImageDispatchIntent, + runs: readonly QualificationWorkflowRun[], + outcome: "none" | "recovered-one" | "multiple", + now: () => number, +): void { + const evidence = { + schemaVersion: 1, + kind: "nemoclaw-exact-image-dispatch-reconciliation", + recordedAt: timestamp(now), + outcome, + correlationId: intent.request.correlationId, + runIds: runs.map((run) => run.id), + producerHeadShas: Object.fromEntries(runs.map((run) => [run.id, run.headSha])), + }; + try { + privateFileRuntime.writePrivateRegularFile( + path.join(workDir, DISPATCH_RECONCILIATION_FILE), + `${JSON.stringify(evidence, null, 2)}\n`, + ); + } catch { + fail("OUTPUT_WRITE_FAILED", "dispatch reconciliation evidence could not be written safely"); + } +} + +function writeAtomicPrivateRegularFile(file: string, contents: string): void { + const temporary = `${file}.${process.pid}.${randomUUID()}.tmp`; + try { + privateFileRuntime.writePrivateRegularFile(temporary, contents); + if (fs.existsSync(file)) { + const current = fs.lstatSync(file); + if (!current.isFile() || current.isSymbolicLink() || current.nlink !== 1) { + fail("OUTPUT_WRITE_FAILED", "existing controller state is not one regular file"); + } + } + fs.renameSync(temporary, file); + } finally { + try { + fs.rmSync(temporary, { force: true }); + } catch { + // A successfully renamed temporary path no longer exists; a failed + // cleanup must not hide the original atomic-write error. + } + } +} + +function writeState(workDir: string, state: ExactImageQualificationState): void { + try { + writeAtomicPrivateRegularFile(statePath(workDir), `${JSON.stringify(state, null, 2)}\n`); + } catch { + fail("OUTPUT_WRITE_FAILED", "controller state could not be written safely"); + } +} + +function preserveUnreadableState(workDir: string): void { + const file = statePath(workDir); + if (!fs.existsSync(file)) return; + const preserved = path.join( + workDir, + `controller-state.corrupt-${Date.now()}-${randomUUID()}.json`, + ); + try { + fs.renameSync(file, preserved); + } catch (error) { + console.warn( + `Could not preserve unreadable controller state before intent recovery: ${error instanceof Error ? error.message : "unknown error"}`, + ); + } +} + +export function readExactImageQualificationState(workDir: string): ExactImageQualificationState { + let contents: string | null; + try { + contents = privateFileRuntime.readPrivateRegularFile(statePath(workDir), { + maxBytes: MAX_STATE_BYTES, + }); + } catch { + fail("PROVENANCE_MISMATCH", "controller state could not be read safely"); + } + if (contents === null) fail("PROVENANCE_MISMATCH", "controller state is missing"); + try { + return JSON.parse(contents) as ExactImageQualificationState; + } catch { + fail("PROVENANCE_MISMATCH", "controller state is not valid JSON"); + } +} + +function canonicalDispatchDetails(runId: string): DispatchDetails { + return { + workflowRunId: runId, + runUrl: `https://api.github.com/repos/${PRODUCER_REPOSITORY}/actions/runs/${runId}`, + htmlUrl: `https://github.com/${PRODUCER_REPOSITORY}/actions/runs/${runId}`, + }; +} + +function dispatchCreationWindow( + intent: ExactImageDispatchIntent, + configured: ReturnType, +): { earliest: number; latest: number } { + const startedAt = Date.parse(intent.requestStartedAt); + return { + earliest: startedAt - DISPATCH_CLOCK_SKEW_MS, + latest: startedAt + configured.apiRequestTimeoutMs + DISPATCH_CLOCK_SKEW_MS, + }; +} + +function reconciledWorkflowRuns( + value: unknown, + intent: ExactImageDispatchIntent, + configured: ReturnType, +): QualificationWorkflowRun[] { + const response = record(value, "workflow run reconciliation response"); + if (!Number.isSafeInteger(response.total_count) || (response.total_count as number) < 0) { + fail("DISPATCH_AMBIGUOUS", "workflow run reconciliation count is invalid"); + } + if (!Array.isArray(response.workflow_runs)) { + fail("DISPATCH_AMBIGUOUS", "workflow run reconciliation list is missing"); + } + if ((response.total_count as number) > response.workflow_runs.length) { + fail("DISPATCH_AMBIGUOUS", "workflow run reconciliation was truncated"); + } + const { earliest, latest } = dispatchCreationWindow(intent, configured); + const title = `Qualify NemoClaw ${intent.request.candidateSha} (${intent.request.correlationId})`; + const matches: QualificationWorkflowRun[] = []; + for (const candidate of response.workflow_runs) { + if (candidate === null || typeof candidate !== "object" || Array.isArray(candidate)) continue; + const run = candidate as Record; + const createdAt = typeof run.created_at === "string" ? Date.parse(run.created_at) : Number.NaN; + const id = Number.isSafeInteger(run.id) && (run.id as number) > 0 ? String(run.id) : ""; + const headSha = typeof run.head_sha === "string" ? run.head_sha : ""; + const details = id ? canonicalDispatchDetails(id) : null; + if ( + details === null || + run.workflow_id !== Number(intent.producer.workflowId) || + run.run_attempt !== 1 || + run.event !== "workflow_dispatch" || + run.head_branch !== PRODUCER_REF || + !FULL_SHA_PATTERN.test(headSha) || + run.path !== PRODUCER_WORKFLOW_PATH || + run.display_title !== title || + run.url !== details.runUrl || + run.html_url !== details.htmlUrl || + (run.repository as { full_name?: unknown } | undefined)?.full_name !== PRODUCER_REPOSITORY || + (run.head_repository as { full_name?: unknown } | undefined)?.full_name !== + PRODUCER_REPOSITORY || + !Number.isFinite(createdAt) || + createdAt < earliest || + createdAt > latest + ) { + continue; + } + matches.push( + validateQualificationWorkflowRun(candidate, { + candidateSha: intent.request.candidateSha, + correlationId: intent.request.correlationId, + producerSha: headSha, + workflowId: intent.producer.workflowId, + runId: id, + runUrl: details.runUrl, + htmlUrl: details.htmlUrl, + }), + ); + } + return matches; +} + +function stateFromIntent( + intent: ExactImageDispatchIntent, + run: QualificationWorkflowRun, +): ExactImageQualificationState { + return { + schemaVersion: 1, + status: "dispatched", + dispatchedAt: intent.requestStartedAt, + request: { + actor: intent.request.actor, + candidateSha: intent.request.candidateSha, + correlationId: intent.request.correlationId, + reason: intent.request.reason, + requesterRunAttempt: intent.request.requesterRunAttempt, + requesterRunId: intent.request.requesterRunId, + workflowSha: intent.request.workflowSha, + }, + producer: { + repository: PRODUCER_REPOSITORY, + repositorySha: run.headSha, + ref: PRODUCER_REF, + workflowPath: PRODUCER_WORKFLOW_PATH, + runId: run.id, + runAttempt: 1, + workflowId: run.workflowId, + runUrl: run.url, + htmlUrl: run.htmlUrl, + }, + }; +} + +async function cancelAndVerifyRecoveredRun( + state: ExactImageQualificationState, + initial: { status: string }, + token: string, + api: GitHubApiClient, + pause: (milliseconds: number) => Promise, + pollIntervalMs: number, + deadline: number, + now: () => number, +): Promise { + if (initial.status === "completed") return false; + let cancellationError: unknown; + try { + await cancelRun(state, token, api); + } catch (error) { + cancellationError = error; + } + for (;;) { + if (now() >= deadline) { + fail("QUALIFICATION_TIMEOUT", "recovered producer run cancellation was not verified in time"); + } + const run = validateCancellationWorkflowRun( + await api( + `repos/${PRODUCER_REPOSITORY}/actions/runs/${state.producer.runId}`, + token, + { apiVersion: GITHUB_API_VERSION, expectedStatus: 200 }, + ), + state.producer.runId, + ); + if (run.status === "completed") return true; + if (cancellationError !== undefined) throw cancellationError; + await pause(pollIntervalMs); + } +} + +async function reconcileAmbiguousDispatch( + options: { + intent: ExactImageDispatchIntent; + workDir: string; + producerToken: string; + mode: "start" | "cleanup"; + }, + dependencies: QualificationDependencies, +): Promise { + const now = dependencies.now ?? Date.now; + const pause = dependencies.sleep ?? sleep; + const configured = limits(dependencies); + const startedAt = now(); + const windowMs = + options.mode === "cleanup" + ? configured.cleanupTimeoutMs + : configured.dispatchReconciliationTimeoutMs; + const deadline = startedAt + windowMs; + const api = boundedApiClient(dependencies.api ?? githubApi, dependencies, deadline); + const creationWindow = dispatchCreationWindow(options.intent, configured); + const earliestQuery = new Date(Math.floor(creationWindow.earliest / 1_000) * 1_000).toISOString(); + const latestQuery = new Date(Math.ceil(creationWindow.latest / 1_000) * 1_000).toISOString(); + const created = encodeURIComponent( + `${earliestQuery.replace(".000Z", "Z")}..${latestQuery.replace(".000Z", "Z")}`, + ); + const listPath = + `repos/${PRODUCER_REPOSITORY}/actions/workflows/${PRODUCER_WORKFLOW_FILE}/runs` + + `?event=workflow_dispatch&branch=${PRODUCER_REF}&created=${created}&per_page=100`; + + for (;;) { + if (now() >= deadline) { + writeDispatchReconciliation(options.workDir, options.intent, [], "none", now); + fail( + "DISPATCH_AMBIGUOUS", + "no strict producer run match appeared before reconciliation timeout", + ); + } + const matches = reconciledWorkflowRuns( + await api(listPath, options.producerToken, { + apiVersion: GITHUB_API_VERSION, + expectedStatus: 200, + }), + options.intent, + configured, + ); + if (matches.length > 1) { + const runIds = matches.map((run) => run.id); + writeDispatchReconciliation(options.workDir, options.intent, matches, "multiple", now); + for (const run of matches) { + if (run.status !== "completed") { + try { + await api( + `repos/${PRODUCER_REPOSITORY}/actions/runs/${run.id}/cancel`, + options.producerToken, + { method: "POST", apiVersion: GITHUB_API_VERSION, expectedStatus: 202 }, + ); + } catch { + // The retained exact IDs are mandatory manual-cleanup evidence. + } + } + } + fail( + "DISPATCH_AMBIGUOUS", + `multiple strict dispatch matches require cleanup: ${runIds.join(",")}`, + ); + } + if (matches.length === 1) { + const recovered = matches[0]; + const state = stateFromIntent(options.intent, recovered); + writeState(options.workDir, state); + writeDispatchReconciliation( + options.workDir, + options.intent, + [recovered], + "recovered-one", + now, + ); + const cancelled = await cancelAndVerifyRecoveredRun( + state, + recovered, + options.producerToken, + api, + pause, + configured.reconciliationPollIntervalMs, + deadline, + now, + ); + if (options.mode === "cleanup") return cancelled; + fail( + "DISPATCH_AMBIGUOUS", + `recovered producer run ${recovered.id} was not accepted and was cleaned up`, + ); + } + await pause(configured.reconciliationPollIntervalMs); + } +} + +export async function startExactImageQualification( + options: { + request: ExactImageQualificationRequest; + coreToken: string; + producerToken: string; + workDir: string; + }, + dependencies: QualificationDependencies = {}, +): Promise { + const now = dependencies.now ?? Date.now; + const initialApi = boundedApiClient(dependencies.api ?? githubApi, dependencies); + await authorizeRequest(options.request, options.coreToken, initialApi); + + const producerRef = await initialApi( + `repos/${PRODUCER_REPOSITORY}/git/ref/heads/${PRODUCER_REF}`, + options.producerToken, + { apiVersion: GITHUB_API_VERSION, expectedStatus: 200 }, + ); + const producerSha = validateMainRef(producerRef, PRODUCER_REPOSITORY); + const workflowId = validateProducerWorkflow( + await initialApi( + `repos/${PRODUCER_REPOSITORY}/actions/workflows/${PRODUCER_WORKFLOW_FILE}`, + options.producerToken, + { apiVersion: GITHUB_API_VERSION, expectedStatus: 200 }, + ), + ); + const correlationId = (dependencies.randomUuid ?? randomUUID)(); + if (!UUID_V4_PATTERN.test(correlationId)) { + fail("REQUEST_INVALID", "generated correlation ID was not a lowercase UUIDv4"); + } + + const intent: ExactImageDispatchIntent = { + schemaVersion: 1, + kind: "nemoclaw-exact-image-dispatch-intent", + requestStartedAt: timestamp(now), + request: { + actor: options.request.actor, + candidateSha: options.request.candidateSha, + correlationId, + reason: options.request.reason, + requesterRunAttempt: options.request.requesterRunAttempt, + requesterRunId: options.request.requesterRunId, + workflowSha: options.request.workflowSha, + }, + producer: { + repository: PRODUCER_REPOSITORY, + repositorySha: producerSha, + ref: PRODUCER_REF, + workflowId, + workflowPath: PRODUCER_WORKFLOW_PATH, + }, + }; + writeDispatchIntent(options.workDir, intent); + const stateDeadline = + Date.parse(intent.requestStartedAt) + limits(dependencies).qualificationTimeoutMs; + const api = boundedApiClient(dependencies.api ?? githubApi, dependencies, stateDeadline); + + let dispatch: DispatchDetails; + try { + const dispatchResponse = await api( + `repos/${PRODUCER_REPOSITORY}/actions/workflows/${PRODUCER_WORKFLOW_FILE}/dispatches`, + options.producerToken, + { + method: "POST", + apiVersion: GITHUB_API_VERSION, + expectedStatus: 200, + body: { + ref: PRODUCER_REF, + inputs: { + nemoclaw_sha: options.request.candidateSha, + correlation_id: correlationId, + requester_workflow_run_id: options.request.requesterRunId, + requester_workflow_run_attempt: String(options.request.requesterRunAttempt), + }, + return_run_details: true, + }, + }, + ); + dispatch = validateWorkflowDispatchDetails(dispatchResponse); + } catch { + await reconcileAmbiguousDispatch( + { intent, workDir: options.workDir, producerToken: options.producerToken, mode: "start" }, + dependencies, + ); + fail("DISPATCH_AMBIGUOUS", "producer dispatch could not be bound safely"); + } + + const state: ExactImageQualificationState = { + schemaVersion: 1, + status: "dispatched", + dispatchedAt: intent.requestStartedAt, + request: { + actor: options.request.actor, + candidateSha: options.request.candidateSha, + correlationId, + reason: options.request.reason, + requesterRunAttempt: options.request.requesterRunAttempt, + requesterRunId: options.request.requesterRunId, + workflowSha: options.request.workflowSha, + }, + producer: { + repository: PRODUCER_REPOSITORY, + repositorySha: producerSha, + ref: PRODUCER_REF, + workflowPath: PRODUCER_WORKFLOW_PATH, + runId: dispatch.workflowRunId, + runAttempt: 1, + runUrl: dispatch.runUrl, + htmlUrl: dispatch.htmlUrl, + workflowId, + }, + }; + // Persist the returned ID before any further API call so the always-run + // cleanup step can cancel exactly this run if identity validation fails. + writeState(options.workDir, state); + + const run = validateQualificationWorkflowRun( + await api( + `repos/${PRODUCER_REPOSITORY}/actions/runs/${dispatch.workflowRunId}`, + options.producerToken, + { apiVersion: GITHUB_API_VERSION, expectedStatus: 200 }, + ), + { + candidateSha: options.request.candidateSha, + correlationId, + producerSha, + workflowId, + runId: dispatch.workflowRunId, + runUrl: dispatch.runUrl, + htmlUrl: dispatch.htmlUrl, + }, + ); + state.producer.workflowId = run.workflowId; + writeState(options.workDir, state); + return state; +} + +function expectedRun(state: ExactImageQualificationState) { + return { + candidateSha: state.request.candidateSha, + correlationId: state.request.correlationId, + producerSha: state.producer.repositorySha, + workflowId: state.producer.workflowId, + runId: state.producer.runId, + runUrl: state.producer.runUrl, + htmlUrl: state.producer.htmlUrl, + }; +} + +async function cancelRun( + state: ExactImageQualificationState, + token: string, + api: GitHubApiClient, +): Promise { + await api( + `repos/${PRODUCER_REPOSITORY}/actions/runs/${state.producer.runId}/cancel`, + token, + { method: "POST", apiVersion: GITHUB_API_VERSION, expectedStatus: 202 }, + ); +} + +export async function waitForExactImageQualification( + options: { workDir: string; producerToken: string }, + dependencies: QualificationDependencies = {}, +): Promise { + const now = dependencies.now ?? Date.now; + const pause = dependencies.sleep ?? sleep; + const configured = limits(dependencies); + const state = readExactImageQualificationState(options.workDir); + const startedAt = Date.parse(state.dispatchedAt); + const hardDeadline = qualificationDeadline(state, configured.qualificationTimeoutMs); + const deadline = acceptanceDeadline(state, dependencies); + const api = boundedApiClient(dependencies.api ?? githubApi, dependencies, deadline); + const cancelApi = boundedApiClient(dependencies.api ?? githubApi, dependencies, hardDeadline); + let warned = false; + + for (;;) { + if (now() >= deadline) { + const finalRun = validateQualificationWorkflowRun( + await cancelApi( + `repos/${PRODUCER_REPOSITORY}/actions/runs/${state.producer.runId}`, + options.producerToken, + { apiVersion: GITHUB_API_VERSION, expectedStatus: 200 }, + ), + expectedRun(state), + ); + if (finalRun.status !== "completed") { + await cancelRun(state, options.producerToken, cancelApi); + } + fail("QUALIFICATION_TIMEOUT", "qualification reached its cancellation reserve"); + } + const run = validateQualificationWorkflowRun( + await api( + `repos/${PRODUCER_REPOSITORY}/actions/runs/${state.producer.runId}`, + options.producerToken, + { apiVersion: GITHUB_API_VERSION, expectedStatus: 200 }, + ), + expectedRun(state), + ); + const observedAt = now(); + const elapsed = observedAt - startedAt; + if (observedAt >= deadline) { + if (run.status !== "completed") { + await cancelRun(state, options.producerToken, cancelApi); + } + fail("QUALIFICATION_TIMEOUT", "qualification reached its cancellation reserve"); + } + if (run.status === "completed") { + if (run.conclusion !== "success") { + fail("PRODUCER_RUN_FAILED", `producer run completed with ${run.conclusion ?? "no result"}`); + } + state.status = "completed"; + state.producer.completedAt = timestamp(now); + state.producer.workflowId = run.workflowId; + writeState(options.workDir, state); + return run; + } + + if (run.status !== "in_progress" && elapsed >= configured.queueTimeoutMs) { + await cancelRun(state, options.producerToken, cancelApi); + fail("RUN_QUEUE_TIMEOUT", "producer run did not start within 10 minutes"); + } + if (!warned && elapsed >= configured.warningAfterMs) { + console.warn( + "Qualification image build has exceeded 25 minutes; continuing to the hard limit.", + ); + warned = true; + } + await pause(configured.pollIntervalMs); + } +} + +export function validateQualificationArtifactList( + value: unknown, + state: ExactImageQualificationState, +): QualificationArtifact | null { + const response = record(value, "artifact list response"); + if (!Number.isSafeInteger(response.total_count) || (response.total_count as number) < 0) { + fail("ARTIFACT_MISSING_OR_INVALID", "artifact total_count is invalid"); + } + if (!Array.isArray(response.artifacts)) { + fail("ARTIFACT_MISSING_OR_INVALID", "artifact list is missing artifacts"); + } + if (response.total_count === 0 && response.artifacts.length === 0) return null; + if (response.total_count !== 1 || response.artifacts.length !== 1) { + fail("ARTIFACT_MISSING_OR_INVALID", "producer run must publish exactly one artifact"); + } + const artifact = record(response.artifacts[0], "qualification artifact"); + const id = safePositiveId(artifact.id, "artifact ID"); + const expectedName = `nemoclaw-image-handoff-v1-${state.producer.runId}-${state.producer.runAttempt}`; + requireExactString(artifact.name, expectedName, "artifact name"); + if (artifact.expired !== false) fail("ARTIFACT_MISSING_OR_INVALID", "artifact is expired"); + if (typeof artifact.digest !== "string" || !ARTIFACT_DIGEST_PATTERN.test(artifact.digest)) { + fail("ARTIFACT_MISSING_OR_INVALID", "artifact digest must be a lowercase SHA-256 digest"); + } + if ( + !Number.isSafeInteger(artifact.size_in_bytes) || + (artifact.size_in_bytes as number) < 1 || + (artifact.size_in_bytes as number) > MAX_ARCHIVE_BYTES + ) { + fail("ARTIFACT_MISSING_OR_INVALID", "artifact archive size is outside the accepted range"); + } + const apiUrl = `https://api.github.com/repos/${PRODUCER_REPOSITORY}/actions/artifacts/${id}`; + const archiveDownloadUrl = `${apiUrl}/zip`; + requireExactString(artifact.url, apiUrl, "artifact API URL"); + requireExactString(artifact.archive_download_url, archiveDownloadUrl, "artifact archive URL"); + const workflowRun = record(artifact.workflow_run, "artifact workflow run"); + if (safePositiveId(workflowRun.id, "artifact workflow run ID") !== state.producer.runId) { + fail("PROVENANCE_MISMATCH", "artifact workflow run ID did not match the dispatched run"); + } + requireExactString( + workflowRun.head_sha, + state.producer.repositorySha, + "artifact workflow run head SHA", + ); + return { + id, + name: expectedName, + digest: artifact.digest, + sizeInBytes: artifact.size_in_bytes as number, + apiUrl, + archiveDownloadUrl, + }; +} + +async function readBoundedResponse(response: Response, maxBytes: number): Promise { + const header = response.headers.get("content-length"); + if (header !== null && (!/^[0-9]+$/u.test(header) || Number(header) > maxBytes)) { + fail("ARTIFACT_MISSING_OR_INVALID", "artifact response is larger than the accepted limit"); + } + if (!response.body) fail("ARTIFACT_DOWNLOAD_TRANSIENT", "artifact response had no body"); + const reader = response.body.getReader(); + const chunks: Buffer[] = []; + let size = 0; + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + size += value.byteLength; + if (size > maxBytes) { + await reader.cancel(); + fail("ARTIFACT_MISSING_OR_INVALID", "artifact response exceeded the accepted limit"); + } + chunks.push(Buffer.from(value)); + } + return Buffer.concat(chunks, size); +} + +function writePrivateBuffer(file: string, contents: Buffer): void { + let descriptor: number; + try { + descriptor = fs.openSync( + file, + fs.constants.O_WRONLY | fs.constants.O_CREAT | fs.constants.O_EXCL | NO_FOLLOW | NON_BLOCK, + 0o600, + ); + } catch { + fail("OUTPUT_WRITE_FAILED", "artifact output could not be created safely"); + } + try { + const stat = fs.fstatSync(descriptor); + if (!stat.isFile() || stat.nlink !== 1) { + fail("OUTPUT_WRITE_FAILED", "artifact output must be one private regular file"); + } + fs.fchmodSync(descriptor, 0o600); + fs.writeFileSync(descriptor, contents); + fs.fsyncSync(descriptor); + } finally { + fs.closeSync(descriptor); + } +} + +function defaultCommandRunner(_command: string, args: readonly string[]): CommandResult { + return spawnSync("/usr/bin/unzip", [...args], { + encoding: null, + env: { + LANG: "C", + LC_ALL: "C", + }, + maxBuffer: MAX_MANIFEST_BYTES + 4096, + timeout: COMMAND_TIMEOUT_MS, + windowsHide: true, + }); +} + +function successfulCommand(result: CommandResult, label: string): Buffer { + if (result.error || result.status !== 0 || result.signal) { + fail("ARTIFACT_MISSING_OR_INVALID", `${label} rejected the artifact archive`); + } + return Buffer.isBuffer(result.stdout) ? result.stdout : Buffer.from(result.stdout); +} + +export function extractExactManifestArchive( + archivePath: string, + outputPath: string, + runCommand: CommandRunner = defaultCommandRunner, +): Buffer { + const entries = successfulCommand(runCommand("unzip", ["-Z1", archivePath]), "ZIP inventory"); + if (!entries.equals(Buffer.from(`${MANIFEST_ARTIFACT_FILE}\n`, "utf8"))) { + fail( + "ARTIFACT_MISSING_OR_INVALID", + `artifact ZIP must contain exactly ${MANIFEST_ARTIFACT_FILE} at its root`, + ); + } + const metadata = successfulCommand( + runCommand("unzip", ["-Zl", archivePath, MANIFEST_ARTIFACT_FILE]), + "ZIP metadata inspection", + ).toString("utf8"); + const matchingMetadata = metadata + .split(/\r?\n/u) + .filter((line) => line.endsWith(` ${MANIFEST_ARTIFACT_FILE}`)); + if (matchingMetadata.length !== 1 || !matchingMetadata[0]?.startsWith("-")) { + fail("ARTIFACT_MISSING_OR_INVALID", "artifact manifest ZIP entry must be a regular file"); + } + const manifest = successfulCommand( + runCommand("unzip", ["-p", archivePath, MANIFEST_ARTIFACT_FILE]), + "ZIP extraction", + ); + if (manifest.length === 0 || manifest.length > MAX_MANIFEST_BYTES) { + fail("ARTIFACT_MISSING_OR_INVALID", "artifact manifest size is outside the accepted range"); + } + writePrivateBuffer(outputPath, manifest); + const stat = fs.lstatSync(outputPath); + if ( + !stat.isFile() || + stat.isSymbolicLink() || + stat.nlink !== 1 || + stat.size !== manifest.length + ) { + fail("ARTIFACT_MISSING_OR_INVALID", "extracted manifest is not one regular direct file"); + } + return manifest; +} + +export async function downloadExactImageManifest( + options: { workDir: string; producerToken: string }, + dependencies: QualificationDependencies = {}, +): Promise { + const state = readExactImageQualificationState(options.workDir); + if (state.status !== "completed") { + fail("PROVENANCE_MISMATCH", "producer run must complete before artifact download"); + } + const pause = dependencies.sleep ?? sleep; + const now = dependencies.now ?? Date.now; + const configured = limits(dependencies); + const deadline = qualificationDeadline(state, configured.qualificationTimeoutMs); + const api = boundedApiClient(dependencies.api ?? githubApi, dependencies, deadline); + requireQualificationTimeRemaining(deadline, now); + const artifactStartedAt = now(); + const artifactDeadline = Math.min( + artifactStartedAt + configured.artifactPropagationTimeoutMs, + deadline, + ); + let artifact: QualificationArtifact | null = null; + while (artifact === null) { + requireQualificationTimeRemaining(deadline, now); + artifact = validateQualificationArtifactList( + await api( + `repos/${PRODUCER_REPOSITORY}/actions/runs/${state.producer.runId}/artifacts?per_page=100`, + options.producerToken, + { apiVersion: GITHUB_API_VERSION, expectedStatus: 200 }, + ), + state, + ); + requireQualificationTimeRemaining(deadline, now); + if (artifact !== null) break; + if (now() >= artifactDeadline) { + fail("ARTIFACT_PENDING", "qualification artifact did not appear within two minutes"); + } + await pause(configured.pollIntervalMs); + } + + const controller = new AbortController(); + const downloadBudget = Math.min( + configured.downloadTimeoutMs, + requireQualificationTimeRemaining(deadline, now), + ); + const timer = setTimeout(() => controller.abort(), downloadBudget); + let archive: Buffer; + try { + const response = await (dependencies.fetch ?? fetch)(artifact.archiveDownloadUrl, { + headers: { + Accept: "application/vnd.github+json", + Authorization: `Bearer ${options.producerToken}`, + "X-GitHub-Api-Version": GITHUB_API_VERSION, + }, + redirect: "follow", + signal: controller.signal, + }); + requireQualificationTimeRemaining(deadline, now); + if (response.status !== 200) { + fail("ARTIFACT_DOWNLOAD_TRANSIENT", `artifact archive download returned ${response.status}`); + } + archive = await readBoundedResponse(response, MAX_ARCHIVE_BYTES); + } catch (error) { + if (error instanceof ExactImageQualificationError) throw error; + requireQualificationTimeRemaining(deadline, now); + fail("ARTIFACT_DOWNLOAD_TRANSIENT", "artifact archive download failed"); + } finally { + clearTimeout(timer); + } + requireQualificationTimeRemaining(deadline, now); + if (archive.length !== artifact.sizeInBytes) { + fail("ARTIFACT_MISSING_OR_INVALID", "downloaded archive size did not match artifact metadata"); + } + const archiveHash = sha256(archive); + if (`sha256:${archiveHash}` !== artifact.digest) { + fail("ARTIFACT_MISSING_OR_INVALID", "downloaded archive digest did not match GitHub metadata"); + } + + const archivePath = path.join(options.workDir, ARCHIVE_FILE); + const manifestPath = path.join(options.workDir, MANIFEST_ARTIFACT_FILE); + writePrivateBuffer(archivePath, archive); + const manifest = extractExactManifestArchive( + archivePath, + manifestPath, + dependencies.runCommand ?? defaultCommandRunner, + ); + requireQualificationTimeRemaining(deadline, now); + state.artifact = { + ...artifact, + archiveSha256: archiveHash, + manifestSha256: sha256(manifest), + }; + state.status = "downloaded"; + writeState(options.workDir, state); + return artifact; +} + +function readPrivateBuffer(file: string, maxBytes: number): Buffer { + let descriptor: number; + try { + descriptor = fs.openSync(file, fs.constants.O_RDONLY | NO_FOLLOW | NON_BLOCK); + } catch { + fail("PROVENANCE_MISMATCH", "qualification evidence file could not be opened safely"); + } + try { + const before = fs.fstatSync(descriptor); + if (!before.isFile() || before.nlink !== 1 || before.size > maxBytes) { + fail("PROVENANCE_MISMATCH", "qualification evidence file failed regular-file checks"); + } + const bytes = fs.readFileSync(descriptor); + const after = fs.fstatSync(descriptor); + if (after.size !== before.size || after.nlink !== 1) { + fail("PROVENANCE_MISMATCH", "qualification evidence file changed while it was read"); + } + return bytes; + } finally { + fs.closeSync(descriptor); + } +} + +export function finalizeExactImageQualification( + workDir: string, + dependencies: QualificationDependencies = {}, +): ExactImageQualificationState { + const state = readExactImageQualificationState(workDir); + if (state.status !== "downloaded" || !state.artifact) { + fail("PROVENANCE_MISMATCH", "artifact must be digest-verified before finalization"); + } + const now = dependencies.now ?? Date.now; + const deadline = qualificationDeadline(state, limits(dependencies).qualificationTimeoutMs); + requireQualificationTimeRemaining(deadline, now); + const manifestHash = sha256( + readPrivateBuffer(path.join(workDir, MANIFEST_ARTIFACT_FILE), MAX_MANIFEST_BYTES), + ); + if (manifestHash !== state.artifact.manifestSha256) { + fail("PROVENANCE_MISMATCH", "manifest changed after archive verification"); + } + const normalizedHash = sha256( + readPrivateBuffer(path.join(workDir, VALIDATED_MANIFEST_FILE), MAX_MANIFEST_BYTES), + ); + requireQualificationTimeRemaining(deadline, now); + state.validation = { + acceptedAt: timestamp(now), + manifestSha256: manifestHash, + normalizedManifestSha256: normalizedHash, + }; + state.status = "validated"; + const evidence = { + schemaVersion: 1, + kind: "nemoclaw-exact-image-qualification-evidence", + qualificationStatus: "accepted", + request: state.request, + producer: state.producer, + artifact: state.artifact, + validation: state.validation, + }; + try { + privateFileRuntime.writePrivateRegularFile( + path.join(workDir, EVIDENCE_FILE), + `${JSON.stringify(evidence, null, 2)}\n`, + ); + } catch { + fail("OUTPUT_WRITE_FAILED", "qualification evidence could not be written safely"); + } + writeState(workDir, state); + return state; +} + +export async function cancelActiveExactImageQualification( + options: { workDir: string; producerToken: string }, + dependencies: QualificationDependencies = {}, +): Promise { + let state: ExactImageQualificationState; + try { + state = readExactImageQualificationState(options.workDir); + } catch (error) { + if (!fs.existsSync(intentPath(options.workDir))) { + if (fs.existsSync(statePath(options.workDir))) throw error; + return false; + } + const intent = readDispatchIntent(options.workDir); + if (intent === null) return false; + preserveUnreadableState(options.workDir); + return reconcileAmbiguousDispatch({ ...options, intent, mode: "cleanup" }, dependencies); + } + const now = dependencies.now ?? Date.now; + const configured = limits(dependencies); + if (state.status !== "dispatched") return false; + const deadline = now() + configured.cleanupTimeoutMs; + const api = boundedApiClient(dependencies.api ?? githubApi, dependencies, deadline); + return cancelAndVerifyRecoveredRun( + state, + { status: "unknown" }, + options.producerToken, + api, + dependencies.sleep ?? sleep, + configured.reconciliationPollIntervalMs, + deadline, + now, + ); +} + +function parseFlags(argv: readonly string[]): Map { + const flags = new Map(); + for (let index = 0; index < argv.length; index += 2) { + const flag = argv[index]; + const value = argv[index + 1]; + if (!flag?.startsWith("--") || value === undefined || value.startsWith("--")) { + fail("REQUEST_INVALID", `${flag ?? "argument"} requires one value`); + } + if (flags.has(flag)) fail("REQUEST_INVALID", `${flag} may be provided only once`); + flags.set(flag, value); + } + return flags; +} + +function requiredFlag(flags: Map, flag: string): string { + const value = flags.get(flag); + if (value === undefined) fail("REQUEST_INVALID", `${flag} is required`); + return value; +} + +export function parseExactImageQualificationCommand(argv: readonly string[]): ControllerCommand { + const flags = parseFlags(argv); + const mode = requiredFlag(flags, "--mode"); + if (["wait", "download", "finalize", "cancel"].includes(mode)) { + const allowed = new Set(["--mode", "--work-dir"]); + for (const flag of flags.keys()) { + if (!allowed.has(flag)) fail("REQUEST_INVALID", `unknown argument ${flag}`); + } + return { + mode: mode as "wait" | "download" | "finalize" | "cancel", + workDir: requiredFlag(flags, "--work-dir"), + }; + } + if (mode !== "preflight" && mode !== "start") { + fail("REQUEST_INVALID", `unsupported mode ${mode}`); + } + const allowed = new Set([ + "--mode", + "--actor", + "--candidate-sha", + "--event-name", + "--reason", + "--ref", + "--requester-run-attempt", + "--requester-run-id", + "--workflow-sha", + ...(mode === "start" ? ["--work-dir"] : []), + ]); + for (const flag of flags.keys()) { + if (!allowed.has(flag)) fail("REQUEST_INVALID", `unknown argument ${flag}`); + } + const request = validateExactImageQualificationRequest({ + actor: requiredFlag(flags, "--actor"), + candidateSha: requiredFlag(flags, "--candidate-sha"), + eventName: requiredFlag(flags, "--event-name"), + reason: requiredFlag(flags, "--reason"), + ref: requiredFlag(flags, "--ref"), + requesterRunAttempt: positiveInteger( + requiredFlag(flags, "--requester-run-attempt"), + "requester run attempt", + ), + requesterRunId: requiredFlag(flags, "--requester-run-id"), + workflowSha: requiredFlag(flags, "--workflow-sha"), + }); + return mode === "preflight" + ? { mode, request } + : { mode, request, workDir: requiredFlag(flags, "--work-dir") }; +} + +function requiredToken(name: string): string { + const token = process.env[name]; + if (!token) fail("REQUEST_INVALID", `${name} is required`); + return token; +} + +function writeOutput(name: string, value: string): void { + const output = process.env.GITHUB_OUTPUT; + if (!output) return; + fs.appendFileSync(output, `${name}=${value}\n`, { encoding: "utf8", mode: 0o600 }); +} + +async function main(): Promise { + const command = parseExactImageQualificationCommand(process.argv.slice(2)); + if (command.mode === "preflight") { + await preflightExactImageQualification(command.request, requiredToken("GITHUB_TOKEN")); + console.log("Draft qualification request passed preflight."); + return; + } + if (command.mode === "start") { + const state = await startExactImageQualification({ + request: command.request, + coreToken: requiredToken("GITHUB_TOKEN"), + producerToken: requiredToken("NEMOCLAW_IMAGE_QUALIFICATION_TOKEN"), + workDir: command.workDir, + }); + writeOutput("correlation_id", state.request.correlationId); + writeOutput("image_repository_sha", state.producer.repositorySha); + writeOutput("producer_run_id", state.producer.runId); + writeOutput("producer_run_attempt", String(state.producer.runAttempt)); + console.log(`Dispatched and bound producer run ${state.producer.runId}.`); + return; + } + if (command.mode === "wait") { + const run = await waitForExactImageQualification({ + workDir: command.workDir, + producerToken: requiredToken("NEMOCLAW_IMAGE_QUALIFICATION_TOKEN"), + }); + console.log(`Producer run ${run.id} completed successfully.`); + return; + } + if (command.mode === "download") { + const artifact = await downloadExactImageManifest({ + workDir: command.workDir, + producerToken: requiredToken("NEMOCLAW_IMAGE_QUALIFICATION_TOKEN"), + }); + console.log(`Verified and extracted artifact ${artifact.id}.`); + return; + } + if (command.mode === "finalize") { + const state = finalizeExactImageQualification(command.workDir); + console.log(`Recorded accepted qualification evidence for run ${state.producer.runId}.`); + return; + } + const cancelled = await cancelActiveExactImageQualification({ + workDir: command.workDir, + producerToken: requiredToken("NEMOCLAW_IMAGE_QUALIFICATION_TOKEN"), + }); + console.log(cancelled ? "Cancelled active producer run." : "No active producer run to cancel."); +} + +export function exactImageQualificationFailureCode( + error: unknown, +): ExactImageQualificationFailureCode { + return error instanceof ExactImageQualificationError ? error.code : "UNKNOWN"; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((error) => { + const code = exactImageQualificationFailureCode(error); + const message = error instanceof Error ? error.message : "unexpected qualification error"; + process.stderr.write(`${code}: ${message}\n`); + process.exitCode = 1; + }); +} diff --git a/tools/e2e/validate-exact-image-manifest.mts b/tools/e2e/validate-exact-image-manifest.mts new file mode 100644 index 0000000000..5e53238f11 --- /dev/null +++ b/tools/e2e/validate-exact-image-manifest.mts @@ -0,0 +1,176 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import { pathToFileURL } from "node:url"; +import { TextDecoder } from "node:util"; +import { + ExactImageManifestError, + type ExactImageManifestExpectations, + exactImageManifestFailureCode, + normalizedExactImageManifestJson, + parseAndValidateExactImageManifest, +} from "./exact-image-manifest.mts"; +import * as privateFile from "./private-file.ts"; + +// The root TypeScript package is exposed as CommonJS under `node --import +// tsx` / `npx tsx`, but as an ESM namespace under Node's strip-types runtime +// and Vitest. Normalize both representations so every supported entrypoint +// uses the same no-follow private-file writer. +const privateFileRuntime = ( + "default" in privateFile && privateFile.default ? privateFile.default : privateFile +) as typeof import("./private-file.ts"); + +const MAX_MANIFEST_BYTES = 64 * 1024; +const NO_FOLLOW = fs.constants.O_NOFOLLOW ?? 0; +const NON_BLOCK = fs.constants.O_NONBLOCK ?? 0; + +type ExactImageManifestCliOptions = ExactImageManifestExpectations & { + manifest: string; + output: string; +}; + +const ARGUMENT_FIELDS = { + "--manifest": "manifest", + "--output": "output", + "--nemoclaw-sha": "nemoclawSha", + "--requester-run-id": "requesterWorkflowRunId", + "--requester-run-attempt": "requesterWorkflowRunAttempt", + "--correlation-id": "correlationId", + "--image-repository-sha": "imageRepositorySha", + "--producer-run-id": "workflowRunId", + "--producer-run-attempt": "workflowRunAttempt", +} as const; + +function requestInvalid(message: string): never { + throw new ExactImageManifestError("REQUEST_INVALID", message); +} + +function positiveIntegerArgument(value: string, flag: string): number { + if (!/^[1-9][0-9]*$/u.test(value)) { + requestInvalid(`${flag} must be a positive integer`); + } + const parsed = Number(value); + if (!Number.isSafeInteger(parsed)) { + requestInvalid(`${flag} exceeds the safe integer range`); + } + return parsed; +} + +export function parseExactImageManifestCliArgs( + argv: readonly string[], +): ExactImageManifestCliOptions { + const values = new Map(); + for (let index = 0; index < argv.length; index += 2) { + const flag = argv[index] as keyof typeof ARGUMENT_FIELDS | undefined; + const value = argv[index + 1]; + if (!flag || !(flag in ARGUMENT_FIELDS)) { + requestInvalid(`unknown argument ${JSON.stringify(flag ?? "")}`); + } + if (value === undefined || value.startsWith("--")) { + requestInvalid(`${flag} requires a value`); + } + const field = ARGUMENT_FIELDS[flag]; + if (values.has(field)) { + requestInvalid(`${flag} may be provided only once`); + } + values.set(field, value); + } + + const requireValue = (field: keyof ExactImageManifestCliOptions, flag: string): string => { + const value = values.get(field); + if (value === undefined) requestInvalid(`${flag} is required`); + return value; + }; + + return { + manifest: requireValue("manifest", "--manifest"), + output: requireValue("output", "--output"), + nemoclawSha: requireValue("nemoclawSha", "--nemoclaw-sha"), + requesterWorkflowRunId: requireValue("requesterWorkflowRunId", "--requester-run-id"), + requesterWorkflowRunAttempt: positiveIntegerArgument( + requireValue("requesterWorkflowRunAttempt", "--requester-run-attempt"), + "--requester-run-attempt", + ), + correlationId: requireValue("correlationId", "--correlation-id"), + imageRepositorySha: requireValue("imageRepositorySha", "--image-repository-sha"), + workflowRunId: requireValue("workflowRunId", "--producer-run-id"), + workflowRunAttempt: positiveIntegerArgument( + requireValue("workflowRunAttempt", "--producer-run-attempt"), + "--producer-run-attempt", + ), + }; +} + +function readManifestFile(file: string): string { + let descriptor: number; + try { + descriptor = fs.openSync(file, fs.constants.O_RDONLY | NO_FOLLOW | NON_BLOCK); + } catch { + throw new ExactImageManifestError( + "ARTIFACT_MISSING_OR_INVALID", + "manifest input could not be opened safely", + ); + } + + try { + const before = fs.fstatSync(descriptor); + if (!before.isFile() || before.nlink !== 1) { + throw new ExactImageManifestError( + "ARTIFACT_MISSING_OR_INVALID", + "manifest input must be one regular file", + ); + } + if (before.size > MAX_MANIFEST_BYTES) { + throw new ExactImageManifestError( + "ARTIFACT_MISSING_OR_INVALID", + `manifest input exceeds ${MAX_MANIFEST_BYTES} bytes`, + ); + } + const bytes = fs.readFileSync(descriptor); + const after = fs.fstatSync(descriptor); + if (bytes.length > MAX_MANIFEST_BYTES || after.size !== before.size || after.nlink !== 1) { + throw new ExactImageManifestError( + "ARTIFACT_MISSING_OR_INVALID", + "manifest input changed while it was read", + ); + } + try { + return new TextDecoder("utf-8", { fatal: true }).decode(bytes); + } catch { + throw new ExactImageManifestError( + "ARTIFACT_MISSING_OR_INVALID", + "manifest input must be valid UTF-8", + ); + } + } finally { + fs.closeSync(descriptor); + } +} + +export function runExactImageManifestCli(argv: readonly string[] = process.argv.slice(2)): void { + const options = parseExactImageManifestCliArgs(argv); + const accepted = parseAndValidateExactImageManifest(readManifestFile(options.manifest), options); + try { + privateFileRuntime.writePrivateRegularFile( + options.output, + normalizedExactImageManifestJson(accepted), + ); + } catch { + throw new ExactImageManifestError( + "OUTPUT_WRITE_FAILED", + "accepted manifest output could not be written safely", + ); + } +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + try { + runExactImageManifestCli(); + } catch (error) { + const code = exactImageManifestFailureCode(error); + const message = error instanceof Error ? error.message : "unexpected manifest validation error"; + process.stderr.write(`${code}: ${message}\n`); + process.exitCode = 1; + } +} From 752e561f2866a94c4a550be0cfccb1583092fbf3 Mon Sep 17 00:00:00 2001 From: Charan Jagwani Date: Thu, 16 Jul 2026 11:58:21 -0700 Subject: [PATCH 02/25] test(e2e): extract qualification controller fixtures Signed-off-by: Charan Jagwani --- ...act-image-qualification-controller.test.ts | 416 ++++++------------ ...-image-qualification-controller-fixture.ts | 249 +++++++++++ 2 files changed, 380 insertions(+), 285 deletions(-) create mode 100644 test/helpers/exact-image-qualification-controller-fixture.ts diff --git a/test/exact-image-qualification-controller.test.ts b/test/exact-image-qualification-controller.test.ts index fd6cfcef29..11441ab786 100644 --- a/test/exact-image-qualification-controller.test.ts +++ b/test/exact-image-qualification-controller.test.ts @@ -4,7 +4,6 @@ import { spawnSync } from "node:child_process"; import { createHash } from "node:crypto"; import fs from "node:fs"; -import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; @@ -18,7 +17,6 @@ import { downloadExactImageManifest, EVIDENCE_FILE, ExactImageQualificationError, - type ExactImageQualificationRequest, extractExactManifestArchive, finalizeExactImageQualification, GITHUB_API_VERSION, @@ -39,167 +37,27 @@ import { validateWorkflowDispatchDetails, waitForExactImageQualification, } from "../tools/e2e/exact-image-qualification-controller.mts"; - -const CANDIDATE_SHA = "a".repeat(40); -const PRODUCER_SHA = "b".repeat(40); -const CORRELATION_ID = "123e4567-e89b-42d3-a456-426614174000"; -const RUN_ID = "24680"; -const WORKFLOW_ID = 13579; -const BASE_TIME = Date.UTC(2026, 0, 1); -const API_RUN_URL = `https://api.github.com/repos/${PRODUCER_REPOSITORY}/actions/runs/${RUN_ID}`; -const HTML_RUN_URL = `https://github.com/${PRODUCER_REPOSITORY}/actions/runs/${RUN_ID}`; - -const REQUEST: ExactImageQualificationRequest = { - actor: "maintainer", - candidateSha: CANDIDATE_SHA, - eventName: "workflow_dispatch", - reason: "Qualify the current daily candidate before tagging", - ref: "refs/heads/main", - requesterRunAttempt: 1, - requesterRunId: "97531", - workflowSha: CANDIDATE_SHA, -}; - -type ApiOptions = { - artifacts?: unknown; - dispatch?: unknown; - permission?: string; - roleName?: string; - producerSha?: string; - requesterSha?: string; - run?: unknown; - runs?: unknown; - workflow?: unknown; -}; - -function mainRef(sha: string) { - return { ref: "refs/heads/main", object: { type: "commit", sha } }; -} - -function dispatchDetails() { - return { - workflow_run_id: Number(RUN_ID), - run_url: API_RUN_URL, - html_url: HTML_RUN_URL, - }; -} - -function workflowRun(overrides: Record = {}) { - return { - id: Number(RUN_ID), - workflow_id: WORKFLOW_ID, - run_attempt: 1, - event: "workflow_dispatch", - head_branch: "main", - head_sha: PRODUCER_SHA, - path: PRODUCER_WORKFLOW_PATH, - display_title: `Qualify NemoClaw ${CANDIDATE_SHA} (${CORRELATION_ID})`, - url: API_RUN_URL, - html_url: HTML_RUN_URL, - repository: { full_name: PRODUCER_REPOSITORY }, - head_repository: { full_name: PRODUCER_REPOSITORY }, - status: "queued", - conclusion: null, - created_at: new Date(BASE_TIME).toISOString(), - ...overrides, - }; -} - -function createApi(options: ApiOptions = {}) { - return vi.fn(async (apiPath: string, _token: string, requestOptions?: unknown) => { - if (apiPath === "repos/NVIDIA/NemoClaw/git/ref/heads/main") { - return mainRef(options.requesterSha ?? CANDIDATE_SHA); - } - if (apiPath.includes("/collaborators/")) { - return { - permission: options.permission ?? "write", - role_name: options.roleName ?? "maintain", - }; - } - if (apiPath === `repos/${PRODUCER_REPOSITORY}/git/ref/heads/main`) { - return mainRef(options.producerSha ?? PRODUCER_SHA); - } - if (apiPath === `repos/${PRODUCER_REPOSITORY}/actions/workflows/${PRODUCER_WORKFLOW_FILE}`) { - return ( - options.workflow ?? { - id: WORKFLOW_ID, - path: PRODUCER_WORKFLOW_PATH, - state: "active", - } - ); - } - if ( - apiPath === - `repos/${PRODUCER_REPOSITORY}/actions/workflows/${PRODUCER_WORKFLOW_FILE}/dispatches` - ) { - return options.dispatch === undefined ? dispatchDetails() : options.dispatch; - } - if (apiPath === `repos/${PRODUCER_REPOSITORY}/actions/runs/${RUN_ID}`) { - return options.run ?? workflowRun(); - } - if (apiPath === `repos/${PRODUCER_REPOSITORY}/actions/runs/${RUN_ID}/cancel`) { - return undefined; - } - if ( - apiPath.startsWith( - `repos/${PRODUCER_REPOSITORY}/actions/workflows/${PRODUCER_WORKFLOW_FILE}/runs?`, - ) - ) { - return options.runs ?? { total_count: 0, workflow_runs: [] }; - } - if (apiPath === `repos/${PRODUCER_REPOSITORY}/actions/runs/${RUN_ID}/artifacts?per_page=100`) { - return options.artifacts; - } - throw new Error(`unexpected API call ${apiPath} ${JSON.stringify(requestOptions)}`); - }); -} - -function dependencies(api: ReturnType, extra: QualificationDependencies = {}) { - return { - now: () => BASE_TIME, - ...extra, - api: api as QualificationDependencies["api"], - randomUuid: () => CORRELATION_ID, - }; -} - -function tempDirectory(): string { - return fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-image-qualification-test-")); -} - -async function startedState(api = createApi()) { - const workDir = tempDirectory(); - const state = await startExactImageQualification( - { - request: REQUEST, - coreToken: "core-token", - producerToken: "producer-token", - workDir, - }, - dependencies(api), - ); - return { api, state, workDir }; -} - -function artifactList(archive: Buffer, overrides: Record = {}) { - const artifactId = 86420; - return { - total_count: 1, - artifacts: [ - { - id: artifactId, - name: `nemoclaw-image-handoff-v1-${RUN_ID}-1`, - expired: false, - digest: `sha256:${createHash("sha256").update(archive).digest("hex")}`, - size_in_bytes: archive.length, - url: `https://api.github.com/repos/${PRODUCER_REPOSITORY}/actions/artifacts/${artifactId}`, - archive_download_url: `https://api.github.com/repos/${PRODUCER_REPOSITORY}/actions/artifacts/${artifactId}/zip`, - workflow_run: { id: Number(RUN_ID), head_sha: PRODUCER_SHA }, - ...overrides, - }, - ], - }; -} +import { + API_RUN_URL, + artifactList, + BASE_TIME, + CANDIDATE_SHA, + CORRELATION_ID, + createApi, + createArchiveRunCommand, + createRoutedApi, + dependencies, + dispatchDetails, + HTML_RUN_URL, + PRODUCER_SHA, + qualificationApiRoute, + REQUEST, + RUN_ID, + startedState, + tempDirectory, + WORKFLOW_ID, + workflowRun, +} from "./helpers/exact-image-qualification-controller-fixture.ts"; afterEach(() => { vi.restoreAllMocks(); @@ -372,23 +230,26 @@ describe("exact producer dispatch binding", () => { const workDir = tempDirectory(); const base = createApi(); let cancelObserved = false; - const api = vi.fn(async (apiPath: string, token: string, requestOptions?: unknown) => { - if (apiPath.endsWith(`/workflows/${PRODUCER_WORKFLOW_FILE}/dispatches`)) { + let stateExistedAtCancel = false; + const api = createRoutedApi(base, [ + qualificationApiRoute.dispatch(() => { throw new Error("response connection reset after server acceptance"); - } - if (apiPath.includes(`/workflows/${PRODUCER_WORKFLOW_FILE}/runs?`)) { - return { total_count: 1, workflow_runs: [workflowRun()] }; - } - if (apiPath === `repos/${PRODUCER_REPOSITORY}/actions/runs/${RUN_ID}/cancel`) { - expect(fs.existsSync(path.join(workDir, STATE_FILE))).toBe(true); + }), + qualificationApiRoute.workflowRuns(() => ({ + total_count: 1, + workflow_runs: [workflowRun()], + })), + qualificationApiRoute.cancelRun(() => { + stateExistedAtCancel = fs.existsSync(path.join(workDir, STATE_FILE)); cancelObserved = true; return undefined; - } - if (apiPath === `repos/${PRODUCER_REPOSITORY}/actions/runs/${RUN_ID}` && cancelObserved) { - return workflowRun({ status: "completed", conclusion: "cancelled" }); - } - return base(apiPath, token, requestOptions); - }); + }), + qualificationApiRoute.run(() => + cancelObserved + ? workflowRun({ status: "completed", conclusion: "cancelled" }) + : workflowRun(), + ), + ]); try { await expect( startExactImageQualification( @@ -407,6 +268,7 @@ describe("exact producer dispatch binding", () => { ), ).toHaveLength(1); expect(cancelObserved).toBe(true); + expect(stateExistedAtCancel).toBe(true); expect(readExactImageQualificationState(workDir).producer.runId).toBe(RUN_ID); expect(fs.existsSync(path.join(workDir, DISPATCH_INTENT_FILE))).toBe(true); expect( @@ -426,36 +288,37 @@ describe("exact producer dispatch binding", () => { const historicalRuns = Array.from({ length: 101 }, (_value, index) => workflowRun({ created_at: new Date(BASE_TIME - (index + 2) * 10 * 60_000).toISOString() }), ); - const api = vi.fn(async (apiPath: string, token: string, requestOptions?: unknown) => { - if (apiPath.endsWith(`/workflows/${PRODUCER_WORKFLOW_FILE}/dispatches`)) { + let observedCreatedFilter: string | null = null; + let observedHeadShaFilter = false; + let observedScopedRuns: unknown[] = []; + const api = createRoutedApi(base, [ + qualificationApiRoute.dispatch(() => { throw new Error("lost response after main advanced"); - } - if (apiPath.includes(`/workflows/${PRODUCER_WORKFLOW_FILE}/runs?`)) { + }), + qualificationApiRoute.workflowRuns((apiPath) => { const query = new URLSearchParams(apiPath.split("?", 2)[1]); - expect(query.get("created")).toBe("2025-12-31T23:59:00Z..2026-01-01T00:01:30Z"); - expect(apiPath).not.toContain("head_sha="); + observedCreatedFilter = query.get("created"); + observedHeadShaFilter = apiPath.includes("head_sha="); const [earliest, latest] = (query.get("created") ?? "").split("..").map(Date.parse); const scoped = [...historicalRuns, movedRun].filter(({ created_at }) => { const createdAt = Date.parse(created_at); return createdAt >= earliest && createdAt <= latest; }); - expect(historicalRuns).toHaveLength(101); - expect(scoped).toEqual([movedRun]); + observedScopedRuns = scoped; return { total_count: scoped.length, workflow_runs: scoped }; - } - if (apiPath === `repos/${PRODUCER_REPOSITORY}/actions/runs/${RUN_ID}/cancel`) { + }), + qualificationApiRoute.cancelRun(() => { cancelled = true; return undefined; - } - if (apiPath === `repos/${PRODUCER_REPOSITORY}/actions/runs/${RUN_ID}` && cancelled) { - return workflowRun({ + }), + qualificationApiRoute.run(() => + workflowRun({ head_sha: movedSha, - status: "completed", - conclusion: "cancelled", - }); - } - return base(apiPath, token, requestOptions); - }); + status: cancelled ? "completed" : "queued", + conclusion: cancelled ? "cancelled" : null, + }), + ), + ]); try { await expect( startExactImageQualification( @@ -469,6 +332,10 @@ describe("exact producer dispatch binding", () => { ), ).rejects.toMatchObject({ code: "DISPATCH_AMBIGUOUS" }); expect(cancelled).toBe(true); + expect(observedCreatedFilter).toBe("2025-12-31T23:59:00Z..2026-01-01T00:01:30Z"); + expect(observedHeadShaFilter).toBe(false); + expect(historicalRuns).toHaveLength(101); + expect(observedScopedRuns).toEqual([movedRun]); expect(readExactImageQualificationState(workDir).producer).toMatchObject({ runId: RUN_ID, repositorySha: movedSha, @@ -504,20 +371,19 @@ describe("exact producer dispatch binding", () => { const base = createApi(); let cancelled = false; - const cleanupApi = vi.fn(async (apiPath: string, token: string, requestOptions?: unknown) => { - if (apiPath === `repos/${PRODUCER_REPOSITORY}/actions/runs/${RUN_ID}/cancel`) { + const cleanupApi = createRoutedApi(base, [ + qualificationApiRoute.cancelRun(() => { cancelled = true; return undefined; - } - if (apiPath === `repos/${PRODUCER_REPOSITORY}/actions/runs/${RUN_ID}`) { - return workflowRun({ + }), + qualificationApiRoute.run(() => + workflowRun({ head_sha: movedSha, status: cancelled ? "completed" : "queued", conclusion: cancelled ? "cancelled" : null, - }); - } - return base(apiPath, token, requestOptions); - }); + }), + ), + ]); await expect( cancelActiveExactImageQualification( { workDir, producerToken: "producer-token" }, @@ -537,15 +403,15 @@ describe("exact producer dispatch binding", () => { const workDir = tempDirectory(); const base = createApi(); const completed = workflowRun({ status: "completed", conclusion: "success" }); - const api = vi.fn(async (apiPath: string, token: string, requestOptions?: unknown) => { - if (apiPath.endsWith(`/workflows/${PRODUCER_WORKFLOW_FILE}/dispatches`)) { + const api = createRoutedApi(base, [ + qualificationApiRoute.dispatch(() => { throw new Error("lost response"); - } - if (apiPath.includes(`/workflows/${PRODUCER_WORKFLOW_FILE}/runs?`)) { - return { total_count: 1, workflow_runs: [completed] }; - } - return base(apiPath, token, requestOptions); - }); + }), + qualificationApiRoute.workflowRuns(() => ({ + total_count: 1, + workflow_runs: [completed], + })), + ]); try { await expect( startExactImageQualification( @@ -570,18 +436,18 @@ describe("exact producer dispatch binding", () => { it("retains the recovered run identity when cancellation fails", async () => { const workDir = tempDirectory(); const base = createApi(); - const api = vi.fn(async (apiPath: string, token: string, requestOptions?: unknown) => { - if (apiPath.endsWith(`/workflows/${PRODUCER_WORKFLOW_FILE}/dispatches`)) { + const api = createRoutedApi(base, [ + qualificationApiRoute.dispatch(() => { throw new Error("lost response"); - } - if (apiPath.includes(`/workflows/${PRODUCER_WORKFLOW_FILE}/runs?`)) { - return { total_count: 1, workflow_runs: [workflowRun()] }; - } - if (apiPath === `repos/${PRODUCER_REPOSITORY}/actions/runs/${RUN_ID}/cancel`) { + }), + qualificationApiRoute.workflowRuns(() => ({ + total_count: 1, + workflow_runs: [workflowRun()], + })), + qualificationApiRoute.cancelRun(() => { throw new Error("cancel transport failed"); - } - return base(apiPath, token, requestOptions); - }); + }), + ]); try { await expect( startExactImageQualification( @@ -607,30 +473,27 @@ describe("exact producer dispatch binding", () => { const workDir = tempDirectory(); const base = createApi(); let clock = BASE_TIME; - const api = vi.fn(async (apiPath: string, token: string, requestOptions?: unknown) => { - if (apiPath.endsWith(`/workflows/${PRODUCER_WORKFLOW_FILE}/dispatches`)) { + const nearMatches = [ + workflowRun({ display_title: "wrong correlation" }), + workflowRun({ workflow_id: WORKFLOW_ID + 1 }), + workflowRun({ run_attempt: 2 }), + workflowRun({ event: "push" }), + workflowRun({ head_branch: "feature" }), + workflowRun({ path: ".github/workflows/other.yml" }), + workflowRun({ repository: { full_name: "other/repository" } }), + workflowRun({ head_repository: { full_name: "other/repository" } }), + workflowRun({ created_at: new Date(BASE_TIME - 2 * 60_000).toISOString() }), + workflowRun({ url: `${API_RUN_URL}/wrong` }), + ]; + const api = createRoutedApi(base, [ + qualificationApiRoute.dispatch(() => { throw new Error("lost response"); - } - if (apiPath.includes(`/workflows/${PRODUCER_WORKFLOW_FILE}/runs?`)) { - const nearMatches = [ - workflowRun({ display_title: "wrong correlation" }), - workflowRun({ workflow_id: WORKFLOW_ID + 1 }), - workflowRun({ run_attempt: 2 }), - workflowRun({ event: "push" }), - workflowRun({ head_branch: "feature" }), - workflowRun({ path: ".github/workflows/other.yml" }), - workflowRun({ repository: { full_name: "other/repository" } }), - workflowRun({ head_repository: { full_name: "other/repository" } }), - workflowRun({ created_at: new Date(BASE_TIME - 2 * 60_000).toISOString() }), - workflowRun({ url: `${API_RUN_URL}/wrong` }), - ]; - return { - total_count: nearMatches.length, - workflow_runs: nearMatches, - }; - } - return base(apiPath, token, requestOptions); - }); + }), + qualificationApiRoute.workflowRuns(() => ({ + total_count: nearMatches.length, + workflow_runs: nearMatches, + })), + ]); try { await expect( startExactImageQualification( @@ -692,19 +555,19 @@ describe("exact producer dispatch binding", () => { const base = createApi(); let cancelled = false; - const cleanupApi = vi.fn(async (apiPath: string, token: string, requestOptions?: unknown) => { - if (apiPath.includes(`/workflows/${PRODUCER_WORKFLOW_FILE}/runs?`)) { - return { total_count: 1, workflow_runs: [workflowRun()] }; - } - if (apiPath === `repos/${PRODUCER_REPOSITORY}/actions/runs/${RUN_ID}/cancel`) { + const cleanupApi = createRoutedApi(base, [ + qualificationApiRoute.workflowRuns(() => ({ + total_count: 1, + workflow_runs: [workflowRun()], + })), + qualificationApiRoute.cancelRun(() => { cancelled = true; return undefined; - } - if (apiPath === `repos/${PRODUCER_REPOSITORY}/actions/runs/${RUN_ID}` && cancelled) { - return workflowRun({ status: "completed", conclusion: "cancelled" }); - } - return base(apiPath, token, requestOptions); - }); + }), + qualificationApiRoute.run(() => + cancelled ? workflowRun({ status: "completed", conclusion: "cancelled" }) : workflowRun(), + ), + ]); await expect( cancelActiveExactImageQualification( { workDir, producerToken: "producer-token" }, @@ -733,16 +596,16 @@ describe("exact producer dispatch binding", () => { url: `https://api.github.com/repos/${PRODUCER_REPOSITORY}/actions/runs/${secondId}`, html_url: `https://github.com/${PRODUCER_REPOSITORY}/actions/runs/${secondId}`, }); - const api = vi.fn(async (apiPath: string, token: string, requestOptions?: unknown) => { - if (apiPath.endsWith(`/workflows/${PRODUCER_WORKFLOW_FILE}/dispatches`)) { + const api = createRoutedApi(base, [ + qualificationApiRoute.dispatch(() => { throw new Error("lost response"); - } - if (apiPath.includes(`/workflows/${PRODUCER_WORKFLOW_FILE}/runs?`)) { - return { total_count: 2, workflow_runs: [workflowRun(), second] }; - } - if (apiPath.endsWith("/cancel")) return undefined; - return base(apiPath, token, requestOptions); - }); + }), + qualificationApiRoute.workflowRuns(() => ({ + total_count: 2, + workflow_runs: [workflowRun(), second], + })), + qualificationApiRoute.cancelAnyRun(() => undefined), + ]); try { await expect( startExactImageQualification( @@ -1026,26 +889,9 @@ describe("qualification artifact integrity", () => { dependencies(createApi({ artifacts: artifactList(archive) }), { now: () => clock, fetch: async () => new Response(archive, { status: 200 }), - runCommand: (_command, args) => { - if (args[0] === "-Z1") { - return { - status: 0, - stdout: Buffer.from(`${MANIFEST_ARTIFACT_FILE}\n`), - stderr: Buffer.alloc(0), - }; - } - if (args[0] === "-Zl") { - return { - status: 0, - stdout: Buffer.from( - `-rw-r--r-- 3.0 unx 20 tx 20 stor ${MANIFEST_ARTIFACT_FILE}\n`, - ), - stderr: Buffer.alloc(0), - }; - } + runCommand: createArchiveRunCommand(manifest, () => { clock = BASE_TIME + 45 * 60_000; - return { status: 0, stdout: manifest, stderr: Buffer.alloc(0) }; - }, + }), }), ), ).rejects.toMatchObject({ code: "QUALIFICATION_TIMEOUT" }); diff --git a/test/helpers/exact-image-qualification-controller-fixture.ts b/test/helpers/exact-image-qualification-controller-fixture.ts new file mode 100644 index 0000000000..f204c72fa6 --- /dev/null +++ b/test/helpers/exact-image-qualification-controller-fixture.ts @@ -0,0 +1,249 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { vi } from "vitest"; + +import { + type ExactImageQualificationRequest, + MANIFEST_ARTIFACT_FILE, + PRODUCER_REPOSITORY, + PRODUCER_WORKFLOW_FILE, + PRODUCER_WORKFLOW_PATH, + type QualificationDependencies, + startExactImageQualification, +} from "../../tools/e2e/exact-image-qualification-controller.mts"; + +export const CANDIDATE_SHA = "a".repeat(40); +export const PRODUCER_SHA = "b".repeat(40); +export const CORRELATION_ID = "123e4567-e89b-42d3-a456-426614174000"; +export const RUN_ID = "24680"; +export const WORKFLOW_ID = 13579; +export const BASE_TIME = Date.UTC(2026, 0, 1); +export const API_RUN_URL = `https://api.github.com/repos/${PRODUCER_REPOSITORY}/actions/runs/${RUN_ID}`; +export const HTML_RUN_URL = `https://github.com/${PRODUCER_REPOSITORY}/actions/runs/${RUN_ID}`; + +export const REQUEST: ExactImageQualificationRequest = { + actor: "maintainer", + candidateSha: CANDIDATE_SHA, + eventName: "workflow_dispatch", + reason: "Qualify the current daily candidate before tagging", + ref: "refs/heads/main", + requesterRunAttempt: 1, + requesterRunId: "97531", + workflowSha: CANDIDATE_SHA, +}; + +type ApiOptions = { + artifacts?: unknown; + dispatch?: unknown; + permission?: string; + roleName?: string; + producerSha?: string; + requesterSha?: string; + run?: unknown; + runs?: unknown; + workflow?: unknown; +}; + +type ApiRouteHandler = ( + apiPath: string, + token: string, + requestOptions?: unknown, +) => unknown | Promise; + +type ApiRoute = { + matches: (apiPath: string) => boolean; + respond: ApiRouteHandler; +}; + +function mainRef(sha: string) { + return { ref: "refs/heads/main", object: { type: "commit", sha } }; +} + +export function dispatchDetails() { + return { + workflow_run_id: Number(RUN_ID), + run_url: API_RUN_URL, + html_url: HTML_RUN_URL, + }; +} + +export function workflowRun(overrides: Record = {}) { + return { + id: Number(RUN_ID), + workflow_id: WORKFLOW_ID, + run_attempt: 1, + event: "workflow_dispatch", + head_branch: "main", + head_sha: PRODUCER_SHA, + path: PRODUCER_WORKFLOW_PATH, + display_title: `Qualify NemoClaw ${CANDIDATE_SHA} (${CORRELATION_ID})`, + url: API_RUN_URL, + html_url: HTML_RUN_URL, + repository: { full_name: PRODUCER_REPOSITORY }, + head_repository: { full_name: PRODUCER_REPOSITORY }, + status: "queued", + conclusion: null, + created_at: new Date(BASE_TIME).toISOString(), + ...overrides, + }; +} + +export function createApi(options: ApiOptions = {}) { + return vi.fn(async (apiPath: string, _token: string, requestOptions?: unknown) => { + if (apiPath === "repos/NVIDIA/NemoClaw/git/ref/heads/main") { + return mainRef(options.requesterSha ?? CANDIDATE_SHA); + } + if (apiPath.includes("/collaborators/")) { + return { + permission: options.permission ?? "write", + role_name: options.roleName ?? "maintain", + }; + } + if (apiPath === `repos/${PRODUCER_REPOSITORY}/git/ref/heads/main`) { + return mainRef(options.producerSha ?? PRODUCER_SHA); + } + if (apiPath === `repos/${PRODUCER_REPOSITORY}/actions/workflows/${PRODUCER_WORKFLOW_FILE}`) { + return ( + options.workflow ?? { + id: WORKFLOW_ID, + path: PRODUCER_WORKFLOW_PATH, + state: "active", + } + ); + } + if ( + apiPath === + `repos/${PRODUCER_REPOSITORY}/actions/workflows/${PRODUCER_WORKFLOW_FILE}/dispatches` + ) { + return options.dispatch === undefined ? dispatchDetails() : options.dispatch; + } + if (apiPath === `repos/${PRODUCER_REPOSITORY}/actions/runs/${RUN_ID}`) { + return options.run ?? workflowRun(); + } + if (apiPath === `repos/${PRODUCER_REPOSITORY}/actions/runs/${RUN_ID}/cancel`) { + return undefined; + } + if ( + apiPath.startsWith( + `repos/${PRODUCER_REPOSITORY}/actions/workflows/${PRODUCER_WORKFLOW_FILE}/runs?`, + ) + ) { + return options.runs ?? { total_count: 0, workflow_runs: [] }; + } + if (apiPath === `repos/${PRODUCER_REPOSITORY}/actions/runs/${RUN_ID}/artifacts?per_page=100`) { + return options.artifacts; + } + throw new Error(`unexpected API call ${apiPath} ${JSON.stringify(requestOptions)}`); + }); +} + +function route(matches: (apiPath: string) => boolean, respond: ApiRouteHandler): ApiRoute { + return { matches, respond }; +} + +export const qualificationApiRoute = { + dispatch: (respond: ApiRouteHandler): ApiRoute => + route( + (apiPath) => apiPath.endsWith(`/workflows/${PRODUCER_WORKFLOW_FILE}/dispatches`), + respond, + ), + workflowRuns: (respond: ApiRouteHandler): ApiRoute => + route((apiPath) => apiPath.includes(`/workflows/${PRODUCER_WORKFLOW_FILE}/runs?`), respond), + run: (respond: ApiRouteHandler): ApiRoute => + route((apiPath) => apiPath === `repos/${PRODUCER_REPOSITORY}/actions/runs/${RUN_ID}`, respond), + cancelRun: (respond: ApiRouteHandler): ApiRoute => + route( + (apiPath) => apiPath === `repos/${PRODUCER_REPOSITORY}/actions/runs/${RUN_ID}/cancel`, + respond, + ), + cancelAnyRun: (respond: ApiRouteHandler): ApiRoute => + route((apiPath) => apiPath.endsWith("/cancel"), respond), +}; + +export function createRoutedApi(base: ReturnType, routes: readonly ApiRoute[]) { + return vi.fn(async (apiPath: string, token: string, requestOptions?: unknown) => { + for (const candidate of routes) { + if (candidate.matches(apiPath)) { + return candidate.respond(apiPath, token, requestOptions); + } + } + return base(apiPath, token, requestOptions); + }); +} + +export function dependencies( + api: ReturnType, + extra: QualificationDependencies = {}, +) { + return { + now: () => BASE_TIME, + ...extra, + api: api as QualificationDependencies["api"], + randomUuid: () => CORRELATION_ID, + }; +} + +export function tempDirectory(): string { + return fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-image-qualification-test-")); +} + +export async function startedState(api = createApi()) { + const workDir = tempDirectory(); + const state = await startExactImageQualification( + { + request: REQUEST, + coreToken: "core-token", + producerToken: "producer-token", + workDir, + }, + dependencies(api), + ); + return { api, state, workDir }; +} + +export function artifactList(archive: Buffer, overrides: Record = {}) { + const artifactId = 86420; + return { + total_count: 1, + artifacts: [ + { + id: artifactId, + name: `nemoclaw-image-handoff-v1-${RUN_ID}-1`, + expired: false, + digest: `sha256:${createHash("sha256").update(archive).digest("hex")}`, + size_in_bytes: archive.length, + url: `https://api.github.com/repos/${PRODUCER_REPOSITORY}/actions/artifacts/${artifactId}`, + archive_download_url: `https://api.github.com/repos/${PRODUCER_REPOSITORY}/actions/artifacts/${artifactId}/zip`, + workflow_run: { id: Number(RUN_ID), head_sha: PRODUCER_SHA }, + ...overrides, + }, + ], + }; +} + +export function createArchiveRunCommand(manifest: Buffer, onExtract: () => void = () => {}) { + return (_command: string, args: readonly string[]) => { + if (args[0] === "-Z1") { + return { + status: 0, + stdout: Buffer.from(`${MANIFEST_ARTIFACT_FILE}\n`), + stderr: Buffer.alloc(0), + }; + } + if (args[0] === "-Zl") { + return { + status: 0, + stdout: Buffer.from(`-rw-r--r-- 3.0 unx 20 tx 20 stor ${MANIFEST_ARTIFACT_FILE}\n`), + stderr: Buffer.alloc(0), + }; + } + onExtract(); + return { status: 0, stdout: manifest, stderr: Buffer.alloc(0) }; + }; +} From 47cfafdc4ec9f4581d8e76106493755277bd6442 Mon Sep 17 00:00:00 2001 From: Charan Jagwani Date: Thu, 16 Jul 2026 12:08:08 -0700 Subject: [PATCH 03/25] fix(e2e): preserve corrupt qualification state Signed-off-by: Charan Jagwani --- ...act-image-qualification-controller.test.ts | 30 +++++++++++++++++++ ...-image-qualification-controller-fixture.ts | 26 ++++++++++++++++ .../exact-image-qualification-controller.mts | 6 ++-- 3 files changed, 58 insertions(+), 4 deletions(-) diff --git a/test/exact-image-qualification-controller.test.ts b/test/exact-image-qualification-controller.test.ts index 11441ab786..37fa129a3c 100644 --- a/test/exact-image-qualification-controller.test.ts +++ b/test/exact-image-qualification-controller.test.ts @@ -48,6 +48,7 @@ import { createRoutedApi, dependencies, dispatchDetails, + dispatchIntent, HTML_RUN_URL, PRODUCER_SHA, qualificationApiRoute, @@ -587,6 +588,35 @@ describe("exact producer dispatch binding", () => { } }); + it("fails closed when unreadable state cannot be preserved before cleanup reconciliation", async () => { + const workDir = tempDirectory(); + const controllerState = "not valid JSON"; + const api = createApi(); + try { + fs.writeFileSync( + path.join(workDir, DISPATCH_INTENT_FILE), + `${JSON.stringify(dispatchIntent(), null, 2)}\n`, + { mode: 0o600 }, + ); + fs.writeFileSync(path.join(workDir, STATE_FILE), controllerState, { mode: 0o600 }); + vi.spyOn(fs, "renameSync").mockImplementation(() => { + throw new Error("preservation denied"); + }); + + await expect( + cancelActiveExactImageQualification( + { workDir, producerToken: "producer-token" }, + dependencies(api), + ), + ).rejects.toMatchObject({ code: "OUTPUT_WRITE_FAILED" }); + expect(api).not.toHaveBeenCalled(); + expect(fs.readFileSync(path.join(workDir, STATE_FILE), "utf8")).toBe(controllerState); + expect(fs.existsSync(path.join(workDir, DISPATCH_RECONCILIATION_FILE))).toBe(false); + } finally { + fs.rmSync(workDir, { recursive: true, force: true }); + } + }); + it("fails closed on multiple strict reconciliation matches and records every exact ID", async () => { const workDir = tempDirectory(); const secondId = "24681"; diff --git a/test/helpers/exact-image-qualification-controller-fixture.ts b/test/helpers/exact-image-qualification-controller-fixture.ts index f204c72fa6..90b133d564 100644 --- a/test/helpers/exact-image-qualification-controller-fixture.ts +++ b/test/helpers/exact-image-qualification-controller-fixture.ts @@ -9,8 +9,10 @@ import path from "node:path"; import { vi } from "vitest"; import { + type ExactImageDispatchIntent, type ExactImageQualificationRequest, MANIFEST_ARTIFACT_FILE, + PRODUCER_REF, PRODUCER_REPOSITORY, PRODUCER_WORKFLOW_FILE, PRODUCER_WORKFLOW_PATH, @@ -38,6 +40,30 @@ export const REQUEST: ExactImageQualificationRequest = { workflowSha: CANDIDATE_SHA, }; +export function dispatchIntent(): ExactImageDispatchIntent { + return { + schemaVersion: 1, + kind: "nemoclaw-exact-image-dispatch-intent", + requestStartedAt: new Date(BASE_TIME).toISOString(), + request: { + actor: REQUEST.actor, + candidateSha: REQUEST.candidateSha, + correlationId: CORRELATION_ID, + reason: REQUEST.reason, + requesterRunAttempt: REQUEST.requesterRunAttempt, + requesterRunId: REQUEST.requesterRunId, + workflowSha: REQUEST.workflowSha, + }, + producer: { + repository: PRODUCER_REPOSITORY, + repositorySha: PRODUCER_SHA, + ref: PRODUCER_REF, + workflowId: String(WORKFLOW_ID), + workflowPath: PRODUCER_WORKFLOW_PATH, + }, + }; +} + type ApiOptions = { artifacts?: unknown; dispatch?: unknown; diff --git a/tools/e2e/exact-image-qualification-controller.mts b/tools/e2e/exact-image-qualification-controller.mts index da75f2be66..0b02215cbe 100755 --- a/tools/e2e/exact-image-qualification-controller.mts +++ b/tools/e2e/exact-image-qualification-controller.mts @@ -667,10 +667,8 @@ function preserveUnreadableState(workDir: string): void { ); try { fs.renameSync(file, preserved); - } catch (error) { - console.warn( - `Could not preserve unreadable controller state before intent recovery: ${error instanceof Error ? error.message : "unknown error"}`, - ); + } catch { + fail("OUTPUT_WRITE_FAILED", "unreadable controller state could not be preserved safely"); } } From 9a74662b6c28b12d5a79e81795a55ab36860391c Mon Sep 17 00:00:00 2001 From: Charan Jagwani Date: Thu, 16 Jul 2026 12:44:50 -0700 Subject: [PATCH 04/25] fix(e2e): validate qualification state provenance Signed-off-by: Charan Jagwani --- test/e2e/support/exact-image-manifest.test.ts | 21 + ...act-image-qualification-controller.test.ts | 329 ++++++++++- ...-image-qualification-controller-fixture.ts | 5 + tools/e2e/exact-image-manifest.mts | 45 +- .../exact-image-qualification-controller.mts | 533 ++++++++++++++++-- 5 files changed, 846 insertions(+), 87 deletions(-) diff --git a/test/e2e/support/exact-image-manifest.test.ts b/test/e2e/support/exact-image-manifest.test.ts index c3d4d0eecb..1b784bba7a 100644 --- a/test/e2e/support/exact-image-manifest.test.ts +++ b/test/e2e/support/exact-image-manifest.test.ts @@ -130,9 +130,13 @@ describe("exact staging image manifest consumer", () => { }); it("requires an immutable self-link reconstructed from project and image name", () => { + const canonical = exactImageManifest().imageSelfLink; for (const imageSelfLink of [ "https://www.googleapis.com/compute/v1/projects/brevdevprod/global/images/family/nemoclaw-brev-staging-cpu", "https://www.googleapis.com/compute/v1/projects/other/global/images/nemoclaw-brev-cpu-v0-1-0-20260716-a-staging-190-1", + `${canonical}?alt=json`, + `${canonical}#fragment`, + `${canonical}\n`, ]) { expectCode( () => @@ -145,6 +149,23 @@ describe("exact staging image manifest consumer", () => { } }); + it("rejects noncanonical GCP project identifiers even when the self-link repeats them", () => { + for (const project of ["short", "-leading", "trailing-", "UPPERCASE", "evil?x=y#z\nproject"]) { + const imageName = exactImageManifest().imageName; + expectCode( + () => + validateExactImageManifest( + exactImageManifest({ + project, + imageSelfLink: `https://www.googleapis.com/compute/v1/projects/${project}/global/images/${imageName}`, + }), + exactImageManifestExpectations(), + ), + "IMAGE_IDENTITY_MISMATCH", + ); + } + }); + it("binds the manifest to the trusted request and producer run", () => { const expectationMismatches = [ { nemoclawSha: "c".repeat(40) }, diff --git a/test/exact-image-qualification-controller.test.ts b/test/exact-image-qualification-controller.test.ts index 37fa129a3c..c341d1bc21 100644 --- a/test/exact-image-qualification-controller.test.ts +++ b/test/exact-image-qualification-controller.test.ts @@ -175,6 +175,32 @@ describe("exact producer dispatch binding", () => { } }); + it("keeps the previous state intact when atomic replacement is interrupted", async () => { + const workDir = tempDirectory(); + const previousState = "previous state evidence\n"; + fs.writeFileSync(path.join(workDir, STATE_FILE), previousState, { mode: 0o600 }); + vi.spyOn(fs, "renameSync").mockImplementation(() => { + throw new Error("simulated interruption before atomic replacement"); + }); + try { + await expect( + startExactImageQualification( + { + request: REQUEST, + coreToken: "core-token", + producerToken: "producer-token", + workDir, + }, + dependencies(createApi()), + ), + ).rejects.toMatchObject({ code: "OUTPUT_WRITE_FAILED" }); + expect(fs.readFileSync(path.join(workDir, STATE_FILE), "utf8")).toBe(previousState); + expect(fs.readdirSync(workDir).filter((name) => name.endsWith(".tmp"))).toEqual([]); + } finally { + fs.rmSync(workDir, { recursive: true, force: true }); + } + }); + it("requires HTTP 200 and sends the explicit API-version header", async () => { const fetchMock = vi.fn( async (_input: string | URL | Request, _init?: RequestInit) => @@ -280,7 +306,7 @@ describe("exact producer dispatch binding", () => { } }); - it("finds the current correlation in a bounded creation window despite history and producer drift", async () => { + it("finds and cleans the exact correlation after producer drift without qualifying it", async () => { const workDir = tempDirectory(); const movedSha = "c".repeat(40); const base = createApi(); @@ -337,9 +363,9 @@ describe("exact producer dispatch binding", () => { expect(observedHeadShaFilter).toBe(false); expect(historicalRuns).toHaveLength(101); expect(observedScopedRuns).toEqual([movedRun]); - expect(readExactImageQualificationState(workDir).producer).toMatchObject({ - runId: RUN_ID, - repositorySha: movedSha, + expect(readExactImageQualificationState(workDir)).toMatchObject({ + status: "dispatched", + producer: { runId: RUN_ID, repositorySha: movedSha }, }); expect( JSON.parse(fs.readFileSync(path.join(workDir, DISPATCH_RECONCILIATION_FILE), "utf8")), @@ -348,12 +374,13 @@ describe("exact producer dispatch binding", () => { runIds: [RUN_ID], producerHeadShas: { [RUN_ID]: movedSha }, }); + expect(fs.existsSync(path.join(workDir, EVIDENCE_FILE))).toBe(false); } finally { fs.rmSync(workDir, { recursive: true, force: true }); } }); - it("uses the response-bound run ID for cleanup after strict normal-path provenance fails", async () => { + it("cleans the response-bound run after producer SHA provenance fails", async () => { const workDir = tempDirectory(); const movedSha = "c".repeat(40); try { @@ -392,9 +419,11 @@ describe("exact producer dispatch binding", () => { ), ).resolves.toBe(true); expect(cancelled).toBe(true); - expect(cleanupApi.mock.calls[0]?.[0]).toBe( - `repos/${PRODUCER_REPOSITORY}/actions/runs/${RUN_ID}/cancel`, - ); + expect( + cleanupApi.mock.calls.filter(([apiPath]) => String(apiPath).endsWith("/cancel")), + ).toHaveLength(1); + expect(readExactImageQualificationState(workDir).status).toBe("dispatched"); + expect(fs.existsSync(path.join(workDir, EVIDENCE_FILE))).toBe(false); } finally { fs.rmSync(workDir, { recursive: true, force: true }); } @@ -527,8 +556,9 @@ describe("exact producer dispatch binding", () => { } }); - it("lets always-run cleanup recover and cancel a run that appeared after start reconciliation", async () => { + it("lets cleanup preserve semantically invalid state and recover only from durable intent", async () => { const workDir = tempDirectory(); + const untrustedRunId = "99999"; let startClock = BASE_TIME; try { await expect( @@ -552,7 +582,15 @@ describe("exact producer dispatch binding", () => { ), ).rejects.toMatchObject({ code: "DISPATCH_AMBIGUOUS" }); expect(fs.existsSync(path.join(workDir, STATE_FILE))).toBe(false); - fs.writeFileSync(path.join(workDir, STATE_FILE), "", { mode: 0o600 }); + fs.writeFileSync( + path.join(workDir, STATE_FILE), + JSON.stringify({ + schemaVersion: 1, + status: "dispatched", + producer: { runId: untrustedRunId }, + }), + { mode: 0o600 }, + ); const base = createApi(); let cancelled = false; @@ -579,6 +617,9 @@ describe("exact producer dispatch binding", () => { ), ).resolves.toBe(true); expect(cancelled).toBe(true); + expect( + cleanupApi.mock.calls.some(([apiPath]) => String(apiPath).includes(untrustedRunId)), + ).toBe(false); expect(readExactImageQualificationState(workDir).producer.runId).toBe(RUN_ID); expect( fs.readdirSync(workDir).some((name) => name.startsWith("controller-state.corrupt-")), @@ -588,6 +629,134 @@ describe("exact producer dispatch binding", () => { } }); + it("binds a valid state tuple to durable intent before any cancellation POST", async () => { + const started = await startedState(); + const untrustedRunId = "99999"; + const untrustedCorrelationId = "87654321-4321-4321-8321-cba987654321"; + const state = readExactImageQualificationState(started.workDir); + state.request.correlationId = untrustedCorrelationId; + state.producer.runId = untrustedRunId; + state.producer.runUrl = `https://api.github.com/repos/${PRODUCER_REPOSITORY}/actions/runs/${untrustedRunId}`; + state.producer.htmlUrl = `https://github.com/${PRODUCER_REPOSITORY}/actions/runs/${untrustedRunId}`; + fs.writeFileSync(path.join(started.workDir, STATE_FILE), `${JSON.stringify(state)}\n`, { + mode: 0o600, + }); + + const base = createApi(); + let cancelled = false; + const cleanupApi = createRoutedApi(base, [ + qualificationApiRoute.workflowRuns(() => ({ + total_count: 1, + workflow_runs: [workflowRun()], + })), + qualificationApiRoute.cancelRun(() => { + cancelled = true; + return undefined; + }), + qualificationApiRoute.run(() => + cancelled ? workflowRun({ status: "completed", conclusion: "cancelled" }) : workflowRun(), + ), + ]); + try { + await expect( + cancelActiveExactImageQualification( + { workDir: started.workDir, producerToken: "producer-token" }, + dependencies(cleanupApi as ReturnType), + ), + ).resolves.toBe(true); + expect(cancelled).toBe(true); + expect( + cleanupApi.mock.calls.some(([apiPath]) => String(apiPath).includes(untrustedRunId)), + ).toBe(false); + expect(readExactImageQualificationState(started.workDir).producer.runId).toBe(RUN_ID); + expect( + fs + .readdirSync(started.workDir) + .some((name) => name.startsWith("controller-state.corrupt-")), + ).toBe(true); + } finally { + fs.rmSync(started.workDir, { recursive: true, force: true }); + } + }); + + it("does not let a forged terminal state suppress cleanup of the active bound run", async () => { + const started = await startedState(); + const state = readExactImageQualificationState(started.workDir); + state.status = "completed"; + state.producer.completedAt = new Date(BASE_TIME + 1_000).toISOString(); + fs.writeFileSync(path.join(started.workDir, STATE_FILE), `${JSON.stringify(state)}\n`, { + mode: 0o600, + }); + + let cancelled = false; + const cleanupApi = createRoutedApi(createApi(), [ + qualificationApiRoute.cancelRun(() => { + cancelled = true; + return undefined; + }), + qualificationApiRoute.run(() => + cancelled + ? workflowRun({ status: "completed", conclusion: "cancelled" }) + : workflowRun({ status: "queued", conclusion: null }), + ), + ]); + try { + await expect( + cancelActiveExactImageQualification( + { workDir: started.workDir, producerToken: "producer-token" }, + dependencies(cleanupApi as ReturnType), + ), + ).resolves.toBe(true); + expect(cancelled).toBe(true); + expect( + cleanupApi.mock.calls.filter(([apiPath]) => String(apiPath).endsWith("/cancel")), + ).toHaveLength(1); + } finally { + fs.rmSync(started.workDir, { recursive: true, force: true }); + } + }); + + it("rejects run-ID-only state corruption before any cancellation POST", async () => { + const started = await startedState(); + const untrustedRunId = "99999"; + const untrustedApiUrl = `https://api.github.com/repos/${PRODUCER_REPOSITORY}/actions/runs/${untrustedRunId}`; + const untrustedHtmlUrl = `https://github.com/${PRODUCER_REPOSITORY}/actions/runs/${untrustedRunId}`; + const state = readExactImageQualificationState(started.workDir); + state.producer.runId = untrustedRunId; + state.producer.runUrl = untrustedApiUrl; + state.producer.htmlUrl = untrustedHtmlUrl; + fs.writeFileSync(path.join(started.workDir, STATE_FILE), `${JSON.stringify(state)}\n`, { + mode: 0o600, + }); + + const cancelAttempt = vi.fn(); + const cleanupApi = createRoutedApi(createApi(), [ + qualificationApiRoute.runAny(() => + workflowRun({ + id: Number(untrustedRunId), + display_title: "Unrelated qualification run", + url: untrustedApiUrl, + html_url: untrustedHtmlUrl, + }), + ), + qualificationApiRoute.cancelAnyRun(cancelAttempt), + ]); + try { + await expect( + cancelActiveExactImageQualification( + { workDir: started.workDir, producerToken: "producer-token" }, + dependencies(cleanupApi as ReturnType), + ), + ).rejects.toMatchObject({ code: "PROVENANCE_MISMATCH" }); + expect(cancelAttempt).not.toHaveBeenCalled(); + expect(cleanupApi.mock.calls.map(([apiPath]) => apiPath)).toEqual([ + `repos/${PRODUCER_REPOSITORY}/actions/runs/${untrustedRunId}`, + ]); + } finally { + fs.rmSync(started.workDir, { recursive: true, force: true }); + } + }); + it("fails closed when unreadable state cannot be preserved before cleanup reconciliation", async () => { const workDir = tempDirectory(); const controllerState = "not valid JSON"; @@ -726,6 +895,70 @@ describe("bound producer polling", () => { } }); + it("rejects repeated wait from a terminal state before polling or mutation", async () => { + const started = await startedState(); + const state = readExactImageQualificationState(started.workDir); + state.status = "completed"; + state.producer.completedAt = new Date(BASE_TIME + 1_000).toISOString(); + fs.writeFileSync(path.join(started.workDir, STATE_FILE), `${JSON.stringify(state)}\n`, { + mode: 0o600, + }); + const api = createApi(); + try { + await expect( + waitForExactImageQualification( + { workDir: started.workDir, producerToken: "producer-token" }, + dependencies(api), + ), + ).rejects.toMatchObject({ code: "PROVENANCE_MISMATCH" }); + expect(api).not.toHaveBeenCalled(); + expect(readExactImageQualificationState(started.workDir)).toEqual(state); + } finally { + fs.rmSync(started.workDir, { recursive: true, force: true }); + } + }); + + it("rejects persisted completion and acceptance after the hard deadline", async () => { + const started = await startedState(); + const state = readExactImageQualificationState(started.workDir); + state.status = "completed"; + state.producer.completedAt = new Date(BASE_TIME + 45 * 60_000 + 1).toISOString(); + fs.writeFileSync(path.join(started.workDir, STATE_FILE), `${JSON.stringify(state)}\n`, { + mode: 0o600, + }); + try { + expect(() => readExactImageQualificationState(started.workDir)).toThrow( + /completedAt exceeds qualification deadline/u, + ); + + state.status = "validated"; + state.producer.completedAt = new Date(BASE_TIME + 1_000).toISOString(); + state.artifact = { + id: "86420", + name: `nemoclaw-image-handoff-v1-${RUN_ID}-1`, + digest: `sha256:${"0".repeat(64)}`, + sizeInBytes: 1, + apiUrl: `https://api.github.com/repos/${PRODUCER_REPOSITORY}/actions/artifacts/86420`, + archiveDownloadUrl: `https://api.github.com/repos/${PRODUCER_REPOSITORY}/actions/artifacts/86420/zip`, + archiveSha256: "0".repeat(64), + manifestSha256: "1".repeat(64), + }; + state.validation = { + acceptedAt: new Date(BASE_TIME + 45 * 60_000 + 1).toISOString(), + manifestSha256: "1".repeat(64), + normalizedManifestSha256: "2".repeat(64), + }; + fs.writeFileSync(path.join(started.workDir, STATE_FILE), `${JSON.stringify(state)}\n`, { + mode: 0o600, + }); + expect(() => readExactImageQualificationState(started.workDir)).toThrow( + /acceptedAt exceeds qualification deadline/u, + ); + } finally { + fs.rmSync(started.workDir, { recursive: true, force: true }); + } + }); + it("cancels a run that remains queued beyond the queue budget", async () => { const { workDir } = await startedState(); const api = createApi({ run: workflowRun({ status: "queued" }) }); @@ -839,6 +1072,41 @@ describe("qualification artifact integrity", () => { } }); + it("rejects persisted artifact metadata whose GitHub digest and archive hash diverge", async () => { + const archive = Buffer.from("deterministic archive bytes"); + const manifest = Buffer.from('{"schemaVersion":1}\n'); + const started = await startedState(); + await waitForExactImageQualification( + { workDir: started.workDir, producerToken: "producer-token" }, + dependencies( + createApi({ run: workflowRun({ status: "completed", conclusion: "success" }) }), + { now: () => BASE_TIME + 1_000 }, + ), + ); + await downloadExactImageManifest( + { workDir: started.workDir, producerToken: "producer-token" }, + dependencies(createApi({ artifacts: artifactList(archive) }), { + fetch: async () => new Response(archive, { status: 200 }), + runCommand: createArchiveRunCommand(manifest), + }), + ); + const state = JSON.parse( + fs.readFileSync(path.join(started.workDir, STATE_FILE), "utf8"), + ) as Record & { artifact: { archiveSha256: string } }; + state.artifact.archiveSha256 = "0".repeat(64); + fs.writeFileSync(path.join(started.workDir, STATE_FILE), `${JSON.stringify(state)}\n`, { + mode: 0o600, + }); + try { + expect(() => readExactImageQualificationState(started.workDir)).toThrow( + /digest does not match archive hash/u, + ); + expect(fs.existsSync(path.join(started.workDir, EVIDENCE_FILE))).toBe(false); + } finally { + fs.rmSync(started.workDir, { recursive: true, force: true }); + } + }); + it("does not inspect or extract an archive whose digest mismatches GitHub metadata", async () => { const archive = Buffer.from("archive bytes"); const started = await startedState(); @@ -1046,6 +1314,41 @@ describe("qualification evidence finalization", () => { } }); + it("never publishes accepted evidence when the validated state transition is rejected", async () => { + const archive = Buffer.from("archive"); + const manifest = Buffer.from('{"schemaVersion":1}\n'); + const normalized = Buffer.from('{"schemaVersion":1,"accepted":true}\n'); + const started = await startedState(); + await waitForExactImageQualification( + { workDir: started.workDir, producerToken: "producer-token" }, + dependencies( + createApi({ run: workflowRun({ status: "completed", conclusion: "success" }) }), + { now: () => BASE_TIME + 3_000 }, + ), + ); + await downloadExactImageManifest( + { workDir: started.workDir, producerToken: "producer-token" }, + dependencies(createApi({ artifacts: artifactList(archive) }), { + fetch: async () => new Response(archive, { status: 200 }), + runCommand: createArchiveRunCommand(manifest), + }), + ); + fs.writeFileSync(path.join(started.workDir, VALIDATED_MANIFEST_FILE), normalized, { + mode: 0o600, + }); + try { + expect(() => + finalizeExactImageQualification(started.workDir, { + now: () => BASE_TIME + 2_000, + }), + ).toThrowError(expect.objectContaining({ code: "OUTPUT_WRITE_FAILED" })); + expect(fs.existsSync(path.join(started.workDir, EVIDENCE_FILE))).toBe(false); + expect(readExactImageQualificationState(started.workDir).status).toBe("downloaded"); + } finally { + fs.rmSync(started.workDir, { recursive: true, force: true }); + } + }); + it("refuses accepted evidence at the shared deadline", () => { const workDir = tempDirectory(); const manifest = Buffer.from('{"schemaVersion":1}\n'); @@ -1074,6 +1377,7 @@ describe("qualification evidence finalization", () => { workflowId: String(WORKFLOW_ID), runUrl: API_RUN_URL, htmlUrl: HTML_RUN_URL, + completedAt: new Date(BASE_TIME + 1_000).toISOString(), }, artifact: { id: "86420", @@ -1086,6 +1390,11 @@ describe("qualification evidence finalization", () => { manifestSha256, }, }; + fs.writeFileSync( + path.join(workDir, DISPATCH_INTENT_FILE), + `${JSON.stringify(dispatchIntent(), null, 2)}\n`, + { mode: 0o600 }, + ); fs.writeFileSync(path.join(workDir, STATE_FILE), `${JSON.stringify(state)}\n`, { mode: 0o600 }); fs.writeFileSync(path.join(workDir, MANIFEST_ARTIFACT_FILE), manifest, { mode: 0o600 }); fs.writeFileSync(path.join(workDir, VALIDATED_MANIFEST_FILE), normalized, { mode: 0o600 }); diff --git a/test/helpers/exact-image-qualification-controller-fixture.ts b/test/helpers/exact-image-qualification-controller-fixture.ts index 90b133d564..c77d2fa134 100644 --- a/test/helpers/exact-image-qualification-controller-fixture.ts +++ b/test/helpers/exact-image-qualification-controller-fixture.ts @@ -183,6 +183,11 @@ export const qualificationApiRoute = { route((apiPath) => apiPath.includes(`/workflows/${PRODUCER_WORKFLOW_FILE}/runs?`), respond), run: (respond: ApiRouteHandler): ApiRoute => route((apiPath) => apiPath === `repos/${PRODUCER_REPOSITORY}/actions/runs/${RUN_ID}`, respond), + runAny: (respond: ApiRouteHandler): ApiRoute => + route( + (apiPath) => /^repos\/brevdev\/nemoclaw-image\/actions\/runs\/[1-9][0-9]*$/u.test(apiPath), + respond, + ), cancelRun: (respond: ApiRouteHandler): ApiRoute => route( (apiPath) => apiPath === `repos/${PRODUCER_REPOSITORY}/actions/runs/${RUN_ID}/cancel`, diff --git a/tools/e2e/exact-image-manifest.mts b/tools/e2e/exact-image-manifest.mts index db4a5f5214..8ee007a411 100644 --- a/tools/e2e/exact-image-manifest.mts +++ b/tools/e2e/exact-image-manifest.mts @@ -102,9 +102,8 @@ const REQUIRED_FIELD_SET = new Set(REQUIRED_FIELDS); const FULL_SHA_PATTERN = /^[0-9a-f]{40}$/u; const UUID_V4_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/u; const DECIMAL_ID_PATTERN = /^[1-9][0-9]*$/u; +const GCP_PROJECT_ID_PATTERN = /^[a-z][a-z0-9-]{4,28}[a-z0-9]$/u; const IMAGE_NAME_PATTERN = /^[a-z](?:[-a-z0-9]{0,61}[a-z0-9])?$/u; -const IMAGE_SELF_LINK_PATTERN = - /^https:\/\/www[.]googleapis[.]com\/compute\/v1\/projects\/[^/]+\/global\/images\/[^/]+$/u; const RFC3339_PATTERN = /^(\d{4})-(\d{2})-(\d{2})T([01]\d|2[0-3]):([0-5]\d):([0-5]\d)(?:[.](\d{1,9}))?(Z|[+-](?:[01]\d|2[0-3]):[0-5]\d)$/u; @@ -155,12 +154,6 @@ function requirePattern(value: string, field: string, pattern: RegExp): void { } } -function requireIdentityPattern(value: string, field: string, pattern: RegExp): void { - if (!pattern.test(value)) { - fail("IMAGE_IDENTITY_MISMATCH", `${field} has an invalid format`); - } -} - function requireConstant( actual: unknown, expected: T, @@ -321,15 +314,38 @@ function buildManifest(record: Record): ExactImageManifest { requirePattern(manifest.imageRepositorySha, "imageRepositorySha", FULL_SHA_PATTERN); requirePattern(manifest.workflowRunId, "workflowRunId", DECIMAL_ID_PATTERN); requirePattern(manifest.imageOriginWorkflowRunId, "imageOriginWorkflowRunId", DECIMAL_ID_PATTERN); + if (!GCP_PROJECT_ID_PATTERN.test(manifest.project)) { + fail("IMAGE_IDENTITY_MISMATCH", "project must be a canonical GCP project ID"); + } requirePattern(manifest.imageName, "imageName", IMAGE_NAME_PATTERN); requirePattern(manifest.imageId, "imageId", DECIMAL_ID_PATTERN); - requireIdentityPattern(manifest.imageSelfLink, "imageSelfLink", IMAGE_SELF_LINK_PATTERN); - if (manifest.project.length === 0) { - fail("ARTIFACT_MISSING_OR_INVALID", "project must not be empty"); - } return manifest; } +function validateImageSelfLink(manifest: ExactImageManifest): void { + const expectedPath = `/compute/v1/projects/${manifest.project}/global/images/${manifest.imageName}`; + const expectedSelfLink = `https://www.googleapis.com${expectedPath}`; + let parsed: URL; + try { + parsed = new URL(manifest.imageSelfLink); + } catch { + fail("IMAGE_IDENTITY_MISMATCH", "imageSelfLink must be an absolute URL"); + } + if ( + parsed.protocol !== "https:" || + parsed.hostname !== "www.googleapis.com" || + parsed.port !== "" || + parsed.username !== "" || + parsed.password !== "" || + parsed.search !== "" || + parsed.hash !== "" || + parsed.pathname !== expectedPath || + manifest.imageSelfLink !== expectedSelfLink + ) { + fail("IMAGE_IDENTITY_MISMATCH", "imageSelfLink does not exactly identify project/imageName"); + } +} + export function parseExactImageManifestJson(contents: string): unknown { try { return JSON.parse(contents) as unknown; @@ -347,10 +363,7 @@ export function validateExactImageManifest( validateExactFields(record); const manifest = buildManifest(record); - const expectedSelfLink = `https://www.googleapis.com/compute/v1/projects/${manifest.project}/global/images/${manifest.imageName}`; - if (manifest.imageSelfLink !== expectedSelfLink) { - fail("IMAGE_IDENTITY_MISMATCH", "imageSelfLink does not exactly match project and imageName"); - } + validateImageSelfLink(manifest); if ( manifest.result === "built" && (manifest.imageOriginWorkflowRunId !== manifest.workflowRunId || diff --git a/tools/e2e/exact-image-qualification-controller.mts b/tools/e2e/exact-image-qualification-controller.mts index 0b02215cbe..e36f5f0225 100755 --- a/tools/e2e/exact-image-qualification-controller.mts +++ b/tools/e2e/exact-image-qualification-controller.mts @@ -37,6 +37,7 @@ const FULL_SHA_PATTERN = /^[0-9a-f]{40}$/u; const UUID_V4_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/u; const DECIMAL_ID_PATTERN = /^[1-9][0-9]*$/u; const ARTIFACT_DIGEST_PATTERN = /^sha256:[0-9a-f]{64}$/u; +const SHA256_PATTERN = /^[0-9a-f]{64}$/u; const GITHUB_LOGIN_PATTERN = /^(?!-)[A-Za-z0-9-]{1,39}(? { return value as Record; } +function requireExactRecordFields( + value: Record, + fields: readonly string[], + label: string, +): void { + const allowed = new Set(fields); + for (const field of fields) { + if (!Object.hasOwn(value, field)) { + fail("PROVENANCE_MISMATCH", `${label} is missing ${field}`); + } + } + const unexpected = Object.keys(value) + .filter((field) => !allowed.has(field)) + .sort(); + if (unexpected.length > 0) { + fail("PROVENANCE_MISMATCH", `${label} contains unexpected field ${unexpected[0]}`); + } +} + +function persistedString( + value: Record, + field: string, + label: string, + pattern?: RegExp, +): string { + const result = value[field]; + if (typeof result !== "string" || (pattern !== undefined && !pattern.test(result))) { + fail("PROVENANCE_MISMATCH", `${label} has an invalid format`); + } + return result; +} + +function persistedTimestamp(value: unknown, label: string): number { + if (typeof value !== "string") { + fail("PROVENANCE_MISMATCH", `${label} must be a timestamp string`); + } + const parsed = Date.parse(value); + if (!Number.isFinite(parsed) || new Date(parsed).toISOString() !== value) { + fail("PROVENANCE_MISMATCH", `${label} must be a canonical UTC timestamp`); + } + return parsed; +} + +function validatePersistedReason(value: unknown, label: string): void { + if ( + typeof value !== "string" || + value.length === 0 || + value !== value.trim() || + Buffer.byteLength(value, "utf8") > MAX_REASON_BYTES || + /[\u0000-\u001f\u007f]/u.test(value) + ) { + fail("PROVENANCE_MISMATCH", `${label} is invalid`); + } +} + function safePositiveId(value: unknown, label: string): string { if (!Number.isSafeInteger(value) || (value as number) < 1) { fail("PROVENANCE_MISMATCH", `${label} must be a positive safe integer`); @@ -515,30 +571,22 @@ export function validateQualificationWorkflowRun( function validateCancellationWorkflowRun( value: unknown, - runId: string, + state: ExactImageQualificationState, ): { status: string; conclusion: string | null } { const run = record(value, "producer cancellation workflow run"); - if (safePositiveId(run.id, "producer cancellation run ID") !== runId) { - fail("PROVENANCE_MISMATCH", "producer cancellation run ID changed"); - } - const details = canonicalDispatchDetails(runId); - requireExactString(run.url, details.runUrl, "producer cancellation run URL"); - requireExactString(run.html_url, details.htmlUrl, "producer cancellation HTML URL"); - const allowedStatuses = new Set([ - "queued", - "in_progress", - "completed", - "waiting", - "requested", - "pending", - ]); - if (typeof run.status !== "string" || !allowedStatuses.has(run.status)) { - fail("PROVENANCE_MISMATCH", "producer cancellation run has an unsupported status"); - } - if (run.conclusion !== null && typeof run.conclusion !== "string") { - fail("PROVENANCE_MISMATCH", "producer cancellation conclusion must be a string or null"); - } - return { status: run.status, conclusion: run.conclusion as string | null }; + const observedHeadSha = persistedString( + run, + "head_sha", + "producer cancellation head SHA", + FULL_SHA_PATTERN, + ); + const validated = validateQualificationWorkflowRun(value, { + ...expectedRun(state), + // A run whose producer ref moved after dispatch remains cleanup-owned but + // can never become qualification evidence; all other identity must bind. + producerSha: observedHeadSha, + }); + return { status: validated.status, conclusion: validated.conclusion }; } function statePath(workDir: string): string { @@ -564,43 +612,130 @@ function readDispatchIntent(workDir: string): ExactImageDispatchIntent | null { let contents: string | null; try { contents = privateFileRuntime.readPrivateRegularFile(intentPath(workDir), { + allowMissing: true, maxBytes: MAX_STATE_BYTES, }); } catch { fail("PROVENANCE_MISMATCH", "dispatch intent could not be read safely"); } if (contents === null) return null; + let parsed: unknown; try { - const intent = JSON.parse(contents) as ExactImageDispatchIntent; - if ( - intent.schemaVersion !== 1 || - intent.kind !== "nemoclaw-exact-image-dispatch-intent" || - !GITHUB_LOGIN_PATTERN.test(intent.request?.actor ?? "") || - !FULL_SHA_PATTERN.test(intent.request?.candidateSha ?? "") || - !UUID_V4_PATTERN.test(intent.request?.correlationId ?? "") || - !FULL_SHA_PATTERN.test(intent.request?.workflowSha ?? "") || - intent.request?.candidateSha !== intent.request?.workflowSha || - intent.request?.requesterRunAttempt !== 1 || - !DECIMAL_ID_PATTERN.test(intent.request?.requesterRunId ?? "") || - typeof intent.request?.reason !== "string" || - intent.request.reason.length === 0 || - intent.request.reason !== intent.request.reason.trim() || - Buffer.byteLength(intent.request.reason, "utf8") > MAX_REASON_BYTES || - /[\u0000-\u001f\u007f]/u.test(intent.request.reason) || - intent.producer?.repository !== PRODUCER_REPOSITORY || - !FULL_SHA_PATTERN.test(intent.producer?.repositorySha ?? "") || - intent.producer?.ref !== PRODUCER_REF || - intent.producer?.workflowPath !== PRODUCER_WORKFLOW_PATH || - !DECIMAL_ID_PATTERN.test(intent.producer?.workflowId ?? "") || - !Number.isFinite(Date.parse(intent.requestStartedAt)) - ) { - fail("PROVENANCE_MISMATCH", "dispatch intent is invalid"); - } - return intent; - } catch (error) { - if (error instanceof ExactImageQualificationError) throw error; + parsed = JSON.parse(contents) as unknown; + } catch { fail("PROVENANCE_MISMATCH", "dispatch intent is not valid JSON"); } + const intent = record(parsed, "dispatch intent"); + requireExactRecordFields( + intent, + ["schemaVersion", "kind", "requestStartedAt", "request", "producer"], + "dispatch intent", + ); + if (intent.schemaVersion !== 1 || intent.kind !== "nemoclaw-exact-image-dispatch-intent") { + fail("PROVENANCE_MISMATCH", "dispatch intent schema identity is invalid"); + } + persistedTimestamp(intent.requestStartedAt, "dispatch intent requestStartedAt"); + + const request = record(intent.request, "dispatch intent request"); + requireExactRecordFields( + request, + [ + "actor", + "candidateSha", + "correlationId", + "reason", + "requesterRunAttempt", + "requesterRunId", + "workflowSha", + ], + "dispatch intent request", + ); + persistedString(request, "actor", "dispatch intent actor", GITHUB_LOGIN_PATTERN); + const candidateSha = persistedString( + request, + "candidateSha", + "dispatch intent candidate SHA", + FULL_SHA_PATTERN, + ); + persistedString(request, "correlationId", "dispatch intent correlation ID", UUID_V4_PATTERN); + validatePersistedReason(request.reason, "dispatch intent reason"); + if (request.requesterRunAttempt !== 1) { + fail("PROVENANCE_MISMATCH", "dispatch intent requester run attempt must equal 1"); + } + persistedString( + request, + "requesterRunId", + "dispatch intent requester run ID", + DECIMAL_ID_PATTERN, + ); + const workflowSha = persistedString( + request, + "workflowSha", + "dispatch intent workflow SHA", + FULL_SHA_PATTERN, + ); + if (candidateSha !== workflowSha) { + fail("PROVENANCE_MISMATCH", "dispatch intent candidate and workflow SHAs differ"); + } + + const producer = record(intent.producer, "dispatch intent producer"); + requireExactRecordFields( + producer, + ["repository", "repositorySha", "ref", "workflowId", "workflowPath"], + "dispatch intent producer", + ); + requireExactString(producer.repository, PRODUCER_REPOSITORY, "dispatch intent repository"); + persistedString(producer, "repositorySha", "dispatch intent producer SHA", FULL_SHA_PATTERN); + requireExactString(producer.ref, PRODUCER_REF, "dispatch intent producer ref"); + persistedString(producer, "workflowId", "dispatch intent workflow ID", DECIMAL_ID_PATTERN); + requireExactString( + producer.workflowPath, + PRODUCER_WORKFLOW_PATH, + "dispatch intent workflow path", + ); + return intent as unknown as ExactImageDispatchIntent; +} + +function requiredDispatchIntent(workDir: string): ExactImageDispatchIntent { + const intent = readDispatchIntent(workDir); + if (intent === null) fail("PROVENANCE_MISMATCH", "dispatch intent is missing"); + return intent; +} + +function validateStateIntentBinding( + state: ExactImageQualificationState, + intent: ExactImageDispatchIntent, +): void { + const comparisons: Array<[unknown, unknown, string]> = [ + [state.dispatchedAt, intent.requestStartedAt, "dispatch timestamp"], + [state.request.actor, intent.request.actor, "actor"], + [state.request.candidateSha, intent.request.candidateSha, "candidate SHA"], + [state.request.correlationId, intent.request.correlationId, "correlation ID"], + [state.request.reason, intent.request.reason, "reason"], + [ + state.request.requesterRunAttempt, + intent.request.requesterRunAttempt, + "requester run attempt", + ], + [state.request.requesterRunId, intent.request.requesterRunId, "requester run ID"], + [state.request.workflowSha, intent.request.workflowSha, "workflow SHA"], + [state.producer.repository, intent.producer.repository, "producer repository"], + [state.producer.repositorySha, intent.producer.repositorySha, "producer SHA"], + [state.producer.ref, intent.producer.ref, "producer ref"], + [state.producer.workflowId, intent.producer.workflowId, "producer workflow ID"], + [state.producer.workflowPath, intent.producer.workflowPath, "producer workflow path"], + ]; + for (const [actual, expected, label] of comparisons) { + if (actual !== expected) { + fail("PROVENANCE_MISMATCH", `controller state ${label} does not match dispatch intent`); + } + } +} + +function readBoundExactImageQualificationState(workDir: string): ExactImageQualificationState { + const state = readExactImageQualificationState(workDir); + validateStateIntentBinding(state, requiredDispatchIntent(workDir)); + return state; } function writeDispatchReconciliation( @@ -650,9 +785,260 @@ function writeAtomicPrivateRegularFile(file: string, contents: string): void { } } +type PersistedQualificationStatus = ExactImageQualificationState["status"]; + +function validatePersistedRequest(value: unknown): void { + const request = record(value, "controller state request"); + requireExactRecordFields( + request, + [ + "actor", + "candidateSha", + "correlationId", + "reason", + "requesterRunAttempt", + "requesterRunId", + "workflowSha", + ], + "controller state request", + ); + persistedString(request, "actor", "controller state actor", GITHUB_LOGIN_PATTERN); + const candidateSha = persistedString( + request, + "candidateSha", + "controller state candidate SHA", + FULL_SHA_PATTERN, + ); + persistedString(request, "correlationId", "controller state correlation ID", UUID_V4_PATTERN); + validatePersistedReason(request.reason, "controller state reason"); + if (request.requesterRunAttempt !== 1) { + fail("PROVENANCE_MISMATCH", "controller state requester run attempt must equal 1"); + } + persistedString( + request, + "requesterRunId", + "controller state requester run ID", + DECIMAL_ID_PATTERN, + ); + const workflowSha = persistedString( + request, + "workflowSha", + "controller state workflow SHA", + FULL_SHA_PATTERN, + ); + if (candidateSha !== workflowSha) { + fail("PROVENANCE_MISMATCH", "controller state candidate and workflow SHAs differ"); + } +} + +function validatePersistedProducer( + value: unknown, + status: PersistedQualificationStatus, + dispatchedAt: number, +): { completedAt: number | null; runId: string } { + const producer = record(value, "controller state producer"); + const completed = status !== "dispatched"; + requireExactRecordFields( + producer, + [ + "repository", + "repositorySha", + "ref", + "workflowPath", + "runId", + "runAttempt", + "workflowId", + "runUrl", + "htmlUrl", + ...(completed ? ["completedAt"] : []), + ], + "controller state producer", + ); + requireExactString(producer.repository, PRODUCER_REPOSITORY, "controller state repository"); + persistedString(producer, "repositorySha", "controller state producer SHA", FULL_SHA_PATTERN); + requireExactString(producer.ref, PRODUCER_REF, "controller state producer ref"); + requireExactString( + producer.workflowPath, + PRODUCER_WORKFLOW_PATH, + "controller state workflow path", + ); + const runId = persistedString( + producer, + "runId", + "controller state producer run ID", + DECIMAL_ID_PATTERN, + ); + if (producer.runAttempt !== 1) { + fail("PROVENANCE_MISMATCH", "controller state producer run attempt must equal 1"); + } + persistedString( + producer, + "workflowId", + "controller state producer workflow ID", + DECIMAL_ID_PATTERN, + ); + const expectedUrls = canonicalDispatchDetails(runId); + requireExactString(producer.runUrl, expectedUrls.runUrl, "controller state producer run URL"); + requireExactString(producer.htmlUrl, expectedUrls.htmlUrl, "controller state producer HTML URL"); + const completedAt = completed + ? persistedTimestamp(producer.completedAt, "controller state completedAt") + : null; + if (completedAt !== null && completedAt < dispatchedAt) { + fail("PROVENANCE_MISMATCH", "controller state completedAt precedes dispatchedAt"); + } + if (completedAt !== null && completedAt > dispatchedAt + QUALIFICATION_TIMEOUT_MS) { + fail("PROVENANCE_MISMATCH", "controller state completedAt exceeds qualification deadline"); + } + return { completedAt, runId }; +} + +function validatePersistedArtifact(value: unknown, runId: string): string { + const artifact = record(value, "controller state artifact"); + requireExactRecordFields( + artifact, + [ + "id", + "name", + "digest", + "sizeInBytes", + "apiUrl", + "archiveDownloadUrl", + "archiveSha256", + "manifestSha256", + ], + "controller state artifact", + ); + const id = persistedString(artifact, "id", "controller state artifact ID", DECIMAL_ID_PATTERN); + requireExactString( + artifact.name, + `nemoclaw-image-handoff-v1-${runId}-1`, + "controller state artifact name", + ); + const digest = persistedString( + artifact, + "digest", + "controller state artifact digest", + ARTIFACT_DIGEST_PATTERN, + ); + if ( + !Number.isSafeInteger(artifact.sizeInBytes) || + (artifact.sizeInBytes as number) < 1 || + (artifact.sizeInBytes as number) > MAX_ARCHIVE_BYTES + ) { + fail("PROVENANCE_MISMATCH", "controller state artifact size is invalid"); + } + const apiUrl = `https://api.github.com/repos/${PRODUCER_REPOSITORY}/actions/artifacts/${id}`; + requireExactString(artifact.apiUrl, apiUrl, "controller state artifact API URL"); + requireExactString( + artifact.archiveDownloadUrl, + `${apiUrl}/zip`, + "controller state artifact archive URL", + ); + const archiveSha256 = persistedString( + artifact, + "archiveSha256", + "controller state archive hash", + SHA256_PATTERN, + ); + if (digest !== `sha256:${archiveSha256}`) { + fail("PROVENANCE_MISMATCH", "controller state artifact digest does not match archive hash"); + } + return persistedString( + artifact, + "manifestSha256", + "controller state manifest hash", + SHA256_PATTERN, + ); +} + +function validatePersistedValidation( + value: unknown, + manifestSha256: string, + completedAt: number, + dispatchedAt: number, +): void { + const validation = record(value, "controller state validation"); + requireExactRecordFields( + validation, + ["acceptedAt", "manifestSha256", "normalizedManifestSha256"], + "controller state validation", + ); + const acceptedAt = persistedTimestamp(validation.acceptedAt, "controller state acceptedAt"); + if (acceptedAt < completedAt) { + fail("PROVENANCE_MISMATCH", "controller state acceptedAt precedes completedAt"); + } + if (acceptedAt > dispatchedAt + QUALIFICATION_TIMEOUT_MS) { + fail("PROVENANCE_MISMATCH", "controller state acceptedAt exceeds qualification deadline"); + } + requireExactString( + validation.manifestSha256, + manifestSha256, + "controller state accepted manifest hash", + ); + persistedString( + validation, + "normalizedManifestSha256", + "controller state normalized manifest hash", + SHA256_PATTERN, + ); +} + +function validatePersistedQualificationState(value: unknown): ExactImageQualificationState { + const state = record(value, "controller state"); + const statuses = new Set([ + "dispatched", + "completed", + "downloaded", + "validated", + ]); + if ( + typeof state.status !== "string" || + !statuses.has(state.status as PersistedQualificationStatus) + ) { + fail("PROVENANCE_MISMATCH", "controller state status is invalid"); + } + const status = state.status as PersistedQualificationStatus; + const hasArtifact = status === "downloaded" || status === "validated"; + requireExactRecordFields( + state, + [ + "schemaVersion", + "status", + "dispatchedAt", + "request", + "producer", + ...(hasArtifact ? ["artifact"] : []), + ...(status === "validated" ? ["validation"] : []), + ], + "controller state", + ); + if (state.schemaVersion !== 1) { + fail("PROVENANCE_MISMATCH", "controller state schema version must equal 1"); + } + const dispatchedAt = persistedTimestamp(state.dispatchedAt, "controller state dispatchedAt"); + validatePersistedRequest(state.request); + const producer = validatePersistedProducer(state.producer, status, dispatchedAt); + const manifestSha256 = hasArtifact + ? validatePersistedArtifact(state.artifact, producer.runId) + : null; + if (status === "validated") { + if (manifestSha256 === null || producer.completedAt === null) { + fail("PROVENANCE_MISMATCH", "validated controller state is incomplete"); + } + validatePersistedValidation( + state.validation, + manifestSha256, + producer.completedAt, + dispatchedAt, + ); + } + return state as unknown as ExactImageQualificationState; +} + function writeState(workDir: string, state: ExactImageQualificationState): void { try { - writeAtomicPrivateRegularFile(statePath(workDir), `${JSON.stringify(state, null, 2)}\n`); + const validated = validatePersistedQualificationState(state); + writeAtomicPrivateRegularFile(statePath(workDir), `${JSON.stringify(validated, null, 2)}\n`); } catch { fail("OUTPUT_WRITE_FAILED", "controller state could not be written safely"); } @@ -682,11 +1068,13 @@ export function readExactImageQualificationState(workDir: string): ExactImageQua fail("PROVENANCE_MISMATCH", "controller state could not be read safely"); } if (contents === null) fail("PROVENANCE_MISMATCH", "controller state is missing"); + let parsed: unknown; try { - return JSON.parse(contents) as ExactImageQualificationState; + parsed = JSON.parse(contents) as unknown; } catch { fail("PROVENANCE_MISMATCH", "controller state is not valid JSON"); } + return validatePersistedQualificationState(parsed); } function canonicalDispatchDetails(runId: string): DispatchDetails { @@ -826,7 +1214,7 @@ async function cancelAndVerifyRecoveredRun( token, { apiVersion: GITHUB_API_VERSION, expectedStatus: 200 }, ), - state.producer.runId, + state, ); if (run.status === "completed") return true; if (cancellationError !== undefined) throw cancellationError; @@ -1099,7 +1487,10 @@ export async function waitForExactImageQualification( const now = dependencies.now ?? Date.now; const pause = dependencies.sleep ?? sleep; const configured = limits(dependencies); - const state = readExactImageQualificationState(options.workDir); + const state = readBoundExactImageQualificationState(options.workDir); + if (state.status !== "dispatched") { + fail("PROVENANCE_MISMATCH", "producer run may only be awaited from dispatched state"); + } const startedAt = Date.parse(state.dispatchedAt); const hardDeadline = qualificationDeadline(state, configured.qualificationTimeoutMs); const deadline = acceptanceDeadline(state, dependencies); @@ -1328,7 +1719,7 @@ export async function downloadExactImageManifest( options: { workDir: string; producerToken: string }, dependencies: QualificationDependencies = {}, ): Promise { - const state = readExactImageQualificationState(options.workDir); + const state = readBoundExactImageQualificationState(options.workDir); if (state.status !== "completed") { fail("PROVENANCE_MISMATCH", "producer run must complete before artifact download"); } @@ -1446,7 +1837,7 @@ export function finalizeExactImageQualification( workDir: string, dependencies: QualificationDependencies = {}, ): ExactImageQualificationState { - const state = readExactImageQualificationState(workDir); + const state = readBoundExactImageQualificationState(workDir); if (state.status !== "downloaded" || !state.artifact) { fail("PROVENANCE_MISMATCH", "artifact must be digest-verified before finalization"); } @@ -1478,6 +1869,10 @@ export function finalizeExactImageQualification( artifact: state.artifact, validation: state.validation, }; + // The state is the durable commit point for acceptance. Validate and persist + // that transition before publishing an accepted receipt so a rejected state + // can never leave accepted evidence behind. + writeState(workDir, state); try { privateFileRuntime.writePrivateRegularFile( path.join(workDir, EVIDENCE_FILE), @@ -1486,7 +1881,6 @@ export function finalizeExactImageQualification( } catch { fail("OUTPUT_WRITE_FAILED", "qualification evidence could not be written safely"); } - writeState(workDir, state); return state; } @@ -1502,19 +1896,36 @@ export async function cancelActiveExactImageQualification( if (fs.existsSync(statePath(options.workDir))) throw error; return false; } - const intent = readDispatchIntent(options.workDir); - if (intent === null) return false; + // State crosses independent workflow steps and is untrusted/damaged-disk input on every read. + // Atomic replacement prevents ordinary partial writes, while this permanent defense-in-depth + // path preserves interruption/filesystem-corruption evidence and reconciles only from the + // separately validated durable intent. preserveUnreadableState(options.workDir); + const intent = requiredDispatchIntent(options.workDir); return reconcileAmbiguousDispatch({ ...options, intent, mode: "cleanup" }, dependencies); } const now = dependencies.now ?? Date.now; const configured = limits(dependencies); - if (state.status !== "dispatched") return false; + const intent = requiredDispatchIntent(options.workDir); + try { + validateStateIntentBinding(state, intent); + } catch { + preserveUnreadableState(options.workDir); + return reconcileAmbiguousDispatch({ ...options, intent, mode: "cleanup" }, dependencies); + } const deadline = now() + configured.cleanupTimeoutMs; const api = boundedApiClient(dependencies.api ?? githubApi, dependencies, deadline); + const initial = validateCancellationWorkflowRun( + await api( + `repos/${PRODUCER_REPOSITORY}/actions/runs/${state.producer.runId}`, + options.producerToken, + { apiVersion: GITHUB_API_VERSION, expectedStatus: 200 }, + ), + state, + ); return cancelAndVerifyRecoveredRun( state, - { status: "unknown" }, + initial, options.producerToken, api, dependencies.sleep ?? sleep, From 5ebf2a0e66ea59f87f5cee446961594fc2132247 Mon Sep 17 00:00:00 2001 From: Charan Jagwani Date: Thu, 16 Jul 2026 13:37:04 -0700 Subject: [PATCH 05/25] ci(e2e): add launchable evidence contract Signed-off-by: Charan Jagwani --- ...ev-launchable-workspace-receipt-fixture.ts | 92 +++ .../brev-launchable-workspace-receipt.test.ts | 516 ++++++++++++ .../e2e/brev-launchable-workspace-receipt.mts | 763 ++++++++++++++++++ 3 files changed, 1371 insertions(+) create mode 100644 test/e2e/support/brev-launchable-workspace-receipt-fixture.ts create mode 100644 test/e2e/support/brev-launchable-workspace-receipt.test.ts create mode 100644 tools/e2e/brev-launchable-workspace-receipt.mts diff --git a/test/e2e/support/brev-launchable-workspace-receipt-fixture.ts b/test/e2e/support/brev-launchable-workspace-receipt-fixture.ts new file mode 100644 index 0000000000..5a47933b49 --- /dev/null +++ b/test/e2e/support/brev-launchable-workspace-receipt-fixture.ts @@ -0,0 +1,92 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { + BrevLaunchableWorkspaceReceipt, + BrevLaunchableWorkspaceReceiptExpectations, + BrevWorkspaceCleanupEvidence, +} from "../../../tools/e2e/brev-launchable-workspace-receipt.mts"; +import { brevLaunchableWorkspaceReceiptSha256 } from "../../../tools/e2e/brev-launchable-workspace-receipt.mts"; + +export const BREV_RECEIPT_CORRELATION_ID = "12345678-1234-4123-8123-123456789abc"; +export const BREV_RECEIPT_CANDIDATE_SHA = "a".repeat(40); +export const BREV_RECEIPT_WINDOW_START = Date.parse("2026-07-16T12:00:00.000Z"); +export const BREV_RECEIPT_WINDOW_END = Date.parse("2026-07-16T13:00:00.000Z"); +export const BREV_CLEANUP_DEADLINE = Date.parse("2026-07-16T13:15:00.000Z"); + +export const BREV_IMAGE = { + project: "brevdevprod", + imageName: "nemoclaw-brev-cpu-v0-1-0-20260716-a-staging-190-1", + imageId: "12345678901234567890", + imageSelfLink: + "https://www.googleapis.com/compute/v1/projects/brevdevprod/global/images/nemoclaw-brev-cpu-v0-1-0-20260716-a-staging-190-1", +} as const; + +export function brevLaunchableWorkspaceReceipt( + overrides: Partial = {}, +): BrevLaunchableWorkspaceReceipt { + return { + schemaVersion: 1, + kind: "nemoclaw-brev-launchable-workspace-receipt", + correlationId: BREV_RECEIPT_CORRELATION_ID, + nemoclawSha: BREV_RECEIPT_CANDIDATE_SHA, + acceptedImageManifestSha256: "c".repeat(64), + organizationId: "org-NemoClaw-staging", + idempotencyKey: "qualification-run-1", + launchable: { + id: "env-3Azt0aYgVNFEuz7opyx3gscmowS", + revision: "revision-01JZZZZZZZZZZZZZZZZZZZZZZZ", + suppliedImageReference: BREV_IMAGE.imageSelfLink, + resolvedImage: { ...BREV_IMAGE }, + }, + workspace: { + id: "workspace-01JZZZZZZZZZZZZZZZZZZZZZZZ", + launchableId: "env-3Azt0aYgVNFEuz7opyx3gscmowS", + launchableRevision: "revision-01JZZZZZZZZZZZZZZZZZZZZZZZ", + bootImage: { ...BREV_IMAGE }, + status: "READY", + }, + recordedAt: "2026-07-16T12:45:00.000Z", + ...overrides, + }; +} + +export function brevLaunchableWorkspaceReceiptExpectations( + overrides: Partial = {}, +): BrevLaunchableWorkspaceReceiptExpectations { + return { + correlationId: BREV_RECEIPT_CORRELATION_ID, + nemoclawSha: BREV_RECEIPT_CANDIDATE_SHA, + acceptedImageManifestSha256: "c".repeat(64), + organizationId: "org-NemoClaw-staging", + idempotencyKey: "qualification-run-1", + launchableId: "env-3Azt0aYgVNFEuz7opyx3gscmowS", + launchableRevision: "revision-01JZZZZZZZZZZZZZZZZZZZZZZZ", + workspaceId: "workspace-01JZZZZZZZZZZZZZZZZZZZZZZZ", + suppliedImageReference: BREV_IMAGE.imageSelfLink, + image: { ...BREV_IMAGE }, + notBeforeMs: BREV_RECEIPT_WINDOW_START, + notAfterMs: BREV_RECEIPT_WINDOW_END, + ...overrides, + }; +} + +export function brevWorkspaceCleanupEvidence( + receipt = brevLaunchableWorkspaceReceipt(), + overrides: Partial = {}, +): BrevWorkspaceCleanupEvidence { + return { + schemaVersion: 1, + kind: "nemoclaw-brev-workspace-cleanup-evidence", + correlationId: receipt.correlationId, + receiptSha256: brevLaunchableWorkspaceReceiptSha256(receipt), + organizationId: receipt.organizationId, + workspaceId: receipt.workspace.id, + launchableId: receipt.launchable.id, + launchableRevision: receipt.launchable.revision, + deleteRequestedAt: "2026-07-16T13:01:00.000Z", + terminalState: "ABSENT", + verifiedAt: "2026-07-16T13:04:00.000Z", + ...overrides, + }; +} diff --git a/test/e2e/support/brev-launchable-workspace-receipt.test.ts b/test/e2e/support/brev-launchable-workspace-receipt.test.ts new file mode 100644 index 0000000000..c1fc7f404f --- /dev/null +++ b/test/e2e/support/brev-launchable-workspace-receipt.test.ts @@ -0,0 +1,516 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { + BrevLaunchableWorkspaceError, + brevLaunchableWorkspaceReceiptSha256, + normalizedBrevLaunchableWorkspaceReceiptJson, + normalizedBrevWorkspaceCleanupEvidenceJson, + parseAndValidateBrevLaunchableWorkspaceReceipt, + parseAndValidateBrevWorkspaceCleanupEvidence, + validateBrevLaunchableWorkspaceReceipt, + validateBrevWorkspaceCleanupEvidence, +} from "../../../tools/e2e/brev-launchable-workspace-receipt.mts"; +import { + BREV_CLEANUP_DEADLINE, + BREV_IMAGE, + BREV_RECEIPT_WINDOW_END, + brevLaunchableWorkspaceReceipt, + brevLaunchableWorkspaceReceiptExpectations, + brevWorkspaceCleanupEvidence, +} from "./brev-launchable-workspace-receipt-fixture.ts"; + +function expectCode(run: () => unknown, code: BrevLaunchableWorkspaceError["code"]): void { + try { + run(); + } catch (error) { + expect(error).toBeInstanceOf(BrevLaunchableWorkspaceError); + expect((error as BrevLaunchableWorkspaceError).code).toBe(code); + return; + } + throw new Error(`expected ${code}`); +} + +describe("Brev Launchable and workspace evidence", () => { + it("accepts only a ready workspace bound to one immutable Launchable revision and exact image", () => { + const source = brevLaunchableWorkspaceReceipt(); + const accepted = validateBrevLaunchableWorkspaceReceipt( + source, + brevLaunchableWorkspaceReceiptExpectations(), + ); + + expect(accepted).toEqual(source); + expect(accepted.workspace.launchableRevision).toBe(accepted.launchable.revision); + expect(accepted.workspace.bootImage.imageId).toBe(BREV_IMAGE.imageId); + expect(normalizedBrevLaunchableWorkspaceReceiptJson(accepted)).toBe( + `${JSON.stringify(source, null, 2)}\n`, + ); + expect(brevLaunchableWorkspaceReceiptSha256(accepted)).toMatch(/^[0-9a-f]{64}$/u); + }); + + it("rejects missing and additional fields at every receipt layer", () => { + const missing = { ...brevLaunchableWorkspaceReceipt() } as Record; + delete missing.acceptedImageManifestSha256; + expectCode( + () => + validateBrevLaunchableWorkspaceReceipt( + missing, + brevLaunchableWorkspaceReceiptExpectations(), + ), + "ARTIFACT_MISSING_OR_INVALID", + ); + + const source = brevLaunchableWorkspaceReceipt(); + const additional = { + ...source, + launchable: { ...source.launchable, mutableFamilyWasCloseEnough: true }, + }; + expectCode( + () => + validateBrevLaunchableWorkspaceReceipt( + additional, + brevLaunchableWorkspaceReceiptExpectations(), + ), + "ARTIFACT_MISSING_OR_INVALID", + ); + + const extraImage = { + ...source, + workspace: { + ...source.workspace, + bootImage: { ...source.workspace.bootImage, imageFamily: "nemoclaw-brev-staging-cpu" }, + }, + }; + expectCode( + () => + validateBrevLaunchableWorkspaceReceipt( + extraImage, + brevLaunchableWorkspaceReceiptExpectations(), + ), + "ARTIFACT_MISSING_OR_INVALID", + ); + + for (const receipt of [ + { + ...source, + launchable: Object.fromEntries( + Object.entries(source.launchable).filter(([field]) => field !== "revision"), + ), + }, + { + ...source, + workspace: Object.fromEntries( + Object.entries(source.workspace).filter(([field]) => field !== "id"), + ), + }, + { + ...source, + launchable: { + ...source.launchable, + resolvedImage: Object.fromEntries( + Object.entries(source.launchable.resolvedImage).filter( + ([field]) => field !== "imageId", + ), + ), + }, + }, + { + ...source, + workspace: { + ...source.workspace, + bootImage: Object.fromEntries( + Object.entries(source.workspace.bootImage).filter( + ([field]) => field !== "imageSelfLink", + ), + ), + }, + }, + ]) { + expectCode( + () => + validateBrevLaunchableWorkspaceReceipt( + receipt, + brevLaunchableWorkspaceReceiptExpectations(), + ), + "ARTIFACT_MISSING_OR_INVALID", + ); + } + }); + + it.each([ + ["correlation ID", { correlationId: "12345678-1234-1123-8123-123456789abc" }], + ["candidate SHA", { nemoclawSha: "A".repeat(40) }], + ["manifest hash", { acceptedImageManifestSha256: "b".repeat(63) }], + ["organization control character", { organizationId: "org\nother" }], + ])("rejects an invalid %s", (_name, overrides) => { + expectCode( + () => + validateBrevLaunchableWorkspaceReceipt( + { ...brevLaunchableWorkspaceReceipt(), ...overrides }, + brevLaunchableWorkspaceReceiptExpectations(), + ), + "ARTIFACT_MISSING_OR_INVALID", + ); + }); + + it("binds request, manifest, organization, idempotency, and supplied reference to trusted state", () => { + const mismatches = [ + { correlationId: "87654321-4321-4321-8321-cba987654321" }, + { nemoclawSha: "d".repeat(40) }, + { acceptedImageManifestSha256: "f".repeat(64) }, + { organizationId: "org-other" }, + { idempotencyKey: "qualification-other" }, + { launchableId: "env-other" }, + { launchableRevision: "revision-other" }, + { workspaceId: "workspace-other" }, + { + suppliedImageReference: + "projects/brevdevprod/global/images/family/nemoclaw-brev-staging-cpu", + }, + ]; + for (const expected of mismatches) { + expectCode( + () => + validateBrevLaunchableWorkspaceReceipt( + brevLaunchableWorkspaceReceipt(), + brevLaunchableWorkspaceReceiptExpectations(expected), + ), + "PROVENANCE_MISMATCH", + ); + } + }); + + it("accepts only the immutable image or canonical staging family as the trusted supplied reference", () => { + expectCode( + () => + validateBrevLaunchableWorkspaceReceipt( + brevLaunchableWorkspaceReceipt(), + brevLaunchableWorkspaceReceiptExpectations({ + suppliedImageReference: "https://attacker.invalid/image", + }), + ), + "REQUEST_INVALID", + ); + + const familyReference = "projects/brevdevprod/global/images/family/nemoclaw-brev-staging-cpu"; + const familyReceipt = brevLaunchableWorkspaceReceipt({ + launchable: { + ...brevLaunchableWorkspaceReceipt().launchable, + suppliedImageReference: familyReference, + }, + }); + expect( + validateBrevLaunchableWorkspaceReceipt( + familyReceipt, + brevLaunchableWorkspaceReceiptExpectations({ suppliedImageReference: familyReference }), + ), + ).toEqual(familyReceipt); + }); + + it("requires structured workspace readiness", () => { + const source = brevLaunchableWorkspaceReceipt(); + expectCode( + () => + validateBrevLaunchableWorkspaceReceipt( + { ...source, workspace: { ...source.workspace, status: "RUNNING" } }, + brevLaunchableWorkspaceReceiptExpectations(), + ), + "BREV_READINESS_FAILED", + ); + }); + + it("binds opaque Brev IDs and revision to durable observations and workspace readback", () => { + const source = brevLaunchableWorkspaceReceipt(); + for (const receipt of [ + { + ...source, + workspace: { ...source.workspace, launchableRevision: "revision-other" }, + }, + { ...source, workspace: { ...source.workspace, launchableId: "env-other" } }, + ]) { + expectCode( + () => + validateBrevLaunchableWorkspaceReceipt( + receipt, + brevLaunchableWorkspaceReceiptExpectations(), + ), + "PROVENANCE_MISMATCH", + ); + } + expectCode( + () => + validateBrevLaunchableWorkspaceReceipt( + { ...source, launchable: { ...source.launchable, revision: "current" } }, + brevLaunchableWorkspaceReceiptExpectations(), + ), + "PROVENANCE_MISMATCH", + ); + }); + + it("rejects family-only, name-only, same-name replacement, and different boot-image evidence", () => { + const source = brevLaunchableWorkspaceReceipt(); + const mismatches = [ + { + ...source, + launchable: { + ...source.launchable, + resolvedImage: { + ...source.launchable.resolvedImage, + imageSelfLink: + "https://www.googleapis.com/compute/v1/projects/brevdevprod/global/images/family/nemoclaw-brev-staging-cpu", + }, + }, + }, + { + ...source, + workspace: { + ...source.workspace, + bootImage: { ...source.workspace.bootImage, imageId: "99999999999999999999" }, + }, + }, + { + ...source, + workspace: { + ...source.workspace, + bootImage: { ...source.workspace.bootImage, imageName: "different-image" }, + }, + }, + ]; + for (const receipt of mismatches) { + expectCode( + () => + validateBrevLaunchableWorkspaceReceipt( + receipt, + brevLaunchableWorkspaceReceiptExpectations(), + ), + "BREV_IMAGE_RESOLUTION_MISMATCH", + ); + } + }); + + it("requires one real canonical UTC receipt time inside the trusted window", () => { + for (const recordedAt of [ + "2026-02-30T12:00:00.000Z", + "2026-07-16T12:45:00Z", + "2026-07-16T12:45:00.000+00:00", + "2026-07-16T13:00:00.001Z", + ]) { + expectCode( + () => + validateBrevLaunchableWorkspaceReceipt( + brevLaunchableWorkspaceReceipt({ recordedAt }), + brevLaunchableWorkspaceReceiptExpectations(), + ), + recordedAt === "2026-07-16T13:00:00.001Z" + ? "PROVENANCE_MISMATCH" + : "ARTIFACT_MISSING_OR_INVALID", + ); + } + expect( + validateBrevLaunchableWorkspaceReceipt( + brevLaunchableWorkspaceReceipt({ + recordedAt: new Date(BREV_RECEIPT_WINDOW_END).toISOString(), + }), + brevLaunchableWorkspaceReceiptExpectations(), + ).recordedAt, + ).toBe("2026-07-16T13:00:00.000Z"); + }); + + it("parses bounded strict JSON and rejects malformed, duplicate, deep, and trailing data", () => { + const source = brevLaunchableWorkspaceReceipt(); + expect( + parseAndValidateBrevLaunchableWorkspaceReceipt( + JSON.stringify(source), + brevLaunchableWorkspaceReceiptExpectations(), + ), + ).toEqual(source); + for (const contents of [ + "{not-json", + '{"a":1,"\\u0061":2}', + "[[[[[[[0]]]]]]]", + `${JSON.stringify(source)}\ntransport output`, + " ".repeat(64 * 1024 + 1), + ]) { + expectCode( + () => + parseAndValidateBrevLaunchableWorkspaceReceipt( + contents, + brevLaunchableWorkspaceReceiptExpectations(), + ), + "ARTIFACT_MISSING_OR_INVALID", + ); + } + }); + + it("accepts terminal absence only when cleanup is bound to the immutable receipt and IDs", () => { + const receipt = brevLaunchableWorkspaceReceipt(); + const cleanup = brevWorkspaceCleanupEvidence(receipt); + const accepted = validateBrevWorkspaceCleanupEvidence(cleanup, { + correlationId: receipt.correlationId, + receiptSha256: brevLaunchableWorkspaceReceiptSha256(receipt), + organizationId: receipt.organizationId, + workspaceId: receipt.workspace.id, + launchableId: receipt.launchable.id, + launchableRevision: receipt.launchable.revision, + notBeforeMs: Date.parse(receipt.recordedAt), + notAfterMs: BREV_CLEANUP_DEADLINE, + }); + + expect(accepted).toEqual(cleanup); + expect(normalizedBrevWorkspaceCleanupEvidenceJson(accepted)).toBe( + `${JSON.stringify(cleanup, null, 2)}\n`, + ); + expect( + parseAndValidateBrevWorkspaceCleanupEvidence(JSON.stringify(cleanup), { + correlationId: receipt.correlationId, + receiptSha256: brevLaunchableWorkspaceReceiptSha256(receipt), + organizationId: receipt.organizationId, + workspaceId: receipt.workspace.id, + launchableId: receipt.launchable.id, + launchableRevision: receipt.launchable.revision, + notBeforeMs: Date.parse(receipt.recordedAt), + notAfterMs: BREV_CLEANUP_DEADLINE, + }), + ).toEqual(cleanup); + }); + + it("rejects delete acknowledgement, deletion in progress, false absence, and unbound cleanup", () => { + const receipt = brevLaunchableWorkspaceReceipt(); + const expected = { + correlationId: receipt.correlationId, + receiptSha256: brevLaunchableWorkspaceReceiptSha256(receipt), + organizationId: receipt.organizationId, + workspaceId: receipt.workspace.id, + launchableId: receipt.launchable.id, + launchableRevision: receipt.launchable.revision, + notBeforeMs: Date.parse(receipt.recordedAt), + notAfterMs: BREV_CLEANUP_DEADLINE, + }; + + for (const cleanup of [ + { ...brevWorkspaceCleanupEvidence(receipt), terminalState: "DELETE_ACCEPTED" }, + { ...brevWorkspaceCleanupEvidence(receipt), terminalState: "DELETING" }, + ]) { + expectCode( + () => validateBrevWorkspaceCleanupEvidence(cleanup, expected), + "BREV_CLEANUP_INCOMPLETE", + ); + } + for (const cleanup of [ + brevWorkspaceCleanupEvidence(receipt, { receiptSha256: "f".repeat(64) }), + brevWorkspaceCleanupEvidence(receipt, { workspaceId: "workspace-other" }), + brevWorkspaceCleanupEvidence(receipt, { launchableRevision: "revision-other" }), + ]) { + expectCode( + () => validateBrevWorkspaceCleanupEvidence(cleanup, expected), + "PROVENANCE_MISMATCH", + ); + } + + const missing = { ...brevWorkspaceCleanupEvidence(receipt) } as Record; + delete missing.receiptSha256; + for (const cleanup of [ + missing, + { ...brevWorkspaceCleanupEvidence(receipt), diagnostic: "ok" }, + ]) { + expectCode( + () => validateBrevWorkspaceCleanupEvidence(cleanup, expected), + "ARTIFACT_MISSING_OR_INVALID", + ); + } + }); + + it("requires ordered cleanup timestamps inside the controller deadline", () => { + const receipt = brevLaunchableWorkspaceReceipt(); + const expected = { + correlationId: receipt.correlationId, + receiptSha256: brevLaunchableWorkspaceReceiptSha256(receipt), + organizationId: receipt.organizationId, + workspaceId: receipt.workspace.id, + launchableId: receipt.launchable.id, + launchableRevision: receipt.launchable.revision, + notBeforeMs: Date.parse(receipt.recordedAt), + notAfterMs: BREV_CLEANUP_DEADLINE, + }; + for (const cleanup of [ + brevWorkspaceCleanupEvidence(receipt, { + deleteRequestedAt: "2026-07-16T13:05:00.000Z", + verifiedAt: "2026-07-16T13:04:00.000Z", + }), + brevWorkspaceCleanupEvidence(receipt, { verifiedAt: "2026-07-16T13:15:00.001Z" }), + ]) { + expectCode( + () => validateBrevWorkspaceCleanupEvidence(cleanup, expected), + cleanup.verifiedAt.endsWith("001Z") ? "PROVENANCE_MISMATCH" : "BREV_CLEANUP_INCOMPLETE", + ); + } + expectCode( + () => + validateBrevWorkspaceCleanupEvidence( + brevWorkspaceCleanupEvidence(receipt, { + deleteRequestedAt: "2026-07-16T13:04:00.000Z", + verifiedAt: "2026-07-16T13:04:00.000Z", + }), + expected, + ), + "BREV_CLEANUP_INCOMPLETE", + ); + expectCode( + () => + validateBrevWorkspaceCleanupEvidence( + brevWorkspaceCleanupEvidence(receipt, { + deleteRequestedAt: "2026-07-16T12:44:59.999Z", + }), + expected, + ), + "PROVENANCE_MISMATCH", + ); + }); + + it("requires separately anchored cleanup and rejects consistent receipt plus cleanup tampering", () => { + const receipt = brevLaunchableWorkspaceReceipt(); + const cleanupExpected = { + correlationId: receipt.correlationId, + receiptSha256: brevLaunchableWorkspaceReceiptSha256(receipt), + organizationId: receipt.organizationId, + workspaceId: receipt.workspace.id, + launchableId: receipt.launchable.id, + launchableRevision: receipt.launchable.revision, + notBeforeMs: Date.parse(receipt.recordedAt), + notAfterMs: BREV_CLEANUP_DEADLINE, + }; + expectCode( + () => validateBrevWorkspaceCleanupEvidence(undefined, cleanupExpected), + "ARTIFACT_MISSING_OR_INVALID", + ); + + const forged = brevLaunchableWorkspaceReceipt({ + launchable: { + ...receipt.launchable, + id: "env-forged", + revision: "revision-forged", + }, + workspace: { + ...receipt.workspace, + id: "workspace-forged", + launchableId: "env-forged", + launchableRevision: "revision-forged", + }, + }); + expectCode( + () => + validateBrevLaunchableWorkspaceReceipt( + forged, + brevLaunchableWorkspaceReceiptExpectations(), + ), + "PROVENANCE_MISMATCH", + ); + expectCode( + () => + validateBrevWorkspaceCleanupEvidence(brevWorkspaceCleanupEvidence(forged), cleanupExpected), + "PROVENANCE_MISMATCH", + ); + }); +}); diff --git a/tools/e2e/brev-launchable-workspace-receipt.mts b/tools/e2e/brev-launchable-workspace-receipt.mts new file mode 100644 index 0000000000..90a242041f --- /dev/null +++ b/tools/e2e/brev-launchable-workspace-receipt.mts @@ -0,0 +1,763 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; + +import { EXACT_IMAGE_STAGING_FAMILY } from "./exact-image-manifest.mts"; + +export const BREV_LAUNCHABLE_WORKSPACE_RECEIPT_KIND = "nemoclaw-brev-launchable-workspace-receipt"; +export const BREV_WORKSPACE_CLEANUP_EVIDENCE_KIND = "nemoclaw-brev-workspace-cleanup-evidence"; + +export type BrevLaunchableWorkspaceFailureCode = + | "REQUEST_INVALID" + | "ARTIFACT_MISSING_OR_INVALID" + | "PROVENANCE_MISMATCH" + | "BREV_IMAGE_RESOLUTION_MISMATCH" + | "BREV_READINESS_FAILED" + | "BREV_CLEANUP_INCOMPLETE" + | "UNKNOWN"; + +export class BrevLaunchableWorkspaceError extends Error { + readonly code: BrevLaunchableWorkspaceFailureCode; + + constructor(code: BrevLaunchableWorkspaceFailureCode, message: string) { + super(message); + this.name = "BrevLaunchableWorkspaceError"; + this.code = code; + } +} + +export type BrevImageIdentity = { + project: string; + imageName: string; + imageId: string; + imageSelfLink: string; +}; + +// resolvedImage and bootImage are independently observed Brev/platform +// readbacks. A transport adapter must never populate either object by copying +// the accepted manifest or the supplied Launchable configuration. +export type BrevLaunchableWorkspaceReceipt = { + schemaVersion: 1; + kind: typeof BREV_LAUNCHABLE_WORKSPACE_RECEIPT_KIND; + correlationId: string; + nemoclawSha: string; + acceptedImageManifestSha256: string; + organizationId: string; + idempotencyKey: string; + launchable: { + id: string; + revision: string; + suppliedImageReference: string; + resolvedImage: BrevImageIdentity; + }; + workspace: { + id: string; + launchableId: string; + launchableRevision: string; + bootImage: BrevImageIdentity; + status: "READY"; + }; + recordedAt: string; +}; + +export type BrevLaunchableWorkspaceReceiptExpectations = { + correlationId: string; + nemoclawSha: string; + // Hash of the normalized manifest already accepted by the image controller. + acceptedImageManifestSha256: string; + organizationId: string; + idempotencyKey: string; + // These generated identities must come from separately persisted, + // authoritative Brev operation results. Never derive them from the receipt + // being validated. + launchableId: string; + launchableRevision: string; + workspaceId: string; + suppliedImageReference: string; + image: BrevImageIdentity; + notBeforeMs: number; + notAfterMs: number; +}; + +export type BrevWorkspaceCleanupEvidence = { + schemaVersion: 1; + kind: typeof BREV_WORKSPACE_CLEANUP_EVIDENCE_KIND; + correlationId: string; + receiptSha256: string; + organizationId: string; + workspaceId: string; + launchableId: string; + launchableRevision: string; + deleteRequestedAt: string; + terminalState: "ABSENT"; + verifiedAt: string; +}; + +export type BrevWorkspaceCleanupEvidenceExpectations = { + // The controller persists this full tuple before issuing delete. Cleanup + // validation must not derive it from the cleanup artifact being validated. + correlationId: string; + receiptSha256: string; + organizationId: string; + workspaceId: string; + launchableId: string; + launchableRevision: string; + notBeforeMs: number; + notAfterMs: number; +}; + +const RECEIPT_FIELDS = [ + "schemaVersion", + "kind", + "correlationId", + "nemoclawSha", + "acceptedImageManifestSha256", + "organizationId", + "idempotencyKey", + "launchable", + "workspace", + "recordedAt", +] as const; +const LAUNCHABLE_FIELDS = ["id", "revision", "suppliedImageReference", "resolvedImage"] as const; +const WORKSPACE_FIELDS = [ + "id", + "launchableId", + "launchableRevision", + "bootImage", + "status", +] as const; +const IMAGE_FIELDS = ["project", "imageName", "imageId", "imageSelfLink"] as const; +const CLEANUP_FIELDS = [ + "schemaVersion", + "kind", + "correlationId", + "receiptSha256", + "organizationId", + "workspaceId", + "launchableId", + "launchableRevision", + "deleteRequestedAt", + "terminalState", + "verifiedAt", +] as const; + +const FULL_SHA_PATTERN = /^[0-9a-f]{40}$/u; +const SHA256_PATTERN = /^[0-9a-f]{64}$/u; +const UUID_V4_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/u; +const DECIMAL_ID_PATTERN = /^[1-9][0-9]*$/u; +const GCP_PROJECT_ID_PATTERN = /^[a-z][a-z0-9-]{4,28}[a-z0-9]$/u; +const IMAGE_NAME_PATTERN = /^[a-z](?:[-a-z0-9]{0,61}[a-z0-9])?$/u; +const CANONICAL_UTC_PATTERN = + /^(\d{4})-(\d{2})-(\d{2})T([01]\d|2[0-3]):([0-5]\d):([0-5]\d)[.](\d{3})Z$/u; +const OPAQUE_ID_MAX_BYTES = 256; +const REVISION_MAX_BYTES = 512; +const IMAGE_REFERENCE_MAX_BYTES = 2_048; +const MAX_EVIDENCE_BYTES = 64 * 1024; +const MAX_JSON_DEPTH = 6; + +function fail(code: BrevLaunchableWorkspaceFailureCode, message: string): never { + throw new BrevLaunchableWorkspaceError(code, message); +} + +function requireRecord( + value: unknown, + label: string, + code: BrevLaunchableWorkspaceFailureCode = "ARTIFACT_MISSING_OR_INVALID", +): Record { + if (value === null || typeof value !== "object" || Array.isArray(value)) { + fail(code, `${label} must be a JSON object`); + } + return value as Record; +} + +function validateExactFields( + record: Record, + fields: readonly string[], + label: string, +): void { + const expected = new Set(fields); + for (const field of fields) { + if (!Object.hasOwn(record, field)) { + fail("ARTIFACT_MISSING_OR_INVALID", `${label} is missing required field ${field}`); + } + } + const unexpected = Object.keys(record) + .filter((field) => !expected.has(field)) + .sort(); + if (unexpected.length > 0) { + fail("ARTIFACT_MISSING_OR_INVALID", `${label} contains unexpected field ${unexpected[0]}`); + } +} + +function requireString(record: Record, field: string, label: string): string { + const value = record[field]; + if (typeof value !== "string") { + fail("ARTIFACT_MISSING_OR_INVALID", `${label}.${field} must be a string`); + } + return value; +} + +function requireConstant( + actual: unknown, + expected: T, + field: string, + code: BrevLaunchableWorkspaceFailureCode = "PROVENANCE_MISMATCH", +): asserts actual is T { + if (actual !== expected) { + fail(code, `${field} must equal ${JSON.stringify(expected)}`); + } +} + +function requirePattern( + value: unknown, + field: string, + pattern: RegExp, + code: BrevLaunchableWorkspaceFailureCode = "ARTIFACT_MISSING_OR_INVALID", +): asserts value is string { + if (typeof value !== "string" || !pattern.test(value)) { + fail(code, `${field} has an invalid format`); + } +} + +function requireOpaqueAscii( + value: unknown, + field: string, + maxBytes: number, +): asserts value is string { + if ( + typeof value !== "string" || + value.length < 1 || + value.length > maxBytes || + value !== value.trim() || + !/^[\x20-\x7e]+$/u.test(value) + ) { + fail( + "ARTIFACT_MISSING_OR_INVALID", + `${field} must be 1-${maxBytes} bytes of trimmed printable ASCII`, + ); + } +} + +function parseCanonicalUtc(value: string, field: string): number { + if (!CANONICAL_UTC_PATTERN.test(value)) { + fail("ARTIFACT_MISSING_OR_INVALID", `${field} must be a canonical UTC timestamp`); + } + const timestamp = Date.parse(value); + if (!Number.isFinite(timestamp) || new Date(timestamp).toISOString() !== value) { + fail("ARTIFACT_MISSING_OR_INVALID", `${field} must be a real canonical UTC timestamp`); + } + return timestamp; +} + +function parseStrictEvidenceJson(contents: string, label: string): unknown { + if (Buffer.byteLength(contents, "utf8") > MAX_EVIDENCE_BYTES) { + fail("ARTIFACT_MISSING_OR_INVALID", `${label} exceeds ${MAX_EVIDENCE_BYTES} bytes`); + } + + let position = 0; + const invalid = (message: string): never => + fail("ARTIFACT_MISSING_OR_INVALID", `${label} ${message}`); + const skipWhitespace = (): void => { + while ( + contents[position] === " " || + contents[position] === "\t" || + contents[position] === "\r" || + contents[position] === "\n" + ) { + position += 1; + } + }; + const scanString = (): string => { + const start = position; + if (contents[position] !== '"') invalid("is not valid JSON"); + position += 1; + while (position < contents.length) { + const character = contents[position]; + if (character === '"') { + position += 1; + try { + const decoded = JSON.parse(contents.slice(start, position)) as unknown; + if (typeof decoded === "string") return decoded; + return invalid("is not valid JSON"); + } catch { + invalid("is not valid JSON"); + } + } + if (character === "\\") { + position += 2; + continue; + } + if (character === undefined || character.charCodeAt(0) < 0x20) { + invalid("is not valid JSON"); + } + position += 1; + } + return invalid("is not valid JSON"); + }; + + const scanValue = (containerDepth: number): void => { + skipWhitespace(); + const character = contents[position]; + if (character === '"') { + scanString(); + return; + } + if (character === "{" || character === "[") { + if (containerDepth >= MAX_JSON_DEPTH) { + invalid(`exceeds the maximum JSON depth of ${MAX_JSON_DEPTH}`); + } + const closing = character === "{" ? "}" : "]"; + const objectKeys = character === "{" ? new Set() : null; + position += 1; + skipWhitespace(); + if (contents[position] === closing) { + position += 1; + return; + } + while (position < contents.length) { + if (objectKeys) { + if (contents[position] !== '"') invalid("is not valid JSON"); + const key = scanString(); + if (objectKeys.has(key)) invalid(`contains duplicate key ${JSON.stringify(key)}`); + objectKeys.add(key); + skipWhitespace(); + if (contents[position] !== ":") invalid("is not valid JSON"); + position += 1; + } + scanValue(containerDepth + 1); + skipWhitespace(); + if (contents[position] === closing) { + position += 1; + return; + } + if (contents[position] !== ",") invalid("is not valid JSON"); + position += 1; + skipWhitespace(); + } + invalid("is not valid JSON"); + } + + const start = position; + while ( + position < contents.length && + contents[position] !== " " && + contents[position] !== "\t" && + contents[position] !== "\r" && + contents[position] !== "\n" && + contents[position] !== "," && + contents[position] !== "]" && + contents[position] !== "}" + ) { + position += 1; + } + if (position === start) invalid("is not valid JSON"); + }; + + scanValue(0); + skipWhitespace(); + if (position !== contents.length) invalid("contains trailing transport output"); + try { + return JSON.parse(contents) as unknown; + } catch { + return invalid("is not valid JSON"); + } +} + +function validateWindow(notBeforeMs: number, notAfterMs: number): void { + if ( + !Number.isSafeInteger(notBeforeMs) || + !Number.isSafeInteger(notAfterMs) || + notBeforeMs < 0 || + notAfterMs < notBeforeMs + ) { + fail("REQUEST_INVALID", "trusted evidence window is invalid"); + } +} + +function requireWithinWindow( + timestamp: number, + notBeforeMs: number, + notAfterMs: number, + field: string, +): void { + if (timestamp < notBeforeMs || timestamp > notAfterMs) { + fail("PROVENANCE_MISMATCH", `${field} is outside the trusted evidence window`); + } +} + +function buildImageIdentity(value: unknown, label: string): BrevImageIdentity { + const record = requireRecord(value, label); + validateExactFields(record, IMAGE_FIELDS, label); + const image: BrevImageIdentity = { + project: requireString(record, "project", label), + imageName: requireString(record, "imageName", label), + imageId: requireString(record, "imageId", label), + imageSelfLink: requireString(record, "imageSelfLink", label), + }; + if (!GCP_PROJECT_ID_PATTERN.test(image.project)) { + fail("BREV_IMAGE_RESOLUTION_MISMATCH", `${label}.project is not a canonical GCP project ID`); + } + requirePattern( + image.imageName, + `${label}.imageName`, + IMAGE_NAME_PATTERN, + "BREV_IMAGE_RESOLUTION_MISMATCH", + ); + requirePattern( + image.imageId, + `${label}.imageId`, + DECIMAL_ID_PATTERN, + "BREV_IMAGE_RESOLUTION_MISMATCH", + ); + const expectedSelfLink = `https://www.googleapis.com/compute/v1/projects/${image.project}/global/images/${image.imageName}`; + if (image.imageSelfLink !== expectedSelfLink) { + fail( + "BREV_IMAGE_RESOLUTION_MISMATCH", + `${label}.imageSelfLink does not exactly identify project/imageName`, + ); + } + return image; +} + +function validateExpectedImage(image: BrevImageIdentity): void { + try { + buildImageIdentity(image, "expected image"); + } catch (error) { + if (error instanceof BrevLaunchableWorkspaceError) { + fail("REQUEST_INVALID", error.message); + } + throw error; + } +} + +function assertExpected(actual: unknown, expected: unknown, field: string): void { + if (actual !== expected) { + fail("PROVENANCE_MISMATCH", `${field} does not match trusted controller state`); + } +} + +function assertExpectedImage( + actual: BrevImageIdentity, + expected: BrevImageIdentity, + label: string, +) { + for (const field of IMAGE_FIELDS) { + if (actual[field] !== expected[field]) { + fail( + "BREV_IMAGE_RESOLUTION_MISMATCH", + `${label}.${field} does not match the accepted exact image manifest`, + ); + } + } +} + +function validateReceiptExpectations(expected: BrevLaunchableWorkspaceReceiptExpectations): void { + requirePattern( + expected.correlationId, + "expected correlationId", + UUID_V4_PATTERN, + "REQUEST_INVALID", + ); + requirePattern(expected.nemoclawSha, "expected nemoclawSha", FULL_SHA_PATTERN, "REQUEST_INVALID"); + requirePattern( + expected.acceptedImageManifestSha256, + "expected acceptedImageManifestSha256", + SHA256_PATTERN, + "REQUEST_INVALID", + ); + for (const [value, field, maxBytes] of [ + [expected.organizationId, "expected organizationId", OPAQUE_ID_MAX_BYTES], + [expected.idempotencyKey, "expected idempotencyKey", OPAQUE_ID_MAX_BYTES], + [expected.launchableId, "expected launchableId", OPAQUE_ID_MAX_BYTES], + [expected.launchableRevision, "expected launchableRevision", REVISION_MAX_BYTES], + [expected.workspaceId, "expected workspaceId", OPAQUE_ID_MAX_BYTES], + [expected.suppliedImageReference, "expected suppliedImageReference", IMAGE_REFERENCE_MAX_BYTES], + ] as const) { + try { + requireOpaqueAscii(value, field, maxBytes); + } catch (error) { + if (error instanceof BrevLaunchableWorkspaceError) fail("REQUEST_INVALID", error.message); + throw error; + } + } + validateExpectedImage(expected.image); + const supportedFamilyReference = `projects/${expected.image.project}/global/images/family/${EXACT_IMAGE_STAGING_FAMILY}`; + if ( + expected.suppliedImageReference !== expected.image.imageSelfLink && + expected.suppliedImageReference !== supportedFamilyReference + ) { + fail( + "REQUEST_INVALID", + "expected suppliedImageReference must be the accepted immutable self-link or canonical staging family", + ); + } + validateWindow(expected.notBeforeMs, expected.notAfterMs); +} + +function buildReceipt(record: Record): BrevLaunchableWorkspaceReceipt { + requireConstant(record.schemaVersion, 1, "schemaVersion"); + requireConstant(record.kind, BREV_LAUNCHABLE_WORKSPACE_RECEIPT_KIND, "kind"); + + const launchable = requireRecord(record.launchable, "launchable"); + validateExactFields(launchable, LAUNCHABLE_FIELDS, "launchable"); + const launchableId = requireString(launchable, "id", "launchable"); + const launchableRevision = requireString(launchable, "revision", "launchable"); + requireOpaqueAscii(launchableId, "launchable.id", OPAQUE_ID_MAX_BYTES); + requireOpaqueAscii(launchableRevision, "launchable.revision", REVISION_MAX_BYTES); + + const workspace = requireRecord(record.workspace, "workspace"); + validateExactFields(workspace, WORKSPACE_FIELDS, "workspace"); + requireConstant(workspace.status, "READY", "workspace.status", "BREV_READINESS_FAILED"); + const workspaceId = requireString(workspace, "id", "workspace"); + const workspaceLaunchableId = requireString(workspace, "launchableId", "workspace"); + const workspaceLaunchableRevision = requireString(workspace, "launchableRevision", "workspace"); + requireOpaqueAscii(workspaceId, "workspace.id", OPAQUE_ID_MAX_BYTES); + requireOpaqueAscii(workspaceLaunchableId, "workspace.launchableId", OPAQUE_ID_MAX_BYTES); + requireOpaqueAscii( + workspaceLaunchableRevision, + "workspace.launchableRevision", + REVISION_MAX_BYTES, + ); + + const receipt: BrevLaunchableWorkspaceReceipt = { + schemaVersion: 1, + kind: BREV_LAUNCHABLE_WORKSPACE_RECEIPT_KIND, + correlationId: requireString(record, "correlationId", "receipt"), + nemoclawSha: requireString(record, "nemoclawSha", "receipt"), + acceptedImageManifestSha256: requireString(record, "acceptedImageManifestSha256", "receipt"), + organizationId: requireString(record, "organizationId", "receipt"), + idempotencyKey: requireString(record, "idempotencyKey", "receipt"), + launchable: { + id: launchableId, + revision: launchableRevision, + suppliedImageReference: requireString(launchable, "suppliedImageReference", "launchable"), + resolvedImage: buildImageIdentity(launchable.resolvedImage, "launchable.resolvedImage"), + }, + workspace: { + id: workspaceId, + launchableId: workspaceLaunchableId, + launchableRevision: workspaceLaunchableRevision, + bootImage: buildImageIdentity(workspace.bootImage, "workspace.bootImage"), + status: "READY", + }, + recordedAt: requireString(record, "recordedAt", "receipt"), + }; + + requirePattern(receipt.correlationId, "correlationId", UUID_V4_PATTERN); + requirePattern(receipt.nemoclawSha, "nemoclawSha", FULL_SHA_PATTERN); + requirePattern( + receipt.acceptedImageManifestSha256, + "acceptedImageManifestSha256", + SHA256_PATTERN, + ); + requireOpaqueAscii(receipt.organizationId, "organizationId", OPAQUE_ID_MAX_BYTES); + requireOpaqueAscii(receipt.idempotencyKey, "idempotencyKey", OPAQUE_ID_MAX_BYTES); + requireOpaqueAscii( + receipt.launchable.suppliedImageReference, + "launchable.suppliedImageReference", + IMAGE_REFERENCE_MAX_BYTES, + ); + parseCanonicalUtc(receipt.recordedAt, "recordedAt"); + return receipt; +} + +export function validateBrevLaunchableWorkspaceReceipt( + value: unknown, + expected: BrevLaunchableWorkspaceReceiptExpectations, +): BrevLaunchableWorkspaceReceipt { + // This is the normalized evidence boundary, not a Brev transport adapter. + // Adapters must retain separate bounded raw-response hashes and populate + // observed image/readiness fields only from authoritative platform readback. + validateReceiptExpectations(expected); + const record = requireRecord(value, "receipt"); + validateExactFields(record, RECEIPT_FIELDS, "receipt"); + const receipt = buildReceipt(record); + + for (const [actual, wanted, field] of [ + [receipt.correlationId, expected.correlationId, "correlationId"], + [receipt.nemoclawSha, expected.nemoclawSha, "nemoclawSha"], + [ + receipt.acceptedImageManifestSha256, + expected.acceptedImageManifestSha256, + "acceptedImageManifestSha256", + ], + [receipt.organizationId, expected.organizationId, "organizationId"], + [receipt.idempotencyKey, expected.idempotencyKey, "idempotencyKey"], + [receipt.launchable.id, expected.launchableId, "launchable.id"], + [receipt.launchable.revision, expected.launchableRevision, "launchable.revision"], + [receipt.workspace.id, expected.workspaceId, "workspace.id"], + [ + receipt.launchable.suppliedImageReference, + expected.suppliedImageReference, + "launchable.suppliedImageReference", + ], + ] as const) { + assertExpected(actual, wanted, field); + } + + if (receipt.workspace.launchableId !== receipt.launchable.id) { + fail( + "PROVENANCE_MISMATCH", + "workspace launchable ID does not match the provisioned Launchable", + ); + } + if (receipt.workspace.launchableRevision !== receipt.launchable.revision) { + fail( + "PROVENANCE_MISMATCH", + "workspace Launchable revision does not match the immutable provisioned revision", + ); + } + assertExpectedImage(receipt.launchable.resolvedImage, expected.image, "launchable.resolvedImage"); + assertExpectedImage(receipt.workspace.bootImage, expected.image, "workspace.bootImage"); + + const recordedAt = parseCanonicalUtc(receipt.recordedAt, "recordedAt"); + requireWithinWindow(recordedAt, expected.notBeforeMs, expected.notAfterMs, "recordedAt"); + return receipt; +} + +export function parseBrevLaunchableWorkspaceReceiptJson(contents: string): unknown { + return parseStrictEvidenceJson(contents, "receipt"); +} + +export function parseAndValidateBrevLaunchableWorkspaceReceipt( + contents: string, + expected: BrevLaunchableWorkspaceReceiptExpectations, +): BrevLaunchableWorkspaceReceipt { + return validateBrevLaunchableWorkspaceReceipt( + parseBrevLaunchableWorkspaceReceiptJson(contents), + expected, + ); +} + +export function normalizedBrevLaunchableWorkspaceReceiptJson( + receipt: BrevLaunchableWorkspaceReceipt, +): string { + return `${JSON.stringify(receipt, null, 2)}\n`; +} + +export function brevLaunchableWorkspaceReceiptSha256( + receipt: BrevLaunchableWorkspaceReceipt, +): string { + return createHash("sha256") + .update(normalizedBrevLaunchableWorkspaceReceiptJson(receipt)) + .digest("hex"); +} + +function validateCleanupExpectations(expected: BrevWorkspaceCleanupEvidenceExpectations): void { + requirePattern( + expected.correlationId, + "expected correlationId", + UUID_V4_PATTERN, + "REQUEST_INVALID", + ); + requirePattern( + expected.receiptSha256, + "expected receiptSha256", + SHA256_PATTERN, + "REQUEST_INVALID", + ); + for (const [value, field, maxBytes] of [ + [expected.organizationId, "expected organizationId", OPAQUE_ID_MAX_BYTES], + [expected.workspaceId, "expected workspaceId", OPAQUE_ID_MAX_BYTES], + [expected.launchableId, "expected launchableId", OPAQUE_ID_MAX_BYTES], + [expected.launchableRevision, "expected launchableRevision", REVISION_MAX_BYTES], + ] as const) { + try { + requireOpaqueAscii(value, field, maxBytes); + } catch (error) { + if (error instanceof BrevLaunchableWorkspaceError) fail("REQUEST_INVALID", error.message); + throw error; + } + } + validateWindow(expected.notBeforeMs, expected.notAfterMs); +} + +function buildCleanupEvidence(record: Record): BrevWorkspaceCleanupEvidence { + requireConstant(record.schemaVersion, 1, "schemaVersion"); + requireConstant(record.kind, BREV_WORKSPACE_CLEANUP_EVIDENCE_KIND, "kind"); + requireConstant(record.terminalState, "ABSENT", "terminalState", "BREV_CLEANUP_INCOMPLETE"); + const cleanup: BrevWorkspaceCleanupEvidence = { + schemaVersion: 1, + kind: BREV_WORKSPACE_CLEANUP_EVIDENCE_KIND, + correlationId: requireString(record, "correlationId", "cleanup evidence"), + receiptSha256: requireString(record, "receiptSha256", "cleanup evidence"), + organizationId: requireString(record, "organizationId", "cleanup evidence"), + workspaceId: requireString(record, "workspaceId", "cleanup evidence"), + launchableId: requireString(record, "launchableId", "cleanup evidence"), + launchableRevision: requireString(record, "launchableRevision", "cleanup evidence"), + deleteRequestedAt: requireString(record, "deleteRequestedAt", "cleanup evidence"), + terminalState: "ABSENT", + verifiedAt: requireString(record, "verifiedAt", "cleanup evidence"), + }; + requirePattern(cleanup.correlationId, "correlationId", UUID_V4_PATTERN); + requirePattern(cleanup.receiptSha256, "receiptSha256", SHA256_PATTERN); + requireOpaqueAscii(cleanup.organizationId, "organizationId", OPAQUE_ID_MAX_BYTES); + requireOpaqueAscii(cleanup.workspaceId, "workspaceId", OPAQUE_ID_MAX_BYTES); + requireOpaqueAscii(cleanup.launchableId, "launchableId", OPAQUE_ID_MAX_BYTES); + requireOpaqueAscii(cleanup.launchableRevision, "launchableRevision", REVISION_MAX_BYTES); + parseCanonicalUtc(cleanup.deleteRequestedAt, "deleteRequestedAt"); + parseCanonicalUtc(cleanup.verifiedAt, "verifiedAt"); + return cleanup; +} + +export function validateBrevWorkspaceCleanupEvidence( + value: unknown, + expected: BrevWorkspaceCleanupEvidenceExpectations, +): BrevWorkspaceCleanupEvidence { + validateCleanupExpectations(expected); + const record = requireRecord(value, "cleanup evidence"); + validateExactFields(record, CLEANUP_FIELDS, "cleanup evidence"); + const cleanup = buildCleanupEvidence(record); + + for (const [actual, wanted, field] of [ + [cleanup.correlationId, expected.correlationId, "correlationId"], + [cleanup.receiptSha256, expected.receiptSha256, "receiptSha256"], + [cleanup.organizationId, expected.organizationId, "organizationId"], + [cleanup.workspaceId, expected.workspaceId, "workspaceId"], + [cleanup.launchableId, expected.launchableId, "launchableId"], + [cleanup.launchableRevision, expected.launchableRevision, "launchableRevision"], + ] as const) { + assertExpected(actual, wanted, field); + } + + const deleteRequestedAt = parseCanonicalUtc(cleanup.deleteRequestedAt, "deleteRequestedAt"); + const verifiedAt = parseCanonicalUtc(cleanup.verifiedAt, "verifiedAt"); + requireWithinWindow( + deleteRequestedAt, + expected.notBeforeMs, + expected.notAfterMs, + "deleteRequestedAt", + ); + requireWithinWindow(verifiedAt, expected.notBeforeMs, expected.notAfterMs, "verifiedAt"); + if (verifiedAt <= deleteRequestedAt) { + fail("BREV_CLEANUP_INCOMPLETE", "verifiedAt must follow deleteRequestedAt"); + } + return cleanup; +} + +export function parseBrevWorkspaceCleanupEvidenceJson(contents: string): unknown { + return parseStrictEvidenceJson(contents, "cleanup evidence"); +} + +export function parseAndValidateBrevWorkspaceCleanupEvidence( + contents: string, + expected: BrevWorkspaceCleanupEvidenceExpectations, +): BrevWorkspaceCleanupEvidence { + return validateBrevWorkspaceCleanupEvidence( + parseBrevWorkspaceCleanupEvidenceJson(contents), + expected, + ); +} + +export function normalizedBrevWorkspaceCleanupEvidenceJson( + evidence: BrevWorkspaceCleanupEvidence, +): string { + return `${JSON.stringify(evidence, null, 2)}\n`; +} + +export function brevLaunchableWorkspaceFailureCode( + error: unknown, +): BrevLaunchableWorkspaceFailureCode { + return error instanceof BrevLaunchableWorkspaceError ? error.code : "UNKNOWN"; +} From 325daa70554d7bb6d053b049a7a11f9f5dae5dc8 Mon Sep 17 00:00:00 2001 From: Charan Jagwani Date: Thu, 16 Jul 2026 13:48:43 -0700 Subject: [PATCH 06/25] chore(e2e): defer launchable receipt validator Signed-off-by: Charan Jagwani --- ...ev-launchable-workspace-receipt-fixture.ts | 92 --- .../brev-launchable-workspace-receipt.test.ts | 516 ------------ .../e2e/brev-launchable-workspace-receipt.mts | 763 ------------------ 3 files changed, 1371 deletions(-) delete mode 100644 test/e2e/support/brev-launchable-workspace-receipt-fixture.ts delete mode 100644 test/e2e/support/brev-launchable-workspace-receipt.test.ts delete mode 100644 tools/e2e/brev-launchable-workspace-receipt.mts diff --git a/test/e2e/support/brev-launchable-workspace-receipt-fixture.ts b/test/e2e/support/brev-launchable-workspace-receipt-fixture.ts deleted file mode 100644 index 5a47933b49..0000000000 --- a/test/e2e/support/brev-launchable-workspace-receipt-fixture.ts +++ /dev/null @@ -1,92 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import type { - BrevLaunchableWorkspaceReceipt, - BrevLaunchableWorkspaceReceiptExpectations, - BrevWorkspaceCleanupEvidence, -} from "../../../tools/e2e/brev-launchable-workspace-receipt.mts"; -import { brevLaunchableWorkspaceReceiptSha256 } from "../../../tools/e2e/brev-launchable-workspace-receipt.mts"; - -export const BREV_RECEIPT_CORRELATION_ID = "12345678-1234-4123-8123-123456789abc"; -export const BREV_RECEIPT_CANDIDATE_SHA = "a".repeat(40); -export const BREV_RECEIPT_WINDOW_START = Date.parse("2026-07-16T12:00:00.000Z"); -export const BREV_RECEIPT_WINDOW_END = Date.parse("2026-07-16T13:00:00.000Z"); -export const BREV_CLEANUP_DEADLINE = Date.parse("2026-07-16T13:15:00.000Z"); - -export const BREV_IMAGE = { - project: "brevdevprod", - imageName: "nemoclaw-brev-cpu-v0-1-0-20260716-a-staging-190-1", - imageId: "12345678901234567890", - imageSelfLink: - "https://www.googleapis.com/compute/v1/projects/brevdevprod/global/images/nemoclaw-brev-cpu-v0-1-0-20260716-a-staging-190-1", -} as const; - -export function brevLaunchableWorkspaceReceipt( - overrides: Partial = {}, -): BrevLaunchableWorkspaceReceipt { - return { - schemaVersion: 1, - kind: "nemoclaw-brev-launchable-workspace-receipt", - correlationId: BREV_RECEIPT_CORRELATION_ID, - nemoclawSha: BREV_RECEIPT_CANDIDATE_SHA, - acceptedImageManifestSha256: "c".repeat(64), - organizationId: "org-NemoClaw-staging", - idempotencyKey: "qualification-run-1", - launchable: { - id: "env-3Azt0aYgVNFEuz7opyx3gscmowS", - revision: "revision-01JZZZZZZZZZZZZZZZZZZZZZZZ", - suppliedImageReference: BREV_IMAGE.imageSelfLink, - resolvedImage: { ...BREV_IMAGE }, - }, - workspace: { - id: "workspace-01JZZZZZZZZZZZZZZZZZZZZZZZ", - launchableId: "env-3Azt0aYgVNFEuz7opyx3gscmowS", - launchableRevision: "revision-01JZZZZZZZZZZZZZZZZZZZZZZZ", - bootImage: { ...BREV_IMAGE }, - status: "READY", - }, - recordedAt: "2026-07-16T12:45:00.000Z", - ...overrides, - }; -} - -export function brevLaunchableWorkspaceReceiptExpectations( - overrides: Partial = {}, -): BrevLaunchableWorkspaceReceiptExpectations { - return { - correlationId: BREV_RECEIPT_CORRELATION_ID, - nemoclawSha: BREV_RECEIPT_CANDIDATE_SHA, - acceptedImageManifestSha256: "c".repeat(64), - organizationId: "org-NemoClaw-staging", - idempotencyKey: "qualification-run-1", - launchableId: "env-3Azt0aYgVNFEuz7opyx3gscmowS", - launchableRevision: "revision-01JZZZZZZZZZZZZZZZZZZZZZZZ", - workspaceId: "workspace-01JZZZZZZZZZZZZZZZZZZZZZZZ", - suppliedImageReference: BREV_IMAGE.imageSelfLink, - image: { ...BREV_IMAGE }, - notBeforeMs: BREV_RECEIPT_WINDOW_START, - notAfterMs: BREV_RECEIPT_WINDOW_END, - ...overrides, - }; -} - -export function brevWorkspaceCleanupEvidence( - receipt = brevLaunchableWorkspaceReceipt(), - overrides: Partial = {}, -): BrevWorkspaceCleanupEvidence { - return { - schemaVersion: 1, - kind: "nemoclaw-brev-workspace-cleanup-evidence", - correlationId: receipt.correlationId, - receiptSha256: brevLaunchableWorkspaceReceiptSha256(receipt), - organizationId: receipt.organizationId, - workspaceId: receipt.workspace.id, - launchableId: receipt.launchable.id, - launchableRevision: receipt.launchable.revision, - deleteRequestedAt: "2026-07-16T13:01:00.000Z", - terminalState: "ABSENT", - verifiedAt: "2026-07-16T13:04:00.000Z", - ...overrides, - }; -} diff --git a/test/e2e/support/brev-launchable-workspace-receipt.test.ts b/test/e2e/support/brev-launchable-workspace-receipt.test.ts deleted file mode 100644 index c1fc7f404f..0000000000 --- a/test/e2e/support/brev-launchable-workspace-receipt.test.ts +++ /dev/null @@ -1,516 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { describe, expect, it } from "vitest"; - -import { - BrevLaunchableWorkspaceError, - brevLaunchableWorkspaceReceiptSha256, - normalizedBrevLaunchableWorkspaceReceiptJson, - normalizedBrevWorkspaceCleanupEvidenceJson, - parseAndValidateBrevLaunchableWorkspaceReceipt, - parseAndValidateBrevWorkspaceCleanupEvidence, - validateBrevLaunchableWorkspaceReceipt, - validateBrevWorkspaceCleanupEvidence, -} from "../../../tools/e2e/brev-launchable-workspace-receipt.mts"; -import { - BREV_CLEANUP_DEADLINE, - BREV_IMAGE, - BREV_RECEIPT_WINDOW_END, - brevLaunchableWorkspaceReceipt, - brevLaunchableWorkspaceReceiptExpectations, - brevWorkspaceCleanupEvidence, -} from "./brev-launchable-workspace-receipt-fixture.ts"; - -function expectCode(run: () => unknown, code: BrevLaunchableWorkspaceError["code"]): void { - try { - run(); - } catch (error) { - expect(error).toBeInstanceOf(BrevLaunchableWorkspaceError); - expect((error as BrevLaunchableWorkspaceError).code).toBe(code); - return; - } - throw new Error(`expected ${code}`); -} - -describe("Brev Launchable and workspace evidence", () => { - it("accepts only a ready workspace bound to one immutable Launchable revision and exact image", () => { - const source = brevLaunchableWorkspaceReceipt(); - const accepted = validateBrevLaunchableWorkspaceReceipt( - source, - brevLaunchableWorkspaceReceiptExpectations(), - ); - - expect(accepted).toEqual(source); - expect(accepted.workspace.launchableRevision).toBe(accepted.launchable.revision); - expect(accepted.workspace.bootImage.imageId).toBe(BREV_IMAGE.imageId); - expect(normalizedBrevLaunchableWorkspaceReceiptJson(accepted)).toBe( - `${JSON.stringify(source, null, 2)}\n`, - ); - expect(brevLaunchableWorkspaceReceiptSha256(accepted)).toMatch(/^[0-9a-f]{64}$/u); - }); - - it("rejects missing and additional fields at every receipt layer", () => { - const missing = { ...brevLaunchableWorkspaceReceipt() } as Record; - delete missing.acceptedImageManifestSha256; - expectCode( - () => - validateBrevLaunchableWorkspaceReceipt( - missing, - brevLaunchableWorkspaceReceiptExpectations(), - ), - "ARTIFACT_MISSING_OR_INVALID", - ); - - const source = brevLaunchableWorkspaceReceipt(); - const additional = { - ...source, - launchable: { ...source.launchable, mutableFamilyWasCloseEnough: true }, - }; - expectCode( - () => - validateBrevLaunchableWorkspaceReceipt( - additional, - brevLaunchableWorkspaceReceiptExpectations(), - ), - "ARTIFACT_MISSING_OR_INVALID", - ); - - const extraImage = { - ...source, - workspace: { - ...source.workspace, - bootImage: { ...source.workspace.bootImage, imageFamily: "nemoclaw-brev-staging-cpu" }, - }, - }; - expectCode( - () => - validateBrevLaunchableWorkspaceReceipt( - extraImage, - brevLaunchableWorkspaceReceiptExpectations(), - ), - "ARTIFACT_MISSING_OR_INVALID", - ); - - for (const receipt of [ - { - ...source, - launchable: Object.fromEntries( - Object.entries(source.launchable).filter(([field]) => field !== "revision"), - ), - }, - { - ...source, - workspace: Object.fromEntries( - Object.entries(source.workspace).filter(([field]) => field !== "id"), - ), - }, - { - ...source, - launchable: { - ...source.launchable, - resolvedImage: Object.fromEntries( - Object.entries(source.launchable.resolvedImage).filter( - ([field]) => field !== "imageId", - ), - ), - }, - }, - { - ...source, - workspace: { - ...source.workspace, - bootImage: Object.fromEntries( - Object.entries(source.workspace.bootImage).filter( - ([field]) => field !== "imageSelfLink", - ), - ), - }, - }, - ]) { - expectCode( - () => - validateBrevLaunchableWorkspaceReceipt( - receipt, - brevLaunchableWorkspaceReceiptExpectations(), - ), - "ARTIFACT_MISSING_OR_INVALID", - ); - } - }); - - it.each([ - ["correlation ID", { correlationId: "12345678-1234-1123-8123-123456789abc" }], - ["candidate SHA", { nemoclawSha: "A".repeat(40) }], - ["manifest hash", { acceptedImageManifestSha256: "b".repeat(63) }], - ["organization control character", { organizationId: "org\nother" }], - ])("rejects an invalid %s", (_name, overrides) => { - expectCode( - () => - validateBrevLaunchableWorkspaceReceipt( - { ...brevLaunchableWorkspaceReceipt(), ...overrides }, - brevLaunchableWorkspaceReceiptExpectations(), - ), - "ARTIFACT_MISSING_OR_INVALID", - ); - }); - - it("binds request, manifest, organization, idempotency, and supplied reference to trusted state", () => { - const mismatches = [ - { correlationId: "87654321-4321-4321-8321-cba987654321" }, - { nemoclawSha: "d".repeat(40) }, - { acceptedImageManifestSha256: "f".repeat(64) }, - { organizationId: "org-other" }, - { idempotencyKey: "qualification-other" }, - { launchableId: "env-other" }, - { launchableRevision: "revision-other" }, - { workspaceId: "workspace-other" }, - { - suppliedImageReference: - "projects/brevdevprod/global/images/family/nemoclaw-brev-staging-cpu", - }, - ]; - for (const expected of mismatches) { - expectCode( - () => - validateBrevLaunchableWorkspaceReceipt( - brevLaunchableWorkspaceReceipt(), - brevLaunchableWorkspaceReceiptExpectations(expected), - ), - "PROVENANCE_MISMATCH", - ); - } - }); - - it("accepts only the immutable image or canonical staging family as the trusted supplied reference", () => { - expectCode( - () => - validateBrevLaunchableWorkspaceReceipt( - brevLaunchableWorkspaceReceipt(), - brevLaunchableWorkspaceReceiptExpectations({ - suppliedImageReference: "https://attacker.invalid/image", - }), - ), - "REQUEST_INVALID", - ); - - const familyReference = "projects/brevdevprod/global/images/family/nemoclaw-brev-staging-cpu"; - const familyReceipt = brevLaunchableWorkspaceReceipt({ - launchable: { - ...brevLaunchableWorkspaceReceipt().launchable, - suppliedImageReference: familyReference, - }, - }); - expect( - validateBrevLaunchableWorkspaceReceipt( - familyReceipt, - brevLaunchableWorkspaceReceiptExpectations({ suppliedImageReference: familyReference }), - ), - ).toEqual(familyReceipt); - }); - - it("requires structured workspace readiness", () => { - const source = brevLaunchableWorkspaceReceipt(); - expectCode( - () => - validateBrevLaunchableWorkspaceReceipt( - { ...source, workspace: { ...source.workspace, status: "RUNNING" } }, - brevLaunchableWorkspaceReceiptExpectations(), - ), - "BREV_READINESS_FAILED", - ); - }); - - it("binds opaque Brev IDs and revision to durable observations and workspace readback", () => { - const source = brevLaunchableWorkspaceReceipt(); - for (const receipt of [ - { - ...source, - workspace: { ...source.workspace, launchableRevision: "revision-other" }, - }, - { ...source, workspace: { ...source.workspace, launchableId: "env-other" } }, - ]) { - expectCode( - () => - validateBrevLaunchableWorkspaceReceipt( - receipt, - brevLaunchableWorkspaceReceiptExpectations(), - ), - "PROVENANCE_MISMATCH", - ); - } - expectCode( - () => - validateBrevLaunchableWorkspaceReceipt( - { ...source, launchable: { ...source.launchable, revision: "current" } }, - brevLaunchableWorkspaceReceiptExpectations(), - ), - "PROVENANCE_MISMATCH", - ); - }); - - it("rejects family-only, name-only, same-name replacement, and different boot-image evidence", () => { - const source = brevLaunchableWorkspaceReceipt(); - const mismatches = [ - { - ...source, - launchable: { - ...source.launchable, - resolvedImage: { - ...source.launchable.resolvedImage, - imageSelfLink: - "https://www.googleapis.com/compute/v1/projects/brevdevprod/global/images/family/nemoclaw-brev-staging-cpu", - }, - }, - }, - { - ...source, - workspace: { - ...source.workspace, - bootImage: { ...source.workspace.bootImage, imageId: "99999999999999999999" }, - }, - }, - { - ...source, - workspace: { - ...source.workspace, - bootImage: { ...source.workspace.bootImage, imageName: "different-image" }, - }, - }, - ]; - for (const receipt of mismatches) { - expectCode( - () => - validateBrevLaunchableWorkspaceReceipt( - receipt, - brevLaunchableWorkspaceReceiptExpectations(), - ), - "BREV_IMAGE_RESOLUTION_MISMATCH", - ); - } - }); - - it("requires one real canonical UTC receipt time inside the trusted window", () => { - for (const recordedAt of [ - "2026-02-30T12:00:00.000Z", - "2026-07-16T12:45:00Z", - "2026-07-16T12:45:00.000+00:00", - "2026-07-16T13:00:00.001Z", - ]) { - expectCode( - () => - validateBrevLaunchableWorkspaceReceipt( - brevLaunchableWorkspaceReceipt({ recordedAt }), - brevLaunchableWorkspaceReceiptExpectations(), - ), - recordedAt === "2026-07-16T13:00:00.001Z" - ? "PROVENANCE_MISMATCH" - : "ARTIFACT_MISSING_OR_INVALID", - ); - } - expect( - validateBrevLaunchableWorkspaceReceipt( - brevLaunchableWorkspaceReceipt({ - recordedAt: new Date(BREV_RECEIPT_WINDOW_END).toISOString(), - }), - brevLaunchableWorkspaceReceiptExpectations(), - ).recordedAt, - ).toBe("2026-07-16T13:00:00.000Z"); - }); - - it("parses bounded strict JSON and rejects malformed, duplicate, deep, and trailing data", () => { - const source = brevLaunchableWorkspaceReceipt(); - expect( - parseAndValidateBrevLaunchableWorkspaceReceipt( - JSON.stringify(source), - brevLaunchableWorkspaceReceiptExpectations(), - ), - ).toEqual(source); - for (const contents of [ - "{not-json", - '{"a":1,"\\u0061":2}', - "[[[[[[[0]]]]]]]", - `${JSON.stringify(source)}\ntransport output`, - " ".repeat(64 * 1024 + 1), - ]) { - expectCode( - () => - parseAndValidateBrevLaunchableWorkspaceReceipt( - contents, - brevLaunchableWorkspaceReceiptExpectations(), - ), - "ARTIFACT_MISSING_OR_INVALID", - ); - } - }); - - it("accepts terminal absence only when cleanup is bound to the immutable receipt and IDs", () => { - const receipt = brevLaunchableWorkspaceReceipt(); - const cleanup = brevWorkspaceCleanupEvidence(receipt); - const accepted = validateBrevWorkspaceCleanupEvidence(cleanup, { - correlationId: receipt.correlationId, - receiptSha256: brevLaunchableWorkspaceReceiptSha256(receipt), - organizationId: receipt.organizationId, - workspaceId: receipt.workspace.id, - launchableId: receipt.launchable.id, - launchableRevision: receipt.launchable.revision, - notBeforeMs: Date.parse(receipt.recordedAt), - notAfterMs: BREV_CLEANUP_DEADLINE, - }); - - expect(accepted).toEqual(cleanup); - expect(normalizedBrevWorkspaceCleanupEvidenceJson(accepted)).toBe( - `${JSON.stringify(cleanup, null, 2)}\n`, - ); - expect( - parseAndValidateBrevWorkspaceCleanupEvidence(JSON.stringify(cleanup), { - correlationId: receipt.correlationId, - receiptSha256: brevLaunchableWorkspaceReceiptSha256(receipt), - organizationId: receipt.organizationId, - workspaceId: receipt.workspace.id, - launchableId: receipt.launchable.id, - launchableRevision: receipt.launchable.revision, - notBeforeMs: Date.parse(receipt.recordedAt), - notAfterMs: BREV_CLEANUP_DEADLINE, - }), - ).toEqual(cleanup); - }); - - it("rejects delete acknowledgement, deletion in progress, false absence, and unbound cleanup", () => { - const receipt = brevLaunchableWorkspaceReceipt(); - const expected = { - correlationId: receipt.correlationId, - receiptSha256: brevLaunchableWorkspaceReceiptSha256(receipt), - organizationId: receipt.organizationId, - workspaceId: receipt.workspace.id, - launchableId: receipt.launchable.id, - launchableRevision: receipt.launchable.revision, - notBeforeMs: Date.parse(receipt.recordedAt), - notAfterMs: BREV_CLEANUP_DEADLINE, - }; - - for (const cleanup of [ - { ...brevWorkspaceCleanupEvidence(receipt), terminalState: "DELETE_ACCEPTED" }, - { ...brevWorkspaceCleanupEvidence(receipt), terminalState: "DELETING" }, - ]) { - expectCode( - () => validateBrevWorkspaceCleanupEvidence(cleanup, expected), - "BREV_CLEANUP_INCOMPLETE", - ); - } - for (const cleanup of [ - brevWorkspaceCleanupEvidence(receipt, { receiptSha256: "f".repeat(64) }), - brevWorkspaceCleanupEvidence(receipt, { workspaceId: "workspace-other" }), - brevWorkspaceCleanupEvidence(receipt, { launchableRevision: "revision-other" }), - ]) { - expectCode( - () => validateBrevWorkspaceCleanupEvidence(cleanup, expected), - "PROVENANCE_MISMATCH", - ); - } - - const missing = { ...brevWorkspaceCleanupEvidence(receipt) } as Record; - delete missing.receiptSha256; - for (const cleanup of [ - missing, - { ...brevWorkspaceCleanupEvidence(receipt), diagnostic: "ok" }, - ]) { - expectCode( - () => validateBrevWorkspaceCleanupEvidence(cleanup, expected), - "ARTIFACT_MISSING_OR_INVALID", - ); - } - }); - - it("requires ordered cleanup timestamps inside the controller deadline", () => { - const receipt = brevLaunchableWorkspaceReceipt(); - const expected = { - correlationId: receipt.correlationId, - receiptSha256: brevLaunchableWorkspaceReceiptSha256(receipt), - organizationId: receipt.organizationId, - workspaceId: receipt.workspace.id, - launchableId: receipt.launchable.id, - launchableRevision: receipt.launchable.revision, - notBeforeMs: Date.parse(receipt.recordedAt), - notAfterMs: BREV_CLEANUP_DEADLINE, - }; - for (const cleanup of [ - brevWorkspaceCleanupEvidence(receipt, { - deleteRequestedAt: "2026-07-16T13:05:00.000Z", - verifiedAt: "2026-07-16T13:04:00.000Z", - }), - brevWorkspaceCleanupEvidence(receipt, { verifiedAt: "2026-07-16T13:15:00.001Z" }), - ]) { - expectCode( - () => validateBrevWorkspaceCleanupEvidence(cleanup, expected), - cleanup.verifiedAt.endsWith("001Z") ? "PROVENANCE_MISMATCH" : "BREV_CLEANUP_INCOMPLETE", - ); - } - expectCode( - () => - validateBrevWorkspaceCleanupEvidence( - brevWorkspaceCleanupEvidence(receipt, { - deleteRequestedAt: "2026-07-16T13:04:00.000Z", - verifiedAt: "2026-07-16T13:04:00.000Z", - }), - expected, - ), - "BREV_CLEANUP_INCOMPLETE", - ); - expectCode( - () => - validateBrevWorkspaceCleanupEvidence( - brevWorkspaceCleanupEvidence(receipt, { - deleteRequestedAt: "2026-07-16T12:44:59.999Z", - }), - expected, - ), - "PROVENANCE_MISMATCH", - ); - }); - - it("requires separately anchored cleanup and rejects consistent receipt plus cleanup tampering", () => { - const receipt = brevLaunchableWorkspaceReceipt(); - const cleanupExpected = { - correlationId: receipt.correlationId, - receiptSha256: brevLaunchableWorkspaceReceiptSha256(receipt), - organizationId: receipt.organizationId, - workspaceId: receipt.workspace.id, - launchableId: receipt.launchable.id, - launchableRevision: receipt.launchable.revision, - notBeforeMs: Date.parse(receipt.recordedAt), - notAfterMs: BREV_CLEANUP_DEADLINE, - }; - expectCode( - () => validateBrevWorkspaceCleanupEvidence(undefined, cleanupExpected), - "ARTIFACT_MISSING_OR_INVALID", - ); - - const forged = brevLaunchableWorkspaceReceipt({ - launchable: { - ...receipt.launchable, - id: "env-forged", - revision: "revision-forged", - }, - workspace: { - ...receipt.workspace, - id: "workspace-forged", - launchableId: "env-forged", - launchableRevision: "revision-forged", - }, - }); - expectCode( - () => - validateBrevLaunchableWorkspaceReceipt( - forged, - brevLaunchableWorkspaceReceiptExpectations(), - ), - "PROVENANCE_MISMATCH", - ); - expectCode( - () => - validateBrevWorkspaceCleanupEvidence(brevWorkspaceCleanupEvidence(forged), cleanupExpected), - "PROVENANCE_MISMATCH", - ); - }); -}); diff --git a/tools/e2e/brev-launchable-workspace-receipt.mts b/tools/e2e/brev-launchable-workspace-receipt.mts deleted file mode 100644 index 90a242041f..0000000000 --- a/tools/e2e/brev-launchable-workspace-receipt.mts +++ /dev/null @@ -1,763 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { createHash } from "node:crypto"; - -import { EXACT_IMAGE_STAGING_FAMILY } from "./exact-image-manifest.mts"; - -export const BREV_LAUNCHABLE_WORKSPACE_RECEIPT_KIND = "nemoclaw-brev-launchable-workspace-receipt"; -export const BREV_WORKSPACE_CLEANUP_EVIDENCE_KIND = "nemoclaw-brev-workspace-cleanup-evidence"; - -export type BrevLaunchableWorkspaceFailureCode = - | "REQUEST_INVALID" - | "ARTIFACT_MISSING_OR_INVALID" - | "PROVENANCE_MISMATCH" - | "BREV_IMAGE_RESOLUTION_MISMATCH" - | "BREV_READINESS_FAILED" - | "BREV_CLEANUP_INCOMPLETE" - | "UNKNOWN"; - -export class BrevLaunchableWorkspaceError extends Error { - readonly code: BrevLaunchableWorkspaceFailureCode; - - constructor(code: BrevLaunchableWorkspaceFailureCode, message: string) { - super(message); - this.name = "BrevLaunchableWorkspaceError"; - this.code = code; - } -} - -export type BrevImageIdentity = { - project: string; - imageName: string; - imageId: string; - imageSelfLink: string; -}; - -// resolvedImage and bootImage are independently observed Brev/platform -// readbacks. A transport adapter must never populate either object by copying -// the accepted manifest or the supplied Launchable configuration. -export type BrevLaunchableWorkspaceReceipt = { - schemaVersion: 1; - kind: typeof BREV_LAUNCHABLE_WORKSPACE_RECEIPT_KIND; - correlationId: string; - nemoclawSha: string; - acceptedImageManifestSha256: string; - organizationId: string; - idempotencyKey: string; - launchable: { - id: string; - revision: string; - suppliedImageReference: string; - resolvedImage: BrevImageIdentity; - }; - workspace: { - id: string; - launchableId: string; - launchableRevision: string; - bootImage: BrevImageIdentity; - status: "READY"; - }; - recordedAt: string; -}; - -export type BrevLaunchableWorkspaceReceiptExpectations = { - correlationId: string; - nemoclawSha: string; - // Hash of the normalized manifest already accepted by the image controller. - acceptedImageManifestSha256: string; - organizationId: string; - idempotencyKey: string; - // These generated identities must come from separately persisted, - // authoritative Brev operation results. Never derive them from the receipt - // being validated. - launchableId: string; - launchableRevision: string; - workspaceId: string; - suppliedImageReference: string; - image: BrevImageIdentity; - notBeforeMs: number; - notAfterMs: number; -}; - -export type BrevWorkspaceCleanupEvidence = { - schemaVersion: 1; - kind: typeof BREV_WORKSPACE_CLEANUP_EVIDENCE_KIND; - correlationId: string; - receiptSha256: string; - organizationId: string; - workspaceId: string; - launchableId: string; - launchableRevision: string; - deleteRequestedAt: string; - terminalState: "ABSENT"; - verifiedAt: string; -}; - -export type BrevWorkspaceCleanupEvidenceExpectations = { - // The controller persists this full tuple before issuing delete. Cleanup - // validation must not derive it from the cleanup artifact being validated. - correlationId: string; - receiptSha256: string; - organizationId: string; - workspaceId: string; - launchableId: string; - launchableRevision: string; - notBeforeMs: number; - notAfterMs: number; -}; - -const RECEIPT_FIELDS = [ - "schemaVersion", - "kind", - "correlationId", - "nemoclawSha", - "acceptedImageManifestSha256", - "organizationId", - "idempotencyKey", - "launchable", - "workspace", - "recordedAt", -] as const; -const LAUNCHABLE_FIELDS = ["id", "revision", "suppliedImageReference", "resolvedImage"] as const; -const WORKSPACE_FIELDS = [ - "id", - "launchableId", - "launchableRevision", - "bootImage", - "status", -] as const; -const IMAGE_FIELDS = ["project", "imageName", "imageId", "imageSelfLink"] as const; -const CLEANUP_FIELDS = [ - "schemaVersion", - "kind", - "correlationId", - "receiptSha256", - "organizationId", - "workspaceId", - "launchableId", - "launchableRevision", - "deleteRequestedAt", - "terminalState", - "verifiedAt", -] as const; - -const FULL_SHA_PATTERN = /^[0-9a-f]{40}$/u; -const SHA256_PATTERN = /^[0-9a-f]{64}$/u; -const UUID_V4_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/u; -const DECIMAL_ID_PATTERN = /^[1-9][0-9]*$/u; -const GCP_PROJECT_ID_PATTERN = /^[a-z][a-z0-9-]{4,28}[a-z0-9]$/u; -const IMAGE_NAME_PATTERN = /^[a-z](?:[-a-z0-9]{0,61}[a-z0-9])?$/u; -const CANONICAL_UTC_PATTERN = - /^(\d{4})-(\d{2})-(\d{2})T([01]\d|2[0-3]):([0-5]\d):([0-5]\d)[.](\d{3})Z$/u; -const OPAQUE_ID_MAX_BYTES = 256; -const REVISION_MAX_BYTES = 512; -const IMAGE_REFERENCE_MAX_BYTES = 2_048; -const MAX_EVIDENCE_BYTES = 64 * 1024; -const MAX_JSON_DEPTH = 6; - -function fail(code: BrevLaunchableWorkspaceFailureCode, message: string): never { - throw new BrevLaunchableWorkspaceError(code, message); -} - -function requireRecord( - value: unknown, - label: string, - code: BrevLaunchableWorkspaceFailureCode = "ARTIFACT_MISSING_OR_INVALID", -): Record { - if (value === null || typeof value !== "object" || Array.isArray(value)) { - fail(code, `${label} must be a JSON object`); - } - return value as Record; -} - -function validateExactFields( - record: Record, - fields: readonly string[], - label: string, -): void { - const expected = new Set(fields); - for (const field of fields) { - if (!Object.hasOwn(record, field)) { - fail("ARTIFACT_MISSING_OR_INVALID", `${label} is missing required field ${field}`); - } - } - const unexpected = Object.keys(record) - .filter((field) => !expected.has(field)) - .sort(); - if (unexpected.length > 0) { - fail("ARTIFACT_MISSING_OR_INVALID", `${label} contains unexpected field ${unexpected[0]}`); - } -} - -function requireString(record: Record, field: string, label: string): string { - const value = record[field]; - if (typeof value !== "string") { - fail("ARTIFACT_MISSING_OR_INVALID", `${label}.${field} must be a string`); - } - return value; -} - -function requireConstant( - actual: unknown, - expected: T, - field: string, - code: BrevLaunchableWorkspaceFailureCode = "PROVENANCE_MISMATCH", -): asserts actual is T { - if (actual !== expected) { - fail(code, `${field} must equal ${JSON.stringify(expected)}`); - } -} - -function requirePattern( - value: unknown, - field: string, - pattern: RegExp, - code: BrevLaunchableWorkspaceFailureCode = "ARTIFACT_MISSING_OR_INVALID", -): asserts value is string { - if (typeof value !== "string" || !pattern.test(value)) { - fail(code, `${field} has an invalid format`); - } -} - -function requireOpaqueAscii( - value: unknown, - field: string, - maxBytes: number, -): asserts value is string { - if ( - typeof value !== "string" || - value.length < 1 || - value.length > maxBytes || - value !== value.trim() || - !/^[\x20-\x7e]+$/u.test(value) - ) { - fail( - "ARTIFACT_MISSING_OR_INVALID", - `${field} must be 1-${maxBytes} bytes of trimmed printable ASCII`, - ); - } -} - -function parseCanonicalUtc(value: string, field: string): number { - if (!CANONICAL_UTC_PATTERN.test(value)) { - fail("ARTIFACT_MISSING_OR_INVALID", `${field} must be a canonical UTC timestamp`); - } - const timestamp = Date.parse(value); - if (!Number.isFinite(timestamp) || new Date(timestamp).toISOString() !== value) { - fail("ARTIFACT_MISSING_OR_INVALID", `${field} must be a real canonical UTC timestamp`); - } - return timestamp; -} - -function parseStrictEvidenceJson(contents: string, label: string): unknown { - if (Buffer.byteLength(contents, "utf8") > MAX_EVIDENCE_BYTES) { - fail("ARTIFACT_MISSING_OR_INVALID", `${label} exceeds ${MAX_EVIDENCE_BYTES} bytes`); - } - - let position = 0; - const invalid = (message: string): never => - fail("ARTIFACT_MISSING_OR_INVALID", `${label} ${message}`); - const skipWhitespace = (): void => { - while ( - contents[position] === " " || - contents[position] === "\t" || - contents[position] === "\r" || - contents[position] === "\n" - ) { - position += 1; - } - }; - const scanString = (): string => { - const start = position; - if (contents[position] !== '"') invalid("is not valid JSON"); - position += 1; - while (position < contents.length) { - const character = contents[position]; - if (character === '"') { - position += 1; - try { - const decoded = JSON.parse(contents.slice(start, position)) as unknown; - if (typeof decoded === "string") return decoded; - return invalid("is not valid JSON"); - } catch { - invalid("is not valid JSON"); - } - } - if (character === "\\") { - position += 2; - continue; - } - if (character === undefined || character.charCodeAt(0) < 0x20) { - invalid("is not valid JSON"); - } - position += 1; - } - return invalid("is not valid JSON"); - }; - - const scanValue = (containerDepth: number): void => { - skipWhitespace(); - const character = contents[position]; - if (character === '"') { - scanString(); - return; - } - if (character === "{" || character === "[") { - if (containerDepth >= MAX_JSON_DEPTH) { - invalid(`exceeds the maximum JSON depth of ${MAX_JSON_DEPTH}`); - } - const closing = character === "{" ? "}" : "]"; - const objectKeys = character === "{" ? new Set() : null; - position += 1; - skipWhitespace(); - if (contents[position] === closing) { - position += 1; - return; - } - while (position < contents.length) { - if (objectKeys) { - if (contents[position] !== '"') invalid("is not valid JSON"); - const key = scanString(); - if (objectKeys.has(key)) invalid(`contains duplicate key ${JSON.stringify(key)}`); - objectKeys.add(key); - skipWhitespace(); - if (contents[position] !== ":") invalid("is not valid JSON"); - position += 1; - } - scanValue(containerDepth + 1); - skipWhitespace(); - if (contents[position] === closing) { - position += 1; - return; - } - if (contents[position] !== ",") invalid("is not valid JSON"); - position += 1; - skipWhitespace(); - } - invalid("is not valid JSON"); - } - - const start = position; - while ( - position < contents.length && - contents[position] !== " " && - contents[position] !== "\t" && - contents[position] !== "\r" && - contents[position] !== "\n" && - contents[position] !== "," && - contents[position] !== "]" && - contents[position] !== "}" - ) { - position += 1; - } - if (position === start) invalid("is not valid JSON"); - }; - - scanValue(0); - skipWhitespace(); - if (position !== contents.length) invalid("contains trailing transport output"); - try { - return JSON.parse(contents) as unknown; - } catch { - return invalid("is not valid JSON"); - } -} - -function validateWindow(notBeforeMs: number, notAfterMs: number): void { - if ( - !Number.isSafeInteger(notBeforeMs) || - !Number.isSafeInteger(notAfterMs) || - notBeforeMs < 0 || - notAfterMs < notBeforeMs - ) { - fail("REQUEST_INVALID", "trusted evidence window is invalid"); - } -} - -function requireWithinWindow( - timestamp: number, - notBeforeMs: number, - notAfterMs: number, - field: string, -): void { - if (timestamp < notBeforeMs || timestamp > notAfterMs) { - fail("PROVENANCE_MISMATCH", `${field} is outside the trusted evidence window`); - } -} - -function buildImageIdentity(value: unknown, label: string): BrevImageIdentity { - const record = requireRecord(value, label); - validateExactFields(record, IMAGE_FIELDS, label); - const image: BrevImageIdentity = { - project: requireString(record, "project", label), - imageName: requireString(record, "imageName", label), - imageId: requireString(record, "imageId", label), - imageSelfLink: requireString(record, "imageSelfLink", label), - }; - if (!GCP_PROJECT_ID_PATTERN.test(image.project)) { - fail("BREV_IMAGE_RESOLUTION_MISMATCH", `${label}.project is not a canonical GCP project ID`); - } - requirePattern( - image.imageName, - `${label}.imageName`, - IMAGE_NAME_PATTERN, - "BREV_IMAGE_RESOLUTION_MISMATCH", - ); - requirePattern( - image.imageId, - `${label}.imageId`, - DECIMAL_ID_PATTERN, - "BREV_IMAGE_RESOLUTION_MISMATCH", - ); - const expectedSelfLink = `https://www.googleapis.com/compute/v1/projects/${image.project}/global/images/${image.imageName}`; - if (image.imageSelfLink !== expectedSelfLink) { - fail( - "BREV_IMAGE_RESOLUTION_MISMATCH", - `${label}.imageSelfLink does not exactly identify project/imageName`, - ); - } - return image; -} - -function validateExpectedImage(image: BrevImageIdentity): void { - try { - buildImageIdentity(image, "expected image"); - } catch (error) { - if (error instanceof BrevLaunchableWorkspaceError) { - fail("REQUEST_INVALID", error.message); - } - throw error; - } -} - -function assertExpected(actual: unknown, expected: unknown, field: string): void { - if (actual !== expected) { - fail("PROVENANCE_MISMATCH", `${field} does not match trusted controller state`); - } -} - -function assertExpectedImage( - actual: BrevImageIdentity, - expected: BrevImageIdentity, - label: string, -) { - for (const field of IMAGE_FIELDS) { - if (actual[field] !== expected[field]) { - fail( - "BREV_IMAGE_RESOLUTION_MISMATCH", - `${label}.${field} does not match the accepted exact image manifest`, - ); - } - } -} - -function validateReceiptExpectations(expected: BrevLaunchableWorkspaceReceiptExpectations): void { - requirePattern( - expected.correlationId, - "expected correlationId", - UUID_V4_PATTERN, - "REQUEST_INVALID", - ); - requirePattern(expected.nemoclawSha, "expected nemoclawSha", FULL_SHA_PATTERN, "REQUEST_INVALID"); - requirePattern( - expected.acceptedImageManifestSha256, - "expected acceptedImageManifestSha256", - SHA256_PATTERN, - "REQUEST_INVALID", - ); - for (const [value, field, maxBytes] of [ - [expected.organizationId, "expected organizationId", OPAQUE_ID_MAX_BYTES], - [expected.idempotencyKey, "expected idempotencyKey", OPAQUE_ID_MAX_BYTES], - [expected.launchableId, "expected launchableId", OPAQUE_ID_MAX_BYTES], - [expected.launchableRevision, "expected launchableRevision", REVISION_MAX_BYTES], - [expected.workspaceId, "expected workspaceId", OPAQUE_ID_MAX_BYTES], - [expected.suppliedImageReference, "expected suppliedImageReference", IMAGE_REFERENCE_MAX_BYTES], - ] as const) { - try { - requireOpaqueAscii(value, field, maxBytes); - } catch (error) { - if (error instanceof BrevLaunchableWorkspaceError) fail("REQUEST_INVALID", error.message); - throw error; - } - } - validateExpectedImage(expected.image); - const supportedFamilyReference = `projects/${expected.image.project}/global/images/family/${EXACT_IMAGE_STAGING_FAMILY}`; - if ( - expected.suppliedImageReference !== expected.image.imageSelfLink && - expected.suppliedImageReference !== supportedFamilyReference - ) { - fail( - "REQUEST_INVALID", - "expected suppliedImageReference must be the accepted immutable self-link or canonical staging family", - ); - } - validateWindow(expected.notBeforeMs, expected.notAfterMs); -} - -function buildReceipt(record: Record): BrevLaunchableWorkspaceReceipt { - requireConstant(record.schemaVersion, 1, "schemaVersion"); - requireConstant(record.kind, BREV_LAUNCHABLE_WORKSPACE_RECEIPT_KIND, "kind"); - - const launchable = requireRecord(record.launchable, "launchable"); - validateExactFields(launchable, LAUNCHABLE_FIELDS, "launchable"); - const launchableId = requireString(launchable, "id", "launchable"); - const launchableRevision = requireString(launchable, "revision", "launchable"); - requireOpaqueAscii(launchableId, "launchable.id", OPAQUE_ID_MAX_BYTES); - requireOpaqueAscii(launchableRevision, "launchable.revision", REVISION_MAX_BYTES); - - const workspace = requireRecord(record.workspace, "workspace"); - validateExactFields(workspace, WORKSPACE_FIELDS, "workspace"); - requireConstant(workspace.status, "READY", "workspace.status", "BREV_READINESS_FAILED"); - const workspaceId = requireString(workspace, "id", "workspace"); - const workspaceLaunchableId = requireString(workspace, "launchableId", "workspace"); - const workspaceLaunchableRevision = requireString(workspace, "launchableRevision", "workspace"); - requireOpaqueAscii(workspaceId, "workspace.id", OPAQUE_ID_MAX_BYTES); - requireOpaqueAscii(workspaceLaunchableId, "workspace.launchableId", OPAQUE_ID_MAX_BYTES); - requireOpaqueAscii( - workspaceLaunchableRevision, - "workspace.launchableRevision", - REVISION_MAX_BYTES, - ); - - const receipt: BrevLaunchableWorkspaceReceipt = { - schemaVersion: 1, - kind: BREV_LAUNCHABLE_WORKSPACE_RECEIPT_KIND, - correlationId: requireString(record, "correlationId", "receipt"), - nemoclawSha: requireString(record, "nemoclawSha", "receipt"), - acceptedImageManifestSha256: requireString(record, "acceptedImageManifestSha256", "receipt"), - organizationId: requireString(record, "organizationId", "receipt"), - idempotencyKey: requireString(record, "idempotencyKey", "receipt"), - launchable: { - id: launchableId, - revision: launchableRevision, - suppliedImageReference: requireString(launchable, "suppliedImageReference", "launchable"), - resolvedImage: buildImageIdentity(launchable.resolvedImage, "launchable.resolvedImage"), - }, - workspace: { - id: workspaceId, - launchableId: workspaceLaunchableId, - launchableRevision: workspaceLaunchableRevision, - bootImage: buildImageIdentity(workspace.bootImage, "workspace.bootImage"), - status: "READY", - }, - recordedAt: requireString(record, "recordedAt", "receipt"), - }; - - requirePattern(receipt.correlationId, "correlationId", UUID_V4_PATTERN); - requirePattern(receipt.nemoclawSha, "nemoclawSha", FULL_SHA_PATTERN); - requirePattern( - receipt.acceptedImageManifestSha256, - "acceptedImageManifestSha256", - SHA256_PATTERN, - ); - requireOpaqueAscii(receipt.organizationId, "organizationId", OPAQUE_ID_MAX_BYTES); - requireOpaqueAscii(receipt.idempotencyKey, "idempotencyKey", OPAQUE_ID_MAX_BYTES); - requireOpaqueAscii( - receipt.launchable.suppliedImageReference, - "launchable.suppliedImageReference", - IMAGE_REFERENCE_MAX_BYTES, - ); - parseCanonicalUtc(receipt.recordedAt, "recordedAt"); - return receipt; -} - -export function validateBrevLaunchableWorkspaceReceipt( - value: unknown, - expected: BrevLaunchableWorkspaceReceiptExpectations, -): BrevLaunchableWorkspaceReceipt { - // This is the normalized evidence boundary, not a Brev transport adapter. - // Adapters must retain separate bounded raw-response hashes and populate - // observed image/readiness fields only from authoritative platform readback. - validateReceiptExpectations(expected); - const record = requireRecord(value, "receipt"); - validateExactFields(record, RECEIPT_FIELDS, "receipt"); - const receipt = buildReceipt(record); - - for (const [actual, wanted, field] of [ - [receipt.correlationId, expected.correlationId, "correlationId"], - [receipt.nemoclawSha, expected.nemoclawSha, "nemoclawSha"], - [ - receipt.acceptedImageManifestSha256, - expected.acceptedImageManifestSha256, - "acceptedImageManifestSha256", - ], - [receipt.organizationId, expected.organizationId, "organizationId"], - [receipt.idempotencyKey, expected.idempotencyKey, "idempotencyKey"], - [receipt.launchable.id, expected.launchableId, "launchable.id"], - [receipt.launchable.revision, expected.launchableRevision, "launchable.revision"], - [receipt.workspace.id, expected.workspaceId, "workspace.id"], - [ - receipt.launchable.suppliedImageReference, - expected.suppliedImageReference, - "launchable.suppliedImageReference", - ], - ] as const) { - assertExpected(actual, wanted, field); - } - - if (receipt.workspace.launchableId !== receipt.launchable.id) { - fail( - "PROVENANCE_MISMATCH", - "workspace launchable ID does not match the provisioned Launchable", - ); - } - if (receipt.workspace.launchableRevision !== receipt.launchable.revision) { - fail( - "PROVENANCE_MISMATCH", - "workspace Launchable revision does not match the immutable provisioned revision", - ); - } - assertExpectedImage(receipt.launchable.resolvedImage, expected.image, "launchable.resolvedImage"); - assertExpectedImage(receipt.workspace.bootImage, expected.image, "workspace.bootImage"); - - const recordedAt = parseCanonicalUtc(receipt.recordedAt, "recordedAt"); - requireWithinWindow(recordedAt, expected.notBeforeMs, expected.notAfterMs, "recordedAt"); - return receipt; -} - -export function parseBrevLaunchableWorkspaceReceiptJson(contents: string): unknown { - return parseStrictEvidenceJson(contents, "receipt"); -} - -export function parseAndValidateBrevLaunchableWorkspaceReceipt( - contents: string, - expected: BrevLaunchableWorkspaceReceiptExpectations, -): BrevLaunchableWorkspaceReceipt { - return validateBrevLaunchableWorkspaceReceipt( - parseBrevLaunchableWorkspaceReceiptJson(contents), - expected, - ); -} - -export function normalizedBrevLaunchableWorkspaceReceiptJson( - receipt: BrevLaunchableWorkspaceReceipt, -): string { - return `${JSON.stringify(receipt, null, 2)}\n`; -} - -export function brevLaunchableWorkspaceReceiptSha256( - receipt: BrevLaunchableWorkspaceReceipt, -): string { - return createHash("sha256") - .update(normalizedBrevLaunchableWorkspaceReceiptJson(receipt)) - .digest("hex"); -} - -function validateCleanupExpectations(expected: BrevWorkspaceCleanupEvidenceExpectations): void { - requirePattern( - expected.correlationId, - "expected correlationId", - UUID_V4_PATTERN, - "REQUEST_INVALID", - ); - requirePattern( - expected.receiptSha256, - "expected receiptSha256", - SHA256_PATTERN, - "REQUEST_INVALID", - ); - for (const [value, field, maxBytes] of [ - [expected.organizationId, "expected organizationId", OPAQUE_ID_MAX_BYTES], - [expected.workspaceId, "expected workspaceId", OPAQUE_ID_MAX_BYTES], - [expected.launchableId, "expected launchableId", OPAQUE_ID_MAX_BYTES], - [expected.launchableRevision, "expected launchableRevision", REVISION_MAX_BYTES], - ] as const) { - try { - requireOpaqueAscii(value, field, maxBytes); - } catch (error) { - if (error instanceof BrevLaunchableWorkspaceError) fail("REQUEST_INVALID", error.message); - throw error; - } - } - validateWindow(expected.notBeforeMs, expected.notAfterMs); -} - -function buildCleanupEvidence(record: Record): BrevWorkspaceCleanupEvidence { - requireConstant(record.schemaVersion, 1, "schemaVersion"); - requireConstant(record.kind, BREV_WORKSPACE_CLEANUP_EVIDENCE_KIND, "kind"); - requireConstant(record.terminalState, "ABSENT", "terminalState", "BREV_CLEANUP_INCOMPLETE"); - const cleanup: BrevWorkspaceCleanupEvidence = { - schemaVersion: 1, - kind: BREV_WORKSPACE_CLEANUP_EVIDENCE_KIND, - correlationId: requireString(record, "correlationId", "cleanup evidence"), - receiptSha256: requireString(record, "receiptSha256", "cleanup evidence"), - organizationId: requireString(record, "organizationId", "cleanup evidence"), - workspaceId: requireString(record, "workspaceId", "cleanup evidence"), - launchableId: requireString(record, "launchableId", "cleanup evidence"), - launchableRevision: requireString(record, "launchableRevision", "cleanup evidence"), - deleteRequestedAt: requireString(record, "deleteRequestedAt", "cleanup evidence"), - terminalState: "ABSENT", - verifiedAt: requireString(record, "verifiedAt", "cleanup evidence"), - }; - requirePattern(cleanup.correlationId, "correlationId", UUID_V4_PATTERN); - requirePattern(cleanup.receiptSha256, "receiptSha256", SHA256_PATTERN); - requireOpaqueAscii(cleanup.organizationId, "organizationId", OPAQUE_ID_MAX_BYTES); - requireOpaqueAscii(cleanup.workspaceId, "workspaceId", OPAQUE_ID_MAX_BYTES); - requireOpaqueAscii(cleanup.launchableId, "launchableId", OPAQUE_ID_MAX_BYTES); - requireOpaqueAscii(cleanup.launchableRevision, "launchableRevision", REVISION_MAX_BYTES); - parseCanonicalUtc(cleanup.deleteRequestedAt, "deleteRequestedAt"); - parseCanonicalUtc(cleanup.verifiedAt, "verifiedAt"); - return cleanup; -} - -export function validateBrevWorkspaceCleanupEvidence( - value: unknown, - expected: BrevWorkspaceCleanupEvidenceExpectations, -): BrevWorkspaceCleanupEvidence { - validateCleanupExpectations(expected); - const record = requireRecord(value, "cleanup evidence"); - validateExactFields(record, CLEANUP_FIELDS, "cleanup evidence"); - const cleanup = buildCleanupEvidence(record); - - for (const [actual, wanted, field] of [ - [cleanup.correlationId, expected.correlationId, "correlationId"], - [cleanup.receiptSha256, expected.receiptSha256, "receiptSha256"], - [cleanup.organizationId, expected.organizationId, "organizationId"], - [cleanup.workspaceId, expected.workspaceId, "workspaceId"], - [cleanup.launchableId, expected.launchableId, "launchableId"], - [cleanup.launchableRevision, expected.launchableRevision, "launchableRevision"], - ] as const) { - assertExpected(actual, wanted, field); - } - - const deleteRequestedAt = parseCanonicalUtc(cleanup.deleteRequestedAt, "deleteRequestedAt"); - const verifiedAt = parseCanonicalUtc(cleanup.verifiedAt, "verifiedAt"); - requireWithinWindow( - deleteRequestedAt, - expected.notBeforeMs, - expected.notAfterMs, - "deleteRequestedAt", - ); - requireWithinWindow(verifiedAt, expected.notBeforeMs, expected.notAfterMs, "verifiedAt"); - if (verifiedAt <= deleteRequestedAt) { - fail("BREV_CLEANUP_INCOMPLETE", "verifiedAt must follow deleteRequestedAt"); - } - return cleanup; -} - -export function parseBrevWorkspaceCleanupEvidenceJson(contents: string): unknown { - return parseStrictEvidenceJson(contents, "cleanup evidence"); -} - -export function parseAndValidateBrevWorkspaceCleanupEvidence( - contents: string, - expected: BrevWorkspaceCleanupEvidenceExpectations, -): BrevWorkspaceCleanupEvidence { - return validateBrevWorkspaceCleanupEvidence( - parseBrevWorkspaceCleanupEvidenceJson(contents), - expected, - ); -} - -export function normalizedBrevWorkspaceCleanupEvidenceJson( - evidence: BrevWorkspaceCleanupEvidence, -): string { - return `${JSON.stringify(evidence, null, 2)}\n`; -} - -export function brevLaunchableWorkspaceFailureCode( - error: unknown, -): BrevLaunchableWorkspaceFailureCode { - return error instanceof BrevLaunchableWorkspaceError ? error.code : "UNKNOWN"; -} From 00b1b78c656bdde05b3cc8afd6ec4ab4a724d924 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Fri, 17 Jul 2026 15:40:13 -0400 Subject: [PATCH 07/25] ci(e2e): qualify exact staging Brev Launchables Signed-off-by: Julie Yaunches --- .../brev-launchable-qualification.yaml | 126 ++++++++- .github/workflows/e2e.yaml | 11 + ci/source-shape-test-budget.json | 2 +- test/brev-launchable-runtime.test.ts | 123 +++++++++ ...act-image-qualification-controller.test.ts | 19 +- ...exact-image-qualification-workflow.test.ts | 30 ++- ...-image-qualification-controller-fixture.ts | 3 + test/helpers/vitest-watch-triggers.ts | 7 + test/vitest-watch-triggers.test.ts | 4 + tools/e2e/brev-launchable-runtime.sh | 241 ++++++++++++++++++ .../exact-image-qualification-controller.mts | 47 ++-- 11 files changed, 572 insertions(+), 41 deletions(-) create mode 100644 test/brev-launchable-runtime.test.ts create mode 100755 tools/e2e/brev-launchable-runtime.sh diff --git a/.github/workflows/brev-launchable-qualification.yaml b/.github/workflows/brev-launchable-qualification.yaml index fbf7b5b4fd..84518b9e40 100644 --- a/.github/workflows/brev-launchable-qualification.yaml +++ b/.github/workflows/brev-launchable-qualification.yaml @@ -1,30 +1,49 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -name: Draft / Brev Launchable Qualification -run-name: Draft image qualification for ${{ inputs.candidate_sha }} +name: E2E / Exact Staging Brev Launchable +run-name: Exact staging Launchable qualification for ${{ inputs.candidate_sha }} on: workflow_dispatch: inputs: candidate_sha: - description: Current lowercase 40-character NemoClaw main commit SHA to qualify. + description: Exact lowercase 40-character NemoClaw commit SHA to qualify. required: true type: string reason: - description: Audit reason for starting this pre-tag qualification image build. + description: Audit reason for starting this exact-image Launchable qualification. required: true type: string + workflow_call: + inputs: + candidate_sha: + description: Exact lowercase 40-character NemoClaw commit SHA to qualify. + required: true + type: string + reason: + description: Audit reason for starting this exact-image Launchable qualification. + required: true + type: string + secrets: + NEMOCLAW_IMAGE_QUALIFICATION_TOKEN: + required: true + BREV_API_KEY: + required: true + BREV_ORG_ID: + required: true + NVIDIA_INFERENCE_API_KEY: + required: true permissions: {} concurrency: - group: draft-brev-launchable-qualification-${{ inputs.candidate_sha }} + group: brev-launchable-qualification-${{ inputs.candidate_sha }} cancel-in-progress: false jobs: preflight: - name: Validate draft qualification request + name: Validate exact qualification request runs-on: ubuntu-latest timeout-minutes: 10 permissions: @@ -66,10 +85,10 @@ jobs: --workflow-sha "$WORKFLOW_SHA" qualify: - name: Build and verify exact staging image + name: Build, deploy, and test exact staging image needs: preflight runs-on: ubuntu-latest - timeout-minutes: 60 + timeout-minutes: 120 environment: name: approve-brev-launchable-qualification deployment: false @@ -170,6 +189,82 @@ jobs: --mode finalize --work-dir "${{ steps.workspace.outputs.work_dir }}" + - name: Install pinned Brev CLI + env: + BREV_CLI_SHA256: 5a6e70374db9be33f85f299161733b4a8409840d47638c781429b96e8d53704f + BREV_CLI_VERSION: 0.6.330 + run: | + set -euo pipefail + archive="${RUNNER_TEMP}/brev-cli.tar.gz" + curl -fsSL -o "$archive" "https://github.com/brevdev/brev-cli/releases/download/v${BREV_CLI_VERSION}/brev-cli_${BREV_CLI_VERSION}_linux_amd64.tar.gz" + printf '%s %s\n' "$BREV_CLI_SHA256" "$archive" | sha256sum -c - + tar -xzf "$archive" -C "${RUNNER_TEMP}" brev + mkdir -p "${RUNNER_TEMP}/bin" + install -m 0755 "${RUNNER_TEMP}/brev" "${RUNNER_TEMP}/bin/brev" + printf '%s\n' "${RUNNER_TEMP}/bin" >> "$GITHUB_PATH" + + - name: Authenticate Brev CLI + env: + BREV_API_KEY: ${{ secrets.BREV_API_KEY }} + BREV_ORG_ID: ${{ secrets.BREV_ORG_ID }} + run: | + set -euo pipefail + brev login --api-key "$BREV_API_KEY" --org-id "$BREV_ORG_ID" + brev ls --json | jq -e '.workspaces | type == "array"' >/dev/null + + - id: brev-workspace + name: Bind unique staging workspace name + env: + CANDIDATE_SHA: ${{ inputs.candidate_sha }} + run: | + set -euo pipefail + short_sha="${CANDIDATE_SHA:0:8}" + instance_name="nclaw-e2e-${short_sha}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + printf 'instance_name=%s\n' "$instance_name" >> "$GITHUB_OUTPUT" + + - name: Deploy the configured staging Launchable + env: + BREV_LAUNCHABLE_ID: ${{ vars.NEMOCLAW_STAGING_LAUNCHABLE_ID }} + INSTANCE_NAME: ${{ steps.brev-workspace.outputs.instance_name }} + WORK_DIR: ${{ steps.workspace.outputs.work_dir }} + run: tools/e2e/brev-launchable-runtime.sh deploy + + - name: Verify exact boot image and run preinstalled-image smoke + env: + CANDIDATE_SHA: ${{ inputs.candidate_sha }} + INSTANCE_NAME: ${{ steps.brev-workspace.outputs.instance_name }} + NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }} + VALIDATED_MANIFEST: ${{ steps.workspace.outputs.work_dir }}/validated-manifest.v1.json + WORK_DIR: ${{ steps.workspace.outputs.work_dir }} + run: tools/e2e/brev-launchable-runtime.sh qualify + + - name: Redact runtime evidence + if: ${{ always() && steps.workspace.outputs.work_dir != '' }} + env: + NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }} + WORK_DIR: ${{ steps.workspace.outputs.work_dir }} + run: | + set -euo pipefail + python3 - <<'PY' + import os + from pathlib import Path + + secret = os.environ.get("NVIDIA_INFERENCE_API_KEY", "") + root = Path(os.environ["WORK_DIR"]) + if secret: + for name in ("brev-prerequisites.log", "brev-quickstart.log", "brev-inference-pong.json", "brev-agent-response.log"): + path = root / name + if path.is_file(): + path.write_text(path.read_text(encoding="utf-8", errors="replace").replace(secret, "[REDACTED]"), encoding="utf-8") + PY + + - name: Delete staging workspace and verify absence + if: ${{ always() && steps.workspace.outputs.work_dir != '' && steps.brev-workspace.outputs.instance_name != '' }} + env: + INSTANCE_NAME: ${{ steps.brev-workspace.outputs.instance_name }} + WORK_DIR: ${{ steps.workspace.outputs.work_dir }} + run: tools/e2e/brev-launchable-runtime.sh cleanup + - name: Cancel an incomplete producer run if: ${{ always() && steps.workspace.outputs.work_dir != '' && steps.start.outcome != 'skipped' }} continue-on-error: true @@ -181,11 +276,11 @@ jobs: --mode cancel --work-dir "${{ steps.workspace.outputs.work_dir }}" - - name: Upload draft qualification evidence + - name: Upload exact staging Launchable evidence if: ${{ always() && steps.workspace.outputs.work_dir != '' }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: draft-brev-launchable-qualification-${{ inputs.candidate_sha }}-${{ github.run_id }} + name: brev-launchable-qualification-${{ inputs.candidate_sha }}-${{ github.run_id }} path: | ${{ steps.workspace.outputs.work_dir }}/dispatch-intent.v1.json ${{ steps.workspace.outputs.work_dir }}/dispatch-reconciliation.v1.json @@ -195,6 +290,17 @@ jobs: ${{ steps.workspace.outputs.work_dir }}/nemoclaw-image-manifest.v1.json ${{ steps.workspace.outputs.work_dir }}/validated-manifest.v1.json ${{ steps.workspace.outputs.work_dir }}/qualification-evidence.v1.json + ${{ steps.workspace.outputs.work_dir }}/brev-deploy-request.json + ${{ steps.workspace.outputs.work_dir }}/brev-workspace-ready.json + ${{ steps.workspace.outputs.work_dir }}/brev-provision.json + ${{ steps.workspace.outputs.work_dir }}/brev-boot-image.json + ${{ steps.workspace.outputs.work_dir }}/brev-identity-evidence.json + ${{ steps.workspace.outputs.work_dir }}/brev-prerequisites.log + ${{ steps.workspace.outputs.work_dir }}/brev-quickstart.log + ${{ steps.workspace.outputs.work_dir }}/brev-inference-pong.json + ${{ steps.workspace.outputs.work_dir }}/brev-agent-response.log + ${{ steps.workspace.outputs.work_dir }}/brev-smoke-evidence.json + ${{ steps.workspace.outputs.work_dir }}/brev-cleanup-evidence.json if-no-files-found: warn retention-days: 90 diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 36a0e27668..fd718fe4bd 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -185,6 +185,16 @@ jobs: fi fi + staging-brev-launchable: + name: Exact staging Brev Launchable + needs: generate-matrix + if: ${{ github.repository == 'NVIDIA/NemoClaw' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') }} + uses: ./.github/workflows/brev-launchable-qualification.yaml + with: + candidate_sha: ${{ inputs.checkout_sha || github.sha }} + reason: E2E run ${{ github.run_id }} qualifying the exact checked-out NemoClaw candidate + secrets: inherit + live: needs: generate-matrix if: ${{ needs.generate-matrix.outputs.matrix != '[]' }} @@ -4906,6 +4916,7 @@ jobs: needs: &e2e-result-jobs [ generate-matrix, + staging-brev-launchable, live, shared-e2e, openshell-gateway-auth-contract, diff --git a/ci/source-shape-test-budget.json b/ci/source-shape-test-budget.json index 87a348c99c..58454d8c40 100644 --- a/ci/source-shape-test-budget.json +++ b/ci/source-shape-test-budget.json @@ -128,7 +128,7 @@ }, { "file": "test/exact-image-qualification-workflow.test.ts", - "test": "keeps draft exact-image qualification manual, protected, and evidence-only", + "test": "keeps exact-image Launchable qualification protected, reusable, and fail-closed", "category": "security" }, { diff --git a/test/brev-launchable-runtime.test.ts b/test/brev-launchable-runtime.test.ts new file mode 100644 index 0000000000..850fbaaff8 --- /dev/null +++ b/test/brev-launchable-runtime.test.ts @@ -0,0 +1,123 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; + +const REPO_ROOT = path.resolve(import.meta.dirname, ".."); +const SCRIPT = path.join(REPO_ROOT, "tools", "e2e", "brev-launchable-runtime.sh"); +const roots: string[] = []; +const candidateSha = "a".repeat(40); + +afterEach(() => { + for (const root of roots.splice(0)) fs.rmSync(root, { recursive: true, force: true }); +}); + +function fixture(repoSha = candidateSha) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-launchable-runtime-")); + roots.push(root); + const bin = path.join(root, "bin"); + const workDir = path.join(root, "evidence"); + const state = path.join(root, "state.json"); + const log = path.join(root, "brev.log"); + const manifest = path.join(workDir, "validated-manifest.v1.json"); + fs.mkdirSync(bin); + fs.mkdirSync(workDir); + fs.writeFileSync(path.join(bin, "timeout"), '#!/usr/bin/env bash\nshift\nexec "$@"\n', { + mode: 0o755, + }); + fs.writeFileSync( + path.join(bin, "brev"), + `#!/usr/bin/env bash +set -euo pipefail +printf '%s\\n' "$*" >> "$FAKE_BREV_LOG" +case "$1" in + ls) + if [ -f "$FAKE_BREV_STATE" ]; then cat "$FAKE_BREV_STATE"; else printf '{"workspaces":[]}\\n'; fi + ;; + create) + printf '{"workspaces":[{"id":"ws-1","name":"%s","status":"RUNNING","shell_status":"READY","health_status":"HEALTHY","build_status":"COMPLETED"}]}\\n' "$INSTANCE_NAME" > "$FAKE_BREV_STATE" + ;; + exec) + shift 3 + command="$*" + case "$command" in + *'/etc/nemoclaw/provision.json'*) printf '{"gitSha":"aaaaaaa","version":"0.0.0"}\\n' ;; + *'git -C'*'rev-parse HEAD'*) printf '%s\\n' "$FAKE_REPO_SHA" ;; + *'metadata.google.internal'*) printf '{"sourceImage":"https://www.googleapis.com/compute/v1/projects/brevdevprod/global/images/image-a","sourceImageId":"123456789"}\\n' ;; + *'brev-quickstart'*) printf 'Ready!\\n' ;; + *'inference.local'*) printf '{"choices":[{"message":{"content":"PONG"}}]}\\n' ;; + *'openclaw agent'*) printf '{"payloads":[{"text":"42"}]}\\n' ;; + *) printf 'ok\\n' ;; + esac + ;; + delete) rm -f "$FAKE_BREV_STATE" ;; + refresh) ;; + *) exit 2 ;; +esac +`, + { mode: 0o755 }, + ); + fs.writeFileSync( + manifest, + `${JSON.stringify({ + imageName: "image-a", + imageId: "123456789", + imageSelfLink: + "https://www.googleapis.com/compute/v1/projects/brevdevprod/global/images/image-a", + })}\n`, + ); + const env = { + ...process.env, + PATH: `${bin}:${process.env.PATH ?? ""}`, + BREV_LAUNCHABLE_ID: "env-staging123", + CANDIDATE_SHA: candidateSha, + FAKE_BREV_LOG: log, + FAKE_BREV_STATE: state, + FAKE_REPO_SHA: repoSha, + INSTANCE_NAME: "nclaw-e2e-test-1", + NVIDIA_INFERENCE_API_KEY: "nvapi-test-value", + VALIDATED_MANIFEST: manifest, + WORK_DIR: workDir, + BREV_POLL_SECONDS: "0", + }; + return { env, log, workDir }; +} + +function run(mode: string, env: NodeJS.ProcessEnv) { + return spawnSync("bash", [SCRIPT, mode], { cwd: REPO_ROOT, encoding: "utf8", env }); +} + +describe("exact staging Brev Launchable runtime", () => { + it("deploys only the configured Launchable, proves identity, smokes the baked install, and deletes", () => { + const { env, log, workDir } = fixture(); + expect(run("deploy", env).status).toBe(0); + expect(run("qualify", env).status).toBe(0); + expect(run("cleanup", env).status).toBe(0); + + const commands = fs.readFileSync(log, "utf8"); + expect(commands).toContain( + "create nclaw-e2e-test-1 --launchable env-staging123 --detached --timeout 900", + ); + expect(commands).not.toMatch(/rsync|install\.sh|npm (?:ci|install)|git clone/u); + expect(commands.indexOf("rev-parse HEAD")).toBeLessThan(commands.indexOf("brev-quickstart")); + expect(fs.existsSync(path.join(workDir, "brev-identity-evidence.json"))).toBe(true); + expect(fs.existsSync(path.join(workDir, "brev-smoke-evidence.json"))).toBe(true); + expect( + JSON.parse(fs.readFileSync(path.join(workDir, "brev-cleanup-evidence.json"), "utf8")), + ).toMatchObject({ terminalState: "ABSENT", workspaceName: "nclaw-e2e-test-1" }); + }); + + it("fails closed on a baked SHA mismatch before onboarding", () => { + const { env, log } = fixture("b".repeat(40)); + expect(run("deploy", env).status).toBe(0); + const qualification = run("qualify", env); + expect(qualification.status).not.toBe(0); + expect(qualification.stderr).toContain("does not match candidate"); + expect(fs.readFileSync(log, "utf8")).not.toContain("brev-quickstart"); + expect(run("cleanup", env).status).toBe(0); + }); +}); diff --git a/test/exact-image-qualification-controller.test.ts b/test/exact-image-qualification-controller.test.ts index c341d1bc21..66ae40e44a 100644 --- a/test/exact-image-qualification-controller.test.ts +++ b/test/exact-image-qualification-controller.test.ts @@ -66,17 +66,26 @@ afterEach(() => { }); describe("exact image qualification request", () => { - it("accepts only a first-attempt manual current-workflow candidate", () => { + it("accepts trusted manual and scheduled candidates while rejecting malformed boundaries", () => { expect(validateExactImageQualificationRequest(REQUEST)).toEqual(REQUEST); - expect(() => + expect( validateExactImageQualificationRequest({ ...REQUEST, candidateSha: "c".repeat(40), + eventName: "schedule", + requesterRunAttempt: 2, }), - ).toThrowError(ExactImageQualificationError); + ).toMatchObject({ + candidateSha: "c".repeat(40), + eventName: "schedule", + requesterRunAttempt: 2, + }); + expect(() => + validateExactImageQualificationRequest({ ...REQUEST, eventName: "pull_request" }), + ).toThrow(/workflow_dispatch or schedule/u); expect(() => - validateExactImageQualificationRequest({ ...REQUEST, requesterRunAttempt: 2 }), - ).toThrow(/reruns/u); + validateExactImageQualificationRequest({ ...REQUEST, ref: "refs/pull/1/merge" }), + ).toThrow(/branch ref/u); expect(() => validateExactImageQualificationRequest({ ...REQUEST, reason: " bad " })).toThrow( /reason/u, ); diff --git a/test/exact-image-qualification-workflow.test.ts b/test/exact-image-qualification-workflow.test.ts index 0d78a07817..5d7643cca0 100644 --- a/test/exact-image-qualification-workflow.test.ts +++ b/test/exact-image-qualification-workflow.test.ts @@ -12,8 +12,9 @@ import { const WORKFLOW_PATH = ".github/workflows/brev-launchable-qualification.yaml"; const CONTROLLER_PATH = "tools/e2e/exact-image-qualification-controller.mts"; +const RUNTIME_PATH = "tools/e2e/brev-launchable-runtime.sh"; -type DraftQualificationWorkflow = Workflow & { +type QualificationWorkflow = Workflow & { name: string; on: Record; permissions: Record; @@ -30,31 +31,36 @@ function strings(value: unknown): string[] { : []; } -function job(workflow: DraftQualificationWorkflow, name: string): WorkflowJob { +function job(workflow: QualificationWorkflow, name: string): WorkflowJob { const value = workflow.jobs[name]; expect(value, `missing ${name} job`).toBeDefined(); return value!; } -// source-shape-contract: security -- The draft cross-repository image handoff must remain manual, protected, fixed-target, least-privilege, and stop before any Brev deployment -it("keeps draft exact-image qualification manual, protected, and evidence-only", () => { - const workflow = readYaml(WORKFLOW_PATH); +// source-shape-contract: security -- Exact-image qualification must remain manual/reusable, protected, fixed-target, least-privilege, identity-gated, and cleanup-verifying +it("keeps exact-image Launchable qualification protected, reusable, and fail-closed", () => { + const workflow = readYaml(WORKFLOW_PATH); const source = readRepoText(WORKFLOW_PATH); const controller = readRepoText(CONTROLLER_PATH); + const runtime = readRepoText(RUNTIME_PATH); const preflight = job(workflow, "preflight"); const qualify = job(workflow, "qualify"); const workflowStrings = strings(workflow); - expect(workflow.name).toBe("Draft / Brev Launchable Qualification"); - expect(Object.keys(workflow.on)).toEqual(["workflow_dispatch"]); + expect(workflow.name).toBe("E2E / Exact Staging Brev Launchable"); + expect(Object.keys(workflow.on)).toEqual(["workflow_dispatch", "workflow_call"]); expect(source).not.toMatch(/^\s+(?:push|schedule|workflow_run|pull_request):/mu); expect(Object.keys((workflow.on.workflow_dispatch as { inputs: object }).inputs)).toEqual([ "candidate_sha", "reason", ]); + expect(Object.keys((workflow.on.workflow_call as { inputs: object }).inputs)).toEqual([ + "candidate_sha", + "reason", + ]); expect(workflow.permissions).toEqual({}); expect(workflow.concurrency).toEqual({ - group: "draft-brev-launchable-qualification-${{ inputs.candidate_sha }}", + group: "brev-launchable-qualification-${{ inputs.candidate_sha }}", "cancel-in-progress": false, }); @@ -116,7 +122,11 @@ it("keeps draft exact-image qualification manual, protected, and evidence-only", expect(source).toContain("dispatch-reconciliation.v1.json"); expect(source).toContain("controller-state.corrupt-*.json"); expect(source).toContain("--mode finalize"); - expect(source).not.toMatch(/\b(?:brev|gcloud)\s+(?:launch|deploy|set|create|update)\b/iu); - expect(source).not.toContain("launchable_id"); + expect(runtime).toContain("brev create"); + expect(runtime).toContain("--launchable"); + expect(source).toContain("brev-launchable-runtime.sh qualify"); + expect(source).toContain("brev-launchable-runtime.sh cleanup"); + expect(source).toContain("brev-cleanup-evidence.json"); + expect(source).toContain("NEMOCLAW_STAGING_LAUNCHABLE_ID"); expect(source).not.toContain("image_family"); }); diff --git a/test/helpers/exact-image-qualification-controller-fixture.ts b/test/helpers/exact-image-qualification-controller-fixture.ts index c77d2fa134..4b91462289 100644 --- a/test/helpers/exact-image-qualification-controller-fixture.ts +++ b/test/helpers/exact-image-qualification-controller-fixture.ts @@ -125,6 +125,9 @@ export function createApi(options: ApiOptions = {}) { if (apiPath === "repos/NVIDIA/NemoClaw/git/ref/heads/main") { return mainRef(options.requesterSha ?? CANDIDATE_SHA); } + if (apiPath === `repos/NVIDIA/NemoClaw/git/commits/${CANDIDATE_SHA}`) { + return { sha: CANDIDATE_SHA }; + } if (apiPath.includes("/collaborators/")) { return { permission: options.permission ?? "write", diff --git a/test/helpers/vitest-watch-triggers.ts b/test/helpers/vitest-watch-triggers.ts index 2c6ba99bf8..1669de37b9 100644 --- a/test/helpers/vitest-watch-triggers.ts +++ b/test/helpers/vitest-watch-triggers.ts @@ -95,6 +95,13 @@ export const vitestWatchTriggerPatterns: VitestWatchTriggerPattern[] = [ pattern: /(?:^|\/)\.github\/workflows\/brev-launchable-qualification\.yaml$/, testsToRun: runTests("test/exact-image-qualification-workflow.test.ts"), }, + { + pattern: /(?:^|\/)tools\/e2e\/brev-launchable-runtime\.sh$/, + testsToRun: runTests( + "test/brev-launchable-runtime.test.ts", + "test/exact-image-qualification-workflow.test.ts", + ), + }, { pattern: /(?:^|\/)(?:\.github\/workflows\/platform-vitest-main\.yaml|ci\/platform-vitest-macos-requirements\.lock)$/, diff --git a/test/vitest-watch-triggers.test.ts b/test/vitest-watch-triggers.test.ts index fac5b90a5c..52e52a12ac 100644 --- a/test/vitest-watch-triggers.test.ts +++ b/test/vitest-watch-triggers.test.ts @@ -110,6 +110,10 @@ describe("Vitest opaque-input watch triggers", () => { expect(triggeredBy(".github/workflows/brev-launchable-qualification.yaml")).toEqual([ "test/exact-image-qualification-workflow.test.ts", ]); + expect(triggeredBy("tools/e2e/brev-launchable-runtime.sh")).toEqual([ + "test/brev-launchable-runtime.test.ts", + "test/exact-image-qualification-workflow.test.ts", + ]); expect(triggeredBy(".github/workflows/platform-vitest-main.yaml")).toEqual([ "test/platform-vitest-main-workflow.test.ts", ]); diff --git a/tools/e2e/brev-launchable-runtime.sh b/tools/e2e/brev-launchable-runtime.sh new file mode 100755 index 0000000000..e25ef47793 --- /dev/null +++ b/tools/e2e/brev-launchable-runtime.sh @@ -0,0 +1,241 @@ +#!/usr/bin/env bash + +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +die() { + printf 'BREV_LAUNCHABLE_QUALIFICATION_FAILED: %s\n' "$*" >&2 + exit 1 +} + +require_env() { + local name="$1" + [ -n "${!name:-}" ] || die "$name is required" +} + +require_tools() { + local tool + for tool in "$@"; do + command -v "$tool" >/dev/null 2>&1 || die "$tool is required" + done +} + +validate_common() { + require_env WORK_DIR + require_env INSTANCE_NAME + [[ "$INSTANCE_NAME" =~ ^[a-z][a-z0-9-]{0,62}$ ]] \ + || die "INSTANCE_NAME must be a lowercase Brev workspace name" + [ -d "$WORK_DIR" ] || die "WORK_DIR must already exist" + require_tools brev jq timeout +} + +workspace_rows() { + timeout 30s brev ls --json | jq -c ' + if type == "array" then . + elif type == "object" and (.workspaces | type) == "array" then .workspaces + else error("unexpected brev ls --json shape") + end + ' +} + +workspace_record() { + workspace_rows | jq -c --arg name "$INSTANCE_NAME" ' + map(select(((.name // .workspaceName // .instanceName // .Name // "") | tostring) == $name)) + | if length == 0 then empty + elif length == 1 then .[0] + else error("workspace name is ambiguous") + end + ' +} + +wait_for_workspace_ready() { + local deadline=$((SECONDS + ${BREV_READY_TIMEOUT_SECONDS:-1200})) + local record status shell_status health_status build_status + while [ "$SECONDS" -lt "$deadline" ]; do + record="$(workspace_record || true)" + if [ -n "$record" ]; then + status="$(jq -r '.status // ""' <<<"$record")" + shell_status="$(jq -r '.shell_status // .shellStatus // ""' <<<"$record")" + health_status="$(jq -r '.health_status // .healthStatus // ""' <<<"$record")" + build_status="$(jq -r '.build_status // .buildStatus // ""' <<<"$record")" + if [ "$status" = "RUNNING" ] && [ "$shell_status" = "READY" ] \ + && [ "$health_status" = "HEALTHY" ] && [ "$build_status" = "COMPLETED" ]; then + printf '%s\n' "$record" >"$WORK_DIR/brev-workspace-ready.json" + return 0 + fi + case "$status:$build_status" in + FAILED:* | ERROR:* | *:FAILED | *:ERROR) die "Brev workspace entered terminal failure ($status/$build_status)" ;; + esac + fi + sleep "${BREV_POLL_SECONDS:-15}" + done + die "Brev workspace did not become structurally ready before the deadline" +} + +deploy() { + validate_common + require_env BREV_LAUNCHABLE_ID + [[ "$BREV_LAUNCHABLE_ID" =~ ^env-[A-Za-z0-9]+$ ]] \ + || die "BREV_LAUNCHABLE_ID must be one opaque env-* ID" + if [ -n "$(workspace_record)" ]; then + die "refusing to reuse pre-existing workspace $INSTANCE_NAME" + fi + printf '{"schemaVersion":1,"launchableId":%s,"workspaceName":%s,"requestedAt":%s}\n' \ + "$(jq -Rn --arg value "$BREV_LAUNCHABLE_ID" '$value')" \ + "$(jq -Rn --arg value "$INSTANCE_NAME" '$value')" \ + "$(jq -Rn --arg value "$(date -u +%Y-%m-%dT%H:%M:%SZ)" '$value')" \ + >"$WORK_DIR/brev-deploy-request.json" + timeout "${BREV_CREATE_TIMEOUT_SECONDS:-900}" \ + brev create "$INSTANCE_NAME" --launchable "$BREV_LAUNCHABLE_ID" --detached \ + --timeout "${BREV_CREATE_TIMEOUT_SECONDS:-900}" + wait_for_workspace_ready +} + +host_exec() { + local command="$1" + timeout "${BREV_HOST_COMMAND_TIMEOUT_SECONDS:-1800}" \ + brev exec "$INSTANCE_NAME" --host "$command" +} + +verify_identity() { + require_env CANDIDATE_SHA + [[ "$CANDIDATE_SHA" =~ ^[0-9a-f]{40}$ ]] || die "CANDIDATE_SHA must be a lowercase full SHA" + require_env VALIDATED_MANIFEST + [ -f "$VALIDATED_MANIFEST" ] || die "VALIDATED_MANIFEST is missing" + + local expected_image expected_image_id expected_self_link repo_sha provision provision_sha disk_json + expected_image="$(jq -er '.imageName' "$VALIDATED_MANIFEST")" + expected_image_id="$(jq -er '.imageId' "$VALIDATED_MANIFEST")" + expected_self_link="$(jq -er '.imageSelfLink' "$VALIDATED_MANIFEST")" + + provision="$(host_exec 'sudo -n cat /etc/nemoclaw/provision.json')" + jq -e 'type == "object" and (.gitSha | type == "string")' <<<"$provision" >/dev/null \ + || die "the baked provision metadata is missing or malformed" + printf '%s\n' "$provision" >"$WORK_DIR/brev-provision.json" + provision_sha="$(jq -r '.gitSha' <<<"$provision")" + + # HOME and repo are intentionally expanded by the remote host shell. + # shellcheck disable=SC2016 + repo_sha="$(host_exec 'set -e; repo="$HOME/NemoClaw"; test -d "$repo/.git"; git -C "$repo" rev-parse HEAD' | tail -n 1)" + [ "$repo_sha" = "$CANDIDATE_SHA" ] \ + || die "baked NemoClaw SHA $repo_sha does not match candidate $CANDIDATE_SHA" + [[ "$CANDIDATE_SHA" == "$provision_sha"* ]] \ + || die "provision metadata SHA $provision_sha does not identify candidate $CANDIDATE_SHA" + + # Metadata variables are intentionally expanded by the remote host shell. + # shellcheck disable=SC2016 + disk_json="$(host_exec 'set -euo pipefail + metadata=http://metadata.google.internal/computeMetadata/v1 + header="Metadata-Flavor: Google" + project=$(curl -fsS -H "$header" "$metadata/project/project-id") + zone_path=$(curl -fsS -H "$header" "$metadata/instance/zone") + zone=${zone_path##*/} + disk=$(curl -fsS -H "$header" "$metadata/instance/disks/0/device-name") + token=$(curl -fsS -H "$header" "$metadata/instance/service-accounts/default/token" | jq -er .access_token) + curl -fsS -H "Authorization: Bearer $token" \ + "https://compute.googleapis.com/compute/v1/projects/$project/zones/$zone/disks/$disk" \ + | jq -c "{sourceImage,sourceImageId}"')" + disk_json="$(tail -n 1 <<<"$disk_json")" + jq -e --arg image "$expected_self_link" --arg id "$expected_image_id" ' + .sourceImage == $image and ((.sourceImageId | tostring) == $id) + ' <<<"$disk_json" >/dev/null \ + || die "workspace boot disk does not match accepted image $expected_image ($expected_image_id)" + printf '%s\n' "$disk_json" >"$WORK_DIR/brev-boot-image.json" + + jq -n \ + --arg candidateSha "$CANDIDATE_SHA" \ + --arg repositorySha "$repo_sha" \ + --arg provisionSha "$provision_sha" \ + --arg imageName "$expected_image" \ + --arg imageId "$expected_image_id" \ + --arg imageSelfLink "$expected_self_link" \ + --arg verifiedAt "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + '{schemaVersion:1,candidateSha:$candidateSha,repositorySha:$repositorySha,provisionSha:$provisionSha,image:{name:$imageName,id:$imageId,selfLink:$imageSelfLink},verifiedAt:$verifiedAt}' \ + >"$WORK_DIR/brev-identity-evidence.json" +} + +run_smoke() { + require_env NVIDIA_INFERENCE_API_KEY + local sandbox="${NEMOCLAW_STAGING_SANDBOX_NAME:-e2e-staging}" + [[ "$sandbox" =~ ^[a-z][a-z0-9-]{0,62}$ ]] || die "invalid staging sandbox name" + local quoted_key quoted_sandbox + printf -v quoted_key '%q' "$NVIDIA_INFERENCE_API_KEY" + printf -v quoted_sandbox '%q' "$sandbox" + + host_exec 'set -e; command -v nemoclaw; command -v openshell; command -v docker; command -v brev-quickstart; docker info >/dev/null; openshell --version; nemoclaw --help >/dev/null' \ + >"$WORK_DIR/brev-prerequisites.log" 2>&1 + host_exec "set -euo pipefail; export NVIDIA_API_KEY=$quoted_key NEMOCLAW_PROVIDER=build NEMOCLAW_AGENT=openclaw; timeout 1500 brev-quickstart $quoted_sandbox" \ + >"$WORK_DIR/brev-quickstart.log" 2>&1 + + host_exec "set -euo pipefail + model=\$(node /usr/local/lib/nemoclaw/launchable-config.mjs /usr/local/share/nemoclaw/launchable-agents.json openclaw cloudModel) + payload=\$(jq -cn --arg model \"\$model\" '{model:\$model,messages:[{role:\"user\",content:\"Reply with exactly one word: PONG\"}],max_tokens:100}') + response=\$(openshell sandbox exec --name $quoted_sandbox -- curl -fsS --max-time 90 \ + https://inference.local/v1/chat/completions -H 'Content-Type: application/json' -d \"\$payload\") + printf '%s' \"\$response\" | jq -er '[.choices[0].message.content,.choices[0].message.reasoning_content,.choices[0].message.reasoning] | map(select(type == \"string\")) | join(\" \") | test(\"PONG\"; \"i\")' >/dev/null + printf '%s\n' \"\$response\"" >"$WORK_DIR/brev-inference-pong.json" 2>&1 + + host_exec "set -euo pipefail + response=\$(openshell sandbox exec --name $quoted_sandbox -- openclaw agent --agent main --json --thinking off \ + --session-id qualification-$(date +%s) -m 'What is 6 multiplied by 7? Reply with only the integer, no extra words.') + printf '%s\n' \"\$response\" + printf '%s' \"\$response\" | grep -Eq '(^|[^0-9])42([^0-9]|$)'" \ + >"$WORK_DIR/brev-agent-response.log" 2>&1 + + jq -n --arg sandbox "$sandbox" --arg completedAt "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + '{schemaVersion:1,sandbox:$sandbox,checks:["baked-tools","onboarding","sandbox-ready","inference-local-pong","openclaw-agent-response"],completedAt:$completedAt}' \ + >"$WORK_DIR/brev-smoke-evidence.json" +} + +qualify() { + validate_common + verify_identity + run_smoke +} + +cleanup() { + validate_common + local requested_at verified_at deadline output record status=0 absent_count=0 + requested_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + record="$(workspace_record)" + if [ -n "$record" ]; then + output="$(timeout 60s brev delete "$INSTANCE_NAME" 2>&1)" || status=$? + printf '%s\n' "$output" + [ "$status" -eq 0 ] || printf 'brev delete returned %s; verifying absence before failing\n' "$status" >&2 + fi + deadline=$((SECONDS + ${BREV_DELETE_TIMEOUT_SECONDS:-600})) + while [ "$SECONDS" -lt "$deadline" ]; do + if record="$(workspace_record)"; then + if [ -n "$record" ]; then + absent_count=0 + timeout 30s brev refresh >/dev/null 2>&1 || true + sleep "${BREV_POLL_SECONDS:-15}" + continue + fi + absent_count=$((absent_count + 1)) + if [ "$absent_count" -lt "${BREV_ABSENCE_CONFIRMATIONS:-4}" ]; then + timeout 30s brev refresh >/dev/null 2>&1 || true + sleep "${BREV_POLL_SECONDS:-15}" + continue + fi + verified_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + jq -n --arg workspaceName "$INSTANCE_NAME" --arg requestedAt "$requested_at" --arg verifiedAt "$verified_at" \ + '{schemaVersion:1,workspaceName:$workspaceName,deleteRequestedAt:$requestedAt,terminalState:"ABSENT",verifiedAt:$verifiedAt}' \ + >"$WORK_DIR/brev-cleanup-evidence.json" + return 0 + fi + printf 'brev ls failed while verifying cleanup; retrying\n' >&2 + timeout 30s brev refresh >/dev/null 2>&1 || true + sleep "${BREV_POLL_SECONDS:-15}" + done + die "Brev workspace still exists after cleanup deadline" +} + +case "${1:-}" in + deploy) deploy ;; + qualify) qualify ;; + cleanup) cleanup ;; + *) die "usage: $0 deploy|qualify|cleanup" ;; +esac diff --git a/tools/e2e/exact-image-qualification-controller.mts b/tools/e2e/exact-image-qualification-controller.mts index e36f5f0225..9cb6ad28fb 100755 --- a/tools/e2e/exact-image-qualification-controller.mts +++ b/tools/e2e/exact-image-qualification-controller.mts @@ -39,6 +39,7 @@ const DECIMAL_ID_PATTERN = /^[1-9][0-9]*$/u; const ARTIFACT_DIGEST_PATTERN = /^sha256:[0-9a-f]{64}$/u; const SHA256_PATTERN = /^[0-9a-f]{64}$/u; const GITHUB_LOGIN_PATTERN = /^(?!-)[A-Za-z0-9-]{1,39}(? { validateExactImageQualificationRequest(request); - const main = await api(`repos/${REQUESTER_REPOSITORY}/git/ref/heads/main`, token); - if (validateMainRef(main, REQUESTER_REPOSITORY) !== request.candidateSha) { - fail("DISPATCH_FORBIDDEN", "candidate SHA is no longer the current NemoClaw main commit"); + const branchPath = request.ref.slice("refs/heads/".length); + const branch = await api( + `repos/${REQUESTER_REPOSITORY}/git/ref/heads/${encodeURIComponent(branchPath)}`, + token, + ); + if (validateRequesterRef(branch, REQUESTER_REPOSITORY, request.ref) !== request.workflowSha) { + fail("DISPATCH_FORBIDDEN", "workflow SHA is no longer the requested branch head"); + } + const candidate = record( + await api(`repos/${REQUESTER_REPOSITORY}/git/commits/${request.candidateSha}`, token), + "candidate commit", + ); + if (candidate.sha !== request.candidateSha) { + fail("DISPATCH_FORBIDDEN", "candidate SHA does not identify an exact NemoClaw commit"); } const permission = record( await api( From 8cd0ad5f64cc737e35f9d5e144153250c7e50367 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Fri, 17 Jul 2026 15:45:06 -0400 Subject: [PATCH 08/25] ci(e2e): reuse image dispatch credential Signed-off-by: Julie Yaunches --- .github/workflows/brev-launchable-qualification.yaml | 10 +++++----- test/exact-image-qualification-workflow.test.ts | 8 +++++++- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/.github/workflows/brev-launchable-qualification.yaml b/.github/workflows/brev-launchable-qualification.yaml index 84518b9e40..a774a084fb 100644 --- a/.github/workflows/brev-launchable-qualification.yaml +++ b/.github/workflows/brev-launchable-qualification.yaml @@ -26,7 +26,7 @@ on: required: true type: string secrets: - NEMOCLAW_IMAGE_QUALIFICATION_TOKEN: + NEMOCLAW_IMAGE_DISPATCH_TOKEN: required: true BREV_API_KEY: required: true @@ -122,7 +122,7 @@ jobs: CANDIDATE_SHA: ${{ inputs.candidate_sha }} EVENT_NAME: ${{ github.event_name }} GITHUB_TOKEN: ${{ github.token }} - NEMOCLAW_IMAGE_QUALIFICATION_TOKEN: ${{ secrets.NEMOCLAW_IMAGE_QUALIFICATION_TOKEN }} + NEMOCLAW_IMAGE_QUALIFICATION_TOKEN: ${{ secrets.NEMOCLAW_IMAGE_DISPATCH_TOKEN }} QUALIFICATION_REASON: ${{ inputs.reason }} REQUESTER_RUN_ATTEMPT: ${{ github.run_attempt }} REQUESTER_RUN_ID: ${{ github.run_id }} @@ -144,7 +144,7 @@ jobs: - name: Wait for the bound producer run env: - NEMOCLAW_IMAGE_QUALIFICATION_TOKEN: ${{ secrets.NEMOCLAW_IMAGE_QUALIFICATION_TOKEN }} + NEMOCLAW_IMAGE_QUALIFICATION_TOKEN: ${{ secrets.NEMOCLAW_IMAGE_DISPATCH_TOKEN }} run: >- node --experimental-strip-types --no-warnings tools/e2e/exact-image-qualification-controller.mts @@ -153,7 +153,7 @@ jobs: - name: Download and verify the bound manifest artifact env: - NEMOCLAW_IMAGE_QUALIFICATION_TOKEN: ${{ secrets.NEMOCLAW_IMAGE_QUALIFICATION_TOKEN }} + NEMOCLAW_IMAGE_QUALIFICATION_TOKEN: ${{ secrets.NEMOCLAW_IMAGE_DISPATCH_TOKEN }} run: >- node --experimental-strip-types --no-warnings tools/e2e/exact-image-qualification-controller.mts @@ -269,7 +269,7 @@ jobs: if: ${{ always() && steps.workspace.outputs.work_dir != '' && steps.start.outcome != 'skipped' }} continue-on-error: true env: - NEMOCLAW_IMAGE_QUALIFICATION_TOKEN: ${{ secrets.NEMOCLAW_IMAGE_QUALIFICATION_TOKEN }} + NEMOCLAW_IMAGE_QUALIFICATION_TOKEN: ${{ secrets.NEMOCLAW_IMAGE_DISPATCH_TOKEN }} run: >- node --experimental-strip-types --no-warnings tools/e2e/exact-image-qualification-controller.mts diff --git a/test/exact-image-qualification-workflow.test.ts b/test/exact-image-qualification-workflow.test.ts index 5d7643cca0..2544bc12ef 100644 --- a/test/exact-image-qualification-workflow.test.ts +++ b/test/exact-image-qualification-workflow.test.ts @@ -58,6 +58,12 @@ it("keeps exact-image Launchable qualification protected, reusable, and fail-clo "candidate_sha", "reason", ]); + expect(Object.keys((workflow.on.workflow_call as { secrets: object }).secrets)).toEqual([ + "NEMOCLAW_IMAGE_DISPATCH_TOKEN", + "BREV_API_KEY", + "BREV_ORG_ID", + "NVIDIA_INFERENCE_API_KEY", + ]); expect(workflow.permissions).toEqual({}); expect(workflow.concurrency).toEqual({ group: "brev-launchable-qualification-${{ inputs.candidate_sha }}", @@ -73,7 +79,7 @@ it("keeps exact-image Launchable qualification protected, reusable, and fail-clo deployment: false, }); expect(JSON.stringify(qualify)).toContain("NEMOCLAW_IMAGE_QUALIFICATION_TOKEN"); - expect(source.match(/secrets\.NEMOCLAW_IMAGE_QUALIFICATION_TOKEN/gu)).toHaveLength(4); + expect(source.match(/secrets\.NEMOCLAW_IMAGE_DISPATCH_TOKEN/gu)).toHaveLength(4); expect(workflowStrings).not.toContain("id-token: write"); expect(source).not.toMatch(/npm (?:ci|install)/u); for (const step of [...(preflight.steps ?? []), ...(qualify.steps ?? [])]) { From be0aa7a1320591599e68ba8bd4a092d0f938cef0 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Fri, 17 Jul 2026 15:59:12 -0400 Subject: [PATCH 09/25] fix(e2e): preserve selected candidate provenance Signed-off-by: Julie Yaunches --- ...act-image-qualification-controller.test.ts | 27 +++++++++++++-- ...-image-qualification-controller-fixture.ts | 5 +-- .../exact-image-qualification-controller.mts | 34 +++---------------- 3 files changed, 32 insertions(+), 34 deletions(-) diff --git a/test/exact-image-qualification-controller.test.ts b/test/exact-image-qualification-controller.test.ts index 66ae40e44a..da2050b42f 100644 --- a/test/exact-image-qualification-controller.test.ts +++ b/test/exact-image-qualification-controller.test.ts @@ -56,6 +56,7 @@ import { RUN_ID, startedState, tempDirectory, + WORKFLOW_SHA, WORKFLOW_ID, workflowRun, } from "./helpers/exact-image-qualification-controller-fixture.ts"; @@ -111,7 +112,7 @@ describe("exact image qualification request", () => { "--requester-run-id", REQUEST.requesterRunId, "--workflow-sha", - CANDIDATE_SHA, + WORKFLOW_SHA, ]), ).toEqual({ mode: "preflight", request: REQUEST }); expect(() => @@ -128,6 +129,28 @@ describe("exact image qualification request", () => { }); describe("exact producer dispatch binding", () => { + it("preserves a selected candidate distinct from the trusted workflow SHA", async () => { + expect(REQUEST.candidateSha).not.toBe(REQUEST.workflowSha); + const { state, workDir } = await startedState(); + try { + expect(readExactImageQualificationState(workDir).request).toMatchObject({ + candidateSha: CANDIDATE_SHA, + workflowSha: WORKFLOW_SHA, + }); + await expect( + waitForExactImageQualification( + { workDir, producerToken: "producer-token" }, + dependencies( + createApi({ run: workflowRun({ status: "completed", conclusion: "success" }) }), + { now: () => BASE_TIME + 1_000 }, + ), + ), + ).resolves.toMatchObject({ id: state.producer.runId, conclusion: "success" }); + } finally { + fs.rmSync(workDir, { recursive: true, force: true }); + } + }); + it("accepts GitHub's maintain role mapping but rejects ordinary write access", async () => { const accepted = await startedState(createApi({ permission: "write", roleName: "maintain" })); fs.rmSync(accepted.workDir, { recursive: true, force: true }); @@ -1374,7 +1397,7 @@ describe("qualification evidence finalization", () => { reason: REQUEST.reason, requesterRunAttempt: 1, requesterRunId: REQUEST.requesterRunId, - workflowSha: CANDIDATE_SHA, + workflowSha: WORKFLOW_SHA, }, producer: { repository: PRODUCER_REPOSITORY, diff --git a/test/helpers/exact-image-qualification-controller-fixture.ts b/test/helpers/exact-image-qualification-controller-fixture.ts index 4b91462289..4cd9e2d7b9 100644 --- a/test/helpers/exact-image-qualification-controller-fixture.ts +++ b/test/helpers/exact-image-qualification-controller-fixture.ts @@ -21,6 +21,7 @@ import { } from "../../tools/e2e/exact-image-qualification-controller.mts"; export const CANDIDATE_SHA = "a".repeat(40); +export const WORKFLOW_SHA = "c".repeat(40); export const PRODUCER_SHA = "b".repeat(40); export const CORRELATION_ID = "123e4567-e89b-42d3-a456-426614174000"; export const RUN_ID = "24680"; @@ -37,7 +38,7 @@ export const REQUEST: ExactImageQualificationRequest = { ref: "refs/heads/main", requesterRunAttempt: 1, requesterRunId: "97531", - workflowSha: CANDIDATE_SHA, + workflowSha: WORKFLOW_SHA, }; export function dispatchIntent(): ExactImageDispatchIntent { @@ -123,7 +124,7 @@ export function workflowRun(overrides: Record = {}) { export function createApi(options: ApiOptions = {}) { return vi.fn(async (apiPath: string, _token: string, requestOptions?: unknown) => { if (apiPath === "repos/NVIDIA/NemoClaw/git/ref/heads/main") { - return mainRef(options.requesterSha ?? CANDIDATE_SHA); + return mainRef(options.requesterSha ?? WORKFLOW_SHA); } if (apiPath === `repos/NVIDIA/NemoClaw/git/commits/${CANDIDATE_SHA}`) { return { sha: CANDIDATE_SHA }; diff --git a/tools/e2e/exact-image-qualification-controller.mts b/tools/e2e/exact-image-qualification-controller.mts index 9cb6ad28fb..59aa4eff8a 100755 --- a/tools/e2e/exact-image-qualification-controller.mts +++ b/tools/e2e/exact-image-qualification-controller.mts @@ -668,12 +668,7 @@ function readDispatchIntent(workDir: string): ExactImageDispatchIntent | null { "dispatch intent request", ); persistedString(request, "actor", "dispatch intent actor", GITHUB_LOGIN_PATTERN); - const candidateSha = persistedString( - request, - "candidateSha", - "dispatch intent candidate SHA", - FULL_SHA_PATTERN, - ); + persistedString(request, "candidateSha", "dispatch intent candidate SHA", FULL_SHA_PATTERN); persistedString(request, "correlationId", "dispatch intent correlation ID", UUID_V4_PATTERN); validatePersistedReason(request.reason, "dispatch intent reason"); if (request.requesterRunAttempt !== 1) { @@ -685,15 +680,7 @@ function readDispatchIntent(workDir: string): ExactImageDispatchIntent | null { "dispatch intent requester run ID", DECIMAL_ID_PATTERN, ); - const workflowSha = persistedString( - request, - "workflowSha", - "dispatch intent workflow SHA", - FULL_SHA_PATTERN, - ); - if (candidateSha !== workflowSha) { - fail("PROVENANCE_MISMATCH", "dispatch intent candidate and workflow SHAs differ"); - } + persistedString(request, "workflowSha", "dispatch intent workflow SHA", FULL_SHA_PATTERN); const producer = record(intent.producer, "dispatch intent producer"); requireExactRecordFields( @@ -820,12 +807,7 @@ function validatePersistedRequest(value: unknown): void { "controller state request", ); persistedString(request, "actor", "controller state actor", GITHUB_LOGIN_PATTERN); - const candidateSha = persistedString( - request, - "candidateSha", - "controller state candidate SHA", - FULL_SHA_PATTERN, - ); + persistedString(request, "candidateSha", "controller state candidate SHA", FULL_SHA_PATTERN); persistedString(request, "correlationId", "controller state correlation ID", UUID_V4_PATTERN); validatePersistedReason(request.reason, "controller state reason"); if (request.requesterRunAttempt !== 1) { @@ -837,15 +819,7 @@ function validatePersistedRequest(value: unknown): void { "controller state requester run ID", DECIMAL_ID_PATTERN, ); - const workflowSha = persistedString( - request, - "workflowSha", - "controller state workflow SHA", - FULL_SHA_PATTERN, - ); - if (candidateSha !== workflowSha) { - fail("PROVENANCE_MISMATCH", "controller state candidate and workflow SHAs differ"); - } + persistedString(request, "workflowSha", "controller state workflow SHA", FULL_SHA_PATTERN); } function validatePersistedProducer( From de8ab46c95cf3f11948702458323c7907488daf6 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Fri, 17 Jul 2026 16:13:15 -0400 Subject: [PATCH 10/25] fix(e2e): harden staging launchable qualification Signed-off-by: Julie Yaunches --- .../brev-launchable-qualification.yaml | 19 +-- .github/workflows/e2e.yaml | 6 +- test/brev-launchable-runtime.test.ts | 125 +++++++++++++++--- .../e2e/live/exact-staging-launchable.test.ts | 55 ++++++++ ...act-image-qualification-controller.test.ts | 7 +- ...exact-image-qualification-workflow.test.ts | 17 ++- ...-image-qualification-controller-fixture.ts | 2 +- tools/e2e/brev-launchable-runtime.sh | 8 +- .../exact-image-qualification-controller.mts | 18 ++- 9 files changed, 215 insertions(+), 42 deletions(-) create mode 100644 test/e2e/live/exact-staging-launchable.test.ts diff --git a/.github/workflows/brev-launchable-qualification.yaml b/.github/workflows/brev-launchable-qualification.yaml index a774a084fb..aa3f5681f6 100644 --- a/.github/workflows/brev-launchable-qualification.yaml +++ b/.github/workflows/brev-launchable-qualification.yaml @@ -106,6 +106,11 @@ jobs: with: node-version: "22" + - name: Prepare E2E workspace + uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@50281ee84c4a6fc759da95ea28fc0b7d9c378a28 + with: + build-cli: "false" + - id: workspace name: Create private evidence workspace shell: bash @@ -222,21 +227,19 @@ jobs: instance_name="nclaw-e2e-${short_sha}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" printf 'instance_name=%s\n' "$instance_name" >> "$GITHUB_OUTPUT" - - name: Deploy the configured staging Launchable + - name: Run exact staging Launchable live test env: BREV_LAUNCHABLE_ID: ${{ vars.NEMOCLAW_STAGING_LAUNCHABLE_ID }} - INSTANCE_NAME: ${{ steps.brev-workspace.outputs.instance_name }} - WORK_DIR: ${{ steps.workspace.outputs.work_dir }} - run: tools/e2e/brev-launchable-runtime.sh deploy - - - name: Verify exact boot image and run preinstalled-image smoke - env: CANDIDATE_SHA: ${{ inputs.candidate_sha }} INSTANCE_NAME: ${{ steps.brev-workspace.outputs.instance_name }} + NEMOCLAW_RUN_EXACT_STAGING_LAUNCHABLE: "1" + NEMOCLAW_RUN_LIVE_E2E: "1" NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }} VALIDATED_MANIFEST: ${{ steps.workspace.outputs.work_dir }}/validated-manifest.v1.json WORK_DIR: ${{ steps.workspace.outputs.work_dir }} - run: tools/e2e/brev-launchable-runtime.sh qualify + run: >- + npx tsx tools/e2e/live-vitest-invocation.mts run + --test-path test/e2e/live/exact-staging-launchable.test.ts - name: Redact runtime evidence if: ${{ always() && steps.workspace.outputs.work_dir != '' }} diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index fd718fe4bd..97209813df 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -193,7 +193,11 @@ jobs: with: candidate_sha: ${{ inputs.checkout_sha || github.sha }} reason: E2E run ${{ github.run_id }} qualifying the exact checked-out NemoClaw candidate - secrets: inherit + secrets: + NEMOCLAW_IMAGE_DISPATCH_TOKEN: ${{ secrets.NEMOCLAW_IMAGE_DISPATCH_TOKEN }} + BREV_API_KEY: ${{ secrets.BREV_API_KEY }} + BREV_ORG_ID: ${{ secrets.BREV_ORG_ID }} + NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }} live: needs: generate-matrix diff --git a/test/brev-launchable-runtime.test.ts b/test/brev-launchable-runtime.test.ts index 850fbaaff8..46c265022e 100644 --- a/test/brev-launchable-runtime.test.ts +++ b/test/brev-launchable-runtime.test.ts @@ -12,30 +12,42 @@ const SCRIPT = path.join(REPO_ROOT, "tools", "e2e", "brev-launchable-runtime.sh" const roots: string[] = []; const candidateSha = "a".repeat(40); +type FixtureOptions = { + lsMode?: "ok" | "fail" | "malformed"; + provisionSha?: string; + repoSha?: string; +}; + afterEach(() => { for (const root of roots.splice(0)) fs.rmSync(root, { recursive: true, force: true }); }); -function fixture(repoSha = candidateSha) { +function writeExecutable(file: string, contents: string): void { + fs.writeFileSync(file, contents, { mode: 0o755 }); +} + +function fixture(options: FixtureOptions = {}) { const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-launchable-runtime-")); roots.push(root); const bin = path.join(root, "bin"); const workDir = path.join(root, "evidence"); + const home = path.join(root, "home"); const state = path.join(root, "state.json"); const log = path.join(root, "brev.log"); const manifest = path.join(workDir, "validated-manifest.v1.json"); fs.mkdirSync(bin); fs.mkdirSync(workDir); - fs.writeFileSync(path.join(bin, "timeout"), '#!/usr/bin/env bash\nshift\nexec "$@"\n', { - mode: 0o755, - }); - fs.writeFileSync( + fs.mkdirSync(path.join(home, "NemoClaw", ".git"), { recursive: true }); + writeExecutable(path.join(bin, "timeout"), '#!/usr/bin/env bash\nshift\nexec "$@"\n'); + writeExecutable( path.join(bin, "brev"), `#!/usr/bin/env bash set -euo pipefail printf '%s\\n' "$*" >> "$FAKE_BREV_LOG" case "$1" in ls) + [ "$FAKE_BREV_LS_MODE" != fail ] || exit 9 + if [ "$FAKE_BREV_LS_MODE" = malformed ]; then printf '{}\\n'; exit 0; fi if [ -f "$FAKE_BREV_STATE" ]; then cat "$FAKE_BREV_STATE"; else printf '{"workspaces":[]}\\n'; fi ;; create) @@ -43,24 +55,70 @@ case "$1" in ;; exec) shift 3 - command="$*" - case "$command" in - *'/etc/nemoclaw/provision.json'*) printf '{"gitSha":"aaaaaaa","version":"0.0.0"}\\n' ;; - *'git -C'*'rev-parse HEAD'*) printf '%s\\n' "$FAKE_REPO_SHA" ;; - *'metadata.google.internal'*) printf '{"sourceImage":"https://www.googleapis.com/compute/v1/projects/brevdevprod/global/images/image-a","sourceImageId":"123456789"}\\n' ;; - *'brev-quickstart'*) printf 'Ready!\\n' ;; - *'inference.local'*) printf '{"choices":[{"message":{"content":"PONG"}}]}\\n' ;; - *'openclaw agent'*) printf '{"payloads":[{"text":"42"}]}\\n' ;; - *) printf 'ok\\n' ;; - esac + bash -c "$*" ;; delete) rm -f "$FAKE_BREV_STATE" ;; refresh) ;; *) exit 2 ;; esac `, - { mode: 0o755 }, ); + writeExecutable( + path.join(bin, "sudo"), + `#!/usr/bin/env bash +set -euo pipefail +[ "\${1:-}" != -n ] || shift +if [ "\${1:-}" = cat ] && [ "\${2:-}" = /etc/nemoclaw/provision.json ]; then + jq -cn --arg sha "$FAKE_PROVISION_SHA" '{gitSha:$sha,version:"0.0.0"}' + exit 0 +fi +exec "$@" +`, + ); + writeExecutable( + path.join(bin, "git"), + `#!/usr/bin/env bash +set -euo pipefail +if [[ "$*" == *'rev-parse HEAD'* ]]; then printf '%s\\n' "$FAKE_REPO_SHA"; exit 0; fi +exit 2 +`, + ); + writeExecutable( + path.join(bin, "curl"), + `#!/usr/bin/env bash +set -euo pipefail +for value in "$@"; do + case "$value" in + */project/project-id) printf 'brevdevprod\\n'; exit 0 ;; + */instance/zone) printf 'projects/1/zones/us-central1-a\\n'; exit 0 ;; + */instance/disks/0/device-name) printf 'disk-1\\n'; exit 0 ;; + */instance/service-accounts/default/token) printf '{"access_token":"token"}\\n'; exit 0 ;; + https://compute.googleapis.com/*) printf '{"sourceImage":"https://www.googleapis.com/compute/v1/projects/brevdevprod/global/images/image-a","sourceImageId":"123456789"}\\n'; exit 0 ;; + https://inference.local/*) printf '{"choices":[{"message":{"content":"PONG"}}]}\\n'; exit 0 ;; + esac +done +exit 2 +`, + ); + writeExecutable( + path.join(bin, "openshell"), + `#!/usr/bin/env bash +set -euo pipefail +if [ "\${1:-}" = --version ]; then printf 'openshell 1.0\\n'; exit 0; fi +while [ "$#" -gt 0 ] && [ "$1" != -- ]; do shift; done +[ "\${1:-}" = -- ] && shift +exec "$@" +`, + ); + for (const [name, body] of [ + ["brev-quickstart", "printf 'Ready!\\n'"], + ["docker", "exit 0"], + ["nemoclaw", "exit 0"], + ["node", "printf 'nvidia/test-model\\n'"], + ["openclaw", 'printf \'{"payloads":[{"text":"42"}]}\\n\''], + ] as const) { + writeExecutable(path.join(bin, name), `#!/usr/bin/env bash\nset -euo pipefail\n${body}\n`); + } fs.writeFileSync( manifest, `${JSON.stringify({ @@ -75,9 +133,12 @@ esac PATH: `${bin}:${process.env.PATH ?? ""}`, BREV_LAUNCHABLE_ID: "env-staging123", CANDIDATE_SHA: candidateSha, + FAKE_BREV_LS_MODE: options.lsMode ?? "ok", FAKE_BREV_LOG: log, FAKE_BREV_STATE: state, - FAKE_REPO_SHA: repoSha, + FAKE_PROVISION_SHA: options.provisionSha ?? candidateSha.slice(0, 7), + FAKE_REPO_SHA: options.repoSha ?? candidateSha, + HOME: home, INSTANCE_NAME: "nclaw-e2e-test-1", NVIDIA_INFERENCE_API_KEY: "nvapi-test-value", VALIDATED_MANIFEST: manifest, @@ -95,7 +156,11 @@ describe("exact staging Brev Launchable runtime", () => { it("deploys only the configured Launchable, proves identity, smokes the baked install, and deletes", () => { const { env, log, workDir } = fixture(); expect(run("deploy", env).status).toBe(0); - expect(run("qualify", env).status).toBe(0); + const qualification = run("qualify", env); + expect( + qualification.status, + [qualification.stderr, qualification.stdout, fs.readFileSync(log, "utf8")].join("\n"), + ).toBe(0); expect(run("cleanup", env).status).toBe(0); const commands = fs.readFileSync(log, "utf8"); @@ -112,11 +177,31 @@ describe("exact staging Brev Launchable runtime", () => { }); it("fails closed on a baked SHA mismatch before onboarding", () => { - const { env, log } = fixture("b".repeat(40)); + const { env, log } = fixture({ repoSha: "b".repeat(40) }); + expect(run("deploy", env).status).toBe(0); + const qualification = run("qualify", env); + expect(qualification.status).not.toBe(0); + expect( + [qualification.stderr, qualification.stdout, fs.readFileSync(log, "utf8")].join("\n"), + ).toContain("does not match candidate"); + expect(fs.readFileSync(log, "utf8")).not.toContain("brev-quickstart"); + expect(run("cleanup", env).status).toBe(0); + }); + + it.each(["fail", "malformed"] as const)("fails closed when Brev inventory is %s", (lsMode) => { + const { env, log } = fixture({ lsMode }); + const deploy = run("deploy", env); + expect(deploy.status).not.toBe(0); + expect(deploy.stderr).toContain("unable to inventory Brev workspaces before deploy"); + expect(fs.readFileSync(log, "utf8")).not.toContain("create "); + }); + + it("rejects an empty provision SHA before onboarding", () => { + const { env, log } = fixture({ provisionSha: "" }); expect(run("deploy", env).status).toBe(0); const qualification = run("qualify", env); expect(qualification.status).not.toBe(0); - expect(qualification.stderr).toContain("does not match candidate"); + expect(qualification.stderr).toContain("provision metadata SHA must be a lowercase Git SHA"); expect(fs.readFileSync(log, "utf8")).not.toContain("brev-quickstart"); expect(run("cleanup", env).status).toBe(0); }); diff --git a/test/e2e/live/exact-staging-launchable.test.ts b/test/e2e/live/exact-staging-launchable.test.ts new file mode 100644 index 0000000000..008b6811ea --- /dev/null +++ b/test/e2e/live/exact-staging-launchable.test.ts @@ -0,0 +1,55 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync, type SpawnSyncReturns } from "node:child_process"; +import path from "node:path"; + +import { expect, test } from "vitest"; + +import { REPO_ROOT } from "../fixtures/paths.ts"; + +const SCRIPT = path.join(REPO_ROOT, "tools", "e2e", "brev-launchable-runtime.sh"); +const enabled = process.env.NEMOCLAW_RUN_EXACT_STAGING_LAUNCHABLE === "1"; +const liveTest = enabled ? test : test.skip; + +function run(mode: "deploy" | "qualify" | "cleanup"): SpawnSyncReturns { + return spawnSync("bash", [SCRIPT, mode], { + cwd: REPO_ROOT, + encoding: "utf8", + env: process.env, + timeout: 45 * 60_000, + }); +} + +function failure(mode: string, result: SpawnSyncReturns): Error { + const detail = [result.error?.message, result.stderr, result.stdout].filter(Boolean).join("\n"); + return new Error(`${mode} failed with status ${String(result.status)}\n${detail}`); +} + +liveTest( + "exact staging Launchable: deploy, prove image identity, smoke the baked install, and clean up", + { timeout: 60 * 60_000 }, + () => { + let journeyFailure: Error | undefined; + try { + const deploy = run("deploy"); + if (deploy.status !== 0) throw failure("deploy", deploy); + + const qualify = run("qualify"); + if (qualify.status !== 0) throw failure("qualify", qualify); + } catch (error) { + journeyFailure = error instanceof Error ? error : new Error(String(error)); + } + + const cleanup = run("cleanup"); + if (cleanup.status !== 0) { + const cleanupFailure = failure("cleanup", cleanup); + throw journeyFailure + ? new AggregateError([journeyFailure, cleanupFailure], "qualification and cleanup failed") + : cleanupFailure; + } + if (journeyFailure) throw journeyFailure; + + expect(cleanup.status).toBe(0); + }, +); diff --git a/test/exact-image-qualification-controller.test.ts b/test/exact-image-qualification-controller.test.ts index da2050b42f..62171862af 100644 --- a/test/exact-image-qualification-controller.test.ts +++ b/test/exact-image-qualification-controller.test.ts @@ -16,7 +16,6 @@ import { DISPATCH_RECONCILIATION_FILE, downloadExactImageManifest, EVIDENCE_FILE, - ExactImageQualificationError, extractExactManifestArchive, finalizeExactImageQualification, GITHUB_API_VERSION, @@ -108,7 +107,7 @@ describe("exact image qualification request", () => { "--ref", "refs/heads/main", "--requester-run-attempt", - "1", + "2", "--requester-run-id", REQUEST.requesterRunId, "--workflow-sha", @@ -190,7 +189,7 @@ describe("exact producer dispatch binding", () => { nemoclaw_sha: CANDIDATE_SHA, correlation_id: CORRELATION_ID, requester_workflow_run_id: REQUEST.requesterRunId, - requester_workflow_run_attempt: "1", + requester_workflow_run_attempt: "2", }, return_run_details: true, }, @@ -1395,7 +1394,7 @@ describe("qualification evidence finalization", () => { candidateSha: CANDIDATE_SHA, correlationId: CORRELATION_ID, reason: REQUEST.reason, - requesterRunAttempt: 1, + requesterRunAttempt: REQUEST.requesterRunAttempt, requesterRunId: REQUEST.requesterRunId, workflowSha: WORKFLOW_SHA, }, diff --git a/test/exact-image-qualification-workflow.test.ts b/test/exact-image-qualification-workflow.test.ts index 2544bc12ef..9e744e40ab 100644 --- a/test/exact-image-qualification-workflow.test.ts +++ b/test/exact-image-qualification-workflow.test.ts @@ -11,6 +11,7 @@ import { } from "./helpers/e2e-workflow-contract"; const WORKFLOW_PATH = ".github/workflows/brev-launchable-qualification.yaml"; +const E2E_WORKFLOW_PATH = ".github/workflows/e2e.yaml"; const CONTROLLER_PATH = "tools/e2e/exact-image-qualification-controller.mts"; const RUNTIME_PATH = "tools/e2e/brev-launchable-runtime.sh"; @@ -31,7 +32,7 @@ function strings(value: unknown): string[] { : []; } -function job(workflow: QualificationWorkflow, name: string): WorkflowJob { +function job(workflow: Workflow, name: string): WorkflowJob { const value = workflow.jobs[name]; expect(value, `missing ${name} job`).toBeDefined(); return value!; @@ -40,11 +41,13 @@ function job(workflow: QualificationWorkflow, name: string): WorkflowJob { // source-shape-contract: security -- Exact-image qualification must remain manual/reusable, protected, fixed-target, least-privilege, identity-gated, and cleanup-verifying it("keeps exact-image Launchable qualification protected, reusable, and fail-closed", () => { const workflow = readYaml(WORKFLOW_PATH); + const e2eWorkflow = readYaml(E2E_WORKFLOW_PATH); const source = readRepoText(WORKFLOW_PATH); const controller = readRepoText(CONTROLLER_PATH); const runtime = readRepoText(RUNTIME_PATH); const preflight = job(workflow, "preflight"); const qualify = job(workflow, "qualify"); + const caller = job(e2eWorkflow, "staging-brev-launchable"); const workflowStrings = strings(workflow); expect(workflow.name).toBe("E2E / Exact Staging Brev Launchable"); @@ -65,6 +68,12 @@ it("keeps exact-image Launchable qualification protected, reusable, and fail-clo "NVIDIA_INFERENCE_API_KEY", ]); expect(workflow.permissions).toEqual({}); + expect(caller.secrets).toEqual({ + NEMOCLAW_IMAGE_DISPATCH_TOKEN: "${{ secrets.NEMOCLAW_IMAGE_DISPATCH_TOKEN }}", + BREV_API_KEY: "${{ secrets.BREV_API_KEY }}", + BREV_ORG_ID: "${{ secrets.BREV_ORG_ID }}", + NVIDIA_INFERENCE_API_KEY: "${{ secrets.NVIDIA_INFERENCE_API_KEY }}", + }); expect(workflow.concurrency).toEqual({ group: "brev-launchable-qualification-${{ inputs.candidate_sha }}", "cancel-in-progress": false, @@ -86,7 +95,7 @@ it("keeps exact-image Launchable qualification protected, reusable, and fail-clo expect(step.run ?? "").not.toContain("${{ inputs."); } - const actionUses = workflowStrings.filter((value) => value.includes("actions/")); + const actionUses = workflowStrings.filter((value) => value.startsWith("actions/")); expect(actionUses.length).toBeGreaterThan(0); for (const use of actionUses) expect(use).toMatch(/^actions\/[a-z-]+@[0-9a-f]{40}$/u); for (const checkout of qualify.steps?.filter((step) => @@ -130,7 +139,9 @@ it("keeps exact-image Launchable qualification protected, reusable, and fail-clo expect(source).toContain("--mode finalize"); expect(runtime).toContain("brev create"); expect(runtime).toContain("--launchable"); - expect(source).toContain("brev-launchable-runtime.sh qualify"); + expect(source).toContain("tools/e2e/live-vitest-invocation.mts run"); + expect(source).toContain("test/e2e/live/exact-staging-launchable.test.ts"); + expect(source).not.toContain("brev-launchable-runtime.sh qualify"); expect(source).toContain("brev-launchable-runtime.sh cleanup"); expect(source).toContain("brev-cleanup-evidence.json"); expect(source).toContain("NEMOCLAW_STAGING_LAUNCHABLE_ID"); diff --git a/test/helpers/exact-image-qualification-controller-fixture.ts b/test/helpers/exact-image-qualification-controller-fixture.ts index 4cd9e2d7b9..3d56e528cb 100644 --- a/test/helpers/exact-image-qualification-controller-fixture.ts +++ b/test/helpers/exact-image-qualification-controller-fixture.ts @@ -36,7 +36,7 @@ export const REQUEST: ExactImageQualificationRequest = { eventName: "workflow_dispatch", reason: "Qualify the current daily candidate before tagging", ref: "refs/heads/main", - requesterRunAttempt: 1, + requesterRunAttempt: 2, requesterRunId: "97531", workflowSha: WORKFLOW_SHA, }; diff --git a/tools/e2e/brev-launchable-runtime.sh b/tools/e2e/brev-launchable-runtime.sh index e25ef47793..e45ecca214 100755 --- a/tools/e2e/brev-launchable-runtime.sh +++ b/tools/e2e/brev-launchable-runtime.sh @@ -76,10 +76,14 @@ wait_for_workspace_ready() { deploy() { validate_common + local existing require_env BREV_LAUNCHABLE_ID [[ "$BREV_LAUNCHABLE_ID" =~ ^env-[A-Za-z0-9]+$ ]] \ || die "BREV_LAUNCHABLE_ID must be one opaque env-* ID" - if [ -n "$(workspace_record)" ]; then + if ! existing="$(workspace_record)"; then + die "unable to inventory Brev workspaces before deploy" + fi + if [ -n "$existing" ]; then die "refusing to reuse pre-existing workspace $INSTANCE_NAME" fi printf '{"schemaVersion":1,"launchableId":%s,"workspaceName":%s,"requestedAt":%s}\n' \ @@ -115,6 +119,8 @@ verify_identity() { || die "the baked provision metadata is missing or malformed" printf '%s\n' "$provision" >"$WORK_DIR/brev-provision.json" provision_sha="$(jq -r '.gitSha' <<<"$provision")" + [[ "$provision_sha" =~ ^[0-9a-f]{7,40}$ ]] \ + || die "provision metadata SHA must be a lowercase Git SHA" # HOME and repo are intentionally expanded by the remote host shell. # shellcheck disable=SC2016 diff --git a/tools/e2e/exact-image-qualification-controller.mts b/tools/e2e/exact-image-qualification-controller.mts index 59aa4eff8a..84ee56b36a 100755 --- a/tools/e2e/exact-image-qualification-controller.mts +++ b/tools/e2e/exact-image-qualification-controller.mts @@ -671,8 +671,13 @@ function readDispatchIntent(workDir: string): ExactImageDispatchIntent | null { persistedString(request, "candidateSha", "dispatch intent candidate SHA", FULL_SHA_PATTERN); persistedString(request, "correlationId", "dispatch intent correlation ID", UUID_V4_PATTERN); validatePersistedReason(request.reason, "dispatch intent reason"); - if (request.requesterRunAttempt !== 1) { - fail("PROVENANCE_MISMATCH", "dispatch intent requester run attempt must equal 1"); + const requesterRunAttempt = request.requesterRunAttempt; + if ( + typeof requesterRunAttempt !== "number" || + !Number.isSafeInteger(requesterRunAttempt) || + requesterRunAttempt < 1 + ) { + fail("PROVENANCE_MISMATCH", "dispatch intent requester run attempt must be positive"); } persistedString( request, @@ -810,8 +815,13 @@ function validatePersistedRequest(value: unknown): void { persistedString(request, "candidateSha", "controller state candidate SHA", FULL_SHA_PATTERN); persistedString(request, "correlationId", "controller state correlation ID", UUID_V4_PATTERN); validatePersistedReason(request.reason, "controller state reason"); - if (request.requesterRunAttempt !== 1) { - fail("PROVENANCE_MISMATCH", "controller state requester run attempt must equal 1"); + const requesterRunAttempt = request.requesterRunAttempt; + if ( + typeof requesterRunAttempt !== "number" || + !Number.isSafeInteger(requesterRunAttempt) || + requesterRunAttempt < 1 + ) { + fail("PROVENANCE_MISMATCH", "controller state requester run attempt must be positive"); } persistedString( request, From 7b540514659d4e6c7047dcc622fd3f0f59c3516f Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Fri, 17 Jul 2026 16:13:15 -0400 Subject: [PATCH 11/25] fix(e2e): harden staging launchable qualification Signed-off-by: Julie Yaunches --- .../e2e/live/exact-staging-launchable.test.ts | 33 ++++--------------- 1 file changed, 7 insertions(+), 26 deletions(-) diff --git a/test/e2e/live/exact-staging-launchable.test.ts b/test/e2e/live/exact-staging-launchable.test.ts index 008b6811ea..476f6d0479 100644 --- a/test/e2e/live/exact-staging-launchable.test.ts +++ b/test/e2e/live/exact-staging-launchable.test.ts @@ -4,13 +4,12 @@ import { spawnSync, type SpawnSyncReturns } from "node:child_process"; import path from "node:path"; -import { expect, test } from "vitest"; +import { expect, onTestFinished, test } from "vitest"; import { REPO_ROOT } from "../fixtures/paths.ts"; const SCRIPT = path.join(REPO_ROOT, "tools", "e2e", "brev-launchable-runtime.sh"); const enabled = process.env.NEMOCLAW_RUN_EXACT_STAGING_LAUNCHABLE === "1"; -const liveTest = enabled ? test : test.skip; function run(mode: "deploy" | "qualify" | "cleanup"): SpawnSyncReturns { return spawnSync("bash", [SCRIPT, mode], { @@ -21,35 +20,17 @@ function run(mode: "deploy" | "qualify" | "cleanup"): SpawnSyncReturns { }); } -function failure(mode: string, result: SpawnSyncReturns): Error { +function expectSuccess(mode: string, result: SpawnSyncReturns): void { const detail = [result.error?.message, result.stderr, result.stdout].filter(Boolean).join("\n"); - return new Error(`${mode} failed with status ${String(result.status)}\n${detail}`); + expect(result.status, `${mode} failed with status ${String(result.status)}\n${detail}`).toBe(0); } -liveTest( +test.runIf(enabled)( "exact staging Launchable: deploy, prove image identity, smoke the baked install, and clean up", { timeout: 60 * 60_000 }, () => { - let journeyFailure: Error | undefined; - try { - const deploy = run("deploy"); - if (deploy.status !== 0) throw failure("deploy", deploy); - - const qualify = run("qualify"); - if (qualify.status !== 0) throw failure("qualify", qualify); - } catch (error) { - journeyFailure = error instanceof Error ? error : new Error(String(error)); - } - - const cleanup = run("cleanup"); - if (cleanup.status !== 0) { - const cleanupFailure = failure("cleanup", cleanup); - throw journeyFailure - ? new AggregateError([journeyFailure, cleanupFailure], "qualification and cleanup failed") - : cleanupFailure; - } - if (journeyFailure) throw journeyFailure; - - expect(cleanup.status).toBe(0); + onTestFinished(() => expectSuccess("cleanup", run("cleanup")), 15 * 60_000); + expectSuccess("deploy", run("deploy")); + expectSuccess("qualify", run("qualify")); }, ); From c9b94687c462ff47b99b0f4f7a40d8703d60f04a Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Fri, 17 Jul 2026 16:24:26 -0400 Subject: [PATCH 12/25] test(e2e): map exact launchable fast coverage Signed-off-by: Julie Yaunches --- test/e2e/mock-parity.json | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/e2e/mock-parity.json b/test/e2e/mock-parity.json index 31da6469b2..926c01829d 100644 --- a/test/e2e/mock-parity.json +++ b/test/e2e/mock-parity.json @@ -2,6 +2,14 @@ "$comment": "SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\nSPDX-License-Identifier: Apache-2.0", "version": 1, "entries": [ + { + "live": "test/e2e/live/exact-staging-launchable.test.ts", + "fast": [ + "test/brev-launchable-runtime.test.ts", + "test/exact-image-qualification-controller.test.ts", + "test/exact-image-qualification-workflow.test.ts" + ] + }, { "live": "test/e2e/live/hermes-gpu-startup.test.ts", "fast": [ From 08a83aac5863fb544fe76938c75bc2ab7bb1f3da Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Sat, 18 Jul 2026 12:50:04 -0700 Subject: [PATCH 13/25] fix(e2e): retry initial Brev cleanup inventory Signed-off-by: Carlos Villela --- test/brev-launchable-runtime.test.ts | 19 ++++++++++++++++++- tools/e2e/brev-launchable-runtime.sh | 13 ++++++++----- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/test/brev-launchable-runtime.test.ts b/test/brev-launchable-runtime.test.ts index 46c265022e..f2c7cc76ab 100644 --- a/test/brev-launchable-runtime.test.ts +++ b/test/brev-launchable-runtime.test.ts @@ -13,7 +13,7 @@ const roots: string[] = []; const candidateSha = "a".repeat(40); type FixtureOptions = { - lsMode?: "ok" | "fail" | "malformed"; + lsMode?: "ok" | "fail" | "fail-once" | "malformed"; provisionSha?: string; repoSha?: string; }; @@ -33,6 +33,7 @@ function fixture(options: FixtureOptions = {}) { const workDir = path.join(root, "evidence"); const home = path.join(root, "home"); const state = path.join(root, "state.json"); + const lsCount = path.join(root, "ls-count"); const log = path.join(root, "brev.log"); const manifest = path.join(workDir, "validated-manifest.v1.json"); fs.mkdirSync(bin); @@ -47,6 +48,10 @@ printf '%s\\n' "$*" >> "$FAKE_BREV_LOG" case "$1" in ls) [ "$FAKE_BREV_LS_MODE" != fail ] || exit 9 + if [ "$FAKE_BREV_LS_MODE" = fail-once ] && [ ! -f "$FAKE_BREV_LS_COUNT" ]; then + : >"$FAKE_BREV_LS_COUNT" + exit 9 + fi if [ "$FAKE_BREV_LS_MODE" = malformed ]; then printf '{}\\n'; exit 0; fi if [ -f "$FAKE_BREV_STATE" ]; then cat "$FAKE_BREV_STATE"; else printf '{"workspaces":[]}\\n'; fi ;; @@ -135,6 +140,7 @@ exec "$@" CANDIDATE_SHA: candidateSha, FAKE_BREV_LS_MODE: options.lsMode ?? "ok", FAKE_BREV_LOG: log, + FAKE_BREV_LS_COUNT: lsCount, FAKE_BREV_STATE: state, FAKE_PROVISION_SHA: options.provisionSha ?? candidateSha.slice(0, 7), FAKE_REPO_SHA: options.repoSha ?? candidateSha, @@ -196,6 +202,17 @@ describe("exact staging Brev Launchable runtime", () => { expect(fs.readFileSync(log, "utf8")).not.toContain("create "); }); + it("retries cleanup when the initial Brev inventory request fails", () => { + const { env, workDir } = fixture({ lsMode: "fail-once" }); + const cleanup = run("cleanup", env); + + expect(cleanup.status, [cleanup.stderr, cleanup.stdout].join("\n")).toBe(0); + expect(cleanup.stderr).toContain("brev ls failed before cleanup"); + expect( + JSON.parse(fs.readFileSync(path.join(workDir, "brev-cleanup-evidence.json"), "utf8")), + ).toMatchObject({ terminalState: "ABSENT", workspaceName: "nclaw-e2e-test-1" }); + }); + it("rejects an empty provision SHA before onboarding", () => { const { env, log } = fixture({ provisionSha: "" }); expect(run("deploy", env).status).toBe(0); diff --git a/tools/e2e/brev-launchable-runtime.sh b/tools/e2e/brev-launchable-runtime.sh index e45ecca214..eee5ae6358 100755 --- a/tools/e2e/brev-launchable-runtime.sh +++ b/tools/e2e/brev-launchable-runtime.sh @@ -205,11 +205,14 @@ cleanup() { validate_common local requested_at verified_at deadline output record status=0 absent_count=0 requested_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)" - record="$(workspace_record)" - if [ -n "$record" ]; then - output="$(timeout 60s brev delete "$INSTANCE_NAME" 2>&1)" || status=$? - printf '%s\n' "$output" - [ "$status" -eq 0 ] || printf 'brev delete returned %s; verifying absence before failing\n' "$status" >&2 + if record="$(workspace_record)"; then + if [ -n "$record" ]; then + output="$(timeout 60s brev delete "$INSTANCE_NAME" 2>&1)" || status=$? + printf '%s\n' "$output" + [ "$status" -eq 0 ] || printf 'brev delete returned %s; verifying absence before failing\n' "$status" >&2 + fi + else + printf 'brev ls failed before cleanup; verifying absence with retries\n' >&2 fi deadline=$((SECONDS + ${BREV_DELETE_TIMEOUT_SECONDS:-600})) while [ "$SECONDS" -lt "$deadline" ]; do From 5e96bdff2c26d77c682864bcd255830f66d444fc Mon Sep 17 00:00:00 2001 From: Charan Jagwani Date: Mon, 20 Jul 2026 07:39:59 -0700 Subject: [PATCH 14/25] fix(e2e): trust main for launchable qualification Signed-off-by: Charan Jagwani --- .../workflows/brev-launchable-qualification.yaml | 4 +++- .../exact-image-qualification-controller.test.ts | 16 +++++++++++++++- test/exact-image-qualification-workflow.test.ts | 4 +++- .../e2e/exact-image-qualification-controller.mts | 16 +++++----------- 4 files changed, 26 insertions(+), 14 deletions(-) diff --git a/.github/workflows/brev-launchable-qualification.yaml b/.github/workflows/brev-launchable-qualification.yaml index aa3f5681f6..50bdd4eeb5 100644 --- a/.github/workflows/brev-launchable-qualification.yaml +++ b/.github/workflows/brev-launchable-qualification.yaml @@ -38,7 +38,8 @@ on: permissions: {} concurrency: - group: brev-launchable-qualification-${{ inputs.candidate_sha }} + # Every candidate advances the same family consumed by the standing Launchable. + group: brev-launchable-qualification-staging-cpu cancel-in-progress: false jobs: @@ -87,6 +88,7 @@ jobs: qualify: name: Build, deploy, and test exact staging image needs: preflight + if: ${{ github.ref == 'refs/heads/main' }} runs-on: ubuntu-latest timeout-minutes: 120 environment: diff --git a/test/exact-image-qualification-controller.test.ts b/test/exact-image-qualification-controller.test.ts index 62171862af..b3f0a498dc 100644 --- a/test/exact-image-qualification-controller.test.ts +++ b/test/exact-image-qualification-controller.test.ts @@ -85,12 +85,26 @@ describe("exact image qualification request", () => { ).toThrow(/workflow_dispatch or schedule/u); expect(() => validateExactImageQualificationRequest({ ...REQUEST, ref: "refs/pull/1/merge" }), - ).toThrow(/branch ref/u); + ).toThrow(/trusted main branch/u); + expect(() => + validateExactImageQualificationRequest({ ...REQUEST, ref: "refs/heads/feature" }), + ).toThrow(/trusted main branch/u); expect(() => validateExactImageQualificationRequest({ ...REQUEST, reason: " bad " })).toThrow( /reason/u, ); }); + it("rejects non-main requests before authorization API access", async () => { + const api = vi.fn(); + + await expect( + preflightExactImageQualification({ ...REQUEST, ref: "refs/heads/feature" }, "core-token", { + api: api as QualificationDependencies["api"], + }), + ).rejects.toMatchObject({ code: "REQUEST_INVALID" }); + expect(api).not.toHaveBeenCalled(); + }); + it("parses the fixed CLI surface and rejects undeclared controls", () => { expect( parseExactImageQualificationCommand([ diff --git a/test/exact-image-qualification-workflow.test.ts b/test/exact-image-qualification-workflow.test.ts index 9e744e40ab..fe1629e9b2 100644 --- a/test/exact-image-qualification-workflow.test.ts +++ b/test/exact-image-qualification-workflow.test.ts @@ -75,7 +75,7 @@ it("keeps exact-image Launchable qualification protected, reusable, and fail-clo NVIDIA_INFERENCE_API_KEY: "${{ secrets.NVIDIA_INFERENCE_API_KEY }}", }); expect(workflow.concurrency).toEqual({ - group: "brev-launchable-qualification-${{ inputs.candidate_sha }}", + group: "brev-launchable-qualification-staging-cpu", "cancel-in-progress": false, }); @@ -83,6 +83,7 @@ it("keeps exact-image Launchable qualification protected, reusable, and fail-clo expect(preflight.environment).toBeUndefined(); expect(JSON.stringify(preflight)).not.toContain("NEMOCLAW_IMAGE_QUALIFICATION_TOKEN"); expect(qualify.permissions).toEqual({ contents: "read" }); + expect(qualify.if).toBe("${{ github.ref == 'refs/heads/main' }}"); expect(qualify.environment).toEqual({ name: "approve-brev-launchable-qualification", deployment: false, @@ -112,6 +113,7 @@ it("keeps exact-image Launchable qualification protected, reusable, and fail-clo 'export const PRODUCER_WORKFLOW_FILE = "build-qualification-image.yml"', ); expect(controller).toContain('export const PRODUCER_REF = "main"'); + expect(controller).toContain('request.ref !== "refs/heads/main"'); expect(controller).toContain('export const GITHUB_API_VERSION = "2026-03-10"'); expect(controller).toContain("return_run_details: true"); expect(controller).toContain("fs.renameSync(temporary, file)"); diff --git a/tools/e2e/exact-image-qualification-controller.mts b/tools/e2e/exact-image-qualification-controller.mts index 84ee56b36a..c39e0deb61 100755 --- a/tools/e2e/exact-image-qualification-controller.mts +++ b/tools/e2e/exact-image-qualification-controller.mts @@ -39,7 +39,6 @@ const DECIMAL_ID_PATTERN = /^[1-9][0-9]*$/u; const ARTIFACT_DIGEST_PATTERN = /^sha256:[0-9a-f]{64}$/u; const SHA256_PATTERN = /^[0-9a-f]{64}$/u; const GITHUB_LOGIN_PATTERN = /^(?!-)[A-Za-z0-9-]{1,39}(? { validateExactImageQualificationRequest(request); - const branchPath = request.ref.slice("refs/heads/".length); const branch = await api( - `repos/${REQUESTER_REPOSITORY}/git/ref/heads/${encodeURIComponent(branchPath)}`, + `repos/${REQUESTER_REPOSITORY}/git/ref/heads/main`, token, ); - if (validateRequesterRef(branch, REQUESTER_REPOSITORY, request.ref) !== request.workflowSha) { - fail("DISPATCH_FORBIDDEN", "workflow SHA is no longer the requested branch head"); + if (validateMainRef(branch, REQUESTER_REPOSITORY) !== request.workflowSha) { + fail("DISPATCH_FORBIDDEN", "workflow SHA is no longer the trusted main branch head"); } const candidate = record( await api(`repos/${REQUESTER_REPOSITORY}/git/commits/${request.candidateSha}`, token), From 639c6f8b83bbdc99a2c237dba7970ceade954aca Mon Sep 17 00:00:00 2001 From: Charan Jagwani Date: Mon, 20 Jul 2026 07:42:49 -0700 Subject: [PATCH 15/25] fix(e2e): follow private file module rename Signed-off-by: Charan Jagwani --- tools/e2e/exact-image-qualification-controller.mts | 4 ++-- tools/e2e/validate-exact-image-manifest.mts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/e2e/exact-image-qualification-controller.mts b/tools/e2e/exact-image-qualification-controller.mts index c39e0deb61..98794605c1 100755 --- a/tools/e2e/exact-image-qualification-controller.mts +++ b/tools/e2e/exact-image-qualification-controller.mts @@ -10,14 +10,14 @@ import path from "node:path"; import { pathToFileURL } from "node:url"; import { type GitHubRequestOptions, githubApi } from "../advisors/github.mts"; -import * as privateFile from "./private-file.ts"; +import * as privateFile from "./private-file.mts"; // Root .ts files are exposed as CommonJS under tsx and as ESM under Node's // strip-types runtime. Normalize both forms so the workflow uses the same // no-follow state-file helpers in either test or production execution. const privateFileRuntime = ( "default" in privateFile && privateFile.default ? privateFile.default : privateFile -) as typeof import("./private-file.ts"); +) as typeof import("./private-file.mts"); export const REQUESTER_REPOSITORY = "NVIDIA/NemoClaw"; export const PRODUCER_REPOSITORY = "brevdev/nemoclaw-image"; diff --git a/tools/e2e/validate-exact-image-manifest.mts b/tools/e2e/validate-exact-image-manifest.mts index 5e53238f11..2bbaf4186a 100644 --- a/tools/e2e/validate-exact-image-manifest.mts +++ b/tools/e2e/validate-exact-image-manifest.mts @@ -11,7 +11,7 @@ import { normalizedExactImageManifestJson, parseAndValidateExactImageManifest, } from "./exact-image-manifest.mts"; -import * as privateFile from "./private-file.ts"; +import * as privateFile from "./private-file.mts"; // The root TypeScript package is exposed as CommonJS under `node --import // tsx` / `npx tsx`, but as an ESM namespace under Node's strip-types runtime @@ -19,7 +19,7 @@ import * as privateFile from "./private-file.ts"; // uses the same no-follow private-file writer. const privateFileRuntime = ( "default" in privateFile && privateFile.default ? privateFile.default : privateFile -) as typeof import("./private-file.ts"); +) as typeof import("./private-file.mts"); const MAX_MANIFEST_BYTES = 64 * 1024; const NO_FOLLOW = fs.constants.O_NOFOLLOW ?? 0; From cc60309d2f95e24831ac08da1a85e82eeff3ea11 Mon Sep 17 00:00:00 2001 From: Charan Jagwani Date: Mon, 20 Jul 2026 07:52:52 -0700 Subject: [PATCH 16/25] fix(e2e): source launchable secrets from environment Signed-off-by: Charan Jagwani --- .../brev-launchable-qualification.yaml | 9 ------- .github/workflows/e2e.yaml | 5 ---- ...exact-image-qualification-workflow.test.ts | 24 +++++++++---------- 3 files changed, 11 insertions(+), 27 deletions(-) diff --git a/.github/workflows/brev-launchable-qualification.yaml b/.github/workflows/brev-launchable-qualification.yaml index 50bdd4eeb5..4040045e99 100644 --- a/.github/workflows/brev-launchable-qualification.yaml +++ b/.github/workflows/brev-launchable-qualification.yaml @@ -25,15 +25,6 @@ on: description: Audit reason for starting this exact-image Launchable qualification. required: true type: string - secrets: - NEMOCLAW_IMAGE_DISPATCH_TOKEN: - required: true - BREV_API_KEY: - required: true - BREV_ORG_ID: - required: true - NVIDIA_INFERENCE_API_KEY: - required: true permissions: {} diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index a5b0d74282..d258b861e1 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -193,11 +193,6 @@ jobs: with: candidate_sha: ${{ inputs.checkout_sha || github.sha }} reason: E2E run ${{ github.run_id }} qualifying the exact checked-out NemoClaw candidate - secrets: - NEMOCLAW_IMAGE_DISPATCH_TOKEN: ${{ secrets.NEMOCLAW_IMAGE_DISPATCH_TOKEN }} - BREV_API_KEY: ${{ secrets.BREV_API_KEY }} - BREV_ORG_ID: ${{ secrets.BREV_ORG_ID }} - NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }} live: needs: generate-matrix diff --git a/test/exact-image-qualification-workflow.test.ts b/test/exact-image-qualification-workflow.test.ts index fe1629e9b2..b10258c3f4 100644 --- a/test/exact-image-qualification-workflow.test.ts +++ b/test/exact-image-qualification-workflow.test.ts @@ -61,19 +61,9 @@ it("keeps exact-image Launchable qualification protected, reusable, and fail-clo "candidate_sha", "reason", ]); - expect(Object.keys((workflow.on.workflow_call as { secrets: object }).secrets)).toEqual([ - "NEMOCLAW_IMAGE_DISPATCH_TOKEN", - "BREV_API_KEY", - "BREV_ORG_ID", - "NVIDIA_INFERENCE_API_KEY", - ]); + expect((workflow.on.workflow_call as { secrets?: object }).secrets).toBeUndefined(); expect(workflow.permissions).toEqual({}); - expect(caller.secrets).toEqual({ - NEMOCLAW_IMAGE_DISPATCH_TOKEN: "${{ secrets.NEMOCLAW_IMAGE_DISPATCH_TOKEN }}", - BREV_API_KEY: "${{ secrets.BREV_API_KEY }}", - BREV_ORG_ID: "${{ secrets.BREV_ORG_ID }}", - NVIDIA_INFERENCE_API_KEY: "${{ secrets.NVIDIA_INFERENCE_API_KEY }}", - }); + expect(caller.secrets).toBeUndefined(); expect(workflow.concurrency).toEqual({ group: "brev-launchable-qualification-staging-cpu", "cancel-in-progress": false, @@ -81,13 +71,21 @@ it("keeps exact-image Launchable qualification protected, reusable, and fail-clo expect(preflight.permissions).toEqual({ contents: "read" }); expect(preflight.environment).toBeUndefined(); - expect(JSON.stringify(preflight)).not.toContain("NEMOCLAW_IMAGE_QUALIFICATION_TOKEN"); expect(qualify.permissions).toEqual({ contents: "read" }); expect(qualify.if).toBe("${{ github.ref == 'refs/heads/main' }}"); expect(qualify.environment).toEqual({ name: "approve-brev-launchable-qualification", deployment: false, }); + for (const secret of [ + "NEMOCLAW_IMAGE_DISPATCH_TOKEN", + "BREV_API_KEY", + "BREV_ORG_ID", + "NVIDIA_INFERENCE_API_KEY", + ]) { + expect(JSON.stringify(preflight)).not.toContain(`secrets.${secret}`); + expect(JSON.stringify(qualify)).toContain(`secrets.${secret}`); + } expect(JSON.stringify(qualify)).toContain("NEMOCLAW_IMAGE_QUALIFICATION_TOKEN"); expect(source.match(/secrets\.NEMOCLAW_IMAGE_DISPATCH_TOKEN/gu)).toHaveLength(4); expect(workflowStrings).not.toContain("id-token: write"); From 8b133129d170e9151417c9156562df54db130e1b Mon Sep 17 00:00:00 2001 From: Charan Jagwani Date: Mon, 20 Jul 2026 08:49:32 -0700 Subject: [PATCH 17/25] ci(e2e): gate staging qualification activation Signed-off-by: Charan Jagwani --- .../brev-launchable-qualification.yaml | 16 ++++- .github/workflows/e2e.yaml | 4 +- ci/source-shape-test-budget.json | 2 +- test/brev-launchable-runtime.test.ts | 61 ++++++++++++++++++- test/e2e/README.md | 27 ++++++++ test/e2e/mock-parity.json | 2 +- ...exact-image-qualification-workflow.test.ts | 29 ++++++++- test/helpers/vitest-watch-triggers.ts | 5 +- test/vitest-watch-triggers.test.ts | 5 +- 9 files changed, 137 insertions(+), 14 deletions(-) rename test/{ => e2e/support}/exact-image-qualification-workflow.test.ts (80%) diff --git a/.github/workflows/brev-launchable-qualification.yaml b/.github/workflows/brev-launchable-qualification.yaml index 4040045e99..80237a0975 100644 --- a/.github/workflows/brev-launchable-qualification.yaml +++ b/.github/workflows/brev-launchable-qualification.yaml @@ -41,6 +41,16 @@ jobs: permissions: contents: read steps: + - name: Require explicit repository activation + env: + QUALIFICATION_ENABLED: ${{ vars.NEMOCLAW_BREV_LAUNCHABLE_QUALIFICATION_ENABLED }} + run: | + set -euo pipefail + if [[ "$QUALIFICATION_ENABLED" != "true" ]]; then + echo "::error::Exact staging Brev Launchable qualification is inactive. Set NEMOCLAW_BREV_LAUNCHABLE_QUALIFICATION_ENABLED=true only after the protected environment and external dependencies are ready." + exit 1 + fi + - name: Checkout trusted controller uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: @@ -79,7 +89,9 @@ jobs: qualify: name: Build, deploy, and test exact staging image needs: preflight - if: ${{ github.ref == 'refs/heads/main' }} + # Keep the repository-owned switch outside the protected environment so an + # inactive run cannot request approval or credentials. + if: ${{ vars.NEMOCLAW_BREV_LAUNCHABLE_QUALIFICATION_ENABLED == 'true' && github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' }} runs-on: ubuntu-latest timeout-minutes: 120 environment: @@ -297,7 +309,7 @@ jobs: ${{ steps.workspace.outputs.work_dir }}/brev-agent-response.log ${{ steps.workspace.outputs.work_dir }}/brev-smoke-evidence.json ${{ steps.workspace.outputs.work_dir }}/brev-cleanup-evidence.json - if-no-files-found: warn + if-no-files-found: error retention-days: 90 - name: Remove private evidence workspace diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index d258b861e1..729768637e 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -188,7 +188,9 @@ jobs: staging-brev-launchable: name: Exact staging Brev Launchable needs: generate-matrix - if: ${{ github.repository == 'NVIDIA/NemoClaw' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') }} + # Repository owners enable this only after the protected environment, + # producer, standing Launchable, and Brev contract are ready. + if: ${{ vars.NEMOCLAW_BREV_LAUNCHABLE_QUALIFICATION_ENABLED == 'true' && github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') }} uses: ./.github/workflows/brev-launchable-qualification.yaml with: candidate_sha: ${{ inputs.checkout_sha || github.sha }} diff --git a/ci/source-shape-test-budget.json b/ci/source-shape-test-budget.json index 8c47bbc771..72b8a11b4c 100644 --- a/ci/source-shape-test-budget.json +++ b/ci/source-shape-test-budget.json @@ -132,7 +132,7 @@ "category": "security" }, { - "file": "test/exact-image-qualification-workflow.test.ts", + "file": "test/e2e/support/exact-image-qualification-workflow.test.ts", "test": "keeps exact-image Launchable qualification protected, reusable, and fail-closed", "category": "security" }, diff --git a/test/brev-launchable-runtime.test.ts b/test/brev-launchable-runtime.test.ts index f2c7cc76ab..75a21acd18 100644 --- a/test/brev-launchable-runtime.test.ts +++ b/test/brev-launchable-runtime.test.ts @@ -13,9 +13,13 @@ const roots: string[] = []; const candidateSha = "a".repeat(40); type FixtureOptions = { + bootImageId?: string; + bootImageSelfLink?: string; + deleteMode?: "remove" | "retain"; lsMode?: "ok" | "fail" | "fail-once" | "malformed"; provisionSha?: string; repoSha?: string; + workspaceMode?: "ready" | "pending"; }; afterEach(() => { @@ -56,13 +60,17 @@ case "$1" in if [ -f "$FAKE_BREV_STATE" ]; then cat "$FAKE_BREV_STATE"; else printf '{"workspaces":[]}\\n'; fi ;; create) - printf '{"workspaces":[{"id":"ws-1","name":"%s","status":"RUNNING","shell_status":"READY","health_status":"HEALTHY","build_status":"COMPLETED"}]}\\n' "$INSTANCE_NAME" > "$FAKE_BREV_STATE" + if [ "$FAKE_BREV_WORKSPACE_MODE" = pending ]; then + printf '{"workspaces":[{"id":"ws-1","name":"%s","status":"PROVISIONING","shell_status":"PENDING","health_status":"PENDING","build_status":"PENDING"}]}\\n' "$INSTANCE_NAME" > "$FAKE_BREV_STATE" + else + printf '{"workspaces":[{"id":"ws-1","name":"%s","status":"RUNNING","shell_status":"READY","health_status":"HEALTHY","build_status":"COMPLETED"}]}\\n' "$INSTANCE_NAME" > "$FAKE_BREV_STATE" + fi ;; exec) shift 3 bash -c "$*" ;; - delete) rm -f "$FAKE_BREV_STATE" ;; + delete) [ "$FAKE_BREV_DELETE_MODE" = retain ] || rm -f "$FAKE_BREV_STATE" ;; refresh) ;; *) exit 2 ;; esac @@ -98,7 +106,10 @@ for value in "$@"; do */instance/zone) printf 'projects/1/zones/us-central1-a\\n'; exit 0 ;; */instance/disks/0/device-name) printf 'disk-1\\n'; exit 0 ;; */instance/service-accounts/default/token) printf '{"access_token":"token"}\\n'; exit 0 ;; - https://compute.googleapis.com/*) printf '{"sourceImage":"https://www.googleapis.com/compute/v1/projects/brevdevprod/global/images/image-a","sourceImageId":"123456789"}\\n'; exit 0 ;; + https://compute.googleapis.com/*) + jq -cn --arg sourceImage "$FAKE_BOOT_IMAGE_SELF_LINK" --arg sourceImageId "$FAKE_BOOT_IMAGE_ID" '{sourceImage:$sourceImage,sourceImageId:$sourceImageId}' + exit 0 + ;; https://inference.local/*) printf '{"choices":[{"message":{"content":"PONG"}}]}\\n'; exit 0 ;; esac done @@ -138,12 +149,18 @@ exec "$@" PATH: `${bin}:${process.env.PATH ?? ""}`, BREV_LAUNCHABLE_ID: "env-staging123", CANDIDATE_SHA: candidateSha, + FAKE_BOOT_IMAGE_ID: options.bootImageId ?? "123456789", + FAKE_BOOT_IMAGE_SELF_LINK: + options.bootImageSelfLink ?? + "https://www.googleapis.com/compute/v1/projects/brevdevprod/global/images/image-a", + FAKE_BREV_DELETE_MODE: options.deleteMode ?? "remove", FAKE_BREV_LS_MODE: options.lsMode ?? "ok", FAKE_BREV_LOG: log, FAKE_BREV_LS_COUNT: lsCount, FAKE_BREV_STATE: state, FAKE_PROVISION_SHA: options.provisionSha ?? candidateSha.slice(0, 7), FAKE_REPO_SHA: options.repoSha ?? candidateSha, + FAKE_BREV_WORKSPACE_MODE: options.workspaceMode ?? "ready", HOME: home, INSTANCE_NAME: "nclaw-e2e-test-1", NVIDIA_INFERENCE_API_KEY: "nvapi-test-value", @@ -194,6 +211,31 @@ describe("exact staging Brev Launchable runtime", () => { expect(run("cleanup", env).status).toBe(0); }); + it("fails closed on a boot-image mismatch before onboarding", () => { + const { env, log } = fixture({ bootImageId: "987654321" }); + expect(run("deploy", env).status).toBe(0); + const qualification = run("qualify", env); + expect(qualification.status).not.toBe(0); + expect(qualification.stderr).toContain("workspace boot disk does not match accepted image"); + expect(fs.readFileSync(log, "utf8")).not.toContain("brev-quickstart"); + expect(run("cleanup", env).status).toBe(0); + }); + + it("fails when workspace readiness does not reach its deadline", () => { + const { env } = fixture({ workspaceMode: "pending" }); + const deploy = run("deploy", { + ...env, + BREV_POLL_SECONDS: "1", + BREV_READY_TIMEOUT_SECONDS: "1", + }); + + expect(deploy.status).not.toBe(0); + expect(deploy.stderr).toContain( + "Brev workspace did not become structurally ready before the deadline", + ); + expect(run("cleanup", env).status).toBe(0); + }); + it.each(["fail", "malformed"] as const)("fails closed when Brev inventory is %s", (lsMode) => { const { env, log } = fixture({ lsMode }); const deploy = run("deploy", env); @@ -213,6 +255,19 @@ describe("exact staging Brev Launchable runtime", () => { ).toMatchObject({ terminalState: "ABSENT", workspaceName: "nclaw-e2e-test-1" }); }); + it("fails cleanup when the workspace remains after the deletion deadline", () => { + const { env } = fixture({ deleteMode: "retain" }); + expect(run("deploy", env).status).toBe(0); + const cleanup = run("cleanup", { + ...env, + BREV_DELETE_TIMEOUT_SECONDS: "1", + BREV_POLL_SECONDS: "1", + }); + + expect(cleanup.status).not.toBe(0); + expect(cleanup.stderr).toContain("Brev workspace still exists after cleanup deadline"); + }); + it("rejects an empty provision SHA before onboarding", () => { const { env, log } = fixture({ provisionSha: "" }); expect(run("deploy", env).status).toBe(0); diff --git a/test/e2e/README.md b/test/e2e/README.md index 2e5734a9f0..ea2faf9c0c 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -85,6 +85,33 @@ artifact so baseline aggregation stays stable. Older issue references to Vitest target artifacts under `e2e-artifacts/vitest/` map to this consolidated `e2e-artifacts/live/` registry-target artifact layout. +## Exact staging Brev Launchable qualification + +The exact-image staging Launchable lane is inactive by default. The scheduled +and manual `e2e.yaml` paths call it only from `main` when the repository +variable `NEMOCLAW_BREV_LAUNCHABLE_QUALIFICATION_ENABLED` is exactly `true`. +A direct dispatch of `brev-launchable-qualification.yaml` applies the same +repository, ref, and activation checks; while inactive, it fails before +checkout, environment approval, credential access, or external API calls. +The skipped `staging-brev-launchable` job in a routine E2E run is an inactive +status, not successful qualification evidence. + +Before setting the activation variable, repository owners must: + +- create and protect the `approve-brev-launchable-qualification` environment + for `main`, with the required reviewers; +- configure that environment with `NEMOCLAW_IMAGE_DISPATCH_TOKEN`, + `BREV_API_KEY`, `BREV_ORG_ID`, and `NVIDIA_INFERENCE_API_KEY`; +- set the environment or repository variable `NEMOCLAW_STAGING_LAUNCHABLE_ID` + to the standing staging Launchable that consumes the staging image family; + and +- verify that the producer workflow and the Brev Launchable contract are ready + for automated image builds, provisioning, validation, and cleanup. + +Set `NEMOCLAW_BREV_LAUNCHABLE_QUALIFICATION_ENABLED=true` only after those +conditions are met. Remove the variable, or set it to any value other than +`true`, to keep routine E2E runs from starting this cost-bearing lane. + ## PR E2E gate The controller, coordination check, and required job deliberately use diff --git a/test/e2e/mock-parity.json b/test/e2e/mock-parity.json index 926c01829d..719db202f5 100644 --- a/test/e2e/mock-parity.json +++ b/test/e2e/mock-parity.json @@ -7,7 +7,7 @@ "fast": [ "test/brev-launchable-runtime.test.ts", "test/exact-image-qualification-controller.test.ts", - "test/exact-image-qualification-workflow.test.ts" + "test/e2e/support/exact-image-qualification-workflow.test.ts" ] }, { diff --git a/test/exact-image-qualification-workflow.test.ts b/test/e2e/support/exact-image-qualification-workflow.test.ts similarity index 80% rename from test/exact-image-qualification-workflow.test.ts rename to test/e2e/support/exact-image-qualification-workflow.test.ts index b10258c3f4..062f6780c6 100644 --- a/test/exact-image-qualification-workflow.test.ts +++ b/test/e2e/support/exact-image-qualification-workflow.test.ts @@ -8,12 +8,13 @@ import { readYaml, type Workflow, type WorkflowJob, -} from "./helpers/e2e-workflow-contract"; +} from "../../helpers/e2e-workflow-contract"; const WORKFLOW_PATH = ".github/workflows/brev-launchable-qualification.yaml"; const E2E_WORKFLOW_PATH = ".github/workflows/e2e.yaml"; const CONTROLLER_PATH = "tools/e2e/exact-image-qualification-controller.mts"; const RUNTIME_PATH = "tools/e2e/brev-launchable-runtime.sh"; +const ACTIVATION_VARIABLE = "NEMOCLAW_BREV_LAUNCHABLE_QUALIFICATION_ENABLED"; type QualificationWorkflow = Workflow & { name: string; @@ -62,21 +63,44 @@ it("keeps exact-image Launchable qualification protected, reusable, and fail-clo "reason", ]); expect((workflow.on.workflow_call as { secrets?: object }).secrets).toBeUndefined(); + expect(JSON.stringify(workflow.on)).not.toContain(ACTIVATION_VARIABLE); expect(workflow.permissions).toEqual({}); expect(caller.secrets).toBeUndefined(); + expect(caller.if).toBe( + `\${{ vars.${ACTIVATION_VARIABLE} == 'true' && github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') }}`, + ); expect(workflow.concurrency).toEqual({ group: "brev-launchable-qualification-staging-cpu", "cancel-in-progress": false, }); expect(preflight.permissions).toEqual({ contents: "read" }); + expect(preflight.if).toBeUndefined(); expect(preflight.environment).toBeUndefined(); + const activation = preflight.steps?.[0]; + expect(activation?.name).toBe("Require explicit repository activation"); + expect(activation?.uses).toBeUndefined(); + expect(activation?.env).toEqual({ + QUALIFICATION_ENABLED: `\${{ vars.${ACTIVATION_VARIABLE} }}`, + }); + expect(activation?.run).toContain('[[ "$QUALIFICATION_ENABLED" != "true" ]]'); + expect(activation?.run).toContain(`${ACTIVATION_VARIABLE}=true`); + expect(activation?.run).toContain("exit 1"); expect(qualify.permissions).toEqual({ contents: "read" }); - expect(qualify.if).toBe("${{ github.ref == 'refs/heads/main' }}"); + expect(qualify.if).toBe( + `\${{ vars.${ACTIVATION_VARIABLE} == 'true' && github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' }}`, + ); expect(qualify.environment).toEqual({ name: "approve-brev-launchable-qualification", deployment: false, }); + const environmentJobs = Object.values(workflow.jobs).filter( + (workflowJob) => workflowJob.environment !== undefined, + ); + expect(environmentJobs).toEqual([qualify]); + for (const environmentJob of environmentJobs) { + expect(environmentJob.if).toContain(`vars.${ACTIVATION_VARIABLE} == 'true'`); + } for (const secret of [ "NEMOCLAW_IMAGE_DISPATCH_TOKEN", "BREV_API_KEY", @@ -133,6 +157,7 @@ it("keeps exact-image Launchable qualification protected, reusable, and fail-clo } expect(source).toContain("retention-days: 90"); + expect(source).toContain("if-no-files-found: error"); expect(source).toContain("dispatch-intent.v1.json"); expect(source).toContain("dispatch-reconciliation.v1.json"); expect(source).toContain("controller-state.corrupt-*.json"); diff --git a/test/helpers/vitest-watch-triggers.ts b/test/helpers/vitest-watch-triggers.ts index c16dbb37f4..43f5b5573b 100644 --- a/test/helpers/vitest-watch-triggers.ts +++ b/test/helpers/vitest-watch-triggers.ts @@ -37,6 +37,7 @@ const E2E_WORKFLOW_CONTRACTS = [ "test/e2e/support/tunnel-lifecycle-workflow-boundary.test.ts", "test/e2e/support/upload-e2e-artifacts-workflow-boundary.test.ts", "test/e2e/support/workflow-plan.test.ts", + "test/e2e/support/exact-image-qualification-workflow.test.ts", ] as const; function runTests(...tests: string[]): () => string[] { @@ -98,13 +99,13 @@ export const vitestWatchTriggerPatterns: VitestWatchTriggerPattern[] = [ }, { pattern: /(?:^|\/)\.github\/workflows\/brev-launchable-qualification\.yaml$/, - testsToRun: runTests("test/exact-image-qualification-workflow.test.ts"), + testsToRun: runTests("test/e2e/support/exact-image-qualification-workflow.test.ts"), }, { pattern: /(?:^|\/)tools\/e2e\/brev-launchable-runtime\.sh$/, testsToRun: runTests( "test/brev-launchable-runtime.test.ts", - "test/exact-image-qualification-workflow.test.ts", + "test/e2e/support/exact-image-qualification-workflow.test.ts", ), }, { diff --git a/test/vitest-watch-triggers.test.ts b/test/vitest-watch-triggers.test.ts index 26242f8928..3fda1fae76 100644 --- a/test/vitest-watch-triggers.test.ts +++ b/test/vitest-watch-triggers.test.ts @@ -43,6 +43,7 @@ const E2E_WORKFLOW_CONTRACTS = [ "test/e2e/support/tunnel-lifecycle-workflow-boundary.test.ts", "test/e2e/support/upload-e2e-artifacts-workflow-boundary.test.ts", "test/e2e/support/workflow-plan.test.ts", + "test/e2e/support/exact-image-qualification-workflow.test.ts", ] as const; const OPAQUE_INPUTS = [ @@ -113,11 +114,11 @@ describe("Vitest opaque-input watch triggers", () => { "test/pr-e2e-required.test.ts", ]); expect(triggeredBy(".github/workflows/brev-launchable-qualification.yaml")).toEqual([ - "test/exact-image-qualification-workflow.test.ts", + "test/e2e/support/exact-image-qualification-workflow.test.ts", ]); expect(triggeredBy("tools/e2e/brev-launchable-runtime.sh")).toEqual([ "test/brev-launchable-runtime.test.ts", - "test/exact-image-qualification-workflow.test.ts", + "test/e2e/support/exact-image-qualification-workflow.test.ts", ]); expect(triggeredBy(".github/workflows/platform-vitest-main.yaml")).toEqual([ "test/platform-vitest-main-workflow.test.ts", From ace05739058145ffdccaae5a93cdf4b6beef3d3a Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Mon, 20 Jul 2026 15:49:15 -0400 Subject: [PATCH 18/25] test(e2e): reuse full suite for Brev Launchable Signed-off-by: Julie Yaunches --- .../brev-launchable-qualification.yaml | 24 +++--- test/brev-launchable-runtime.test.ts | 62 ++++++++++++-- test/e2e/README.md | 9 ++ .../e2e/live/exact-staging-launchable.test.ts | 36 -------- test/e2e/live/full-e2e.test.ts | 82 +++++++++++++------ test/e2e/mock-parity.json | 10 +-- ...exact-image-qualification-workflow.test.ts | 14 +++- tools/e2e/brev-launchable-runtime.sh | 67 +++++++++------ 8 files changed, 189 insertions(+), 115 deletions(-) delete mode 100644 test/e2e/live/exact-staging-launchable.test.ts diff --git a/.github/workflows/brev-launchable-qualification.yaml b/.github/workflows/brev-launchable-qualification.yaml index 80237a0975..d907d5d7c7 100644 --- a/.github/workflows/brev-launchable-qualification.yaml +++ b/.github/workflows/brev-launchable-qualification.yaml @@ -232,19 +232,21 @@ jobs: instance_name="nclaw-e2e-${short_sha}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" printf 'instance_name=%s\n' "$instance_name" >> "$GITHUB_OUTPUT" - - name: Run exact staging Launchable live test + - name: Deploy exact staging Brev Launchable env: BREV_LAUNCHABLE_ID: ${{ vars.NEMOCLAW_STAGING_LAUNCHABLE_ID }} + INSTANCE_NAME: ${{ steps.brev-workspace.outputs.instance_name }} + WORK_DIR: ${{ steps.workspace.outputs.work_dir }} + run: tools/e2e/brev-launchable-runtime.sh deploy + + - name: Run existing full E2E suite against the Brev Launchable + env: CANDIDATE_SHA: ${{ inputs.candidate_sha }} INSTANCE_NAME: ${{ steps.brev-workspace.outputs.instance_name }} - NEMOCLAW_RUN_EXACT_STAGING_LAUNCHABLE: "1" - NEMOCLAW_RUN_LIVE_E2E: "1" NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }} VALIDATED_MANIFEST: ${{ steps.workspace.outputs.work_dir }}/validated-manifest.v1.json WORK_DIR: ${{ steps.workspace.outputs.work_dir }} - run: >- - npx tsx tools/e2e/live-vitest-invocation.mts run - --test-path test/e2e/live/exact-staging-launchable.test.ts + run: tools/e2e/brev-launchable-runtime.sh qualify - name: Redact runtime evidence if: ${{ always() && steps.workspace.outputs.work_dir != '' }} @@ -260,7 +262,7 @@ jobs: secret = os.environ.get("NVIDIA_INFERENCE_API_KEY", "") root = Path(os.environ["WORK_DIR"]) if secret: - for name in ("brev-prerequisites.log", "brev-quickstart.log", "brev-inference-pong.json", "brev-agent-response.log"): + for name in ("brev-launchable-e2e.log",): path = root / name if path.is_file(): path.write_text(path.read_text(encoding="utf-8", errors="replace").replace(secret, "[REDACTED]"), encoding="utf-8") @@ -303,11 +305,9 @@ jobs: ${{ steps.workspace.outputs.work_dir }}/brev-provision.json ${{ steps.workspace.outputs.work_dir }}/brev-boot-image.json ${{ steps.workspace.outputs.work_dir }}/brev-identity-evidence.json - ${{ steps.workspace.outputs.work_dir }}/brev-prerequisites.log - ${{ steps.workspace.outputs.work_dir }}/brev-quickstart.log - ${{ steps.workspace.outputs.work_dir }}/brev-inference-pong.json - ${{ steps.workspace.outputs.work_dir }}/brev-agent-response.log - ${{ steps.workspace.outputs.work_dir }}/brev-smoke-evidence.json + ${{ steps.workspace.outputs.work_dir }}/brev-launchable-e2e.log + ${{ steps.workspace.outputs.work_dir }}/brev-launchable-e2e-evidence.json + ${{ steps.workspace.outputs.work_dir }}/brev-launchable-cloud-openclaw/ ${{ steps.workspace.outputs.work_dir }}/brev-cleanup-evidence.json if-no-files-found: error retention-days: 90 diff --git a/test/brev-launchable-runtime.test.ts b/test/brev-launchable-runtime.test.ts index 75a21acd18..fedab567c5 100644 --- a/test/brev-launchable-runtime.test.ts +++ b/test/brev-launchable-runtime.test.ts @@ -19,6 +19,7 @@ type FixtureOptions = { lsMode?: "ok" | "fail" | "fail-once" | "malformed"; provisionSha?: string; repoSha?: string; + supportsLaunchableMode?: boolean; workspaceMode?: "ready" | "pending"; }; @@ -43,6 +44,23 @@ function fixture(options: FixtureOptions = {}) { fs.mkdirSync(bin); fs.mkdirSync(workDir); fs.mkdirSync(path.join(home, "NemoClaw", ".git"), { recursive: true }); + fs.mkdirSync(path.join(home, "NemoClaw", "test", "e2e", "live"), { recursive: true }); + fs.writeFileSync( + path.join(home, "NemoClaw", "test", "e2e", "live", "full-e2e.test.ts"), + options.supportsLaunchableMode === false + ? "// legacy full E2E fixture\n" + : 'const mode = process.env.NEMOCLAW_E2E_SETUP_MODE; const target = "brev-launchable-cloud-openclaw";\n', + ); + fs.mkdirSync(path.join(home, "NemoClaw", "node_modules", ".bin"), { recursive: true }); + writeExecutable( + path.join(home, "NemoClaw", "node_modules", ".bin", "vitest"), + `#!/usr/bin/env bash +set -euo pipefail +artifact_dir="$E2E_ARTIFACT_DIR/brev-launchable-cloud-openclaw-onboard-inference-cli-operations-and-cleanup" +mkdir -p "$artifact_dir" +jq -n '{id:"brev-launchable-cloud-openclaw",status:"passed",runner:"vitest"}' > "$artifact_dir/target-result.json" +`, + ); writeExecutable(path.join(bin, "timeout"), '#!/usr/bin/env bash\nshift\nexec "$@"\n'); writeExecutable( path.join(bin, "brev"), @@ -70,6 +88,11 @@ case "$1" in shift 3 bash -c "$*" ;; + copy) + source_path="\${2#*:}" + destination="$3" + cp -R "$source_path"/. "$destination"/ + ;; delete) [ "$FAKE_BREV_DELETE_MODE" = retain ] || rm -f "$FAKE_BREV_STATE" ;; refresh) ;; *) exit 2 ;; @@ -176,7 +199,7 @@ function run(mode: string, env: NodeJS.ProcessEnv) { } describe("exact staging Brev Launchable runtime", () => { - it("deploys only the configured Launchable, proves identity, smokes the baked install, and deletes", () => { + it("deploys only the configured Launchable, proves identity, runs the existing E2E, and deletes", () => { const { env, log, workDir } = fixture(); expect(run("deploy", env).status).toBe(0); const qualification = run("qualify", env); @@ -191,9 +214,24 @@ describe("exact staging Brev Launchable runtime", () => { "create nclaw-e2e-test-1 --launchable env-staging123 --detached --timeout 900", ); expect(commands).not.toMatch(/rsync|install\.sh|npm (?:ci|install)|git clone/u); - expect(commands.indexOf("rev-parse HEAD")).toBeLessThan(commands.indexOf("brev-quickstart")); + expect(commands).toContain("test/e2e/live/full-e2e.test.ts"); + expect(commands).toContain("NEMOCLAW_E2E_SETUP_MODE=preinstalled-launchable"); + expect(commands).toContain("E2E_TARGET_ID=brev-launchable-cloud-openclaw"); + expect(commands.indexOf("rev-parse HEAD")).toBeLessThan( + commands.indexOf("test/e2e/live/full-e2e.test.ts"), + ); expect(fs.existsSync(path.join(workDir, "brev-identity-evidence.json"))).toBe(true); - expect(fs.existsSync(path.join(workDir, "brev-smoke-evidence.json"))).toBe(true); + expect(fs.existsSync(path.join(workDir, "brev-launchable-e2e-evidence.json"))).toBe(true); + expect( + fs.existsSync( + path.join( + workDir, + "brev-launchable-cloud-openclaw", + "brev-launchable-cloud-openclaw-onboard-inference-cli-operations-and-cleanup", + "target-result.json", + ), + ), + ).toBe(true); expect( JSON.parse(fs.readFileSync(path.join(workDir, "brev-cleanup-evidence.json"), "utf8")), ).toMatchObject({ terminalState: "ABSENT", workspaceName: "nclaw-e2e-test-1" }); @@ -207,7 +245,7 @@ describe("exact staging Brev Launchable runtime", () => { expect( [qualification.stderr, qualification.stdout, fs.readFileSync(log, "utf8")].join("\n"), ).toContain("does not match candidate"); - expect(fs.readFileSync(log, "utf8")).not.toContain("brev-quickstart"); + expect(fs.readFileSync(log, "utf8")).not.toContain("test/e2e/live/full-e2e.test.ts"); expect(run("cleanup", env).status).toBe(0); }); @@ -217,7 +255,19 @@ describe("exact staging Brev Launchable runtime", () => { const qualification = run("qualify", env); expect(qualification.status).not.toBe(0); expect(qualification.stderr).toContain("workspace boot disk does not match accepted image"); - expect(fs.readFileSync(log, "utf8")).not.toContain("brev-quickstart"); + expect(fs.readFileSync(log, "utf8")).not.toContain("test/e2e/live/full-e2e.test.ts"); + expect(run("cleanup", env).status).toBe(0); + }); + + it("fails rather than reinstalling when the candidate full E2E lacks Launchable mode", () => { + const { env, workDir } = fixture({ supportsLaunchableMode: false }); + expect(run("deploy", env).status).toBe(0); + const qualification = run("qualify", env); + expect(qualification.status).not.toBe(0); + expect(fs.readFileSync(path.join(workDir, "brev-launchable-e2e.log"), "utf8")).toContain( + "does not support preinstalled Launchable setup", + ); + expect(fs.existsSync(path.join(workDir, "brev-launchable-e2e-evidence.json"))).toBe(false); expect(run("cleanup", env).status).toBe(0); }); @@ -274,7 +324,7 @@ describe("exact staging Brev Launchable runtime", () => { const qualification = run("qualify", env); expect(qualification.status).not.toBe(0); expect(qualification.stderr).toContain("provision metadata SHA must be a lowercase Git SHA"); - expect(fs.readFileSync(log, "utf8")).not.toContain("brev-quickstart"); + expect(fs.readFileSync(log, "utf8")).not.toContain("test/e2e/live/full-e2e.test.ts"); expect(run("cleanup", env).status).toBe(0); }); }); diff --git a/test/e2e/README.md b/test/e2e/README.md index ea2faf9c0c..32b80ebeb6 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -112,6 +112,15 @@ Set `NEMOCLAW_BREV_LAUNCHABLE_QUALIFICATION_ENABLED=true` only after those conditions are met. Remove the variable, or set it to any value other than `true`, to keep routine E2E runs from starting this cost-bearing lane. +The runtime target is `brev-launchable-cloud-openclaw`. After proving the +workspace booted the accepted image and exact NemoClaw SHA, the lane runs the +existing `test/e2e/live/full-e2e.test.ts` suite in `preinstalled-launchable` +setup mode. That mode onboards through the baked `brev-quickstart` entry point +and reuses the suite's CLI, policy, hosted-inference, `inference.local`, logs, +and cleanup assertions. It fails if the baked checkout does not already +contain the pinned Vitest harness; it does not rsync source, run `install.sh`, +install packages, rebuild, or relink the product under test. + ## PR E2E gate The controller, coordination check, and required job deliberately use diff --git a/test/e2e/live/exact-staging-launchable.test.ts b/test/e2e/live/exact-staging-launchable.test.ts deleted file mode 100644 index 476f6d0479..0000000000 --- a/test/e2e/live/exact-staging-launchable.test.ts +++ /dev/null @@ -1,36 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { spawnSync, type SpawnSyncReturns } from "node:child_process"; -import path from "node:path"; - -import { expect, onTestFinished, test } from "vitest"; - -import { REPO_ROOT } from "../fixtures/paths.ts"; - -const SCRIPT = path.join(REPO_ROOT, "tools", "e2e", "brev-launchable-runtime.sh"); -const enabled = process.env.NEMOCLAW_RUN_EXACT_STAGING_LAUNCHABLE === "1"; - -function run(mode: "deploy" | "qualify" | "cleanup"): SpawnSyncReturns { - return spawnSync("bash", [SCRIPT, mode], { - cwd: REPO_ROOT, - encoding: "utf8", - env: process.env, - timeout: 45 * 60_000, - }); -} - -function expectSuccess(mode: string, result: SpawnSyncReturns): void { - const detail = [result.error?.message, result.stderr, result.stdout].filter(Boolean).join("\n"); - expect(result.status, `${mode} failed with status ${String(result.status)}\n${detail}`).toBe(0); -} - -test.runIf(enabled)( - "exact staging Launchable: deploy, prove image identity, smoke the baked install, and clean up", - { timeout: 60 * 60_000 }, - () => { - onTestFinished(() => expectSuccess("cleanup", run("cleanup")), 15 * 60_000); - expectSuccess("deploy", run("deploy")); - expectSuccess("qualify", run("qualify")); - }, -); diff --git a/test/e2e/live/full-e2e.test.ts b/test/e2e/live/full-e2e.test.ts index 05ee1613be..3d3dbaa8b4 100644 --- a/test/e2e/live/full-e2e.test.ts +++ b/test/e2e/live/full-e2e.test.ts @@ -34,11 +34,20 @@ import type { ShellProbeOutputEvent, ShellProbeResult } from "../fixtures/shell- import { extractOpenClawAgentPayloadText } from "./agent-turn-latency-helpers.ts"; const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? "e2e-full"; +const SETUP_MODE = process.env.NEMOCLAW_E2E_SETUP_MODE ?? "source-install"; +const USE_PREINSTALLED_LAUNCHABLE = SETUP_MODE === "preinstalled-launchable"; +const EVIDENCE_TARGET_ID = USE_PREINSTALLED_LAUNCHABLE + ? "brev-launchable-cloud-openclaw" + : "full-e2e"; +const TEST_NAME = USE_PREINSTALLED_LAUNCHABLE + ? "brev-launchable-cloud-openclaw: onboard, inference, cli operations, and cleanup" + : "full e2e: install, onboard, inference, cli operations, and cleanup"; const LIVE_TIMEOUT_MS = 50 * 60_000; const FIRST_TURN_TIMEOUT_MS = 240_000; const MAX_SILENCE_SECS = 60; const EXPECTED_FIRST_REPLY = "NEMOCLAW_E2E_READY_6002"; -const MEASURE_COLD_ONBOARD = process.env.E2E_TARGET_ID === "full-e2e"; +const MEASURE_COLD_ONBOARD = + !USE_PREINSTALLED_LAUNCHABLE && process.env.E2E_TARGET_ID === "full-e2e"; interface ColdOnboardCapture { outputEvents: ShellProbeOutputEvent[]; @@ -46,7 +55,10 @@ interface ColdOnboardCapture { traceFile: string; } -process.env.NEMOCLAW_CLI_BIN ??= CLI_ENTRYPOINT; +if (!new Set(["source-install", "preinstalled-launchable"]).has(SETUP_MODE)) { + throw new Error(`unsupported NEMOCLAW_E2E_SETUP_MODE: ${SETUP_MODE}`); +} +process.env.NEMOCLAW_CLI_BIN ??= USE_PREINSTALLED_LAUNCHABLE ? "nemoclaw" : CLI_ENTRYPOINT; validateSandboxName(SANDBOX_NAME); function env(extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { @@ -70,7 +82,9 @@ async function repoNemoclaw( extraEnv: NodeJS.ProcessEnv = {}, timeoutMs = 120_000, ): Promise { - return await host.command(process.execPath, [CLI_ENTRYPOINT, ...args], { + const command = USE_PREINSTALLED_LAUNCHABLE ? "nemoclaw" : process.execPath; + const commandArgs = USE_PREINSTALLED_LAUNCHABLE ? args : [CLI_ENTRYPOINT, ...args]; + return await host.command(command, commandArgs, { artifactName, env: env(extraEnv), timeoutMs, @@ -261,19 +275,21 @@ async function assertColdOnboardPerformance(input: { ).toBe(true); } -test("full e2e: install, onboard, inference, cli operations, and cleanup", { +test(TEST_NAME, { timeout: LIVE_TIMEOUT_MS, }, async ({ artifacts, cleanup: cleanupRegistry, host, sandbox, secrets, skip }) => { const hosted = requireHostedInferenceConfig(secrets); - const coldOnboardBudget = readFullE2eColdPathBudget(); + const coldOnboardBudget = USE_PREINSTALLED_LAUNCHABLE ? null : readFullE2eColdPathBudget(); const redactionValues = [hosted.apiKey]; await artifacts.target.declare({ - id: "full-e2e", + id: EVIDENCE_TARGET_ID, sandboxName: SANDBOX_NAME, endpointUrl: hosted.endpointUrl, model: hosted.model, contracts: [ - "install.sh --non-interactive completes onboarding", + USE_PREINSTALLED_LAUNCHABLE + ? "the baked Launchable completes onboarding without installing from source" + : "install.sh --non-interactive completes onboarding", "nemoclaw and openshell are installed and usable", "sandbox appears in list/status and has policy/inference configuration", "direct hosted inference and sandbox inference.local both respond", @@ -322,29 +338,41 @@ test("full e2e: install, onboard, inference, cli operations, and cleanup", { fs.rmSync(coldOnboard.traceDirectory, { recursive: true, force: true }); }); - const install = await host.command("bash", ["install.sh", "--non-interactive", "--fresh"], { - artifactName: "phase-1-install-sh", - cwd: REPO_ROOT, - env: env({ - ...hosted.env, - NVIDIA_INFERENCE_API_KEY: hosted.apiKey, - ...(coldOnboard ? { NEMOCLAW_TRACE_FILE: coldOnboard.traceFile } : {}), - }), - ...(coldOnboard - ? { onOutput: (event: ShellProbeOutputEvent) => coldOnboard.outputEvents.push(event) } - : {}), - redactionValues, - timeoutMs: 25 * 60_000, - }); - const installCompletedAtMs = Date.now(); - expect(install.exitCode, resultText(install)).toBe(0); + const preparation = USE_PREINSTALLED_LAUNCHABLE + ? await host.command("brev-quickstart", [SANDBOX_NAME], { + artifactName: "phase-1-brev-launchable-quickstart", + env: env({ + ...hosted.env, + NVIDIA_API_KEY: hosted.apiKey, + NEMOCLAW_AGENT: "openclaw", + NEMOCLAW_PROVIDER: "build", + }), + redactionValues, + timeoutMs: 25 * 60_000, + }) + : await host.command("bash", ["install.sh", "--non-interactive", "--fresh"], { + artifactName: "phase-1-install-sh", + cwd: REPO_ROOT, + env: env({ + ...hosted.env, + NVIDIA_INFERENCE_API_KEY: hosted.apiKey, + ...(coldOnboard ? { NEMOCLAW_TRACE_FILE: coldOnboard.traceFile } : {}), + }), + ...(coldOnboard + ? { onOutput: (event: ShellProbeOutputEvent) => coldOnboard.outputEvents.push(event) } + : {}), + redactionValues, + timeoutMs: 25 * 60_000, + }); + const preparationCompletedAtMs = Date.now(); + expect(preparation.exitCode, resultText(preparation)).toBe(0); await (coldOnboard ? assertColdOnboardPerformance({ apiKey: hosted.apiKey, artifacts, - budget: coldOnboardBudget, - install, - installCompletedAtMs, + budget: coldOnboardBudget!, + install: preparation, + installCompletedAtMs: preparationCompletedAtMs, outputEvents: coldOnboard.outputEvents, sandbox, traceDirectory: coldOnboard.traceDirectory, @@ -443,7 +471,7 @@ test("full e2e: install, onboard, inference, cli operations, and cleanup", { expect(registryText).not.toContain(SANDBOX_NAME); await artifacts.target.complete({ - id: "full-e2e", + id: EVIDENCE_TARGET_ID, securityPosture, status: "passed", }); diff --git a/test/e2e/mock-parity.json b/test/e2e/mock-parity.json index 67c93d5457..4e02191be1 100644 --- a/test/e2e/mock-parity.json +++ b/test/e2e/mock-parity.json @@ -2,14 +2,6 @@ "$comment": "SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\nSPDX-License-Identifier: Apache-2.0", "version": 1, "entries": [ - { - "live": "test/e2e/live/exact-staging-launchable.test.ts", - "fast": [ - "test/brev-launchable-runtime.test.ts", - "test/exact-image-qualification-controller.test.ts", - "test/e2e/support/exact-image-qualification-workflow.test.ts" - ] - }, { "live": "test/e2e/live/hermes-gpu-startup.test.ts", "fast": [ @@ -35,6 +27,8 @@ { "live": "test/e2e/live/full-e2e.test.ts", "fast": [ + "test/brev-launchable-runtime.test.ts", + "test/e2e/support/exact-image-qualification-workflow.test.ts", "test/e2e/support/onboard-performance.test.ts", "test/e2e/support/e2e-cleanup-resources.test.ts", "test/e2e/support/e2e-clients.test.ts" diff --git a/test/e2e/support/exact-image-qualification-workflow.test.ts b/test/e2e/support/exact-image-qualification-workflow.test.ts index 062f6780c6..20e1d9e3d1 100644 --- a/test/e2e/support/exact-image-qualification-workflow.test.ts +++ b/test/e2e/support/exact-image-qualification-workflow.test.ts @@ -14,6 +14,7 @@ const WORKFLOW_PATH = ".github/workflows/brev-launchable-qualification.yaml"; const E2E_WORKFLOW_PATH = ".github/workflows/e2e.yaml"; const CONTROLLER_PATH = "tools/e2e/exact-image-qualification-controller.mts"; const RUNTIME_PATH = "tools/e2e/brev-launchable-runtime.sh"; +const FULL_E2E_PATH = "test/e2e/live/full-e2e.test.ts"; const ACTIVATION_VARIABLE = "NEMOCLAW_BREV_LAUNCHABLE_QUALIFICATION_ENABLED"; type QualificationWorkflow = Workflow & { @@ -46,6 +47,7 @@ it("keeps exact-image Launchable qualification protected, reusable, and fail-clo const source = readRepoText(WORKFLOW_PATH); const controller = readRepoText(CONTROLLER_PATH); const runtime = readRepoText(RUNTIME_PATH); + const fullE2e = readRepoText(FULL_E2E_PATH); const preflight = job(workflow, "preflight"); const qualify = job(workflow, "qualify"); const caller = job(e2eWorkflow, "staging-brev-launchable"); @@ -164,10 +166,16 @@ it("keeps exact-image Launchable qualification protected, reusable, and fail-clo expect(source).toContain("--mode finalize"); expect(runtime).toContain("brev create"); expect(runtime).toContain("--launchable"); - expect(source).toContain("tools/e2e/live-vitest-invocation.mts run"); - expect(source).toContain("test/e2e/live/exact-staging-launchable.test.ts"); - expect(source).not.toContain("brev-launchable-runtime.sh qualify"); + expect(runtime).toContain("test/e2e/live/full-e2e.test.ts"); + expect(runtime).toContain("NEMOCLAW_E2E_SETUP_MODE=preinstalled-launchable"); + expect(runtime).not.toContain("brev-quickstart"); + expect(fullE2e).toContain('host.command("brev-quickstart"'); + expect(fullE2e).toContain('"brev-launchable-cloud-openclaw"'); + expect(source).not.toContain("test/e2e/live/exact-staging-launchable.test.ts"); + expect(source).toContain("brev-launchable-runtime.sh deploy"); + expect(source).toContain("brev-launchable-runtime.sh qualify"); expect(source).toContain("brev-launchable-runtime.sh cleanup"); + expect(source).toContain("brev-launchable-cloud-openclaw/"); expect(source).toContain("brev-cleanup-evidence.json"); expect(source).toContain("NEMOCLAW_STAGING_LAUNCHABLE_ID"); expect(source).not.toContain("image_family"); diff --git a/tools/e2e/brev-launchable-runtime.sh b/tools/e2e/brev-launchable-runtime.sh index eee5ae6358..3cc0d6f844 100755 --- a/tools/e2e/brev-launchable-runtime.sh +++ b/tools/e2e/brev-launchable-runtime.sh @@ -162,43 +162,64 @@ verify_identity() { >"$WORK_DIR/brev-identity-evidence.json" } -run_smoke() { +run_existing_e2e() { require_env NVIDIA_INFERENCE_API_KEY local sandbox="${NEMOCLAW_STAGING_SANDBOX_NAME:-e2e-staging}" [[ "$sandbox" =~ ^[a-z][a-z0-9-]{0,62}$ ]] || die "invalid staging sandbox name" - local quoted_key quoted_sandbox + local remote_artifact_dir="/tmp/nemoclaw-launchable-e2e-$INSTANCE_NAME" + local local_artifact_dir="$WORK_DIR/brev-launchable-cloud-openclaw" + local quoted_artifact_dir quoted_key quoted_sandbox + printf -v quoted_artifact_dir '%q' "$remote_artifact_dir" printf -v quoted_key '%q' "$NVIDIA_INFERENCE_API_KEY" printf -v quoted_sandbox '%q' "$sandbox" - host_exec 'set -e; command -v nemoclaw; command -v openshell; command -v docker; command -v brev-quickstart; docker info >/dev/null; openshell --version; nemoclaw --help >/dev/null' \ - >"$WORK_DIR/brev-prerequisites.log" 2>&1 - host_exec "set -euo pipefail; export NVIDIA_API_KEY=$quoted_key NEMOCLAW_PROVIDER=build NEMOCLAW_AGENT=openclaw; timeout 1500 brev-quickstart $quoted_sandbox" \ - >"$WORK_DIR/brev-quickstart.log" 2>&1 - host_exec "set -euo pipefail + repo=\$HOME/NemoClaw + cd \"\$repo\" + test -x ./node_modules/.bin/vitest + grep -q 'NEMOCLAW_E2E_SETUP_MODE' test/e2e/live/full-e2e.test.ts \ + || { printf 'candidate full-e2e.test.ts does not support preinstalled Launchable setup\n' >&2; exit 1; } + grep -q 'brev-launchable-cloud-openclaw' test/e2e/live/full-e2e.test.ts \ + || { printf 'candidate full-e2e.test.ts does not declare the Brev Launchable target\n' >&2; exit 1; } + rm -rf -- $quoted_artifact_dir + install -d -m 700 $quoted_artifact_dir model=\$(node /usr/local/lib/nemoclaw/launchable-config.mjs /usr/local/share/nemoclaw/launchable-agents.json openclaw cloudModel) - payload=\$(jq -cn --arg model \"\$model\" '{model:\$model,messages:[{role:\"user\",content:\"Reply with exactly one word: PONG\"}],max_tokens:100}') - response=\$(openshell sandbox exec --name $quoted_sandbox -- curl -fsS --max-time 90 \ - https://inference.local/v1/chat/completions -H 'Content-Type: application/json' -d \"\$payload\") - printf '%s' \"\$response\" | jq -er '[.choices[0].message.content,.choices[0].message.reasoning_content,.choices[0].message.reasoning] | map(select(type == \"string\")) | join(\" \") | test(\"PONG\"; \"i\")' >/dev/null - printf '%s\n' \"\$response\"" >"$WORK_DIR/brev-inference-pong.json" 2>&1 + export CI=true GITHUB_ACTIONS=true + export E2E_ARTIFACT_DIR=$quoted_artifact_dir + export E2E_TARGET_ID=brev-launchable-cloud-openclaw + export NEMOCLAW_CLI_BIN=nemoclaw + export NEMOCLAW_E2E_SETUP_MODE=preinstalled-launchable + export NEMOCLAW_MODEL=\"\$model\" + export NEMOCLAW_RUN_LIVE_E2E=1 + export NEMOCLAW_SANDBOX_NAME=$quoted_sandbox + export NVIDIA_INFERENCE_API_KEY=$quoted_key + ./node_modules/.bin/vitest run --project e2e-live test/e2e/live/full-e2e.test.ts --silent=false --reporter=default" \ + >"$WORK_DIR/brev-launchable-e2e.log" 2>&1 - host_exec "set -euo pipefail - response=\$(openshell sandbox exec --name $quoted_sandbox -- openclaw agent --agent main --json --thinking off \ - --session-id qualification-$(date +%s) -m 'What is 6 multiplied by 7? Reply with only the integer, no extra words.') - printf '%s\n' \"\$response\" - printf '%s' \"\$response\" | grep -Eq '(^|[^0-9])42([^0-9]|$)'" \ - >"$WORK_DIR/brev-agent-response.log" 2>&1 - - jq -n --arg sandbox "$sandbox" --arg completedAt "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ - '{schemaVersion:1,sandbox:$sandbox,checks:["baked-tools","onboarding","sandbox-ready","inference-local-pong","openclaw-agent-response"],completedAt:$completedAt}' \ - >"$WORK_DIR/brev-smoke-evidence.json" + mkdir -m 700 "$local_artifact_dir" + timeout "${BREV_COPY_TIMEOUT_SECONDS:-300}" \ + brev copy "$INSTANCE_NAME:$remote_artifact_dir/" "$local_artifact_dir/" --host + + local target_result + target_result="$(find "$local_artifact_dir" -type f -name target-result.json -print -quit)" + [ -n "$target_result" ] || die "the existing full E2E suite did not produce target-result.json" + jq -e '.id == "brev-launchable-cloud-openclaw" and .status == "passed" and .runner == "vitest"' \ + "$target_result" >/dev/null \ + || die "the existing full E2E suite did not produce passing Launchable evidence" + + jq -n \ + --arg sandbox "$sandbox" \ + --arg target "brev-launchable-cloud-openclaw" \ + --arg testFile "test/e2e/live/full-e2e.test.ts" \ + --arg completedAt "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + '{schemaVersion:1,target:$target,testFile:$testFile,setupMode:"preinstalled-launchable",sandbox:$sandbox,completedAt:$completedAt}' \ + >"$WORK_DIR/brev-launchable-e2e-evidence.json" } qualify() { validate_common verify_identity - run_smoke + run_existing_e2e } cleanup() { From c589eef1e41170c154bf818e25fe56cd034c62c7 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Mon, 20 Jul 2026 15:58:14 -0400 Subject: [PATCH 19/25] test(e2e): satisfy test conditional guardrail Signed-off-by: Julie Yaunches --- test/e2e/live/full-e2e.test.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/e2e/live/full-e2e.test.ts b/test/e2e/live/full-e2e.test.ts index 3d3dbaa8b4..4f1c09d3cd 100644 --- a/test/e2e/live/full-e2e.test.ts +++ b/test/e2e/live/full-e2e.test.ts @@ -55,9 +55,10 @@ interface ColdOnboardCapture { traceFile: string; } -if (!new Set(["source-install", "preinstalled-launchable"]).has(SETUP_MODE)) { - throw new Error(`unsupported NEMOCLAW_E2E_SETUP_MODE: ${SETUP_MODE}`); -} +expect( + ["source-install", "preinstalled-launchable"], + `unsupported NEMOCLAW_E2E_SETUP_MODE: ${SETUP_MODE}`, +).toContain(SETUP_MODE); process.env.NEMOCLAW_CLI_BIN ??= USE_PREINSTALLED_LAUNCHABLE ? "nemoclaw" : CLI_ENTRYPOINT; validateSandboxName(SANDBOX_NAME); From 68d004f8ad66337fb1e1919b92a9536609a643b4 Mon Sep 17 00:00:00 2001 From: Charan Jagwani Date: Mon, 20 Jul 2026 13:23:58 -0700 Subject: [PATCH 20/25] fix(e2e): secure Launchable qualification evidence Signed-off-by: Charan Jagwani --- .../brev-launchable-qualification.yaml | 54 ++++++- .github/workflows/e2e.yaml | 4 +- test/brev-launchable-runtime.test.ts | 135 ++++++++++++++++- test/e2e/README.md | 29 ++-- test/e2e/live/full-e2e.test.ts | 88 ++++++----- ...exact-image-qualification-workflow.test.ts | 78 +++++++++- ...act-image-qualification-controller.test.ts | 20 ++- ...-image-qualification-controller-fixture.ts | 2 +- tools/e2e/brev-launchable-runtime.sh | 140 +++++++++++++++++- .../exact-image-qualification-controller.mts | 8 +- 10 files changed, 482 insertions(+), 76 deletions(-) diff --git a/.github/workflows/brev-launchable-qualification.yaml b/.github/workflows/brev-launchable-qualification.yaml index d907d5d7c7..cbebea4e39 100644 --- a/.github/workflows/brev-launchable-qualification.yaml +++ b/.github/workflows/brev-launchable-qualification.yaml @@ -248,7 +248,8 @@ jobs: WORK_DIR: ${{ steps.workspace.outputs.work_dir }} run: tools/e2e/brev-launchable-runtime.sh qualify - - name: Redact runtime evidence + - id: redact-runtime-evidence + name: Redact runtime evidence if: ${{ always() && steps.workspace.outputs.work_dir != '' }} env: NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }} @@ -257,15 +258,52 @@ jobs: set -euo pipefail python3 - <<'PY' import os + import stat from pathlib import Path - secret = os.environ.get("NVIDIA_INFERENCE_API_KEY", "") + MAX_ENTRIES = 2_500 + MAX_FILES = 1_250 + MAX_TOTAL_BYTES = 128 * 1024 * 1024 + + secret = os.environ.get("NVIDIA_INFERENCE_API_KEY", "").encode() root = Path(os.environ["WORK_DIR"]) - if secret: - for name in ("brev-launchable-e2e.log",): - path = root / name - if path.is_file(): - path.write_text(path.read_text(encoding="utf-8", errors="replace").replace(secret, "[REDACTED]"), encoding="utf-8") + if root.is_symlink() or not root.is_dir(): + raise SystemExit("runtime evidence root must be a real directory") + + entries = 0 + total_bytes = 0 + paths = [] + stack = [root] + while stack: + directory = stack.pop() + with os.scandir(directory) as children: + for child in children: + entries += 1 + if entries > MAX_ENTRIES: + raise SystemExit(f"runtime evidence exceeds {MAX_ENTRIES} entries") + path = Path(child.path) + if child.is_symlink(): + raise SystemExit(f"runtime evidence must not contain symlinks: {path}") + if child.is_dir(follow_symlinks=False): + stack.append(path) + continue + if not child.is_file(follow_symlinks=False): + raise SystemExit(f"runtime evidence contains a non-regular file: {path}") + metadata = child.stat(follow_symlinks=False) + if not stat.S_ISREG(metadata.st_mode): + raise SystemExit(f"runtime evidence contains a non-regular file: {path}") + paths.append(path) + total_bytes += metadata.st_size + if len(paths) > MAX_FILES: + raise SystemExit(f"runtime evidence exceeds {MAX_FILES} files") + if total_bytes > MAX_TOTAL_BYTES: + raise SystemExit(f"runtime evidence exceeds {MAX_TOTAL_BYTES} bytes") + + for path in paths: + if secret: + content = path.read_bytes() + if secret in content: + path.write_bytes(content.replace(secret, b"[REDACTED]")) PY - name: Delete staging workspace and verify absence @@ -287,7 +325,7 @@ jobs: --work-dir "${{ steps.workspace.outputs.work_dir }}" - name: Upload exact staging Launchable evidence - if: ${{ always() && steps.workspace.outputs.work_dir != '' }} + if: ${{ always() && steps.workspace.outputs.work_dir != '' && steps.redact-runtime-evidence.outcome == 'success' }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: brev-launchable-qualification-${{ inputs.candidate_sha }}-${{ github.run_id }} diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index ccdccc71ed..2fbfec406d 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -190,10 +190,10 @@ jobs: needs: generate-matrix # Repository owners enable this only after the protected environment, # producer, standing Launchable, and Brev contract are ready. - if: ${{ vars.NEMOCLAW_BREV_LAUNCHABLE_QUALIFICATION_ENABLED == 'true' && github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') }} + if: ${{ vars.NEMOCLAW_BREV_LAUNCHABLE_QUALIFICATION_ENABLED == 'true' && github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && inputs.checkout_sha == '' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') }} uses: ./.github/workflows/brev-launchable-qualification.yaml with: - candidate_sha: ${{ inputs.checkout_sha || github.sha }} + candidate_sha: ${{ github.sha }} reason: E2E run ${{ github.run_id }} qualifying the exact checked-out NemoClaw candidate live: diff --git a/test/brev-launchable-runtime.test.ts b/test/brev-launchable-runtime.test.ts index fedab567c5..7e47a4a3ee 100644 --- a/test/brev-launchable-runtime.test.ts +++ b/test/brev-launchable-runtime.test.ts @@ -16,10 +16,21 @@ type FixtureOptions = { bootImageId?: string; bootImageSelfLink?: string; deleteMode?: "remove" | "retain"; + evidenceMode?: + | "duplicate" + | "failed-turn" + | "fifo" + | "malformed" + | "oversized" + | "symlink" + | "valid" + | "wrong-path"; lsMode?: "ok" | "fail" | "fail-once" | "malformed"; + model?: string; provisionSha?: string; repoSha?: string; supportsLaunchableMode?: boolean; + trackedDrift?: "index" | "none" | "worktree"; workspaceMode?: "ready" | "pending"; }; @@ -40,6 +51,7 @@ function fixture(options: FixtureOptions = {}) { const state = path.join(root, "state.json"); const lsCount = path.join(root, "ls-count"); const log = path.join(root, "brev.log"); + const e2eExecuted = path.join(root, "e2e-executed"); const manifest = path.join(workDir, "validated-manifest.v1.json"); fs.mkdirSync(bin); fs.mkdirSync(workDir); @@ -57,8 +69,52 @@ function fixture(options: FixtureOptions = {}) { `#!/usr/bin/env bash set -euo pipefail artifact_dir="$E2E_ARTIFACT_DIR/brev-launchable-cloud-openclaw-onboard-inference-cli-operations-and-cleanup" -mkdir -p "$artifact_dir" -jq -n '{id:"brev-launchable-cloud-openclaw",status:"passed",runner:"vitest"}' > "$artifact_dir/target-result.json" +: > "$FAKE_E2E_EXECUTED" +write_result() { + mkdir -p "$(dirname "$1")" + jq -n '{ + firstAgentTurn:{commandMs:125,responseChars:24,status:"passed"}, + id:"brev-launchable-cloud-openclaw", + runner:"vitest", + securityPosture:{ + configureGuard:true, + entrypoint:{capBnd:"0",capEff:"0",dangerousBoundingCapabilities:[],dangerousEffectiveCapabilities:[],noNewPrivs:"1",uid:"1000"}, + hostNonRoot:true, + rcFilesLocked:true, + runtimeProxyEnvLocked:true, + startupLogClean:true + }, + status:"passed" + }' > "$1" +} +case "$FAKE_EVIDENCE_MODE" in + valid) write_result "$artifact_dir/target-result.json" ;; + duplicate) + write_result "$artifact_dir/target-result.json" + write_result "$E2E_ARTIFACT_DIR/duplicate/target-result.json" + ;; + malformed) + mkdir -p "$artifact_dir" + jq -n '{id:"brev-launchable-cloud-openclaw",runner:"vitest",status:"passed"}' > "$artifact_dir/target-result.json" + ;; + failed-turn) + write_result "$artifact_dir/target-result.json" + jq '.firstAgentTurn.status = "failed"' "$artifact_dir/target-result.json" > "$artifact_dir/result.tmp" + mv "$artifact_dir/result.tmp" "$artifact_dir/target-result.json" + ;; + fifo) write_result "$artifact_dir/target-result.json" ;; + oversized) + write_result "$artifact_dir/target-result.json" + dd if=/dev/null of="$artifact_dir/oversized.bin" bs=1 seek=104857601 2>/dev/null + ;; + symlink) + mkdir -p "$artifact_dir" + write_result "$artifact_dir/real-result.json" + ln -s real-result.json "$artifact_dir/target-result.json" + ;; + wrong-path) write_result "$E2E_ARTIFACT_DIR/wrong/target-result.json" ;; + *) exit 2 ;; +esac `, ); writeExecutable(path.join(bin, "timeout"), '#!/usr/bin/env bash\nshift\nexec "$@"\n'); @@ -92,6 +148,7 @@ case "$1" in source_path="\${2#*:}" destination="$3" cp -R "$source_path"/. "$destination"/ + [ "$FAKE_EVIDENCE_MODE" != fifo ] || mkfifo "$destination/untrusted.fifo" ;; delete) [ "$FAKE_BREV_DELETE_MODE" = retain ] || rm -f "$FAKE_BREV_STATE" ;; refresh) ;; @@ -115,8 +172,12 @@ exec "$@" path.join(bin, "git"), `#!/usr/bin/env bash set -euo pipefail -if [[ "$*" == *'rev-parse HEAD'* ]]; then printf '%s\\n' "$FAKE_REPO_SHA"; exit 0; fi -exit 2 +case "$*" in + *'diff --cached --quiet --no-ext-diff HEAD --'*) [ "$FAKE_TRACKED_DRIFT" != index ] ;; + *'diff --quiet --no-ext-diff HEAD --'*) [ "$FAKE_TRACKED_DRIFT" != worktree ] ;; + *'rev-parse HEAD'*) printf '%s\\n' "$FAKE_REPO_SHA" ;; + *) exit 2 ;; +esac `, ); writeExecutable( @@ -153,7 +214,7 @@ exec "$@" ["brev-quickstart", "printf 'Ready!\\n'"], ["docker", "exit 0"], ["nemoclaw", "exit 0"], - ["node", "printf 'nvidia/test-model\\n'"], + ["node", "printf '%s\\n' \"$FAKE_MODEL\""], ["openclaw", 'printf \'{"payloads":[{"text":"42"}]}\\n\''], ] as const) { writeExecutable(path.join(bin, name), `#!/usr/bin/env bash\nset -euo pipefail\n${body}\n`); @@ -181,8 +242,12 @@ exec "$@" FAKE_BREV_LOG: log, FAKE_BREV_LS_COUNT: lsCount, FAKE_BREV_STATE: state, + FAKE_E2E_EXECUTED: e2eExecuted, + FAKE_EVIDENCE_MODE: options.evidenceMode ?? "valid", + FAKE_MODEL: options.model ?? "nvidia/test-model", FAKE_PROVISION_SHA: options.provisionSha ?? candidateSha.slice(0, 7), FAKE_REPO_SHA: options.repoSha ?? candidateSha, + FAKE_TRACKED_DRIFT: options.trackedDrift ?? "none", FAKE_BREV_WORKSPACE_MODE: options.workspaceMode ?? "ready", HOME: home, INSTANCE_NAME: "nclaw-e2e-test-1", @@ -191,7 +256,7 @@ exec "$@" WORK_DIR: workDir, BREV_POLL_SECONDS: "0", }; - return { env, log, workDir }; + return { e2eExecuted, env, log, workDir }; } function run(mode: string, env: NodeJS.ProcessEnv) { @@ -216,12 +281,25 @@ describe("exact staging Brev Launchable runtime", () => { expect(commands).not.toMatch(/rsync|install\.sh|npm (?:ci|install)|git clone/u); expect(commands).toContain("test/e2e/live/full-e2e.test.ts"); expect(commands).toContain("NEMOCLAW_E2E_SETUP_MODE=preinstalled-launchable"); + expect(commands).toContain("NEMOCLAW_E2E_SECURITY_POSTURE=1"); expect(commands).toContain("E2E_TARGET_ID=brev-launchable-cloud-openclaw"); expect(commands.indexOf("rev-parse HEAD")).toBeLessThan( commands.indexOf("test/e2e/live/full-e2e.test.ts"), ); expect(fs.existsSync(path.join(workDir, "brev-identity-evidence.json"))).toBe(true); expect(fs.existsSync(path.join(workDir, "brev-launchable-e2e-evidence.json"))).toBe(true); + expect( + JSON.parse(fs.readFileSync(path.join(workDir, "brev-launchable-e2e-evidence.json"), "utf8")), + ).toMatchObject({ + artifacts: { + fileCount: expect.any(Number), + targetResultPath: + "brev-launchable-cloud-openclaw-onboard-inference-cli-operations-and-cleanup/target-result.json", + targetResultSha256: expect.stringMatching(/^[0-9a-f]{64}$/u), + totalBytes: expect.any(Number), + }, + setupMode: "preinstalled-launchable", + }); expect( fs.existsSync( path.join( @@ -249,6 +327,20 @@ describe("exact staging Brev Launchable runtime", () => { expect(run("cleanup", env).status).toBe(0); }); + it.each([ + "index", + "worktree", + ] as const)("fails closed on tracked %s drift before executing the E2E harness", (trackedDrift) => { + const { e2eExecuted, env } = fixture({ trackedDrift }); + const qualification = run("qualify", env); + + expect(qualification.status).not.toBe(0); + expect([qualification.stderr, qualification.stdout].join("\n")).toContain( + trackedDrift === "index" ? "staged changes" : "tracked worktree changes", + ); + expect(fs.existsSync(e2eExecuted)).toBe(false); + }); + it("fails closed on a boot-image mismatch before onboarding", () => { const { env, log } = fixture({ bootImageId: "987654321" }); expect(run("deploy", env).status).toBe(0); @@ -271,6 +363,37 @@ describe("exact staging Brev Launchable runtime", () => { expect(run("cleanup", env).status).toBe(0); }); + it("rejects an unsafe image-derived model before executing the E2E harness", () => { + const { e2eExecuted, env, workDir } = fixture(); + const injectionMarker = `${e2eExecuted}-model-injection`; + env.FAKE_MODEL = `nvidia/model'; touch ${injectionMarker}; #`; + const qualification = run("qualify", env); + + expect(qualification.status).not.toBe(0); + expect(fs.readFileSync(path.join(workDir, "brev-launchable-e2e.log"), "utf8")).toContain( + "Launchable cloud model is not a safe model ID", + ); + expect(fs.existsSync(e2eExecuted)).toBe(false); + expect(fs.existsSync(injectionMarker)).toBe(false); + }); + + it.each([ + "duplicate", + "failed-turn", + "fifo", + "malformed", + "oversized", + "symlink", + "wrong-path", + ] as const)("rejects %s copied E2E evidence", (evidenceMode) => { + const { e2eExecuted, env, workDir } = fixture({ evidenceMode }); + const qualification = run("qualify", env); + + expect(qualification.status).not.toBe(0); + expect(fs.existsSync(e2eExecuted)).toBe(true); + expect(fs.existsSync(path.join(workDir, "brev-launchable-e2e-evidence.json"))).toBe(false); + }); + it("fails when workspace readiness does not reach its deadline", () => { const { env } = fixture({ workspaceMode: "pending" }); const deploy = run("deploy", { diff --git a/test/e2e/README.md b/test/e2e/README.md index 32b80ebeb6..b05af6c6b4 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -87,12 +87,15 @@ map to this consolidated `e2e-artifacts/live/` registry-target artifact layout. ## Exact staging Brev Launchable qualification -The exact-image staging Launchable lane is inactive by default. The scheduled -and manual `e2e.yaml` paths call it only from `main` when the repository -variable `NEMOCLAW_BREV_LAUNCHABLE_QUALIFICATION_ENABLED` is exactly `true`. -A direct dispatch of `brev-launchable-qualification.yaml` applies the same -repository, ref, and activation checks; while inactive, it fails before -checkout, environment approval, credential access, or external API calls. +The exact-image staging Launchable lane is inactive by default. Only scheduled +and manual `e2e.yaml` runs on `main` with an empty `checkout_sha` may call it, +and only when the repository variable +`NEMOCLAW_BREV_LAUNCHABLE_QUALIFICATION_ENABLED` is exactly `true`. PR-gate +runs cannot enter this credentialed lane. A direct dispatch of +`brev-launchable-qualification.yaml` applies the same repository, ref, and +activation checks and requires `candidate_sha` to equal the trusted current +`main` workflow SHA. While inactive, it fails before checkout, environment +approval, credential access, or external API calls. The skipped `staging-brev-launchable` job in a routine E2E run is an inactive status, not successful qualification evidence. @@ -116,10 +119,16 @@ The runtime target is `brev-launchable-cloud-openclaw`. After proving the workspace booted the accepted image and exact NemoClaw SHA, the lane runs the existing `test/e2e/live/full-e2e.test.ts` suite in `preinstalled-launchable` setup mode. That mode onboards through the baked `brev-quickstart` entry point -and reuses the suite's CLI, policy, hosted-inference, `inference.local`, logs, -and cleanup assertions. It fails if the baked checkout does not already -contain the pinned Vitest harness; it does not rsync source, run `install.sh`, -install packages, rebuild, or relink the product under test. +and reuses the suite's CLI, policy, real first-agent-turn, hosted-inference, +`inference.local`, security-posture, logs, and cleanup assertions. Before the +trusted harness runs, the runtime rejects tracked checkout drift and unsafe +image-derived model IDs. Copied evidence must contain exactly the canonical +passing target result, stay within fixed file and byte limits, contain no +symlinks or special files, and produce a recorded SHA-256 digest. Recursive +redaction must succeed before evidence can be uploaded. The lane fails if the +baked checkout does not already contain the pinned Vitest harness; it does not +rsync source, run `install.sh`, install packages, rebuild, or relink the +product under test. ## PR E2E gate diff --git a/test/e2e/live/full-e2e.test.ts b/test/e2e/live/full-e2e.test.ts index 4f1c09d3cd..202f394fbc 100644 --- a/test/e2e/live/full-e2e.test.ts +++ b/test/e2e/live/full-e2e.test.ts @@ -46,6 +46,7 @@ const LIVE_TIMEOUT_MS = 50 * 60_000; const FIRST_TURN_TIMEOUT_MS = 240_000; const MAX_SILENCE_SECS = 60; const EXPECTED_FIRST_REPLY = "NEMOCLAW_E2E_READY_6002"; +const SAFE_MODEL_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:/-]{0,255}$/u; const MEASURE_COLD_ONBOARD = !USE_PREINSTALLED_LAUNCHABLE && process.env.E2E_TARGET_ID === "full-e2e"; @@ -55,6 +56,12 @@ interface ColdOnboardCapture { traceFile: string; } +interface FirstAgentTurnResult { + completedAtMs: number; + commandMs: number; + responseChars: number; +} + expect( ["source-install", "preinstalled-launchable"], `unsupported NEMOCLAW_E2E_SETUP_MODE: ${SETUP_MODE}`, @@ -170,6 +177,39 @@ function readFullE2eColdPathBudget() { } } +async function assertFirstAgentTurn(input: { + apiKey: string; + sandbox: SandboxClient; +}): Promise { + const startedAtMs = Date.now(); + const turn = await input.sandbox.execShell( + SANDBOX_NAME, + trustedSandboxShellScript( + "openclaw agent --agent main --json --thinking off --session-id e2e-6002 " + + `-m 'Reply with exactly: ${EXPECTED_FIRST_REPLY}'`, + ), + { + artifactName: "phase-1-first-agent-turn", + env: env(), + redactionValues: [input.apiKey], + timeoutMs: FIRST_TURN_TIMEOUT_MS, + }, + ); + const completedAtMs = Date.now(); + const turnText = resultText(turn); + const assistantReply = extractOpenClawAgentPayloadText(turnText).trim(); + + expect(turn.exitCode, turnText).toBe(0); + expect(assistantReply, `expected the sentinel first agent reply, got: ${turnText}`).toBe( + EXPECTED_FIRST_REPLY, + ); + return { + completedAtMs, + commandMs: completedAtMs - startedAtMs, + responseChars: assistantReply.length, + }; +} + async function assertColdOnboardPerformance(input: { apiKey: string; artifacts: ArtifactSink; @@ -180,7 +220,7 @@ async function assertColdOnboardPerformance(input: { sandbox: SandboxClient; traceDirectory: string; traceFile: string; -}): Promise { +}): Promise { const traceWindow = readAndDeleteTraceWindow(input.traceFile, input.traceDirectory); expect( input.installCompletedAtMs, @@ -199,45 +239,25 @@ async function assertColdOnboardPerformance(input: { const maxSilenceSecs = Math.ceil(maxSilenceMs / 1_000); const rootEndToInstallCompletionMs = input.installCompletedAtMs - traceWindow.finishedAtMs; - const firstTurnStartedAtMs = Date.now(); - const turn = await input.sandbox.execShell( - SANDBOX_NAME, - trustedSandboxShellScript( - "openclaw agent --agent main --json --thinking off --session-id e2e-6002 " + - `-m 'Reply with exactly: ${EXPECTED_FIRST_REPLY}'`, - ), - { - artifactName: "phase-1-first-agent-turn", - env: env(), - redactionValues: [input.apiKey], - timeoutMs: FIRST_TURN_TIMEOUT_MS, - }, - ); - const firstTurnCompletedAtMs = Date.now(); - const firstTurnCommandMs = firstTurnCompletedAtMs - firstTurnStartedAtMs; + const firstTurn = await assertFirstAgentTurn(input); const performanceEvaluation = evaluateColdOnboardPerformance( traceWindow, - firstTurnCompletedAtMs, + firstTurn.completedAtMs, input.budget, ); const rootStartToFirstTurnCompletionSecs = Math.ceil( performanceEvaluation.rootStartToFirstTurnCompletionMs / 1_000, ); - const turnText = resultText(turn); - const assistantReply = extractOpenClawAgentPayloadText(turnText).trim(); - const compactAssistantReply = assistantReply.replace(/\s+/gu, ""); - const responseChars = assistantReply.length; - await input.artifacts.writeJson("onboard-progress-budget.json", { schemaVersion: "nemoclaw.full_e2e_cold_performance.v2", sandbox: SANDBOX_NAME, installExitCode: input.install.exitCode, - firstTurnExitCode: turn.exitCode, + firstTurnExitCode: 0, phaseMeasurements: { onboardRootMs: traceWindow.durationMs, rootStartToFirstTurnCompletionMs: performanceEvaluation.rootStartToFirstTurnCompletionMs, rootEndToInstallCompletionMs, - firstTurnCommandMs, + firstTurnCommandMs: firstTurn.commandMs, rootEndToFirstTurnCompletionMs: performanceEvaluation.rootEndToFirstTurnCompletionMs, tracePhasesMs: traceWindow.phaseDurationsMs, }, @@ -254,7 +274,7 @@ async function assertColdOnboardPerformance(input: { buildKitFallback, usedBuildKitPrebuild, classicBuildSteps, - responseChars, + responseChars: firstTurn.responseChars, }); expect(plain, "expected literal wizard step [1/8] in installer output").toContain("[1/8]"); @@ -265,21 +285,18 @@ async function assertColdOnboardPerformance(input: { maxSilenceSecs, `longest silent gap ${maxSilenceSecs}s exceeds the ${MAX_SILENCE_SECS}s guarantee`, ).toBeLessThanOrEqual(MAX_SILENCE_SECS); - expect(turn.exitCode, turnText).toBe(0); - expect( - compactAssistantReply, - `expected the sentinel first agent reply, got: ${turnText}`, - ).toContain(EXPECTED_FIRST_REPLY); expect( performanceEvaluation.passed, `onboard-root-start-to-first-turn-completion took ${rootStartToFirstTurnCompletionSecs}s; ${performanceEvaluation.violations.join("; ")}`, ).toBe(true); + return firstTurn; } test(TEST_NAME, { timeout: LIVE_TIMEOUT_MS, }, async ({ artifacts, cleanup: cleanupRegistry, host, sandbox, secrets, skip }) => { const hosted = requireHostedInferenceConfig(secrets); + expect(hosted.model, "hosted model must be a shell-safe model ID").toMatch(SAFE_MODEL_ID_PATTERN); const coldOnboardBudget = USE_PREINSTALLED_LAUNCHABLE ? null : readFullE2eColdPathBudget(); const redactionValues = [hosted.apiKey]; await artifacts.target.declare({ @@ -367,7 +384,7 @@ test(TEST_NAME, { }); const preparationCompletedAtMs = Date.now(); expect(preparation.exitCode, resultText(preparation)).toBe(0); - await (coldOnboard + const firstAgentTurn = await (coldOnboard ? assertColdOnboardPerformance({ apiKey: hosted.apiKey, artifacts, @@ -379,7 +396,7 @@ test(TEST_NAME, { traceDirectory: coldOnboard.traceDirectory, traceFile: coldOnboard.traceFile, }) - : Promise.resolve()); + : assertFirstAgentTurn({ apiKey: hosted.apiKey, sandbox })); const pathProbe = await host.command( "bash", @@ -472,6 +489,11 @@ test(TEST_NAME, { expect(registryText).not.toContain(SANDBOX_NAME); await artifacts.target.complete({ + firstAgentTurn: { + commandMs: firstAgentTurn.commandMs, + responseChars: firstAgentTurn.responseChars, + status: "passed", + }, id: EVIDENCE_TARGET_ID, securityPosture, status: "passed", diff --git a/test/e2e/support/exact-image-qualification-workflow.test.ts b/test/e2e/support/exact-image-qualification-workflow.test.ts index 20e1d9e3d1..4c691c99bf 100644 --- a/test/e2e/support/exact-image-qualification-workflow.test.ts +++ b/test/e2e/support/exact-image-qualification-workflow.test.ts @@ -1,6 +1,11 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + import { expect, it } from "vitest"; import { @@ -52,6 +57,33 @@ it("keeps exact-image Launchable qualification protected, reusable, and fail-clo const qualify = job(workflow, "qualify"); const caller = job(e2eWorkflow, "staging-brev-launchable"); const workflowStrings = strings(workflow); + const steps = qualify.steps ?? []; + const redactIndex = steps.findIndex((step) => step.name === "Redact runtime evidence"); + const uploadIndex = steps.findIndex( + (step) => step.name === "Upload exact staging Launchable evidence", + ); + + expect(redactIndex).toBeGreaterThanOrEqual(0); + expect(uploadIndex).toBeGreaterThan(redactIndex); + const redact = steps[redactIndex]!; + const upload = steps[uploadIndex]!; + expect(redact.if).toBe("${{ always() && steps.workspace.outputs.work_dir != '' }}"); + expect(redact.env).toEqual({ + NVIDIA_INFERENCE_API_KEY: "${{ secrets.NVIDIA_INFERENCE_API_KEY }}", + WORK_DIR: "${{ steps.workspace.outputs.work_dir }}", + }); + expect(redact.run).toContain('content.replace(secret, b"[REDACTED]")'); + expect(redact.run).toContain("os.scandir(directory)"); + expect(redact.run).toContain("child.is_symlink()"); + expect(redact.run).toContain("MAX_TOTAL_BYTES"); + expect(upload.if).toBe( + "${{ always() && steps.workspace.outputs.work_dir != '' && steps.redact-runtime-evidence.outcome == 'success' }}", + ); + for (const path of ["brev-launchable-e2e.log", "brev-launchable-cloud-openclaw"]) { + expect(String(upload.with?.path), `retained evidence must include ${path}`).toContain( + `/${path}`, + ); + } expect(workflow.name).toBe("E2E / Exact Staging Brev Launchable"); expect(Object.keys(workflow.on)).toEqual(["workflow_dispatch", "workflow_call"]); @@ -69,8 +101,9 @@ it("keeps exact-image Launchable qualification protected, reusable, and fail-clo expect(workflow.permissions).toEqual({}); expect(caller.secrets).toBeUndefined(); expect(caller.if).toBe( - `\${{ vars.${ACTIVATION_VARIABLE} == 'true' && github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') }}`, + `\${{ vars.${ACTIVATION_VARIABLE} == 'true' && github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && inputs.checkout_sha == '' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') }}`, ); + expect(caller.with?.candidate_sha).toBe("${{ github.sha }}"); expect(workflow.concurrency).toEqual({ group: "brev-launchable-qualification-staging-cpu", "cancel-in-progress": false, @@ -138,6 +171,7 @@ it("keeps exact-image Launchable qualification protected, reusable, and fail-clo ); expect(controller).toContain('export const PRODUCER_REF = "main"'); expect(controller).toContain('request.ref !== "refs/heads/main"'); + expect(controller).toContain("request.candidateSha !== request.workflowSha"); expect(controller).toContain('export const GITHUB_API_VERSION = "2026-03-10"'); expect(controller).toContain("return_run_details: true"); expect(controller).toContain("fs.renameSync(temporary, file)"); @@ -168,9 +202,19 @@ it("keeps exact-image Launchable qualification protected, reusable, and fail-clo expect(runtime).toContain("--launchable"); expect(runtime).toContain("test/e2e/live/full-e2e.test.ts"); expect(runtime).toContain("NEMOCLAW_E2E_SETUP_MODE=preinstalled-launchable"); + expect(runtime).toContain("NEMOCLAW_E2E_SECURITY_POSTURE=1"); + expect(runtime).toContain("diff --quiet --no-ext-diff HEAD --"); + expect(runtime).toContain("validate_copied_artifact_tree"); + expect(runtime).toContain("targetResultSha256"); + expect(runtime).toContain("firstAgentTurn"); expect(runtime).not.toContain("brev-quickstart"); expect(fullE2e).toContain('host.command("brev-quickstart"'); expect(fullE2e).toContain('"brev-launchable-cloud-openclaw"'); + expect(fullE2e).toContain("assertFirstAgentTurn({ apiKey: hosted.apiKey, sandbox })"); + expect(fullE2e).toMatch( + /expect\(assistantReply,[\s\S]{0,200}\)\.toBe\(\s*EXPECTED_FIRST_REPLY,\s*\);/u, + ); + expect(fullE2e).toContain("firstAgentTurn:"); expect(source).not.toContain("test/e2e/live/exact-staging-launchable.test.ts"); expect(source).toContain("brev-launchable-runtime.sh deploy"); expect(source).toContain("brev-launchable-runtime.sh qualify"); @@ -180,3 +224,35 @@ it("keeps exact-image Launchable qualification protected, reusable, and fail-clo expect(source).toContain("NEMOCLAW_STAGING_LAUNCHABLE_ID"); expect(source).not.toContain("image_family"); }); + +it("recursively redacts nested runtime evidence and rejects symlinks before upload", () => { + const workflow = readYaml(WORKFLOW_PATH); + const redact = job(workflow, "qualify").steps?.find( + (step) => step.id === "redact-runtime-evidence", + ); + const script = redact?.run as string; + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-evidence-redaction-")); + const secret = "nvapi-nested-secret"; + + try { + const nested = path.join(root, "brev-launchable-cloud-openclaw", "target", "raw.log"); + fs.mkdirSync(path.dirname(nested), { recursive: true }); + fs.writeFileSync(nested, `before ${secret} after\n`); + const redacted = spawnSync("bash", ["-c", script], { + encoding: "utf8", + env: { ...process.env, NVIDIA_INFERENCE_API_KEY: secret, WORK_DIR: root }, + }); + expect(redacted.status, redacted.stderr).toBe(0); + expect(fs.readFileSync(nested, "utf8")).toBe("before [REDACTED] after\n"); + + fs.symlinkSync(nested, path.join(root, "untrusted-link")); + const rejected = spawnSync("bash", ["-c", script], { + encoding: "utf8", + env: { ...process.env, NVIDIA_INFERENCE_API_KEY: secret, WORK_DIR: root }, + }); + expect(rejected.status).not.toBe(0); + expect(rejected.stderr).toContain("must not contain symlinks"); + } finally { + fs.rmSync(root, { force: true, recursive: true }); + } +}); diff --git a/test/exact-image-qualification-controller.test.ts b/test/exact-image-qualification-controller.test.ts index b3f0a498dc..d8a7bb01ad 100644 --- a/test/exact-image-qualification-controller.test.ts +++ b/test/exact-image-qualification-controller.test.ts @@ -66,7 +66,7 @@ afterEach(() => { }); describe("exact image qualification request", () => { - it("accepts trusted manual and scheduled candidates while rejecting malformed boundaries", () => { + it("accepts the current trusted main candidate while rejecting malformed boundaries", () => { expect(validateExactImageQualificationRequest(REQUEST)).toEqual(REQUEST); expect( validateExactImageQualificationRequest({ @@ -89,6 +89,9 @@ describe("exact image qualification request", () => { expect(() => validateExactImageQualificationRequest({ ...REQUEST, ref: "refs/heads/feature" }), ).toThrow(/trusted main branch/u); + expect(() => + validateExactImageQualificationRequest({ ...REQUEST, candidateSha: "d".repeat(40) }), + ).toThrow(/must equal the trusted main workflow SHA/u); expect(() => validateExactImageQualificationRequest({ ...REQUEST, reason: " bad " })).toThrow( /reason/u, ); @@ -105,6 +108,17 @@ describe("exact image qualification request", () => { expect(api).not.toHaveBeenCalled(); }); + it("rejects a candidate other than the trusted workflow SHA before authorization API access", async () => { + const api = vi.fn(); + + await expect( + preflightExactImageQualification({ ...REQUEST, candidateSha: "d".repeat(40) }, "core-token", { + api: api as QualificationDependencies["api"], + }), + ).rejects.toMatchObject({ code: "REQUEST_INVALID" }); + expect(api).not.toHaveBeenCalled(); + }); + it("parses the fixed CLI surface and rejects undeclared controls", () => { expect( parseExactImageQualificationCommand([ @@ -142,8 +156,8 @@ describe("exact image qualification request", () => { }); describe("exact producer dispatch binding", () => { - it("preserves a selected candidate distinct from the trusted workflow SHA", async () => { - expect(REQUEST.candidateSha).not.toBe(REQUEST.workflowSha); + it("binds the selected candidate to the trusted main workflow SHA", async () => { + expect(REQUEST.candidateSha).toBe(REQUEST.workflowSha); const { state, workDir } = await startedState(); try { expect(readExactImageQualificationState(workDir).request).toMatchObject({ diff --git a/test/helpers/exact-image-qualification-controller-fixture.ts b/test/helpers/exact-image-qualification-controller-fixture.ts index 3d56e528cb..1dba1cd812 100644 --- a/test/helpers/exact-image-qualification-controller-fixture.ts +++ b/test/helpers/exact-image-qualification-controller-fixture.ts @@ -20,8 +20,8 @@ import { startExactImageQualification, } from "../../tools/e2e/exact-image-qualification-controller.mts"; -export const CANDIDATE_SHA = "a".repeat(40); export const WORKFLOW_SHA = "c".repeat(40); +export const CANDIDATE_SHA = WORKFLOW_SHA; export const PRODUCER_SHA = "b".repeat(40); export const CORRELATION_ID = "123e4567-e89b-42d3-a456-426614174000"; export const RUN_ID = "24680"; diff --git a/tools/e2e/brev-launchable-runtime.sh b/tools/e2e/brev-launchable-runtime.sh index 3cc0d6f844..da29d54cb1 100755 --- a/tools/e2e/brev-launchable-runtime.sh +++ b/tools/e2e/brev-launchable-runtime.sh @@ -124,7 +124,14 @@ verify_identity() { # HOME and repo are intentionally expanded by the remote host shell. # shellcheck disable=SC2016 - repo_sha="$(host_exec 'set -e; repo="$HOME/NemoClaw"; test -d "$repo/.git"; git -C "$repo" rev-parse HEAD' | tail -n 1)" + repo_sha="$(host_exec 'set -e + repo="$HOME/NemoClaw" + test -d "$repo/.git" + git -C "$repo" diff --quiet --no-ext-diff HEAD -- \ + || { printf "baked NemoClaw checkout has tracked worktree changes\n" >&2; exit 1; } + git -C "$repo" diff --cached --quiet --no-ext-diff HEAD -- \ + || { printf "baked NemoClaw checkout has staged changes\n" >&2; exit 1; } + git -C "$repo" rev-parse HEAD' | tail -n 1)" [ "$repo_sha" = "$CANDIDATE_SHA" ] \ || die "baked NemoClaw SHA $repo_sha does not match candidate $CANDIDATE_SHA" [[ "$CANDIDATE_SHA" == "$provision_sha"* ]] \ @@ -162,17 +169,126 @@ verify_identity() { >"$WORK_DIR/brev-identity-evidence.json" } +validate_copied_artifact_tree() { + local root="$1" + local canonical_result="$2" + python3 - "$root" "$canonical_result" <<'PY' +import hashlib +import json +import os +import stat +import sys +from pathlib import Path + +MAX_ENTRIES = 2_000 +MAX_FILES = 1_000 +MAX_TOTAL_BYTES = 100 * 1024 * 1024 + +root = Path(sys.argv[1]) +expected_result = sys.argv[2] +if root.is_symlink() or not root.is_dir(): + raise SystemExit("copied E2E artifact root must be a real directory") + +entries = 0 +files = 0 +total_bytes = 0 +target_results = [] +stack = [root] +while stack: + directory = stack.pop() + with os.scandir(directory) as children: + for child in children: + entries += 1 + if entries > MAX_ENTRIES: + raise SystemExit(f"copied E2E artifact tree exceeds {MAX_ENTRIES} entries") + path = Path(child.path) + relative = path.relative_to(root).as_posix() + if child.is_symlink(): + raise SystemExit(f"copied E2E artifacts must not contain symlinks: {relative}") + if child.is_dir(follow_symlinks=False): + stack.append(path) + continue + if not child.is_file(follow_symlinks=False): + raise SystemExit(f"copied E2E artifacts must contain only directories and regular files: {relative}") + metadata = child.stat(follow_symlinks=False) + if not stat.S_ISREG(metadata.st_mode): + raise SystemExit(f"copied E2E artifact is not a regular file: {relative}") + files += 1 + total_bytes += metadata.st_size + if files > MAX_FILES: + raise SystemExit(f"copied E2E artifact tree exceeds {MAX_FILES} files") + if total_bytes > MAX_TOTAL_BYTES: + raise SystemExit(f"copied E2E artifact tree exceeds {MAX_TOTAL_BYTES} bytes") + if child.name == "target-result.json": + target_results.append(relative) + +target_results.sort() +if target_results != [expected_result]: + raise SystemExit( + "copied E2E artifacts must contain exactly the canonical target-result.json; " + f"found {target_results}" + ) + +target_result = root / expected_result +raw_result = target_result.read_bytes() + +def unique_object(pairs): + result = {} + for key, value in pairs: + if key in result: + raise ValueError(f"duplicate JSON key: {key}") + result[key] = value + return result + +try: + result = json.loads(raw_result, object_pairs_hook=unique_object) +except (UnicodeDecodeError, json.JSONDecodeError, ValueError) as error: + raise SystemExit(f"canonical target-result.json is invalid: {error}") from error + +expected_keys = {"firstAgentTurn", "id", "runner", "securityPosture", "status"} +if not isinstance(result, dict) or set(result) != expected_keys: + raise SystemExit("canonical target-result.json has an unexpected schema") +if result["id"] != "brev-launchable-cloud-openclaw" or result["runner"] != "vitest" or result["status"] != "passed": + raise SystemExit("canonical target-result.json does not report a passing Launchable target") +first_turn = result["firstAgentTurn"] +if not isinstance(first_turn, dict) or set(first_turn) != {"commandMs", "responseChars", "status"}: + raise SystemExit("canonical target-result.json has invalid first-agent-turn evidence") +if first_turn["status"] != "passed" or type(first_turn["commandMs"]) is not int or first_turn["commandMs"] < 0: + raise SystemExit("canonical target-result.json does not report a passing first agent turn") +if type(first_turn["responseChars"]) is not int or first_turn["responseChars"] <= 0: + raise SystemExit("canonical target-result.json has invalid first-agent-turn response evidence") +posture = result["securityPosture"] +if not isinstance(posture, dict) or not isinstance(posture.get("entrypoint"), dict): + raise SystemExit("canonical target-result.json has invalid security-posture evidence") +for field in ("configureGuard", "hostNonRoot", "rcFilesLocked", "runtimeProxyEnvLocked", "startupLogClean"): + if posture.get(field) is not True: + raise SystemExit(f"canonical target-result.json security posture did not pass {field}") + +digest = hashlib.sha256(raw_result).hexdigest() +print(json.dumps({ + "fileCount": files, + "totalBytes": total_bytes, + "targetResultPath": expected_result, + "targetResultSha256": digest, +}, separators=(",", ":"))) +PY +} + run_existing_e2e() { require_env NVIDIA_INFERENCE_API_KEY + require_tools python3 local sandbox="${NEMOCLAW_STAGING_SANDBOX_NAME:-e2e-staging}" [[ "$sandbox" =~ ^[a-z][a-z0-9-]{0,62}$ ]] || die "invalid staging sandbox name" local remote_artifact_dir="/tmp/nemoclaw-launchable-e2e-$INSTANCE_NAME" local local_artifact_dir="$WORK_DIR/brev-launchable-cloud-openclaw" + local canonical_result_relative="brev-launchable-cloud-openclaw-onboard-inference-cli-operations-and-cleanup/target-result.json" local quoted_artifact_dir quoted_key quoted_sandbox printf -v quoted_artifact_dir '%q' "$remote_artifact_dir" printf -v quoted_key '%q' "$NVIDIA_INFERENCE_API_KEY" printf -v quoted_sandbox '%q' "$sandbox" + # The escaped model expansions and case patterns are evaluated by the remote host shell. + # shellcheck disable=SC1083,SC2140 host_exec "set -euo pipefail repo=\$HOME/NemoClaw cd \"\$repo\" @@ -184,10 +300,20 @@ run_existing_e2e() { rm -rf -- $quoted_artifact_dir install -d -m 700 $quoted_artifact_dir model=\$(node /usr/local/lib/nemoclaw/launchable-config.mjs /usr/local/share/nemoclaw/launchable-agents.json openclaw cloudModel) + [ "\${#model}" -le 256 ] \ + || { printf 'Launchable cloud model is not a safe model ID\n' >&2; exit 1; } + case "\$model" in + [A-Za-z0-9]*) ;; + *) printf 'Launchable cloud model is not a safe model ID\n' >&2; exit 1 ;; + esac + case "\$model" in + *[!A-Za-z0-9._:/-]*) printf 'Launchable cloud model is not a safe model ID\n' >&2; exit 1 ;; + esac export CI=true GITHUB_ACTIONS=true export E2E_ARTIFACT_DIR=$quoted_artifact_dir export E2E_TARGET_ID=brev-launchable-cloud-openclaw export NEMOCLAW_CLI_BIN=nemoclaw + export NEMOCLAW_E2E_SECURITY_POSTURE=1 export NEMOCLAW_E2E_SETUP_MODE=preinstalled-launchable export NEMOCLAW_MODEL=\"\$model\" export NEMOCLAW_RUN_LIVE_E2E=1 @@ -200,19 +326,17 @@ run_existing_e2e() { timeout "${BREV_COPY_TIMEOUT_SECONDS:-300}" \ brev copy "$INSTANCE_NAME:$remote_artifact_dir/" "$local_artifact_dir/" --host - local target_result - target_result="$(find "$local_artifact_dir" -type f -name target-result.json -print -quit)" - [ -n "$target_result" ] || die "the existing full E2E suite did not produce target-result.json" - jq -e '.id == "brev-launchable-cloud-openclaw" and .status == "passed" and .runner == "vitest"' \ - "$target_result" >/dev/null \ - || die "the existing full E2E suite did not produce passing Launchable evidence" + local artifact_summary + artifact_summary="$(validate_copied_artifact_tree "$local_artifact_dir" "$canonical_result_relative")" \ + || die "copied full E2E artifacts failed validation" jq -n \ + --argjson artifacts "$artifact_summary" \ --arg sandbox "$sandbox" \ --arg target "brev-launchable-cloud-openclaw" \ --arg testFile "test/e2e/live/full-e2e.test.ts" \ --arg completedAt "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ - '{schemaVersion:1,target:$target,testFile:$testFile,setupMode:"preinstalled-launchable",sandbox:$sandbox,completedAt:$completedAt}' \ + '{schemaVersion:1,target:$target,testFile:$testFile,setupMode:"preinstalled-launchable",sandbox:$sandbox,artifacts:$artifacts,completedAt:$completedAt}' \ >"$WORK_DIR/brev-launchable-e2e-evidence.json" } diff --git a/tools/e2e/exact-image-qualification-controller.mts b/tools/e2e/exact-image-qualification-controller.mts index 98794605c1..aeb4cdf62e 100755 --- a/tools/e2e/exact-image-qualification-controller.mts +++ b/tools/e2e/exact-image-qualification-controller.mts @@ -419,6 +419,9 @@ export function validateExactImageQualificationRequest( if (!FULL_SHA_PATTERN.test(request.workflowSha)) { fail("REQUEST_INVALID", "workflow SHA must be a lowercase full commit SHA"); } + if (request.candidateSha !== request.workflowSha) { + fail("REQUEST_INVALID", "candidate SHA must equal the trusted main workflow SHA"); + } if (!DECIMAL_ID_PATTERN.test(request.requesterRunId)) { fail("REQUEST_INVALID", "requester run ID must be a positive decimal string"); } @@ -465,10 +468,7 @@ async function authorizeRequest( api: GitHubApiClient, ): Promise { validateExactImageQualificationRequest(request); - const branch = await api( - `repos/${REQUESTER_REPOSITORY}/git/ref/heads/main`, - token, - ); + const branch = await api(`repos/${REQUESTER_REPOSITORY}/git/ref/heads/main`, token); if (validateMainRef(branch, REQUESTER_REPOSITORY) !== request.workflowSha) { fail("DISPATCH_FORBIDDEN", "workflow SHA is no longer the trusted main branch head"); } From e61f90ffd953da56f4ab3af4f72388bcc27e56d7 Mon Sep 17 00:00:00 2001 From: Charan Jagwani Date: Mon, 20 Jul 2026 14:28:17 -0700 Subject: [PATCH 21/25] fix(e2e): align Brev workspace states Signed-off-by: Charan Jagwani --- test/brev-launchable-runtime.test.ts | 58 ++++++++++++++++++++++++---- tools/e2e/brev-launchable-runtime.sh | 3 +- 2 files changed, 53 insertions(+), 8 deletions(-) diff --git a/test/brev-launchable-runtime.test.ts b/test/brev-launchable-runtime.test.ts index 7e47a4a3ee..3793b38d32 100644 --- a/test/brev-launchable-runtime.test.ts +++ b/test/brev-launchable-runtime.test.ts @@ -16,6 +16,7 @@ type FixtureOptions = { bootImageId?: string; bootImageSelfLink?: string; deleteMode?: "remove" | "retain"; + emptyWorkspacesMode?: "array" | "null"; evidenceMode?: | "duplicate" | "failed-turn" @@ -31,7 +32,7 @@ type FixtureOptions = { repoSha?: string; supportsLaunchableMode?: boolean; trackedDrift?: "index" | "none" | "worktree"; - workspaceMode?: "ready" | "pending"; + workspaceMode?: "create-failed" | "failure" | "pending" | "ready"; }; afterEach(() => { @@ -131,15 +132,30 @@ case "$1" in exit 9 fi if [ "$FAKE_BREV_LS_MODE" = malformed ]; then printf '{}\\n'; exit 0; fi - if [ -f "$FAKE_BREV_STATE" ]; then cat "$FAKE_BREV_STATE"; else printf '{"workspaces":[]}\\n'; fi - ;; - create) - if [ "$FAKE_BREV_WORKSPACE_MODE" = pending ]; then - printf '{"workspaces":[{"id":"ws-1","name":"%s","status":"PROVISIONING","shell_status":"PENDING","health_status":"PENDING","build_status":"PENDING"}]}\\n' "$INSTANCE_NAME" > "$FAKE_BREV_STATE" + if [ -f "$FAKE_BREV_STATE" ]; then + cat "$FAKE_BREV_STATE" + elif [ "$FAKE_BREV_EMPTY_WORKSPACES_MODE" = null ]; then + printf '{"workspaces":null}\\n' else - printf '{"workspaces":[{"id":"ws-1","name":"%s","status":"RUNNING","shell_status":"READY","health_status":"HEALTHY","build_status":"COMPLETED"}]}\\n' "$INSTANCE_NAME" > "$FAKE_BREV_STATE" + printf '{"workspaces":[]}\\n' fi ;; + create) + case "$FAKE_BREV_WORKSPACE_MODE" in + pending) + printf '{"workspaces":[{"id":"ws-1","name":"%s","status":"PROVISIONING","shell_status":"PENDING","health_status":"PENDING","build_status":"PENDING"}]}\\n' "$INSTANCE_NAME" > "$FAKE_BREV_STATE" + ;; + failure) + printf '{"workspaces":[{"id":"ws-1","name":"%s","status":"FAILURE","shell_status":"NOT_READY","health_status":"UNHEALTHY","build_status":"PENDING"}]}\\n' "$INSTANCE_NAME" > "$FAKE_BREV_STATE" + ;; + create-failed) + printf '{"workspaces":[{"id":"ws-1","name":"%s","status":"PROVISIONING","shell_status":"NOT_READY","health_status":"UNHEALTHY","build_status":"CREATE_FAILED"}]}\\n' "$INSTANCE_NAME" > "$FAKE_BREV_STATE" + ;; + *) + printf '{"workspaces":[{"id":"ws-1","name":"%s","status":"RUNNING","shell_status":"READY","health_status":"HEALTHY","build_status":"COMPLETED"}]}\\n' "$INSTANCE_NAME" > "$FAKE_BREV_STATE" + ;; + esac + ;; exec) shift 3 bash -c "$*" @@ -238,6 +254,7 @@ exec "$@" options.bootImageSelfLink ?? "https://www.googleapis.com/compute/v1/projects/brevdevprod/global/images/image-a", FAKE_BREV_DELETE_MODE: options.deleteMode ?? "remove", + FAKE_BREV_EMPTY_WORKSPACES_MODE: options.emptyWorkspacesMode ?? "array", FAKE_BREV_LS_MODE: options.lsMode ?? "ok", FAKE_BREV_LOG: log, FAKE_BREV_LS_COUNT: lsCount, @@ -409,6 +426,21 @@ describe("exact staging Brev Launchable runtime", () => { expect(run("cleanup", env).status).toBe(0); }); + it.each([ + "failure", + "create-failed", + ] as const)("fails immediately when Brev reports terminal %s state", (workspaceMode) => { + const { env } = fixture({ workspaceMode }); + const deploy = run("deploy", { + ...env, + BREV_POLL_SECONDS: "1", + BREV_READY_TIMEOUT_SECONDS: "1", + }); + + expect(deploy.status).not.toBe(0); + expect(deploy.stderr).toContain("Brev workspace entered terminal failure"); + }); + it.each(["fail", "malformed"] as const)("fails closed when Brev inventory is %s", (lsMode) => { const { env, log } = fixture({ lsMode }); const deploy = run("deploy", env); @@ -428,6 +460,18 @@ describe("exact staging Brev Launchable runtime", () => { ).toMatchObject({ terminalState: "ABSENT", workspaceName: "nclaw-e2e-test-1" }); }); + it("accepts Brev's null workspaces collection after deleting the final workspace", () => { + const { env, workDir } = fixture({ emptyWorkspacesMode: "null" }); + expect(run("deploy", env).status).toBe(0); + + const cleanup = run("cleanup", env); + + expect(cleanup.status, [cleanup.stderr, cleanup.stdout].join("\n")).toBe(0); + expect( + JSON.parse(fs.readFileSync(path.join(workDir, "brev-cleanup-evidence.json"), "utf8")), + ).toMatchObject({ terminalState: "ABSENT", workspaceName: "nclaw-e2e-test-1" }); + }); + it("fails cleanup when the workspace remains after the deletion deadline", () => { const { env } = fixture({ deleteMode: "retain" }); expect(run("deploy", env).status).toBe(0); diff --git a/tools/e2e/brev-launchable-runtime.sh b/tools/e2e/brev-launchable-runtime.sh index da29d54cb1..1dc7ac0308 100755 --- a/tools/e2e/brev-launchable-runtime.sh +++ b/tools/e2e/brev-launchable-runtime.sh @@ -34,6 +34,7 @@ validate_common() { workspace_rows() { timeout 30s brev ls --json | jq -c ' if type == "array" then . + elif type == "object" and has("workspaces") and .workspaces == null then [] elif type == "object" and (.workspaces | type) == "array" then .workspaces else error("unexpected brev ls --json shape") end @@ -66,7 +67,7 @@ wait_for_workspace_ready() { return 0 fi case "$status:$build_status" in - FAILED:* | ERROR:* | *:FAILED | *:ERROR) die "Brev workspace entered terminal failure ($status/$build_status)" ;; + FAILURE:* | FAILED:* | ERROR:* | *:CREATE_FAILED | *:FAILED | *:ERROR) die "Brev workspace entered terminal failure ($status/$build_status)" ;; esac fi sleep "${BREV_POLL_SECONDS:-15}" From e88cc4a495d8c2ed3e13d33b04543bf56b441d6e Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Mon, 20 Jul 2026 17:43:52 -0400 Subject: [PATCH 22/25] ci(e2e): simplify Brev Launchable coverage Signed-off-by: Julie Yaunches --- .../brev-launchable-qualification.yaml | 355 --- .github/workflows/e2e.yaml | 235 +- ci/source-shape-test-budget.json | 5 - test/brev-launchable-runtime.test.ts | 374 +-- test/e2e/README.md | 61 +- test/e2e/mock-parity.json | 1 - .../support/exact-image-manifest-cli.test.ts | 158 -- .../support/exact-image-manifest-fixture.ts | 61 - test/e2e/support/exact-image-manifest.test.ts | 253 -- ...exact-image-qualification-workflow.test.ts | 258 -- ...act-image-qualification-controller.test.ts | 1494 ------------ ...-image-qualification-controller-fixture.ts | 284 --- test/helpers/vitest-watch-triggers.ts | 10 +- test/vitest-watch-triggers.test.ts | 6 - tools/advisors/github.mts | 9 +- tools/e2e/brev-launchable-runtime.sh | 297 +-- tools/e2e/exact-image-manifest.mts | 396 ---- .../exact-image-qualification-controller.mts | 2076 ----------------- tools/e2e/validate-exact-image-manifest.mts | 176 -- 19 files changed, 405 insertions(+), 6104 deletions(-) delete mode 100644 .github/workflows/brev-launchable-qualification.yaml delete mode 100644 test/e2e/support/exact-image-manifest-cli.test.ts delete mode 100644 test/e2e/support/exact-image-manifest-fixture.ts delete mode 100644 test/e2e/support/exact-image-manifest.test.ts delete mode 100644 test/e2e/support/exact-image-qualification-workflow.test.ts delete mode 100644 test/exact-image-qualification-controller.test.ts delete mode 100644 test/helpers/exact-image-qualification-controller-fixture.ts delete mode 100644 tools/e2e/exact-image-manifest.mts delete mode 100755 tools/e2e/exact-image-qualification-controller.mts delete mode 100644 tools/e2e/validate-exact-image-manifest.mts diff --git a/.github/workflows/brev-launchable-qualification.yaml b/.github/workflows/brev-launchable-qualification.yaml deleted file mode 100644 index cbebea4e39..0000000000 --- a/.github/workflows/brev-launchable-qualification.yaml +++ /dev/null @@ -1,355 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -name: E2E / Exact Staging Brev Launchable -run-name: Exact staging Launchable qualification for ${{ inputs.candidate_sha }} - -on: - workflow_dispatch: - inputs: - candidate_sha: - description: Exact lowercase 40-character NemoClaw commit SHA to qualify. - required: true - type: string - reason: - description: Audit reason for starting this exact-image Launchable qualification. - required: true - type: string - workflow_call: - inputs: - candidate_sha: - description: Exact lowercase 40-character NemoClaw commit SHA to qualify. - required: true - type: string - reason: - description: Audit reason for starting this exact-image Launchable qualification. - required: true - type: string - -permissions: {} - -concurrency: - # Every candidate advances the same family consumed by the standing Launchable. - group: brev-launchable-qualification-staging-cpu - cancel-in-progress: false - -jobs: - preflight: - name: Validate exact qualification request - runs-on: ubuntu-latest - timeout-minutes: 10 - permissions: - contents: read - steps: - - name: Require explicit repository activation - env: - QUALIFICATION_ENABLED: ${{ vars.NEMOCLAW_BREV_LAUNCHABLE_QUALIFICATION_ENABLED }} - run: | - set -euo pipefail - if [[ "$QUALIFICATION_ENABLED" != "true" ]]; then - echo "::error::Exact staging Brev Launchable qualification is inactive. Set NEMOCLAW_BREV_LAUNCHABLE_QUALIFICATION_ENABLED=true only after the protected environment and external dependencies are ready." - exit 1 - fi - - - name: Checkout trusted controller - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - with: - ref: ${{ github.workflow_sha }} - persist-credentials: false - - - name: Setup Node - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0 - with: - node-version: "22" - - - name: Validate candidate and maintainer authority - env: - ACTOR: ${{ github.triggering_actor }} - CANDIDATE_SHA: ${{ inputs.candidate_sha }} - EVENT_NAME: ${{ github.event_name }} - GITHUB_TOKEN: ${{ github.token }} - QUALIFICATION_REASON: ${{ inputs.reason }} - REQUESTER_RUN_ATTEMPT: ${{ github.run_attempt }} - REQUESTER_RUN_ID: ${{ github.run_id }} - REQUEST_REF: ${{ github.ref }} - WORKFLOW_SHA: ${{ github.workflow_sha }} - run: >- - node --experimental-strip-types --no-warnings - tools/e2e/exact-image-qualification-controller.mts - --mode preflight - --actor "$ACTOR" - --candidate-sha "$CANDIDATE_SHA" - --event-name "$EVENT_NAME" - --reason "$QUALIFICATION_REASON" - --ref "$REQUEST_REF" - --requester-run-attempt "$REQUESTER_RUN_ATTEMPT" - --requester-run-id "$REQUESTER_RUN_ID" - --workflow-sha "$WORKFLOW_SHA" - - qualify: - name: Build, deploy, and test exact staging image - needs: preflight - # Keep the repository-owned switch outside the protected environment so an - # inactive run cannot request approval or credentials. - if: ${{ vars.NEMOCLAW_BREV_LAUNCHABLE_QUALIFICATION_ENABLED == 'true' && github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' }} - runs-on: ubuntu-latest - timeout-minutes: 120 - environment: - name: approve-brev-launchable-qualification - deployment: false - permissions: - contents: read - steps: - - name: Checkout trusted controller - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - with: - ref: ${{ github.workflow_sha }} - persist-credentials: false - - - name: Setup Node - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0 - with: - node-version: "22" - - - name: Prepare E2E workspace - uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@50281ee84c4a6fc759da95ea28fc0b7d9c378a28 - with: - build-cli: "false" - - - id: workspace - name: Create private evidence workspace - shell: bash - run: | - set -euo pipefail - work_dir="$(mktemp -d "${RUNNER_TEMP}/nemoclaw-image-qualification.XXXXXX")" - chmod 700 "$work_dir" - printf 'work_dir=%s\n' "$work_dir" >> "$GITHUB_OUTPUT" - - - id: start - name: Dispatch exact qualification image build - env: - ACTOR: ${{ github.triggering_actor }} - CANDIDATE_SHA: ${{ inputs.candidate_sha }} - EVENT_NAME: ${{ github.event_name }} - GITHUB_TOKEN: ${{ github.token }} - NEMOCLAW_IMAGE_QUALIFICATION_TOKEN: ${{ secrets.NEMOCLAW_IMAGE_DISPATCH_TOKEN }} - QUALIFICATION_REASON: ${{ inputs.reason }} - REQUESTER_RUN_ATTEMPT: ${{ github.run_attempt }} - REQUESTER_RUN_ID: ${{ github.run_id }} - REQUEST_REF: ${{ github.ref }} - WORKFLOW_SHA: ${{ github.workflow_sha }} - run: >- - node --experimental-strip-types --no-warnings - tools/e2e/exact-image-qualification-controller.mts - --mode start - --actor "$ACTOR" - --candidate-sha "$CANDIDATE_SHA" - --event-name "$EVENT_NAME" - --reason "$QUALIFICATION_REASON" - --ref "$REQUEST_REF" - --requester-run-attempt "$REQUESTER_RUN_ATTEMPT" - --requester-run-id "$REQUESTER_RUN_ID" - --workflow-sha "$WORKFLOW_SHA" - --work-dir "${{ steps.workspace.outputs.work_dir }}" - - - name: Wait for the bound producer run - env: - NEMOCLAW_IMAGE_QUALIFICATION_TOKEN: ${{ secrets.NEMOCLAW_IMAGE_DISPATCH_TOKEN }} - run: >- - node --experimental-strip-types --no-warnings - tools/e2e/exact-image-qualification-controller.mts - --mode wait - --work-dir "${{ steps.workspace.outputs.work_dir }}" - - - name: Download and verify the bound manifest artifact - env: - NEMOCLAW_IMAGE_QUALIFICATION_TOKEN: ${{ secrets.NEMOCLAW_IMAGE_DISPATCH_TOKEN }} - run: >- - node --experimental-strip-types --no-warnings - tools/e2e/exact-image-qualification-controller.mts - --mode download - --work-dir "${{ steps.workspace.outputs.work_dir }}" - - - name: Validate the exact image manifest - env: - CANDIDATE_SHA: ${{ inputs.candidate_sha }} - CORRELATION_ID: ${{ steps.start.outputs.correlation_id }} - IMAGE_REPOSITORY_SHA: ${{ steps.start.outputs.image_repository_sha }} - PRODUCER_RUN_ATTEMPT: ${{ steps.start.outputs.producer_run_attempt }} - PRODUCER_RUN_ID: ${{ steps.start.outputs.producer_run_id }} - REQUESTER_RUN_ATTEMPT: ${{ github.run_attempt }} - REQUESTER_RUN_ID: ${{ github.run_id }} - run: >- - node --experimental-strip-types --no-warnings - tools/e2e/validate-exact-image-manifest.mts - --manifest "${{ steps.workspace.outputs.work_dir }}/nemoclaw-image-manifest.v1.json" - --output "${{ steps.workspace.outputs.work_dir }}/validated-manifest.v1.json" - --nemoclaw-sha "$CANDIDATE_SHA" - --requester-run-id "$REQUESTER_RUN_ID" - --requester-run-attempt "$REQUESTER_RUN_ATTEMPT" - --correlation-id "$CORRELATION_ID" - --image-repository-sha "$IMAGE_REPOSITORY_SHA" - --producer-run-id "$PRODUCER_RUN_ID" - --producer-run-attempt "$PRODUCER_RUN_ATTEMPT" - - - name: Record accepted provenance and hashes - run: >- - node --experimental-strip-types --no-warnings - tools/e2e/exact-image-qualification-controller.mts - --mode finalize - --work-dir "${{ steps.workspace.outputs.work_dir }}" - - - name: Install pinned Brev CLI - env: - BREV_CLI_SHA256: 5a6e70374db9be33f85f299161733b4a8409840d47638c781429b96e8d53704f - BREV_CLI_VERSION: 0.6.330 - run: | - set -euo pipefail - archive="${RUNNER_TEMP}/brev-cli.tar.gz" - curl -fsSL -o "$archive" "https://github.com/brevdev/brev-cli/releases/download/v${BREV_CLI_VERSION}/brev-cli_${BREV_CLI_VERSION}_linux_amd64.tar.gz" - printf '%s %s\n' "$BREV_CLI_SHA256" "$archive" | sha256sum -c - - tar -xzf "$archive" -C "${RUNNER_TEMP}" brev - mkdir -p "${RUNNER_TEMP}/bin" - install -m 0755 "${RUNNER_TEMP}/brev" "${RUNNER_TEMP}/bin/brev" - printf '%s\n' "${RUNNER_TEMP}/bin" >> "$GITHUB_PATH" - - - name: Authenticate Brev CLI - env: - BREV_API_KEY: ${{ secrets.BREV_API_KEY }} - BREV_ORG_ID: ${{ secrets.BREV_ORG_ID }} - run: | - set -euo pipefail - brev login --api-key "$BREV_API_KEY" --org-id "$BREV_ORG_ID" - brev ls --json | jq -e '.workspaces | type == "array"' >/dev/null - - - id: brev-workspace - name: Bind unique staging workspace name - env: - CANDIDATE_SHA: ${{ inputs.candidate_sha }} - run: | - set -euo pipefail - short_sha="${CANDIDATE_SHA:0:8}" - instance_name="nclaw-e2e-${short_sha}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" - printf 'instance_name=%s\n' "$instance_name" >> "$GITHUB_OUTPUT" - - - name: Deploy exact staging Brev Launchable - env: - BREV_LAUNCHABLE_ID: ${{ vars.NEMOCLAW_STAGING_LAUNCHABLE_ID }} - INSTANCE_NAME: ${{ steps.brev-workspace.outputs.instance_name }} - WORK_DIR: ${{ steps.workspace.outputs.work_dir }} - run: tools/e2e/brev-launchable-runtime.sh deploy - - - name: Run existing full E2E suite against the Brev Launchable - env: - CANDIDATE_SHA: ${{ inputs.candidate_sha }} - INSTANCE_NAME: ${{ steps.brev-workspace.outputs.instance_name }} - NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }} - VALIDATED_MANIFEST: ${{ steps.workspace.outputs.work_dir }}/validated-manifest.v1.json - WORK_DIR: ${{ steps.workspace.outputs.work_dir }} - run: tools/e2e/brev-launchable-runtime.sh qualify - - - id: redact-runtime-evidence - name: Redact runtime evidence - if: ${{ always() && steps.workspace.outputs.work_dir != '' }} - env: - NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }} - WORK_DIR: ${{ steps.workspace.outputs.work_dir }} - run: | - set -euo pipefail - python3 - <<'PY' - import os - import stat - from pathlib import Path - - MAX_ENTRIES = 2_500 - MAX_FILES = 1_250 - MAX_TOTAL_BYTES = 128 * 1024 * 1024 - - secret = os.environ.get("NVIDIA_INFERENCE_API_KEY", "").encode() - root = Path(os.environ["WORK_DIR"]) - if root.is_symlink() or not root.is_dir(): - raise SystemExit("runtime evidence root must be a real directory") - - entries = 0 - total_bytes = 0 - paths = [] - stack = [root] - while stack: - directory = stack.pop() - with os.scandir(directory) as children: - for child in children: - entries += 1 - if entries > MAX_ENTRIES: - raise SystemExit(f"runtime evidence exceeds {MAX_ENTRIES} entries") - path = Path(child.path) - if child.is_symlink(): - raise SystemExit(f"runtime evidence must not contain symlinks: {path}") - if child.is_dir(follow_symlinks=False): - stack.append(path) - continue - if not child.is_file(follow_symlinks=False): - raise SystemExit(f"runtime evidence contains a non-regular file: {path}") - metadata = child.stat(follow_symlinks=False) - if not stat.S_ISREG(metadata.st_mode): - raise SystemExit(f"runtime evidence contains a non-regular file: {path}") - paths.append(path) - total_bytes += metadata.st_size - if len(paths) > MAX_FILES: - raise SystemExit(f"runtime evidence exceeds {MAX_FILES} files") - if total_bytes > MAX_TOTAL_BYTES: - raise SystemExit(f"runtime evidence exceeds {MAX_TOTAL_BYTES} bytes") - - for path in paths: - if secret: - content = path.read_bytes() - if secret in content: - path.write_bytes(content.replace(secret, b"[REDACTED]")) - PY - - - name: Delete staging workspace and verify absence - if: ${{ always() && steps.workspace.outputs.work_dir != '' && steps.brev-workspace.outputs.instance_name != '' }} - env: - INSTANCE_NAME: ${{ steps.brev-workspace.outputs.instance_name }} - WORK_DIR: ${{ steps.workspace.outputs.work_dir }} - run: tools/e2e/brev-launchable-runtime.sh cleanup - - - name: Cancel an incomplete producer run - if: ${{ always() && steps.workspace.outputs.work_dir != '' && steps.start.outcome != 'skipped' }} - continue-on-error: true - env: - NEMOCLAW_IMAGE_QUALIFICATION_TOKEN: ${{ secrets.NEMOCLAW_IMAGE_DISPATCH_TOKEN }} - run: >- - node --experimental-strip-types --no-warnings - tools/e2e/exact-image-qualification-controller.mts - --mode cancel - --work-dir "${{ steps.workspace.outputs.work_dir }}" - - - name: Upload exact staging Launchable evidence - if: ${{ always() && steps.workspace.outputs.work_dir != '' && steps.redact-runtime-evidence.outcome == 'success' }} - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: brev-launchable-qualification-${{ inputs.candidate_sha }}-${{ github.run_id }} - path: | - ${{ steps.workspace.outputs.work_dir }}/dispatch-intent.v1.json - ${{ steps.workspace.outputs.work_dir }}/dispatch-reconciliation.v1.json - ${{ steps.workspace.outputs.work_dir }}/controller-state.json - ${{ steps.workspace.outputs.work_dir }}/controller-state.corrupt-*.json - ${{ steps.workspace.outputs.work_dir }}/nemoclaw-image-handoff.zip - ${{ steps.workspace.outputs.work_dir }}/nemoclaw-image-manifest.v1.json - ${{ steps.workspace.outputs.work_dir }}/validated-manifest.v1.json - ${{ steps.workspace.outputs.work_dir }}/qualification-evidence.v1.json - ${{ steps.workspace.outputs.work_dir }}/brev-deploy-request.json - ${{ steps.workspace.outputs.work_dir }}/brev-workspace-ready.json - ${{ steps.workspace.outputs.work_dir }}/brev-provision.json - ${{ steps.workspace.outputs.work_dir }}/brev-boot-image.json - ${{ steps.workspace.outputs.work_dir }}/brev-identity-evidence.json - ${{ steps.workspace.outputs.work_dir }}/brev-launchable-e2e.log - ${{ steps.workspace.outputs.work_dir }}/brev-launchable-e2e-evidence.json - ${{ steps.workspace.outputs.work_dir }}/brev-launchable-cloud-openclaw/ - ${{ steps.workspace.outputs.work_dir }}/brev-cleanup-evidence.json - if-no-files-found: error - retention-days: 90 - - - name: Remove private evidence workspace - if: ${{ always() && steps.workspace.outputs.work_dir != '' }} - run: rm -rf -- "${{ steps.workspace.outputs.work_dir }}" diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 2fbfec406d..fc74879899 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -186,15 +186,224 @@ jobs: fi staging-brev-launchable: - name: Exact staging Brev Launchable needs: generate-matrix - # Repository owners enable this only after the protected environment, - # producer, standing Launchable, and Brev contract are ready. - if: ${{ vars.NEMOCLAW_BREV_LAUNCHABLE_QUALIFICATION_ENABLED == 'true' && github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && inputs.checkout_sha == '' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') }} - uses: ./.github/workflows/brev-launchable-qualification.yaml - with: - candidate_sha: ${{ github.sha }} - reason: E2E run ${{ github.run_id }} qualifying the exact checked-out NemoClaw candidate + if: ${{ contains(format(',{0},', inputs.jobs), ',staging-brev-launchable,') || contains(format(',{0},', inputs.targets), ',staging-brev-launchable,') }} + runs-on: ubuntu-latest + timeout-minutes: 120 + permissions: + contents: read + environment: + name: approve-brev-launchable-e2e + deployment: false + env: + E2E_JOB: "1" + E2E_DEFAULT_ENABLED: "0" + E2E_TARGET_ID: staging-brev-launchable + E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/staging-brev-launchable + CANDIDATE_SHA: ${{ inputs.checkout_sha || github.sha }} + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + ref: ${{ inputs.checkout_sha || github.sha }} + persist-credentials: false + + # Keep only the credential-bearing step anchored. Cleanup mappings stay + # explicit because strict YAML decoders reject 100 or more aliases here. + - &dockerhub-auth + name: Authenticate to Docker Hub + uses: NVIDIA/NemoClaw/.github/actions/docker-auth-setup@78091da47e290f49b8fe3f3e70b72362a0853928 + with: + auth-required: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && inputs.checkout_sha == '' && '1' || '0' }} + username: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && inputs.checkout_sha == '' && secrets.DOCKERHUB_USERNAME || '' }} + token: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && inputs.checkout_sha == '' && secrets.DOCKERHUB_TOKEN || '' }} + + - name: Prepare E2E workspace + uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@50281ee84c4a6fc759da95ea28fc0b7d9c378a28 + + - id: workspace + name: Create evidence workspace + run: | + set -euo pipefail + work_dir="$E2E_ARTIFACT_DIR" + mkdir -p "$work_dir" + chmod 700 "$work_dir" + printf 'work_dir=%s\n' "$work_dir" >> "$GITHUB_OUTPUT" + + - id: image-build + name: Build or reuse the exact staging image + env: + IMAGE_TOKEN: ${{ secrets.NEMOCLAW_IMAGE_DISPATCH_TOKEN }} + WORK_DIR: ${{ steps.workspace.outputs.work_dir }} + run: | + set -euo pipefail + [[ "$CANDIDATE_SHA" =~ ^[0-9a-f]{40}$ ]] \ + || { echo "::error::candidate SHA must be a lowercase full SHA"; exit 1; } + api=https://api.github.com/repos/brevdev/nemoclaw-image + producer_sha="$(curl --fail-with-body --silent --show-error --proto '=https' \ + -H "Authorization: Bearer $IMAGE_TOKEN" \ + -H 'Accept: application/vnd.github+json' \ + -H 'X-GitHub-Api-Version: 2026-03-10' \ + "$api/git/ref/heads/main" | jq -er .object.sha)" + response="$({ + jq -n \ + --arg ref main \ + --arg sha "$CANDIDATE_SHA" \ + '{ref:$ref,inputs:{nemoclaw_ref:$sha},return_run_details:true}' + } | curl --fail-with-body --silent --show-error --proto '=https' \ + -X POST \ + -H "Authorization: Bearer $IMAGE_TOKEN" \ + -H 'Accept: application/vnd.github+json' \ + -H 'X-GitHub-Api-Version: 2026-03-10' \ + -H 'Content-Type: application/json' \ + --data-binary @- \ + "$api/actions/workflows/build-on-push.yml/dispatches")" + run_id="$(jq -er '.workflow_run_id | tostring' <<< "$response")" + run_url="$(jq -er .html_url <<< "$response")" + jq -n \ + --arg candidateSha "$CANDIDATE_SHA" \ + --arg producerSha "$producer_sha" \ + --arg runId "$run_id" \ + --arg runUrl "$run_url" \ + '{candidateSha:$candidateSha,producerSha:$producerSha,runId:$runId,runUrl:$runUrl,status:"dispatched"}' \ + > "$WORK_DIR/image-build.json" + printf 'producer_sha=%s\nrun_id=%s\nrun_url=%s\n' \ + "$producer_sha" "$run_id" "$run_url" >> "$GITHUB_OUTPUT" + + - name: Wait for the exact image build + env: + IMAGE_TOKEN: ${{ secrets.NEMOCLAW_IMAGE_DISPATCH_TOKEN }} + PRODUCER_SHA: ${{ steps.image-build.outputs.producer_sha }} + RUN_ID: ${{ steps.image-build.outputs.run_id }} + WORK_DIR: ${{ steps.workspace.outputs.work_dir }} + run: | + set -euo pipefail + api=https://api.github.com/repos/brevdev/nemoclaw-image + deadline=$((SECONDS + 5400)) + while [ "$SECONDS" -lt "$deadline" ]; do + run="$(curl --fail-with-body --silent --show-error --proto '=https' \ + -H "Authorization: Bearer $IMAGE_TOKEN" \ + -H 'Accept: application/vnd.github+json' \ + -H 'X-GitHub-Api-Version: 2026-03-10' \ + "$api/actions/runs/$RUN_ID")" + jq -e \ + --arg id "$RUN_ID" \ + --arg sha "$PRODUCER_SHA" ' + (.id | tostring) == $id + and .head_sha == $sha + and .event == "workflow_dispatch" + and .path == ".github/workflows/build-on-push.yml" + and .run_attempt == 1 + ' <<< "$run" >/dev/null \ + || { echo "::error::image build identity changed"; exit 1; } + status="$(jq -r .status <<< "$run")" + if [ "$status" = completed ]; then + conclusion="$(jq -r .conclusion <<< "$run")" + jq --arg status "$status" --arg conclusion "$conclusion" \ + '.status=$status | .conclusion=$conclusion' \ + "$WORK_DIR/image-build.json" > "$WORK_DIR/image-build.tmp" + mv "$WORK_DIR/image-build.tmp" "$WORK_DIR/image-build.json" + [ "$conclusion" = success ] \ + || { echo "::error::image build completed with $conclusion"; exit 1; } + exit 0 + fi + sleep 15 + done + echo "::error::image build did not complete within 90 minutes" + exit 1 + + - name: Install pinned Brev CLI + env: + BREV_CLI_SHA256: 5a6e70374db9be33f85f299161733b4a8409840d47638c781429b96e8d53704f + BREV_CLI_VERSION: 0.6.330 + run: | + set -euo pipefail + archive="${RUNNER_TEMP}/brev-cli.tar.gz" + curl -fsSL -o "$archive" "https://github.com/brevdev/brev-cli/releases/download/v${BREV_CLI_VERSION}/brev-cli_${BREV_CLI_VERSION}_linux_amd64.tar.gz" + printf '%s %s\n' "$BREV_CLI_SHA256" "$archive" | sha256sum -c - + tar -xzf "$archive" -C "${RUNNER_TEMP}" brev + install -Dm 0755 "${RUNNER_TEMP}/brev" "${RUNNER_TEMP}/bin/brev" + printf '%s\n' "${RUNNER_TEMP}/bin" >> "$GITHUB_PATH" + + - name: Authenticate Brev CLI + env: + BREV_API_KEY: ${{ secrets.BREV_API_KEY }} + BREV_ORG_ID: ${{ secrets.BREV_ORG_ID }} + run: | + set -euo pipefail + brev login --api-key "$BREV_API_KEY" --org-id "$BREV_ORG_ID" + brev ls --json | jq -e '.workspaces | type == "array"' >/dev/null + + - id: brev-workspace + name: Choose workspace name + run: | + set -euo pipefail + instance_name="nclaw-e2e-${CANDIDATE_SHA:0:8}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + printf 'instance_name=%s\n' "$instance_name" >> "$GITHUB_OUTPUT" + + - name: Deploy the staging Launchable + env: + BREV_LAUNCHABLE_ID: ${{ vars.NEMOCLAW_STAGING_LAUNCHABLE_ID }} + INSTANCE_NAME: ${{ steps.brev-workspace.outputs.instance_name }} + WORK_DIR: ${{ steps.workspace.outputs.work_dir }} + run: tools/e2e/brev-launchable-runtime.sh deploy + + - name: Run full E2E against the baked image + env: + INSTANCE_NAME: ${{ steps.brev-workspace.outputs.instance_name }} + NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }} + WORK_DIR: ${{ steps.workspace.outputs.work_dir }} + run: tools/e2e/brev-launchable-runtime.sh run + + - name: Redact the runtime log + if: ${{ always() && steps.workspace.outputs.work_dir != '' }} + env: + NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }} + WORK_DIR: ${{ steps.workspace.outputs.work_dir }} + run: | + set -euo pipefail + log="$WORK_DIR/brev-launchable-e2e.log" + if [ -f "$log" ] && [ -n "$NVIDIA_INFERENCE_API_KEY" ]; then + sed "s|${NVIDIA_INFERENCE_API_KEY//|/\\|}|[REDACTED]|g" "$log" > "$log.redacted" + mv "$log.redacted" "$log" + fi + + - name: Delete the Brev workspace + if: ${{ always() && steps.brev-workspace.outputs.instance_name != '' }} + env: + INSTANCE_NAME: ${{ steps.brev-workspace.outputs.instance_name }} + WORK_DIR: ${{ steps.workspace.outputs.work_dir }} + run: tools/e2e/brev-launchable-runtime.sh cleanup + + - name: Cancel an unfinished image build + if: ${{ always() && steps.image-build.outputs.run_id != '' }} + continue-on-error: true + env: + IMAGE_TOKEN: ${{ secrets.NEMOCLAW_IMAGE_DISPATCH_TOKEN }} + RUN_ID: ${{ steps.image-build.outputs.run_id }} + run: | + set -euo pipefail + api=https://api.github.com/repos/brevdev/nemoclaw-image + status="$(curl --fail-with-body --silent --show-error --proto '=https' \ + -H "Authorization: Bearer $IMAGE_TOKEN" \ + -H 'Accept: application/vnd.github+json' \ + -H 'X-GitHub-Api-Version: 2026-03-10' \ + "$api/actions/runs/$RUN_ID" | jq -er .status)" + [ "$status" = completed ] \ + || curl --fail-with-body --silent --show-error --proto '=https' \ + -X POST \ + -H "Authorization: Bearer $IMAGE_TOKEN" \ + -H 'Accept: application/vnd.github+json' \ + -H 'X-GitHub-Api-Version: 2026-03-10' \ + "$api/actions/runs/$RUN_ID/cancel" + + - name: Upload Brev Launchable E2E evidence + if: always() + uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 + + - name: Clean up Docker auth + if: always() + shell: bash + run: bash .github/scripts/docker-auth-cleanup.sh live: needs: generate-matrix @@ -216,15 +425,7 @@ jobs: ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false - # Keep only the credential-bearing step anchored. Cleanup mappings stay - # explicit because strict YAML decoders reject 100 or more aliases here. - - &dockerhub-auth - name: Authenticate to Docker Hub - uses: NVIDIA/NemoClaw/.github/actions/docker-auth-setup@78091da47e290f49b8fe3f3e70b72362a0853928 - with: - auth-required: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && inputs.checkout_sha == '' && '1' || '0' }} - username: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && inputs.checkout_sha == '' && secrets.DOCKERHUB_USERNAME || '' }} - token: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && inputs.checkout_sha == '' && secrets.DOCKERHUB_TOKEN || '' }} + - *dockerhub-auth - name: Configure live E2E trace directory env: diff --git a/ci/source-shape-test-budget.json b/ci/source-shape-test-budget.json index 72b8a11b4c..57d1ac7d22 100644 --- a/ci/source-shape-test-budget.json +++ b/ci/source-shape-test-budget.json @@ -131,11 +131,6 @@ "test": "replaces legacy target_ref dispatches with the validated checkout contract", "category": "security" }, - { - "file": "test/e2e/support/exact-image-qualification-workflow.test.ts", - "test": "keeps exact-image Launchable qualification protected, reusable, and fail-closed", - "category": "security" - }, { "file": "test/e2e/live/hermes-e2e.test.ts", "test": "hermes-e2e: install.sh onboards Hermes and proves health plus live inference", diff --git a/test/brev-launchable-runtime.test.ts b/test/brev-launchable-runtime.test.ts index 7e47a4a3ee..587567c2da 100644 --- a/test/brev-launchable-runtime.test.ts +++ b/test/brev-launchable-runtime.test.ts @@ -9,36 +9,24 @@ import { afterEach, describe, expect, it } from "vitest"; const REPO_ROOT = path.resolve(import.meta.dirname, ".."); const SCRIPT = path.join(REPO_ROOT, "tools", "e2e", "brev-launchable-runtime.sh"); -const roots: string[] = []; const candidateSha = "a".repeat(40); +const roots: string[] = []; type FixtureOptions = { bootImageId?: string; - bootImageSelfLink?: string; deleteMode?: "remove" | "retain"; - evidenceMode?: - | "duplicate" - | "failed-turn" - | "fifo" - | "malformed" - | "oversized" - | "symlink" - | "valid" - | "wrong-path"; - lsMode?: "ok" | "fail" | "fail-once" | "malformed"; - model?: string; + e2eStatus?: "failed" | "passed"; + imageLabelSha?: string; provisionSha?: string; repoSha?: string; - supportsLaunchableMode?: boolean; - trackedDrift?: "index" | "none" | "worktree"; - workspaceMode?: "ready" | "pending"; + workspaceMode?: "pending" | "ready"; }; afterEach(() => { for (const root of roots.splice(0)) fs.rmSync(root, { recursive: true, force: true }); }); -function writeExecutable(file: string, contents: string): void { +function executable(file: string, contents: string): void { fs.writeFileSync(file, contents, { mode: 0o755 }); } @@ -49,106 +37,51 @@ function fixture(options: FixtureOptions = {}) { const workDir = path.join(root, "evidence"); const home = path.join(root, "home"); const state = path.join(root, "state.json"); - const lsCount = path.join(root, "ls-count"); const log = path.join(root, "brev.log"); const e2eExecuted = path.join(root, "e2e-executed"); - const manifest = path.join(workDir, "validated-manifest.v1.json"); fs.mkdirSync(bin); fs.mkdirSync(workDir); fs.mkdirSync(path.join(home, "NemoClaw", ".git"), { recursive: true }); - fs.mkdirSync(path.join(home, "NemoClaw", "test", "e2e", "live"), { recursive: true }); - fs.writeFileSync( - path.join(home, "NemoClaw", "test", "e2e", "live", "full-e2e.test.ts"), - options.supportsLaunchableMode === false - ? "// legacy full E2E fixture\n" - : 'const mode = process.env.NEMOCLAW_E2E_SETUP_MODE; const target = "brev-launchable-cloud-openclaw";\n', - ); fs.mkdirSync(path.join(home, "NemoClaw", "node_modules", ".bin"), { recursive: true }); - writeExecutable( + + executable( path.join(home, "NemoClaw", "node_modules", ".bin", "vitest"), `#!/usr/bin/env bash set -euo pipefail -artifact_dir="$E2E_ARTIFACT_DIR/brev-launchable-cloud-openclaw-onboard-inference-cli-operations-and-cleanup" : > "$FAKE_E2E_EXECUTED" -write_result() { - mkdir -p "$(dirname "$1")" - jq -n '{ - firstAgentTurn:{commandMs:125,responseChars:24,status:"passed"}, - id:"brev-launchable-cloud-openclaw", - runner:"vitest", - securityPosture:{ - configureGuard:true, - entrypoint:{capBnd:"0",capEff:"0",dangerousBoundingCapabilities:[],dangerousEffectiveCapabilities:[],noNewPrivs:"1",uid:"1000"}, - hostNonRoot:true, - rcFilesLocked:true, - runtimeProxyEnvLocked:true, - startupLogClean:true - }, - status:"passed" - }' > "$1" -} -case "$FAKE_EVIDENCE_MODE" in - valid) write_result "$artifact_dir/target-result.json" ;; - duplicate) - write_result "$artifact_dir/target-result.json" - write_result "$E2E_ARTIFACT_DIR/duplicate/target-result.json" - ;; - malformed) - mkdir -p "$artifact_dir" - jq -n '{id:"brev-launchable-cloud-openclaw",runner:"vitest",status:"passed"}' > "$artifact_dir/target-result.json" - ;; - failed-turn) - write_result "$artifact_dir/target-result.json" - jq '.firstAgentTurn.status = "failed"' "$artifact_dir/target-result.json" > "$artifact_dir/result.tmp" - mv "$artifact_dir/result.tmp" "$artifact_dir/target-result.json" - ;; - fifo) write_result "$artifact_dir/target-result.json" ;; - oversized) - write_result "$artifact_dir/target-result.json" - dd if=/dev/null of="$artifact_dir/oversized.bin" bs=1 seek=104857601 2>/dev/null - ;; - symlink) - mkdir -p "$artifact_dir" - write_result "$artifact_dir/real-result.json" - ln -s real-result.json "$artifact_dir/target-result.json" - ;; - wrong-path) write_result "$E2E_ARTIFACT_DIR/wrong/target-result.json" ;; - *) exit 2 ;; -esac +result="$E2E_ARTIFACT_DIR/brev-launchable-cloud-openclaw-onboard-inference-cli-operations-and-cleanup/target-result.json" +mkdir -p "$(dirname "$result")" +jq -n --arg status "$FAKE_E2E_STATUS" '{ + firstAgentTurn:{commandMs:125,responseChars:24,status:$status}, + id:"brev-launchable-cloud-openclaw", + runner:"vitest", + securityPosture:{configureGuard:true,hostNonRoot:true,rcFilesLocked:true,runtimeProxyEnvLocked:true,startupLogClean:true}, + status:$status +}' > "$result" +[ "$FAKE_E2E_STATUS" = passed ] `, ); - writeExecutable(path.join(bin, "timeout"), '#!/usr/bin/env bash\nshift\nexec "$@"\n'); - writeExecutable( + executable(path.join(bin, "timeout"), '#!/usr/bin/env bash\nshift\nexec "$@"\n'); + executable( path.join(bin, "brev"), `#!/usr/bin/env bash set -euo pipefail -printf '%s\\n' "$*" >> "$FAKE_BREV_LOG" +printf '%s\n' "$*" >> "$FAKE_BREV_LOG" case "$1" in ls) - [ "$FAKE_BREV_LS_MODE" != fail ] || exit 9 - if [ "$FAKE_BREV_LS_MODE" = fail-once ] && [ ! -f "$FAKE_BREV_LS_COUNT" ]; then - : >"$FAKE_BREV_LS_COUNT" - exit 9 - fi - if [ "$FAKE_BREV_LS_MODE" = malformed ]; then printf '{}\\n'; exit 0; fi - if [ -f "$FAKE_BREV_STATE" ]; then cat "$FAKE_BREV_STATE"; else printf '{"workspaces":[]}\\n'; fi + if [ -f "$FAKE_BREV_STATE" ]; then cat "$FAKE_BREV_STATE"; else printf '{"workspaces":[]}\n'; fi ;; create) if [ "$FAKE_BREV_WORKSPACE_MODE" = pending ]; then - printf '{"workspaces":[{"id":"ws-1","name":"%s","status":"PROVISIONING","shell_status":"PENDING","health_status":"PENDING","build_status":"PENDING"}]}\\n' "$INSTANCE_NAME" > "$FAKE_BREV_STATE" + printf '{"workspaces":[{"name":"%s","status":"PROVISIONING","shell_status":"PENDING","health_status":"PENDING","build_status":"PENDING"}]}\n' "$INSTANCE_NAME" > "$FAKE_BREV_STATE" else - printf '{"workspaces":[{"id":"ws-1","name":"%s","status":"RUNNING","shell_status":"READY","health_status":"HEALTHY","build_status":"COMPLETED"}]}\\n' "$INSTANCE_NAME" > "$FAKE_BREV_STATE" + printf '{"workspaces":[{"name":"%s","status":"RUNNING","shell_status":"READY","health_status":"HEALTHY","build_status":"COMPLETED"}]}\n' "$INSTANCE_NAME" > "$FAKE_BREV_STATE" fi ;; - exec) - shift 3 - bash -c "$*" - ;; + exec) shift 3; bash -c "$*" ;; copy) source_path="\${2#*:}" - destination="$3" - cp -R "$source_path"/. "$destination"/ - [ "$FAKE_EVIDENCE_MODE" != fifo ] || mkfifo "$destination/untrusted.fifo" + cp -R "$source_path"/. "$3"/ ;; delete) [ "$FAKE_BREV_DELETE_MODE" = retain ] || rm -f "$FAKE_BREV_STATE" ;; refresh) ;; @@ -156,103 +89,82 @@ case "$1" in esac `, ); - writeExecutable( + executable( path.join(bin, "sudo"), `#!/usr/bin/env bash set -euo pipefail [ "\${1:-}" != -n ] || shift if [ "\${1:-}" = cat ] && [ "\${2:-}" = /etc/nemoclaw/provision.json ]; then - jq -cn --arg sha "$FAKE_PROVISION_SHA" '{gitSha:$sha,version:"0.0.0"}' + jq -cn --arg sha "$FAKE_PROVISION_SHA" '{gitSha:$sha}' exit 0 fi exec "$@" `, ); - writeExecutable( + executable( path.join(bin, "git"), `#!/usr/bin/env bash set -euo pipefail case "$*" in - *'diff --cached --quiet --no-ext-diff HEAD --'*) [ "$FAKE_TRACKED_DRIFT" != index ] ;; - *'diff --quiet --no-ext-diff HEAD --'*) [ "$FAKE_TRACKED_DRIFT" != worktree ] ;; - *'rev-parse HEAD'*) printf '%s\\n' "$FAKE_REPO_SHA" ;; + *'diff --quiet --no-ext-diff HEAD --'*) exit 0 ;; + *'diff --cached --quiet --no-ext-diff HEAD --'*) exit 0 ;; + *'rev-parse HEAD'*) printf '%s\n' "$FAKE_REPO_SHA" ;; *) exit 2 ;; esac `, ); - writeExecutable( + executable( path.join(bin, "curl"), `#!/usr/bin/env bash set -euo pipefail for value in "$@"; do case "$value" in - */project/project-id) printf 'brevdevprod\\n'; exit 0 ;; - */instance/zone) printf 'projects/1/zones/us-central1-a\\n'; exit 0 ;; - */instance/disks/0/device-name) printf 'disk-1\\n'; exit 0 ;; - */instance/service-accounts/default/token) printf '{"access_token":"token"}\\n'; exit 0 ;; - https://compute.googleapis.com/*) - jq -cn --arg sourceImage "$FAKE_BOOT_IMAGE_SELF_LINK" --arg sourceImageId "$FAKE_BOOT_IMAGE_ID" '{sourceImage:$sourceImage,sourceImageId:$sourceImageId}' + */project/project-id) printf 'brevdevprod\n'; exit 0 ;; + */instance/zone) printf 'projects/1/zones/us-central1-a\n'; exit 0 ;; + */instance/disks/0/device-name) printf 'disk-1\n'; exit 0 ;; + */instance/service-accounts/default/token) printf '{"access_token":"token"}\n'; exit 0 ;; + https://compute.googleapis.com/*/disks/*) + jq -cn --arg id "$FAKE_BOOT_IMAGE_ID" '{sourceImage:"https://www.googleapis.com/compute/v1/projects/brevdevprod/global/images/image-a",sourceImageId:$id}' + exit 0 + ;; + https://www.googleapis.com/compute/v1/projects/brevdevprod/global/images/image-a) + jq -cn --arg sha "$FAKE_IMAGE_LABEL_SHA" '{name:"image-a",id:"123456789",selfLink:"https://www.googleapis.com/compute/v1/projects/brevdevprod/global/images/image-a",status:"READY",labels:{"nemoclaw-sha":$sha,channel:"staging",variant:"cpu"}}' exit 0 ;; - https://inference.local/*) printf '{"choices":[{"message":{"content":"PONG"}}]}\\n'; exit 0 ;; esac done exit 2 -`, - ); - writeExecutable( - path.join(bin, "openshell"), - `#!/usr/bin/env bash -set -euo pipefail -if [ "\${1:-}" = --version ]; then printf 'openshell 1.0\\n'; exit 0; fi -while [ "$#" -gt 0 ] && [ "$1" != -- ]; do shift; done -[ "\${1:-}" = -- ] && shift -exec "$@" `, ); for (const [name, body] of [ - ["brev-quickstart", "printf 'Ready!\\n'"], - ["docker", "exit 0"], + ["node", "printf '%s\\n' 'nvidia/test-model'"], + ["brev-quickstart", "exit 0"], ["nemoclaw", "exit 0"], - ["node", "printf '%s\\n' \"$FAKE_MODEL\""], - ["openclaw", 'printf \'{"payloads":[{"text":"42"}]}\\n\''], + ["openshell", "exit 0"], + ["docker", "exit 0"], + ["openclaw", "exit 0"], ] as const) { - writeExecutable(path.join(bin, name), `#!/usr/bin/env bash\nset -euo pipefail\n${body}\n`); + executable(path.join(bin, name), `#!/usr/bin/env bash\nset -euo pipefail\n${body}\n`); } - fs.writeFileSync( - manifest, - `${JSON.stringify({ - imageName: "image-a", - imageId: "123456789", - imageSelfLink: - "https://www.googleapis.com/compute/v1/projects/brevdevprod/global/images/image-a", - })}\n`, - ); + const env = { ...process.env, PATH: `${bin}:${process.env.PATH ?? ""}`, BREV_LAUNCHABLE_ID: "env-staging123", CANDIDATE_SHA: candidateSha, FAKE_BOOT_IMAGE_ID: options.bootImageId ?? "123456789", - FAKE_BOOT_IMAGE_SELF_LINK: - options.bootImageSelfLink ?? - "https://www.googleapis.com/compute/v1/projects/brevdevprod/global/images/image-a", FAKE_BREV_DELETE_MODE: options.deleteMode ?? "remove", - FAKE_BREV_LS_MODE: options.lsMode ?? "ok", FAKE_BREV_LOG: log, - FAKE_BREV_LS_COUNT: lsCount, FAKE_BREV_STATE: state, + FAKE_BREV_WORKSPACE_MODE: options.workspaceMode ?? "ready", FAKE_E2E_EXECUTED: e2eExecuted, - FAKE_EVIDENCE_MODE: options.evidenceMode ?? "valid", - FAKE_MODEL: options.model ?? "nvidia/test-model", + FAKE_E2E_STATUS: options.e2eStatus ?? "passed", + FAKE_IMAGE_LABEL_SHA: options.imageLabelSha ?? candidateSha, FAKE_PROVISION_SHA: options.provisionSha ?? candidateSha.slice(0, 7), FAKE_REPO_SHA: options.repoSha ?? candidateSha, - FAKE_TRACKED_DRIFT: options.trackedDrift ?? "none", - FAKE_BREV_WORKSPACE_MODE: options.workspaceMode ?? "ready", HOME: home, INSTANCE_NAME: "nclaw-e2e-test-1", NVIDIA_INFERENCE_API_KEY: "nvapi-test-value", - VALIDATED_MANIFEST: manifest, WORK_DIR: workDir, BREV_POLL_SECONDS: "0", }; @@ -263,191 +175,71 @@ function run(mode: string, env: NodeJS.ProcessEnv) { return spawnSync("bash", [SCRIPT, mode], { cwd: REPO_ROOT, encoding: "utf8", env }); } -describe("exact staging Brev Launchable runtime", () => { - it("deploys only the configured Launchable, proves identity, runs the existing E2E, and deletes", () => { +describe("Brev Launchable E2E runtime", () => { + it("deploys the configured Launchable, verifies the exact image, runs full E2E, and cleans up", () => { const { env, log, workDir } = fixture(); expect(run("deploy", env).status).toBe(0); - const qualification = run("qualify", env); - expect( - qualification.status, - [qualification.stderr, qualification.stdout, fs.readFileSync(log, "utf8")].join("\n"), - ).toBe(0); + const e2e = run("run", env); + expect(e2e.status, [e2e.stderr, e2e.stdout].join("\n")).toBe(0); expect(run("cleanup", env).status).toBe(0); const commands = fs.readFileSync(log, "utf8"); expect(commands).toContain( "create nclaw-e2e-test-1 --launchable env-staging123 --detached --timeout 900", ); - expect(commands).not.toMatch(/rsync|install\.sh|npm (?:ci|install)|git clone/u); expect(commands).toContain("test/e2e/live/full-e2e.test.ts"); expect(commands).toContain("NEMOCLAW_E2E_SETUP_MODE=preinstalled-launchable"); - expect(commands).toContain("NEMOCLAW_E2E_SECURITY_POSTURE=1"); - expect(commands).toContain("E2E_TARGET_ID=brev-launchable-cloud-openclaw"); - expect(commands.indexOf("rev-parse HEAD")).toBeLessThan( - commands.indexOf("test/e2e/live/full-e2e.test.ts"), - ); - expect(fs.existsSync(path.join(workDir, "brev-identity-evidence.json"))).toBe(true); - expect(fs.existsSync(path.join(workDir, "brev-launchable-e2e-evidence.json"))).toBe(true); - expect( - JSON.parse(fs.readFileSync(path.join(workDir, "brev-launchable-e2e-evidence.json"), "utf8")), - ).toMatchObject({ - artifacts: { - fileCount: expect.any(Number), - targetResultPath: - "brev-launchable-cloud-openclaw-onboard-inference-cli-operations-and-cleanup/target-result.json", - targetResultSha256: expect.stringMatching(/^[0-9a-f]{64}$/u), - totalBytes: expect.any(Number), - }, - setupMode: "preinstalled-launchable", - }); - expect( - fs.existsSync( - path.join( - workDir, - "brev-launchable-cloud-openclaw", - "brev-launchable-cloud-openclaw-onboard-inference-cli-operations-and-cleanup", - "target-result.json", - ), - ), - ).toBe(true); + expect(commands).not.toMatch(/rsync|install\.sh|npm (?:ci|install)|git clone/u); expect( JSON.parse(fs.readFileSync(path.join(workDir, "brev-cleanup-evidence.json"), "utf8")), - ).toMatchObject({ terminalState: "ABSENT", workspaceName: "nclaw-e2e-test-1" }); + ).toMatchObject({ terminalState: "ABSENT" }); }); - it("fails closed on a baked SHA mismatch before onboarding", () => { - const { env, log } = fixture({ repoSha: "b".repeat(40) }); - expect(run("deploy", env).status).toBe(0); - const qualification = run("qualify", env); - expect(qualification.status).not.toBe(0); - expect( - [qualification.stderr, qualification.stdout, fs.readFileSync(log, "utf8")].join("\n"), - ).toContain("does not match candidate"); - expect(fs.readFileSync(log, "utf8")).not.toContain("test/e2e/live/full-e2e.test.ts"); - expect(run("cleanup", env).status).toBe(0); + it("rejects a baked checkout for a different candidate before E2E", () => { + const { e2eExecuted, env } = fixture({ repoSha: "b".repeat(40) }); + const result = run("run", env); + expect(result.status).not.toBe(0); + expect(result.stderr).toContain("does not match candidate"); + expect(fs.existsSync(e2eExecuted)).toBe(false); }); it.each([ - "index", - "worktree", - ] as const)("fails closed on tracked %s drift before executing the E2E harness", (trackedDrift) => { - const { e2eExecuted, env } = fixture({ trackedDrift }); - const qualification = run("qualify", env); - - expect(qualification.status).not.toBe(0); - expect([qualification.stderr, qualification.stdout].join("\n")).toContain( - trackedDrift === "index" ? "staged changes" : "tracked worktree changes", - ); + ["image ID", { bootImageId: "987654321" }], + ["candidate label", { imageLabelSha: "b".repeat(40) }], + ] as const)("rejects a mismatched boot %s before E2E", (_name, options) => { + const { e2eExecuted, env } = fixture(options); + const result = run("run", env); + expect(result.status).not.toBe(0); + expect(result.stderr).toContain("boot image does not match"); expect(fs.existsSync(e2eExecuted)).toBe(false); }); - it("fails closed on a boot-image mismatch before onboarding", () => { - const { env, log } = fixture({ bootImageId: "987654321" }); - expect(run("deploy", env).status).toBe(0); - const qualification = run("qualify", env); - expect(qualification.status).not.toBe(0); - expect(qualification.stderr).toContain("workspace boot disk does not match accepted image"); - expect(fs.readFileSync(log, "utf8")).not.toContain("test/e2e/live/full-e2e.test.ts"); - expect(run("cleanup", env).status).toBe(0); - }); - - it("fails rather than reinstalling when the candidate full E2E lacks Launchable mode", () => { - const { env, workDir } = fixture({ supportsLaunchableMode: false }); - expect(run("deploy", env).status).toBe(0); - const qualification = run("qualify", env); - expect(qualification.status).not.toBe(0); - expect(fs.readFileSync(path.join(workDir, "brev-launchable-e2e.log"), "utf8")).toContain( - "does not support preinstalled Launchable setup", - ); - expect(fs.existsSync(path.join(workDir, "brev-launchable-e2e-evidence.json"))).toBe(false); - expect(run("cleanup", env).status).toBe(0); - }); - - it("rejects an unsafe image-derived model before executing the E2E harness", () => { - const { e2eExecuted, env, workDir } = fixture(); - const injectionMarker = `${e2eExecuted}-model-injection`; - env.FAKE_MODEL = `nvidia/model'; touch ${injectionMarker}; #`; - const qualification = run("qualify", env); - - expect(qualification.status).not.toBe(0); - expect(fs.readFileSync(path.join(workDir, "brev-launchable-e2e.log"), "utf8")).toContain( - "Launchable cloud model is not a safe model ID", - ); - expect(fs.existsSync(e2eExecuted)).toBe(false); - expect(fs.existsSync(injectionMarker)).toBe(false); + it("reports a failing full E2E result", () => { + const { env } = fixture({ e2eStatus: "failed" }); + const result = run("run", env); + expect(result.status).not.toBe(0); }); - it.each([ - "duplicate", - "failed-turn", - "fifo", - "malformed", - "oversized", - "symlink", - "wrong-path", - ] as const)("rejects %s copied E2E evidence", (evidenceMode) => { - const { e2eExecuted, env, workDir } = fixture({ evidenceMode }); - const qualification = run("qualify", env); - - expect(qualification.status).not.toBe(0); - expect(fs.existsSync(e2eExecuted)).toBe(true); - expect(fs.existsSync(path.join(workDir, "brev-launchable-e2e-evidence.json"))).toBe(false); - }); - - it("fails when workspace readiness does not reach its deadline", () => { + it("times out when the workspace never becomes ready", () => { const { env } = fixture({ workspaceMode: "pending" }); - const deploy = run("deploy", { + const result = run("deploy", { ...env, BREV_POLL_SECONDS: "1", BREV_READY_TIMEOUT_SECONDS: "1", }); - - expect(deploy.status).not.toBe(0); - expect(deploy.stderr).toContain( - "Brev workspace did not become structurally ready before the deadline", - ); - expect(run("cleanup", env).status).toBe(0); + expect(result.status).not.toBe(0); + expect(result.stderr).toContain("did not become ready"); }); - it.each(["fail", "malformed"] as const)("fails closed when Brev inventory is %s", (lsMode) => { - const { env, log } = fixture({ lsMode }); - const deploy = run("deploy", env); - expect(deploy.status).not.toBe(0); - expect(deploy.stderr).toContain("unable to inventory Brev workspaces before deploy"); - expect(fs.readFileSync(log, "utf8")).not.toContain("create "); - }); - - it("retries cleanup when the initial Brev inventory request fails", () => { - const { env, workDir } = fixture({ lsMode: "fail-once" }); - const cleanup = run("cleanup", env); - - expect(cleanup.status, [cleanup.stderr, cleanup.stdout].join("\n")).toBe(0); - expect(cleanup.stderr).toContain("brev ls failed before cleanup"); - expect( - JSON.parse(fs.readFileSync(path.join(workDir, "brev-cleanup-evidence.json"), "utf8")), - ).toMatchObject({ terminalState: "ABSENT", workspaceName: "nclaw-e2e-test-1" }); - }); - - it("fails cleanup when the workspace remains after the deletion deadline", () => { + it("fails cleanup while the workspace still exists", () => { const { env } = fixture({ deleteMode: "retain" }); expect(run("deploy", env).status).toBe(0); - const cleanup = run("cleanup", { + const result = run("cleanup", { ...env, BREV_DELETE_TIMEOUT_SECONDS: "1", BREV_POLL_SECONDS: "1", }); - - expect(cleanup.status).not.toBe(0); - expect(cleanup.stderr).toContain("Brev workspace still exists after cleanup deadline"); - }); - - it("rejects an empty provision SHA before onboarding", () => { - const { env, log } = fixture({ provisionSha: "" }); - expect(run("deploy", env).status).toBe(0); - const qualification = run("qualify", env); - expect(qualification.status).not.toBe(0); - expect(qualification.stderr).toContain("provision metadata SHA must be a lowercase Git SHA"); - expect(fs.readFileSync(log, "utf8")).not.toContain("test/e2e/live/full-e2e.test.ts"); - expect(run("cleanup", env).status).toBe(0); + expect(result.status).not.toBe(0); + expect(result.stderr).toContain("still exists"); }); }); diff --git a/test/e2e/README.md b/test/e2e/README.md index b05af6c6b4..1726fccfea 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -85,50 +85,23 @@ artifact so baseline aggregation stays stable. Older issue references to Vitest target artifacts under `e2e-artifacts/vitest/` map to this consolidated `e2e-artifacts/live/` registry-target artifact layout. -## Exact staging Brev Launchable qualification - -The exact-image staging Launchable lane is inactive by default. Only scheduled -and manual `e2e.yaml` runs on `main` with an empty `checkout_sha` may call it, -and only when the repository variable -`NEMOCLAW_BREV_LAUNCHABLE_QUALIFICATION_ENABLED` is exactly `true`. PR-gate -runs cannot enter this credentialed lane. A direct dispatch of -`brev-launchable-qualification.yaml` applies the same repository, ref, and -activation checks and requires `candidate_sha` to equal the trusted current -`main` workflow SHA. While inactive, it fails before checkout, environment -approval, credential access, or external API calls. -The skipped `staging-brev-launchable` job in a routine E2E run is an inactive -status, not successful qualification evidence. - -Before setting the activation variable, repository owners must: - -- create and protect the `approve-brev-launchable-qualification` environment - for `main`, with the required reviewers; -- configure that environment with `NEMOCLAW_IMAGE_DISPATCH_TOKEN`, - `BREV_API_KEY`, `BREV_ORG_ID`, and `NVIDIA_INFERENCE_API_KEY`; -- set the environment or repository variable `NEMOCLAW_STAGING_LAUNCHABLE_ID` - to the standing staging Launchable that consumes the staging image family; - and -- verify that the producer workflow and the Brev Launchable contract are ready - for automated image builds, provisioning, validation, and cleanup. - -Set `NEMOCLAW_BREV_LAUNCHABLE_QUALIFICATION_ENABLED=true` only after those -conditions are met. Remove the variable, or set it to any value other than -`true`, to keep routine E2E runs from starting this cost-bearing lane. - -The runtime target is `brev-launchable-cloud-openclaw`. After proving the -workspace booted the accepted image and exact NemoClaw SHA, the lane runs the -existing `test/e2e/live/full-e2e.test.ts` suite in `preinstalled-launchable` -setup mode. That mode onboards through the baked `brev-quickstart` entry point -and reuses the suite's CLI, policy, real first-agent-turn, hosted-inference, -`inference.local`, security-posture, logs, and cleanup assertions. Before the -trusted harness runs, the runtime rejects tracked checkout drift and unsafe -image-derived model IDs. Copied evidence must contain exactly the canonical -passing target result, stay within fixed file and byte limits, contain no -symlinks or special files, and produce a recorded SHA-256 digest. Recursive -redaction must succeed before evidence can be uploaded. The lane fails if the -baked checkout does not already contain the pinned Vitest harness; it does not -rsync source, run `install.sh`, install packages, rebuild, or relink the -product under test. +## Brev Launchable E2E + +Select `jobs=staging-brev-launchable` in a trusted manual E2E dispatch. The +`approve-brev-launchable-e2e` environment protects the image-builder, Brev, +and inference credentials used by this cost-bearing lane. Configure +`NEMOCLAW_STAGING_LAUNCHABLE_ID` as the standing staging Launchable and provide +the `NEMOCLAW_IMAGE_DISPATCH_TOKEN`, `BREV_API_KEY`, `BREV_ORG_ID`, and +`NVIDIA_INFERENCE_API_KEY` repository secrets before approving the run. + +The job asks the existing image workflow to build or reuse the staging CPU +image for the exact selected NemoClaw SHA, deploys the configured standing +Launchable, and verifies the workspace's baked checkout and actual GCP boot +image labels before testing. It then runs +`test/e2e/live/full-e2e.test.ts` in `preinstalled-launchable` mode through the +baked `brev-quickstart` entry point and always deletes the workspace. The job +does not rsync source, invoke `install.sh`, install dependencies, rebuild, or +relink NemoClaw. ## PR E2E gate diff --git a/test/e2e/mock-parity.json b/test/e2e/mock-parity.json index 4e02191be1..f567549bdf 100644 --- a/test/e2e/mock-parity.json +++ b/test/e2e/mock-parity.json @@ -28,7 +28,6 @@ "live": "test/e2e/live/full-e2e.test.ts", "fast": [ "test/brev-launchable-runtime.test.ts", - "test/e2e/support/exact-image-qualification-workflow.test.ts", "test/e2e/support/onboard-performance.test.ts", "test/e2e/support/e2e-cleanup-resources.test.ts", "test/e2e/support/e2e-clients.test.ts" diff --git a/test/e2e/support/exact-image-manifest-cli.test.ts b/test/e2e/support/exact-image-manifest-cli.test.ts deleted file mode 100644 index fe0ac65037..0000000000 --- a/test/e2e/support/exact-image-manifest-cli.test.ts +++ /dev/null @@ -1,158 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { spawnSync } from "node:child_process"; -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; -import { afterEach, describe, expect, it } from "vitest"; - -import { normalizedExactImageManifestJson } from "../../../tools/e2e/exact-image-manifest.mts"; -import { - CANDIDATE_SHA, - CORRELATION_ID, - exactImageManifest, - IMAGE_REPOSITORY_SHA, -} from "./exact-image-manifest-fixture.ts"; - -const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); -const CLI = path.join(REPO_ROOT, "tools/e2e/validate-exact-image-manifest.mts"); -const tempDirs: string[] = []; - -afterEach(() => { - for (const directory of tempDirs.splice(0)) { - fs.rmSync(directory, { recursive: true, force: true }); - } -}); - -function tempDir(): string { - const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-image-manifest-")); - tempDirs.push(directory); - return directory; -} - -function args(manifest: string, output: string, overrides: Record = {}): string[] { - const values = { - "--manifest": manifest, - "--output": output, - "--nemoclaw-sha": CANDIDATE_SHA, - "--requester-run-id": "8001", - "--requester-run-attempt": "1", - "--correlation-id": CORRELATION_ID, - "--image-repository-sha": IMAGE_REPOSITORY_SHA, - "--producer-run-id": "9002", - "--producer-run-attempt": "1", - ...overrides, - }; - return Object.entries(values).flat(); -} - -function runCli(cliArgs: string[]) { - return spawnSync(process.execPath, ["--experimental-strip-types", CLI, ...cliArgs], { - cwd: REPO_ROOT, - encoding: "utf8", - env: { ...process.env, NODE_NO_WARNINGS: "1" }, - }); -} - -function runCliWithTsx(cliArgs: string[]) { - return spawnSync(process.execPath, ["--import", "tsx", CLI, ...cliArgs], { - cwd: REPO_ROOT, - encoding: "utf8", - env: { ...process.env, NODE_NO_WARNINGS: "1" }, - }); -} - -describe("exact staging image manifest CLI", () => { - it("writes only normalized accepted JSON with private permissions", () => { - const directory = tempDir(); - const input = path.join(directory, "nemoclaw-image-manifest.v1.json"); - const output = path.join(directory, "accepted.json"); - const manifest = exactImageManifest(); - fs.writeFileSync(input, JSON.stringify(manifest), "utf8"); - - const result = runCli(args(input, output)); - - expect(result.status).toBe(0); - expect(result.stderr).toBe(""); - expect(fs.readFileSync(output, "utf8")).toBe(normalizedExactImageManifestJson(manifest)); - expect(fs.statSync(output).mode & 0o777).toBe(0o600); - }); - - it("loads through the repository tsx runtime", () => { - const directory = tempDir(); - const input = path.join(directory, "nemoclaw-image-manifest.v1.json"); - const output = path.join(directory, "accepted.json"); - fs.writeFileSync(input, JSON.stringify(exactImageManifest()), "utf8"); - - const result = runCliWithTsx(args(input, output)); - - expect(result.status).toBe(0); - expect(result.stderr).toBe(""); - expect(fs.existsSync(output)).toBe(true); - }); - - it("reports a stable provenance failure code and leaves no accepted output", () => { - const directory = tempDir(); - const input = path.join(directory, "manifest.json"); - const output = path.join(directory, "accepted.json"); - fs.writeFileSync(input, JSON.stringify(exactImageManifest()), "utf8"); - - const result = runCli(args(input, output, { "--producer-run-id": "9003" })); - - expect(result.status).toBe(1); - expect(result.stderr).toMatch(/^PROVENANCE_MISMATCH: workflowRunId/u); - expect(fs.existsSync(output)).toBe(false); - }); - - it("rejects symlinked, oversized, and non-UTF-8 inputs", () => { - const directory = tempDir(); - const valid = path.join(directory, "valid.json"); - fs.writeFileSync(valid, JSON.stringify(exactImageManifest()), "utf8"); - const symlink = path.join(directory, "symlink.json"); - fs.symlinkSync(valid, symlink); - - const oversized = path.join(directory, "oversized.json"); - fs.writeFileSync(oversized, Buffer.alloc(64 * 1024 + 1, 0x20)); - - const invalidUtf8 = path.join(directory, "invalid-utf8.json"); - fs.writeFileSync(invalidUtf8, Buffer.from([0xc3, 0x28])); - - for (const [name, input, message] of [ - ["symlink", symlink, "manifest input could not be opened safely"], - ["oversized", oversized, "manifest input exceeds 65536 bytes"], - ["non-UTF-8", invalidUtf8, "manifest input must be valid UTF-8"], - ]) { - const output = path.join(directory, `${name}-accepted.json`); - const result = runCli(args(input, output)); - expect(result.status, name).toBe(1); - expect(result.stderr, name).toContain(`ARTIFACT_MISSING_OR_INVALID: ${message}`); - expect(fs.existsSync(output), name).toBe(false); - } - }); - - it("refuses to replace a symlinked accepted-output path", () => { - const directory = tempDir(); - const input = path.join(directory, "manifest.json"); - const target = path.join(directory, "target.json"); - const output = path.join(directory, "accepted.json"); - fs.writeFileSync(input, JSON.stringify(exactImageManifest()), "utf8"); - fs.writeFileSync(target, "unchanged\n", "utf8"); - fs.symlinkSync(target, output); - - const result = runCli(args(input, output)); - - expect(result.status).toBe(1); - expect(result.stderr).toBe( - "OUTPUT_WRITE_FAILED: accepted manifest output could not be written safely\n", - ); - expect(fs.readFileSync(target, "utf8")).toBe("unchanged\n"); - }); - - it("reports invalid or incomplete invocation as a request failure", () => { - const result = runCli(["--manifest", "manifest.json"]); - - expect(result.status).toBe(1); - expect(result.stderr).toBe("REQUEST_INVALID: --output is required\n"); - }); -}); diff --git a/test/e2e/support/exact-image-manifest-fixture.ts b/test/e2e/support/exact-image-manifest-fixture.ts deleted file mode 100644 index 5c6ab66f8e..0000000000 --- a/test/e2e/support/exact-image-manifest-fixture.ts +++ /dev/null @@ -1,61 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import type { - ExactImageManifest, - ExactImageManifestExpectations, -} from "../../../tools/e2e/exact-image-manifest.mts"; - -export const CANDIDATE_SHA = "a".repeat(40); -export const IMAGE_REPOSITORY_SHA = "b".repeat(40); -export const CORRELATION_ID = "12345678-1234-4123-8123-123456789abc"; - -export function exactImageManifest( - overrides: Partial = {}, -): ExactImageManifest { - return { - schemaVersion: 1, - kind: "nemoclaw-exact-image-manifest", - correlationId: CORRELATION_ID, - requesterRepository: "NVIDIA/NemoClaw", - requesterWorkflowRunId: "8001", - requesterWorkflowRunAttempt: 1, - nemoclawSha: CANDIDATE_SHA, - imageRepository: "brevdev/nemoclaw-image", - imageRepositorySha: IMAGE_REPOSITORY_SHA, - producerWorkflow: ".github/workflows/build-qualification-image.yml", - workflowRunId: "9002", - workflowRunAttempt: 1, - imageOriginWorkflowRunId: "9002", - imageOriginWorkflowRunAttempt: 1, - imageKind: "compute#image", - project: "brevdevprod", - imageName: "nemoclaw-brev-cpu-v0-1-0-20260716-a-staging-190-1", - imageId: "12345678901234567890", - imageSelfLink: - "https://www.googleapis.com/compute/v1/projects/brevdevprod/global/images/nemoclaw-brev-cpu-v0-1-0-20260716-a-staging-190-1", - status: "READY", - imageCreationTimestamp: "2026-07-16T12:00:00.000Z", - manifestCreatedAt: "2026-07-16T12:01:00.000Z", - channel: "staging", - variant: "cpu", - observedFamily: "nemoclaw-brev-staging-cpu", - result: "built", - ...overrides, - }; -} - -export function exactImageManifestExpectations( - overrides: Partial = {}, -): ExactImageManifestExpectations { - return { - correlationId: CORRELATION_ID, - requesterWorkflowRunId: "8001", - requesterWorkflowRunAttempt: 1, - nemoclawSha: CANDIDATE_SHA, - imageRepositorySha: IMAGE_REPOSITORY_SHA, - workflowRunId: "9002", - workflowRunAttempt: 1, - ...overrides, - }; -} diff --git a/test/e2e/support/exact-image-manifest.test.ts b/test/e2e/support/exact-image-manifest.test.ts deleted file mode 100644 index 1b784bba7a..0000000000 --- a/test/e2e/support/exact-image-manifest.test.ts +++ /dev/null @@ -1,253 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { describe, expect, it } from "vitest"; - -import { - EXACT_IMAGE_MANIFEST_MAX_CLOCK_SKEW_MS, - ExactImageManifestError, - normalizedExactImageManifestJson, - parseAndValidateExactImageManifest, - validateExactImageManifest, -} from "../../../tools/e2e/exact-image-manifest.mts"; -import { - CANDIDATE_SHA, - CORRELATION_ID, - exactImageManifest, - exactImageManifestExpectations, - IMAGE_REPOSITORY_SHA, -} from "./exact-image-manifest-fixture.ts"; - -function expectCode(run: () => unknown, code: ExactImageManifestError["code"]): void { - try { - run(); - } catch (error) { - expect(error).toBeInstanceOf(ExactImageManifestError); - expect((error as ExactImageManifestError).code).toBe(code); - return; - } - throw new Error(`expected ${code}`); -} - -describe("exact staging image manifest consumer", () => { - it("accepts and normalizes one exact built CPU image", () => { - const source = exactImageManifest(); - const accepted = validateExactImageManifest(source, exactImageManifestExpectations()); - - expect(accepted).toEqual(source); - expect(accepted.imageId).toBe("12345678901234567890"); - expect(normalizedExactImageManifestJson(accepted)).toBe(`${JSON.stringify(source, null, 2)}\n`); - }); - - it("accepts a reused image with required origin evidence at the 24-hour boundary", () => { - const reused = exactImageManifest({ - result: "reused", - imageOriginWorkflowRunId: "7000", - imageOriginWorkflowRunAttempt: 3, - imageCreationTimestamp: "2026-07-15T12:01:00.000Z", - }); - - expect(validateExactImageManifest(reused, exactImageManifestExpectations())).toEqual(reused); - }); - - it("requires reused manifests to carry both origin fields", () => { - for (const field of ["imageOriginWorkflowRunId", "imageOriginWorkflowRunAttempt"] as const) { - const reused = { ...exactImageManifest({ result: "reused" }) } as Record; - delete reused[field]; - expectCode( - () => validateExactImageManifest(reused, exactImageManifestExpectations()), - "ARTIFACT_MISSING_OR_INVALID", - ); - } - }); - - it("rejects missing and additional fields", () => { - const missing = { ...exactImageManifest() } as Record; - delete missing.imageId; - expectCode( - () => validateExactImageManifest(missing, exactImageManifestExpectations()), - "ARTIFACT_MISSING_OR_INVALID", - ); - - const additional = { ...exactImageManifest(), mutableImageFamilyFallback: true }; - expectCode( - () => validateExactImageManifest(additional, exactImageManifestExpectations()), - "ARTIFACT_MISSING_OR_INVALID", - ); - }); - - it.each([ - ["uppercase candidate SHA", { nemoclawSha: CANDIDATE_SHA.toUpperCase() }], - ["short image repository SHA", { imageRepositorySha: IMAGE_REPOSITORY_SHA.slice(0, 12) }], - ["uppercase correlation UUID", { correlationId: CORRELATION_ID.toUpperCase() }], - ["zero requester run ID", { requesterWorkflowRunId: "0" }], - ["numeric image ID", { imageId: 123456789 }], - ["zero image ID", { imageId: "0" }], - ["fractional workflow attempt", { workflowRunAttempt: 1.5 }], - ])("rejects an invalid %s", (_name, overrides) => { - expectCode( - () => - validateExactImageManifest( - { ...exactImageManifest(), ...overrides }, - exactImageManifestExpectations(), - ), - "ARTIFACT_MISSING_OR_INVALID", - ); - }); - - it.each([ - ["requester repository", { requesterRepository: "somewhere/NemoClaw" }], - ["image repository", { imageRepository: "brevdev/other-image" }], - ["producer workflow", { producerWorkflow: ".github/workflows/build-image.yml" }], - ["image kind", { imageKind: "compute#family" }], - ["status", { status: "PENDING" }], - ["channel", { channel: "production" }], - ["observed family", { observedFamily: "nemoclaw-brev-production-cpu" }], - ])("rejects the wrong fixed %s", (_name, overrides) => { - expectCode( - () => - validateExactImageManifest( - { ...exactImageManifest(), ...overrides }, - exactImageManifestExpectations(), - ), - "PROVENANCE_MISMATCH", - ); - }); - - it("rejects GPU before any later dispatch can consume it", () => { - expectCode( - () => - validateExactImageManifest( - { - ...exactImageManifest(), - variant: "gpu", - observedFamily: "nemoclaw-brev-staging-gpu", - }, - exactImageManifestExpectations(), - ), - "UNSUPPORTED_VARIANT", - ); - }); - - it("requires an immutable self-link reconstructed from project and image name", () => { - const canonical = exactImageManifest().imageSelfLink; - for (const imageSelfLink of [ - "https://www.googleapis.com/compute/v1/projects/brevdevprod/global/images/family/nemoclaw-brev-staging-cpu", - "https://www.googleapis.com/compute/v1/projects/other/global/images/nemoclaw-brev-cpu-v0-1-0-20260716-a-staging-190-1", - `${canonical}?alt=json`, - `${canonical}#fragment`, - `${canonical}\n`, - ]) { - expectCode( - () => - validateExactImageManifest( - exactImageManifest({ imageSelfLink }), - exactImageManifestExpectations(), - ), - "IMAGE_IDENTITY_MISMATCH", - ); - } - }); - - it("rejects noncanonical GCP project identifiers even when the self-link repeats them", () => { - for (const project of ["short", "-leading", "trailing-", "UPPERCASE", "evil?x=y#z\nproject"]) { - const imageName = exactImageManifest().imageName; - expectCode( - () => - validateExactImageManifest( - exactImageManifest({ - project, - imageSelfLink: `https://www.googleapis.com/compute/v1/projects/${project}/global/images/${imageName}`, - }), - exactImageManifestExpectations(), - ), - "IMAGE_IDENTITY_MISMATCH", - ); - } - }); - - it("binds the manifest to the trusted request and producer run", () => { - const expectationMismatches = [ - { nemoclawSha: "c".repeat(40) }, - { requesterWorkflowRunId: "8002" }, - { requesterWorkflowRunAttempt: 2 }, - { correlationId: "87654321-4321-4321-8321-cba987654321" }, - { imageRepositorySha: "d".repeat(40) }, - { workflowRunId: "9003" }, - { workflowRunAttempt: 2 }, - ]; - for (const expected of expectationMismatches) { - expectCode( - () => - validateExactImageManifest( - exactImageManifest(), - exactImageManifestExpectations(expected), - ), - "PROVENANCE_MISMATCH", - ); - } - }); - - it("requires built images to originate in the current run", () => { - expectCode( - () => - validateExactImageManifest( - exactImageManifest({ imageOriginWorkflowRunId: "7000" }), - exactImageManifestExpectations(), - ), - "PROVENANCE_MISMATCH", - ); - }); - - it("enforces real timestamps, bounded clock skew, and the reused-image age", () => { - expectCode( - () => - validateExactImageManifest( - exactImageManifest({ imageCreationTimestamp: "2026-02-30T12:00:00Z" }), - exactImageManifestExpectations(), - ), - "ARTIFACT_MISSING_OR_INVALID", - ); - - const beyondSkew = new Date( - Date.parse("2026-07-16T12:01:00.000Z") + EXACT_IMAGE_MANIFEST_MAX_CLOCK_SKEW_MS + 1, - ).toISOString(); - const atSkew = new Date( - Date.parse("2026-07-16T12:01:00.000Z") + EXACT_IMAGE_MANIFEST_MAX_CLOCK_SKEW_MS, - ).toISOString(); - expect( - validateExactImageManifest( - exactImageManifest({ imageCreationTimestamp: atSkew }), - exactImageManifestExpectations(), - ).imageCreationTimestamp, - ).toBe(atSkew); - expectCode( - () => - validateExactImageManifest( - exactImageManifest({ imageCreationTimestamp: beyondSkew }), - exactImageManifestExpectations(), - ), - "PROVENANCE_MISMATCH", - ); - - expectCode( - () => - validateExactImageManifest( - exactImageManifest({ - result: "reused", - imageOriginWorkflowRunId: "7000", - imageCreationTimestamp: "2026-07-15T12:00:59.999Z", - }), - exactImageManifestExpectations(), - ), - "PROVENANCE_MISMATCH", - ); - }); - - it("rejects invalid JSON before semantic validation", () => { - expectCode( - () => parseAndValidateExactImageManifest("{not-json", exactImageManifestExpectations()), - "ARTIFACT_MISSING_OR_INVALID", - ); - }); -}); diff --git a/test/e2e/support/exact-image-qualification-workflow.test.ts b/test/e2e/support/exact-image-qualification-workflow.test.ts deleted file mode 100644 index 4c691c99bf..0000000000 --- a/test/e2e/support/exact-image-qualification-workflow.test.ts +++ /dev/null @@ -1,258 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { spawnSync } from "node:child_process"; -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; - -import { expect, it } from "vitest"; - -import { - readRepoText, - readYaml, - type Workflow, - type WorkflowJob, -} from "../../helpers/e2e-workflow-contract"; - -const WORKFLOW_PATH = ".github/workflows/brev-launchable-qualification.yaml"; -const E2E_WORKFLOW_PATH = ".github/workflows/e2e.yaml"; -const CONTROLLER_PATH = "tools/e2e/exact-image-qualification-controller.mts"; -const RUNTIME_PATH = "tools/e2e/brev-launchable-runtime.sh"; -const FULL_E2E_PATH = "test/e2e/live/full-e2e.test.ts"; -const ACTIVATION_VARIABLE = "NEMOCLAW_BREV_LAUNCHABLE_QUALIFICATION_ENABLED"; - -type QualificationWorkflow = Workflow & { - name: string; - on: Record; - permissions: Record; - concurrency: { group: string; "cancel-in-progress": boolean }; -}; - -function strings(value: unknown): string[] { - return typeof value === "string" - ? [value] - : Array.isArray(value) - ? value.flatMap(strings) - : value && typeof value === "object" - ? Object.values(value).flatMap(strings) - : []; -} - -function job(workflow: Workflow, name: string): WorkflowJob { - const value = workflow.jobs[name]; - expect(value, `missing ${name} job`).toBeDefined(); - return value!; -} - -// source-shape-contract: security -- Exact-image qualification must remain manual/reusable, protected, fixed-target, least-privilege, identity-gated, and cleanup-verifying -it("keeps exact-image Launchable qualification protected, reusable, and fail-closed", () => { - const workflow = readYaml(WORKFLOW_PATH); - const e2eWorkflow = readYaml(E2E_WORKFLOW_PATH); - const source = readRepoText(WORKFLOW_PATH); - const controller = readRepoText(CONTROLLER_PATH); - const runtime = readRepoText(RUNTIME_PATH); - const fullE2e = readRepoText(FULL_E2E_PATH); - const preflight = job(workflow, "preflight"); - const qualify = job(workflow, "qualify"); - const caller = job(e2eWorkflow, "staging-brev-launchable"); - const workflowStrings = strings(workflow); - const steps = qualify.steps ?? []; - const redactIndex = steps.findIndex((step) => step.name === "Redact runtime evidence"); - const uploadIndex = steps.findIndex( - (step) => step.name === "Upload exact staging Launchable evidence", - ); - - expect(redactIndex).toBeGreaterThanOrEqual(0); - expect(uploadIndex).toBeGreaterThan(redactIndex); - const redact = steps[redactIndex]!; - const upload = steps[uploadIndex]!; - expect(redact.if).toBe("${{ always() && steps.workspace.outputs.work_dir != '' }}"); - expect(redact.env).toEqual({ - NVIDIA_INFERENCE_API_KEY: "${{ secrets.NVIDIA_INFERENCE_API_KEY }}", - WORK_DIR: "${{ steps.workspace.outputs.work_dir }}", - }); - expect(redact.run).toContain('content.replace(secret, b"[REDACTED]")'); - expect(redact.run).toContain("os.scandir(directory)"); - expect(redact.run).toContain("child.is_symlink()"); - expect(redact.run).toContain("MAX_TOTAL_BYTES"); - expect(upload.if).toBe( - "${{ always() && steps.workspace.outputs.work_dir != '' && steps.redact-runtime-evidence.outcome == 'success' }}", - ); - for (const path of ["brev-launchable-e2e.log", "brev-launchable-cloud-openclaw"]) { - expect(String(upload.with?.path), `retained evidence must include ${path}`).toContain( - `/${path}`, - ); - } - - expect(workflow.name).toBe("E2E / Exact Staging Brev Launchable"); - expect(Object.keys(workflow.on)).toEqual(["workflow_dispatch", "workflow_call"]); - expect(source).not.toMatch(/^\s+(?:push|schedule|workflow_run|pull_request):/mu); - expect(Object.keys((workflow.on.workflow_dispatch as { inputs: object }).inputs)).toEqual([ - "candidate_sha", - "reason", - ]); - expect(Object.keys((workflow.on.workflow_call as { inputs: object }).inputs)).toEqual([ - "candidate_sha", - "reason", - ]); - expect((workflow.on.workflow_call as { secrets?: object }).secrets).toBeUndefined(); - expect(JSON.stringify(workflow.on)).not.toContain(ACTIVATION_VARIABLE); - expect(workflow.permissions).toEqual({}); - expect(caller.secrets).toBeUndefined(); - expect(caller.if).toBe( - `\${{ vars.${ACTIVATION_VARIABLE} == 'true' && github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && inputs.checkout_sha == '' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') }}`, - ); - expect(caller.with?.candidate_sha).toBe("${{ github.sha }}"); - expect(workflow.concurrency).toEqual({ - group: "brev-launchable-qualification-staging-cpu", - "cancel-in-progress": false, - }); - - expect(preflight.permissions).toEqual({ contents: "read" }); - expect(preflight.if).toBeUndefined(); - expect(preflight.environment).toBeUndefined(); - const activation = preflight.steps?.[0]; - expect(activation?.name).toBe("Require explicit repository activation"); - expect(activation?.uses).toBeUndefined(); - expect(activation?.env).toEqual({ - QUALIFICATION_ENABLED: `\${{ vars.${ACTIVATION_VARIABLE} }}`, - }); - expect(activation?.run).toContain('[[ "$QUALIFICATION_ENABLED" != "true" ]]'); - expect(activation?.run).toContain(`${ACTIVATION_VARIABLE}=true`); - expect(activation?.run).toContain("exit 1"); - expect(qualify.permissions).toEqual({ contents: "read" }); - expect(qualify.if).toBe( - `\${{ vars.${ACTIVATION_VARIABLE} == 'true' && github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' }}`, - ); - expect(qualify.environment).toEqual({ - name: "approve-brev-launchable-qualification", - deployment: false, - }); - const environmentJobs = Object.values(workflow.jobs).filter( - (workflowJob) => workflowJob.environment !== undefined, - ); - expect(environmentJobs).toEqual([qualify]); - for (const environmentJob of environmentJobs) { - expect(environmentJob.if).toContain(`vars.${ACTIVATION_VARIABLE} == 'true'`); - } - for (const secret of [ - "NEMOCLAW_IMAGE_DISPATCH_TOKEN", - "BREV_API_KEY", - "BREV_ORG_ID", - "NVIDIA_INFERENCE_API_KEY", - ]) { - expect(JSON.stringify(preflight)).not.toContain(`secrets.${secret}`); - expect(JSON.stringify(qualify)).toContain(`secrets.${secret}`); - } - expect(JSON.stringify(qualify)).toContain("NEMOCLAW_IMAGE_QUALIFICATION_TOKEN"); - expect(source.match(/secrets\.NEMOCLAW_IMAGE_DISPATCH_TOKEN/gu)).toHaveLength(4); - expect(workflowStrings).not.toContain("id-token: write"); - expect(source).not.toMatch(/npm (?:ci|install)/u); - for (const step of [...(preflight.steps ?? []), ...(qualify.steps ?? [])]) { - expect(step.run ?? "").not.toContain("${{ inputs."); - } - - const actionUses = workflowStrings.filter((value) => value.startsWith("actions/")); - expect(actionUses.length).toBeGreaterThan(0); - for (const use of actionUses) expect(use).toMatch(/^actions\/[a-z-]+@[0-9a-f]{40}$/u); - for (const checkout of qualify.steps?.filter((step) => - step.uses?.startsWith("actions/checkout@"), - ) ?? []) { - expect(checkout.with).toMatchObject({ - ref: "${{ github.workflow_sha }}", - "persist-credentials": false, - }); - } - - expect(controller).toContain('export const PRODUCER_REPOSITORY = "brevdev/nemoclaw-image"'); - expect(controller).toContain( - 'export const PRODUCER_WORKFLOW_FILE = "build-qualification-image.yml"', - ); - expect(controller).toContain('export const PRODUCER_REF = "main"'); - expect(controller).toContain('request.ref !== "refs/heads/main"'); - expect(controller).toContain("request.candidateSha !== request.workflowSha"); - expect(controller).toContain('export const GITHUB_API_VERSION = "2026-03-10"'); - expect(controller).toContain("return_run_details: true"); - expect(controller).toContain("fs.renameSync(temporary, file)"); - expect(controller).not.toMatch(/actions\/runs\?/u); - expect(controller).toContain("actions/workflows/${PRODUCER_WORKFLOW_FILE}/runs"); - - const validate = qualify.steps?.find((step) => step.name === "Validate the exact image manifest"); - expect(validate?.run).toContain("tools/e2e/validate-exact-image-manifest.mts"); - for (const flag of [ - "--nemoclaw-sha", - "--requester-run-id", - "--requester-run-attempt", - "--correlation-id", - "--image-repository-sha", - "--producer-run-id", - "--producer-run-attempt", - ]) { - expect(validate?.run).toContain(flag); - } - - expect(source).toContain("retention-days: 90"); - expect(source).toContain("if-no-files-found: error"); - expect(source).toContain("dispatch-intent.v1.json"); - expect(source).toContain("dispatch-reconciliation.v1.json"); - expect(source).toContain("controller-state.corrupt-*.json"); - expect(source).toContain("--mode finalize"); - expect(runtime).toContain("brev create"); - expect(runtime).toContain("--launchable"); - expect(runtime).toContain("test/e2e/live/full-e2e.test.ts"); - expect(runtime).toContain("NEMOCLAW_E2E_SETUP_MODE=preinstalled-launchable"); - expect(runtime).toContain("NEMOCLAW_E2E_SECURITY_POSTURE=1"); - expect(runtime).toContain("diff --quiet --no-ext-diff HEAD --"); - expect(runtime).toContain("validate_copied_artifact_tree"); - expect(runtime).toContain("targetResultSha256"); - expect(runtime).toContain("firstAgentTurn"); - expect(runtime).not.toContain("brev-quickstart"); - expect(fullE2e).toContain('host.command("brev-quickstart"'); - expect(fullE2e).toContain('"brev-launchable-cloud-openclaw"'); - expect(fullE2e).toContain("assertFirstAgentTurn({ apiKey: hosted.apiKey, sandbox })"); - expect(fullE2e).toMatch( - /expect\(assistantReply,[\s\S]{0,200}\)\.toBe\(\s*EXPECTED_FIRST_REPLY,\s*\);/u, - ); - expect(fullE2e).toContain("firstAgentTurn:"); - expect(source).not.toContain("test/e2e/live/exact-staging-launchable.test.ts"); - expect(source).toContain("brev-launchable-runtime.sh deploy"); - expect(source).toContain("brev-launchable-runtime.sh qualify"); - expect(source).toContain("brev-launchable-runtime.sh cleanup"); - expect(source).toContain("brev-launchable-cloud-openclaw/"); - expect(source).toContain("brev-cleanup-evidence.json"); - expect(source).toContain("NEMOCLAW_STAGING_LAUNCHABLE_ID"); - expect(source).not.toContain("image_family"); -}); - -it("recursively redacts nested runtime evidence and rejects symlinks before upload", () => { - const workflow = readYaml(WORKFLOW_PATH); - const redact = job(workflow, "qualify").steps?.find( - (step) => step.id === "redact-runtime-evidence", - ); - const script = redact?.run as string; - const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-evidence-redaction-")); - const secret = "nvapi-nested-secret"; - - try { - const nested = path.join(root, "brev-launchable-cloud-openclaw", "target", "raw.log"); - fs.mkdirSync(path.dirname(nested), { recursive: true }); - fs.writeFileSync(nested, `before ${secret} after\n`); - const redacted = spawnSync("bash", ["-c", script], { - encoding: "utf8", - env: { ...process.env, NVIDIA_INFERENCE_API_KEY: secret, WORK_DIR: root }, - }); - expect(redacted.status, redacted.stderr).toBe(0); - expect(fs.readFileSync(nested, "utf8")).toBe("before [REDACTED] after\n"); - - fs.symlinkSync(nested, path.join(root, "untrusted-link")); - const rejected = spawnSync("bash", ["-c", script], { - encoding: "utf8", - env: { ...process.env, NVIDIA_INFERENCE_API_KEY: secret, WORK_DIR: root }, - }); - expect(rejected.status).not.toBe(0); - expect(rejected.stderr).toContain("must not contain symlinks"); - } finally { - fs.rmSync(root, { force: true, recursive: true }); - } -}); diff --git a/test/exact-image-qualification-controller.test.ts b/test/exact-image-qualification-controller.test.ts deleted file mode 100644 index d8a7bb01ad..0000000000 --- a/test/exact-image-qualification-controller.test.ts +++ /dev/null @@ -1,1494 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { spawnSync } from "node:child_process"; -import { createHash } from "node:crypto"; -import fs from "node:fs"; -import path from "node:path"; - -import { afterEach, describe, expect, it, vi } from "vitest"; - -import { githubApi } from "../tools/advisors/github.mts"; -import { - ARCHIVE_FILE, - cancelActiveExactImageQualification, - DISPATCH_INTENT_FILE, - DISPATCH_RECONCILIATION_FILE, - downloadExactImageManifest, - EVIDENCE_FILE, - extractExactManifestArchive, - finalizeExactImageQualification, - GITHUB_API_VERSION, - MANIFEST_ARTIFACT_FILE, - PRODUCER_REPOSITORY, - PRODUCER_WORKFLOW_FILE, - PRODUCER_WORKFLOW_PATH, - parseExactImageQualificationCommand, - preflightExactImageQualification, - type QualificationDependencies, - readExactImageQualificationState, - STATE_FILE, - startExactImageQualification, - VALIDATED_MANIFEST_FILE, - validateExactImageQualificationRequest, - validateQualificationArtifactList, - validateQualificationWorkflowRun, - validateWorkflowDispatchDetails, - waitForExactImageQualification, -} from "../tools/e2e/exact-image-qualification-controller.mts"; -import { - API_RUN_URL, - artifactList, - BASE_TIME, - CANDIDATE_SHA, - CORRELATION_ID, - createApi, - createArchiveRunCommand, - createRoutedApi, - dependencies, - dispatchDetails, - dispatchIntent, - HTML_RUN_URL, - PRODUCER_SHA, - qualificationApiRoute, - REQUEST, - RUN_ID, - startedState, - tempDirectory, - WORKFLOW_SHA, - WORKFLOW_ID, - workflowRun, -} from "./helpers/exact-image-qualification-controller-fixture.ts"; - -afterEach(() => { - vi.restoreAllMocks(); - vi.unstubAllGlobals(); -}); - -describe("exact image qualification request", () => { - it("accepts the current trusted main candidate while rejecting malformed boundaries", () => { - expect(validateExactImageQualificationRequest(REQUEST)).toEqual(REQUEST); - expect( - validateExactImageQualificationRequest({ - ...REQUEST, - candidateSha: "c".repeat(40), - eventName: "schedule", - requesterRunAttempt: 2, - }), - ).toMatchObject({ - candidateSha: "c".repeat(40), - eventName: "schedule", - requesterRunAttempt: 2, - }); - expect(() => - validateExactImageQualificationRequest({ ...REQUEST, eventName: "pull_request" }), - ).toThrow(/workflow_dispatch or schedule/u); - expect(() => - validateExactImageQualificationRequest({ ...REQUEST, ref: "refs/pull/1/merge" }), - ).toThrow(/trusted main branch/u); - expect(() => - validateExactImageQualificationRequest({ ...REQUEST, ref: "refs/heads/feature" }), - ).toThrow(/trusted main branch/u); - expect(() => - validateExactImageQualificationRequest({ ...REQUEST, candidateSha: "d".repeat(40) }), - ).toThrow(/must equal the trusted main workflow SHA/u); - expect(() => validateExactImageQualificationRequest({ ...REQUEST, reason: " bad " })).toThrow( - /reason/u, - ); - }); - - it("rejects non-main requests before authorization API access", async () => { - const api = vi.fn(); - - await expect( - preflightExactImageQualification({ ...REQUEST, ref: "refs/heads/feature" }, "core-token", { - api: api as QualificationDependencies["api"], - }), - ).rejects.toMatchObject({ code: "REQUEST_INVALID" }); - expect(api).not.toHaveBeenCalled(); - }); - - it("rejects a candidate other than the trusted workflow SHA before authorization API access", async () => { - const api = vi.fn(); - - await expect( - preflightExactImageQualification({ ...REQUEST, candidateSha: "d".repeat(40) }, "core-token", { - api: api as QualificationDependencies["api"], - }), - ).rejects.toMatchObject({ code: "REQUEST_INVALID" }); - expect(api).not.toHaveBeenCalled(); - }); - - it("parses the fixed CLI surface and rejects undeclared controls", () => { - expect( - parseExactImageQualificationCommand([ - "--mode", - "preflight", - "--actor", - REQUEST.actor, - "--candidate-sha", - CANDIDATE_SHA, - "--event-name", - "workflow_dispatch", - "--reason", - REQUEST.reason, - "--ref", - "refs/heads/main", - "--requester-run-attempt", - "2", - "--requester-run-id", - REQUEST.requesterRunId, - "--workflow-sha", - WORKFLOW_SHA, - ]), - ).toEqual({ mode: "preflight", request: REQUEST }); - expect(() => - parseExactImageQualificationCommand([ - "--mode", - "wait", - "--work-dir", - "/tmp/work", - "--producer-ref", - "feature", - ]), - ).toThrow(/unknown argument/u); - }); -}); - -describe("exact producer dispatch binding", () => { - it("binds the selected candidate to the trusted main workflow SHA", async () => { - expect(REQUEST.candidateSha).toBe(REQUEST.workflowSha); - const { state, workDir } = await startedState(); - try { - expect(readExactImageQualificationState(workDir).request).toMatchObject({ - candidateSha: CANDIDATE_SHA, - workflowSha: WORKFLOW_SHA, - }); - await expect( - waitForExactImageQualification( - { workDir, producerToken: "producer-token" }, - dependencies( - createApi({ run: workflowRun({ status: "completed", conclusion: "success" }) }), - { now: () => BASE_TIME + 1_000 }, - ), - ), - ).resolves.toMatchObject({ id: state.producer.runId, conclusion: "success" }); - } finally { - fs.rmSync(workDir, { recursive: true, force: true }); - } - }); - - it("accepts GitHub's maintain role mapping but rejects ordinary write access", async () => { - const accepted = await startedState(createApi({ permission: "write", roleName: "maintain" })); - fs.rmSync(accepted.workDir, { recursive: true, force: true }); - - const workDir = tempDirectory(); - try { - await expect( - startExactImageQualification( - { - request: REQUEST, - coreToken: "core-token", - producerToken: "producer-token", - workDir, - }, - dependencies(createApi({ permission: "write", roleName: "write" })), - ), - ).rejects.toMatchObject({ code: "DISPATCH_FORBIDDEN" }); - } finally { - fs.rmSync(workDir, { recursive: true, force: true }); - } - }); - - it("uses the 2026 API contract and binds the returned run without listing runs", async () => { - const api = createApi(); - const { state, workDir } = await startedState(api); - try { - const dispatchCall = api.mock.calls.find(([apiPath]) => - String(apiPath).endsWith(`/workflows/${PRODUCER_WORKFLOW_FILE}/dispatches`), - ); - expect(dispatchCall?.[2]).toMatchObject({ - method: "POST", - apiVersion: GITHUB_API_VERSION, - expectedStatus: 200, - body: { - ref: "main", - inputs: { - nemoclaw_sha: CANDIDATE_SHA, - correlation_id: CORRELATION_ID, - requester_workflow_run_id: REQUEST.requesterRunId, - requester_workflow_run_attempt: "2", - }, - return_run_details: true, - }, - signal: expect.any(AbortSignal), - }); - expect(api.mock.calls.some(([apiPath]) => /actions\/runs\?/u.test(String(apiPath)))).toBe( - false, - ); - expect(state.producer.runId).toBe(RUN_ID); - expect(state.producer.repositorySha).toBe(PRODUCER_SHA); - expect(readExactImageQualificationState(workDir)).toEqual(state); - } finally { - fs.rmSync(workDir, { recursive: true, force: true }); - } - }); - - it("keeps the previous state intact when atomic replacement is interrupted", async () => { - const workDir = tempDirectory(); - const previousState = "previous state evidence\n"; - fs.writeFileSync(path.join(workDir, STATE_FILE), previousState, { mode: 0o600 }); - vi.spyOn(fs, "renameSync").mockImplementation(() => { - throw new Error("simulated interruption before atomic replacement"); - }); - try { - await expect( - startExactImageQualification( - { - request: REQUEST, - coreToken: "core-token", - producerToken: "producer-token", - workDir, - }, - dependencies(createApi()), - ), - ).rejects.toMatchObject({ code: "OUTPUT_WRITE_FAILED" }); - expect(fs.readFileSync(path.join(workDir, STATE_FILE), "utf8")).toBe(previousState); - expect(fs.readdirSync(workDir).filter((name) => name.endsWith(".tmp"))).toEqual([]); - } finally { - fs.rmSync(workDir, { recursive: true, force: true }); - } - }); - - it("requires HTTP 200 and sends the explicit API-version header", async () => { - const fetchMock = vi.fn( - async (_input: string | URL | Request, _init?: RequestInit) => - new Response(null, { status: 204 }), - ); - vi.stubGlobal("fetch", fetchMock); - await expect( - githubApi("repos/example/actions/workflows/build.yml/dispatches", "token", { - method: "POST", - apiVersion: GITHUB_API_VERSION, - expectedStatus: 200, - body: { return_run_details: true }, - }), - ).rejects.toThrow(/204/u); - const headers = new Headers(fetchMock.mock.calls[0]?.[1]?.headers); - expect(headers.get("X-GitHub-Api-Version")).toBe(GITHUB_API_VERSION); - }); - - it.each([ - null, - {}, - { workflow_run_id: Number(RUN_ID) }, - ])("fails closed when dispatch returns no complete run details: %j", async (dispatch) => { - const workDir = tempDirectory(); - try { - const api = createApi({ dispatch }); - let clock = BASE_TIME; - await expect( - startExactImageQualification( - { - request: REQUEST, - coreToken: "core-token", - producerToken: "producer-token", - workDir, - }, - dependencies(api, { - now: () => clock, - sleep: async () => { - clock += 2; - }, - limits: { - dispatchReconciliationTimeoutMs: 1, - reconciliationPollIntervalMs: 1, - }, - }), - ), - ).rejects.toMatchObject({ code: "DISPATCH_AMBIGUOUS" }); - } finally { - fs.rmSync(workDir, { recursive: true, force: true }); - } - }); - - it("records, reconciles, cancels, and rejects a server-accepted dispatch whose response is lost", async () => { - const workDir = tempDirectory(); - const base = createApi(); - let cancelObserved = false; - let stateExistedAtCancel = false; - const api = createRoutedApi(base, [ - qualificationApiRoute.dispatch(() => { - throw new Error("response connection reset after server acceptance"); - }), - qualificationApiRoute.workflowRuns(() => ({ - total_count: 1, - workflow_runs: [workflowRun()], - })), - qualificationApiRoute.cancelRun(() => { - stateExistedAtCancel = fs.existsSync(path.join(workDir, STATE_FILE)); - cancelObserved = true; - return undefined; - }), - qualificationApiRoute.run(() => - cancelObserved - ? workflowRun({ status: "completed", conclusion: "cancelled" }) - : workflowRun(), - ), - ]); - try { - await expect( - startExactImageQualification( - { - request: REQUEST, - coreToken: "core-token", - producerToken: "producer-token", - workDir, - }, - dependencies(api as ReturnType), - ), - ).rejects.toMatchObject({ code: "DISPATCH_AMBIGUOUS" }); - expect( - api.mock.calls.filter(([apiPath]) => - String(apiPath).endsWith(`/workflows/${PRODUCER_WORKFLOW_FILE}/dispatches`), - ), - ).toHaveLength(1); - expect(cancelObserved).toBe(true); - expect(stateExistedAtCancel).toBe(true); - expect(readExactImageQualificationState(workDir).producer.runId).toBe(RUN_ID); - expect(fs.existsSync(path.join(workDir, DISPATCH_INTENT_FILE))).toBe(true); - expect( - JSON.parse(fs.readFileSync(path.join(workDir, DISPATCH_RECONCILIATION_FILE), "utf8")), - ).toMatchObject({ outcome: "recovered-one", runIds: [RUN_ID] }); - } finally { - fs.rmSync(workDir, { recursive: true, force: true }); - } - }); - - it("finds and cleans the exact correlation after producer drift without qualifying it", async () => { - const workDir = tempDirectory(); - const movedSha = "c".repeat(40); - const base = createApi(); - let cancelled = false; - const movedRun = workflowRun({ head_sha: movedSha }); - const historicalRuns = Array.from({ length: 101 }, (_value, index) => - workflowRun({ created_at: new Date(BASE_TIME - (index + 2) * 10 * 60_000).toISOString() }), - ); - let observedCreatedFilter: string | null = null; - let observedHeadShaFilter = false; - let observedScopedRuns: unknown[] = []; - const api = createRoutedApi(base, [ - qualificationApiRoute.dispatch(() => { - throw new Error("lost response after main advanced"); - }), - qualificationApiRoute.workflowRuns((apiPath) => { - const query = new URLSearchParams(apiPath.split("?", 2)[1]); - observedCreatedFilter = query.get("created"); - observedHeadShaFilter = apiPath.includes("head_sha="); - const [earliest, latest] = (query.get("created") ?? "").split("..").map(Date.parse); - const scoped = [...historicalRuns, movedRun].filter(({ created_at }) => { - const createdAt = Date.parse(created_at); - return createdAt >= earliest && createdAt <= latest; - }); - observedScopedRuns = scoped; - return { total_count: scoped.length, workflow_runs: scoped }; - }), - qualificationApiRoute.cancelRun(() => { - cancelled = true; - return undefined; - }), - qualificationApiRoute.run(() => - workflowRun({ - head_sha: movedSha, - status: cancelled ? "completed" : "queued", - conclusion: cancelled ? "cancelled" : null, - }), - ), - ]); - try { - await expect( - startExactImageQualification( - { - request: REQUEST, - coreToken: "core-token", - producerToken: "producer-token", - workDir, - }, - dependencies(api as ReturnType), - ), - ).rejects.toMatchObject({ code: "DISPATCH_AMBIGUOUS" }); - expect(cancelled).toBe(true); - expect(observedCreatedFilter).toBe("2025-12-31T23:59:00Z..2026-01-01T00:01:30Z"); - expect(observedHeadShaFilter).toBe(false); - expect(historicalRuns).toHaveLength(101); - expect(observedScopedRuns).toEqual([movedRun]); - expect(readExactImageQualificationState(workDir)).toMatchObject({ - status: "dispatched", - producer: { runId: RUN_ID, repositorySha: movedSha }, - }); - expect( - JSON.parse(fs.readFileSync(path.join(workDir, DISPATCH_RECONCILIATION_FILE), "utf8")), - ).toMatchObject({ - outcome: "recovered-one", - runIds: [RUN_ID], - producerHeadShas: { [RUN_ID]: movedSha }, - }); - expect(fs.existsSync(path.join(workDir, EVIDENCE_FILE))).toBe(false); - } finally { - fs.rmSync(workDir, { recursive: true, force: true }); - } - }); - - it("cleans the response-bound run after producer SHA provenance fails", async () => { - const workDir = tempDirectory(); - const movedSha = "c".repeat(40); - try { - await expect( - startExactImageQualification( - { - request: REQUEST, - coreToken: "core-token", - producerToken: "producer-token", - workDir, - }, - dependencies(createApi({ run: workflowRun({ head_sha: movedSha }) })), - ), - ).rejects.toMatchObject({ code: "PROVENANCE_MISMATCH" }); - expect(readExactImageQualificationState(workDir).producer.runId).toBe(RUN_ID); - - const base = createApi(); - let cancelled = false; - const cleanupApi = createRoutedApi(base, [ - qualificationApiRoute.cancelRun(() => { - cancelled = true; - return undefined; - }), - qualificationApiRoute.run(() => - workflowRun({ - head_sha: movedSha, - status: cancelled ? "completed" : "queued", - conclusion: cancelled ? "cancelled" : null, - }), - ), - ]); - await expect( - cancelActiveExactImageQualification( - { workDir, producerToken: "producer-token" }, - dependencies(cleanupApi as ReturnType), - ), - ).resolves.toBe(true); - expect(cancelled).toBe(true); - expect( - cleanupApi.mock.calls.filter(([apiPath]) => String(apiPath).endsWith("/cancel")), - ).toHaveLength(1); - expect(readExactImageQualificationState(workDir).status).toBe("dispatched"); - expect(fs.existsSync(path.join(workDir, EVIDENCE_FILE))).toBe(false); - } finally { - fs.rmSync(workDir, { recursive: true, force: true }); - } - }); - - it("never accepts a recovered ambiguous dispatch that already completed successfully", async () => { - const workDir = tempDirectory(); - const base = createApi(); - const completed = workflowRun({ status: "completed", conclusion: "success" }); - const api = createRoutedApi(base, [ - qualificationApiRoute.dispatch(() => { - throw new Error("lost response"); - }), - qualificationApiRoute.workflowRuns(() => ({ - total_count: 1, - workflow_runs: [completed], - })), - ]); - try { - await expect( - startExactImageQualification( - { - request: REQUEST, - coreToken: "core-token", - producerToken: "producer-token", - workDir, - }, - dependencies(api as ReturnType), - ), - ).rejects.toMatchObject({ code: "DISPATCH_AMBIGUOUS" }); - expect( - api.mock.calls.some(([apiPath]) => String(apiPath).endsWith(`/${RUN_ID}/cancel`)), - ).toBe(false); - expect(readExactImageQualificationState(workDir).status).toBe("dispatched"); - } finally { - fs.rmSync(workDir, { recursive: true, force: true }); - } - }); - - it("retains the recovered run identity when cancellation fails", async () => { - const workDir = tempDirectory(); - const base = createApi(); - const api = createRoutedApi(base, [ - qualificationApiRoute.dispatch(() => { - throw new Error("lost response"); - }), - qualificationApiRoute.workflowRuns(() => ({ - total_count: 1, - workflow_runs: [workflowRun()], - })), - qualificationApiRoute.cancelRun(() => { - throw new Error("cancel transport failed"); - }), - ]); - try { - await expect( - startExactImageQualification( - { - request: REQUEST, - coreToken: "core-token", - producerToken: "producer-token", - workDir, - }, - dependencies(api as ReturnType), - ), - ).rejects.toThrow(/cancel transport failed/u); - expect(readExactImageQualificationState(workDir).producer.runId).toBe(RUN_ID); - expect( - JSON.parse(fs.readFileSync(path.join(workDir, DISPATCH_RECONCILIATION_FILE), "utf8")), - ).toMatchObject({ outcome: "recovered-one", runIds: [RUN_ID] }); - } finally { - fs.rmSync(workDir, { recursive: true, force: true }); - } - }); - - it("ignores near-match runs and retains a zero-match reconciliation audit", async () => { - const workDir = tempDirectory(); - const base = createApi(); - let clock = BASE_TIME; - const nearMatches = [ - workflowRun({ display_title: "wrong correlation" }), - workflowRun({ workflow_id: WORKFLOW_ID + 1 }), - workflowRun({ run_attempt: 2 }), - workflowRun({ event: "push" }), - workflowRun({ head_branch: "feature" }), - workflowRun({ path: ".github/workflows/other.yml" }), - workflowRun({ repository: { full_name: "other/repository" } }), - workflowRun({ head_repository: { full_name: "other/repository" } }), - workflowRun({ created_at: new Date(BASE_TIME - 2 * 60_000).toISOString() }), - workflowRun({ url: `${API_RUN_URL}/wrong` }), - ]; - const api = createRoutedApi(base, [ - qualificationApiRoute.dispatch(() => { - throw new Error("lost response"); - }), - qualificationApiRoute.workflowRuns(() => ({ - total_count: nearMatches.length, - workflow_runs: nearMatches, - })), - ]); - try { - await expect( - startExactImageQualification( - { - request: REQUEST, - coreToken: "core-token", - producerToken: "producer-token", - workDir, - }, - dependencies(api as ReturnType, { - now: () => clock, - sleep: async () => { - clock += 2; - }, - limits: { - dispatchReconciliationTimeoutMs: 1, - reconciliationPollIntervalMs: 1, - }, - }), - ), - ).rejects.toMatchObject({ code: "DISPATCH_AMBIGUOUS" }); - expect( - api.mock.calls.some(([apiPath]) => String(apiPath).endsWith(`/${RUN_ID}/cancel`)), - ).toBe(false); - expect( - JSON.parse(fs.readFileSync(path.join(workDir, DISPATCH_RECONCILIATION_FILE), "utf8")), - ).toMatchObject({ outcome: "none", runIds: [] }); - } finally { - fs.rmSync(workDir, { recursive: true, force: true }); - } - }); - - it("lets cleanup preserve semantically invalid state and recover only from durable intent", async () => { - const workDir = tempDirectory(); - const untrustedRunId = "99999"; - let startClock = BASE_TIME; - try { - await expect( - startExactImageQualification( - { - request: REQUEST, - coreToken: "core-token", - producerToken: "producer-token", - workDir, - }, - dependencies(createApi({ dispatch: null }), { - now: () => startClock, - sleep: async () => { - startClock += 2; - }, - limits: { - dispatchReconciliationTimeoutMs: 1, - reconciliationPollIntervalMs: 1, - }, - }), - ), - ).rejects.toMatchObject({ code: "DISPATCH_AMBIGUOUS" }); - expect(fs.existsSync(path.join(workDir, STATE_FILE))).toBe(false); - fs.writeFileSync( - path.join(workDir, STATE_FILE), - JSON.stringify({ - schemaVersion: 1, - status: "dispatched", - producer: { runId: untrustedRunId }, - }), - { mode: 0o600 }, - ); - - const base = createApi(); - let cancelled = false; - const cleanupApi = createRoutedApi(base, [ - qualificationApiRoute.workflowRuns(() => ({ - total_count: 1, - workflow_runs: [workflowRun()], - })), - qualificationApiRoute.cancelRun(() => { - cancelled = true; - return undefined; - }), - qualificationApiRoute.run(() => - cancelled ? workflowRun({ status: "completed", conclusion: "cancelled" }) : workflowRun(), - ), - ]); - await expect( - cancelActiveExactImageQualification( - { workDir, producerToken: "producer-token" }, - dependencies(cleanupApi as ReturnType, { - now: () => BASE_TIME + 10_000, - limits: { cleanupTimeoutMs: 10_000 }, - }), - ), - ).resolves.toBe(true); - expect(cancelled).toBe(true); - expect( - cleanupApi.mock.calls.some(([apiPath]) => String(apiPath).includes(untrustedRunId)), - ).toBe(false); - expect(readExactImageQualificationState(workDir).producer.runId).toBe(RUN_ID); - expect( - fs.readdirSync(workDir).some((name) => name.startsWith("controller-state.corrupt-")), - ).toBe(true); - } finally { - fs.rmSync(workDir, { recursive: true, force: true }); - } - }); - - it("binds a valid state tuple to durable intent before any cancellation POST", async () => { - const started = await startedState(); - const untrustedRunId = "99999"; - const untrustedCorrelationId = "87654321-4321-4321-8321-cba987654321"; - const state = readExactImageQualificationState(started.workDir); - state.request.correlationId = untrustedCorrelationId; - state.producer.runId = untrustedRunId; - state.producer.runUrl = `https://api.github.com/repos/${PRODUCER_REPOSITORY}/actions/runs/${untrustedRunId}`; - state.producer.htmlUrl = `https://github.com/${PRODUCER_REPOSITORY}/actions/runs/${untrustedRunId}`; - fs.writeFileSync(path.join(started.workDir, STATE_FILE), `${JSON.stringify(state)}\n`, { - mode: 0o600, - }); - - const base = createApi(); - let cancelled = false; - const cleanupApi = createRoutedApi(base, [ - qualificationApiRoute.workflowRuns(() => ({ - total_count: 1, - workflow_runs: [workflowRun()], - })), - qualificationApiRoute.cancelRun(() => { - cancelled = true; - return undefined; - }), - qualificationApiRoute.run(() => - cancelled ? workflowRun({ status: "completed", conclusion: "cancelled" }) : workflowRun(), - ), - ]); - try { - await expect( - cancelActiveExactImageQualification( - { workDir: started.workDir, producerToken: "producer-token" }, - dependencies(cleanupApi as ReturnType), - ), - ).resolves.toBe(true); - expect(cancelled).toBe(true); - expect( - cleanupApi.mock.calls.some(([apiPath]) => String(apiPath).includes(untrustedRunId)), - ).toBe(false); - expect(readExactImageQualificationState(started.workDir).producer.runId).toBe(RUN_ID); - expect( - fs - .readdirSync(started.workDir) - .some((name) => name.startsWith("controller-state.corrupt-")), - ).toBe(true); - } finally { - fs.rmSync(started.workDir, { recursive: true, force: true }); - } - }); - - it("does not let a forged terminal state suppress cleanup of the active bound run", async () => { - const started = await startedState(); - const state = readExactImageQualificationState(started.workDir); - state.status = "completed"; - state.producer.completedAt = new Date(BASE_TIME + 1_000).toISOString(); - fs.writeFileSync(path.join(started.workDir, STATE_FILE), `${JSON.stringify(state)}\n`, { - mode: 0o600, - }); - - let cancelled = false; - const cleanupApi = createRoutedApi(createApi(), [ - qualificationApiRoute.cancelRun(() => { - cancelled = true; - return undefined; - }), - qualificationApiRoute.run(() => - cancelled - ? workflowRun({ status: "completed", conclusion: "cancelled" }) - : workflowRun({ status: "queued", conclusion: null }), - ), - ]); - try { - await expect( - cancelActiveExactImageQualification( - { workDir: started.workDir, producerToken: "producer-token" }, - dependencies(cleanupApi as ReturnType), - ), - ).resolves.toBe(true); - expect(cancelled).toBe(true); - expect( - cleanupApi.mock.calls.filter(([apiPath]) => String(apiPath).endsWith("/cancel")), - ).toHaveLength(1); - } finally { - fs.rmSync(started.workDir, { recursive: true, force: true }); - } - }); - - it("rejects run-ID-only state corruption before any cancellation POST", async () => { - const started = await startedState(); - const untrustedRunId = "99999"; - const untrustedApiUrl = `https://api.github.com/repos/${PRODUCER_REPOSITORY}/actions/runs/${untrustedRunId}`; - const untrustedHtmlUrl = `https://github.com/${PRODUCER_REPOSITORY}/actions/runs/${untrustedRunId}`; - const state = readExactImageQualificationState(started.workDir); - state.producer.runId = untrustedRunId; - state.producer.runUrl = untrustedApiUrl; - state.producer.htmlUrl = untrustedHtmlUrl; - fs.writeFileSync(path.join(started.workDir, STATE_FILE), `${JSON.stringify(state)}\n`, { - mode: 0o600, - }); - - const cancelAttempt = vi.fn(); - const cleanupApi = createRoutedApi(createApi(), [ - qualificationApiRoute.runAny(() => - workflowRun({ - id: Number(untrustedRunId), - display_title: "Unrelated qualification run", - url: untrustedApiUrl, - html_url: untrustedHtmlUrl, - }), - ), - qualificationApiRoute.cancelAnyRun(cancelAttempt), - ]); - try { - await expect( - cancelActiveExactImageQualification( - { workDir: started.workDir, producerToken: "producer-token" }, - dependencies(cleanupApi as ReturnType), - ), - ).rejects.toMatchObject({ code: "PROVENANCE_MISMATCH" }); - expect(cancelAttempt).not.toHaveBeenCalled(); - expect(cleanupApi.mock.calls.map(([apiPath]) => apiPath)).toEqual([ - `repos/${PRODUCER_REPOSITORY}/actions/runs/${untrustedRunId}`, - ]); - } finally { - fs.rmSync(started.workDir, { recursive: true, force: true }); - } - }); - - it("fails closed when unreadable state cannot be preserved before cleanup reconciliation", async () => { - const workDir = tempDirectory(); - const controllerState = "not valid JSON"; - const api = createApi(); - try { - fs.writeFileSync( - path.join(workDir, DISPATCH_INTENT_FILE), - `${JSON.stringify(dispatchIntent(), null, 2)}\n`, - { mode: 0o600 }, - ); - fs.writeFileSync(path.join(workDir, STATE_FILE), controllerState, { mode: 0o600 }); - vi.spyOn(fs, "renameSync").mockImplementation(() => { - throw new Error("preservation denied"); - }); - - await expect( - cancelActiveExactImageQualification( - { workDir, producerToken: "producer-token" }, - dependencies(api), - ), - ).rejects.toMatchObject({ code: "OUTPUT_WRITE_FAILED" }); - expect(api).not.toHaveBeenCalled(); - expect(fs.readFileSync(path.join(workDir, STATE_FILE), "utf8")).toBe(controllerState); - expect(fs.existsSync(path.join(workDir, DISPATCH_RECONCILIATION_FILE))).toBe(false); - } finally { - fs.rmSync(workDir, { recursive: true, force: true }); - } - }); - - it("fails closed on multiple strict reconciliation matches and records every exact ID", async () => { - const workDir = tempDirectory(); - const secondId = "24681"; - const base = createApi(); - const second = workflowRun({ - id: Number(secondId), - url: `https://api.github.com/repos/${PRODUCER_REPOSITORY}/actions/runs/${secondId}`, - html_url: `https://github.com/${PRODUCER_REPOSITORY}/actions/runs/${secondId}`, - }); - const api = createRoutedApi(base, [ - qualificationApiRoute.dispatch(() => { - throw new Error("lost response"); - }), - qualificationApiRoute.workflowRuns(() => ({ - total_count: 2, - workflow_runs: [workflowRun(), second], - })), - qualificationApiRoute.cancelAnyRun(() => undefined), - ]); - try { - await expect( - startExactImageQualification( - { - request: REQUEST, - coreToken: "core-token", - producerToken: "producer-token", - workDir, - }, - dependencies(api as ReturnType), - ), - ).rejects.toMatchObject({ code: "DISPATCH_AMBIGUOUS" }); - const audit = JSON.parse( - fs.readFileSync(path.join(workDir, DISPATCH_RECONCILIATION_FILE), "utf8"), - ); - expect(audit).toMatchObject({ outcome: "multiple", runIds: [RUN_ID, secondId] }); - expect( - api.mock.calls.filter(([apiPath]) => String(apiPath).endsWith("/cancel")), - ).toHaveLength(2); - } finally { - fs.rmSync(workDir, { recursive: true, force: true }); - } - }); - - it("aborts a GitHub REST call at the configured per-request cap", async () => { - let observedSignal: AbortSignal | undefined; - const api = vi.fn( - async (_apiPath: string, _token: string, requestOptions?: { signal?: AbortSignal }) => - new Promise((_resolve, reject) => { - observedSignal = requestOptions?.signal; - requestOptions?.signal?.addEventListener("abort", () => reject(new Error("aborted")), { - once: true, - }); - }), - ); - await expect( - preflightExactImageQualification(REQUEST, "core-token", { - api: api as QualificationDependencies["api"], - now: Date.now, - limits: { apiRequestTimeoutMs: 5 }, - }), - ).rejects.toMatchObject({ code: "QUALIFICATION_TIMEOUT" }); - expect(observedSignal?.aborted).toBe(true); - }); - - it("validates the exact returned URLs", () => { - expect(validateWorkflowDispatchDetails(dispatchDetails())).toEqual({ - workflowRunId: RUN_ID, - runUrl: API_RUN_URL, - htmlUrl: HTML_RUN_URL, - }); - expect(() => - validateWorkflowDispatchDetails({ - ...dispatchDetails(), - html_url: `${HTML_RUN_URL}/attempts/1`, - }), - ).toThrow(/html_url/u); - }); - - it("rejects producer workflow identity drift", () => { - expect(() => - validateQualificationWorkflowRun(workflowRun({ head_sha: "c".repeat(40) }), { - candidateSha: CANDIDATE_SHA, - correlationId: CORRELATION_ID, - producerSha: PRODUCER_SHA, - runId: RUN_ID, - runUrl: API_RUN_URL, - htmlUrl: HTML_RUN_URL, - }), - ).toThrow(/head SHA/u); - }); -}); - -describe("bound producer polling", () => { - it("accepts success only from the exact dispatched run", async () => { - const { workDir } = await startedState(); - const api = createApi({ run: workflowRun({ status: "completed", conclusion: "success" }) }); - try { - await expect( - waitForExactImageQualification( - { workDir, producerToken: "producer-token" }, - dependencies(api, { now: () => BASE_TIME + 1_000 }), - ), - ).resolves.toMatchObject({ id: RUN_ID, conclusion: "success" }); - expect(readExactImageQualificationState(workDir).status).toBe("completed"); - } finally { - fs.rmSync(workDir, { recursive: true, force: true }); - } - }); - - it("rejects repeated wait from a terminal state before polling or mutation", async () => { - const started = await startedState(); - const state = readExactImageQualificationState(started.workDir); - state.status = "completed"; - state.producer.completedAt = new Date(BASE_TIME + 1_000).toISOString(); - fs.writeFileSync(path.join(started.workDir, STATE_FILE), `${JSON.stringify(state)}\n`, { - mode: 0o600, - }); - const api = createApi(); - try { - await expect( - waitForExactImageQualification( - { workDir: started.workDir, producerToken: "producer-token" }, - dependencies(api), - ), - ).rejects.toMatchObject({ code: "PROVENANCE_MISMATCH" }); - expect(api).not.toHaveBeenCalled(); - expect(readExactImageQualificationState(started.workDir)).toEqual(state); - } finally { - fs.rmSync(started.workDir, { recursive: true, force: true }); - } - }); - - it("rejects persisted completion and acceptance after the hard deadline", async () => { - const started = await startedState(); - const state = readExactImageQualificationState(started.workDir); - state.status = "completed"; - state.producer.completedAt = new Date(BASE_TIME + 45 * 60_000 + 1).toISOString(); - fs.writeFileSync(path.join(started.workDir, STATE_FILE), `${JSON.stringify(state)}\n`, { - mode: 0o600, - }); - try { - expect(() => readExactImageQualificationState(started.workDir)).toThrow( - /completedAt exceeds qualification deadline/u, - ); - - state.status = "validated"; - state.producer.completedAt = new Date(BASE_TIME + 1_000).toISOString(); - state.artifact = { - id: "86420", - name: `nemoclaw-image-handoff-v1-${RUN_ID}-1`, - digest: `sha256:${"0".repeat(64)}`, - sizeInBytes: 1, - apiUrl: `https://api.github.com/repos/${PRODUCER_REPOSITORY}/actions/artifacts/86420`, - archiveDownloadUrl: `https://api.github.com/repos/${PRODUCER_REPOSITORY}/actions/artifacts/86420/zip`, - archiveSha256: "0".repeat(64), - manifestSha256: "1".repeat(64), - }; - state.validation = { - acceptedAt: new Date(BASE_TIME + 45 * 60_000 + 1).toISOString(), - manifestSha256: "1".repeat(64), - normalizedManifestSha256: "2".repeat(64), - }; - fs.writeFileSync(path.join(started.workDir, STATE_FILE), `${JSON.stringify(state)}\n`, { - mode: 0o600, - }); - expect(() => readExactImageQualificationState(started.workDir)).toThrow( - /acceptedAt exceeds qualification deadline/u, - ); - } finally { - fs.rmSync(started.workDir, { recursive: true, force: true }); - } - }); - - it("cancels a run that remains queued beyond the queue budget", async () => { - const { workDir } = await startedState(); - const api = createApi({ run: workflowRun({ status: "queued" }) }); - try { - await expect( - waitForExactImageQualification( - { workDir, producerToken: "producer-token" }, - dependencies(api, { - now: () => BASE_TIME + 10 * 60_000 + 1, - limits: { queueTimeoutMs: 10 * 60_000 }, - }), - ), - ).rejects.toMatchObject({ code: "RUN_QUEUE_TIMEOUT" }); - expect( - api.mock.calls.some( - ([apiPath]) => apiPath === `repos/${PRODUCER_REPOSITORY}/actions/runs/${RUN_ID}/cancel`, - ), - ).toBe(true); - } finally { - fs.rmSync(workDir, { recursive: true, force: true }); - } - }); - - it("rejects a successful producer completion observed at the shared deadline", async () => { - const { workDir } = await startedState(); - const api = createApi({ run: workflowRun({ status: "completed", conclusion: "success" }) }); - try { - await expect( - waitForExactImageQualification( - { workDir, producerToken: "producer-token" }, - dependencies(api, { now: () => BASE_TIME + 45 * 60_000 }), - ), - ).rejects.toMatchObject({ code: "QUALIFICATION_TIMEOUT" }); - expect(readExactImageQualificationState(workDir).status).toBe("dispatched"); - } finally { - fs.rmSync(workDir, { recursive: true, force: true }); - } - }); -}); - -describe("qualification artifact integrity", () => { - it("requires one non-expired digest-bound artifact from the exact run and SHA", async () => { - const archive = Buffer.from("archive"); - const { state, workDir } = await startedState(); - try { - expect(validateQualificationArtifactList(artifactList(archive), state)).toMatchObject({ - id: "86420", - digest: `sha256:${createHash("sha256").update(archive).digest("hex")}`, - }); - expect(() => - validateQualificationArtifactList( - artifactList(archive, { workflow_run: { id: Number(RUN_ID), head_sha: "c".repeat(40) } }), - state, - ), - ).toThrow(/head SHA/u); - expect(() => - validateQualificationArtifactList(artifactList(archive, { digest: null }), state), - ).toThrow(/digest/u); - } finally { - fs.rmSync(workDir, { recursive: true, force: true }); - } - }); - - it("checks the archive digest before accepting the single root manifest entry", async () => { - const archive = Buffer.from("deterministic archive bytes"); - const manifest = Buffer.from('{"schemaVersion":1}\n'); - const started = await startedState(); - const completedApi = createApi({ - run: workflowRun({ status: "completed", conclusion: "success" }), - }); - await waitForExactImageQualification( - { workDir: started.workDir, producerToken: "producer-token" }, - dependencies(completedApi, { now: () => BASE_TIME + 1_000 }), - ); - const artifactApi = createApi({ artifacts: artifactList(archive) }); - const runCommand = vi.fn((command: string, args: readonly string[]) => ({ - status: 0, - stdout: - args[0] === "-Z1" - ? Buffer.from(`${MANIFEST_ARTIFACT_FILE}\n`) - : args[0] === "-Zl" - ? Buffer.from(`-rw-r--r-- 3.0 unx 20 tx 20 stor ${MANIFEST_ARTIFACT_FILE}\n`) - : manifest, - stderr: Buffer.alloc(0), - })); - const fetchMock = vi.fn( - async (_input: string | URL | Request, _init?: RequestInit) => - new Response(archive, { status: 200 }), - ); - try { - await downloadExactImageManifest( - { workDir: started.workDir, producerToken: "producer-token" }, - dependencies(artifactApi, { fetch: fetchMock, runCommand }), - ); - const fetchHeaders = new Headers(fetchMock.mock.calls[0]?.[1]?.headers); - expect(fetchMock.mock.calls[0]?.[0]).toBe( - `https://api.github.com/repos/${PRODUCER_REPOSITORY}/actions/artifacts/86420/zip`, - ); - expect(fetchHeaders.get("X-GitHub-Api-Version")).toBe(GITHUB_API_VERSION); - expect(fs.readFileSync(path.join(started.workDir, ARCHIVE_FILE))).toEqual(archive); - expect(fs.readFileSync(path.join(started.workDir, MANIFEST_ARTIFACT_FILE))).toEqual(manifest); - expect(readExactImageQualificationState(started.workDir)).toMatchObject({ - status: "downloaded", - artifact: { - archiveSha256: createHash("sha256").update(archive).digest("hex"), - manifestSha256: createHash("sha256").update(manifest).digest("hex"), - }, - }); - } finally { - fs.rmSync(started.workDir, { recursive: true, force: true }); - } - }); - - it("rejects persisted artifact metadata whose GitHub digest and archive hash diverge", async () => { - const archive = Buffer.from("deterministic archive bytes"); - const manifest = Buffer.from('{"schemaVersion":1}\n'); - const started = await startedState(); - await waitForExactImageQualification( - { workDir: started.workDir, producerToken: "producer-token" }, - dependencies( - createApi({ run: workflowRun({ status: "completed", conclusion: "success" }) }), - { now: () => BASE_TIME + 1_000 }, - ), - ); - await downloadExactImageManifest( - { workDir: started.workDir, producerToken: "producer-token" }, - dependencies(createApi({ artifacts: artifactList(archive) }), { - fetch: async () => new Response(archive, { status: 200 }), - runCommand: createArchiveRunCommand(manifest), - }), - ); - const state = JSON.parse( - fs.readFileSync(path.join(started.workDir, STATE_FILE), "utf8"), - ) as Record & { artifact: { archiveSha256: string } }; - state.artifact.archiveSha256 = "0".repeat(64); - fs.writeFileSync(path.join(started.workDir, STATE_FILE), `${JSON.stringify(state)}\n`, { - mode: 0o600, - }); - try { - expect(() => readExactImageQualificationState(started.workDir)).toThrow( - /digest does not match archive hash/u, - ); - expect(fs.existsSync(path.join(started.workDir, EVIDENCE_FILE))).toBe(false); - } finally { - fs.rmSync(started.workDir, { recursive: true, force: true }); - } - }); - - it("does not inspect or extract an archive whose digest mismatches GitHub metadata", async () => { - const archive = Buffer.from("archive bytes"); - const started = await startedState(); - await waitForExactImageQualification( - { workDir: started.workDir, producerToken: "producer-token" }, - dependencies( - createApi({ run: workflowRun({ status: "completed", conclusion: "success" }) }), - { now: () => BASE_TIME + 1_000 }, - ), - ); - const runCommand = vi.fn(); - try { - await expect( - downloadExactImageManifest( - { workDir: started.workDir, producerToken: "producer-token" }, - dependencies( - createApi({ artifacts: artifactList(archive, { digest: `sha256:${"0".repeat(64)}` }) }), - { - fetch: async () => new Response(archive, { status: 200 }), - runCommand, - }, - ), - ), - ).rejects.toMatchObject({ code: "ARTIFACT_MISSING_OR_INVALID" }); - expect(runCommand).not.toHaveBeenCalled(); - } finally { - fs.rmSync(started.workDir, { recursive: true, force: true }); - } - }); - - it("caps artifact propagation at the shared qualification deadline", async () => { - const started = await startedState(); - await waitForExactImageQualification( - { workDir: started.workDir, producerToken: "producer-token" }, - dependencies( - createApi({ run: workflowRun({ status: "completed", conclusion: "success" }) }), - { now: () => BASE_TIME + 43 * 60_000 }, - ), - ); - let clock = BASE_TIME + 44 * 60_000; - const api = createApi({ artifacts: { total_count: 0, artifacts: [] } }); - try { - await expect( - downloadExactImageManifest( - { workDir: started.workDir, producerToken: "producer-token" }, - dependencies(api, { - now: () => clock, - sleep: async () => { - clock = BASE_TIME + 45 * 60_000; - }, - }), - ), - ).rejects.toMatchObject({ code: "QUALIFICATION_TIMEOUT" }); - expect( - api.mock.calls.filter(([apiPath]) => String(apiPath).includes("/artifacts?")), - ).toHaveLength(1); - } finally { - fs.rmSync(started.workDir, { recursive: true, force: true }); - } - }); - - it("does not accept an archive whose extraction crosses the shared deadline", async () => { - const archive = Buffer.from("archive"); - const manifest = Buffer.from('{"schemaVersion":1}\n'); - const started = await startedState(); - await waitForExactImageQualification( - { workDir: started.workDir, producerToken: "producer-token" }, - dependencies( - createApi({ run: workflowRun({ status: "completed", conclusion: "success" }) }), - { now: () => BASE_TIME + 43 * 60_000 }, - ), - ); - let clock = BASE_TIME + 44 * 60_000; - try { - await expect( - downloadExactImageManifest( - { workDir: started.workDir, producerToken: "producer-token" }, - dependencies(createApi({ artifacts: artifactList(archive) }), { - now: () => clock, - fetch: async () => new Response(archive, { status: 200 }), - runCommand: createArchiveRunCommand(manifest, () => { - clock = BASE_TIME + 45 * 60_000; - }), - }), - ), - ).rejects.toMatchObject({ code: "QUALIFICATION_TIMEOUT" }); - expect(readExactImageQualificationState(started.workDir).status).toBe("completed"); - } finally { - fs.rmSync(started.workDir, { recursive: true, force: true }); - } - }); - - it("rejects duplicate or nested ZIP entry inventories", () => { - const tempDir = tempDirectory(); - const archivePath = path.join(tempDir, ARCHIVE_FILE); - fs.writeFileSync(archivePath, "not inspected by the command seam"); - try { - expect(() => - extractExactManifestArchive( - archivePath, - path.join(tempDir, MANIFEST_ARTIFACT_FILE), - () => ({ - status: 0, - stdout: Buffer.from(`${MANIFEST_ARTIFACT_FILE}\n${MANIFEST_ARTIFACT_FILE}\n`), - stderr: Buffer.alloc(0), - }), - ), - ).toThrow(/exactly/u); - expect(() => - extractExactManifestArchive( - archivePath, - path.join(tempDir, MANIFEST_ARTIFACT_FILE), - () => ({ - status: 0, - stdout: Buffer.from(`nested/${MANIFEST_ARTIFACT_FILE}\n`), - stderr: Buffer.alloc(0), - }), - ), - ).toThrow(/exactly/u); - } finally { - fs.rmSync(tempDir, { recursive: true, force: true }); - } - }); - - it("rejects a ZIP entry marked as a symbolic link before extraction", () => { - const tempDir = tempDirectory(); - const archivePath = path.join(tempDir, ARCHIVE_FILE); - fs.writeFileSync(archivePath, "not inspected by the command seam"); - const runCommand = vi.fn((_command: string, args: readonly string[]) => ({ - status: 0, - stdout: - args[0] === "-Z1" - ? Buffer.from(`${MANIFEST_ARTIFACT_FILE}\n`) - : Buffer.from(`lrwxrwxrwx 3.0 unx 20 tx 20 stor ${MANIFEST_ARTIFACT_FILE}\n`), - stderr: Buffer.alloc(0), - })); - try { - expect(() => - extractExactManifestArchive( - archivePath, - path.join(tempDir, MANIFEST_ARTIFACT_FILE), - runCommand, - ), - ).toThrow(/regular file/u); - expect(runCommand).toHaveBeenCalledTimes(2); - } finally { - fs.rmSync(tempDir, { recursive: true, force: true }); - } - }); -}); - -describe("qualification evidence finalization", () => { - it("records immutable producer, artifact, and accepted-manifest hashes", async () => { - const archive = Buffer.from("archive"); - const manifest = Buffer.from('{"schemaVersion":1}\n'); - const normalized = Buffer.from('{"schemaVersion":1,"accepted":true}\n'); - const started = await startedState(); - await waitForExactImageQualification( - { workDir: started.workDir, producerToken: "producer-token" }, - dependencies( - createApi({ run: workflowRun({ status: "completed", conclusion: "success" }) }), - { now: () => BASE_TIME + 1_000 }, - ), - ); - await downloadExactImageManifest( - { workDir: started.workDir, producerToken: "producer-token" }, - dependencies(createApi({ artifacts: artifactList(archive) }), { - fetch: async () => new Response(archive, { status: 200 }), - runCommand: (_command, args) => ({ - status: 0, - stdout: - args[0] === "-Z1" - ? Buffer.from(`${MANIFEST_ARTIFACT_FILE}\n`) - : args[0] === "-Zl" - ? Buffer.from(`-rw-r--r-- 3.0 unx 20 tx 20 stor ${MANIFEST_ARTIFACT_FILE}\n`) - : manifest, - stderr: Buffer.alloc(0), - }), - }), - ); - fs.writeFileSync(path.join(started.workDir, VALIDATED_MANIFEST_FILE), normalized, { - mode: 0o600, - }); - try { - const finalState = finalizeExactImageQualification(started.workDir, { - now: () => BASE_TIME + 2_000, - }); - expect(finalState).toMatchObject({ - status: "validated", - validation: { - manifestSha256: createHash("sha256").update(manifest).digest("hex"), - normalizedManifestSha256: createHash("sha256").update(normalized).digest("hex"), - }, - }); - const evidence = JSON.parse( - fs.readFileSync(path.join(started.workDir, EVIDENCE_FILE), "utf8"), - ); - expect(evidence).toMatchObject({ - qualificationStatus: "accepted", - producer: { runId: RUN_ID, repositorySha: PRODUCER_SHA }, - artifact: { id: "86420" }, - }); - } finally { - fs.rmSync(started.workDir, { recursive: true, force: true }); - } - }); - - it("never publishes accepted evidence when the validated state transition is rejected", async () => { - const archive = Buffer.from("archive"); - const manifest = Buffer.from('{"schemaVersion":1}\n'); - const normalized = Buffer.from('{"schemaVersion":1,"accepted":true}\n'); - const started = await startedState(); - await waitForExactImageQualification( - { workDir: started.workDir, producerToken: "producer-token" }, - dependencies( - createApi({ run: workflowRun({ status: "completed", conclusion: "success" }) }), - { now: () => BASE_TIME + 3_000 }, - ), - ); - await downloadExactImageManifest( - { workDir: started.workDir, producerToken: "producer-token" }, - dependencies(createApi({ artifacts: artifactList(archive) }), { - fetch: async () => new Response(archive, { status: 200 }), - runCommand: createArchiveRunCommand(manifest), - }), - ); - fs.writeFileSync(path.join(started.workDir, VALIDATED_MANIFEST_FILE), normalized, { - mode: 0o600, - }); - try { - expect(() => - finalizeExactImageQualification(started.workDir, { - now: () => BASE_TIME + 2_000, - }), - ).toThrowError(expect.objectContaining({ code: "OUTPUT_WRITE_FAILED" })); - expect(fs.existsSync(path.join(started.workDir, EVIDENCE_FILE))).toBe(false); - expect(readExactImageQualificationState(started.workDir).status).toBe("downloaded"); - } finally { - fs.rmSync(started.workDir, { recursive: true, force: true }); - } - }); - - it("refuses accepted evidence at the shared deadline", () => { - const workDir = tempDirectory(); - const manifest = Buffer.from('{"schemaVersion":1}\n'); - const normalized = Buffer.from('{"schemaVersion":1,"accepted":true}\n'); - const manifestSha256 = createHash("sha256").update(manifest).digest("hex"); - const state = { - schemaVersion: 1, - status: "downloaded", - dispatchedAt: new Date(BASE_TIME).toISOString(), - request: { - actor: REQUEST.actor, - candidateSha: CANDIDATE_SHA, - correlationId: CORRELATION_ID, - reason: REQUEST.reason, - requesterRunAttempt: REQUEST.requesterRunAttempt, - requesterRunId: REQUEST.requesterRunId, - workflowSha: WORKFLOW_SHA, - }, - producer: { - repository: PRODUCER_REPOSITORY, - repositorySha: PRODUCER_SHA, - ref: "main", - workflowPath: PRODUCER_WORKFLOW_PATH, - runId: RUN_ID, - runAttempt: 1, - workflowId: String(WORKFLOW_ID), - runUrl: API_RUN_URL, - htmlUrl: HTML_RUN_URL, - completedAt: new Date(BASE_TIME + 1_000).toISOString(), - }, - artifact: { - id: "86420", - name: `nemoclaw-image-handoff-v1-${RUN_ID}-1`, - digest: `sha256:${"0".repeat(64)}`, - sizeInBytes: 7, - apiUrl: `https://api.github.com/repos/${PRODUCER_REPOSITORY}/actions/artifacts/86420`, - archiveDownloadUrl: `https://api.github.com/repos/${PRODUCER_REPOSITORY}/actions/artifacts/86420/zip`, - archiveSha256: "0".repeat(64), - manifestSha256, - }, - }; - fs.writeFileSync( - path.join(workDir, DISPATCH_INTENT_FILE), - `${JSON.stringify(dispatchIntent(), null, 2)}\n`, - { mode: 0o600 }, - ); - fs.writeFileSync(path.join(workDir, STATE_FILE), `${JSON.stringify(state)}\n`, { mode: 0o600 }); - fs.writeFileSync(path.join(workDir, MANIFEST_ARTIFACT_FILE), manifest, { mode: 0o600 }); - fs.writeFileSync(path.join(workDir, VALIDATED_MANIFEST_FILE), normalized, { mode: 0o600 }); - try { - expect(() => - finalizeExactImageQualification(workDir, { - now: () => BASE_TIME + 45 * 60_000, - }), - ).toThrowError(expect.objectContaining({ code: "QUALIFICATION_TIMEOUT" })); - expect(fs.existsSync(path.join(workDir, EVIDENCE_FILE))).toBe(false); - } finally { - fs.rmSync(workDir, { recursive: true, force: true }); - } - }); - - it("loads under the workflow's dependency-free Node strip-types runtime", () => { - const script = path.resolve("tools/e2e/exact-image-qualification-controller.mts"); - const result = spawnSync( - process.execPath, - [ - "--experimental-strip-types", - "--no-warnings", - script, - "--mode", - "cancel", - "--work-dir", - "/does/not/exist", - ], - { - encoding: "utf8", - env: { ...process.env, NEMOCLAW_IMAGE_QUALIFICATION_TOKEN: "test-token" }, - timeout: 10_000, - }, - ); - expect(result.status, result.stderr).toBe(0); - expect(result.stdout).toContain("No active producer run"); - }); -}); diff --git a/test/helpers/exact-image-qualification-controller-fixture.ts b/test/helpers/exact-image-qualification-controller-fixture.ts deleted file mode 100644 index 1dba1cd812..0000000000 --- a/test/helpers/exact-image-qualification-controller-fixture.ts +++ /dev/null @@ -1,284 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { createHash } from "node:crypto"; -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; - -import { vi } from "vitest"; - -import { - type ExactImageDispatchIntent, - type ExactImageQualificationRequest, - MANIFEST_ARTIFACT_FILE, - PRODUCER_REF, - PRODUCER_REPOSITORY, - PRODUCER_WORKFLOW_FILE, - PRODUCER_WORKFLOW_PATH, - type QualificationDependencies, - startExactImageQualification, -} from "../../tools/e2e/exact-image-qualification-controller.mts"; - -export const WORKFLOW_SHA = "c".repeat(40); -export const CANDIDATE_SHA = WORKFLOW_SHA; -export const PRODUCER_SHA = "b".repeat(40); -export const CORRELATION_ID = "123e4567-e89b-42d3-a456-426614174000"; -export const RUN_ID = "24680"; -export const WORKFLOW_ID = 13579; -export const BASE_TIME = Date.UTC(2026, 0, 1); -export const API_RUN_URL = `https://api.github.com/repos/${PRODUCER_REPOSITORY}/actions/runs/${RUN_ID}`; -export const HTML_RUN_URL = `https://github.com/${PRODUCER_REPOSITORY}/actions/runs/${RUN_ID}`; - -export const REQUEST: ExactImageQualificationRequest = { - actor: "maintainer", - candidateSha: CANDIDATE_SHA, - eventName: "workflow_dispatch", - reason: "Qualify the current daily candidate before tagging", - ref: "refs/heads/main", - requesterRunAttempt: 2, - requesterRunId: "97531", - workflowSha: WORKFLOW_SHA, -}; - -export function dispatchIntent(): ExactImageDispatchIntent { - return { - schemaVersion: 1, - kind: "nemoclaw-exact-image-dispatch-intent", - requestStartedAt: new Date(BASE_TIME).toISOString(), - request: { - actor: REQUEST.actor, - candidateSha: REQUEST.candidateSha, - correlationId: CORRELATION_ID, - reason: REQUEST.reason, - requesterRunAttempt: REQUEST.requesterRunAttempt, - requesterRunId: REQUEST.requesterRunId, - workflowSha: REQUEST.workflowSha, - }, - producer: { - repository: PRODUCER_REPOSITORY, - repositorySha: PRODUCER_SHA, - ref: PRODUCER_REF, - workflowId: String(WORKFLOW_ID), - workflowPath: PRODUCER_WORKFLOW_PATH, - }, - }; -} - -type ApiOptions = { - artifacts?: unknown; - dispatch?: unknown; - permission?: string; - roleName?: string; - producerSha?: string; - requesterSha?: string; - run?: unknown; - runs?: unknown; - workflow?: unknown; -}; - -type ApiRouteHandler = ( - apiPath: string, - token: string, - requestOptions?: unknown, -) => unknown | Promise; - -type ApiRoute = { - matches: (apiPath: string) => boolean; - respond: ApiRouteHandler; -}; - -function mainRef(sha: string) { - return { ref: "refs/heads/main", object: { type: "commit", sha } }; -} - -export function dispatchDetails() { - return { - workflow_run_id: Number(RUN_ID), - run_url: API_RUN_URL, - html_url: HTML_RUN_URL, - }; -} - -export function workflowRun(overrides: Record = {}) { - return { - id: Number(RUN_ID), - workflow_id: WORKFLOW_ID, - run_attempt: 1, - event: "workflow_dispatch", - head_branch: "main", - head_sha: PRODUCER_SHA, - path: PRODUCER_WORKFLOW_PATH, - display_title: `Qualify NemoClaw ${CANDIDATE_SHA} (${CORRELATION_ID})`, - url: API_RUN_URL, - html_url: HTML_RUN_URL, - repository: { full_name: PRODUCER_REPOSITORY }, - head_repository: { full_name: PRODUCER_REPOSITORY }, - status: "queued", - conclusion: null, - created_at: new Date(BASE_TIME).toISOString(), - ...overrides, - }; -} - -export function createApi(options: ApiOptions = {}) { - return vi.fn(async (apiPath: string, _token: string, requestOptions?: unknown) => { - if (apiPath === "repos/NVIDIA/NemoClaw/git/ref/heads/main") { - return mainRef(options.requesterSha ?? WORKFLOW_SHA); - } - if (apiPath === `repos/NVIDIA/NemoClaw/git/commits/${CANDIDATE_SHA}`) { - return { sha: CANDIDATE_SHA }; - } - if (apiPath.includes("/collaborators/")) { - return { - permission: options.permission ?? "write", - role_name: options.roleName ?? "maintain", - }; - } - if (apiPath === `repos/${PRODUCER_REPOSITORY}/git/ref/heads/main`) { - return mainRef(options.producerSha ?? PRODUCER_SHA); - } - if (apiPath === `repos/${PRODUCER_REPOSITORY}/actions/workflows/${PRODUCER_WORKFLOW_FILE}`) { - return ( - options.workflow ?? { - id: WORKFLOW_ID, - path: PRODUCER_WORKFLOW_PATH, - state: "active", - } - ); - } - if ( - apiPath === - `repos/${PRODUCER_REPOSITORY}/actions/workflows/${PRODUCER_WORKFLOW_FILE}/dispatches` - ) { - return options.dispatch === undefined ? dispatchDetails() : options.dispatch; - } - if (apiPath === `repos/${PRODUCER_REPOSITORY}/actions/runs/${RUN_ID}`) { - return options.run ?? workflowRun(); - } - if (apiPath === `repos/${PRODUCER_REPOSITORY}/actions/runs/${RUN_ID}/cancel`) { - return undefined; - } - if ( - apiPath.startsWith( - `repos/${PRODUCER_REPOSITORY}/actions/workflows/${PRODUCER_WORKFLOW_FILE}/runs?`, - ) - ) { - return options.runs ?? { total_count: 0, workflow_runs: [] }; - } - if (apiPath === `repos/${PRODUCER_REPOSITORY}/actions/runs/${RUN_ID}/artifacts?per_page=100`) { - return options.artifacts; - } - throw new Error(`unexpected API call ${apiPath} ${JSON.stringify(requestOptions)}`); - }); -} - -function route(matches: (apiPath: string) => boolean, respond: ApiRouteHandler): ApiRoute { - return { matches, respond }; -} - -export const qualificationApiRoute = { - dispatch: (respond: ApiRouteHandler): ApiRoute => - route( - (apiPath) => apiPath.endsWith(`/workflows/${PRODUCER_WORKFLOW_FILE}/dispatches`), - respond, - ), - workflowRuns: (respond: ApiRouteHandler): ApiRoute => - route((apiPath) => apiPath.includes(`/workflows/${PRODUCER_WORKFLOW_FILE}/runs?`), respond), - run: (respond: ApiRouteHandler): ApiRoute => - route((apiPath) => apiPath === `repos/${PRODUCER_REPOSITORY}/actions/runs/${RUN_ID}`, respond), - runAny: (respond: ApiRouteHandler): ApiRoute => - route( - (apiPath) => /^repos\/brevdev\/nemoclaw-image\/actions\/runs\/[1-9][0-9]*$/u.test(apiPath), - respond, - ), - cancelRun: (respond: ApiRouteHandler): ApiRoute => - route( - (apiPath) => apiPath === `repos/${PRODUCER_REPOSITORY}/actions/runs/${RUN_ID}/cancel`, - respond, - ), - cancelAnyRun: (respond: ApiRouteHandler): ApiRoute => - route((apiPath) => apiPath.endsWith("/cancel"), respond), -}; - -export function createRoutedApi(base: ReturnType, routes: readonly ApiRoute[]) { - return vi.fn(async (apiPath: string, token: string, requestOptions?: unknown) => { - for (const candidate of routes) { - if (candidate.matches(apiPath)) { - return candidate.respond(apiPath, token, requestOptions); - } - } - return base(apiPath, token, requestOptions); - }); -} - -export function dependencies( - api: ReturnType, - extra: QualificationDependencies = {}, -) { - return { - now: () => BASE_TIME, - ...extra, - api: api as QualificationDependencies["api"], - randomUuid: () => CORRELATION_ID, - }; -} - -export function tempDirectory(): string { - return fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-image-qualification-test-")); -} - -export async function startedState(api = createApi()) { - const workDir = tempDirectory(); - const state = await startExactImageQualification( - { - request: REQUEST, - coreToken: "core-token", - producerToken: "producer-token", - workDir, - }, - dependencies(api), - ); - return { api, state, workDir }; -} - -export function artifactList(archive: Buffer, overrides: Record = {}) { - const artifactId = 86420; - return { - total_count: 1, - artifacts: [ - { - id: artifactId, - name: `nemoclaw-image-handoff-v1-${RUN_ID}-1`, - expired: false, - digest: `sha256:${createHash("sha256").update(archive).digest("hex")}`, - size_in_bytes: archive.length, - url: `https://api.github.com/repos/${PRODUCER_REPOSITORY}/actions/artifacts/${artifactId}`, - archive_download_url: `https://api.github.com/repos/${PRODUCER_REPOSITORY}/actions/artifacts/${artifactId}/zip`, - workflow_run: { id: Number(RUN_ID), head_sha: PRODUCER_SHA }, - ...overrides, - }, - ], - }; -} - -export function createArchiveRunCommand(manifest: Buffer, onExtract: () => void = () => {}) { - return (_command: string, args: readonly string[]) => { - if (args[0] === "-Z1") { - return { - status: 0, - stdout: Buffer.from(`${MANIFEST_ARTIFACT_FILE}\n`), - stderr: Buffer.alloc(0), - }; - } - if (args[0] === "-Zl") { - return { - status: 0, - stdout: Buffer.from(`-rw-r--r-- 3.0 unx 20 tx 20 stor ${MANIFEST_ARTIFACT_FILE}\n`), - stderr: Buffer.alloc(0), - }; - } - onExtract(); - return { status: 0, stdout: manifest, stderr: Buffer.alloc(0) }; - }; -} diff --git a/test/helpers/vitest-watch-triggers.ts b/test/helpers/vitest-watch-triggers.ts index 43f5b5573b..c501b063f3 100644 --- a/test/helpers/vitest-watch-triggers.ts +++ b/test/helpers/vitest-watch-triggers.ts @@ -37,7 +37,6 @@ const E2E_WORKFLOW_CONTRACTS = [ "test/e2e/support/tunnel-lifecycle-workflow-boundary.test.ts", "test/e2e/support/upload-e2e-artifacts-workflow-boundary.test.ts", "test/e2e/support/workflow-plan.test.ts", - "test/e2e/support/exact-image-qualification-workflow.test.ts", ] as const; function runTests(...tests: string[]): () => string[] { @@ -97,16 +96,9 @@ export const vitestWatchTriggerPatterns: VitestWatchTriggerPattern[] = [ pattern: /(?:^|\/)\.github\/workflows\/pr-e2e-gate\.yaml$/, testsToRun: runTests("test/pr-e2e-gate-workflow.test.ts", "test/pr-e2e-required.test.ts"), }, - { - pattern: /(?:^|\/)\.github\/workflows\/brev-launchable-qualification\.yaml$/, - testsToRun: runTests("test/e2e/support/exact-image-qualification-workflow.test.ts"), - }, { pattern: /(?:^|\/)tools\/e2e\/brev-launchable-runtime\.sh$/, - testsToRun: runTests( - "test/brev-launchable-runtime.test.ts", - "test/e2e/support/exact-image-qualification-workflow.test.ts", - ), + testsToRun: runTests("test/brev-launchable-runtime.test.ts"), }, { pattern: diff --git a/test/vitest-watch-triggers.test.ts b/test/vitest-watch-triggers.test.ts index 3fda1fae76..8005617f70 100644 --- a/test/vitest-watch-triggers.test.ts +++ b/test/vitest-watch-triggers.test.ts @@ -43,7 +43,6 @@ const E2E_WORKFLOW_CONTRACTS = [ "test/e2e/support/tunnel-lifecycle-workflow-boundary.test.ts", "test/e2e/support/upload-e2e-artifacts-workflow-boundary.test.ts", "test/e2e/support/workflow-plan.test.ts", - "test/e2e/support/exact-image-qualification-workflow.test.ts", ] as const; const OPAQUE_INPUTS = [ @@ -59,7 +58,6 @@ const OPAQUE_INPUTS = [ ".github/workflows/e2e.yaml", ".github/workflows/code-scanning.yaml", ".github/workflows/pr-e2e-gate.yaml", - ".github/workflows/brev-launchable-qualification.yaml", ".github/workflows/platform-vitest-main.yaml", "ci/platform-vitest-macos-requirements.lock", ] as const; @@ -113,12 +111,8 @@ describe("Vitest opaque-input watch triggers", () => { "test/pr-e2e-gate-workflow.test.ts", "test/pr-e2e-required.test.ts", ]); - expect(triggeredBy(".github/workflows/brev-launchable-qualification.yaml")).toEqual([ - "test/e2e/support/exact-image-qualification-workflow.test.ts", - ]); expect(triggeredBy("tools/e2e/brev-launchable-runtime.sh")).toEqual([ "test/brev-launchable-runtime.test.ts", - "test/e2e/support/exact-image-qualification-workflow.test.ts", ]); expect(triggeredBy(".github/workflows/platform-vitest-main.yaml")).toEqual([ "test/platform-vitest-main-workflow.test.ts", diff --git a/tools/advisors/github.mts b/tools/advisors/github.mts index 1cbdcb4d04..c9a7f58a50 100644 --- a/tools/advisors/github.mts +++ b/tools/advisors/github.mts @@ -10,8 +10,6 @@ export type GitHubComment = { export type GitHubRequestOptions = { method?: string; body?: unknown; - apiVersion?: string; - expectedStatus?: number; userAgent?: string; signal?: AbortSignal; }; @@ -93,17 +91,14 @@ export async function githubApi( Accept: "application/vnd.github+json", Authorization: `Bearer ${token}`, "Content-Type": "application/json", - "X-GitHub-Api-Version": options.apiVersion ?? "2022-11-28", + "X-GitHub-Api-Version": "2022-11-28", ...(options.userAgent ? { "User-Agent": options.userAgent } : {}), }, body: options.body === undefined ? undefined : JSON.stringify(options.body), signal: options.signal, }); const text = await response.text(); - if ( - !response.ok || - (options.expectedStatus !== undefined && response.status !== options.expectedStatus) - ) { + if (!response.ok) { throw new Error(`GitHub API ${apiPath} failed: ${response.status} ${text}`); } return (text ? JSON.parse(text) : undefined) as T; diff --git a/tools/e2e/brev-launchable-runtime.sh b/tools/e2e/brev-launchable-runtime.sh index da29d54cb1..e5a38d487b 100755 --- a/tools/e2e/brev-launchable-runtime.sh +++ b/tools/e2e/brev-launchable-runtime.sh @@ -6,7 +6,7 @@ set -euo pipefail die() { - printf 'BREV_LAUNCHABLE_QUALIFICATION_FAILED: %s\n' "$*" >&2 + printf 'BREV_LAUNCHABLE_E2E_FAILED: %s\n' "$*" >&2 exit 1 } @@ -15,20 +15,15 @@ require_env() { [ -n "${!name:-}" ] || die "$name is required" } -require_tools() { - local tool - for tool in "$@"; do - command -v "$tool" >/dev/null 2>&1 || die "$tool is required" - done -} - validate_common() { require_env WORK_DIR require_env INSTANCE_NAME [[ "$INSTANCE_NAME" =~ ^[a-z][a-z0-9-]{0,62}$ ]] \ || die "INSTANCE_NAME must be a lowercase Brev workspace name" [ -d "$WORK_DIR" ] || die "WORK_DIR must already exist" - require_tools brev jq timeout + command -v brev >/dev/null 2>&1 || die "brev is required" + command -v jq >/dev/null 2>&1 || die "jq is required" + command -v timeout >/dev/null 2>&1 || die "timeout is required" } workspace_rows() { @@ -71,26 +66,18 @@ wait_for_workspace_ready() { fi sleep "${BREV_POLL_SECONDS:-15}" done - die "Brev workspace did not become structurally ready before the deadline" + die "Brev workspace did not become ready before the deadline" } deploy() { validate_common - local existing require_env BREV_LAUNCHABLE_ID [[ "$BREV_LAUNCHABLE_ID" =~ ^env-[A-Za-z0-9]+$ ]] \ || die "BREV_LAUNCHABLE_ID must be one opaque env-* ID" - if ! existing="$(workspace_record)"; then - die "unable to inventory Brev workspaces before deploy" - fi - if [ -n "$existing" ]; then - die "refusing to reuse pre-existing workspace $INSTANCE_NAME" - fi - printf '{"schemaVersion":1,"launchableId":%s,"workspaceName":%s,"requestedAt":%s}\n' \ - "$(jq -Rn --arg value "$BREV_LAUNCHABLE_ID" '$value')" \ - "$(jq -Rn --arg value "$INSTANCE_NAME" '$value')" \ - "$(jq -Rn --arg value "$(date -u +%Y-%m-%dT%H:%M:%SZ)" '$value')" \ - >"$WORK_DIR/brev-deploy-request.json" + local existing + existing="$(workspace_record)" || die "unable to inventory Brev workspaces before deploy" + [ -z "$existing" ] || die "refusing to reuse pre-existing workspace $INSTANCE_NAME" + timeout "${BREV_CREATE_TIMEOUT_SECONDS:-900}" \ brev create "$INSTANCE_NAME" --launchable "$BREV_LAUNCHABLE_ID" --detached \ --timeout "${BREV_CREATE_TIMEOUT_SECONDS:-900}" @@ -105,41 +92,34 @@ host_exec() { verify_identity() { require_env CANDIDATE_SHA - [[ "$CANDIDATE_SHA" =~ ^[0-9a-f]{40}$ ]] || die "CANDIDATE_SHA must be a lowercase full SHA" - require_env VALIDATED_MANIFEST - [ -f "$VALIDATED_MANIFEST" ] || die "VALIDATED_MANIFEST is missing" - - local expected_image expected_image_id expected_self_link repo_sha provision provision_sha disk_json - expected_image="$(jq -er '.imageName' "$VALIDATED_MANIFEST")" - expected_image_id="$(jq -er '.imageId' "$VALIDATED_MANIFEST")" - expected_self_link="$(jq -er '.imageSelfLink' "$VALIDATED_MANIFEST")" + [[ "$CANDIDATE_SHA" =~ ^[0-9a-f]{40}$ ]] \ + || die "CANDIDATE_SHA must be a lowercase full SHA" + local provision repo_sha identity provision="$(host_exec 'sudo -n cat /etc/nemoclaw/provision.json')" + provision="$(tail -n 1 <<<"$provision")" jq -e 'type == "object" and (.gitSha | type == "string")' <<<"$provision" >/dev/null \ || die "the baked provision metadata is missing or malformed" printf '%s\n' "$provision" >"$WORK_DIR/brev-provision.json" - provision_sha="$(jq -r '.gitSha' <<<"$provision")" - [[ "$provision_sha" =~ ^[0-9a-f]{7,40}$ ]] \ - || die "provision metadata SHA must be a lowercase Git SHA" - # HOME and repo are intentionally expanded by the remote host shell. + # HOME is expanded by the remote host shell. # shellcheck disable=SC2016 - repo_sha="$(host_exec 'set -e + repo_sha="$(host_exec 'set -euo pipefail repo="$HOME/NemoClaw" test -d "$repo/.git" - git -C "$repo" diff --quiet --no-ext-diff HEAD -- \ - || { printf "baked NemoClaw checkout has tracked worktree changes\n" >&2; exit 1; } - git -C "$repo" diff --cached --quiet --no-ext-diff HEAD -- \ - || { printf "baked NemoClaw checkout has staged changes\n" >&2; exit 1; } + git -C "$repo" diff --quiet --no-ext-diff HEAD -- + git -C "$repo" diff --cached --quiet --no-ext-diff HEAD -- git -C "$repo" rev-parse HEAD' | tail -n 1)" [ "$repo_sha" = "$CANDIDATE_SHA" ] \ || die "baked NemoClaw SHA $repo_sha does not match candidate $CANDIDATE_SHA" - [[ "$CANDIDATE_SHA" == "$provision_sha"* ]] \ - || die "provision metadata SHA $provision_sha does not identify candidate $CANDIDATE_SHA" + [[ "$CANDIDATE_SHA" == "$(jq -r '.gitSha' <<<"$provision")"* ]] \ + || die "provision metadata does not identify candidate $CANDIDATE_SHA" - # Metadata variables are intentionally expanded by the remote host shell. + # Metadata variables are expanded by the remote host shell. The image labels + # are the producer contract; the disk and image API responses are independent + # observations of what the Launchable actually booted. # shellcheck disable=SC2016 - disk_json="$(host_exec 'set -euo pipefail + identity="$(host_exec 'set -euo pipefail metadata=http://metadata.google.internal/computeMetadata/v1 header="Metadata-Flavor: Google" project=$(curl -fsS -H "$header" "$metadata/project/project-id") @@ -147,167 +127,58 @@ verify_identity() { zone=${zone_path##*/} disk=$(curl -fsS -H "$header" "$metadata/instance/disks/0/device-name") token=$(curl -fsS -H "$header" "$metadata/instance/service-accounts/default/token" | jq -er .access_token) - curl -fsS -H "Authorization: Bearer $token" \ - "https://compute.googleapis.com/compute/v1/projects/$project/zones/$zone/disks/$disk" \ - | jq -c "{sourceImage,sourceImageId}"')" - disk_json="$(tail -n 1 <<<"$disk_json")" - jq -e --arg image "$expected_self_link" --arg id "$expected_image_id" ' - .sourceImage == $image and ((.sourceImageId | tostring) == $id) - ' <<<"$disk_json" >/dev/null \ - || die "workspace boot disk does not match accepted image $expected_image ($expected_image_id)" - printf '%s\n' "$disk_json" >"$WORK_DIR/brev-boot-image.json" + disk_json=$(curl -fsS -H "Authorization: Bearer $token" \ + "https://compute.googleapis.com/compute/v1/projects/$project/zones/$zone/disks/$disk") + image_url=$(jq -er .sourceImage <<<"$disk_json") + image_json=$(curl -fsS -H "Authorization: Bearer $token" "$image_url") + jq -cn --argjson disk "$disk_json" --argjson image "$image_json" \ + "{disk:{sourceImage:\$disk.sourceImage,sourceImageId:(\$disk.sourceImageId|tostring)},image:{name:\$image.name,id:(\$image.id|tostring),selfLink:\$image.selfLink,status:\$image.status,labels:\$image.labels}}"' | tail -n 1)" + jq -e --arg sha "$CANDIDATE_SHA" ' + .disk.sourceImage == .image.selfLink + and .disk.sourceImageId == .image.id + and .image.status == "READY" + and .image.labels["nemoclaw-sha"] == $sha + and .image.labels.channel == "staging" + and .image.labels.variant == "cpu" + ' <<<"$identity" >/dev/null \ + || die "workspace boot image does not match the exact staging image for $CANDIDATE_SHA" + printf '%s\n' "$identity" >"$WORK_DIR/brev-boot-image.json" jq -n \ --arg candidateSha "$CANDIDATE_SHA" \ --arg repositorySha "$repo_sha" \ - --arg provisionSha "$provision_sha" \ - --arg imageName "$expected_image" \ - --arg imageId "$expected_image_id" \ - --arg imageSelfLink "$expected_self_link" \ + --argjson bootImage "$identity" \ --arg verifiedAt "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ - '{schemaVersion:1,candidateSha:$candidateSha,repositorySha:$repositorySha,provisionSha:$provisionSha,image:{name:$imageName,id:$imageId,selfLink:$imageSelfLink},verifiedAt:$verifiedAt}' \ + '{schemaVersion:1,candidateSha:$candidateSha,repositorySha:$repositorySha,bootImage:$bootImage,verifiedAt:$verifiedAt}' \ >"$WORK_DIR/brev-identity-evidence.json" } -validate_copied_artifact_tree() { - local root="$1" - local canonical_result="$2" - python3 - "$root" "$canonical_result" <<'PY' -import hashlib -import json -import os -import stat -import sys -from pathlib import Path - -MAX_ENTRIES = 2_000 -MAX_FILES = 1_000 -MAX_TOTAL_BYTES = 100 * 1024 * 1024 - -root = Path(sys.argv[1]) -expected_result = sys.argv[2] -if root.is_symlink() or not root.is_dir(): - raise SystemExit("copied E2E artifact root must be a real directory") - -entries = 0 -files = 0 -total_bytes = 0 -target_results = [] -stack = [root] -while stack: - directory = stack.pop() - with os.scandir(directory) as children: - for child in children: - entries += 1 - if entries > MAX_ENTRIES: - raise SystemExit(f"copied E2E artifact tree exceeds {MAX_ENTRIES} entries") - path = Path(child.path) - relative = path.relative_to(root).as_posix() - if child.is_symlink(): - raise SystemExit(f"copied E2E artifacts must not contain symlinks: {relative}") - if child.is_dir(follow_symlinks=False): - stack.append(path) - continue - if not child.is_file(follow_symlinks=False): - raise SystemExit(f"copied E2E artifacts must contain only directories and regular files: {relative}") - metadata = child.stat(follow_symlinks=False) - if not stat.S_ISREG(metadata.st_mode): - raise SystemExit(f"copied E2E artifact is not a regular file: {relative}") - files += 1 - total_bytes += metadata.st_size - if files > MAX_FILES: - raise SystemExit(f"copied E2E artifact tree exceeds {MAX_FILES} files") - if total_bytes > MAX_TOTAL_BYTES: - raise SystemExit(f"copied E2E artifact tree exceeds {MAX_TOTAL_BYTES} bytes") - if child.name == "target-result.json": - target_results.append(relative) - -target_results.sort() -if target_results != [expected_result]: - raise SystemExit( - "copied E2E artifacts must contain exactly the canonical target-result.json; " - f"found {target_results}" - ) - -target_result = root / expected_result -raw_result = target_result.read_bytes() - -def unique_object(pairs): - result = {} - for key, value in pairs: - if key in result: - raise ValueError(f"duplicate JSON key: {key}") - result[key] = value - return result - -try: - result = json.loads(raw_result, object_pairs_hook=unique_object) -except (UnicodeDecodeError, json.JSONDecodeError, ValueError) as error: - raise SystemExit(f"canonical target-result.json is invalid: {error}") from error - -expected_keys = {"firstAgentTurn", "id", "runner", "securityPosture", "status"} -if not isinstance(result, dict) or set(result) != expected_keys: - raise SystemExit("canonical target-result.json has an unexpected schema") -if result["id"] != "brev-launchable-cloud-openclaw" or result["runner"] != "vitest" or result["status"] != "passed": - raise SystemExit("canonical target-result.json does not report a passing Launchable target") -first_turn = result["firstAgentTurn"] -if not isinstance(first_turn, dict) or set(first_turn) != {"commandMs", "responseChars", "status"}: - raise SystemExit("canonical target-result.json has invalid first-agent-turn evidence") -if first_turn["status"] != "passed" or type(first_turn["commandMs"]) is not int or first_turn["commandMs"] < 0: - raise SystemExit("canonical target-result.json does not report a passing first agent turn") -if type(first_turn["responseChars"]) is not int or first_turn["responseChars"] <= 0: - raise SystemExit("canonical target-result.json has invalid first-agent-turn response evidence") -posture = result["securityPosture"] -if not isinstance(posture, dict) or not isinstance(posture.get("entrypoint"), dict): - raise SystemExit("canonical target-result.json has invalid security-posture evidence") -for field in ("configureGuard", "hostNonRoot", "rcFilesLocked", "runtimeProxyEnvLocked", "startupLogClean"): - if posture.get(field) is not True: - raise SystemExit(f"canonical target-result.json security posture did not pass {field}") - -digest = hashlib.sha256(raw_result).hexdigest() -print(json.dumps({ - "fileCount": files, - "totalBytes": total_bytes, - "targetResultPath": expected_result, - "targetResultSha256": digest, -}, separators=(",", ":"))) -PY -} - run_existing_e2e() { require_env NVIDIA_INFERENCE_API_KEY - require_tools python3 local sandbox="${NEMOCLAW_STAGING_SANDBOX_NAME:-e2e-staging}" [[ "$sandbox" =~ ^[a-z][a-z0-9-]{0,62}$ ]] || die "invalid staging sandbox name" local remote_artifact_dir="/tmp/nemoclaw-launchable-e2e-$INSTANCE_NAME" local local_artifact_dir="$WORK_DIR/brev-launchable-cloud-openclaw" - local canonical_result_relative="brev-launchable-cloud-openclaw-onboard-inference-cli-operations-and-cleanup/target-result.json" + local result_relative="brev-launchable-cloud-openclaw-onboard-inference-cli-operations-and-cleanup/target-result.json" local quoted_artifact_dir quoted_key quoted_sandbox printf -v quoted_artifact_dir '%q' "$remote_artifact_dir" printf -v quoted_key '%q' "$NVIDIA_INFERENCE_API_KEY" printf -v quoted_sandbox '%q' "$sandbox" - # The escaped model expansions and case patterns are evaluated by the remote host shell. + # Escaped values and model checks are evaluated by the remote host shell. # shellcheck disable=SC1083,SC2140 host_exec "set -euo pipefail - repo=\$HOME/NemoClaw - cd \"\$repo\" + cd \"\$HOME/NemoClaw\" test -x ./node_modules/.bin/vitest - grep -q 'NEMOCLAW_E2E_SETUP_MODE' test/e2e/live/full-e2e.test.ts \ - || { printf 'candidate full-e2e.test.ts does not support preinstalled Launchable setup\n' >&2; exit 1; } - grep -q 'brev-launchable-cloud-openclaw' test/e2e/live/full-e2e.test.ts \ - || { printf 'candidate full-e2e.test.ts does not declare the Brev Launchable target\n' >&2; exit 1; } rm -rf -- $quoted_artifact_dir install -d -m 700 $quoted_artifact_dir model=\$(node /usr/local/lib/nemoclaw/launchable-config.mjs /usr/local/share/nemoclaw/launchable-agents.json openclaw cloudModel) - [ "\${#model}" -le 256 ] \ - || { printf 'Launchable cloud model is not a safe model ID\n' >&2; exit 1; } - case "\$model" in - [A-Za-z0-9]*) ;; - *) printf 'Launchable cloud model is not a safe model ID\n' >&2; exit 1 ;; + case \"\$model\" in + [A-Za-z0-9]* ) ;; + * ) printf 'Launchable cloud model is not a safe model ID\n' >&2; exit 1 ;; esac - case "\$model" in - *[!A-Za-z0-9._:/-]*) printf 'Launchable cloud model is not a safe model ID\n' >&2; exit 1 ;; + case \"\$model\" in + *[!A-Za-z0-9._:/-]* ) printf 'Launchable cloud model is not a safe model ID\n' >&2; exit 1 ;; esac export CI=true GITHUB_ACTIONS=true export E2E_ARTIFACT_DIR=$quoted_artifact_dir @@ -325,22 +196,30 @@ run_existing_e2e() { mkdir -m 700 "$local_artifact_dir" timeout "${BREV_COPY_TIMEOUT_SECONDS:-300}" \ brev copy "$INSTANCE_NAME:$remote_artifact_dir/" "$local_artifact_dir/" --host - - local artifact_summary - artifact_summary="$(validate_copied_artifact_tree "$local_artifact_dir" "$canonical_result_relative")" \ - || die "copied full E2E artifacts failed validation" + local result="$local_artifact_dir/$result_relative" + [ -f "$result" ] && [ ! -L "$result" ] || die "full E2E result is missing" + jq -e ' + .id == "brev-launchable-cloud-openclaw" + and .runner == "vitest" + and .status == "passed" + and .firstAgentTurn.status == "passed" + and .securityPosture.configureGuard == true + and .securityPosture.hostNonRoot == true + and .securityPosture.rcFilesLocked == true + and .securityPosture.runtimeProxyEnvLocked == true + and .securityPosture.startupLogClean == true + ' "$result" >/dev/null || die "full E2E result did not pass" jq -n \ - --argjson artifacts "$artifact_summary" \ - --arg sandbox "$sandbox" \ --arg target "brev-launchable-cloud-openclaw" \ --arg testFile "test/e2e/live/full-e2e.test.ts" \ + --arg targetResultSha256 "$(sha256sum "$result" | awk '{print $1}')" \ --arg completedAt "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ - '{schemaVersion:1,target:$target,testFile:$testFile,setupMode:"preinstalled-launchable",sandbox:$sandbox,artifacts:$artifacts,completedAt:$completedAt}' \ + '{schemaVersion:1,target:$target,testFile:$testFile,setupMode:"preinstalled-launchable",targetResultSha256:$targetResultSha256,completedAt:$completedAt}' \ >"$WORK_DIR/brev-launchable-e2e-evidence.json" } -qualify() { +run_e2e() { validate_common verify_identity run_existing_e2e @@ -348,39 +227,31 @@ qualify() { cleanup() { validate_common - local requested_at verified_at deadline output record status=0 absent_count=0 + local requested_at deadline record absent_count=0 requested_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)" - if record="$(workspace_record)"; then - if [ -n "$record" ]; then - output="$(timeout 60s brev delete "$INSTANCE_NAME" 2>&1)" || status=$? - printf '%s\n' "$output" - [ "$status" -eq 0 ] || printf 'brev delete returned %s; verifying absence before failing\n' "$status" >&2 - fi - else - printf 'brev ls failed before cleanup; verifying absence with retries\n' >&2 + record="$(workspace_record || true)" + if [ -n "$record" ]; then + timeout 60s brev delete "$INSTANCE_NAME" || true fi + deadline=$((SECONDS + ${BREV_DELETE_TIMEOUT_SECONDS:-600})) while [ "$SECONDS" -lt "$deadline" ]; do if record="$(workspace_record)"; then - if [ -n "$record" ]; then + if [ -z "$record" ]; then + absent_count=$((absent_count + 1)) + if [ "$absent_count" -ge "${BREV_ABSENCE_CONFIRMATIONS:-2}" ]; then + jq -n \ + --arg workspaceName "$INSTANCE_NAME" \ + --arg requestedAt "$requested_at" \ + --arg verifiedAt "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + '{schemaVersion:1,workspaceName:$workspaceName,deleteRequestedAt:$requestedAt,terminalState:"ABSENT",verifiedAt:$verifiedAt}' \ + >"$WORK_DIR/brev-cleanup-evidence.json" + return 0 + fi + else absent_count=0 - timeout 30s brev refresh >/dev/null 2>&1 || true - sleep "${BREV_POLL_SECONDS:-15}" - continue - fi - absent_count=$((absent_count + 1)) - if [ "$absent_count" -lt "${BREV_ABSENCE_CONFIRMATIONS:-4}" ]; then - timeout 30s brev refresh >/dev/null 2>&1 || true - sleep "${BREV_POLL_SECONDS:-15}" - continue fi - verified_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)" - jq -n --arg workspaceName "$INSTANCE_NAME" --arg requestedAt "$requested_at" --arg verifiedAt "$verified_at" \ - '{schemaVersion:1,workspaceName:$workspaceName,deleteRequestedAt:$requestedAt,terminalState:"ABSENT",verifiedAt:$verifiedAt}' \ - >"$WORK_DIR/brev-cleanup-evidence.json" - return 0 fi - printf 'brev ls failed while verifying cleanup; retrying\n' >&2 timeout 30s brev refresh >/dev/null 2>&1 || true sleep "${BREV_POLL_SECONDS:-15}" done @@ -389,7 +260,7 @@ cleanup() { case "${1:-}" in deploy) deploy ;; - qualify) qualify ;; + run) run_e2e ;; cleanup) cleanup ;; - *) die "usage: $0 deploy|qualify|cleanup" ;; + *) die "usage: $0 deploy|run|cleanup" ;; esac diff --git a/tools/e2e/exact-image-manifest.mts b/tools/e2e/exact-image-manifest.mts deleted file mode 100644 index 8ee007a411..0000000000 --- a/tools/e2e/exact-image-manifest.mts +++ /dev/null @@ -1,396 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -export const EXACT_IMAGE_MANIFEST_KIND = "nemoclaw-exact-image-manifest"; -export const EXACT_IMAGE_REQUESTER_REPOSITORY = "NVIDIA/NemoClaw"; -export const EXACT_IMAGE_REPOSITORY = "brevdev/nemoclaw-image"; -export const EXACT_IMAGE_PRODUCER_WORKFLOW = ".github/workflows/build-qualification-image.yml"; -// This mutable family is publication evidence only. Consumers accept the -// immutable name, numeric ID, and self-link below as the image identity. -export const EXACT_IMAGE_STAGING_FAMILY = "nemoclaw-brev-staging-cpu"; -export const EXACT_IMAGE_MANIFEST_MAX_CLOCK_SKEW_MS = 5 * 60_000; -export const EXACT_IMAGE_MANIFEST_MAX_REUSED_AGE_MS = 24 * 60 * 60_000; - -export type ExactImageManifestFailureCode = - | "REQUEST_INVALID" - | "ARTIFACT_MISSING_OR_INVALID" - | "UNSUPPORTED_VARIANT" - | "PROVENANCE_MISMATCH" - | "IMAGE_IDENTITY_MISMATCH" - | "OUTPUT_WRITE_FAILED" - | "UNKNOWN"; - -export class ExactImageManifestError extends Error { - readonly code: ExactImageManifestFailureCode; - - constructor(code: ExactImageManifestFailureCode, message: string) { - super(message); - this.name = "ExactImageManifestError"; - this.code = code; - } -} - -export type ExactImageManifest = { - schemaVersion: 1; - kind: typeof EXACT_IMAGE_MANIFEST_KIND; - correlationId: string; - requesterRepository: typeof EXACT_IMAGE_REQUESTER_REPOSITORY; - requesterWorkflowRunId: string; - requesterWorkflowRunAttempt: number; - nemoclawSha: string; - imageRepository: typeof EXACT_IMAGE_REPOSITORY; - imageRepositorySha: string; - producerWorkflow: typeof EXACT_IMAGE_PRODUCER_WORKFLOW; - workflowRunId: string; - workflowRunAttempt: number; - imageOriginWorkflowRunId: string; - imageOriginWorkflowRunAttempt: number; - imageKind: "compute#image"; - project: string; - imageName: string; - imageId: string; - imageSelfLink: string; - status: "READY"; - imageCreationTimestamp: string; - manifestCreatedAt: string; - channel: "staging"; - variant: "cpu"; - observedFamily: typeof EXACT_IMAGE_STAGING_FAMILY; - result: "built" | "reused"; -}; - -export type ExactImageManifestExpectations = { - correlationId: string; - requesterWorkflowRunId: string; - requesterWorkflowRunAttempt: number; - nemoclawSha: string; - imageRepositorySha: string; - workflowRunId: string; - workflowRunAttempt: number; -}; - -const REQUIRED_FIELDS = [ - "schemaVersion", - "kind", - "correlationId", - "requesterRepository", - "requesterWorkflowRunId", - "requesterWorkflowRunAttempt", - "nemoclawSha", - "imageRepository", - "imageRepositorySha", - "producerWorkflow", - "workflowRunId", - "workflowRunAttempt", - "imageOriginWorkflowRunId", - "imageOriginWorkflowRunAttempt", - "imageKind", - "project", - "imageName", - "imageId", - "imageSelfLink", - "status", - "imageCreationTimestamp", - "manifestCreatedAt", - "channel", - "variant", - "observedFamily", - "result", -] as const; - -const REQUIRED_FIELD_SET = new Set(REQUIRED_FIELDS); -const FULL_SHA_PATTERN = /^[0-9a-f]{40}$/u; -const UUID_V4_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/u; -const DECIMAL_ID_PATTERN = /^[1-9][0-9]*$/u; -const GCP_PROJECT_ID_PATTERN = /^[a-z][a-z0-9-]{4,28}[a-z0-9]$/u; -const IMAGE_NAME_PATTERN = /^[a-z](?:[-a-z0-9]{0,61}[a-z0-9])?$/u; -const RFC3339_PATTERN = - /^(\d{4})-(\d{2})-(\d{2})T([01]\d|2[0-3]):([0-5]\d):([0-5]\d)(?:[.](\d{1,9}))?(Z|[+-](?:[01]\d|2[0-3]):[0-5]\d)$/u; - -function fail(code: ExactImageManifestFailureCode, message: string): never { - throw new ExactImageManifestError(code, message); -} - -function requireRecord(value: unknown): Record { - if (value === null || typeof value !== "object" || Array.isArray(value)) { - fail("ARTIFACT_MISSING_OR_INVALID", "manifest must be a JSON object"); - } - return value as Record; -} - -function validateExactFields(record: Record): void { - for (const field of REQUIRED_FIELDS) { - if (!Object.hasOwn(record, field)) { - fail("ARTIFACT_MISSING_OR_INVALID", `manifest is missing required field ${field}`); - } - } - const unexpected = Object.keys(record) - .filter((field) => !REQUIRED_FIELD_SET.has(field)) - .sort(); - if (unexpected.length > 0) { - fail("ARTIFACT_MISSING_OR_INVALID", `manifest contains unexpected field ${unexpected[0]}`); - } -} - -function requireString(record: Record, field: string): string { - const value = record[field]; - if (typeof value !== "string") { - fail("ARTIFACT_MISSING_OR_INVALID", `${field} must be a string`); - } - return value; -} - -function requirePositiveInteger(record: Record, field: string): number { - const value = record[field]; - if (!Number.isSafeInteger(value) || (value as number) < 1) { - fail("ARTIFACT_MISSING_OR_INVALID", `${field} must be a positive safe integer`); - } - return value as number; -} - -function requirePattern(value: string, field: string, pattern: RegExp): void { - if (!pattern.test(value)) { - fail("ARTIFACT_MISSING_OR_INVALID", `${field} has an invalid format`); - } -} - -function requireConstant( - actual: unknown, - expected: T, - field: string, -): asserts actual is T { - if (actual !== expected) { - fail("PROVENANCE_MISMATCH", `${field} must equal ${JSON.stringify(expected)}`); - } -} - -function daysInMonth(year: number, month: number): number { - if (month === 2) { - const leap = year % 4 === 0 && (year % 100 !== 0 || year % 400 === 0); - return leap ? 29 : 28; - } - return [4, 6, 9, 11].includes(month) ? 30 : 31; -} - -function parseRfc3339(value: string, field: string): number { - const match = RFC3339_PATTERN.exec(value); - if (!match) { - fail("ARTIFACT_MISSING_OR_INVALID", `${field} must be an RFC 3339 date-time`); - } - const year = Number(match[1]); - const month = Number(match[2]); - const day = Number(match[3]); - if (month < 1 || month > 12 || day < 1 || day > daysInMonth(year, month)) { - fail("ARTIFACT_MISSING_OR_INVALID", `${field} must be a real calendar date`); - } - const timestamp = Date.parse(value); - if (!Number.isFinite(timestamp)) { - fail("ARTIFACT_MISSING_OR_INVALID", `${field} must be an RFC 3339 date-time`); - } - return timestamp; -} - -function validateExpectations(expected: ExactImageManifestExpectations): void { - const stringPatterns: Array<[string, string, RegExp]> = [ - [expected.correlationId, "expected correlationId", UUID_V4_PATTERN], - [expected.requesterWorkflowRunId, "expected requesterWorkflowRunId", DECIMAL_ID_PATTERN], - [expected.nemoclawSha, "expected nemoclawSha", FULL_SHA_PATTERN], - [expected.imageRepositorySha, "expected imageRepositorySha", FULL_SHA_PATTERN], - [expected.workflowRunId, "expected workflowRunId", DECIMAL_ID_PATTERN], - ]; - for (const [value, field, pattern] of stringPatterns) { - if (typeof value !== "string" || !pattern.test(value)) { - fail("REQUEST_INVALID", `${field} has an invalid format`); - } - } - for (const [value, field] of [ - [expected.requesterWorkflowRunAttempt, "expected requesterWorkflowRunAttempt"], - [expected.workflowRunAttempt, "expected workflowRunAttempt"], - ] as const) { - if (!Number.isSafeInteger(value) || value < 1) { - fail("REQUEST_INVALID", `${field} must be a positive safe integer`); - } - } -} - -function assertExpected( - manifest: ExactImageManifest, - expected: ExactImageManifestExpectations, -): void { - const comparisons: Array<[unknown, unknown, string]> = [ - [manifest.correlationId, expected.correlationId, "correlationId"], - [manifest.requesterWorkflowRunId, expected.requesterWorkflowRunId, "requesterWorkflowRunId"], - [ - manifest.requesterWorkflowRunAttempt, - expected.requesterWorkflowRunAttempt, - "requesterWorkflowRunAttempt", - ], - [manifest.nemoclawSha, expected.nemoclawSha, "nemoclawSha"], - [manifest.imageRepositorySha, expected.imageRepositorySha, "imageRepositorySha"], - [manifest.workflowRunId, expected.workflowRunId, "workflowRunId"], - [manifest.workflowRunAttempt, expected.workflowRunAttempt, "workflowRunAttempt"], - ]; - for (const [actual, wanted, field] of comparisons) { - if (actual !== wanted) { - fail("PROVENANCE_MISMATCH", `${field} does not match the trusted request`); - } - } -} - -function validateTemporalContract(manifest: ExactImageManifest): void { - const imageCreated = parseRfc3339(manifest.imageCreationTimestamp, "imageCreationTimestamp"); - const manifestCreated = parseRfc3339(manifest.manifestCreatedAt, "manifestCreatedAt"); - if (imageCreated > manifestCreated + EXACT_IMAGE_MANIFEST_MAX_CLOCK_SKEW_MS) { - fail( - "PROVENANCE_MISMATCH", - "imageCreationTimestamp is later than manifestCreatedAt beyond allowed clock skew", - ); - } - if ( - manifest.result === "reused" && - manifestCreated - imageCreated > EXACT_IMAGE_MANIFEST_MAX_REUSED_AGE_MS - ) { - fail("PROVENANCE_MISMATCH", "reused image is older than 24 hours"); - } -} - -function buildManifest(record: Record): ExactImageManifest { - requireConstant(record.schemaVersion, 1, "schemaVersion"); - requireConstant(record.kind, EXACT_IMAGE_MANIFEST_KIND, "kind"); - requireConstant( - record.requesterRepository, - EXACT_IMAGE_REQUESTER_REPOSITORY, - "requesterRepository", - ); - requireConstant(record.imageRepository, EXACT_IMAGE_REPOSITORY, "imageRepository"); - requireConstant(record.producerWorkflow, EXACT_IMAGE_PRODUCER_WORKFLOW, "producerWorkflow"); - requireConstant(record.imageKind, "compute#image", "imageKind"); - requireConstant(record.status, "READY", "status"); - requireConstant(record.channel, "staging", "channel"); - - const variant = requireString(record, "variant"); - if (variant !== "cpu") { - fail("UNSUPPORTED_VARIANT", 'variant must equal "cpu"'); - } - requireConstant(record.observedFamily, EXACT_IMAGE_STAGING_FAMILY, "observedFamily"); - - const result = requireString(record, "result"); - if (result !== "built" && result !== "reused") { - fail("ARTIFACT_MISSING_OR_INVALID", 'result must equal "built" or "reused"'); - } - - const manifest: ExactImageManifest = { - schemaVersion: 1, - kind: EXACT_IMAGE_MANIFEST_KIND, - correlationId: requireString(record, "correlationId"), - requesterRepository: EXACT_IMAGE_REQUESTER_REPOSITORY, - requesterWorkflowRunId: requireString(record, "requesterWorkflowRunId"), - requesterWorkflowRunAttempt: requirePositiveInteger(record, "requesterWorkflowRunAttempt"), - nemoclawSha: requireString(record, "nemoclawSha"), - imageRepository: EXACT_IMAGE_REPOSITORY, - imageRepositorySha: requireString(record, "imageRepositorySha"), - producerWorkflow: EXACT_IMAGE_PRODUCER_WORKFLOW, - workflowRunId: requireString(record, "workflowRunId"), - workflowRunAttempt: requirePositiveInteger(record, "workflowRunAttempt"), - imageOriginWorkflowRunId: requireString(record, "imageOriginWorkflowRunId"), - imageOriginWorkflowRunAttempt: requirePositiveInteger(record, "imageOriginWorkflowRunAttempt"), - imageKind: "compute#image", - project: requireString(record, "project"), - imageName: requireString(record, "imageName"), - imageId: requireString(record, "imageId"), - imageSelfLink: requireString(record, "imageSelfLink"), - status: "READY", - imageCreationTimestamp: requireString(record, "imageCreationTimestamp"), - manifestCreatedAt: requireString(record, "manifestCreatedAt"), - channel: "staging", - variant, - observedFamily: EXACT_IMAGE_STAGING_FAMILY, - result, - }; - - requirePattern(manifest.correlationId, "correlationId", UUID_V4_PATTERN); - requirePattern(manifest.requesterWorkflowRunId, "requesterWorkflowRunId", DECIMAL_ID_PATTERN); - requirePattern(manifest.nemoclawSha, "nemoclawSha", FULL_SHA_PATTERN); - requirePattern(manifest.imageRepositorySha, "imageRepositorySha", FULL_SHA_PATTERN); - requirePattern(manifest.workflowRunId, "workflowRunId", DECIMAL_ID_PATTERN); - requirePattern(manifest.imageOriginWorkflowRunId, "imageOriginWorkflowRunId", DECIMAL_ID_PATTERN); - if (!GCP_PROJECT_ID_PATTERN.test(manifest.project)) { - fail("IMAGE_IDENTITY_MISMATCH", "project must be a canonical GCP project ID"); - } - requirePattern(manifest.imageName, "imageName", IMAGE_NAME_PATTERN); - requirePattern(manifest.imageId, "imageId", DECIMAL_ID_PATTERN); - return manifest; -} - -function validateImageSelfLink(manifest: ExactImageManifest): void { - const expectedPath = `/compute/v1/projects/${manifest.project}/global/images/${manifest.imageName}`; - const expectedSelfLink = `https://www.googleapis.com${expectedPath}`; - let parsed: URL; - try { - parsed = new URL(manifest.imageSelfLink); - } catch { - fail("IMAGE_IDENTITY_MISMATCH", "imageSelfLink must be an absolute URL"); - } - if ( - parsed.protocol !== "https:" || - parsed.hostname !== "www.googleapis.com" || - parsed.port !== "" || - parsed.username !== "" || - parsed.password !== "" || - parsed.search !== "" || - parsed.hash !== "" || - parsed.pathname !== expectedPath || - manifest.imageSelfLink !== expectedSelfLink - ) { - fail("IMAGE_IDENTITY_MISMATCH", "imageSelfLink does not exactly identify project/imageName"); - } -} - -export function parseExactImageManifestJson(contents: string): unknown { - try { - return JSON.parse(contents) as unknown; - } catch { - fail("ARTIFACT_MISSING_OR_INVALID", "manifest is not valid JSON"); - } -} - -export function validateExactImageManifest( - value: unknown, - expected: ExactImageManifestExpectations, -): ExactImageManifest { - validateExpectations(expected); - const record = requireRecord(value); - validateExactFields(record); - const manifest = buildManifest(record); - - validateImageSelfLink(manifest); - if ( - manifest.result === "built" && - (manifest.imageOriginWorkflowRunId !== manifest.workflowRunId || - manifest.imageOriginWorkflowRunAttempt !== manifest.workflowRunAttempt) - ) { - fail( - "PROVENANCE_MISMATCH", - "built image origin run and attempt must match the current producer run", - ); - } - - validateTemporalContract(manifest); - assertExpected(manifest, expected); - return manifest; -} - -export function parseAndValidateExactImageManifest( - contents: string, - expected: ExactImageManifestExpectations, -): ExactImageManifest { - return validateExactImageManifest(parseExactImageManifestJson(contents), expected); -} - -export function normalizedExactImageManifestJson(manifest: ExactImageManifest): string { - return `${JSON.stringify(manifest, null, 2)}\n`; -} - -export function exactImageManifestFailureCode(error: unknown): ExactImageManifestFailureCode { - return error instanceof ExactImageManifestError ? error.code : "UNKNOWN"; -} diff --git a/tools/e2e/exact-image-qualification-controller.mts b/tools/e2e/exact-image-qualification-controller.mts deleted file mode 100755 index aeb4cdf62e..0000000000 --- a/tools/e2e/exact-image-qualification-controller.mts +++ /dev/null @@ -1,2076 +0,0 @@ -#!/usr/bin/env node - -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { spawnSync } from "node:child_process"; -import { createHash, randomUUID } from "node:crypto"; -import fs from "node:fs"; -import path from "node:path"; -import { pathToFileURL } from "node:url"; - -import { type GitHubRequestOptions, githubApi } from "../advisors/github.mts"; -import * as privateFile from "./private-file.mts"; - -// Root .ts files are exposed as CommonJS under tsx and as ESM under Node's -// strip-types runtime. Normalize both forms so the workflow uses the same -// no-follow state-file helpers in either test or production execution. -const privateFileRuntime = ( - "default" in privateFile && privateFile.default ? privateFile.default : privateFile -) as typeof import("./private-file.mts"); - -export const REQUESTER_REPOSITORY = "NVIDIA/NemoClaw"; -export const PRODUCER_REPOSITORY = "brevdev/nemoclaw-image"; -export const PRODUCER_WORKFLOW_FILE = "build-qualification-image.yml"; -export const PRODUCER_WORKFLOW_PATH = `.github/workflows/${PRODUCER_WORKFLOW_FILE}`; -export const PRODUCER_REF = "main"; -export const GITHUB_API_VERSION = "2026-03-10"; -export const MANIFEST_ARTIFACT_FILE = "nemoclaw-image-manifest.v1.json"; -export const ARCHIVE_FILE = "nemoclaw-image-handoff.zip"; -export const VALIDATED_MANIFEST_FILE = "validated-manifest.v1.json"; -export const EVIDENCE_FILE = "qualification-evidence.v1.json"; -export const STATE_FILE = "controller-state.json"; -export const DISPATCH_INTENT_FILE = "dispatch-intent.v1.json"; -export const DISPATCH_RECONCILIATION_FILE = "dispatch-reconciliation.v1.json"; - -const FULL_SHA_PATTERN = /^[0-9a-f]{40}$/u; -const UUID_V4_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/u; -const DECIMAL_ID_PATTERN = /^[1-9][0-9]*$/u; -const ARTIFACT_DIGEST_PATTERN = /^sha256:[0-9a-f]{64}$/u; -const SHA256_PATTERN = /^[0-9a-f]{64}$/u; -const GITHUB_LOGIN_PATTERN = /^(?!-)[A-Za-z0-9-]{1,39}(?( - apiPath: string, - token: string, - options?: GitHubRequestOptions, -) => Promise; - -type CommandResult = { - status: number | null; - signal?: NodeJS.Signals | null; - stdout: Buffer | string; - stderr: Buffer | string; - error?: Error; -}; - -type CommandRunner = (command: string, args: readonly string[]) => CommandResult; - -export type QualificationDependencies = { - api?: GitHubApiClient; - fetch?: typeof fetch; - now?: () => number; - randomUuid?: () => string; - runCommand?: CommandRunner; - sleep?: (milliseconds: number) => Promise; - limits?: Partial<{ - artifactPropagationTimeoutMs: number; - apiRequestTimeoutMs: number; - cancelReserveMs: number; - cleanupTimeoutMs: number; - dispatchReconciliationTimeoutMs: number; - downloadTimeoutMs: number; - pollIntervalMs: number; - qualificationTimeoutMs: number; - queueTimeoutMs: number; - reconciliationPollIntervalMs: number; - warningAfterMs: number; - }>; -}; - -export type ExactImageQualificationRequest = { - actor: string; - candidateSha: string; - eventName: string; - reason: string; - ref: string; - requesterRunAttempt: number; - requesterRunId: string; - workflowSha: string; -}; - -export type DispatchDetails = { - workflowRunId: string; - runUrl: string; - htmlUrl: string; -}; - -export type ExactImageDispatchIntent = { - schemaVersion: 1; - kind: "nemoclaw-exact-image-dispatch-intent"; - requestStartedAt: string; - request: { - actor: string; - candidateSha: string; - correlationId: string; - reason: string; - requesterRunAttempt: number; - requesterRunId: string; - workflowSha: string; - }; - producer: { - repository: typeof PRODUCER_REPOSITORY; - repositorySha: string; - ref: typeof PRODUCER_REF; - workflowId: string; - workflowPath: typeof PRODUCER_WORKFLOW_PATH; - }; -}; - -export type QualificationWorkflowRun = { - id: string; - workflowId: string; - headSha: string; - runAttempt: number; - status: string; - conclusion: string | null; - url: string; - htmlUrl: string; -}; - -export type QualificationArtifact = { - id: string; - name: string; - digest: string; - sizeInBytes: number; - apiUrl: string; - archiveDownloadUrl: string; -}; - -export type ExactImageQualificationState = { - schemaVersion: 1; - status: "dispatched" | "completed" | "downloaded" | "validated"; - dispatchedAt: string; - request: { - actor: string; - candidateSha: string; - correlationId: string; - reason: string; - requesterRunAttempt: number; - requesterRunId: string; - workflowSha: string; - }; - producer: { - repository: typeof PRODUCER_REPOSITORY; - repositorySha: string; - ref: typeof PRODUCER_REF; - workflowPath: typeof PRODUCER_WORKFLOW_PATH; - runId: string; - runAttempt: 1; - workflowId?: string; - runUrl: string; - htmlUrl: string; - completedAt?: string; - }; - artifact?: QualificationArtifact & { - archiveSha256: string; - manifestSha256: string; - }; - validation?: { - acceptedAt: string; - manifestSha256: string; - normalizedManifestSha256: string; - }; -}; - -type ControllerCommand = - | { mode: "preflight"; request: ExactImageQualificationRequest } - | { - mode: "start"; - request: ExactImageQualificationRequest; - workDir: string; - } - | { mode: "wait" | "download" | "finalize" | "cancel"; workDir: string }; - -function fail(code: ExactImageQualificationFailureCode, message: string): never { - throw new ExactImageQualificationError(code, message); -} - -function record(value: unknown, label: string): Record { - if (value === null || typeof value !== "object" || Array.isArray(value)) { - fail("PROVENANCE_MISMATCH", `${label} must be an object`); - } - return value as Record; -} - -function requireExactRecordFields( - value: Record, - fields: readonly string[], - label: string, -): void { - const allowed = new Set(fields); - for (const field of fields) { - if (!Object.hasOwn(value, field)) { - fail("PROVENANCE_MISMATCH", `${label} is missing ${field}`); - } - } - const unexpected = Object.keys(value) - .filter((field) => !allowed.has(field)) - .sort(); - if (unexpected.length > 0) { - fail("PROVENANCE_MISMATCH", `${label} contains unexpected field ${unexpected[0]}`); - } -} - -function persistedString( - value: Record, - field: string, - label: string, - pattern?: RegExp, -): string { - const result = value[field]; - if (typeof result !== "string" || (pattern !== undefined && !pattern.test(result))) { - fail("PROVENANCE_MISMATCH", `${label} has an invalid format`); - } - return result; -} - -function persistedTimestamp(value: unknown, label: string): number { - if (typeof value !== "string") { - fail("PROVENANCE_MISMATCH", `${label} must be a timestamp string`); - } - const parsed = Date.parse(value); - if (!Number.isFinite(parsed) || new Date(parsed).toISOString() !== value) { - fail("PROVENANCE_MISMATCH", `${label} must be a canonical UTC timestamp`); - } - return parsed; -} - -function validatePersistedReason(value: unknown, label: string): void { - if ( - typeof value !== "string" || - value.length === 0 || - value !== value.trim() || - Buffer.byteLength(value, "utf8") > MAX_REASON_BYTES || - /[\u0000-\u001f\u007f]/u.test(value) - ) { - fail("PROVENANCE_MISMATCH", `${label} is invalid`); - } -} - -function safePositiveId(value: unknown, label: string): string { - if (!Number.isSafeInteger(value) || (value as number) < 1) { - fail("PROVENANCE_MISMATCH", `${label} must be a positive safe integer`); - } - return String(value); -} - -function positiveInteger(value: string, label: string): number { - if (!DECIMAL_ID_PATTERN.test(value)) fail("REQUEST_INVALID", `${label} must be positive`); - const parsed = Number(value); - if (!Number.isSafeInteger(parsed)) fail("REQUEST_INVALID", `${label} is outside the safe range`); - return parsed; -} - -function requireExactString(value: unknown, expected: string, label: string): void { - if (value !== expected) fail("PROVENANCE_MISMATCH", `${label} did not match the trusted request`); -} - -function sha256(bytes: Uint8Array): string { - return createHash("sha256").update(bytes).digest("hex"); -} - -function timestamp(now: () => number): string { - return new Date(now()).toISOString(); -} - -function sleep(milliseconds: number): Promise { - return new Promise((resolve) => setTimeout(resolve, milliseconds)); -} - -function limits(dependencies: QualificationDependencies) { - return { - artifactPropagationTimeoutMs: - dependencies.limits?.artifactPropagationTimeoutMs ?? ARTIFACT_PROPAGATION_TIMEOUT_MS, - apiRequestTimeoutMs: dependencies.limits?.apiRequestTimeoutMs ?? API_REQUEST_TIMEOUT_MS, - cancelReserveMs: dependencies.limits?.cancelReserveMs ?? CANCEL_RESERVE_MS, - cleanupTimeoutMs: dependencies.limits?.cleanupTimeoutMs ?? CLEANUP_TIMEOUT_MS, - dispatchReconciliationTimeoutMs: - dependencies.limits?.dispatchReconciliationTimeoutMs ?? DISPATCH_RECONCILIATION_TIMEOUT_MS, - downloadTimeoutMs: dependencies.limits?.downloadTimeoutMs ?? DOWNLOAD_TIMEOUT_MS, - pollIntervalMs: dependencies.limits?.pollIntervalMs ?? POLL_INTERVAL_MS, - qualificationTimeoutMs: dependencies.limits?.qualificationTimeoutMs ?? QUALIFICATION_TIMEOUT_MS, - queueTimeoutMs: dependencies.limits?.queueTimeoutMs ?? QUEUE_TIMEOUT_MS, - reconciliationPollIntervalMs: - dependencies.limits?.reconciliationPollIntervalMs ?? RECONCILIATION_POLL_INTERVAL_MS, - warningAfterMs: dependencies.limits?.warningAfterMs ?? WARNING_AFTER_MS, - }; -} - -function qualificationDeadline( - state: ExactImageQualificationState, - qualificationTimeoutMs: number, -): number { - const dispatchedAt = Date.parse(state.dispatchedAt); - if (!Number.isFinite(dispatchedAt)) { - fail("PROVENANCE_MISMATCH", "controller state has an invalid dispatch timestamp"); - } - return dispatchedAt + qualificationTimeoutMs; -} - -function requireQualificationTimeRemaining(deadline: number, now: () => number): number { - const remaining = deadline - now(); - if (remaining <= 0) { - fail("QUALIFICATION_TIMEOUT", "qualification exceeded the shared 45-minute budget"); - } - return remaining; -} - -function boundedApiClient( - api: GitHubApiClient, - dependencies: QualificationDependencies, - deadline?: number, -): GitHubApiClient { - const now = dependencies.now ?? Date.now; - const requestCap = limits(dependencies).apiRequestTimeoutMs; - return async (apiPath: string, token: string, options: GitHubRequestOptions = {}) => { - const remaining = deadline === undefined ? requestCap : deadline - now(); - if (remaining <= 0) { - fail("QUALIFICATION_TIMEOUT", "no time remains for the bounded GitHub API request"); - } - const controller = new AbortController(); - const timeout = Math.min(requestCap, remaining); - const timer = setTimeout(() => controller.abort(), timeout); - timer.unref?.(); - const signal = options.signal - ? AbortSignal.any([options.signal, controller.signal]) - : controller.signal; - try { - return await api(apiPath, token, { ...options, signal }); - } catch (error) { - if (controller.signal.aborted) { - fail("QUALIFICATION_TIMEOUT", `GitHub API request exceeded its ${timeout}ms budget`); - } - throw error; - } finally { - clearTimeout(timer); - } - }; -} - -function acceptanceDeadline( - state: ExactImageQualificationState, - dependencies: QualificationDependencies, -): number { - const configured = limits(dependencies); - return ( - qualificationDeadline(state, configured.qualificationTimeoutMs) - configured.cancelReserveMs - ); -} - -export function validateExactImageQualificationRequest( - request: ExactImageQualificationRequest, -): ExactImageQualificationRequest { - if (request.eventName !== "workflow_dispatch" && request.eventName !== "schedule") { - fail("REQUEST_INVALID", "qualification must be started by workflow_dispatch or schedule"); - } - if (request.ref !== "refs/heads/main") { - fail("REQUEST_INVALID", "qualification must run from the trusted main branch"); - } - if (!Number.isSafeInteger(request.requesterRunAttempt) || request.requesterRunAttempt < 1) { - fail("REQUEST_INVALID", "requester run attempt must be a positive integer"); - } - if (!FULL_SHA_PATTERN.test(request.candidateSha)) { - fail("REQUEST_INVALID", "candidate SHA must be a lowercase full commit SHA"); - } - if (!FULL_SHA_PATTERN.test(request.workflowSha)) { - fail("REQUEST_INVALID", "workflow SHA must be a lowercase full commit SHA"); - } - if (request.candidateSha !== request.workflowSha) { - fail("REQUEST_INVALID", "candidate SHA must equal the trusted main workflow SHA"); - } - if (!DECIMAL_ID_PATTERN.test(request.requesterRunId)) { - fail("REQUEST_INVALID", "requester run ID must be a positive decimal string"); - } - if (!GITHUB_LOGIN_PATTERN.test(request.actor)) { - fail("REQUEST_INVALID", "triggering actor has an invalid GitHub login"); - } - if ( - request.reason.length === 0 || - request.reason !== request.reason.trim() || - Buffer.byteLength(request.reason, "utf8") > MAX_REASON_BYTES || - /[\u0000-\u001f\u007f]/u.test(request.reason) - ) { - fail("REQUEST_INVALID", "reason must be trimmed, nonempty, bounded text without controls"); - } - return request; -} - -function validateRequesterRef(value: unknown, repository: string, expectedRef: string): string { - const payload = record(value, `${repository} requester ref`); - requireExactString(payload.ref, expectedRef, `${repository} ref`); - const object = record(payload.object, `${repository} ref object`); - requireExactString(object.type, "commit", `${repository} ref object type`); - if (typeof object.sha !== "string" || !FULL_SHA_PATTERN.test(object.sha)) { - fail("PROVENANCE_MISMATCH", `${repository} main did not resolve to a full commit SHA`); - } - return object.sha; -} - -function validateMainRef(value: unknown, repository: string): string { - return validateRequesterRef(value, repository, "refs/heads/main"); -} - -function validateProducerWorkflow(value: unknown): string { - const workflow = record(value, "producer workflow"); - const workflowId = safePositiveId(workflow.id, "producer workflow ID"); - requireExactString(workflow.path, PRODUCER_WORKFLOW_PATH, "producer workflow path"); - requireExactString(workflow.state, "active", "producer workflow state"); - return workflowId; -} - -async function authorizeRequest( - request: ExactImageQualificationRequest, - token: string, - api: GitHubApiClient, -): Promise { - validateExactImageQualificationRequest(request); - const branch = await api(`repos/${REQUESTER_REPOSITORY}/git/ref/heads/main`, token); - if (validateMainRef(branch, REQUESTER_REPOSITORY) !== request.workflowSha) { - fail("DISPATCH_FORBIDDEN", "workflow SHA is no longer the trusted main branch head"); - } - const candidate = record( - await api(`repos/${REQUESTER_REPOSITORY}/git/commits/${request.candidateSha}`, token), - "candidate commit", - ); - if (candidate.sha !== request.candidateSha) { - fail("DISPATCH_FORBIDDEN", "candidate SHA does not identify an exact NemoClaw commit"); - } - const permission = record( - await api( - `repos/${REQUESTER_REPOSITORY}/collaborators/${encodeURIComponent(request.actor)}/permission`, - token, - ), - "collaborator permission", - ); - if (permission.permission !== "admin" && permission.role_name !== "maintain") { - fail("DISPATCH_FORBIDDEN", "triggering actor must have maintain or admin permission"); - } -} - -export async function preflightExactImageQualification( - request: ExactImageQualificationRequest, - token: string, - dependencies: QualificationDependencies = {}, -): Promise { - await authorizeRequest( - request, - token, - boundedApiClient(dependencies.api ?? githubApi, dependencies), - ); -} - -export function validateWorkflowDispatchDetails(value: unknown): DispatchDetails { - const payload = record(value, "workflow dispatch response"); - const workflowRunId = safePositiveId(payload.workflow_run_id, "workflow_run_id"); - const runUrl = `https://api.github.com/repos/${PRODUCER_REPOSITORY}/actions/runs/${workflowRunId}`; - const htmlUrl = `https://github.com/${PRODUCER_REPOSITORY}/actions/runs/${workflowRunId}`; - requireExactString(payload.run_url, runUrl, "dispatch run_url"); - requireExactString(payload.html_url, htmlUrl, "dispatch html_url"); - return { workflowRunId, runUrl, htmlUrl }; -} - -export function validateQualificationWorkflowRun( - value: unknown, - expected: { - candidateSha: string; - correlationId: string; - producerSha: string; - runId: string; - runUrl: string; - htmlUrl: string; - workflowId?: string; - }, -): QualificationWorkflowRun { - const run = record(value, "producer workflow run"); - const id = safePositiveId(run.id, "producer run ID"); - if (id !== expected.runId) fail("PROVENANCE_MISMATCH", "producer run ID changed"); - const workflowId = safePositiveId(run.workflow_id, "producer workflow ID"); - if (expected.workflowId !== undefined && workflowId !== expected.workflowId) { - fail("PROVENANCE_MISMATCH", "producer workflow ID did not match the dispatch intent"); - } - if (run.run_attempt !== 1) fail("PROVENANCE_MISMATCH", "producer run attempt must equal 1"); - requireExactString(run.event, "workflow_dispatch", "producer event"); - requireExactString(run.head_branch, PRODUCER_REF, "producer head branch"); - requireExactString(run.head_sha, expected.producerSha, "producer head SHA"); - requireExactString(run.path, PRODUCER_WORKFLOW_PATH, "producer workflow path"); - requireExactString( - run.display_title, - `Qualify NemoClaw ${expected.candidateSha} (${expected.correlationId})`, - "producer display title", - ); - requireExactString(run.url, expected.runUrl, "producer run URL"); - requireExactString(run.html_url, expected.htmlUrl, "producer HTML URL"); - requireExactString( - record(run.repository, "producer repository").full_name, - PRODUCER_REPOSITORY, - "producer repository", - ); - requireExactString( - record(run.head_repository, "producer head repository").full_name, - PRODUCER_REPOSITORY, - "producer head repository", - ); - const allowedStatuses = new Set([ - "queued", - "in_progress", - "completed", - "waiting", - "requested", - "pending", - ]); - if (typeof run.status !== "string" || !allowedStatuses.has(run.status)) { - fail("PROVENANCE_MISMATCH", "producer run has an unsupported status"); - } - if (run.conclusion !== null && typeof run.conclusion !== "string") { - fail("PROVENANCE_MISMATCH", "producer conclusion must be a string or null"); - } - return { - id, - workflowId, - headSha: expected.producerSha, - runAttempt: 1, - status: run.status, - conclusion: run.conclusion as string | null, - url: expected.runUrl, - htmlUrl: expected.htmlUrl, - }; -} - -function validateCancellationWorkflowRun( - value: unknown, - state: ExactImageQualificationState, -): { status: string; conclusion: string | null } { - const run = record(value, "producer cancellation workflow run"); - const observedHeadSha = persistedString( - run, - "head_sha", - "producer cancellation head SHA", - FULL_SHA_PATTERN, - ); - const validated = validateQualificationWorkflowRun(value, { - ...expectedRun(state), - // A run whose producer ref moved after dispatch remains cleanup-owned but - // can never become qualification evidence; all other identity must bind. - producerSha: observedHeadSha, - }); - return { status: validated.status, conclusion: validated.conclusion }; -} - -function statePath(workDir: string): string { - return path.join(workDir, STATE_FILE); -} - -function intentPath(workDir: string): string { - return path.join(workDir, DISPATCH_INTENT_FILE); -} - -function writeDispatchIntent(workDir: string, intent: ExactImageDispatchIntent): void { - try { - privateFileRuntime.writePrivateRegularFile( - intentPath(workDir), - `${JSON.stringify(intent, null, 2)}\n`, - ); - } catch { - fail("OUTPUT_WRITE_FAILED", "dispatch intent could not be written safely"); - } -} - -function readDispatchIntent(workDir: string): ExactImageDispatchIntent | null { - let contents: string | null; - try { - contents = privateFileRuntime.readPrivateRegularFile(intentPath(workDir), { - allowMissing: true, - maxBytes: MAX_STATE_BYTES, - }); - } catch { - fail("PROVENANCE_MISMATCH", "dispatch intent could not be read safely"); - } - if (contents === null) return null; - let parsed: unknown; - try { - parsed = JSON.parse(contents) as unknown; - } catch { - fail("PROVENANCE_MISMATCH", "dispatch intent is not valid JSON"); - } - const intent = record(parsed, "dispatch intent"); - requireExactRecordFields( - intent, - ["schemaVersion", "kind", "requestStartedAt", "request", "producer"], - "dispatch intent", - ); - if (intent.schemaVersion !== 1 || intent.kind !== "nemoclaw-exact-image-dispatch-intent") { - fail("PROVENANCE_MISMATCH", "dispatch intent schema identity is invalid"); - } - persistedTimestamp(intent.requestStartedAt, "dispatch intent requestStartedAt"); - - const request = record(intent.request, "dispatch intent request"); - requireExactRecordFields( - request, - [ - "actor", - "candidateSha", - "correlationId", - "reason", - "requesterRunAttempt", - "requesterRunId", - "workflowSha", - ], - "dispatch intent request", - ); - persistedString(request, "actor", "dispatch intent actor", GITHUB_LOGIN_PATTERN); - persistedString(request, "candidateSha", "dispatch intent candidate SHA", FULL_SHA_PATTERN); - persistedString(request, "correlationId", "dispatch intent correlation ID", UUID_V4_PATTERN); - validatePersistedReason(request.reason, "dispatch intent reason"); - const requesterRunAttempt = request.requesterRunAttempt; - if ( - typeof requesterRunAttempt !== "number" || - !Number.isSafeInteger(requesterRunAttempt) || - requesterRunAttempt < 1 - ) { - fail("PROVENANCE_MISMATCH", "dispatch intent requester run attempt must be positive"); - } - persistedString( - request, - "requesterRunId", - "dispatch intent requester run ID", - DECIMAL_ID_PATTERN, - ); - persistedString(request, "workflowSha", "dispatch intent workflow SHA", FULL_SHA_PATTERN); - - const producer = record(intent.producer, "dispatch intent producer"); - requireExactRecordFields( - producer, - ["repository", "repositorySha", "ref", "workflowId", "workflowPath"], - "dispatch intent producer", - ); - requireExactString(producer.repository, PRODUCER_REPOSITORY, "dispatch intent repository"); - persistedString(producer, "repositorySha", "dispatch intent producer SHA", FULL_SHA_PATTERN); - requireExactString(producer.ref, PRODUCER_REF, "dispatch intent producer ref"); - persistedString(producer, "workflowId", "dispatch intent workflow ID", DECIMAL_ID_PATTERN); - requireExactString( - producer.workflowPath, - PRODUCER_WORKFLOW_PATH, - "dispatch intent workflow path", - ); - return intent as unknown as ExactImageDispatchIntent; -} - -function requiredDispatchIntent(workDir: string): ExactImageDispatchIntent { - const intent = readDispatchIntent(workDir); - if (intent === null) fail("PROVENANCE_MISMATCH", "dispatch intent is missing"); - return intent; -} - -function validateStateIntentBinding( - state: ExactImageQualificationState, - intent: ExactImageDispatchIntent, -): void { - const comparisons: Array<[unknown, unknown, string]> = [ - [state.dispatchedAt, intent.requestStartedAt, "dispatch timestamp"], - [state.request.actor, intent.request.actor, "actor"], - [state.request.candidateSha, intent.request.candidateSha, "candidate SHA"], - [state.request.correlationId, intent.request.correlationId, "correlation ID"], - [state.request.reason, intent.request.reason, "reason"], - [ - state.request.requesterRunAttempt, - intent.request.requesterRunAttempt, - "requester run attempt", - ], - [state.request.requesterRunId, intent.request.requesterRunId, "requester run ID"], - [state.request.workflowSha, intent.request.workflowSha, "workflow SHA"], - [state.producer.repository, intent.producer.repository, "producer repository"], - [state.producer.repositorySha, intent.producer.repositorySha, "producer SHA"], - [state.producer.ref, intent.producer.ref, "producer ref"], - [state.producer.workflowId, intent.producer.workflowId, "producer workflow ID"], - [state.producer.workflowPath, intent.producer.workflowPath, "producer workflow path"], - ]; - for (const [actual, expected, label] of comparisons) { - if (actual !== expected) { - fail("PROVENANCE_MISMATCH", `controller state ${label} does not match dispatch intent`); - } - } -} - -function readBoundExactImageQualificationState(workDir: string): ExactImageQualificationState { - const state = readExactImageQualificationState(workDir); - validateStateIntentBinding(state, requiredDispatchIntent(workDir)); - return state; -} - -function writeDispatchReconciliation( - workDir: string, - intent: ExactImageDispatchIntent, - runs: readonly QualificationWorkflowRun[], - outcome: "none" | "recovered-one" | "multiple", - now: () => number, -): void { - const evidence = { - schemaVersion: 1, - kind: "nemoclaw-exact-image-dispatch-reconciliation", - recordedAt: timestamp(now), - outcome, - correlationId: intent.request.correlationId, - runIds: runs.map((run) => run.id), - producerHeadShas: Object.fromEntries(runs.map((run) => [run.id, run.headSha])), - }; - try { - privateFileRuntime.writePrivateRegularFile( - path.join(workDir, DISPATCH_RECONCILIATION_FILE), - `${JSON.stringify(evidence, null, 2)}\n`, - ); - } catch { - fail("OUTPUT_WRITE_FAILED", "dispatch reconciliation evidence could not be written safely"); - } -} - -function writeAtomicPrivateRegularFile(file: string, contents: string): void { - const temporary = `${file}.${process.pid}.${randomUUID()}.tmp`; - try { - privateFileRuntime.writePrivateRegularFile(temporary, contents); - if (fs.existsSync(file)) { - const current = fs.lstatSync(file); - if (!current.isFile() || current.isSymbolicLink() || current.nlink !== 1) { - fail("OUTPUT_WRITE_FAILED", "existing controller state is not one regular file"); - } - } - fs.renameSync(temporary, file); - } finally { - try { - fs.rmSync(temporary, { force: true }); - } catch { - // A successfully renamed temporary path no longer exists; a failed - // cleanup must not hide the original atomic-write error. - } - } -} - -type PersistedQualificationStatus = ExactImageQualificationState["status"]; - -function validatePersistedRequest(value: unknown): void { - const request = record(value, "controller state request"); - requireExactRecordFields( - request, - [ - "actor", - "candidateSha", - "correlationId", - "reason", - "requesterRunAttempt", - "requesterRunId", - "workflowSha", - ], - "controller state request", - ); - persistedString(request, "actor", "controller state actor", GITHUB_LOGIN_PATTERN); - persistedString(request, "candidateSha", "controller state candidate SHA", FULL_SHA_PATTERN); - persistedString(request, "correlationId", "controller state correlation ID", UUID_V4_PATTERN); - validatePersistedReason(request.reason, "controller state reason"); - const requesterRunAttempt = request.requesterRunAttempt; - if ( - typeof requesterRunAttempt !== "number" || - !Number.isSafeInteger(requesterRunAttempt) || - requesterRunAttempt < 1 - ) { - fail("PROVENANCE_MISMATCH", "controller state requester run attempt must be positive"); - } - persistedString( - request, - "requesterRunId", - "controller state requester run ID", - DECIMAL_ID_PATTERN, - ); - persistedString(request, "workflowSha", "controller state workflow SHA", FULL_SHA_PATTERN); -} - -function validatePersistedProducer( - value: unknown, - status: PersistedQualificationStatus, - dispatchedAt: number, -): { completedAt: number | null; runId: string } { - const producer = record(value, "controller state producer"); - const completed = status !== "dispatched"; - requireExactRecordFields( - producer, - [ - "repository", - "repositorySha", - "ref", - "workflowPath", - "runId", - "runAttempt", - "workflowId", - "runUrl", - "htmlUrl", - ...(completed ? ["completedAt"] : []), - ], - "controller state producer", - ); - requireExactString(producer.repository, PRODUCER_REPOSITORY, "controller state repository"); - persistedString(producer, "repositorySha", "controller state producer SHA", FULL_SHA_PATTERN); - requireExactString(producer.ref, PRODUCER_REF, "controller state producer ref"); - requireExactString( - producer.workflowPath, - PRODUCER_WORKFLOW_PATH, - "controller state workflow path", - ); - const runId = persistedString( - producer, - "runId", - "controller state producer run ID", - DECIMAL_ID_PATTERN, - ); - if (producer.runAttempt !== 1) { - fail("PROVENANCE_MISMATCH", "controller state producer run attempt must equal 1"); - } - persistedString( - producer, - "workflowId", - "controller state producer workflow ID", - DECIMAL_ID_PATTERN, - ); - const expectedUrls = canonicalDispatchDetails(runId); - requireExactString(producer.runUrl, expectedUrls.runUrl, "controller state producer run URL"); - requireExactString(producer.htmlUrl, expectedUrls.htmlUrl, "controller state producer HTML URL"); - const completedAt = completed - ? persistedTimestamp(producer.completedAt, "controller state completedAt") - : null; - if (completedAt !== null && completedAt < dispatchedAt) { - fail("PROVENANCE_MISMATCH", "controller state completedAt precedes dispatchedAt"); - } - if (completedAt !== null && completedAt > dispatchedAt + QUALIFICATION_TIMEOUT_MS) { - fail("PROVENANCE_MISMATCH", "controller state completedAt exceeds qualification deadline"); - } - return { completedAt, runId }; -} - -function validatePersistedArtifact(value: unknown, runId: string): string { - const artifact = record(value, "controller state artifact"); - requireExactRecordFields( - artifact, - [ - "id", - "name", - "digest", - "sizeInBytes", - "apiUrl", - "archiveDownloadUrl", - "archiveSha256", - "manifestSha256", - ], - "controller state artifact", - ); - const id = persistedString(artifact, "id", "controller state artifact ID", DECIMAL_ID_PATTERN); - requireExactString( - artifact.name, - `nemoclaw-image-handoff-v1-${runId}-1`, - "controller state artifact name", - ); - const digest = persistedString( - artifact, - "digest", - "controller state artifact digest", - ARTIFACT_DIGEST_PATTERN, - ); - if ( - !Number.isSafeInteger(artifact.sizeInBytes) || - (artifact.sizeInBytes as number) < 1 || - (artifact.sizeInBytes as number) > MAX_ARCHIVE_BYTES - ) { - fail("PROVENANCE_MISMATCH", "controller state artifact size is invalid"); - } - const apiUrl = `https://api.github.com/repos/${PRODUCER_REPOSITORY}/actions/artifacts/${id}`; - requireExactString(artifact.apiUrl, apiUrl, "controller state artifact API URL"); - requireExactString( - artifact.archiveDownloadUrl, - `${apiUrl}/zip`, - "controller state artifact archive URL", - ); - const archiveSha256 = persistedString( - artifact, - "archiveSha256", - "controller state archive hash", - SHA256_PATTERN, - ); - if (digest !== `sha256:${archiveSha256}`) { - fail("PROVENANCE_MISMATCH", "controller state artifact digest does not match archive hash"); - } - return persistedString( - artifact, - "manifestSha256", - "controller state manifest hash", - SHA256_PATTERN, - ); -} - -function validatePersistedValidation( - value: unknown, - manifestSha256: string, - completedAt: number, - dispatchedAt: number, -): void { - const validation = record(value, "controller state validation"); - requireExactRecordFields( - validation, - ["acceptedAt", "manifestSha256", "normalizedManifestSha256"], - "controller state validation", - ); - const acceptedAt = persistedTimestamp(validation.acceptedAt, "controller state acceptedAt"); - if (acceptedAt < completedAt) { - fail("PROVENANCE_MISMATCH", "controller state acceptedAt precedes completedAt"); - } - if (acceptedAt > dispatchedAt + QUALIFICATION_TIMEOUT_MS) { - fail("PROVENANCE_MISMATCH", "controller state acceptedAt exceeds qualification deadline"); - } - requireExactString( - validation.manifestSha256, - manifestSha256, - "controller state accepted manifest hash", - ); - persistedString( - validation, - "normalizedManifestSha256", - "controller state normalized manifest hash", - SHA256_PATTERN, - ); -} - -function validatePersistedQualificationState(value: unknown): ExactImageQualificationState { - const state = record(value, "controller state"); - const statuses = new Set([ - "dispatched", - "completed", - "downloaded", - "validated", - ]); - if ( - typeof state.status !== "string" || - !statuses.has(state.status as PersistedQualificationStatus) - ) { - fail("PROVENANCE_MISMATCH", "controller state status is invalid"); - } - const status = state.status as PersistedQualificationStatus; - const hasArtifact = status === "downloaded" || status === "validated"; - requireExactRecordFields( - state, - [ - "schemaVersion", - "status", - "dispatchedAt", - "request", - "producer", - ...(hasArtifact ? ["artifact"] : []), - ...(status === "validated" ? ["validation"] : []), - ], - "controller state", - ); - if (state.schemaVersion !== 1) { - fail("PROVENANCE_MISMATCH", "controller state schema version must equal 1"); - } - const dispatchedAt = persistedTimestamp(state.dispatchedAt, "controller state dispatchedAt"); - validatePersistedRequest(state.request); - const producer = validatePersistedProducer(state.producer, status, dispatchedAt); - const manifestSha256 = hasArtifact - ? validatePersistedArtifact(state.artifact, producer.runId) - : null; - if (status === "validated") { - if (manifestSha256 === null || producer.completedAt === null) { - fail("PROVENANCE_MISMATCH", "validated controller state is incomplete"); - } - validatePersistedValidation( - state.validation, - manifestSha256, - producer.completedAt, - dispatchedAt, - ); - } - return state as unknown as ExactImageQualificationState; -} - -function writeState(workDir: string, state: ExactImageQualificationState): void { - try { - const validated = validatePersistedQualificationState(state); - writeAtomicPrivateRegularFile(statePath(workDir), `${JSON.stringify(validated, null, 2)}\n`); - } catch { - fail("OUTPUT_WRITE_FAILED", "controller state could not be written safely"); - } -} - -function preserveUnreadableState(workDir: string): void { - const file = statePath(workDir); - if (!fs.existsSync(file)) return; - const preserved = path.join( - workDir, - `controller-state.corrupt-${Date.now()}-${randomUUID()}.json`, - ); - try { - fs.renameSync(file, preserved); - } catch { - fail("OUTPUT_WRITE_FAILED", "unreadable controller state could not be preserved safely"); - } -} - -export function readExactImageQualificationState(workDir: string): ExactImageQualificationState { - let contents: string | null; - try { - contents = privateFileRuntime.readPrivateRegularFile(statePath(workDir), { - maxBytes: MAX_STATE_BYTES, - }); - } catch { - fail("PROVENANCE_MISMATCH", "controller state could not be read safely"); - } - if (contents === null) fail("PROVENANCE_MISMATCH", "controller state is missing"); - let parsed: unknown; - try { - parsed = JSON.parse(contents) as unknown; - } catch { - fail("PROVENANCE_MISMATCH", "controller state is not valid JSON"); - } - return validatePersistedQualificationState(parsed); -} - -function canonicalDispatchDetails(runId: string): DispatchDetails { - return { - workflowRunId: runId, - runUrl: `https://api.github.com/repos/${PRODUCER_REPOSITORY}/actions/runs/${runId}`, - htmlUrl: `https://github.com/${PRODUCER_REPOSITORY}/actions/runs/${runId}`, - }; -} - -function dispatchCreationWindow( - intent: ExactImageDispatchIntent, - configured: ReturnType, -): { earliest: number; latest: number } { - const startedAt = Date.parse(intent.requestStartedAt); - return { - earliest: startedAt - DISPATCH_CLOCK_SKEW_MS, - latest: startedAt + configured.apiRequestTimeoutMs + DISPATCH_CLOCK_SKEW_MS, - }; -} - -function reconciledWorkflowRuns( - value: unknown, - intent: ExactImageDispatchIntent, - configured: ReturnType, -): QualificationWorkflowRun[] { - const response = record(value, "workflow run reconciliation response"); - if (!Number.isSafeInteger(response.total_count) || (response.total_count as number) < 0) { - fail("DISPATCH_AMBIGUOUS", "workflow run reconciliation count is invalid"); - } - if (!Array.isArray(response.workflow_runs)) { - fail("DISPATCH_AMBIGUOUS", "workflow run reconciliation list is missing"); - } - if ((response.total_count as number) > response.workflow_runs.length) { - fail("DISPATCH_AMBIGUOUS", "workflow run reconciliation was truncated"); - } - const { earliest, latest } = dispatchCreationWindow(intent, configured); - const title = `Qualify NemoClaw ${intent.request.candidateSha} (${intent.request.correlationId})`; - const matches: QualificationWorkflowRun[] = []; - for (const candidate of response.workflow_runs) { - if (candidate === null || typeof candidate !== "object" || Array.isArray(candidate)) continue; - const run = candidate as Record; - const createdAt = typeof run.created_at === "string" ? Date.parse(run.created_at) : Number.NaN; - const id = Number.isSafeInteger(run.id) && (run.id as number) > 0 ? String(run.id) : ""; - const headSha = typeof run.head_sha === "string" ? run.head_sha : ""; - const details = id ? canonicalDispatchDetails(id) : null; - if ( - details === null || - run.workflow_id !== Number(intent.producer.workflowId) || - run.run_attempt !== 1 || - run.event !== "workflow_dispatch" || - run.head_branch !== PRODUCER_REF || - !FULL_SHA_PATTERN.test(headSha) || - run.path !== PRODUCER_WORKFLOW_PATH || - run.display_title !== title || - run.url !== details.runUrl || - run.html_url !== details.htmlUrl || - (run.repository as { full_name?: unknown } | undefined)?.full_name !== PRODUCER_REPOSITORY || - (run.head_repository as { full_name?: unknown } | undefined)?.full_name !== - PRODUCER_REPOSITORY || - !Number.isFinite(createdAt) || - createdAt < earliest || - createdAt > latest - ) { - continue; - } - matches.push( - validateQualificationWorkflowRun(candidate, { - candidateSha: intent.request.candidateSha, - correlationId: intent.request.correlationId, - producerSha: headSha, - workflowId: intent.producer.workflowId, - runId: id, - runUrl: details.runUrl, - htmlUrl: details.htmlUrl, - }), - ); - } - return matches; -} - -function stateFromIntent( - intent: ExactImageDispatchIntent, - run: QualificationWorkflowRun, -): ExactImageQualificationState { - return { - schemaVersion: 1, - status: "dispatched", - dispatchedAt: intent.requestStartedAt, - request: { - actor: intent.request.actor, - candidateSha: intent.request.candidateSha, - correlationId: intent.request.correlationId, - reason: intent.request.reason, - requesterRunAttempt: intent.request.requesterRunAttempt, - requesterRunId: intent.request.requesterRunId, - workflowSha: intent.request.workflowSha, - }, - producer: { - repository: PRODUCER_REPOSITORY, - repositorySha: run.headSha, - ref: PRODUCER_REF, - workflowPath: PRODUCER_WORKFLOW_PATH, - runId: run.id, - runAttempt: 1, - workflowId: run.workflowId, - runUrl: run.url, - htmlUrl: run.htmlUrl, - }, - }; -} - -async function cancelAndVerifyRecoveredRun( - state: ExactImageQualificationState, - initial: { status: string }, - token: string, - api: GitHubApiClient, - pause: (milliseconds: number) => Promise, - pollIntervalMs: number, - deadline: number, - now: () => number, -): Promise { - if (initial.status === "completed") return false; - let cancellationError: unknown; - try { - await cancelRun(state, token, api); - } catch (error) { - cancellationError = error; - } - for (;;) { - if (now() >= deadline) { - fail("QUALIFICATION_TIMEOUT", "recovered producer run cancellation was not verified in time"); - } - const run = validateCancellationWorkflowRun( - await api( - `repos/${PRODUCER_REPOSITORY}/actions/runs/${state.producer.runId}`, - token, - { apiVersion: GITHUB_API_VERSION, expectedStatus: 200 }, - ), - state, - ); - if (run.status === "completed") return true; - if (cancellationError !== undefined) throw cancellationError; - await pause(pollIntervalMs); - } -} - -async function reconcileAmbiguousDispatch( - options: { - intent: ExactImageDispatchIntent; - workDir: string; - producerToken: string; - mode: "start" | "cleanup"; - }, - dependencies: QualificationDependencies, -): Promise { - const now = dependencies.now ?? Date.now; - const pause = dependencies.sleep ?? sleep; - const configured = limits(dependencies); - const startedAt = now(); - const windowMs = - options.mode === "cleanup" - ? configured.cleanupTimeoutMs - : configured.dispatchReconciliationTimeoutMs; - const deadline = startedAt + windowMs; - const api = boundedApiClient(dependencies.api ?? githubApi, dependencies, deadline); - const creationWindow = dispatchCreationWindow(options.intent, configured); - const earliestQuery = new Date(Math.floor(creationWindow.earliest / 1_000) * 1_000).toISOString(); - const latestQuery = new Date(Math.ceil(creationWindow.latest / 1_000) * 1_000).toISOString(); - const created = encodeURIComponent( - `${earliestQuery.replace(".000Z", "Z")}..${latestQuery.replace(".000Z", "Z")}`, - ); - const listPath = - `repos/${PRODUCER_REPOSITORY}/actions/workflows/${PRODUCER_WORKFLOW_FILE}/runs` + - `?event=workflow_dispatch&branch=${PRODUCER_REF}&created=${created}&per_page=100`; - - for (;;) { - if (now() >= deadline) { - writeDispatchReconciliation(options.workDir, options.intent, [], "none", now); - fail( - "DISPATCH_AMBIGUOUS", - "no strict producer run match appeared before reconciliation timeout", - ); - } - const matches = reconciledWorkflowRuns( - await api(listPath, options.producerToken, { - apiVersion: GITHUB_API_VERSION, - expectedStatus: 200, - }), - options.intent, - configured, - ); - if (matches.length > 1) { - const runIds = matches.map((run) => run.id); - writeDispatchReconciliation(options.workDir, options.intent, matches, "multiple", now); - for (const run of matches) { - if (run.status !== "completed") { - try { - await api( - `repos/${PRODUCER_REPOSITORY}/actions/runs/${run.id}/cancel`, - options.producerToken, - { method: "POST", apiVersion: GITHUB_API_VERSION, expectedStatus: 202 }, - ); - } catch { - // The retained exact IDs are mandatory manual-cleanup evidence. - } - } - } - fail( - "DISPATCH_AMBIGUOUS", - `multiple strict dispatch matches require cleanup: ${runIds.join(",")}`, - ); - } - if (matches.length === 1) { - const recovered = matches[0]; - const state = stateFromIntent(options.intent, recovered); - writeState(options.workDir, state); - writeDispatchReconciliation( - options.workDir, - options.intent, - [recovered], - "recovered-one", - now, - ); - const cancelled = await cancelAndVerifyRecoveredRun( - state, - recovered, - options.producerToken, - api, - pause, - configured.reconciliationPollIntervalMs, - deadline, - now, - ); - if (options.mode === "cleanup") return cancelled; - fail( - "DISPATCH_AMBIGUOUS", - `recovered producer run ${recovered.id} was not accepted and was cleaned up`, - ); - } - await pause(configured.reconciliationPollIntervalMs); - } -} - -export async function startExactImageQualification( - options: { - request: ExactImageQualificationRequest; - coreToken: string; - producerToken: string; - workDir: string; - }, - dependencies: QualificationDependencies = {}, -): Promise { - const now = dependencies.now ?? Date.now; - const initialApi = boundedApiClient(dependencies.api ?? githubApi, dependencies); - await authorizeRequest(options.request, options.coreToken, initialApi); - - const producerRef = await initialApi( - `repos/${PRODUCER_REPOSITORY}/git/ref/heads/${PRODUCER_REF}`, - options.producerToken, - { apiVersion: GITHUB_API_VERSION, expectedStatus: 200 }, - ); - const producerSha = validateMainRef(producerRef, PRODUCER_REPOSITORY); - const workflowId = validateProducerWorkflow( - await initialApi( - `repos/${PRODUCER_REPOSITORY}/actions/workflows/${PRODUCER_WORKFLOW_FILE}`, - options.producerToken, - { apiVersion: GITHUB_API_VERSION, expectedStatus: 200 }, - ), - ); - const correlationId = (dependencies.randomUuid ?? randomUUID)(); - if (!UUID_V4_PATTERN.test(correlationId)) { - fail("REQUEST_INVALID", "generated correlation ID was not a lowercase UUIDv4"); - } - - const intent: ExactImageDispatchIntent = { - schemaVersion: 1, - kind: "nemoclaw-exact-image-dispatch-intent", - requestStartedAt: timestamp(now), - request: { - actor: options.request.actor, - candidateSha: options.request.candidateSha, - correlationId, - reason: options.request.reason, - requesterRunAttempt: options.request.requesterRunAttempt, - requesterRunId: options.request.requesterRunId, - workflowSha: options.request.workflowSha, - }, - producer: { - repository: PRODUCER_REPOSITORY, - repositorySha: producerSha, - ref: PRODUCER_REF, - workflowId, - workflowPath: PRODUCER_WORKFLOW_PATH, - }, - }; - writeDispatchIntent(options.workDir, intent); - const stateDeadline = - Date.parse(intent.requestStartedAt) + limits(dependencies).qualificationTimeoutMs; - const api = boundedApiClient(dependencies.api ?? githubApi, dependencies, stateDeadline); - - let dispatch: DispatchDetails; - try { - const dispatchResponse = await api( - `repos/${PRODUCER_REPOSITORY}/actions/workflows/${PRODUCER_WORKFLOW_FILE}/dispatches`, - options.producerToken, - { - method: "POST", - apiVersion: GITHUB_API_VERSION, - expectedStatus: 200, - body: { - ref: PRODUCER_REF, - inputs: { - nemoclaw_sha: options.request.candidateSha, - correlation_id: correlationId, - requester_workflow_run_id: options.request.requesterRunId, - requester_workflow_run_attempt: String(options.request.requesterRunAttempt), - }, - return_run_details: true, - }, - }, - ); - dispatch = validateWorkflowDispatchDetails(dispatchResponse); - } catch { - await reconcileAmbiguousDispatch( - { intent, workDir: options.workDir, producerToken: options.producerToken, mode: "start" }, - dependencies, - ); - fail("DISPATCH_AMBIGUOUS", "producer dispatch could not be bound safely"); - } - - const state: ExactImageQualificationState = { - schemaVersion: 1, - status: "dispatched", - dispatchedAt: intent.requestStartedAt, - request: { - actor: options.request.actor, - candidateSha: options.request.candidateSha, - correlationId, - reason: options.request.reason, - requesterRunAttempt: options.request.requesterRunAttempt, - requesterRunId: options.request.requesterRunId, - workflowSha: options.request.workflowSha, - }, - producer: { - repository: PRODUCER_REPOSITORY, - repositorySha: producerSha, - ref: PRODUCER_REF, - workflowPath: PRODUCER_WORKFLOW_PATH, - runId: dispatch.workflowRunId, - runAttempt: 1, - runUrl: dispatch.runUrl, - htmlUrl: dispatch.htmlUrl, - workflowId, - }, - }; - // Persist the returned ID before any further API call so the always-run - // cleanup step can cancel exactly this run if identity validation fails. - writeState(options.workDir, state); - - const run = validateQualificationWorkflowRun( - await api( - `repos/${PRODUCER_REPOSITORY}/actions/runs/${dispatch.workflowRunId}`, - options.producerToken, - { apiVersion: GITHUB_API_VERSION, expectedStatus: 200 }, - ), - { - candidateSha: options.request.candidateSha, - correlationId, - producerSha, - workflowId, - runId: dispatch.workflowRunId, - runUrl: dispatch.runUrl, - htmlUrl: dispatch.htmlUrl, - }, - ); - state.producer.workflowId = run.workflowId; - writeState(options.workDir, state); - return state; -} - -function expectedRun(state: ExactImageQualificationState) { - return { - candidateSha: state.request.candidateSha, - correlationId: state.request.correlationId, - producerSha: state.producer.repositorySha, - workflowId: state.producer.workflowId, - runId: state.producer.runId, - runUrl: state.producer.runUrl, - htmlUrl: state.producer.htmlUrl, - }; -} - -async function cancelRun( - state: ExactImageQualificationState, - token: string, - api: GitHubApiClient, -): Promise { - await api( - `repos/${PRODUCER_REPOSITORY}/actions/runs/${state.producer.runId}/cancel`, - token, - { method: "POST", apiVersion: GITHUB_API_VERSION, expectedStatus: 202 }, - ); -} - -export async function waitForExactImageQualification( - options: { workDir: string; producerToken: string }, - dependencies: QualificationDependencies = {}, -): Promise { - const now = dependencies.now ?? Date.now; - const pause = dependencies.sleep ?? sleep; - const configured = limits(dependencies); - const state = readBoundExactImageQualificationState(options.workDir); - if (state.status !== "dispatched") { - fail("PROVENANCE_MISMATCH", "producer run may only be awaited from dispatched state"); - } - const startedAt = Date.parse(state.dispatchedAt); - const hardDeadline = qualificationDeadline(state, configured.qualificationTimeoutMs); - const deadline = acceptanceDeadline(state, dependencies); - const api = boundedApiClient(dependencies.api ?? githubApi, dependencies, deadline); - const cancelApi = boundedApiClient(dependencies.api ?? githubApi, dependencies, hardDeadline); - let warned = false; - - for (;;) { - if (now() >= deadline) { - const finalRun = validateQualificationWorkflowRun( - await cancelApi( - `repos/${PRODUCER_REPOSITORY}/actions/runs/${state.producer.runId}`, - options.producerToken, - { apiVersion: GITHUB_API_VERSION, expectedStatus: 200 }, - ), - expectedRun(state), - ); - if (finalRun.status !== "completed") { - await cancelRun(state, options.producerToken, cancelApi); - } - fail("QUALIFICATION_TIMEOUT", "qualification reached its cancellation reserve"); - } - const run = validateQualificationWorkflowRun( - await api( - `repos/${PRODUCER_REPOSITORY}/actions/runs/${state.producer.runId}`, - options.producerToken, - { apiVersion: GITHUB_API_VERSION, expectedStatus: 200 }, - ), - expectedRun(state), - ); - const observedAt = now(); - const elapsed = observedAt - startedAt; - if (observedAt >= deadline) { - if (run.status !== "completed") { - await cancelRun(state, options.producerToken, cancelApi); - } - fail("QUALIFICATION_TIMEOUT", "qualification reached its cancellation reserve"); - } - if (run.status === "completed") { - if (run.conclusion !== "success") { - fail("PRODUCER_RUN_FAILED", `producer run completed with ${run.conclusion ?? "no result"}`); - } - state.status = "completed"; - state.producer.completedAt = timestamp(now); - state.producer.workflowId = run.workflowId; - writeState(options.workDir, state); - return run; - } - - if (run.status !== "in_progress" && elapsed >= configured.queueTimeoutMs) { - await cancelRun(state, options.producerToken, cancelApi); - fail("RUN_QUEUE_TIMEOUT", "producer run did not start within 10 minutes"); - } - if (!warned && elapsed >= configured.warningAfterMs) { - console.warn( - "Qualification image build has exceeded 25 minutes; continuing to the hard limit.", - ); - warned = true; - } - await pause(configured.pollIntervalMs); - } -} - -export function validateQualificationArtifactList( - value: unknown, - state: ExactImageQualificationState, -): QualificationArtifact | null { - const response = record(value, "artifact list response"); - if (!Number.isSafeInteger(response.total_count) || (response.total_count as number) < 0) { - fail("ARTIFACT_MISSING_OR_INVALID", "artifact total_count is invalid"); - } - if (!Array.isArray(response.artifacts)) { - fail("ARTIFACT_MISSING_OR_INVALID", "artifact list is missing artifacts"); - } - if (response.total_count === 0 && response.artifacts.length === 0) return null; - if (response.total_count !== 1 || response.artifacts.length !== 1) { - fail("ARTIFACT_MISSING_OR_INVALID", "producer run must publish exactly one artifact"); - } - const artifact = record(response.artifacts[0], "qualification artifact"); - const id = safePositiveId(artifact.id, "artifact ID"); - const expectedName = `nemoclaw-image-handoff-v1-${state.producer.runId}-${state.producer.runAttempt}`; - requireExactString(artifact.name, expectedName, "artifact name"); - if (artifact.expired !== false) fail("ARTIFACT_MISSING_OR_INVALID", "artifact is expired"); - if (typeof artifact.digest !== "string" || !ARTIFACT_DIGEST_PATTERN.test(artifact.digest)) { - fail("ARTIFACT_MISSING_OR_INVALID", "artifact digest must be a lowercase SHA-256 digest"); - } - if ( - !Number.isSafeInteger(artifact.size_in_bytes) || - (artifact.size_in_bytes as number) < 1 || - (artifact.size_in_bytes as number) > MAX_ARCHIVE_BYTES - ) { - fail("ARTIFACT_MISSING_OR_INVALID", "artifact archive size is outside the accepted range"); - } - const apiUrl = `https://api.github.com/repos/${PRODUCER_REPOSITORY}/actions/artifacts/${id}`; - const archiveDownloadUrl = `${apiUrl}/zip`; - requireExactString(artifact.url, apiUrl, "artifact API URL"); - requireExactString(artifact.archive_download_url, archiveDownloadUrl, "artifact archive URL"); - const workflowRun = record(artifact.workflow_run, "artifact workflow run"); - if (safePositiveId(workflowRun.id, "artifact workflow run ID") !== state.producer.runId) { - fail("PROVENANCE_MISMATCH", "artifact workflow run ID did not match the dispatched run"); - } - requireExactString( - workflowRun.head_sha, - state.producer.repositorySha, - "artifact workflow run head SHA", - ); - return { - id, - name: expectedName, - digest: artifact.digest, - sizeInBytes: artifact.size_in_bytes as number, - apiUrl, - archiveDownloadUrl, - }; -} - -async function readBoundedResponse(response: Response, maxBytes: number): Promise { - const header = response.headers.get("content-length"); - if (header !== null && (!/^[0-9]+$/u.test(header) || Number(header) > maxBytes)) { - fail("ARTIFACT_MISSING_OR_INVALID", "artifact response is larger than the accepted limit"); - } - if (!response.body) fail("ARTIFACT_DOWNLOAD_TRANSIENT", "artifact response had no body"); - const reader = response.body.getReader(); - const chunks: Buffer[] = []; - let size = 0; - for (;;) { - const { done, value } = await reader.read(); - if (done) break; - size += value.byteLength; - if (size > maxBytes) { - await reader.cancel(); - fail("ARTIFACT_MISSING_OR_INVALID", "artifact response exceeded the accepted limit"); - } - chunks.push(Buffer.from(value)); - } - return Buffer.concat(chunks, size); -} - -function writePrivateBuffer(file: string, contents: Buffer): void { - let descriptor: number; - try { - descriptor = fs.openSync( - file, - fs.constants.O_WRONLY | fs.constants.O_CREAT | fs.constants.O_EXCL | NO_FOLLOW | NON_BLOCK, - 0o600, - ); - } catch { - fail("OUTPUT_WRITE_FAILED", "artifact output could not be created safely"); - } - try { - const stat = fs.fstatSync(descriptor); - if (!stat.isFile() || stat.nlink !== 1) { - fail("OUTPUT_WRITE_FAILED", "artifact output must be one private regular file"); - } - fs.fchmodSync(descriptor, 0o600); - fs.writeFileSync(descriptor, contents); - fs.fsyncSync(descriptor); - } finally { - fs.closeSync(descriptor); - } -} - -function defaultCommandRunner(_command: string, args: readonly string[]): CommandResult { - return spawnSync("/usr/bin/unzip", [...args], { - encoding: null, - env: { - LANG: "C", - LC_ALL: "C", - }, - maxBuffer: MAX_MANIFEST_BYTES + 4096, - timeout: COMMAND_TIMEOUT_MS, - windowsHide: true, - }); -} - -function successfulCommand(result: CommandResult, label: string): Buffer { - if (result.error || result.status !== 0 || result.signal) { - fail("ARTIFACT_MISSING_OR_INVALID", `${label} rejected the artifact archive`); - } - return Buffer.isBuffer(result.stdout) ? result.stdout : Buffer.from(result.stdout); -} - -export function extractExactManifestArchive( - archivePath: string, - outputPath: string, - runCommand: CommandRunner = defaultCommandRunner, -): Buffer { - const entries = successfulCommand(runCommand("unzip", ["-Z1", archivePath]), "ZIP inventory"); - if (!entries.equals(Buffer.from(`${MANIFEST_ARTIFACT_FILE}\n`, "utf8"))) { - fail( - "ARTIFACT_MISSING_OR_INVALID", - `artifact ZIP must contain exactly ${MANIFEST_ARTIFACT_FILE} at its root`, - ); - } - const metadata = successfulCommand( - runCommand("unzip", ["-Zl", archivePath, MANIFEST_ARTIFACT_FILE]), - "ZIP metadata inspection", - ).toString("utf8"); - const matchingMetadata = metadata - .split(/\r?\n/u) - .filter((line) => line.endsWith(` ${MANIFEST_ARTIFACT_FILE}`)); - if (matchingMetadata.length !== 1 || !matchingMetadata[0]?.startsWith("-")) { - fail("ARTIFACT_MISSING_OR_INVALID", "artifact manifest ZIP entry must be a regular file"); - } - const manifest = successfulCommand( - runCommand("unzip", ["-p", archivePath, MANIFEST_ARTIFACT_FILE]), - "ZIP extraction", - ); - if (manifest.length === 0 || manifest.length > MAX_MANIFEST_BYTES) { - fail("ARTIFACT_MISSING_OR_INVALID", "artifact manifest size is outside the accepted range"); - } - writePrivateBuffer(outputPath, manifest); - const stat = fs.lstatSync(outputPath); - if ( - !stat.isFile() || - stat.isSymbolicLink() || - stat.nlink !== 1 || - stat.size !== manifest.length - ) { - fail("ARTIFACT_MISSING_OR_INVALID", "extracted manifest is not one regular direct file"); - } - return manifest; -} - -export async function downloadExactImageManifest( - options: { workDir: string; producerToken: string }, - dependencies: QualificationDependencies = {}, -): Promise { - const state = readBoundExactImageQualificationState(options.workDir); - if (state.status !== "completed") { - fail("PROVENANCE_MISMATCH", "producer run must complete before artifact download"); - } - const pause = dependencies.sleep ?? sleep; - const now = dependencies.now ?? Date.now; - const configured = limits(dependencies); - const deadline = qualificationDeadline(state, configured.qualificationTimeoutMs); - const api = boundedApiClient(dependencies.api ?? githubApi, dependencies, deadline); - requireQualificationTimeRemaining(deadline, now); - const artifactStartedAt = now(); - const artifactDeadline = Math.min( - artifactStartedAt + configured.artifactPropagationTimeoutMs, - deadline, - ); - let artifact: QualificationArtifact | null = null; - while (artifact === null) { - requireQualificationTimeRemaining(deadline, now); - artifact = validateQualificationArtifactList( - await api( - `repos/${PRODUCER_REPOSITORY}/actions/runs/${state.producer.runId}/artifacts?per_page=100`, - options.producerToken, - { apiVersion: GITHUB_API_VERSION, expectedStatus: 200 }, - ), - state, - ); - requireQualificationTimeRemaining(deadline, now); - if (artifact !== null) break; - if (now() >= artifactDeadline) { - fail("ARTIFACT_PENDING", "qualification artifact did not appear within two minutes"); - } - await pause(configured.pollIntervalMs); - } - - const controller = new AbortController(); - const downloadBudget = Math.min( - configured.downloadTimeoutMs, - requireQualificationTimeRemaining(deadline, now), - ); - const timer = setTimeout(() => controller.abort(), downloadBudget); - let archive: Buffer; - try { - const response = await (dependencies.fetch ?? fetch)(artifact.archiveDownloadUrl, { - headers: { - Accept: "application/vnd.github+json", - Authorization: `Bearer ${options.producerToken}`, - "X-GitHub-Api-Version": GITHUB_API_VERSION, - }, - redirect: "follow", - signal: controller.signal, - }); - requireQualificationTimeRemaining(deadline, now); - if (response.status !== 200) { - fail("ARTIFACT_DOWNLOAD_TRANSIENT", `artifact archive download returned ${response.status}`); - } - archive = await readBoundedResponse(response, MAX_ARCHIVE_BYTES); - } catch (error) { - if (error instanceof ExactImageQualificationError) throw error; - requireQualificationTimeRemaining(deadline, now); - fail("ARTIFACT_DOWNLOAD_TRANSIENT", "artifact archive download failed"); - } finally { - clearTimeout(timer); - } - requireQualificationTimeRemaining(deadline, now); - if (archive.length !== artifact.sizeInBytes) { - fail("ARTIFACT_MISSING_OR_INVALID", "downloaded archive size did not match artifact metadata"); - } - const archiveHash = sha256(archive); - if (`sha256:${archiveHash}` !== artifact.digest) { - fail("ARTIFACT_MISSING_OR_INVALID", "downloaded archive digest did not match GitHub metadata"); - } - - const archivePath = path.join(options.workDir, ARCHIVE_FILE); - const manifestPath = path.join(options.workDir, MANIFEST_ARTIFACT_FILE); - writePrivateBuffer(archivePath, archive); - const manifest = extractExactManifestArchive( - archivePath, - manifestPath, - dependencies.runCommand ?? defaultCommandRunner, - ); - requireQualificationTimeRemaining(deadline, now); - state.artifact = { - ...artifact, - archiveSha256: archiveHash, - manifestSha256: sha256(manifest), - }; - state.status = "downloaded"; - writeState(options.workDir, state); - return artifact; -} - -function readPrivateBuffer(file: string, maxBytes: number): Buffer { - let descriptor: number; - try { - descriptor = fs.openSync(file, fs.constants.O_RDONLY | NO_FOLLOW | NON_BLOCK); - } catch { - fail("PROVENANCE_MISMATCH", "qualification evidence file could not be opened safely"); - } - try { - const before = fs.fstatSync(descriptor); - if (!before.isFile() || before.nlink !== 1 || before.size > maxBytes) { - fail("PROVENANCE_MISMATCH", "qualification evidence file failed regular-file checks"); - } - const bytes = fs.readFileSync(descriptor); - const after = fs.fstatSync(descriptor); - if (after.size !== before.size || after.nlink !== 1) { - fail("PROVENANCE_MISMATCH", "qualification evidence file changed while it was read"); - } - return bytes; - } finally { - fs.closeSync(descriptor); - } -} - -export function finalizeExactImageQualification( - workDir: string, - dependencies: QualificationDependencies = {}, -): ExactImageQualificationState { - const state = readBoundExactImageQualificationState(workDir); - if (state.status !== "downloaded" || !state.artifact) { - fail("PROVENANCE_MISMATCH", "artifact must be digest-verified before finalization"); - } - const now = dependencies.now ?? Date.now; - const deadline = qualificationDeadline(state, limits(dependencies).qualificationTimeoutMs); - requireQualificationTimeRemaining(deadline, now); - const manifestHash = sha256( - readPrivateBuffer(path.join(workDir, MANIFEST_ARTIFACT_FILE), MAX_MANIFEST_BYTES), - ); - if (manifestHash !== state.artifact.manifestSha256) { - fail("PROVENANCE_MISMATCH", "manifest changed after archive verification"); - } - const normalizedHash = sha256( - readPrivateBuffer(path.join(workDir, VALIDATED_MANIFEST_FILE), MAX_MANIFEST_BYTES), - ); - requireQualificationTimeRemaining(deadline, now); - state.validation = { - acceptedAt: timestamp(now), - manifestSha256: manifestHash, - normalizedManifestSha256: normalizedHash, - }; - state.status = "validated"; - const evidence = { - schemaVersion: 1, - kind: "nemoclaw-exact-image-qualification-evidence", - qualificationStatus: "accepted", - request: state.request, - producer: state.producer, - artifact: state.artifact, - validation: state.validation, - }; - // The state is the durable commit point for acceptance. Validate and persist - // that transition before publishing an accepted receipt so a rejected state - // can never leave accepted evidence behind. - writeState(workDir, state); - try { - privateFileRuntime.writePrivateRegularFile( - path.join(workDir, EVIDENCE_FILE), - `${JSON.stringify(evidence, null, 2)}\n`, - ); - } catch { - fail("OUTPUT_WRITE_FAILED", "qualification evidence could not be written safely"); - } - return state; -} - -export async function cancelActiveExactImageQualification( - options: { workDir: string; producerToken: string }, - dependencies: QualificationDependencies = {}, -): Promise { - let state: ExactImageQualificationState; - try { - state = readExactImageQualificationState(options.workDir); - } catch (error) { - if (!fs.existsSync(intentPath(options.workDir))) { - if (fs.existsSync(statePath(options.workDir))) throw error; - return false; - } - // State crosses independent workflow steps and is untrusted/damaged-disk input on every read. - // Atomic replacement prevents ordinary partial writes, while this permanent defense-in-depth - // path preserves interruption/filesystem-corruption evidence and reconciles only from the - // separately validated durable intent. - preserveUnreadableState(options.workDir); - const intent = requiredDispatchIntent(options.workDir); - return reconcileAmbiguousDispatch({ ...options, intent, mode: "cleanup" }, dependencies); - } - const now = dependencies.now ?? Date.now; - const configured = limits(dependencies); - const intent = requiredDispatchIntent(options.workDir); - try { - validateStateIntentBinding(state, intent); - } catch { - preserveUnreadableState(options.workDir); - return reconcileAmbiguousDispatch({ ...options, intent, mode: "cleanup" }, dependencies); - } - const deadline = now() + configured.cleanupTimeoutMs; - const api = boundedApiClient(dependencies.api ?? githubApi, dependencies, deadline); - const initial = validateCancellationWorkflowRun( - await api( - `repos/${PRODUCER_REPOSITORY}/actions/runs/${state.producer.runId}`, - options.producerToken, - { apiVersion: GITHUB_API_VERSION, expectedStatus: 200 }, - ), - state, - ); - return cancelAndVerifyRecoveredRun( - state, - initial, - options.producerToken, - api, - dependencies.sleep ?? sleep, - configured.reconciliationPollIntervalMs, - deadline, - now, - ); -} - -function parseFlags(argv: readonly string[]): Map { - const flags = new Map(); - for (let index = 0; index < argv.length; index += 2) { - const flag = argv[index]; - const value = argv[index + 1]; - if (!flag?.startsWith("--") || value === undefined || value.startsWith("--")) { - fail("REQUEST_INVALID", `${flag ?? "argument"} requires one value`); - } - if (flags.has(flag)) fail("REQUEST_INVALID", `${flag} may be provided only once`); - flags.set(flag, value); - } - return flags; -} - -function requiredFlag(flags: Map, flag: string): string { - const value = flags.get(flag); - if (value === undefined) fail("REQUEST_INVALID", `${flag} is required`); - return value; -} - -export function parseExactImageQualificationCommand(argv: readonly string[]): ControllerCommand { - const flags = parseFlags(argv); - const mode = requiredFlag(flags, "--mode"); - if (["wait", "download", "finalize", "cancel"].includes(mode)) { - const allowed = new Set(["--mode", "--work-dir"]); - for (const flag of flags.keys()) { - if (!allowed.has(flag)) fail("REQUEST_INVALID", `unknown argument ${flag}`); - } - return { - mode: mode as "wait" | "download" | "finalize" | "cancel", - workDir: requiredFlag(flags, "--work-dir"), - }; - } - if (mode !== "preflight" && mode !== "start") { - fail("REQUEST_INVALID", `unsupported mode ${mode}`); - } - const allowed = new Set([ - "--mode", - "--actor", - "--candidate-sha", - "--event-name", - "--reason", - "--ref", - "--requester-run-attempt", - "--requester-run-id", - "--workflow-sha", - ...(mode === "start" ? ["--work-dir"] : []), - ]); - for (const flag of flags.keys()) { - if (!allowed.has(flag)) fail("REQUEST_INVALID", `unknown argument ${flag}`); - } - const request = validateExactImageQualificationRequest({ - actor: requiredFlag(flags, "--actor"), - candidateSha: requiredFlag(flags, "--candidate-sha"), - eventName: requiredFlag(flags, "--event-name"), - reason: requiredFlag(flags, "--reason"), - ref: requiredFlag(flags, "--ref"), - requesterRunAttempt: positiveInteger( - requiredFlag(flags, "--requester-run-attempt"), - "requester run attempt", - ), - requesterRunId: requiredFlag(flags, "--requester-run-id"), - workflowSha: requiredFlag(flags, "--workflow-sha"), - }); - return mode === "preflight" - ? { mode, request } - : { mode, request, workDir: requiredFlag(flags, "--work-dir") }; -} - -function requiredToken(name: string): string { - const token = process.env[name]; - if (!token) fail("REQUEST_INVALID", `${name} is required`); - return token; -} - -function writeOutput(name: string, value: string): void { - const output = process.env.GITHUB_OUTPUT; - if (!output) return; - fs.appendFileSync(output, `${name}=${value}\n`, { encoding: "utf8", mode: 0o600 }); -} - -async function main(): Promise { - const command = parseExactImageQualificationCommand(process.argv.slice(2)); - if (command.mode === "preflight") { - await preflightExactImageQualification(command.request, requiredToken("GITHUB_TOKEN")); - console.log("Draft qualification request passed preflight."); - return; - } - if (command.mode === "start") { - const state = await startExactImageQualification({ - request: command.request, - coreToken: requiredToken("GITHUB_TOKEN"), - producerToken: requiredToken("NEMOCLAW_IMAGE_QUALIFICATION_TOKEN"), - workDir: command.workDir, - }); - writeOutput("correlation_id", state.request.correlationId); - writeOutput("image_repository_sha", state.producer.repositorySha); - writeOutput("producer_run_id", state.producer.runId); - writeOutput("producer_run_attempt", String(state.producer.runAttempt)); - console.log(`Dispatched and bound producer run ${state.producer.runId}.`); - return; - } - if (command.mode === "wait") { - const run = await waitForExactImageQualification({ - workDir: command.workDir, - producerToken: requiredToken("NEMOCLAW_IMAGE_QUALIFICATION_TOKEN"), - }); - console.log(`Producer run ${run.id} completed successfully.`); - return; - } - if (command.mode === "download") { - const artifact = await downloadExactImageManifest({ - workDir: command.workDir, - producerToken: requiredToken("NEMOCLAW_IMAGE_QUALIFICATION_TOKEN"), - }); - console.log(`Verified and extracted artifact ${artifact.id}.`); - return; - } - if (command.mode === "finalize") { - const state = finalizeExactImageQualification(command.workDir); - console.log(`Recorded accepted qualification evidence for run ${state.producer.runId}.`); - return; - } - const cancelled = await cancelActiveExactImageQualification({ - workDir: command.workDir, - producerToken: requiredToken("NEMOCLAW_IMAGE_QUALIFICATION_TOKEN"), - }); - console.log(cancelled ? "Cancelled active producer run." : "No active producer run to cancel."); -} - -export function exactImageQualificationFailureCode( - error: unknown, -): ExactImageQualificationFailureCode { - return error instanceof ExactImageQualificationError ? error.code : "UNKNOWN"; -} - -if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { - main().catch((error) => { - const code = exactImageQualificationFailureCode(error); - const message = error instanceof Error ? error.message : "unexpected qualification error"; - process.stderr.write(`${code}: ${message}\n`); - process.exitCode = 1; - }); -} diff --git a/tools/e2e/validate-exact-image-manifest.mts b/tools/e2e/validate-exact-image-manifest.mts deleted file mode 100644 index 2bbaf4186a..0000000000 --- a/tools/e2e/validate-exact-image-manifest.mts +++ /dev/null @@ -1,176 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import fs from "node:fs"; -import { pathToFileURL } from "node:url"; -import { TextDecoder } from "node:util"; -import { - ExactImageManifestError, - type ExactImageManifestExpectations, - exactImageManifestFailureCode, - normalizedExactImageManifestJson, - parseAndValidateExactImageManifest, -} from "./exact-image-manifest.mts"; -import * as privateFile from "./private-file.mts"; - -// The root TypeScript package is exposed as CommonJS under `node --import -// tsx` / `npx tsx`, but as an ESM namespace under Node's strip-types runtime -// and Vitest. Normalize both representations so every supported entrypoint -// uses the same no-follow private-file writer. -const privateFileRuntime = ( - "default" in privateFile && privateFile.default ? privateFile.default : privateFile -) as typeof import("./private-file.mts"); - -const MAX_MANIFEST_BYTES = 64 * 1024; -const NO_FOLLOW = fs.constants.O_NOFOLLOW ?? 0; -const NON_BLOCK = fs.constants.O_NONBLOCK ?? 0; - -type ExactImageManifestCliOptions = ExactImageManifestExpectations & { - manifest: string; - output: string; -}; - -const ARGUMENT_FIELDS = { - "--manifest": "manifest", - "--output": "output", - "--nemoclaw-sha": "nemoclawSha", - "--requester-run-id": "requesterWorkflowRunId", - "--requester-run-attempt": "requesterWorkflowRunAttempt", - "--correlation-id": "correlationId", - "--image-repository-sha": "imageRepositorySha", - "--producer-run-id": "workflowRunId", - "--producer-run-attempt": "workflowRunAttempt", -} as const; - -function requestInvalid(message: string): never { - throw new ExactImageManifestError("REQUEST_INVALID", message); -} - -function positiveIntegerArgument(value: string, flag: string): number { - if (!/^[1-9][0-9]*$/u.test(value)) { - requestInvalid(`${flag} must be a positive integer`); - } - const parsed = Number(value); - if (!Number.isSafeInteger(parsed)) { - requestInvalid(`${flag} exceeds the safe integer range`); - } - return parsed; -} - -export function parseExactImageManifestCliArgs( - argv: readonly string[], -): ExactImageManifestCliOptions { - const values = new Map(); - for (let index = 0; index < argv.length; index += 2) { - const flag = argv[index] as keyof typeof ARGUMENT_FIELDS | undefined; - const value = argv[index + 1]; - if (!flag || !(flag in ARGUMENT_FIELDS)) { - requestInvalid(`unknown argument ${JSON.stringify(flag ?? "")}`); - } - if (value === undefined || value.startsWith("--")) { - requestInvalid(`${flag} requires a value`); - } - const field = ARGUMENT_FIELDS[flag]; - if (values.has(field)) { - requestInvalid(`${flag} may be provided only once`); - } - values.set(field, value); - } - - const requireValue = (field: keyof ExactImageManifestCliOptions, flag: string): string => { - const value = values.get(field); - if (value === undefined) requestInvalid(`${flag} is required`); - return value; - }; - - return { - manifest: requireValue("manifest", "--manifest"), - output: requireValue("output", "--output"), - nemoclawSha: requireValue("nemoclawSha", "--nemoclaw-sha"), - requesterWorkflowRunId: requireValue("requesterWorkflowRunId", "--requester-run-id"), - requesterWorkflowRunAttempt: positiveIntegerArgument( - requireValue("requesterWorkflowRunAttempt", "--requester-run-attempt"), - "--requester-run-attempt", - ), - correlationId: requireValue("correlationId", "--correlation-id"), - imageRepositorySha: requireValue("imageRepositorySha", "--image-repository-sha"), - workflowRunId: requireValue("workflowRunId", "--producer-run-id"), - workflowRunAttempt: positiveIntegerArgument( - requireValue("workflowRunAttempt", "--producer-run-attempt"), - "--producer-run-attempt", - ), - }; -} - -function readManifestFile(file: string): string { - let descriptor: number; - try { - descriptor = fs.openSync(file, fs.constants.O_RDONLY | NO_FOLLOW | NON_BLOCK); - } catch { - throw new ExactImageManifestError( - "ARTIFACT_MISSING_OR_INVALID", - "manifest input could not be opened safely", - ); - } - - try { - const before = fs.fstatSync(descriptor); - if (!before.isFile() || before.nlink !== 1) { - throw new ExactImageManifestError( - "ARTIFACT_MISSING_OR_INVALID", - "manifest input must be one regular file", - ); - } - if (before.size > MAX_MANIFEST_BYTES) { - throw new ExactImageManifestError( - "ARTIFACT_MISSING_OR_INVALID", - `manifest input exceeds ${MAX_MANIFEST_BYTES} bytes`, - ); - } - const bytes = fs.readFileSync(descriptor); - const after = fs.fstatSync(descriptor); - if (bytes.length > MAX_MANIFEST_BYTES || after.size !== before.size || after.nlink !== 1) { - throw new ExactImageManifestError( - "ARTIFACT_MISSING_OR_INVALID", - "manifest input changed while it was read", - ); - } - try { - return new TextDecoder("utf-8", { fatal: true }).decode(bytes); - } catch { - throw new ExactImageManifestError( - "ARTIFACT_MISSING_OR_INVALID", - "manifest input must be valid UTF-8", - ); - } - } finally { - fs.closeSync(descriptor); - } -} - -export function runExactImageManifestCli(argv: readonly string[] = process.argv.slice(2)): void { - const options = parseExactImageManifestCliArgs(argv); - const accepted = parseAndValidateExactImageManifest(readManifestFile(options.manifest), options); - try { - privateFileRuntime.writePrivateRegularFile( - options.output, - normalizedExactImageManifestJson(accepted), - ); - } catch { - throw new ExactImageManifestError( - "OUTPUT_WRITE_FAILED", - "accepted manifest output could not be written safely", - ); - } -} - -if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { - try { - runExactImageManifestCli(); - } catch (error) { - const code = exactImageManifestFailureCode(error); - const message = error instanceof Error ? error.message : "unexpected manifest validation error"; - process.stderr.write(`${code}: ${message}\n`); - process.exitCode = 1; - } -} From 47409d51f0c7e441c2f2e05f3e219d260c7f346d Mon Sep 17 00:00:00 2001 From: Charan Jagwani Date: Mon, 20 Jul 2026 14:45:53 -0700 Subject: [PATCH 23/25] refactor(e2e): simplify exact-image Launchable lane Signed-off-by: Charan Jagwani --- .../brev-launchable-qualification.yaml | 355 --- .github/workflows/e2e.yaml | 347 ++- ci/source-shape-test-budget.json | 4 +- test/e2e/README.md | 36 +- test/e2e/mock-parity.json | 2 +- .../brev-launchable-e2e-workflow.test.ts | 150 ++ .../support/exact-image-manifest-cli.test.ts | 158 -- .../support/exact-image-manifest-fixture.ts | 61 - test/e2e/support/exact-image-manifest.test.ts | 253 -- ...exact-image-qualification-workflow.test.ts | 258 -- ...act-image-qualification-controller.test.ts | 1494 ------------ ...-image-qualification-controller-fixture.ts | 284 --- test/helpers/vitest-watch-triggers.ts | 8 +- test/vitest-watch-triggers.test.ts | 8 +- tools/e2e/exact-image-manifest.mts | 396 ---- .../exact-image-qualification-controller.mts | 2076 ----------------- tools/e2e/operations-workflow-boundary.mts | 7 +- ...upload-e2e-artifacts-workflow-boundary.mts | 25 + tools/e2e/validate-exact-image-manifest.mts | 176 -- 19 files changed, 547 insertions(+), 5551 deletions(-) delete mode 100644 .github/workflows/brev-launchable-qualification.yaml create mode 100644 test/e2e/support/brev-launchable-e2e-workflow.test.ts delete mode 100644 test/e2e/support/exact-image-manifest-cli.test.ts delete mode 100644 test/e2e/support/exact-image-manifest-fixture.ts delete mode 100644 test/e2e/support/exact-image-manifest.test.ts delete mode 100644 test/e2e/support/exact-image-qualification-workflow.test.ts delete mode 100644 test/exact-image-qualification-controller.test.ts delete mode 100644 test/helpers/exact-image-qualification-controller-fixture.ts delete mode 100644 tools/e2e/exact-image-manifest.mts delete mode 100755 tools/e2e/exact-image-qualification-controller.mts delete mode 100644 tools/e2e/validate-exact-image-manifest.mts diff --git a/.github/workflows/brev-launchable-qualification.yaml b/.github/workflows/brev-launchable-qualification.yaml deleted file mode 100644 index cbebea4e39..0000000000 --- a/.github/workflows/brev-launchable-qualification.yaml +++ /dev/null @@ -1,355 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -name: E2E / Exact Staging Brev Launchable -run-name: Exact staging Launchable qualification for ${{ inputs.candidate_sha }} - -on: - workflow_dispatch: - inputs: - candidate_sha: - description: Exact lowercase 40-character NemoClaw commit SHA to qualify. - required: true - type: string - reason: - description: Audit reason for starting this exact-image Launchable qualification. - required: true - type: string - workflow_call: - inputs: - candidate_sha: - description: Exact lowercase 40-character NemoClaw commit SHA to qualify. - required: true - type: string - reason: - description: Audit reason for starting this exact-image Launchable qualification. - required: true - type: string - -permissions: {} - -concurrency: - # Every candidate advances the same family consumed by the standing Launchable. - group: brev-launchable-qualification-staging-cpu - cancel-in-progress: false - -jobs: - preflight: - name: Validate exact qualification request - runs-on: ubuntu-latest - timeout-minutes: 10 - permissions: - contents: read - steps: - - name: Require explicit repository activation - env: - QUALIFICATION_ENABLED: ${{ vars.NEMOCLAW_BREV_LAUNCHABLE_QUALIFICATION_ENABLED }} - run: | - set -euo pipefail - if [[ "$QUALIFICATION_ENABLED" != "true" ]]; then - echo "::error::Exact staging Brev Launchable qualification is inactive. Set NEMOCLAW_BREV_LAUNCHABLE_QUALIFICATION_ENABLED=true only after the protected environment and external dependencies are ready." - exit 1 - fi - - - name: Checkout trusted controller - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - with: - ref: ${{ github.workflow_sha }} - persist-credentials: false - - - name: Setup Node - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0 - with: - node-version: "22" - - - name: Validate candidate and maintainer authority - env: - ACTOR: ${{ github.triggering_actor }} - CANDIDATE_SHA: ${{ inputs.candidate_sha }} - EVENT_NAME: ${{ github.event_name }} - GITHUB_TOKEN: ${{ github.token }} - QUALIFICATION_REASON: ${{ inputs.reason }} - REQUESTER_RUN_ATTEMPT: ${{ github.run_attempt }} - REQUESTER_RUN_ID: ${{ github.run_id }} - REQUEST_REF: ${{ github.ref }} - WORKFLOW_SHA: ${{ github.workflow_sha }} - run: >- - node --experimental-strip-types --no-warnings - tools/e2e/exact-image-qualification-controller.mts - --mode preflight - --actor "$ACTOR" - --candidate-sha "$CANDIDATE_SHA" - --event-name "$EVENT_NAME" - --reason "$QUALIFICATION_REASON" - --ref "$REQUEST_REF" - --requester-run-attempt "$REQUESTER_RUN_ATTEMPT" - --requester-run-id "$REQUESTER_RUN_ID" - --workflow-sha "$WORKFLOW_SHA" - - qualify: - name: Build, deploy, and test exact staging image - needs: preflight - # Keep the repository-owned switch outside the protected environment so an - # inactive run cannot request approval or credentials. - if: ${{ vars.NEMOCLAW_BREV_LAUNCHABLE_QUALIFICATION_ENABLED == 'true' && github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' }} - runs-on: ubuntu-latest - timeout-minutes: 120 - environment: - name: approve-brev-launchable-qualification - deployment: false - permissions: - contents: read - steps: - - name: Checkout trusted controller - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - with: - ref: ${{ github.workflow_sha }} - persist-credentials: false - - - name: Setup Node - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0 - with: - node-version: "22" - - - name: Prepare E2E workspace - uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@50281ee84c4a6fc759da95ea28fc0b7d9c378a28 - with: - build-cli: "false" - - - id: workspace - name: Create private evidence workspace - shell: bash - run: | - set -euo pipefail - work_dir="$(mktemp -d "${RUNNER_TEMP}/nemoclaw-image-qualification.XXXXXX")" - chmod 700 "$work_dir" - printf 'work_dir=%s\n' "$work_dir" >> "$GITHUB_OUTPUT" - - - id: start - name: Dispatch exact qualification image build - env: - ACTOR: ${{ github.triggering_actor }} - CANDIDATE_SHA: ${{ inputs.candidate_sha }} - EVENT_NAME: ${{ github.event_name }} - GITHUB_TOKEN: ${{ github.token }} - NEMOCLAW_IMAGE_QUALIFICATION_TOKEN: ${{ secrets.NEMOCLAW_IMAGE_DISPATCH_TOKEN }} - QUALIFICATION_REASON: ${{ inputs.reason }} - REQUESTER_RUN_ATTEMPT: ${{ github.run_attempt }} - REQUESTER_RUN_ID: ${{ github.run_id }} - REQUEST_REF: ${{ github.ref }} - WORKFLOW_SHA: ${{ github.workflow_sha }} - run: >- - node --experimental-strip-types --no-warnings - tools/e2e/exact-image-qualification-controller.mts - --mode start - --actor "$ACTOR" - --candidate-sha "$CANDIDATE_SHA" - --event-name "$EVENT_NAME" - --reason "$QUALIFICATION_REASON" - --ref "$REQUEST_REF" - --requester-run-attempt "$REQUESTER_RUN_ATTEMPT" - --requester-run-id "$REQUESTER_RUN_ID" - --workflow-sha "$WORKFLOW_SHA" - --work-dir "${{ steps.workspace.outputs.work_dir }}" - - - name: Wait for the bound producer run - env: - NEMOCLAW_IMAGE_QUALIFICATION_TOKEN: ${{ secrets.NEMOCLAW_IMAGE_DISPATCH_TOKEN }} - run: >- - node --experimental-strip-types --no-warnings - tools/e2e/exact-image-qualification-controller.mts - --mode wait - --work-dir "${{ steps.workspace.outputs.work_dir }}" - - - name: Download and verify the bound manifest artifact - env: - NEMOCLAW_IMAGE_QUALIFICATION_TOKEN: ${{ secrets.NEMOCLAW_IMAGE_DISPATCH_TOKEN }} - run: >- - node --experimental-strip-types --no-warnings - tools/e2e/exact-image-qualification-controller.mts - --mode download - --work-dir "${{ steps.workspace.outputs.work_dir }}" - - - name: Validate the exact image manifest - env: - CANDIDATE_SHA: ${{ inputs.candidate_sha }} - CORRELATION_ID: ${{ steps.start.outputs.correlation_id }} - IMAGE_REPOSITORY_SHA: ${{ steps.start.outputs.image_repository_sha }} - PRODUCER_RUN_ATTEMPT: ${{ steps.start.outputs.producer_run_attempt }} - PRODUCER_RUN_ID: ${{ steps.start.outputs.producer_run_id }} - REQUESTER_RUN_ATTEMPT: ${{ github.run_attempt }} - REQUESTER_RUN_ID: ${{ github.run_id }} - run: >- - node --experimental-strip-types --no-warnings - tools/e2e/validate-exact-image-manifest.mts - --manifest "${{ steps.workspace.outputs.work_dir }}/nemoclaw-image-manifest.v1.json" - --output "${{ steps.workspace.outputs.work_dir }}/validated-manifest.v1.json" - --nemoclaw-sha "$CANDIDATE_SHA" - --requester-run-id "$REQUESTER_RUN_ID" - --requester-run-attempt "$REQUESTER_RUN_ATTEMPT" - --correlation-id "$CORRELATION_ID" - --image-repository-sha "$IMAGE_REPOSITORY_SHA" - --producer-run-id "$PRODUCER_RUN_ID" - --producer-run-attempt "$PRODUCER_RUN_ATTEMPT" - - - name: Record accepted provenance and hashes - run: >- - node --experimental-strip-types --no-warnings - tools/e2e/exact-image-qualification-controller.mts - --mode finalize - --work-dir "${{ steps.workspace.outputs.work_dir }}" - - - name: Install pinned Brev CLI - env: - BREV_CLI_SHA256: 5a6e70374db9be33f85f299161733b4a8409840d47638c781429b96e8d53704f - BREV_CLI_VERSION: 0.6.330 - run: | - set -euo pipefail - archive="${RUNNER_TEMP}/brev-cli.tar.gz" - curl -fsSL -o "$archive" "https://github.com/brevdev/brev-cli/releases/download/v${BREV_CLI_VERSION}/brev-cli_${BREV_CLI_VERSION}_linux_amd64.tar.gz" - printf '%s %s\n' "$BREV_CLI_SHA256" "$archive" | sha256sum -c - - tar -xzf "$archive" -C "${RUNNER_TEMP}" brev - mkdir -p "${RUNNER_TEMP}/bin" - install -m 0755 "${RUNNER_TEMP}/brev" "${RUNNER_TEMP}/bin/brev" - printf '%s\n' "${RUNNER_TEMP}/bin" >> "$GITHUB_PATH" - - - name: Authenticate Brev CLI - env: - BREV_API_KEY: ${{ secrets.BREV_API_KEY }} - BREV_ORG_ID: ${{ secrets.BREV_ORG_ID }} - run: | - set -euo pipefail - brev login --api-key "$BREV_API_KEY" --org-id "$BREV_ORG_ID" - brev ls --json | jq -e '.workspaces | type == "array"' >/dev/null - - - id: brev-workspace - name: Bind unique staging workspace name - env: - CANDIDATE_SHA: ${{ inputs.candidate_sha }} - run: | - set -euo pipefail - short_sha="${CANDIDATE_SHA:0:8}" - instance_name="nclaw-e2e-${short_sha}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" - printf 'instance_name=%s\n' "$instance_name" >> "$GITHUB_OUTPUT" - - - name: Deploy exact staging Brev Launchable - env: - BREV_LAUNCHABLE_ID: ${{ vars.NEMOCLAW_STAGING_LAUNCHABLE_ID }} - INSTANCE_NAME: ${{ steps.brev-workspace.outputs.instance_name }} - WORK_DIR: ${{ steps.workspace.outputs.work_dir }} - run: tools/e2e/brev-launchable-runtime.sh deploy - - - name: Run existing full E2E suite against the Brev Launchable - env: - CANDIDATE_SHA: ${{ inputs.candidate_sha }} - INSTANCE_NAME: ${{ steps.brev-workspace.outputs.instance_name }} - NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }} - VALIDATED_MANIFEST: ${{ steps.workspace.outputs.work_dir }}/validated-manifest.v1.json - WORK_DIR: ${{ steps.workspace.outputs.work_dir }} - run: tools/e2e/brev-launchable-runtime.sh qualify - - - id: redact-runtime-evidence - name: Redact runtime evidence - if: ${{ always() && steps.workspace.outputs.work_dir != '' }} - env: - NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }} - WORK_DIR: ${{ steps.workspace.outputs.work_dir }} - run: | - set -euo pipefail - python3 - <<'PY' - import os - import stat - from pathlib import Path - - MAX_ENTRIES = 2_500 - MAX_FILES = 1_250 - MAX_TOTAL_BYTES = 128 * 1024 * 1024 - - secret = os.environ.get("NVIDIA_INFERENCE_API_KEY", "").encode() - root = Path(os.environ["WORK_DIR"]) - if root.is_symlink() or not root.is_dir(): - raise SystemExit("runtime evidence root must be a real directory") - - entries = 0 - total_bytes = 0 - paths = [] - stack = [root] - while stack: - directory = stack.pop() - with os.scandir(directory) as children: - for child in children: - entries += 1 - if entries > MAX_ENTRIES: - raise SystemExit(f"runtime evidence exceeds {MAX_ENTRIES} entries") - path = Path(child.path) - if child.is_symlink(): - raise SystemExit(f"runtime evidence must not contain symlinks: {path}") - if child.is_dir(follow_symlinks=False): - stack.append(path) - continue - if not child.is_file(follow_symlinks=False): - raise SystemExit(f"runtime evidence contains a non-regular file: {path}") - metadata = child.stat(follow_symlinks=False) - if not stat.S_ISREG(metadata.st_mode): - raise SystemExit(f"runtime evidence contains a non-regular file: {path}") - paths.append(path) - total_bytes += metadata.st_size - if len(paths) > MAX_FILES: - raise SystemExit(f"runtime evidence exceeds {MAX_FILES} files") - if total_bytes > MAX_TOTAL_BYTES: - raise SystemExit(f"runtime evidence exceeds {MAX_TOTAL_BYTES} bytes") - - for path in paths: - if secret: - content = path.read_bytes() - if secret in content: - path.write_bytes(content.replace(secret, b"[REDACTED]")) - PY - - - name: Delete staging workspace and verify absence - if: ${{ always() && steps.workspace.outputs.work_dir != '' && steps.brev-workspace.outputs.instance_name != '' }} - env: - INSTANCE_NAME: ${{ steps.brev-workspace.outputs.instance_name }} - WORK_DIR: ${{ steps.workspace.outputs.work_dir }} - run: tools/e2e/brev-launchable-runtime.sh cleanup - - - name: Cancel an incomplete producer run - if: ${{ always() && steps.workspace.outputs.work_dir != '' && steps.start.outcome != 'skipped' }} - continue-on-error: true - env: - NEMOCLAW_IMAGE_QUALIFICATION_TOKEN: ${{ secrets.NEMOCLAW_IMAGE_DISPATCH_TOKEN }} - run: >- - node --experimental-strip-types --no-warnings - tools/e2e/exact-image-qualification-controller.mts - --mode cancel - --work-dir "${{ steps.workspace.outputs.work_dir }}" - - - name: Upload exact staging Launchable evidence - if: ${{ always() && steps.workspace.outputs.work_dir != '' && steps.redact-runtime-evidence.outcome == 'success' }} - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: brev-launchable-qualification-${{ inputs.candidate_sha }}-${{ github.run_id }} - path: | - ${{ steps.workspace.outputs.work_dir }}/dispatch-intent.v1.json - ${{ steps.workspace.outputs.work_dir }}/dispatch-reconciliation.v1.json - ${{ steps.workspace.outputs.work_dir }}/controller-state.json - ${{ steps.workspace.outputs.work_dir }}/controller-state.corrupt-*.json - ${{ steps.workspace.outputs.work_dir }}/nemoclaw-image-handoff.zip - ${{ steps.workspace.outputs.work_dir }}/nemoclaw-image-manifest.v1.json - ${{ steps.workspace.outputs.work_dir }}/validated-manifest.v1.json - ${{ steps.workspace.outputs.work_dir }}/qualification-evidence.v1.json - ${{ steps.workspace.outputs.work_dir }}/brev-deploy-request.json - ${{ steps.workspace.outputs.work_dir }}/brev-workspace-ready.json - ${{ steps.workspace.outputs.work_dir }}/brev-provision.json - ${{ steps.workspace.outputs.work_dir }}/brev-boot-image.json - ${{ steps.workspace.outputs.work_dir }}/brev-identity-evidence.json - ${{ steps.workspace.outputs.work_dir }}/brev-launchable-e2e.log - ${{ steps.workspace.outputs.work_dir }}/brev-launchable-e2e-evidence.json - ${{ steps.workspace.outputs.work_dir }}/brev-launchable-cloud-openclaw/ - ${{ steps.workspace.outputs.work_dir }}/brev-cleanup-evidence.json - if-no-files-found: error - retention-days: 90 - - - name: Remove private evidence workspace - if: ${{ always() && steps.workspace.outputs.work_dir != '' }} - run: rm -rf -- "${{ steps.workspace.outputs.work_dir }}" diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 2fbfec406d..0ede0b10b9 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -186,15 +186,346 @@ jobs: fi staging-brev-launchable: - name: Exact staging Brev Launchable + name: Exact-image Brev Launchable E2E needs: generate-matrix - # Repository owners enable this only after the protected environment, - # producer, standing Launchable, and Brev contract are ready. - if: ${{ vars.NEMOCLAW_BREV_LAUNCHABLE_QUALIFICATION_ENABLED == 'true' && github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && inputs.checkout_sha == '' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') }} - uses: ./.github/workflows/brev-launchable-qualification.yaml - with: - candidate_sha: ${{ github.sha }} - reason: E2E run ${{ github.run_id }} qualifying the exact checked-out NemoClaw candidate + if: >- + ${{ + vars.NEMOCLAW_BREV_LAUNCHABLE_E2E_ENABLED == 'true' && + github.repository == 'NVIDIA/NemoClaw' && + github.ref == 'refs/heads/main' && + ( + github.event_name == 'schedule' || + ( + github.event_name == 'workflow_dispatch' && + ( + contains(format(',{0},', inputs.targets), ',brev-launchable-cloud-openclaw,') || + (inputs.checkout_sha == '' && inputs.targets == '' && inputs.jobs == '') + ) + ) + ) + }} + runs-on: ubuntu-latest + timeout-minutes: 120 + concurrency: + group: brev-launchable-staging-cpu + cancel-in-progress: false + environment: + name: approve-brev-launchable-e2e + deployment: false + permissions: + contents: read + steps: + # The workflow and helper come from trusted main. Candidate code is used + # only after the producer bakes it into the staging image. + - name: Check out trusted E2E control code + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + ref: ${{ github.workflow_sha }} + persist-credentials: false + + - id: request + name: Bind exact image request + env: + CANDIDATE_SHA: ${{ inputs.checkout_sha || github.sha }} + WORKFLOW_SHA: ${{ github.workflow_sha }} + run: | + set -euo pipefail + [[ "$CANDIDATE_SHA" =~ ^[0-9a-f]{40}$ ]] \ + || { echo "::error::Candidate must be a lowercase 40-character SHA."; exit 1; } + [[ "$WORKFLOW_SHA" =~ ^[0-9a-f]{40}$ ]] \ + || { echo "::error::Workflow SHA must be a lowercase 40-character SHA."; exit 1; } + [ "$(git rev-parse HEAD)" = "$WORKFLOW_SHA" ] \ + || { echo "::error::Trusted E2E checkout does not match the workflow SHA."; exit 1; } + + work_dir="$(mktemp -d "${RUNNER_TEMP}/nemoclaw-launchable-e2e.XXXXXX")" + chmod 700 "$work_dir" + correlation_id="$(python3 -c 'import uuid; print(uuid.uuid4())')" + instance_name="nclaw-e2e-${CANDIDATE_SHA:0:8}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + { + echo "candidate_sha=$CANDIDATE_SHA" + echo "correlation_id=$correlation_id" + echo "instance_name=$instance_name" + echo "work_dir=$work_dir" + } >> "$GITHUB_OUTPUT" + + - id: producer + name: Build or reuse the exact staging image + env: + CANDIDATE_SHA: ${{ steps.request.outputs.candidate_sha }} + CORRELATION_ID: ${{ steps.request.outputs.correlation_id }} + PRODUCER_TOKEN: ${{ secrets.NEMOCLAW_IMAGE_DISPATCH_TOKEN }} + WORK_DIR: ${{ steps.request.outputs.work_dir }} + run: | + set -euo pipefail + [ -n "$PRODUCER_TOKEN" ] \ + || { echo "::error::NEMOCLAW_IMAGE_DISPATCH_TOKEN is required."; exit 1; } + + payload="$(jq -cn \ + --arg sha "$CANDIDATE_SHA" \ + --arg correlation "$CORRELATION_ID" \ + --arg run_id "$GITHUB_RUN_ID" \ + --arg run_attempt "$GITHUB_RUN_ATTEMPT" \ + '{ref:"main",inputs:{nemoclaw_sha:$sha,correlation_id:$correlation,requester_workflow_run_id:$run_id,requester_workflow_run_attempt:$run_attempt},return_run_details:true}')" + response="$WORK_DIR/producer-dispatch.json" + status="$(curl --silent --show-error --proto '=https' \ + --request POST \ + --header "Authorization: Bearer $PRODUCER_TOKEN" \ + --header "Accept: application/vnd.github+json" \ + --header "X-GitHub-Api-Version: 2026-03-10" \ + --data "$payload" \ + --output "$response" \ + --write-out '%{http_code}' \ + https://api.github.com/repos/brevdev/nemoclaw-image/actions/workflows/build-qualification-image.yml/dispatches)" + [ "$status" = "200" ] \ + || { echo "::error::Image producer dispatch returned HTTP $status."; exit 1; } + + run_id="$(jq -er '.workflow_run_id | tostring | select(test("^[1-9][0-9]*$"))' "$response")" + run_url="$(jq -er '.run_url' "$response")" + html_url="$(jq -er '.html_url' "$response")" + [ "$run_url" = "https://api.github.com/repos/brevdev/nemoclaw-image/actions/runs/$run_id" ] + [ "$html_url" = "https://github.com/brevdev/nemoclaw-image/actions/runs/$run_id" ] + echo "run_id=$run_id" >> "$GITHUB_OUTPUT" + + deadline=$((SECONDS + 45 * 60)) + while [ "$SECONDS" -lt "$deadline" ]; do + curl --fail --silent --show-error --proto '=https' \ + --header "Authorization: Bearer $PRODUCER_TOKEN" \ + --header "Accept: application/vnd.github+json" \ + --header "X-GitHub-Api-Version: 2026-03-10" \ + "$run_url" > "$WORK_DIR/producer-run.json" + jq -e \ + --arg sha "$CANDIDATE_SHA" \ + --arg title "Qualify NemoClaw $CANDIDATE_SHA ($CORRELATION_ID)" \ + --argjson run_id "$run_id" ' + .id == $run_id and + .event == "workflow_dispatch" and + .head_branch == "main" and + .path == ".github/workflows/build-qualification-image.yml" and + .display_title == $title and + .run_attempt == 1 and + .repository.full_name == "brevdev/nemoclaw-image" and + .head_repository.full_name == "brevdev/nemoclaw-image" and + (.head_sha | test("^[0-9a-f]{40}$")) + ' "$WORK_DIR/producer-run.json" >/dev/null + + run_status="$(jq -r '.status' "$WORK_DIR/producer-run.json")" + if [ "$run_status" = "completed" ]; then + conclusion="$(jq -r '.conclusion // ""' "$WORK_DIR/producer-run.json")" + [ "$conclusion" = "success" ] \ + || { echo "::error::Image producer completed with $conclusion."; exit 1; } + jq -r '.head_sha' "$WORK_DIR/producer-run.json" \ + | sed 's/^/producer_sha=/' >> "$GITHUB_OUTPUT" + exit 0 + fi + sleep 15 + done + echo "::error::Image producer did not complete within 45 minutes." + exit 1 + + - name: Read immutable image handoff + env: + CANDIDATE_SHA: ${{ steps.request.outputs.candidate_sha }} + CORRELATION_ID: ${{ steps.request.outputs.correlation_id }} + PRODUCER_RUN_ID: ${{ steps.producer.outputs.run_id }} + PRODUCER_SHA: ${{ steps.producer.outputs.producer_sha }} + PRODUCER_TOKEN: ${{ secrets.NEMOCLAW_IMAGE_DISPATCH_TOKEN }} + WORK_DIR: ${{ steps.request.outputs.work_dir }} + run: | + set -euo pipefail + artifact_name="nemoclaw-image-handoff-v1-${PRODUCER_RUN_ID}-1" + deadline=$((SECONDS + 2 * 60)) + while [ "$SECONDS" -lt "$deadline" ]; do + curl --fail --silent --show-error --proto '=https' \ + --header "Authorization: Bearer $PRODUCER_TOKEN" \ + --header "Accept: application/vnd.github+json" \ + --header "X-GitHub-Api-Version: 2026-03-10" \ + "https://api.github.com/repos/brevdev/nemoclaw-image/actions/runs/${PRODUCER_RUN_ID}/artifacts" \ + > "$WORK_DIR/producer-artifacts.json" + count="$(jq --arg name "$artifact_name" '[.artifacts[] | select(.name == $name)] | length' "$WORK_DIR/producer-artifacts.json")" + [ "$count" = "0" ] || break + sleep 5 + done + [ "$count" = "1" ] \ + || { echo "::error::Expected one immutable image handoff; found $count."; exit 1; } + + artifact_url="$(jq -er \ + --arg name "$artifact_name" \ + --arg sha "$PRODUCER_SHA" \ + --argjson run_id "$PRODUCER_RUN_ID" ' + .artifacts[] | + select(.name == $name) | + select(.expired == false) | + select(.size_in_bytes > 0 and .size_in_bytes <= 1048576) | + select(.workflow_run.id == $run_id and .workflow_run.head_sha == $sha) | + .archive_download_url + ' "$WORK_DIR/producer-artifacts.json")" + curl --fail --location --silent --show-error --proto '=https' \ + --max-filesize 1048576 \ + --header "Authorization: Bearer $PRODUCER_TOKEN" \ + --header "Accept: application/vnd.github+json" \ + --header "X-GitHub-Api-Version: 2026-03-10" \ + "$artifact_url" > "$WORK_DIR/image-handoff.zip" + [ "$(unzip -Z1 "$WORK_DIR/image-handoff.zip")" = "nemoclaw-image-manifest.v1.json" ] \ + || { echo "::error::Image handoff archive has an unexpected layout."; exit 1; } + unzip -p "$WORK_DIR/image-handoff.zip" nemoclaw-image-manifest.v1.json \ + > "$WORK_DIR/image-handoff.json" + + jq -e \ + --arg candidate "$CANDIDATE_SHA" \ + --arg correlation "$CORRELATION_ID" \ + --arg producer_sha "$PRODUCER_SHA" \ + --arg requester_run "$GITHUB_RUN_ID" \ + --argjson requester_attempt "$GITHUB_RUN_ATTEMPT" \ + --arg producer_run "$PRODUCER_RUN_ID" ' + type == "object" and + .schemaVersion == 1 and + .kind == "nemoclaw-exact-image-manifest" and + .correlationId == $correlation and + .requesterRepository == "NVIDIA/NemoClaw" and + .requesterWorkflowRunId == $requester_run and + .requesterWorkflowRunAttempt == $requester_attempt and + .nemoclawSha == $candidate and + .imageRepository == "brevdev/nemoclaw-image" and + .imageRepositorySha == $producer_sha and + .producerWorkflow == ".github/workflows/build-qualification-image.yml" and + .workflowRunId == $producer_run and + .workflowRunAttempt == 1 and + .imageKind == "compute#image" and + .status == "READY" and + .channel == "staging" and + .variant == "cpu" and + .observedFamily == "nemoclaw-brev-staging-cpu" and + (.result == "built" or .result == "reused") and + (.project | test("^[a-z][a-z0-9-]{4,28}[a-z0-9]$")) and + (.imageName | test("^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$")) and + (.imageId | test("^[1-9][0-9]*$")) and + .imageSelfLink == ("https://www.googleapis.com/compute/v1/projects/" + .project + "/global/images/" + .imageName) + ' "$WORK_DIR/image-handoff.json" >/dev/null + + - name: Install pinned Brev CLI + env: + BREV_CLI_SHA256: 5a6e70374db9be33f85f299161733b4a8409840d47638c781429b96e8d53704f + BREV_CLI_VERSION: 0.6.330 + run: | + set -euo pipefail + archive="${RUNNER_TEMP}/brev-cli.tar.gz" + curl -fsSL --proto '=https' -o "$archive" \ + "https://github.com/brevdev/brev-cli/releases/download/v${BREV_CLI_VERSION}/brev-cli_${BREV_CLI_VERSION}_linux_amd64.tar.gz" + printf '%s %s\n' "$BREV_CLI_SHA256" "$archive" | sha256sum -c - + tar -xzf "$archive" -C "$RUNNER_TEMP" brev + mkdir -p "$RUNNER_TEMP/bin" + install -m 0755 "$RUNNER_TEMP/brev" "$RUNNER_TEMP/bin/brev" + echo "$RUNNER_TEMP/bin" >> "$GITHUB_PATH" + + - name: Authenticate Brev CLI + env: + BREV_API_KEY: ${{ secrets.BREV_API_KEY }} + BREV_ORG_ID: ${{ secrets.BREV_ORG_ID }} + run: | + set -euo pipefail + brev login --api-key "$BREV_API_KEY" --org-id "$BREV_ORG_ID" + brev ls --json | jq -e '.workspaces | type == "array"' >/dev/null + + - id: deploy + name: Deploy standing staging Launchable + env: + BREV_LAUNCHABLE_ID: ${{ vars.NEMOCLAW_STAGING_LAUNCHABLE_ID }} + INSTANCE_NAME: ${{ steps.request.outputs.instance_name }} + WORK_DIR: ${{ steps.request.outputs.work_dir }} + run: tools/e2e/brev-launchable-runtime.sh deploy + + - name: Verify exact image and run existing full E2E + env: + CANDIDATE_SHA: ${{ steps.request.outputs.candidate_sha }} + INSTANCE_NAME: ${{ steps.request.outputs.instance_name }} + NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }} + VALIDATED_MANIFEST: ${{ steps.request.outputs.work_dir }}/image-handoff.json + WORK_DIR: ${{ steps.request.outputs.work_dir }} + run: tools/e2e/brev-launchable-runtime.sh qualify + + - id: redact-launchable-evidence + name: Redact Launchable evidence + if: ${{ always() && steps.request.outputs.work_dir != '' }} + env: + NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }} + WORK_DIR: ${{ steps.request.outputs.work_dir }} + run: | + set -euo pipefail + python3 - <<'PY' + import os + from pathlib import Path + + root = Path(os.environ["WORK_DIR"]) + secret = os.environ.get("NVIDIA_INFERENCE_API_KEY", "").encode() + if root.is_symlink() or not root.is_dir(): + raise SystemExit("evidence root must be a real directory") + paths = [] + total = 0 + for path in root.rglob("*"): + if path.is_symlink(): + raise SystemExit(f"evidence must not contain symlinks: {path}") + if path.is_dir(): + continue + if not path.is_file(): + raise SystemExit(f"evidence contains a special file: {path}") + paths.append(path) + total += path.stat().st_size + if len(paths) > 1250 or total > 128 * 1024 * 1024: + raise SystemExit("evidence exceeds upload bounds") + if secret: + for path in paths: + contents = path.read_bytes() + if secret in contents: + path.write_bytes(contents.replace(secret, b"[REDACTED]")) + PY + + - name: Delete staging workspace and verify absence + if: ${{ always() && steps.deploy.outcome != 'skipped' && steps.request.outputs.work_dir != '' && steps.request.outputs.instance_name != '' }} + env: + INSTANCE_NAME: ${{ steps.request.outputs.instance_name }} + WORK_DIR: ${{ steps.request.outputs.work_dir }} + run: tools/e2e/brev-launchable-runtime.sh cleanup + + - name: Cancel an incomplete image build + if: ${{ always() && steps.producer.outputs.run_id != '' }} + continue-on-error: true + env: + PRODUCER_RUN_ID: ${{ steps.producer.outputs.run_id }} + PRODUCER_TOKEN: ${{ secrets.NEMOCLAW_IMAGE_DISPATCH_TOKEN }} + run: | + set -euo pipefail + run_url="https://api.github.com/repos/brevdev/nemoclaw-image/actions/runs/${PRODUCER_RUN_ID}" + status="$(curl --fail --silent --show-error --proto '=https' \ + --header "Authorization: Bearer $PRODUCER_TOKEN" \ + --header "Accept: application/vnd.github+json" \ + --header "X-GitHub-Api-Version: 2026-03-10" \ + "$run_url" | jq -r '.status')" + if [ "$status" != "completed" ]; then + curl --fail --silent --show-error --proto '=https' \ + --request POST \ + --header "Authorization: Bearer $PRODUCER_TOKEN" \ + --header "Accept: application/vnd.github+json" \ + --header "X-GitHub-Api-Version: 2026-03-10" \ + "$run_url/cancel" >/dev/null + fi + + - name: Upload exact-image Launchable evidence + if: ${{ always() && steps.request.outputs.work_dir != '' && steps.redact-launchable-evidence.outcome == 'success' }} + uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 + with: + name: brev-launchable-e2e-${{ steps.request.outputs.candidate_sha }}-${{ github.run_id }} + path: | + ${{ steps.request.outputs.work_dir }}/producer-run.json + ${{ steps.request.outputs.work_dir }}/image-handoff.json + ${{ steps.request.outputs.work_dir }}/brev-deploy-request.json + ${{ steps.request.outputs.work_dir }}/brev-workspace-ready.json + ${{ steps.request.outputs.work_dir }}/brev-provision.json + ${{ steps.request.outputs.work_dir }}/brev-boot-image.json + ${{ steps.request.outputs.work_dir }}/brev-identity-evidence.json + ${{ steps.request.outputs.work_dir }}/brev-launchable-e2e.log + ${{ steps.request.outputs.work_dir }}/brev-launchable-e2e-evidence.json + ${{ steps.request.outputs.work_dir }}/brev-launchable-cloud-openclaw/ + ${{ steps.request.outputs.work_dir }}/brev-cleanup-evidence.json live: needs: generate-matrix diff --git a/ci/source-shape-test-budget.json b/ci/source-shape-test-budget.json index 72b8a11b4c..8d50f474e7 100644 --- a/ci/source-shape-test-budget.json +++ b/ci/source-shape-test-budget.json @@ -132,8 +132,8 @@ "category": "security" }, { - "file": "test/e2e/support/exact-image-qualification-workflow.test.ts", - "test": "keeps exact-image Launchable qualification protected, reusable, and fail-closed", + "file": "test/e2e/support/brev-launchable-e2e-workflow.test.ts", + "test": "keeps exact-image Launchable coverage inside the existing E2E workflow", "category": "security" }, { diff --git a/test/e2e/README.md b/test/e2e/README.md index b05af6c6b4..7bf545a814 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -85,23 +85,27 @@ artifact so baseline aggregation stays stable. Older issue references to Vitest target artifacts under `e2e-artifacts/vitest/` map to this consolidated `e2e-artifacts/live/` registry-target artifact layout. -## Exact staging Brev Launchable qualification - -The exact-image staging Launchable lane is inactive by default. Only scheduled -and manual `e2e.yaml` runs on `main` with an empty `checkout_sha` may call it, -and only when the repository variable -`NEMOCLAW_BREV_LAUNCHABLE_QUALIFICATION_ENABLED` is exactly `true`. PR-gate -runs cannot enter this credentialed lane. A direct dispatch of -`brev-launchable-qualification.yaml` applies the same repository, ref, and -activation checks and requires `candidate_sha` to equal the trusted current -`main` workflow SHA. While inactive, it fails before checkout, environment -approval, credential access, or external API calls. -The skipped `staging-brev-launchable` job in a routine E2E run is an inactive -status, not successful qualification evidence. +## Exact-image Brev Launchable E2E + +The `staging-brev-launchable` job is part of the existing `e2e.yaml` workflow +and is inactive by default. It runs for scheduled E2E on `main`, an ordinary +manual run with no selection, or an authorized dispatch that selects +`brev-launchable-cloud-openclaw`. The repository variable +`NEMOCLAW_BREV_LAUNCHABLE_E2E_ENABLED` must be exactly `true`. The job checks +out its control code from the trusted workflow SHA. It sends the candidate SHA +to the image producer without checking out or executing that candidate on the +credentialed runner. + +The job dispatches the existing `brevdev/nemoclaw-image` exact-image producer, +waits for its returned run ID, and reads the producer's immutable image handoff. +It does not implement a separate qualification workflow, controller, state +machine, or manifest framework. The mutable staging family is publication +state, not proof: the job requires the booted image ID and baked NemoClaw SHA to +match the producer handoff before E2E starts. Before setting the activation variable, repository owners must: -- create and protect the `approve-brev-launchable-qualification` environment +- create and protect the `approve-brev-launchable-e2e` environment for `main`, with the required reviewers; - configure that environment with `NEMOCLAW_IMAGE_DISPATCH_TOKEN`, `BREV_API_KEY`, `BREV_ORG_ID`, and `NVIDIA_INFERENCE_API_KEY`; @@ -111,12 +115,12 @@ Before setting the activation variable, repository owners must: - verify that the producer workflow and the Brev Launchable contract are ready for automated image builds, provisioning, validation, and cleanup. -Set `NEMOCLAW_BREV_LAUNCHABLE_QUALIFICATION_ENABLED=true` only after those +Set `NEMOCLAW_BREV_LAUNCHABLE_E2E_ENABLED=true` only after those conditions are met. Remove the variable, or set it to any value other than `true`, to keep routine E2E runs from starting this cost-bearing lane. The runtime target is `brev-launchable-cloud-openclaw`. After proving the -workspace booted the accepted image and exact NemoClaw SHA, the lane runs the +workspace booted the producer image and exact NemoClaw SHA, the lane runs the existing `test/e2e/live/full-e2e.test.ts` suite in `preinstalled-launchable` setup mode. That mode onboards through the baked `brev-quickstart` entry point and reuses the suite's CLI, policy, real first-agent-turn, hosted-inference, diff --git a/test/e2e/mock-parity.json b/test/e2e/mock-parity.json index 4e02191be1..661d34bcb7 100644 --- a/test/e2e/mock-parity.json +++ b/test/e2e/mock-parity.json @@ -28,7 +28,7 @@ "live": "test/e2e/live/full-e2e.test.ts", "fast": [ "test/brev-launchable-runtime.test.ts", - "test/e2e/support/exact-image-qualification-workflow.test.ts", + "test/e2e/support/brev-launchable-e2e-workflow.test.ts", "test/e2e/support/onboard-performance.test.ts", "test/e2e/support/e2e-cleanup-resources.test.ts", "test/e2e/support/e2e-clients.test.ts" diff --git a/test/e2e/support/brev-launchable-e2e-workflow.test.ts b/test/e2e/support/brev-launchable-e2e-workflow.test.ts new file mode 100644 index 0000000000..e80270151a --- /dev/null +++ b/test/e2e/support/brev-launchable-e2e-workflow.test.ts @@ -0,0 +1,150 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { expect, it } from "vitest"; + +import { readRepoText, readYaml, type Workflow } from "../../helpers/e2e-workflow-contract"; + +const E2E_WORKFLOW_PATH = ".github/workflows/e2e.yaml"; +const RUNTIME_PATH = "tools/e2e/brev-launchable-runtime.sh"; +const FULL_E2E_PATH = "test/e2e/live/full-e2e.test.ts"; + +function job(workflow: Workflow, name: string) { + const value = workflow.jobs[name]; + expect(value, `missing ${name} job`).toBeDefined(); + return value!; +} + +// source-shape-contract: security -- The exact-image Launchable path stays a single trusted E2E job with immutable identity proof and terminal cleanup +it("keeps exact-image Launchable coverage inside the existing E2E workflow", () => { + const workflow = readYaml(E2E_WORKFLOW_PATH); + const source = readRepoText(E2E_WORKFLOW_PATH); + const runtime = readRepoText(RUNTIME_PATH); + const fullE2e = readRepoText(FULL_E2E_PATH); + const launchable = job(workflow, "staging-brev-launchable"); + const steps = launchable.steps ?? []; + + expect(fs.existsSync(".github/workflows/brev-launchable-qualification.yaml")).toBe(false); + expect(fs.existsSync("tools/e2e/exact-image-qualification-controller.mts")).toBe(false); + expect(fs.existsSync("tools/e2e/exact-image-manifest.mts")).toBe(false); + expect(launchable["runs-on"]).toBe("ubuntu-latest"); + expect(launchable["timeout-minutes"]).toBe(120); + expect(launchable.permissions).toEqual({ contents: "read" }); + expect(launchable.environment).toEqual({ + name: "approve-brev-launchable-e2e", + deployment: false, + }); + expect(launchable.concurrency).toEqual({ + group: "brev-launchable-staging-cpu", + "cancel-in-progress": false, + }); + expect(launchable.if).toContain("vars.NEMOCLAW_BREV_LAUNCHABLE_E2E_ENABLED == 'true'"); + expect(launchable.if).toContain("github.repository == 'NVIDIA/NemoClaw'"); + expect(launchable.if).toContain("github.ref == 'refs/heads/main'"); + expect(launchable.if).toContain("brev-launchable-cloud-openclaw"); + expect(launchable.if).not.toContain("inputs.checkout_sha == '' && (github.event_name"); + + const checkout = steps.find((step) => step.name === "Check out trusted E2E control code"); + expect(checkout?.uses).toMatch(/^actions\/checkout@[0-9a-f]{40}$/u); + expect(checkout?.with).toEqual({ + ref: "${{ github.workflow_sha }}", + "persist-credentials": false, + }); + + const request = steps.find((step) => step.name === "Bind exact image request"); + expect(request?.env?.CANDIDATE_SHA).toBe("${{ inputs.checkout_sha || github.sha }}"); + expect(request?.env?.WORKFLOW_SHA).toBe("${{ github.workflow_sha }}"); + expect(request?.run).toContain('git rev-parse HEAD)" = "$WORKFLOW_SHA"'); + expect(request?.run).toContain("mktemp -d"); + + const producer = steps.find((step) => step.name === "Build or reuse the exact staging image"); + expect(producer?.env?.PRODUCER_TOKEN).toBe("${{ secrets.NEMOCLAW_IMAGE_DISPATCH_TOKEN }}"); + expect(producer?.run).toContain("build-qualification-image.yml/dispatches"); + expect(producer?.run).toContain("return_run_details:true"); + expect(producer?.run).toContain("workflow_run_id"); + expect(producer?.run).toContain('.path == ".github/workflows/build-qualification-image.yml"'); + expect(producer?.run).toContain(".display_title == $title"); + expect(producer?.run).not.toContain("actions/runs?"); + + const handoff = steps.find((step) => step.name === "Read immutable image handoff"); + expect(handoff?.run).toContain("nemoclaw-image-handoff-v1-${PRODUCER_RUN_ID}-1"); + expect(handoff?.run).toContain(".nemoclawSha == $candidate"); + expect(handoff?.run).toContain('.status == "READY"'); + expect(handoff?.run).toContain('.observedFamily == "nemoclaw-brev-staging-cpu"'); + expect(handoff?.run).toContain(".imageSelfLink =="); + expect(handoff?.run).not.toContain("validate-exact-image-manifest"); + + expect(source).toContain("brev-launchable-runtime.sh deploy"); + expect(source).toContain("brev-launchable-runtime.sh qualify"); + expect(source).toContain("brev-launchable-runtime.sh cleanup"); + expect(source).toContain("brev-cleanup-evidence.json"); + expect(source).toContain("NEMOCLAW_STAGING_LAUNCHABLE_ID"); + expect(source).not.toContain("dispatch-intent.v1.json"); + expect(source).not.toContain("controller-state.json"); + expect(source).not.toContain("qualification-evidence.v1.json"); + expect(source).not.toMatch(/npm (?:ci|install)/u); + + const cleanup = steps.find((step) => step.name === "Delete staging workspace and verify absence"); + expect(cleanup?.if).toContain("always()"); + expect(cleanup?.if).toContain("steps.deploy.outcome != 'skipped'"); + const redactIndex = steps.findIndex((step) => step.id === "redact-launchable-evidence"); + const uploadIndex = steps.findIndex( + (step) => step.name === "Upload exact-image Launchable evidence", + ); + expect(redactIndex).toBeGreaterThanOrEqual(0); + expect(uploadIndex).toBeGreaterThan(redactIndex); + expect(steps[uploadIndex]?.if).toContain("steps.redact-launchable-evidence.outcome == 'success'"); + expect(steps[uploadIndex]?.uses).toBe( + "NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57", + ); + + expect(runtime).toContain("brev create"); + expect(runtime).toContain("--launchable"); + expect(runtime).toContain("sourceImageId"); + expect(runtime).toContain("test/e2e/live/full-e2e.test.ts"); + expect(runtime).toContain("NEMOCLAW_E2E_SETUP_MODE=preinstalled-launchable"); + expect(runtime).toContain("NEMOCLAW_E2E_SECURITY_POSTURE=1"); + expect(runtime).toContain("diff --quiet --no-ext-diff HEAD --"); + expect(runtime).toContain("targetResultSha256"); + expect(runtime).not.toContain("brev-quickstart"); + expect(fullE2e).toContain('host.command("brev-quickstart"'); + expect(fullE2e).toContain('"brev-launchable-cloud-openclaw"'); + expect(fullE2e).toContain("assertFirstAgentTurn({ apiKey: hosted.apiKey, sandbox })"); +}); + +it("redacts nested Launchable evidence and rejects symlinks before upload", () => { + const workflow = readYaml(E2E_WORKFLOW_PATH); + const redact = job(workflow, "staging-brev-launchable").steps?.find( + (step) => step.id === "redact-launchable-evidence", + ); + const script = redact?.run as string; + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-launchable-redaction-")); + const secret = "nvapi-nested-secret"; + + try { + const nested = path.join(root, "brev-launchable-cloud-openclaw", "target", "raw.log"); + fs.mkdirSync(path.dirname(nested), { recursive: true }); + fs.writeFileSync(nested, `before ${secret} after\n`); + const redacted = spawnSync("bash", ["-c", script], { + encoding: "utf8", + env: { ...process.env, NVIDIA_INFERENCE_API_KEY: secret, WORK_DIR: root }, + }); + expect(redacted.status, redacted.stderr).toBe(0); + expect(fs.readFileSync(nested, "utf8")).toBe("before [REDACTED] after\n"); + + fs.symlinkSync(nested, path.join(root, "untrusted-link")); + const rejected = spawnSync("bash", ["-c", script], { + encoding: "utf8", + env: { ...process.env, NVIDIA_INFERENCE_API_KEY: secret, WORK_DIR: root }, + }); + expect(rejected.status).not.toBe(0); + expect(rejected.stderr).toContain("must not contain symlinks"); + } finally { + fs.rmSync(root, { force: true, recursive: true }); + } +}); diff --git a/test/e2e/support/exact-image-manifest-cli.test.ts b/test/e2e/support/exact-image-manifest-cli.test.ts deleted file mode 100644 index fe0ac65037..0000000000 --- a/test/e2e/support/exact-image-manifest-cli.test.ts +++ /dev/null @@ -1,158 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { spawnSync } from "node:child_process"; -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; -import { afterEach, describe, expect, it } from "vitest"; - -import { normalizedExactImageManifestJson } from "../../../tools/e2e/exact-image-manifest.mts"; -import { - CANDIDATE_SHA, - CORRELATION_ID, - exactImageManifest, - IMAGE_REPOSITORY_SHA, -} from "./exact-image-manifest-fixture.ts"; - -const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); -const CLI = path.join(REPO_ROOT, "tools/e2e/validate-exact-image-manifest.mts"); -const tempDirs: string[] = []; - -afterEach(() => { - for (const directory of tempDirs.splice(0)) { - fs.rmSync(directory, { recursive: true, force: true }); - } -}); - -function tempDir(): string { - const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-image-manifest-")); - tempDirs.push(directory); - return directory; -} - -function args(manifest: string, output: string, overrides: Record = {}): string[] { - const values = { - "--manifest": manifest, - "--output": output, - "--nemoclaw-sha": CANDIDATE_SHA, - "--requester-run-id": "8001", - "--requester-run-attempt": "1", - "--correlation-id": CORRELATION_ID, - "--image-repository-sha": IMAGE_REPOSITORY_SHA, - "--producer-run-id": "9002", - "--producer-run-attempt": "1", - ...overrides, - }; - return Object.entries(values).flat(); -} - -function runCli(cliArgs: string[]) { - return spawnSync(process.execPath, ["--experimental-strip-types", CLI, ...cliArgs], { - cwd: REPO_ROOT, - encoding: "utf8", - env: { ...process.env, NODE_NO_WARNINGS: "1" }, - }); -} - -function runCliWithTsx(cliArgs: string[]) { - return spawnSync(process.execPath, ["--import", "tsx", CLI, ...cliArgs], { - cwd: REPO_ROOT, - encoding: "utf8", - env: { ...process.env, NODE_NO_WARNINGS: "1" }, - }); -} - -describe("exact staging image manifest CLI", () => { - it("writes only normalized accepted JSON with private permissions", () => { - const directory = tempDir(); - const input = path.join(directory, "nemoclaw-image-manifest.v1.json"); - const output = path.join(directory, "accepted.json"); - const manifest = exactImageManifest(); - fs.writeFileSync(input, JSON.stringify(manifest), "utf8"); - - const result = runCli(args(input, output)); - - expect(result.status).toBe(0); - expect(result.stderr).toBe(""); - expect(fs.readFileSync(output, "utf8")).toBe(normalizedExactImageManifestJson(manifest)); - expect(fs.statSync(output).mode & 0o777).toBe(0o600); - }); - - it("loads through the repository tsx runtime", () => { - const directory = tempDir(); - const input = path.join(directory, "nemoclaw-image-manifest.v1.json"); - const output = path.join(directory, "accepted.json"); - fs.writeFileSync(input, JSON.stringify(exactImageManifest()), "utf8"); - - const result = runCliWithTsx(args(input, output)); - - expect(result.status).toBe(0); - expect(result.stderr).toBe(""); - expect(fs.existsSync(output)).toBe(true); - }); - - it("reports a stable provenance failure code and leaves no accepted output", () => { - const directory = tempDir(); - const input = path.join(directory, "manifest.json"); - const output = path.join(directory, "accepted.json"); - fs.writeFileSync(input, JSON.stringify(exactImageManifest()), "utf8"); - - const result = runCli(args(input, output, { "--producer-run-id": "9003" })); - - expect(result.status).toBe(1); - expect(result.stderr).toMatch(/^PROVENANCE_MISMATCH: workflowRunId/u); - expect(fs.existsSync(output)).toBe(false); - }); - - it("rejects symlinked, oversized, and non-UTF-8 inputs", () => { - const directory = tempDir(); - const valid = path.join(directory, "valid.json"); - fs.writeFileSync(valid, JSON.stringify(exactImageManifest()), "utf8"); - const symlink = path.join(directory, "symlink.json"); - fs.symlinkSync(valid, symlink); - - const oversized = path.join(directory, "oversized.json"); - fs.writeFileSync(oversized, Buffer.alloc(64 * 1024 + 1, 0x20)); - - const invalidUtf8 = path.join(directory, "invalid-utf8.json"); - fs.writeFileSync(invalidUtf8, Buffer.from([0xc3, 0x28])); - - for (const [name, input, message] of [ - ["symlink", symlink, "manifest input could not be opened safely"], - ["oversized", oversized, "manifest input exceeds 65536 bytes"], - ["non-UTF-8", invalidUtf8, "manifest input must be valid UTF-8"], - ]) { - const output = path.join(directory, `${name}-accepted.json`); - const result = runCli(args(input, output)); - expect(result.status, name).toBe(1); - expect(result.stderr, name).toContain(`ARTIFACT_MISSING_OR_INVALID: ${message}`); - expect(fs.existsSync(output), name).toBe(false); - } - }); - - it("refuses to replace a symlinked accepted-output path", () => { - const directory = tempDir(); - const input = path.join(directory, "manifest.json"); - const target = path.join(directory, "target.json"); - const output = path.join(directory, "accepted.json"); - fs.writeFileSync(input, JSON.stringify(exactImageManifest()), "utf8"); - fs.writeFileSync(target, "unchanged\n", "utf8"); - fs.symlinkSync(target, output); - - const result = runCli(args(input, output)); - - expect(result.status).toBe(1); - expect(result.stderr).toBe( - "OUTPUT_WRITE_FAILED: accepted manifest output could not be written safely\n", - ); - expect(fs.readFileSync(target, "utf8")).toBe("unchanged\n"); - }); - - it("reports invalid or incomplete invocation as a request failure", () => { - const result = runCli(["--manifest", "manifest.json"]); - - expect(result.status).toBe(1); - expect(result.stderr).toBe("REQUEST_INVALID: --output is required\n"); - }); -}); diff --git a/test/e2e/support/exact-image-manifest-fixture.ts b/test/e2e/support/exact-image-manifest-fixture.ts deleted file mode 100644 index 5c6ab66f8e..0000000000 --- a/test/e2e/support/exact-image-manifest-fixture.ts +++ /dev/null @@ -1,61 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import type { - ExactImageManifest, - ExactImageManifestExpectations, -} from "../../../tools/e2e/exact-image-manifest.mts"; - -export const CANDIDATE_SHA = "a".repeat(40); -export const IMAGE_REPOSITORY_SHA = "b".repeat(40); -export const CORRELATION_ID = "12345678-1234-4123-8123-123456789abc"; - -export function exactImageManifest( - overrides: Partial = {}, -): ExactImageManifest { - return { - schemaVersion: 1, - kind: "nemoclaw-exact-image-manifest", - correlationId: CORRELATION_ID, - requesterRepository: "NVIDIA/NemoClaw", - requesterWorkflowRunId: "8001", - requesterWorkflowRunAttempt: 1, - nemoclawSha: CANDIDATE_SHA, - imageRepository: "brevdev/nemoclaw-image", - imageRepositorySha: IMAGE_REPOSITORY_SHA, - producerWorkflow: ".github/workflows/build-qualification-image.yml", - workflowRunId: "9002", - workflowRunAttempt: 1, - imageOriginWorkflowRunId: "9002", - imageOriginWorkflowRunAttempt: 1, - imageKind: "compute#image", - project: "brevdevprod", - imageName: "nemoclaw-brev-cpu-v0-1-0-20260716-a-staging-190-1", - imageId: "12345678901234567890", - imageSelfLink: - "https://www.googleapis.com/compute/v1/projects/brevdevprod/global/images/nemoclaw-brev-cpu-v0-1-0-20260716-a-staging-190-1", - status: "READY", - imageCreationTimestamp: "2026-07-16T12:00:00.000Z", - manifestCreatedAt: "2026-07-16T12:01:00.000Z", - channel: "staging", - variant: "cpu", - observedFamily: "nemoclaw-brev-staging-cpu", - result: "built", - ...overrides, - }; -} - -export function exactImageManifestExpectations( - overrides: Partial = {}, -): ExactImageManifestExpectations { - return { - correlationId: CORRELATION_ID, - requesterWorkflowRunId: "8001", - requesterWorkflowRunAttempt: 1, - nemoclawSha: CANDIDATE_SHA, - imageRepositorySha: IMAGE_REPOSITORY_SHA, - workflowRunId: "9002", - workflowRunAttempt: 1, - ...overrides, - }; -} diff --git a/test/e2e/support/exact-image-manifest.test.ts b/test/e2e/support/exact-image-manifest.test.ts deleted file mode 100644 index 1b784bba7a..0000000000 --- a/test/e2e/support/exact-image-manifest.test.ts +++ /dev/null @@ -1,253 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { describe, expect, it } from "vitest"; - -import { - EXACT_IMAGE_MANIFEST_MAX_CLOCK_SKEW_MS, - ExactImageManifestError, - normalizedExactImageManifestJson, - parseAndValidateExactImageManifest, - validateExactImageManifest, -} from "../../../tools/e2e/exact-image-manifest.mts"; -import { - CANDIDATE_SHA, - CORRELATION_ID, - exactImageManifest, - exactImageManifestExpectations, - IMAGE_REPOSITORY_SHA, -} from "./exact-image-manifest-fixture.ts"; - -function expectCode(run: () => unknown, code: ExactImageManifestError["code"]): void { - try { - run(); - } catch (error) { - expect(error).toBeInstanceOf(ExactImageManifestError); - expect((error as ExactImageManifestError).code).toBe(code); - return; - } - throw new Error(`expected ${code}`); -} - -describe("exact staging image manifest consumer", () => { - it("accepts and normalizes one exact built CPU image", () => { - const source = exactImageManifest(); - const accepted = validateExactImageManifest(source, exactImageManifestExpectations()); - - expect(accepted).toEqual(source); - expect(accepted.imageId).toBe("12345678901234567890"); - expect(normalizedExactImageManifestJson(accepted)).toBe(`${JSON.stringify(source, null, 2)}\n`); - }); - - it("accepts a reused image with required origin evidence at the 24-hour boundary", () => { - const reused = exactImageManifest({ - result: "reused", - imageOriginWorkflowRunId: "7000", - imageOriginWorkflowRunAttempt: 3, - imageCreationTimestamp: "2026-07-15T12:01:00.000Z", - }); - - expect(validateExactImageManifest(reused, exactImageManifestExpectations())).toEqual(reused); - }); - - it("requires reused manifests to carry both origin fields", () => { - for (const field of ["imageOriginWorkflowRunId", "imageOriginWorkflowRunAttempt"] as const) { - const reused = { ...exactImageManifest({ result: "reused" }) } as Record; - delete reused[field]; - expectCode( - () => validateExactImageManifest(reused, exactImageManifestExpectations()), - "ARTIFACT_MISSING_OR_INVALID", - ); - } - }); - - it("rejects missing and additional fields", () => { - const missing = { ...exactImageManifest() } as Record; - delete missing.imageId; - expectCode( - () => validateExactImageManifest(missing, exactImageManifestExpectations()), - "ARTIFACT_MISSING_OR_INVALID", - ); - - const additional = { ...exactImageManifest(), mutableImageFamilyFallback: true }; - expectCode( - () => validateExactImageManifest(additional, exactImageManifestExpectations()), - "ARTIFACT_MISSING_OR_INVALID", - ); - }); - - it.each([ - ["uppercase candidate SHA", { nemoclawSha: CANDIDATE_SHA.toUpperCase() }], - ["short image repository SHA", { imageRepositorySha: IMAGE_REPOSITORY_SHA.slice(0, 12) }], - ["uppercase correlation UUID", { correlationId: CORRELATION_ID.toUpperCase() }], - ["zero requester run ID", { requesterWorkflowRunId: "0" }], - ["numeric image ID", { imageId: 123456789 }], - ["zero image ID", { imageId: "0" }], - ["fractional workflow attempt", { workflowRunAttempt: 1.5 }], - ])("rejects an invalid %s", (_name, overrides) => { - expectCode( - () => - validateExactImageManifest( - { ...exactImageManifest(), ...overrides }, - exactImageManifestExpectations(), - ), - "ARTIFACT_MISSING_OR_INVALID", - ); - }); - - it.each([ - ["requester repository", { requesterRepository: "somewhere/NemoClaw" }], - ["image repository", { imageRepository: "brevdev/other-image" }], - ["producer workflow", { producerWorkflow: ".github/workflows/build-image.yml" }], - ["image kind", { imageKind: "compute#family" }], - ["status", { status: "PENDING" }], - ["channel", { channel: "production" }], - ["observed family", { observedFamily: "nemoclaw-brev-production-cpu" }], - ])("rejects the wrong fixed %s", (_name, overrides) => { - expectCode( - () => - validateExactImageManifest( - { ...exactImageManifest(), ...overrides }, - exactImageManifestExpectations(), - ), - "PROVENANCE_MISMATCH", - ); - }); - - it("rejects GPU before any later dispatch can consume it", () => { - expectCode( - () => - validateExactImageManifest( - { - ...exactImageManifest(), - variant: "gpu", - observedFamily: "nemoclaw-brev-staging-gpu", - }, - exactImageManifestExpectations(), - ), - "UNSUPPORTED_VARIANT", - ); - }); - - it("requires an immutable self-link reconstructed from project and image name", () => { - const canonical = exactImageManifest().imageSelfLink; - for (const imageSelfLink of [ - "https://www.googleapis.com/compute/v1/projects/brevdevprod/global/images/family/nemoclaw-brev-staging-cpu", - "https://www.googleapis.com/compute/v1/projects/other/global/images/nemoclaw-brev-cpu-v0-1-0-20260716-a-staging-190-1", - `${canonical}?alt=json`, - `${canonical}#fragment`, - `${canonical}\n`, - ]) { - expectCode( - () => - validateExactImageManifest( - exactImageManifest({ imageSelfLink }), - exactImageManifestExpectations(), - ), - "IMAGE_IDENTITY_MISMATCH", - ); - } - }); - - it("rejects noncanonical GCP project identifiers even when the self-link repeats them", () => { - for (const project of ["short", "-leading", "trailing-", "UPPERCASE", "evil?x=y#z\nproject"]) { - const imageName = exactImageManifest().imageName; - expectCode( - () => - validateExactImageManifest( - exactImageManifest({ - project, - imageSelfLink: `https://www.googleapis.com/compute/v1/projects/${project}/global/images/${imageName}`, - }), - exactImageManifestExpectations(), - ), - "IMAGE_IDENTITY_MISMATCH", - ); - } - }); - - it("binds the manifest to the trusted request and producer run", () => { - const expectationMismatches = [ - { nemoclawSha: "c".repeat(40) }, - { requesterWorkflowRunId: "8002" }, - { requesterWorkflowRunAttempt: 2 }, - { correlationId: "87654321-4321-4321-8321-cba987654321" }, - { imageRepositorySha: "d".repeat(40) }, - { workflowRunId: "9003" }, - { workflowRunAttempt: 2 }, - ]; - for (const expected of expectationMismatches) { - expectCode( - () => - validateExactImageManifest( - exactImageManifest(), - exactImageManifestExpectations(expected), - ), - "PROVENANCE_MISMATCH", - ); - } - }); - - it("requires built images to originate in the current run", () => { - expectCode( - () => - validateExactImageManifest( - exactImageManifest({ imageOriginWorkflowRunId: "7000" }), - exactImageManifestExpectations(), - ), - "PROVENANCE_MISMATCH", - ); - }); - - it("enforces real timestamps, bounded clock skew, and the reused-image age", () => { - expectCode( - () => - validateExactImageManifest( - exactImageManifest({ imageCreationTimestamp: "2026-02-30T12:00:00Z" }), - exactImageManifestExpectations(), - ), - "ARTIFACT_MISSING_OR_INVALID", - ); - - const beyondSkew = new Date( - Date.parse("2026-07-16T12:01:00.000Z") + EXACT_IMAGE_MANIFEST_MAX_CLOCK_SKEW_MS + 1, - ).toISOString(); - const atSkew = new Date( - Date.parse("2026-07-16T12:01:00.000Z") + EXACT_IMAGE_MANIFEST_MAX_CLOCK_SKEW_MS, - ).toISOString(); - expect( - validateExactImageManifest( - exactImageManifest({ imageCreationTimestamp: atSkew }), - exactImageManifestExpectations(), - ).imageCreationTimestamp, - ).toBe(atSkew); - expectCode( - () => - validateExactImageManifest( - exactImageManifest({ imageCreationTimestamp: beyondSkew }), - exactImageManifestExpectations(), - ), - "PROVENANCE_MISMATCH", - ); - - expectCode( - () => - validateExactImageManifest( - exactImageManifest({ - result: "reused", - imageOriginWorkflowRunId: "7000", - imageCreationTimestamp: "2026-07-15T12:00:59.999Z", - }), - exactImageManifestExpectations(), - ), - "PROVENANCE_MISMATCH", - ); - }); - - it("rejects invalid JSON before semantic validation", () => { - expectCode( - () => parseAndValidateExactImageManifest("{not-json", exactImageManifestExpectations()), - "ARTIFACT_MISSING_OR_INVALID", - ); - }); -}); diff --git a/test/e2e/support/exact-image-qualification-workflow.test.ts b/test/e2e/support/exact-image-qualification-workflow.test.ts deleted file mode 100644 index 4c691c99bf..0000000000 --- a/test/e2e/support/exact-image-qualification-workflow.test.ts +++ /dev/null @@ -1,258 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { spawnSync } from "node:child_process"; -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; - -import { expect, it } from "vitest"; - -import { - readRepoText, - readYaml, - type Workflow, - type WorkflowJob, -} from "../../helpers/e2e-workflow-contract"; - -const WORKFLOW_PATH = ".github/workflows/brev-launchable-qualification.yaml"; -const E2E_WORKFLOW_PATH = ".github/workflows/e2e.yaml"; -const CONTROLLER_PATH = "tools/e2e/exact-image-qualification-controller.mts"; -const RUNTIME_PATH = "tools/e2e/brev-launchable-runtime.sh"; -const FULL_E2E_PATH = "test/e2e/live/full-e2e.test.ts"; -const ACTIVATION_VARIABLE = "NEMOCLAW_BREV_LAUNCHABLE_QUALIFICATION_ENABLED"; - -type QualificationWorkflow = Workflow & { - name: string; - on: Record; - permissions: Record; - concurrency: { group: string; "cancel-in-progress": boolean }; -}; - -function strings(value: unknown): string[] { - return typeof value === "string" - ? [value] - : Array.isArray(value) - ? value.flatMap(strings) - : value && typeof value === "object" - ? Object.values(value).flatMap(strings) - : []; -} - -function job(workflow: Workflow, name: string): WorkflowJob { - const value = workflow.jobs[name]; - expect(value, `missing ${name} job`).toBeDefined(); - return value!; -} - -// source-shape-contract: security -- Exact-image qualification must remain manual/reusable, protected, fixed-target, least-privilege, identity-gated, and cleanup-verifying -it("keeps exact-image Launchable qualification protected, reusable, and fail-closed", () => { - const workflow = readYaml(WORKFLOW_PATH); - const e2eWorkflow = readYaml(E2E_WORKFLOW_PATH); - const source = readRepoText(WORKFLOW_PATH); - const controller = readRepoText(CONTROLLER_PATH); - const runtime = readRepoText(RUNTIME_PATH); - const fullE2e = readRepoText(FULL_E2E_PATH); - const preflight = job(workflow, "preflight"); - const qualify = job(workflow, "qualify"); - const caller = job(e2eWorkflow, "staging-brev-launchable"); - const workflowStrings = strings(workflow); - const steps = qualify.steps ?? []; - const redactIndex = steps.findIndex((step) => step.name === "Redact runtime evidence"); - const uploadIndex = steps.findIndex( - (step) => step.name === "Upload exact staging Launchable evidence", - ); - - expect(redactIndex).toBeGreaterThanOrEqual(0); - expect(uploadIndex).toBeGreaterThan(redactIndex); - const redact = steps[redactIndex]!; - const upload = steps[uploadIndex]!; - expect(redact.if).toBe("${{ always() && steps.workspace.outputs.work_dir != '' }}"); - expect(redact.env).toEqual({ - NVIDIA_INFERENCE_API_KEY: "${{ secrets.NVIDIA_INFERENCE_API_KEY }}", - WORK_DIR: "${{ steps.workspace.outputs.work_dir }}", - }); - expect(redact.run).toContain('content.replace(secret, b"[REDACTED]")'); - expect(redact.run).toContain("os.scandir(directory)"); - expect(redact.run).toContain("child.is_symlink()"); - expect(redact.run).toContain("MAX_TOTAL_BYTES"); - expect(upload.if).toBe( - "${{ always() && steps.workspace.outputs.work_dir != '' && steps.redact-runtime-evidence.outcome == 'success' }}", - ); - for (const path of ["brev-launchable-e2e.log", "brev-launchable-cloud-openclaw"]) { - expect(String(upload.with?.path), `retained evidence must include ${path}`).toContain( - `/${path}`, - ); - } - - expect(workflow.name).toBe("E2E / Exact Staging Brev Launchable"); - expect(Object.keys(workflow.on)).toEqual(["workflow_dispatch", "workflow_call"]); - expect(source).not.toMatch(/^\s+(?:push|schedule|workflow_run|pull_request):/mu); - expect(Object.keys((workflow.on.workflow_dispatch as { inputs: object }).inputs)).toEqual([ - "candidate_sha", - "reason", - ]); - expect(Object.keys((workflow.on.workflow_call as { inputs: object }).inputs)).toEqual([ - "candidate_sha", - "reason", - ]); - expect((workflow.on.workflow_call as { secrets?: object }).secrets).toBeUndefined(); - expect(JSON.stringify(workflow.on)).not.toContain(ACTIVATION_VARIABLE); - expect(workflow.permissions).toEqual({}); - expect(caller.secrets).toBeUndefined(); - expect(caller.if).toBe( - `\${{ vars.${ACTIVATION_VARIABLE} == 'true' && github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && inputs.checkout_sha == '' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') }}`, - ); - expect(caller.with?.candidate_sha).toBe("${{ github.sha }}"); - expect(workflow.concurrency).toEqual({ - group: "brev-launchable-qualification-staging-cpu", - "cancel-in-progress": false, - }); - - expect(preflight.permissions).toEqual({ contents: "read" }); - expect(preflight.if).toBeUndefined(); - expect(preflight.environment).toBeUndefined(); - const activation = preflight.steps?.[0]; - expect(activation?.name).toBe("Require explicit repository activation"); - expect(activation?.uses).toBeUndefined(); - expect(activation?.env).toEqual({ - QUALIFICATION_ENABLED: `\${{ vars.${ACTIVATION_VARIABLE} }}`, - }); - expect(activation?.run).toContain('[[ "$QUALIFICATION_ENABLED" != "true" ]]'); - expect(activation?.run).toContain(`${ACTIVATION_VARIABLE}=true`); - expect(activation?.run).toContain("exit 1"); - expect(qualify.permissions).toEqual({ contents: "read" }); - expect(qualify.if).toBe( - `\${{ vars.${ACTIVATION_VARIABLE} == 'true' && github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' }}`, - ); - expect(qualify.environment).toEqual({ - name: "approve-brev-launchable-qualification", - deployment: false, - }); - const environmentJobs = Object.values(workflow.jobs).filter( - (workflowJob) => workflowJob.environment !== undefined, - ); - expect(environmentJobs).toEqual([qualify]); - for (const environmentJob of environmentJobs) { - expect(environmentJob.if).toContain(`vars.${ACTIVATION_VARIABLE} == 'true'`); - } - for (const secret of [ - "NEMOCLAW_IMAGE_DISPATCH_TOKEN", - "BREV_API_KEY", - "BREV_ORG_ID", - "NVIDIA_INFERENCE_API_KEY", - ]) { - expect(JSON.stringify(preflight)).not.toContain(`secrets.${secret}`); - expect(JSON.stringify(qualify)).toContain(`secrets.${secret}`); - } - expect(JSON.stringify(qualify)).toContain("NEMOCLAW_IMAGE_QUALIFICATION_TOKEN"); - expect(source.match(/secrets\.NEMOCLAW_IMAGE_DISPATCH_TOKEN/gu)).toHaveLength(4); - expect(workflowStrings).not.toContain("id-token: write"); - expect(source).not.toMatch(/npm (?:ci|install)/u); - for (const step of [...(preflight.steps ?? []), ...(qualify.steps ?? [])]) { - expect(step.run ?? "").not.toContain("${{ inputs."); - } - - const actionUses = workflowStrings.filter((value) => value.startsWith("actions/")); - expect(actionUses.length).toBeGreaterThan(0); - for (const use of actionUses) expect(use).toMatch(/^actions\/[a-z-]+@[0-9a-f]{40}$/u); - for (const checkout of qualify.steps?.filter((step) => - step.uses?.startsWith("actions/checkout@"), - ) ?? []) { - expect(checkout.with).toMatchObject({ - ref: "${{ github.workflow_sha }}", - "persist-credentials": false, - }); - } - - expect(controller).toContain('export const PRODUCER_REPOSITORY = "brevdev/nemoclaw-image"'); - expect(controller).toContain( - 'export const PRODUCER_WORKFLOW_FILE = "build-qualification-image.yml"', - ); - expect(controller).toContain('export const PRODUCER_REF = "main"'); - expect(controller).toContain('request.ref !== "refs/heads/main"'); - expect(controller).toContain("request.candidateSha !== request.workflowSha"); - expect(controller).toContain('export const GITHUB_API_VERSION = "2026-03-10"'); - expect(controller).toContain("return_run_details: true"); - expect(controller).toContain("fs.renameSync(temporary, file)"); - expect(controller).not.toMatch(/actions\/runs\?/u); - expect(controller).toContain("actions/workflows/${PRODUCER_WORKFLOW_FILE}/runs"); - - const validate = qualify.steps?.find((step) => step.name === "Validate the exact image manifest"); - expect(validate?.run).toContain("tools/e2e/validate-exact-image-manifest.mts"); - for (const flag of [ - "--nemoclaw-sha", - "--requester-run-id", - "--requester-run-attempt", - "--correlation-id", - "--image-repository-sha", - "--producer-run-id", - "--producer-run-attempt", - ]) { - expect(validate?.run).toContain(flag); - } - - expect(source).toContain("retention-days: 90"); - expect(source).toContain("if-no-files-found: error"); - expect(source).toContain("dispatch-intent.v1.json"); - expect(source).toContain("dispatch-reconciliation.v1.json"); - expect(source).toContain("controller-state.corrupt-*.json"); - expect(source).toContain("--mode finalize"); - expect(runtime).toContain("brev create"); - expect(runtime).toContain("--launchable"); - expect(runtime).toContain("test/e2e/live/full-e2e.test.ts"); - expect(runtime).toContain("NEMOCLAW_E2E_SETUP_MODE=preinstalled-launchable"); - expect(runtime).toContain("NEMOCLAW_E2E_SECURITY_POSTURE=1"); - expect(runtime).toContain("diff --quiet --no-ext-diff HEAD --"); - expect(runtime).toContain("validate_copied_artifact_tree"); - expect(runtime).toContain("targetResultSha256"); - expect(runtime).toContain("firstAgentTurn"); - expect(runtime).not.toContain("brev-quickstart"); - expect(fullE2e).toContain('host.command("brev-quickstart"'); - expect(fullE2e).toContain('"brev-launchable-cloud-openclaw"'); - expect(fullE2e).toContain("assertFirstAgentTurn({ apiKey: hosted.apiKey, sandbox })"); - expect(fullE2e).toMatch( - /expect\(assistantReply,[\s\S]{0,200}\)\.toBe\(\s*EXPECTED_FIRST_REPLY,\s*\);/u, - ); - expect(fullE2e).toContain("firstAgentTurn:"); - expect(source).not.toContain("test/e2e/live/exact-staging-launchable.test.ts"); - expect(source).toContain("brev-launchable-runtime.sh deploy"); - expect(source).toContain("brev-launchable-runtime.sh qualify"); - expect(source).toContain("brev-launchable-runtime.sh cleanup"); - expect(source).toContain("brev-launchable-cloud-openclaw/"); - expect(source).toContain("brev-cleanup-evidence.json"); - expect(source).toContain("NEMOCLAW_STAGING_LAUNCHABLE_ID"); - expect(source).not.toContain("image_family"); -}); - -it("recursively redacts nested runtime evidence and rejects symlinks before upload", () => { - const workflow = readYaml(WORKFLOW_PATH); - const redact = job(workflow, "qualify").steps?.find( - (step) => step.id === "redact-runtime-evidence", - ); - const script = redact?.run as string; - const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-evidence-redaction-")); - const secret = "nvapi-nested-secret"; - - try { - const nested = path.join(root, "brev-launchable-cloud-openclaw", "target", "raw.log"); - fs.mkdirSync(path.dirname(nested), { recursive: true }); - fs.writeFileSync(nested, `before ${secret} after\n`); - const redacted = spawnSync("bash", ["-c", script], { - encoding: "utf8", - env: { ...process.env, NVIDIA_INFERENCE_API_KEY: secret, WORK_DIR: root }, - }); - expect(redacted.status, redacted.stderr).toBe(0); - expect(fs.readFileSync(nested, "utf8")).toBe("before [REDACTED] after\n"); - - fs.symlinkSync(nested, path.join(root, "untrusted-link")); - const rejected = spawnSync("bash", ["-c", script], { - encoding: "utf8", - env: { ...process.env, NVIDIA_INFERENCE_API_KEY: secret, WORK_DIR: root }, - }); - expect(rejected.status).not.toBe(0); - expect(rejected.stderr).toContain("must not contain symlinks"); - } finally { - fs.rmSync(root, { force: true, recursive: true }); - } -}); diff --git a/test/exact-image-qualification-controller.test.ts b/test/exact-image-qualification-controller.test.ts deleted file mode 100644 index d8a7bb01ad..0000000000 --- a/test/exact-image-qualification-controller.test.ts +++ /dev/null @@ -1,1494 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { spawnSync } from "node:child_process"; -import { createHash } from "node:crypto"; -import fs from "node:fs"; -import path from "node:path"; - -import { afterEach, describe, expect, it, vi } from "vitest"; - -import { githubApi } from "../tools/advisors/github.mts"; -import { - ARCHIVE_FILE, - cancelActiveExactImageQualification, - DISPATCH_INTENT_FILE, - DISPATCH_RECONCILIATION_FILE, - downloadExactImageManifest, - EVIDENCE_FILE, - extractExactManifestArchive, - finalizeExactImageQualification, - GITHUB_API_VERSION, - MANIFEST_ARTIFACT_FILE, - PRODUCER_REPOSITORY, - PRODUCER_WORKFLOW_FILE, - PRODUCER_WORKFLOW_PATH, - parseExactImageQualificationCommand, - preflightExactImageQualification, - type QualificationDependencies, - readExactImageQualificationState, - STATE_FILE, - startExactImageQualification, - VALIDATED_MANIFEST_FILE, - validateExactImageQualificationRequest, - validateQualificationArtifactList, - validateQualificationWorkflowRun, - validateWorkflowDispatchDetails, - waitForExactImageQualification, -} from "../tools/e2e/exact-image-qualification-controller.mts"; -import { - API_RUN_URL, - artifactList, - BASE_TIME, - CANDIDATE_SHA, - CORRELATION_ID, - createApi, - createArchiveRunCommand, - createRoutedApi, - dependencies, - dispatchDetails, - dispatchIntent, - HTML_RUN_URL, - PRODUCER_SHA, - qualificationApiRoute, - REQUEST, - RUN_ID, - startedState, - tempDirectory, - WORKFLOW_SHA, - WORKFLOW_ID, - workflowRun, -} from "./helpers/exact-image-qualification-controller-fixture.ts"; - -afterEach(() => { - vi.restoreAllMocks(); - vi.unstubAllGlobals(); -}); - -describe("exact image qualification request", () => { - it("accepts the current trusted main candidate while rejecting malformed boundaries", () => { - expect(validateExactImageQualificationRequest(REQUEST)).toEqual(REQUEST); - expect( - validateExactImageQualificationRequest({ - ...REQUEST, - candidateSha: "c".repeat(40), - eventName: "schedule", - requesterRunAttempt: 2, - }), - ).toMatchObject({ - candidateSha: "c".repeat(40), - eventName: "schedule", - requesterRunAttempt: 2, - }); - expect(() => - validateExactImageQualificationRequest({ ...REQUEST, eventName: "pull_request" }), - ).toThrow(/workflow_dispatch or schedule/u); - expect(() => - validateExactImageQualificationRequest({ ...REQUEST, ref: "refs/pull/1/merge" }), - ).toThrow(/trusted main branch/u); - expect(() => - validateExactImageQualificationRequest({ ...REQUEST, ref: "refs/heads/feature" }), - ).toThrow(/trusted main branch/u); - expect(() => - validateExactImageQualificationRequest({ ...REQUEST, candidateSha: "d".repeat(40) }), - ).toThrow(/must equal the trusted main workflow SHA/u); - expect(() => validateExactImageQualificationRequest({ ...REQUEST, reason: " bad " })).toThrow( - /reason/u, - ); - }); - - it("rejects non-main requests before authorization API access", async () => { - const api = vi.fn(); - - await expect( - preflightExactImageQualification({ ...REQUEST, ref: "refs/heads/feature" }, "core-token", { - api: api as QualificationDependencies["api"], - }), - ).rejects.toMatchObject({ code: "REQUEST_INVALID" }); - expect(api).not.toHaveBeenCalled(); - }); - - it("rejects a candidate other than the trusted workflow SHA before authorization API access", async () => { - const api = vi.fn(); - - await expect( - preflightExactImageQualification({ ...REQUEST, candidateSha: "d".repeat(40) }, "core-token", { - api: api as QualificationDependencies["api"], - }), - ).rejects.toMatchObject({ code: "REQUEST_INVALID" }); - expect(api).not.toHaveBeenCalled(); - }); - - it("parses the fixed CLI surface and rejects undeclared controls", () => { - expect( - parseExactImageQualificationCommand([ - "--mode", - "preflight", - "--actor", - REQUEST.actor, - "--candidate-sha", - CANDIDATE_SHA, - "--event-name", - "workflow_dispatch", - "--reason", - REQUEST.reason, - "--ref", - "refs/heads/main", - "--requester-run-attempt", - "2", - "--requester-run-id", - REQUEST.requesterRunId, - "--workflow-sha", - WORKFLOW_SHA, - ]), - ).toEqual({ mode: "preflight", request: REQUEST }); - expect(() => - parseExactImageQualificationCommand([ - "--mode", - "wait", - "--work-dir", - "/tmp/work", - "--producer-ref", - "feature", - ]), - ).toThrow(/unknown argument/u); - }); -}); - -describe("exact producer dispatch binding", () => { - it("binds the selected candidate to the trusted main workflow SHA", async () => { - expect(REQUEST.candidateSha).toBe(REQUEST.workflowSha); - const { state, workDir } = await startedState(); - try { - expect(readExactImageQualificationState(workDir).request).toMatchObject({ - candidateSha: CANDIDATE_SHA, - workflowSha: WORKFLOW_SHA, - }); - await expect( - waitForExactImageQualification( - { workDir, producerToken: "producer-token" }, - dependencies( - createApi({ run: workflowRun({ status: "completed", conclusion: "success" }) }), - { now: () => BASE_TIME + 1_000 }, - ), - ), - ).resolves.toMatchObject({ id: state.producer.runId, conclusion: "success" }); - } finally { - fs.rmSync(workDir, { recursive: true, force: true }); - } - }); - - it("accepts GitHub's maintain role mapping but rejects ordinary write access", async () => { - const accepted = await startedState(createApi({ permission: "write", roleName: "maintain" })); - fs.rmSync(accepted.workDir, { recursive: true, force: true }); - - const workDir = tempDirectory(); - try { - await expect( - startExactImageQualification( - { - request: REQUEST, - coreToken: "core-token", - producerToken: "producer-token", - workDir, - }, - dependencies(createApi({ permission: "write", roleName: "write" })), - ), - ).rejects.toMatchObject({ code: "DISPATCH_FORBIDDEN" }); - } finally { - fs.rmSync(workDir, { recursive: true, force: true }); - } - }); - - it("uses the 2026 API contract and binds the returned run without listing runs", async () => { - const api = createApi(); - const { state, workDir } = await startedState(api); - try { - const dispatchCall = api.mock.calls.find(([apiPath]) => - String(apiPath).endsWith(`/workflows/${PRODUCER_WORKFLOW_FILE}/dispatches`), - ); - expect(dispatchCall?.[2]).toMatchObject({ - method: "POST", - apiVersion: GITHUB_API_VERSION, - expectedStatus: 200, - body: { - ref: "main", - inputs: { - nemoclaw_sha: CANDIDATE_SHA, - correlation_id: CORRELATION_ID, - requester_workflow_run_id: REQUEST.requesterRunId, - requester_workflow_run_attempt: "2", - }, - return_run_details: true, - }, - signal: expect.any(AbortSignal), - }); - expect(api.mock.calls.some(([apiPath]) => /actions\/runs\?/u.test(String(apiPath)))).toBe( - false, - ); - expect(state.producer.runId).toBe(RUN_ID); - expect(state.producer.repositorySha).toBe(PRODUCER_SHA); - expect(readExactImageQualificationState(workDir)).toEqual(state); - } finally { - fs.rmSync(workDir, { recursive: true, force: true }); - } - }); - - it("keeps the previous state intact when atomic replacement is interrupted", async () => { - const workDir = tempDirectory(); - const previousState = "previous state evidence\n"; - fs.writeFileSync(path.join(workDir, STATE_FILE), previousState, { mode: 0o600 }); - vi.spyOn(fs, "renameSync").mockImplementation(() => { - throw new Error("simulated interruption before atomic replacement"); - }); - try { - await expect( - startExactImageQualification( - { - request: REQUEST, - coreToken: "core-token", - producerToken: "producer-token", - workDir, - }, - dependencies(createApi()), - ), - ).rejects.toMatchObject({ code: "OUTPUT_WRITE_FAILED" }); - expect(fs.readFileSync(path.join(workDir, STATE_FILE), "utf8")).toBe(previousState); - expect(fs.readdirSync(workDir).filter((name) => name.endsWith(".tmp"))).toEqual([]); - } finally { - fs.rmSync(workDir, { recursive: true, force: true }); - } - }); - - it("requires HTTP 200 and sends the explicit API-version header", async () => { - const fetchMock = vi.fn( - async (_input: string | URL | Request, _init?: RequestInit) => - new Response(null, { status: 204 }), - ); - vi.stubGlobal("fetch", fetchMock); - await expect( - githubApi("repos/example/actions/workflows/build.yml/dispatches", "token", { - method: "POST", - apiVersion: GITHUB_API_VERSION, - expectedStatus: 200, - body: { return_run_details: true }, - }), - ).rejects.toThrow(/204/u); - const headers = new Headers(fetchMock.mock.calls[0]?.[1]?.headers); - expect(headers.get("X-GitHub-Api-Version")).toBe(GITHUB_API_VERSION); - }); - - it.each([ - null, - {}, - { workflow_run_id: Number(RUN_ID) }, - ])("fails closed when dispatch returns no complete run details: %j", async (dispatch) => { - const workDir = tempDirectory(); - try { - const api = createApi({ dispatch }); - let clock = BASE_TIME; - await expect( - startExactImageQualification( - { - request: REQUEST, - coreToken: "core-token", - producerToken: "producer-token", - workDir, - }, - dependencies(api, { - now: () => clock, - sleep: async () => { - clock += 2; - }, - limits: { - dispatchReconciliationTimeoutMs: 1, - reconciliationPollIntervalMs: 1, - }, - }), - ), - ).rejects.toMatchObject({ code: "DISPATCH_AMBIGUOUS" }); - } finally { - fs.rmSync(workDir, { recursive: true, force: true }); - } - }); - - it("records, reconciles, cancels, and rejects a server-accepted dispatch whose response is lost", async () => { - const workDir = tempDirectory(); - const base = createApi(); - let cancelObserved = false; - let stateExistedAtCancel = false; - const api = createRoutedApi(base, [ - qualificationApiRoute.dispatch(() => { - throw new Error("response connection reset after server acceptance"); - }), - qualificationApiRoute.workflowRuns(() => ({ - total_count: 1, - workflow_runs: [workflowRun()], - })), - qualificationApiRoute.cancelRun(() => { - stateExistedAtCancel = fs.existsSync(path.join(workDir, STATE_FILE)); - cancelObserved = true; - return undefined; - }), - qualificationApiRoute.run(() => - cancelObserved - ? workflowRun({ status: "completed", conclusion: "cancelled" }) - : workflowRun(), - ), - ]); - try { - await expect( - startExactImageQualification( - { - request: REQUEST, - coreToken: "core-token", - producerToken: "producer-token", - workDir, - }, - dependencies(api as ReturnType), - ), - ).rejects.toMatchObject({ code: "DISPATCH_AMBIGUOUS" }); - expect( - api.mock.calls.filter(([apiPath]) => - String(apiPath).endsWith(`/workflows/${PRODUCER_WORKFLOW_FILE}/dispatches`), - ), - ).toHaveLength(1); - expect(cancelObserved).toBe(true); - expect(stateExistedAtCancel).toBe(true); - expect(readExactImageQualificationState(workDir).producer.runId).toBe(RUN_ID); - expect(fs.existsSync(path.join(workDir, DISPATCH_INTENT_FILE))).toBe(true); - expect( - JSON.parse(fs.readFileSync(path.join(workDir, DISPATCH_RECONCILIATION_FILE), "utf8")), - ).toMatchObject({ outcome: "recovered-one", runIds: [RUN_ID] }); - } finally { - fs.rmSync(workDir, { recursive: true, force: true }); - } - }); - - it("finds and cleans the exact correlation after producer drift without qualifying it", async () => { - const workDir = tempDirectory(); - const movedSha = "c".repeat(40); - const base = createApi(); - let cancelled = false; - const movedRun = workflowRun({ head_sha: movedSha }); - const historicalRuns = Array.from({ length: 101 }, (_value, index) => - workflowRun({ created_at: new Date(BASE_TIME - (index + 2) * 10 * 60_000).toISOString() }), - ); - let observedCreatedFilter: string | null = null; - let observedHeadShaFilter = false; - let observedScopedRuns: unknown[] = []; - const api = createRoutedApi(base, [ - qualificationApiRoute.dispatch(() => { - throw new Error("lost response after main advanced"); - }), - qualificationApiRoute.workflowRuns((apiPath) => { - const query = new URLSearchParams(apiPath.split("?", 2)[1]); - observedCreatedFilter = query.get("created"); - observedHeadShaFilter = apiPath.includes("head_sha="); - const [earliest, latest] = (query.get("created") ?? "").split("..").map(Date.parse); - const scoped = [...historicalRuns, movedRun].filter(({ created_at }) => { - const createdAt = Date.parse(created_at); - return createdAt >= earliest && createdAt <= latest; - }); - observedScopedRuns = scoped; - return { total_count: scoped.length, workflow_runs: scoped }; - }), - qualificationApiRoute.cancelRun(() => { - cancelled = true; - return undefined; - }), - qualificationApiRoute.run(() => - workflowRun({ - head_sha: movedSha, - status: cancelled ? "completed" : "queued", - conclusion: cancelled ? "cancelled" : null, - }), - ), - ]); - try { - await expect( - startExactImageQualification( - { - request: REQUEST, - coreToken: "core-token", - producerToken: "producer-token", - workDir, - }, - dependencies(api as ReturnType), - ), - ).rejects.toMatchObject({ code: "DISPATCH_AMBIGUOUS" }); - expect(cancelled).toBe(true); - expect(observedCreatedFilter).toBe("2025-12-31T23:59:00Z..2026-01-01T00:01:30Z"); - expect(observedHeadShaFilter).toBe(false); - expect(historicalRuns).toHaveLength(101); - expect(observedScopedRuns).toEqual([movedRun]); - expect(readExactImageQualificationState(workDir)).toMatchObject({ - status: "dispatched", - producer: { runId: RUN_ID, repositorySha: movedSha }, - }); - expect( - JSON.parse(fs.readFileSync(path.join(workDir, DISPATCH_RECONCILIATION_FILE), "utf8")), - ).toMatchObject({ - outcome: "recovered-one", - runIds: [RUN_ID], - producerHeadShas: { [RUN_ID]: movedSha }, - }); - expect(fs.existsSync(path.join(workDir, EVIDENCE_FILE))).toBe(false); - } finally { - fs.rmSync(workDir, { recursive: true, force: true }); - } - }); - - it("cleans the response-bound run after producer SHA provenance fails", async () => { - const workDir = tempDirectory(); - const movedSha = "c".repeat(40); - try { - await expect( - startExactImageQualification( - { - request: REQUEST, - coreToken: "core-token", - producerToken: "producer-token", - workDir, - }, - dependencies(createApi({ run: workflowRun({ head_sha: movedSha }) })), - ), - ).rejects.toMatchObject({ code: "PROVENANCE_MISMATCH" }); - expect(readExactImageQualificationState(workDir).producer.runId).toBe(RUN_ID); - - const base = createApi(); - let cancelled = false; - const cleanupApi = createRoutedApi(base, [ - qualificationApiRoute.cancelRun(() => { - cancelled = true; - return undefined; - }), - qualificationApiRoute.run(() => - workflowRun({ - head_sha: movedSha, - status: cancelled ? "completed" : "queued", - conclusion: cancelled ? "cancelled" : null, - }), - ), - ]); - await expect( - cancelActiveExactImageQualification( - { workDir, producerToken: "producer-token" }, - dependencies(cleanupApi as ReturnType), - ), - ).resolves.toBe(true); - expect(cancelled).toBe(true); - expect( - cleanupApi.mock.calls.filter(([apiPath]) => String(apiPath).endsWith("/cancel")), - ).toHaveLength(1); - expect(readExactImageQualificationState(workDir).status).toBe("dispatched"); - expect(fs.existsSync(path.join(workDir, EVIDENCE_FILE))).toBe(false); - } finally { - fs.rmSync(workDir, { recursive: true, force: true }); - } - }); - - it("never accepts a recovered ambiguous dispatch that already completed successfully", async () => { - const workDir = tempDirectory(); - const base = createApi(); - const completed = workflowRun({ status: "completed", conclusion: "success" }); - const api = createRoutedApi(base, [ - qualificationApiRoute.dispatch(() => { - throw new Error("lost response"); - }), - qualificationApiRoute.workflowRuns(() => ({ - total_count: 1, - workflow_runs: [completed], - })), - ]); - try { - await expect( - startExactImageQualification( - { - request: REQUEST, - coreToken: "core-token", - producerToken: "producer-token", - workDir, - }, - dependencies(api as ReturnType), - ), - ).rejects.toMatchObject({ code: "DISPATCH_AMBIGUOUS" }); - expect( - api.mock.calls.some(([apiPath]) => String(apiPath).endsWith(`/${RUN_ID}/cancel`)), - ).toBe(false); - expect(readExactImageQualificationState(workDir).status).toBe("dispatched"); - } finally { - fs.rmSync(workDir, { recursive: true, force: true }); - } - }); - - it("retains the recovered run identity when cancellation fails", async () => { - const workDir = tempDirectory(); - const base = createApi(); - const api = createRoutedApi(base, [ - qualificationApiRoute.dispatch(() => { - throw new Error("lost response"); - }), - qualificationApiRoute.workflowRuns(() => ({ - total_count: 1, - workflow_runs: [workflowRun()], - })), - qualificationApiRoute.cancelRun(() => { - throw new Error("cancel transport failed"); - }), - ]); - try { - await expect( - startExactImageQualification( - { - request: REQUEST, - coreToken: "core-token", - producerToken: "producer-token", - workDir, - }, - dependencies(api as ReturnType), - ), - ).rejects.toThrow(/cancel transport failed/u); - expect(readExactImageQualificationState(workDir).producer.runId).toBe(RUN_ID); - expect( - JSON.parse(fs.readFileSync(path.join(workDir, DISPATCH_RECONCILIATION_FILE), "utf8")), - ).toMatchObject({ outcome: "recovered-one", runIds: [RUN_ID] }); - } finally { - fs.rmSync(workDir, { recursive: true, force: true }); - } - }); - - it("ignores near-match runs and retains a zero-match reconciliation audit", async () => { - const workDir = tempDirectory(); - const base = createApi(); - let clock = BASE_TIME; - const nearMatches = [ - workflowRun({ display_title: "wrong correlation" }), - workflowRun({ workflow_id: WORKFLOW_ID + 1 }), - workflowRun({ run_attempt: 2 }), - workflowRun({ event: "push" }), - workflowRun({ head_branch: "feature" }), - workflowRun({ path: ".github/workflows/other.yml" }), - workflowRun({ repository: { full_name: "other/repository" } }), - workflowRun({ head_repository: { full_name: "other/repository" } }), - workflowRun({ created_at: new Date(BASE_TIME - 2 * 60_000).toISOString() }), - workflowRun({ url: `${API_RUN_URL}/wrong` }), - ]; - const api = createRoutedApi(base, [ - qualificationApiRoute.dispatch(() => { - throw new Error("lost response"); - }), - qualificationApiRoute.workflowRuns(() => ({ - total_count: nearMatches.length, - workflow_runs: nearMatches, - })), - ]); - try { - await expect( - startExactImageQualification( - { - request: REQUEST, - coreToken: "core-token", - producerToken: "producer-token", - workDir, - }, - dependencies(api as ReturnType, { - now: () => clock, - sleep: async () => { - clock += 2; - }, - limits: { - dispatchReconciliationTimeoutMs: 1, - reconciliationPollIntervalMs: 1, - }, - }), - ), - ).rejects.toMatchObject({ code: "DISPATCH_AMBIGUOUS" }); - expect( - api.mock.calls.some(([apiPath]) => String(apiPath).endsWith(`/${RUN_ID}/cancel`)), - ).toBe(false); - expect( - JSON.parse(fs.readFileSync(path.join(workDir, DISPATCH_RECONCILIATION_FILE), "utf8")), - ).toMatchObject({ outcome: "none", runIds: [] }); - } finally { - fs.rmSync(workDir, { recursive: true, force: true }); - } - }); - - it("lets cleanup preserve semantically invalid state and recover only from durable intent", async () => { - const workDir = tempDirectory(); - const untrustedRunId = "99999"; - let startClock = BASE_TIME; - try { - await expect( - startExactImageQualification( - { - request: REQUEST, - coreToken: "core-token", - producerToken: "producer-token", - workDir, - }, - dependencies(createApi({ dispatch: null }), { - now: () => startClock, - sleep: async () => { - startClock += 2; - }, - limits: { - dispatchReconciliationTimeoutMs: 1, - reconciliationPollIntervalMs: 1, - }, - }), - ), - ).rejects.toMatchObject({ code: "DISPATCH_AMBIGUOUS" }); - expect(fs.existsSync(path.join(workDir, STATE_FILE))).toBe(false); - fs.writeFileSync( - path.join(workDir, STATE_FILE), - JSON.stringify({ - schemaVersion: 1, - status: "dispatched", - producer: { runId: untrustedRunId }, - }), - { mode: 0o600 }, - ); - - const base = createApi(); - let cancelled = false; - const cleanupApi = createRoutedApi(base, [ - qualificationApiRoute.workflowRuns(() => ({ - total_count: 1, - workflow_runs: [workflowRun()], - })), - qualificationApiRoute.cancelRun(() => { - cancelled = true; - return undefined; - }), - qualificationApiRoute.run(() => - cancelled ? workflowRun({ status: "completed", conclusion: "cancelled" }) : workflowRun(), - ), - ]); - await expect( - cancelActiveExactImageQualification( - { workDir, producerToken: "producer-token" }, - dependencies(cleanupApi as ReturnType, { - now: () => BASE_TIME + 10_000, - limits: { cleanupTimeoutMs: 10_000 }, - }), - ), - ).resolves.toBe(true); - expect(cancelled).toBe(true); - expect( - cleanupApi.mock.calls.some(([apiPath]) => String(apiPath).includes(untrustedRunId)), - ).toBe(false); - expect(readExactImageQualificationState(workDir).producer.runId).toBe(RUN_ID); - expect( - fs.readdirSync(workDir).some((name) => name.startsWith("controller-state.corrupt-")), - ).toBe(true); - } finally { - fs.rmSync(workDir, { recursive: true, force: true }); - } - }); - - it("binds a valid state tuple to durable intent before any cancellation POST", async () => { - const started = await startedState(); - const untrustedRunId = "99999"; - const untrustedCorrelationId = "87654321-4321-4321-8321-cba987654321"; - const state = readExactImageQualificationState(started.workDir); - state.request.correlationId = untrustedCorrelationId; - state.producer.runId = untrustedRunId; - state.producer.runUrl = `https://api.github.com/repos/${PRODUCER_REPOSITORY}/actions/runs/${untrustedRunId}`; - state.producer.htmlUrl = `https://github.com/${PRODUCER_REPOSITORY}/actions/runs/${untrustedRunId}`; - fs.writeFileSync(path.join(started.workDir, STATE_FILE), `${JSON.stringify(state)}\n`, { - mode: 0o600, - }); - - const base = createApi(); - let cancelled = false; - const cleanupApi = createRoutedApi(base, [ - qualificationApiRoute.workflowRuns(() => ({ - total_count: 1, - workflow_runs: [workflowRun()], - })), - qualificationApiRoute.cancelRun(() => { - cancelled = true; - return undefined; - }), - qualificationApiRoute.run(() => - cancelled ? workflowRun({ status: "completed", conclusion: "cancelled" }) : workflowRun(), - ), - ]); - try { - await expect( - cancelActiveExactImageQualification( - { workDir: started.workDir, producerToken: "producer-token" }, - dependencies(cleanupApi as ReturnType), - ), - ).resolves.toBe(true); - expect(cancelled).toBe(true); - expect( - cleanupApi.mock.calls.some(([apiPath]) => String(apiPath).includes(untrustedRunId)), - ).toBe(false); - expect(readExactImageQualificationState(started.workDir).producer.runId).toBe(RUN_ID); - expect( - fs - .readdirSync(started.workDir) - .some((name) => name.startsWith("controller-state.corrupt-")), - ).toBe(true); - } finally { - fs.rmSync(started.workDir, { recursive: true, force: true }); - } - }); - - it("does not let a forged terminal state suppress cleanup of the active bound run", async () => { - const started = await startedState(); - const state = readExactImageQualificationState(started.workDir); - state.status = "completed"; - state.producer.completedAt = new Date(BASE_TIME + 1_000).toISOString(); - fs.writeFileSync(path.join(started.workDir, STATE_FILE), `${JSON.stringify(state)}\n`, { - mode: 0o600, - }); - - let cancelled = false; - const cleanupApi = createRoutedApi(createApi(), [ - qualificationApiRoute.cancelRun(() => { - cancelled = true; - return undefined; - }), - qualificationApiRoute.run(() => - cancelled - ? workflowRun({ status: "completed", conclusion: "cancelled" }) - : workflowRun({ status: "queued", conclusion: null }), - ), - ]); - try { - await expect( - cancelActiveExactImageQualification( - { workDir: started.workDir, producerToken: "producer-token" }, - dependencies(cleanupApi as ReturnType), - ), - ).resolves.toBe(true); - expect(cancelled).toBe(true); - expect( - cleanupApi.mock.calls.filter(([apiPath]) => String(apiPath).endsWith("/cancel")), - ).toHaveLength(1); - } finally { - fs.rmSync(started.workDir, { recursive: true, force: true }); - } - }); - - it("rejects run-ID-only state corruption before any cancellation POST", async () => { - const started = await startedState(); - const untrustedRunId = "99999"; - const untrustedApiUrl = `https://api.github.com/repos/${PRODUCER_REPOSITORY}/actions/runs/${untrustedRunId}`; - const untrustedHtmlUrl = `https://github.com/${PRODUCER_REPOSITORY}/actions/runs/${untrustedRunId}`; - const state = readExactImageQualificationState(started.workDir); - state.producer.runId = untrustedRunId; - state.producer.runUrl = untrustedApiUrl; - state.producer.htmlUrl = untrustedHtmlUrl; - fs.writeFileSync(path.join(started.workDir, STATE_FILE), `${JSON.stringify(state)}\n`, { - mode: 0o600, - }); - - const cancelAttempt = vi.fn(); - const cleanupApi = createRoutedApi(createApi(), [ - qualificationApiRoute.runAny(() => - workflowRun({ - id: Number(untrustedRunId), - display_title: "Unrelated qualification run", - url: untrustedApiUrl, - html_url: untrustedHtmlUrl, - }), - ), - qualificationApiRoute.cancelAnyRun(cancelAttempt), - ]); - try { - await expect( - cancelActiveExactImageQualification( - { workDir: started.workDir, producerToken: "producer-token" }, - dependencies(cleanupApi as ReturnType), - ), - ).rejects.toMatchObject({ code: "PROVENANCE_MISMATCH" }); - expect(cancelAttempt).not.toHaveBeenCalled(); - expect(cleanupApi.mock.calls.map(([apiPath]) => apiPath)).toEqual([ - `repos/${PRODUCER_REPOSITORY}/actions/runs/${untrustedRunId}`, - ]); - } finally { - fs.rmSync(started.workDir, { recursive: true, force: true }); - } - }); - - it("fails closed when unreadable state cannot be preserved before cleanup reconciliation", async () => { - const workDir = tempDirectory(); - const controllerState = "not valid JSON"; - const api = createApi(); - try { - fs.writeFileSync( - path.join(workDir, DISPATCH_INTENT_FILE), - `${JSON.stringify(dispatchIntent(), null, 2)}\n`, - { mode: 0o600 }, - ); - fs.writeFileSync(path.join(workDir, STATE_FILE), controllerState, { mode: 0o600 }); - vi.spyOn(fs, "renameSync").mockImplementation(() => { - throw new Error("preservation denied"); - }); - - await expect( - cancelActiveExactImageQualification( - { workDir, producerToken: "producer-token" }, - dependencies(api), - ), - ).rejects.toMatchObject({ code: "OUTPUT_WRITE_FAILED" }); - expect(api).not.toHaveBeenCalled(); - expect(fs.readFileSync(path.join(workDir, STATE_FILE), "utf8")).toBe(controllerState); - expect(fs.existsSync(path.join(workDir, DISPATCH_RECONCILIATION_FILE))).toBe(false); - } finally { - fs.rmSync(workDir, { recursive: true, force: true }); - } - }); - - it("fails closed on multiple strict reconciliation matches and records every exact ID", async () => { - const workDir = tempDirectory(); - const secondId = "24681"; - const base = createApi(); - const second = workflowRun({ - id: Number(secondId), - url: `https://api.github.com/repos/${PRODUCER_REPOSITORY}/actions/runs/${secondId}`, - html_url: `https://github.com/${PRODUCER_REPOSITORY}/actions/runs/${secondId}`, - }); - const api = createRoutedApi(base, [ - qualificationApiRoute.dispatch(() => { - throw new Error("lost response"); - }), - qualificationApiRoute.workflowRuns(() => ({ - total_count: 2, - workflow_runs: [workflowRun(), second], - })), - qualificationApiRoute.cancelAnyRun(() => undefined), - ]); - try { - await expect( - startExactImageQualification( - { - request: REQUEST, - coreToken: "core-token", - producerToken: "producer-token", - workDir, - }, - dependencies(api as ReturnType), - ), - ).rejects.toMatchObject({ code: "DISPATCH_AMBIGUOUS" }); - const audit = JSON.parse( - fs.readFileSync(path.join(workDir, DISPATCH_RECONCILIATION_FILE), "utf8"), - ); - expect(audit).toMatchObject({ outcome: "multiple", runIds: [RUN_ID, secondId] }); - expect( - api.mock.calls.filter(([apiPath]) => String(apiPath).endsWith("/cancel")), - ).toHaveLength(2); - } finally { - fs.rmSync(workDir, { recursive: true, force: true }); - } - }); - - it("aborts a GitHub REST call at the configured per-request cap", async () => { - let observedSignal: AbortSignal | undefined; - const api = vi.fn( - async (_apiPath: string, _token: string, requestOptions?: { signal?: AbortSignal }) => - new Promise((_resolve, reject) => { - observedSignal = requestOptions?.signal; - requestOptions?.signal?.addEventListener("abort", () => reject(new Error("aborted")), { - once: true, - }); - }), - ); - await expect( - preflightExactImageQualification(REQUEST, "core-token", { - api: api as QualificationDependencies["api"], - now: Date.now, - limits: { apiRequestTimeoutMs: 5 }, - }), - ).rejects.toMatchObject({ code: "QUALIFICATION_TIMEOUT" }); - expect(observedSignal?.aborted).toBe(true); - }); - - it("validates the exact returned URLs", () => { - expect(validateWorkflowDispatchDetails(dispatchDetails())).toEqual({ - workflowRunId: RUN_ID, - runUrl: API_RUN_URL, - htmlUrl: HTML_RUN_URL, - }); - expect(() => - validateWorkflowDispatchDetails({ - ...dispatchDetails(), - html_url: `${HTML_RUN_URL}/attempts/1`, - }), - ).toThrow(/html_url/u); - }); - - it("rejects producer workflow identity drift", () => { - expect(() => - validateQualificationWorkflowRun(workflowRun({ head_sha: "c".repeat(40) }), { - candidateSha: CANDIDATE_SHA, - correlationId: CORRELATION_ID, - producerSha: PRODUCER_SHA, - runId: RUN_ID, - runUrl: API_RUN_URL, - htmlUrl: HTML_RUN_URL, - }), - ).toThrow(/head SHA/u); - }); -}); - -describe("bound producer polling", () => { - it("accepts success only from the exact dispatched run", async () => { - const { workDir } = await startedState(); - const api = createApi({ run: workflowRun({ status: "completed", conclusion: "success" }) }); - try { - await expect( - waitForExactImageQualification( - { workDir, producerToken: "producer-token" }, - dependencies(api, { now: () => BASE_TIME + 1_000 }), - ), - ).resolves.toMatchObject({ id: RUN_ID, conclusion: "success" }); - expect(readExactImageQualificationState(workDir).status).toBe("completed"); - } finally { - fs.rmSync(workDir, { recursive: true, force: true }); - } - }); - - it("rejects repeated wait from a terminal state before polling or mutation", async () => { - const started = await startedState(); - const state = readExactImageQualificationState(started.workDir); - state.status = "completed"; - state.producer.completedAt = new Date(BASE_TIME + 1_000).toISOString(); - fs.writeFileSync(path.join(started.workDir, STATE_FILE), `${JSON.stringify(state)}\n`, { - mode: 0o600, - }); - const api = createApi(); - try { - await expect( - waitForExactImageQualification( - { workDir: started.workDir, producerToken: "producer-token" }, - dependencies(api), - ), - ).rejects.toMatchObject({ code: "PROVENANCE_MISMATCH" }); - expect(api).not.toHaveBeenCalled(); - expect(readExactImageQualificationState(started.workDir)).toEqual(state); - } finally { - fs.rmSync(started.workDir, { recursive: true, force: true }); - } - }); - - it("rejects persisted completion and acceptance after the hard deadline", async () => { - const started = await startedState(); - const state = readExactImageQualificationState(started.workDir); - state.status = "completed"; - state.producer.completedAt = new Date(BASE_TIME + 45 * 60_000 + 1).toISOString(); - fs.writeFileSync(path.join(started.workDir, STATE_FILE), `${JSON.stringify(state)}\n`, { - mode: 0o600, - }); - try { - expect(() => readExactImageQualificationState(started.workDir)).toThrow( - /completedAt exceeds qualification deadline/u, - ); - - state.status = "validated"; - state.producer.completedAt = new Date(BASE_TIME + 1_000).toISOString(); - state.artifact = { - id: "86420", - name: `nemoclaw-image-handoff-v1-${RUN_ID}-1`, - digest: `sha256:${"0".repeat(64)}`, - sizeInBytes: 1, - apiUrl: `https://api.github.com/repos/${PRODUCER_REPOSITORY}/actions/artifacts/86420`, - archiveDownloadUrl: `https://api.github.com/repos/${PRODUCER_REPOSITORY}/actions/artifacts/86420/zip`, - archiveSha256: "0".repeat(64), - manifestSha256: "1".repeat(64), - }; - state.validation = { - acceptedAt: new Date(BASE_TIME + 45 * 60_000 + 1).toISOString(), - manifestSha256: "1".repeat(64), - normalizedManifestSha256: "2".repeat(64), - }; - fs.writeFileSync(path.join(started.workDir, STATE_FILE), `${JSON.stringify(state)}\n`, { - mode: 0o600, - }); - expect(() => readExactImageQualificationState(started.workDir)).toThrow( - /acceptedAt exceeds qualification deadline/u, - ); - } finally { - fs.rmSync(started.workDir, { recursive: true, force: true }); - } - }); - - it("cancels a run that remains queued beyond the queue budget", async () => { - const { workDir } = await startedState(); - const api = createApi({ run: workflowRun({ status: "queued" }) }); - try { - await expect( - waitForExactImageQualification( - { workDir, producerToken: "producer-token" }, - dependencies(api, { - now: () => BASE_TIME + 10 * 60_000 + 1, - limits: { queueTimeoutMs: 10 * 60_000 }, - }), - ), - ).rejects.toMatchObject({ code: "RUN_QUEUE_TIMEOUT" }); - expect( - api.mock.calls.some( - ([apiPath]) => apiPath === `repos/${PRODUCER_REPOSITORY}/actions/runs/${RUN_ID}/cancel`, - ), - ).toBe(true); - } finally { - fs.rmSync(workDir, { recursive: true, force: true }); - } - }); - - it("rejects a successful producer completion observed at the shared deadline", async () => { - const { workDir } = await startedState(); - const api = createApi({ run: workflowRun({ status: "completed", conclusion: "success" }) }); - try { - await expect( - waitForExactImageQualification( - { workDir, producerToken: "producer-token" }, - dependencies(api, { now: () => BASE_TIME + 45 * 60_000 }), - ), - ).rejects.toMatchObject({ code: "QUALIFICATION_TIMEOUT" }); - expect(readExactImageQualificationState(workDir).status).toBe("dispatched"); - } finally { - fs.rmSync(workDir, { recursive: true, force: true }); - } - }); -}); - -describe("qualification artifact integrity", () => { - it("requires one non-expired digest-bound artifact from the exact run and SHA", async () => { - const archive = Buffer.from("archive"); - const { state, workDir } = await startedState(); - try { - expect(validateQualificationArtifactList(artifactList(archive), state)).toMatchObject({ - id: "86420", - digest: `sha256:${createHash("sha256").update(archive).digest("hex")}`, - }); - expect(() => - validateQualificationArtifactList( - artifactList(archive, { workflow_run: { id: Number(RUN_ID), head_sha: "c".repeat(40) } }), - state, - ), - ).toThrow(/head SHA/u); - expect(() => - validateQualificationArtifactList(artifactList(archive, { digest: null }), state), - ).toThrow(/digest/u); - } finally { - fs.rmSync(workDir, { recursive: true, force: true }); - } - }); - - it("checks the archive digest before accepting the single root manifest entry", async () => { - const archive = Buffer.from("deterministic archive bytes"); - const manifest = Buffer.from('{"schemaVersion":1}\n'); - const started = await startedState(); - const completedApi = createApi({ - run: workflowRun({ status: "completed", conclusion: "success" }), - }); - await waitForExactImageQualification( - { workDir: started.workDir, producerToken: "producer-token" }, - dependencies(completedApi, { now: () => BASE_TIME + 1_000 }), - ); - const artifactApi = createApi({ artifacts: artifactList(archive) }); - const runCommand = vi.fn((command: string, args: readonly string[]) => ({ - status: 0, - stdout: - args[0] === "-Z1" - ? Buffer.from(`${MANIFEST_ARTIFACT_FILE}\n`) - : args[0] === "-Zl" - ? Buffer.from(`-rw-r--r-- 3.0 unx 20 tx 20 stor ${MANIFEST_ARTIFACT_FILE}\n`) - : manifest, - stderr: Buffer.alloc(0), - })); - const fetchMock = vi.fn( - async (_input: string | URL | Request, _init?: RequestInit) => - new Response(archive, { status: 200 }), - ); - try { - await downloadExactImageManifest( - { workDir: started.workDir, producerToken: "producer-token" }, - dependencies(artifactApi, { fetch: fetchMock, runCommand }), - ); - const fetchHeaders = new Headers(fetchMock.mock.calls[0]?.[1]?.headers); - expect(fetchMock.mock.calls[0]?.[0]).toBe( - `https://api.github.com/repos/${PRODUCER_REPOSITORY}/actions/artifacts/86420/zip`, - ); - expect(fetchHeaders.get("X-GitHub-Api-Version")).toBe(GITHUB_API_VERSION); - expect(fs.readFileSync(path.join(started.workDir, ARCHIVE_FILE))).toEqual(archive); - expect(fs.readFileSync(path.join(started.workDir, MANIFEST_ARTIFACT_FILE))).toEqual(manifest); - expect(readExactImageQualificationState(started.workDir)).toMatchObject({ - status: "downloaded", - artifact: { - archiveSha256: createHash("sha256").update(archive).digest("hex"), - manifestSha256: createHash("sha256").update(manifest).digest("hex"), - }, - }); - } finally { - fs.rmSync(started.workDir, { recursive: true, force: true }); - } - }); - - it("rejects persisted artifact metadata whose GitHub digest and archive hash diverge", async () => { - const archive = Buffer.from("deterministic archive bytes"); - const manifest = Buffer.from('{"schemaVersion":1}\n'); - const started = await startedState(); - await waitForExactImageQualification( - { workDir: started.workDir, producerToken: "producer-token" }, - dependencies( - createApi({ run: workflowRun({ status: "completed", conclusion: "success" }) }), - { now: () => BASE_TIME + 1_000 }, - ), - ); - await downloadExactImageManifest( - { workDir: started.workDir, producerToken: "producer-token" }, - dependencies(createApi({ artifacts: artifactList(archive) }), { - fetch: async () => new Response(archive, { status: 200 }), - runCommand: createArchiveRunCommand(manifest), - }), - ); - const state = JSON.parse( - fs.readFileSync(path.join(started.workDir, STATE_FILE), "utf8"), - ) as Record & { artifact: { archiveSha256: string } }; - state.artifact.archiveSha256 = "0".repeat(64); - fs.writeFileSync(path.join(started.workDir, STATE_FILE), `${JSON.stringify(state)}\n`, { - mode: 0o600, - }); - try { - expect(() => readExactImageQualificationState(started.workDir)).toThrow( - /digest does not match archive hash/u, - ); - expect(fs.existsSync(path.join(started.workDir, EVIDENCE_FILE))).toBe(false); - } finally { - fs.rmSync(started.workDir, { recursive: true, force: true }); - } - }); - - it("does not inspect or extract an archive whose digest mismatches GitHub metadata", async () => { - const archive = Buffer.from("archive bytes"); - const started = await startedState(); - await waitForExactImageQualification( - { workDir: started.workDir, producerToken: "producer-token" }, - dependencies( - createApi({ run: workflowRun({ status: "completed", conclusion: "success" }) }), - { now: () => BASE_TIME + 1_000 }, - ), - ); - const runCommand = vi.fn(); - try { - await expect( - downloadExactImageManifest( - { workDir: started.workDir, producerToken: "producer-token" }, - dependencies( - createApi({ artifacts: artifactList(archive, { digest: `sha256:${"0".repeat(64)}` }) }), - { - fetch: async () => new Response(archive, { status: 200 }), - runCommand, - }, - ), - ), - ).rejects.toMatchObject({ code: "ARTIFACT_MISSING_OR_INVALID" }); - expect(runCommand).not.toHaveBeenCalled(); - } finally { - fs.rmSync(started.workDir, { recursive: true, force: true }); - } - }); - - it("caps artifact propagation at the shared qualification deadline", async () => { - const started = await startedState(); - await waitForExactImageQualification( - { workDir: started.workDir, producerToken: "producer-token" }, - dependencies( - createApi({ run: workflowRun({ status: "completed", conclusion: "success" }) }), - { now: () => BASE_TIME + 43 * 60_000 }, - ), - ); - let clock = BASE_TIME + 44 * 60_000; - const api = createApi({ artifacts: { total_count: 0, artifacts: [] } }); - try { - await expect( - downloadExactImageManifest( - { workDir: started.workDir, producerToken: "producer-token" }, - dependencies(api, { - now: () => clock, - sleep: async () => { - clock = BASE_TIME + 45 * 60_000; - }, - }), - ), - ).rejects.toMatchObject({ code: "QUALIFICATION_TIMEOUT" }); - expect( - api.mock.calls.filter(([apiPath]) => String(apiPath).includes("/artifacts?")), - ).toHaveLength(1); - } finally { - fs.rmSync(started.workDir, { recursive: true, force: true }); - } - }); - - it("does not accept an archive whose extraction crosses the shared deadline", async () => { - const archive = Buffer.from("archive"); - const manifest = Buffer.from('{"schemaVersion":1}\n'); - const started = await startedState(); - await waitForExactImageQualification( - { workDir: started.workDir, producerToken: "producer-token" }, - dependencies( - createApi({ run: workflowRun({ status: "completed", conclusion: "success" }) }), - { now: () => BASE_TIME + 43 * 60_000 }, - ), - ); - let clock = BASE_TIME + 44 * 60_000; - try { - await expect( - downloadExactImageManifest( - { workDir: started.workDir, producerToken: "producer-token" }, - dependencies(createApi({ artifacts: artifactList(archive) }), { - now: () => clock, - fetch: async () => new Response(archive, { status: 200 }), - runCommand: createArchiveRunCommand(manifest, () => { - clock = BASE_TIME + 45 * 60_000; - }), - }), - ), - ).rejects.toMatchObject({ code: "QUALIFICATION_TIMEOUT" }); - expect(readExactImageQualificationState(started.workDir).status).toBe("completed"); - } finally { - fs.rmSync(started.workDir, { recursive: true, force: true }); - } - }); - - it("rejects duplicate or nested ZIP entry inventories", () => { - const tempDir = tempDirectory(); - const archivePath = path.join(tempDir, ARCHIVE_FILE); - fs.writeFileSync(archivePath, "not inspected by the command seam"); - try { - expect(() => - extractExactManifestArchive( - archivePath, - path.join(tempDir, MANIFEST_ARTIFACT_FILE), - () => ({ - status: 0, - stdout: Buffer.from(`${MANIFEST_ARTIFACT_FILE}\n${MANIFEST_ARTIFACT_FILE}\n`), - stderr: Buffer.alloc(0), - }), - ), - ).toThrow(/exactly/u); - expect(() => - extractExactManifestArchive( - archivePath, - path.join(tempDir, MANIFEST_ARTIFACT_FILE), - () => ({ - status: 0, - stdout: Buffer.from(`nested/${MANIFEST_ARTIFACT_FILE}\n`), - stderr: Buffer.alloc(0), - }), - ), - ).toThrow(/exactly/u); - } finally { - fs.rmSync(tempDir, { recursive: true, force: true }); - } - }); - - it("rejects a ZIP entry marked as a symbolic link before extraction", () => { - const tempDir = tempDirectory(); - const archivePath = path.join(tempDir, ARCHIVE_FILE); - fs.writeFileSync(archivePath, "not inspected by the command seam"); - const runCommand = vi.fn((_command: string, args: readonly string[]) => ({ - status: 0, - stdout: - args[0] === "-Z1" - ? Buffer.from(`${MANIFEST_ARTIFACT_FILE}\n`) - : Buffer.from(`lrwxrwxrwx 3.0 unx 20 tx 20 stor ${MANIFEST_ARTIFACT_FILE}\n`), - stderr: Buffer.alloc(0), - })); - try { - expect(() => - extractExactManifestArchive( - archivePath, - path.join(tempDir, MANIFEST_ARTIFACT_FILE), - runCommand, - ), - ).toThrow(/regular file/u); - expect(runCommand).toHaveBeenCalledTimes(2); - } finally { - fs.rmSync(tempDir, { recursive: true, force: true }); - } - }); -}); - -describe("qualification evidence finalization", () => { - it("records immutable producer, artifact, and accepted-manifest hashes", async () => { - const archive = Buffer.from("archive"); - const manifest = Buffer.from('{"schemaVersion":1}\n'); - const normalized = Buffer.from('{"schemaVersion":1,"accepted":true}\n'); - const started = await startedState(); - await waitForExactImageQualification( - { workDir: started.workDir, producerToken: "producer-token" }, - dependencies( - createApi({ run: workflowRun({ status: "completed", conclusion: "success" }) }), - { now: () => BASE_TIME + 1_000 }, - ), - ); - await downloadExactImageManifest( - { workDir: started.workDir, producerToken: "producer-token" }, - dependencies(createApi({ artifacts: artifactList(archive) }), { - fetch: async () => new Response(archive, { status: 200 }), - runCommand: (_command, args) => ({ - status: 0, - stdout: - args[0] === "-Z1" - ? Buffer.from(`${MANIFEST_ARTIFACT_FILE}\n`) - : args[0] === "-Zl" - ? Buffer.from(`-rw-r--r-- 3.0 unx 20 tx 20 stor ${MANIFEST_ARTIFACT_FILE}\n`) - : manifest, - stderr: Buffer.alloc(0), - }), - }), - ); - fs.writeFileSync(path.join(started.workDir, VALIDATED_MANIFEST_FILE), normalized, { - mode: 0o600, - }); - try { - const finalState = finalizeExactImageQualification(started.workDir, { - now: () => BASE_TIME + 2_000, - }); - expect(finalState).toMatchObject({ - status: "validated", - validation: { - manifestSha256: createHash("sha256").update(manifest).digest("hex"), - normalizedManifestSha256: createHash("sha256").update(normalized).digest("hex"), - }, - }); - const evidence = JSON.parse( - fs.readFileSync(path.join(started.workDir, EVIDENCE_FILE), "utf8"), - ); - expect(evidence).toMatchObject({ - qualificationStatus: "accepted", - producer: { runId: RUN_ID, repositorySha: PRODUCER_SHA }, - artifact: { id: "86420" }, - }); - } finally { - fs.rmSync(started.workDir, { recursive: true, force: true }); - } - }); - - it("never publishes accepted evidence when the validated state transition is rejected", async () => { - const archive = Buffer.from("archive"); - const manifest = Buffer.from('{"schemaVersion":1}\n'); - const normalized = Buffer.from('{"schemaVersion":1,"accepted":true}\n'); - const started = await startedState(); - await waitForExactImageQualification( - { workDir: started.workDir, producerToken: "producer-token" }, - dependencies( - createApi({ run: workflowRun({ status: "completed", conclusion: "success" }) }), - { now: () => BASE_TIME + 3_000 }, - ), - ); - await downloadExactImageManifest( - { workDir: started.workDir, producerToken: "producer-token" }, - dependencies(createApi({ artifacts: artifactList(archive) }), { - fetch: async () => new Response(archive, { status: 200 }), - runCommand: createArchiveRunCommand(manifest), - }), - ); - fs.writeFileSync(path.join(started.workDir, VALIDATED_MANIFEST_FILE), normalized, { - mode: 0o600, - }); - try { - expect(() => - finalizeExactImageQualification(started.workDir, { - now: () => BASE_TIME + 2_000, - }), - ).toThrowError(expect.objectContaining({ code: "OUTPUT_WRITE_FAILED" })); - expect(fs.existsSync(path.join(started.workDir, EVIDENCE_FILE))).toBe(false); - expect(readExactImageQualificationState(started.workDir).status).toBe("downloaded"); - } finally { - fs.rmSync(started.workDir, { recursive: true, force: true }); - } - }); - - it("refuses accepted evidence at the shared deadline", () => { - const workDir = tempDirectory(); - const manifest = Buffer.from('{"schemaVersion":1}\n'); - const normalized = Buffer.from('{"schemaVersion":1,"accepted":true}\n'); - const manifestSha256 = createHash("sha256").update(manifest).digest("hex"); - const state = { - schemaVersion: 1, - status: "downloaded", - dispatchedAt: new Date(BASE_TIME).toISOString(), - request: { - actor: REQUEST.actor, - candidateSha: CANDIDATE_SHA, - correlationId: CORRELATION_ID, - reason: REQUEST.reason, - requesterRunAttempt: REQUEST.requesterRunAttempt, - requesterRunId: REQUEST.requesterRunId, - workflowSha: WORKFLOW_SHA, - }, - producer: { - repository: PRODUCER_REPOSITORY, - repositorySha: PRODUCER_SHA, - ref: "main", - workflowPath: PRODUCER_WORKFLOW_PATH, - runId: RUN_ID, - runAttempt: 1, - workflowId: String(WORKFLOW_ID), - runUrl: API_RUN_URL, - htmlUrl: HTML_RUN_URL, - completedAt: new Date(BASE_TIME + 1_000).toISOString(), - }, - artifact: { - id: "86420", - name: `nemoclaw-image-handoff-v1-${RUN_ID}-1`, - digest: `sha256:${"0".repeat(64)}`, - sizeInBytes: 7, - apiUrl: `https://api.github.com/repos/${PRODUCER_REPOSITORY}/actions/artifacts/86420`, - archiveDownloadUrl: `https://api.github.com/repos/${PRODUCER_REPOSITORY}/actions/artifacts/86420/zip`, - archiveSha256: "0".repeat(64), - manifestSha256, - }, - }; - fs.writeFileSync( - path.join(workDir, DISPATCH_INTENT_FILE), - `${JSON.stringify(dispatchIntent(), null, 2)}\n`, - { mode: 0o600 }, - ); - fs.writeFileSync(path.join(workDir, STATE_FILE), `${JSON.stringify(state)}\n`, { mode: 0o600 }); - fs.writeFileSync(path.join(workDir, MANIFEST_ARTIFACT_FILE), manifest, { mode: 0o600 }); - fs.writeFileSync(path.join(workDir, VALIDATED_MANIFEST_FILE), normalized, { mode: 0o600 }); - try { - expect(() => - finalizeExactImageQualification(workDir, { - now: () => BASE_TIME + 45 * 60_000, - }), - ).toThrowError(expect.objectContaining({ code: "QUALIFICATION_TIMEOUT" })); - expect(fs.existsSync(path.join(workDir, EVIDENCE_FILE))).toBe(false); - } finally { - fs.rmSync(workDir, { recursive: true, force: true }); - } - }); - - it("loads under the workflow's dependency-free Node strip-types runtime", () => { - const script = path.resolve("tools/e2e/exact-image-qualification-controller.mts"); - const result = spawnSync( - process.execPath, - [ - "--experimental-strip-types", - "--no-warnings", - script, - "--mode", - "cancel", - "--work-dir", - "/does/not/exist", - ], - { - encoding: "utf8", - env: { ...process.env, NEMOCLAW_IMAGE_QUALIFICATION_TOKEN: "test-token" }, - timeout: 10_000, - }, - ); - expect(result.status, result.stderr).toBe(0); - expect(result.stdout).toContain("No active producer run"); - }); -}); diff --git a/test/helpers/exact-image-qualification-controller-fixture.ts b/test/helpers/exact-image-qualification-controller-fixture.ts deleted file mode 100644 index 1dba1cd812..0000000000 --- a/test/helpers/exact-image-qualification-controller-fixture.ts +++ /dev/null @@ -1,284 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { createHash } from "node:crypto"; -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; - -import { vi } from "vitest"; - -import { - type ExactImageDispatchIntent, - type ExactImageQualificationRequest, - MANIFEST_ARTIFACT_FILE, - PRODUCER_REF, - PRODUCER_REPOSITORY, - PRODUCER_WORKFLOW_FILE, - PRODUCER_WORKFLOW_PATH, - type QualificationDependencies, - startExactImageQualification, -} from "../../tools/e2e/exact-image-qualification-controller.mts"; - -export const WORKFLOW_SHA = "c".repeat(40); -export const CANDIDATE_SHA = WORKFLOW_SHA; -export const PRODUCER_SHA = "b".repeat(40); -export const CORRELATION_ID = "123e4567-e89b-42d3-a456-426614174000"; -export const RUN_ID = "24680"; -export const WORKFLOW_ID = 13579; -export const BASE_TIME = Date.UTC(2026, 0, 1); -export const API_RUN_URL = `https://api.github.com/repos/${PRODUCER_REPOSITORY}/actions/runs/${RUN_ID}`; -export const HTML_RUN_URL = `https://github.com/${PRODUCER_REPOSITORY}/actions/runs/${RUN_ID}`; - -export const REQUEST: ExactImageQualificationRequest = { - actor: "maintainer", - candidateSha: CANDIDATE_SHA, - eventName: "workflow_dispatch", - reason: "Qualify the current daily candidate before tagging", - ref: "refs/heads/main", - requesterRunAttempt: 2, - requesterRunId: "97531", - workflowSha: WORKFLOW_SHA, -}; - -export function dispatchIntent(): ExactImageDispatchIntent { - return { - schemaVersion: 1, - kind: "nemoclaw-exact-image-dispatch-intent", - requestStartedAt: new Date(BASE_TIME).toISOString(), - request: { - actor: REQUEST.actor, - candidateSha: REQUEST.candidateSha, - correlationId: CORRELATION_ID, - reason: REQUEST.reason, - requesterRunAttempt: REQUEST.requesterRunAttempt, - requesterRunId: REQUEST.requesterRunId, - workflowSha: REQUEST.workflowSha, - }, - producer: { - repository: PRODUCER_REPOSITORY, - repositorySha: PRODUCER_SHA, - ref: PRODUCER_REF, - workflowId: String(WORKFLOW_ID), - workflowPath: PRODUCER_WORKFLOW_PATH, - }, - }; -} - -type ApiOptions = { - artifacts?: unknown; - dispatch?: unknown; - permission?: string; - roleName?: string; - producerSha?: string; - requesterSha?: string; - run?: unknown; - runs?: unknown; - workflow?: unknown; -}; - -type ApiRouteHandler = ( - apiPath: string, - token: string, - requestOptions?: unknown, -) => unknown | Promise; - -type ApiRoute = { - matches: (apiPath: string) => boolean; - respond: ApiRouteHandler; -}; - -function mainRef(sha: string) { - return { ref: "refs/heads/main", object: { type: "commit", sha } }; -} - -export function dispatchDetails() { - return { - workflow_run_id: Number(RUN_ID), - run_url: API_RUN_URL, - html_url: HTML_RUN_URL, - }; -} - -export function workflowRun(overrides: Record = {}) { - return { - id: Number(RUN_ID), - workflow_id: WORKFLOW_ID, - run_attempt: 1, - event: "workflow_dispatch", - head_branch: "main", - head_sha: PRODUCER_SHA, - path: PRODUCER_WORKFLOW_PATH, - display_title: `Qualify NemoClaw ${CANDIDATE_SHA} (${CORRELATION_ID})`, - url: API_RUN_URL, - html_url: HTML_RUN_URL, - repository: { full_name: PRODUCER_REPOSITORY }, - head_repository: { full_name: PRODUCER_REPOSITORY }, - status: "queued", - conclusion: null, - created_at: new Date(BASE_TIME).toISOString(), - ...overrides, - }; -} - -export function createApi(options: ApiOptions = {}) { - return vi.fn(async (apiPath: string, _token: string, requestOptions?: unknown) => { - if (apiPath === "repos/NVIDIA/NemoClaw/git/ref/heads/main") { - return mainRef(options.requesterSha ?? WORKFLOW_SHA); - } - if (apiPath === `repos/NVIDIA/NemoClaw/git/commits/${CANDIDATE_SHA}`) { - return { sha: CANDIDATE_SHA }; - } - if (apiPath.includes("/collaborators/")) { - return { - permission: options.permission ?? "write", - role_name: options.roleName ?? "maintain", - }; - } - if (apiPath === `repos/${PRODUCER_REPOSITORY}/git/ref/heads/main`) { - return mainRef(options.producerSha ?? PRODUCER_SHA); - } - if (apiPath === `repos/${PRODUCER_REPOSITORY}/actions/workflows/${PRODUCER_WORKFLOW_FILE}`) { - return ( - options.workflow ?? { - id: WORKFLOW_ID, - path: PRODUCER_WORKFLOW_PATH, - state: "active", - } - ); - } - if ( - apiPath === - `repos/${PRODUCER_REPOSITORY}/actions/workflows/${PRODUCER_WORKFLOW_FILE}/dispatches` - ) { - return options.dispatch === undefined ? dispatchDetails() : options.dispatch; - } - if (apiPath === `repos/${PRODUCER_REPOSITORY}/actions/runs/${RUN_ID}`) { - return options.run ?? workflowRun(); - } - if (apiPath === `repos/${PRODUCER_REPOSITORY}/actions/runs/${RUN_ID}/cancel`) { - return undefined; - } - if ( - apiPath.startsWith( - `repos/${PRODUCER_REPOSITORY}/actions/workflows/${PRODUCER_WORKFLOW_FILE}/runs?`, - ) - ) { - return options.runs ?? { total_count: 0, workflow_runs: [] }; - } - if (apiPath === `repos/${PRODUCER_REPOSITORY}/actions/runs/${RUN_ID}/artifacts?per_page=100`) { - return options.artifacts; - } - throw new Error(`unexpected API call ${apiPath} ${JSON.stringify(requestOptions)}`); - }); -} - -function route(matches: (apiPath: string) => boolean, respond: ApiRouteHandler): ApiRoute { - return { matches, respond }; -} - -export const qualificationApiRoute = { - dispatch: (respond: ApiRouteHandler): ApiRoute => - route( - (apiPath) => apiPath.endsWith(`/workflows/${PRODUCER_WORKFLOW_FILE}/dispatches`), - respond, - ), - workflowRuns: (respond: ApiRouteHandler): ApiRoute => - route((apiPath) => apiPath.includes(`/workflows/${PRODUCER_WORKFLOW_FILE}/runs?`), respond), - run: (respond: ApiRouteHandler): ApiRoute => - route((apiPath) => apiPath === `repos/${PRODUCER_REPOSITORY}/actions/runs/${RUN_ID}`, respond), - runAny: (respond: ApiRouteHandler): ApiRoute => - route( - (apiPath) => /^repos\/brevdev\/nemoclaw-image\/actions\/runs\/[1-9][0-9]*$/u.test(apiPath), - respond, - ), - cancelRun: (respond: ApiRouteHandler): ApiRoute => - route( - (apiPath) => apiPath === `repos/${PRODUCER_REPOSITORY}/actions/runs/${RUN_ID}/cancel`, - respond, - ), - cancelAnyRun: (respond: ApiRouteHandler): ApiRoute => - route((apiPath) => apiPath.endsWith("/cancel"), respond), -}; - -export function createRoutedApi(base: ReturnType, routes: readonly ApiRoute[]) { - return vi.fn(async (apiPath: string, token: string, requestOptions?: unknown) => { - for (const candidate of routes) { - if (candidate.matches(apiPath)) { - return candidate.respond(apiPath, token, requestOptions); - } - } - return base(apiPath, token, requestOptions); - }); -} - -export function dependencies( - api: ReturnType, - extra: QualificationDependencies = {}, -) { - return { - now: () => BASE_TIME, - ...extra, - api: api as QualificationDependencies["api"], - randomUuid: () => CORRELATION_ID, - }; -} - -export function tempDirectory(): string { - return fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-image-qualification-test-")); -} - -export async function startedState(api = createApi()) { - const workDir = tempDirectory(); - const state = await startExactImageQualification( - { - request: REQUEST, - coreToken: "core-token", - producerToken: "producer-token", - workDir, - }, - dependencies(api), - ); - return { api, state, workDir }; -} - -export function artifactList(archive: Buffer, overrides: Record = {}) { - const artifactId = 86420; - return { - total_count: 1, - artifacts: [ - { - id: artifactId, - name: `nemoclaw-image-handoff-v1-${RUN_ID}-1`, - expired: false, - digest: `sha256:${createHash("sha256").update(archive).digest("hex")}`, - size_in_bytes: archive.length, - url: `https://api.github.com/repos/${PRODUCER_REPOSITORY}/actions/artifacts/${artifactId}`, - archive_download_url: `https://api.github.com/repos/${PRODUCER_REPOSITORY}/actions/artifacts/${artifactId}/zip`, - workflow_run: { id: Number(RUN_ID), head_sha: PRODUCER_SHA }, - ...overrides, - }, - ], - }; -} - -export function createArchiveRunCommand(manifest: Buffer, onExtract: () => void = () => {}) { - return (_command: string, args: readonly string[]) => { - if (args[0] === "-Z1") { - return { - status: 0, - stdout: Buffer.from(`${MANIFEST_ARTIFACT_FILE}\n`), - stderr: Buffer.alloc(0), - }; - } - if (args[0] === "-Zl") { - return { - status: 0, - stdout: Buffer.from(`-rw-r--r-- 3.0 unx 20 tx 20 stor ${MANIFEST_ARTIFACT_FILE}\n`), - stderr: Buffer.alloc(0), - }; - } - onExtract(); - return { status: 0, stdout: manifest, stderr: Buffer.alloc(0) }; - }; -} diff --git a/test/helpers/vitest-watch-triggers.ts b/test/helpers/vitest-watch-triggers.ts index 43f5b5573b..242cc14695 100644 --- a/test/helpers/vitest-watch-triggers.ts +++ b/test/helpers/vitest-watch-triggers.ts @@ -37,7 +37,7 @@ const E2E_WORKFLOW_CONTRACTS = [ "test/e2e/support/tunnel-lifecycle-workflow-boundary.test.ts", "test/e2e/support/upload-e2e-artifacts-workflow-boundary.test.ts", "test/e2e/support/workflow-plan.test.ts", - "test/e2e/support/exact-image-qualification-workflow.test.ts", + "test/e2e/support/brev-launchable-e2e-workflow.test.ts", ] as const; function runTests(...tests: string[]): () => string[] { @@ -97,15 +97,11 @@ export const vitestWatchTriggerPatterns: VitestWatchTriggerPattern[] = [ pattern: /(?:^|\/)\.github\/workflows\/pr-e2e-gate\.yaml$/, testsToRun: runTests("test/pr-e2e-gate-workflow.test.ts", "test/pr-e2e-required.test.ts"), }, - { - pattern: /(?:^|\/)\.github\/workflows\/brev-launchable-qualification\.yaml$/, - testsToRun: runTests("test/e2e/support/exact-image-qualification-workflow.test.ts"), - }, { pattern: /(?:^|\/)tools\/e2e\/brev-launchable-runtime\.sh$/, testsToRun: runTests( "test/brev-launchable-runtime.test.ts", - "test/e2e/support/exact-image-qualification-workflow.test.ts", + "test/e2e/support/brev-launchable-e2e-workflow.test.ts", ), }, { diff --git a/test/vitest-watch-triggers.test.ts b/test/vitest-watch-triggers.test.ts index 3fda1fae76..ac0f42cdb7 100644 --- a/test/vitest-watch-triggers.test.ts +++ b/test/vitest-watch-triggers.test.ts @@ -43,7 +43,7 @@ const E2E_WORKFLOW_CONTRACTS = [ "test/e2e/support/tunnel-lifecycle-workflow-boundary.test.ts", "test/e2e/support/upload-e2e-artifacts-workflow-boundary.test.ts", "test/e2e/support/workflow-plan.test.ts", - "test/e2e/support/exact-image-qualification-workflow.test.ts", + "test/e2e/support/brev-launchable-e2e-workflow.test.ts", ] as const; const OPAQUE_INPUTS = [ @@ -59,7 +59,6 @@ const OPAQUE_INPUTS = [ ".github/workflows/e2e.yaml", ".github/workflows/code-scanning.yaml", ".github/workflows/pr-e2e-gate.yaml", - ".github/workflows/brev-launchable-qualification.yaml", ".github/workflows/platform-vitest-main.yaml", "ci/platform-vitest-macos-requirements.lock", ] as const; @@ -113,12 +112,9 @@ describe("Vitest opaque-input watch triggers", () => { "test/pr-e2e-gate-workflow.test.ts", "test/pr-e2e-required.test.ts", ]); - expect(triggeredBy(".github/workflows/brev-launchable-qualification.yaml")).toEqual([ - "test/e2e/support/exact-image-qualification-workflow.test.ts", - ]); expect(triggeredBy("tools/e2e/brev-launchable-runtime.sh")).toEqual([ "test/brev-launchable-runtime.test.ts", - "test/e2e/support/exact-image-qualification-workflow.test.ts", + "test/e2e/support/brev-launchable-e2e-workflow.test.ts", ]); expect(triggeredBy(".github/workflows/platform-vitest-main.yaml")).toEqual([ "test/platform-vitest-main-workflow.test.ts", diff --git a/tools/e2e/exact-image-manifest.mts b/tools/e2e/exact-image-manifest.mts deleted file mode 100644 index 8ee007a411..0000000000 --- a/tools/e2e/exact-image-manifest.mts +++ /dev/null @@ -1,396 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -export const EXACT_IMAGE_MANIFEST_KIND = "nemoclaw-exact-image-manifest"; -export const EXACT_IMAGE_REQUESTER_REPOSITORY = "NVIDIA/NemoClaw"; -export const EXACT_IMAGE_REPOSITORY = "brevdev/nemoclaw-image"; -export const EXACT_IMAGE_PRODUCER_WORKFLOW = ".github/workflows/build-qualification-image.yml"; -// This mutable family is publication evidence only. Consumers accept the -// immutable name, numeric ID, and self-link below as the image identity. -export const EXACT_IMAGE_STAGING_FAMILY = "nemoclaw-brev-staging-cpu"; -export const EXACT_IMAGE_MANIFEST_MAX_CLOCK_SKEW_MS = 5 * 60_000; -export const EXACT_IMAGE_MANIFEST_MAX_REUSED_AGE_MS = 24 * 60 * 60_000; - -export type ExactImageManifestFailureCode = - | "REQUEST_INVALID" - | "ARTIFACT_MISSING_OR_INVALID" - | "UNSUPPORTED_VARIANT" - | "PROVENANCE_MISMATCH" - | "IMAGE_IDENTITY_MISMATCH" - | "OUTPUT_WRITE_FAILED" - | "UNKNOWN"; - -export class ExactImageManifestError extends Error { - readonly code: ExactImageManifestFailureCode; - - constructor(code: ExactImageManifestFailureCode, message: string) { - super(message); - this.name = "ExactImageManifestError"; - this.code = code; - } -} - -export type ExactImageManifest = { - schemaVersion: 1; - kind: typeof EXACT_IMAGE_MANIFEST_KIND; - correlationId: string; - requesterRepository: typeof EXACT_IMAGE_REQUESTER_REPOSITORY; - requesterWorkflowRunId: string; - requesterWorkflowRunAttempt: number; - nemoclawSha: string; - imageRepository: typeof EXACT_IMAGE_REPOSITORY; - imageRepositorySha: string; - producerWorkflow: typeof EXACT_IMAGE_PRODUCER_WORKFLOW; - workflowRunId: string; - workflowRunAttempt: number; - imageOriginWorkflowRunId: string; - imageOriginWorkflowRunAttempt: number; - imageKind: "compute#image"; - project: string; - imageName: string; - imageId: string; - imageSelfLink: string; - status: "READY"; - imageCreationTimestamp: string; - manifestCreatedAt: string; - channel: "staging"; - variant: "cpu"; - observedFamily: typeof EXACT_IMAGE_STAGING_FAMILY; - result: "built" | "reused"; -}; - -export type ExactImageManifestExpectations = { - correlationId: string; - requesterWorkflowRunId: string; - requesterWorkflowRunAttempt: number; - nemoclawSha: string; - imageRepositorySha: string; - workflowRunId: string; - workflowRunAttempt: number; -}; - -const REQUIRED_FIELDS = [ - "schemaVersion", - "kind", - "correlationId", - "requesterRepository", - "requesterWorkflowRunId", - "requesterWorkflowRunAttempt", - "nemoclawSha", - "imageRepository", - "imageRepositorySha", - "producerWorkflow", - "workflowRunId", - "workflowRunAttempt", - "imageOriginWorkflowRunId", - "imageOriginWorkflowRunAttempt", - "imageKind", - "project", - "imageName", - "imageId", - "imageSelfLink", - "status", - "imageCreationTimestamp", - "manifestCreatedAt", - "channel", - "variant", - "observedFamily", - "result", -] as const; - -const REQUIRED_FIELD_SET = new Set(REQUIRED_FIELDS); -const FULL_SHA_PATTERN = /^[0-9a-f]{40}$/u; -const UUID_V4_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/u; -const DECIMAL_ID_PATTERN = /^[1-9][0-9]*$/u; -const GCP_PROJECT_ID_PATTERN = /^[a-z][a-z0-9-]{4,28}[a-z0-9]$/u; -const IMAGE_NAME_PATTERN = /^[a-z](?:[-a-z0-9]{0,61}[a-z0-9])?$/u; -const RFC3339_PATTERN = - /^(\d{4})-(\d{2})-(\d{2})T([01]\d|2[0-3]):([0-5]\d):([0-5]\d)(?:[.](\d{1,9}))?(Z|[+-](?:[01]\d|2[0-3]):[0-5]\d)$/u; - -function fail(code: ExactImageManifestFailureCode, message: string): never { - throw new ExactImageManifestError(code, message); -} - -function requireRecord(value: unknown): Record { - if (value === null || typeof value !== "object" || Array.isArray(value)) { - fail("ARTIFACT_MISSING_OR_INVALID", "manifest must be a JSON object"); - } - return value as Record; -} - -function validateExactFields(record: Record): void { - for (const field of REQUIRED_FIELDS) { - if (!Object.hasOwn(record, field)) { - fail("ARTIFACT_MISSING_OR_INVALID", `manifest is missing required field ${field}`); - } - } - const unexpected = Object.keys(record) - .filter((field) => !REQUIRED_FIELD_SET.has(field)) - .sort(); - if (unexpected.length > 0) { - fail("ARTIFACT_MISSING_OR_INVALID", `manifest contains unexpected field ${unexpected[0]}`); - } -} - -function requireString(record: Record, field: string): string { - const value = record[field]; - if (typeof value !== "string") { - fail("ARTIFACT_MISSING_OR_INVALID", `${field} must be a string`); - } - return value; -} - -function requirePositiveInteger(record: Record, field: string): number { - const value = record[field]; - if (!Number.isSafeInteger(value) || (value as number) < 1) { - fail("ARTIFACT_MISSING_OR_INVALID", `${field} must be a positive safe integer`); - } - return value as number; -} - -function requirePattern(value: string, field: string, pattern: RegExp): void { - if (!pattern.test(value)) { - fail("ARTIFACT_MISSING_OR_INVALID", `${field} has an invalid format`); - } -} - -function requireConstant( - actual: unknown, - expected: T, - field: string, -): asserts actual is T { - if (actual !== expected) { - fail("PROVENANCE_MISMATCH", `${field} must equal ${JSON.stringify(expected)}`); - } -} - -function daysInMonth(year: number, month: number): number { - if (month === 2) { - const leap = year % 4 === 0 && (year % 100 !== 0 || year % 400 === 0); - return leap ? 29 : 28; - } - return [4, 6, 9, 11].includes(month) ? 30 : 31; -} - -function parseRfc3339(value: string, field: string): number { - const match = RFC3339_PATTERN.exec(value); - if (!match) { - fail("ARTIFACT_MISSING_OR_INVALID", `${field} must be an RFC 3339 date-time`); - } - const year = Number(match[1]); - const month = Number(match[2]); - const day = Number(match[3]); - if (month < 1 || month > 12 || day < 1 || day > daysInMonth(year, month)) { - fail("ARTIFACT_MISSING_OR_INVALID", `${field} must be a real calendar date`); - } - const timestamp = Date.parse(value); - if (!Number.isFinite(timestamp)) { - fail("ARTIFACT_MISSING_OR_INVALID", `${field} must be an RFC 3339 date-time`); - } - return timestamp; -} - -function validateExpectations(expected: ExactImageManifestExpectations): void { - const stringPatterns: Array<[string, string, RegExp]> = [ - [expected.correlationId, "expected correlationId", UUID_V4_PATTERN], - [expected.requesterWorkflowRunId, "expected requesterWorkflowRunId", DECIMAL_ID_PATTERN], - [expected.nemoclawSha, "expected nemoclawSha", FULL_SHA_PATTERN], - [expected.imageRepositorySha, "expected imageRepositorySha", FULL_SHA_PATTERN], - [expected.workflowRunId, "expected workflowRunId", DECIMAL_ID_PATTERN], - ]; - for (const [value, field, pattern] of stringPatterns) { - if (typeof value !== "string" || !pattern.test(value)) { - fail("REQUEST_INVALID", `${field} has an invalid format`); - } - } - for (const [value, field] of [ - [expected.requesterWorkflowRunAttempt, "expected requesterWorkflowRunAttempt"], - [expected.workflowRunAttempt, "expected workflowRunAttempt"], - ] as const) { - if (!Number.isSafeInteger(value) || value < 1) { - fail("REQUEST_INVALID", `${field} must be a positive safe integer`); - } - } -} - -function assertExpected( - manifest: ExactImageManifest, - expected: ExactImageManifestExpectations, -): void { - const comparisons: Array<[unknown, unknown, string]> = [ - [manifest.correlationId, expected.correlationId, "correlationId"], - [manifest.requesterWorkflowRunId, expected.requesterWorkflowRunId, "requesterWorkflowRunId"], - [ - manifest.requesterWorkflowRunAttempt, - expected.requesterWorkflowRunAttempt, - "requesterWorkflowRunAttempt", - ], - [manifest.nemoclawSha, expected.nemoclawSha, "nemoclawSha"], - [manifest.imageRepositorySha, expected.imageRepositorySha, "imageRepositorySha"], - [manifest.workflowRunId, expected.workflowRunId, "workflowRunId"], - [manifest.workflowRunAttempt, expected.workflowRunAttempt, "workflowRunAttempt"], - ]; - for (const [actual, wanted, field] of comparisons) { - if (actual !== wanted) { - fail("PROVENANCE_MISMATCH", `${field} does not match the trusted request`); - } - } -} - -function validateTemporalContract(manifest: ExactImageManifest): void { - const imageCreated = parseRfc3339(manifest.imageCreationTimestamp, "imageCreationTimestamp"); - const manifestCreated = parseRfc3339(manifest.manifestCreatedAt, "manifestCreatedAt"); - if (imageCreated > manifestCreated + EXACT_IMAGE_MANIFEST_MAX_CLOCK_SKEW_MS) { - fail( - "PROVENANCE_MISMATCH", - "imageCreationTimestamp is later than manifestCreatedAt beyond allowed clock skew", - ); - } - if ( - manifest.result === "reused" && - manifestCreated - imageCreated > EXACT_IMAGE_MANIFEST_MAX_REUSED_AGE_MS - ) { - fail("PROVENANCE_MISMATCH", "reused image is older than 24 hours"); - } -} - -function buildManifest(record: Record): ExactImageManifest { - requireConstant(record.schemaVersion, 1, "schemaVersion"); - requireConstant(record.kind, EXACT_IMAGE_MANIFEST_KIND, "kind"); - requireConstant( - record.requesterRepository, - EXACT_IMAGE_REQUESTER_REPOSITORY, - "requesterRepository", - ); - requireConstant(record.imageRepository, EXACT_IMAGE_REPOSITORY, "imageRepository"); - requireConstant(record.producerWorkflow, EXACT_IMAGE_PRODUCER_WORKFLOW, "producerWorkflow"); - requireConstant(record.imageKind, "compute#image", "imageKind"); - requireConstant(record.status, "READY", "status"); - requireConstant(record.channel, "staging", "channel"); - - const variant = requireString(record, "variant"); - if (variant !== "cpu") { - fail("UNSUPPORTED_VARIANT", 'variant must equal "cpu"'); - } - requireConstant(record.observedFamily, EXACT_IMAGE_STAGING_FAMILY, "observedFamily"); - - const result = requireString(record, "result"); - if (result !== "built" && result !== "reused") { - fail("ARTIFACT_MISSING_OR_INVALID", 'result must equal "built" or "reused"'); - } - - const manifest: ExactImageManifest = { - schemaVersion: 1, - kind: EXACT_IMAGE_MANIFEST_KIND, - correlationId: requireString(record, "correlationId"), - requesterRepository: EXACT_IMAGE_REQUESTER_REPOSITORY, - requesterWorkflowRunId: requireString(record, "requesterWorkflowRunId"), - requesterWorkflowRunAttempt: requirePositiveInteger(record, "requesterWorkflowRunAttempt"), - nemoclawSha: requireString(record, "nemoclawSha"), - imageRepository: EXACT_IMAGE_REPOSITORY, - imageRepositorySha: requireString(record, "imageRepositorySha"), - producerWorkflow: EXACT_IMAGE_PRODUCER_WORKFLOW, - workflowRunId: requireString(record, "workflowRunId"), - workflowRunAttempt: requirePositiveInteger(record, "workflowRunAttempt"), - imageOriginWorkflowRunId: requireString(record, "imageOriginWorkflowRunId"), - imageOriginWorkflowRunAttempt: requirePositiveInteger(record, "imageOriginWorkflowRunAttempt"), - imageKind: "compute#image", - project: requireString(record, "project"), - imageName: requireString(record, "imageName"), - imageId: requireString(record, "imageId"), - imageSelfLink: requireString(record, "imageSelfLink"), - status: "READY", - imageCreationTimestamp: requireString(record, "imageCreationTimestamp"), - manifestCreatedAt: requireString(record, "manifestCreatedAt"), - channel: "staging", - variant, - observedFamily: EXACT_IMAGE_STAGING_FAMILY, - result, - }; - - requirePattern(manifest.correlationId, "correlationId", UUID_V4_PATTERN); - requirePattern(manifest.requesterWorkflowRunId, "requesterWorkflowRunId", DECIMAL_ID_PATTERN); - requirePattern(manifest.nemoclawSha, "nemoclawSha", FULL_SHA_PATTERN); - requirePattern(manifest.imageRepositorySha, "imageRepositorySha", FULL_SHA_PATTERN); - requirePattern(manifest.workflowRunId, "workflowRunId", DECIMAL_ID_PATTERN); - requirePattern(manifest.imageOriginWorkflowRunId, "imageOriginWorkflowRunId", DECIMAL_ID_PATTERN); - if (!GCP_PROJECT_ID_PATTERN.test(manifest.project)) { - fail("IMAGE_IDENTITY_MISMATCH", "project must be a canonical GCP project ID"); - } - requirePattern(manifest.imageName, "imageName", IMAGE_NAME_PATTERN); - requirePattern(manifest.imageId, "imageId", DECIMAL_ID_PATTERN); - return manifest; -} - -function validateImageSelfLink(manifest: ExactImageManifest): void { - const expectedPath = `/compute/v1/projects/${manifest.project}/global/images/${manifest.imageName}`; - const expectedSelfLink = `https://www.googleapis.com${expectedPath}`; - let parsed: URL; - try { - parsed = new URL(manifest.imageSelfLink); - } catch { - fail("IMAGE_IDENTITY_MISMATCH", "imageSelfLink must be an absolute URL"); - } - if ( - parsed.protocol !== "https:" || - parsed.hostname !== "www.googleapis.com" || - parsed.port !== "" || - parsed.username !== "" || - parsed.password !== "" || - parsed.search !== "" || - parsed.hash !== "" || - parsed.pathname !== expectedPath || - manifest.imageSelfLink !== expectedSelfLink - ) { - fail("IMAGE_IDENTITY_MISMATCH", "imageSelfLink does not exactly identify project/imageName"); - } -} - -export function parseExactImageManifestJson(contents: string): unknown { - try { - return JSON.parse(contents) as unknown; - } catch { - fail("ARTIFACT_MISSING_OR_INVALID", "manifest is not valid JSON"); - } -} - -export function validateExactImageManifest( - value: unknown, - expected: ExactImageManifestExpectations, -): ExactImageManifest { - validateExpectations(expected); - const record = requireRecord(value); - validateExactFields(record); - const manifest = buildManifest(record); - - validateImageSelfLink(manifest); - if ( - manifest.result === "built" && - (manifest.imageOriginWorkflowRunId !== manifest.workflowRunId || - manifest.imageOriginWorkflowRunAttempt !== manifest.workflowRunAttempt) - ) { - fail( - "PROVENANCE_MISMATCH", - "built image origin run and attempt must match the current producer run", - ); - } - - validateTemporalContract(manifest); - assertExpected(manifest, expected); - return manifest; -} - -export function parseAndValidateExactImageManifest( - contents: string, - expected: ExactImageManifestExpectations, -): ExactImageManifest { - return validateExactImageManifest(parseExactImageManifestJson(contents), expected); -} - -export function normalizedExactImageManifestJson(manifest: ExactImageManifest): string { - return `${JSON.stringify(manifest, null, 2)}\n`; -} - -export function exactImageManifestFailureCode(error: unknown): ExactImageManifestFailureCode { - return error instanceof ExactImageManifestError ? error.code : "UNKNOWN"; -} diff --git a/tools/e2e/exact-image-qualification-controller.mts b/tools/e2e/exact-image-qualification-controller.mts deleted file mode 100755 index aeb4cdf62e..0000000000 --- a/tools/e2e/exact-image-qualification-controller.mts +++ /dev/null @@ -1,2076 +0,0 @@ -#!/usr/bin/env node - -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { spawnSync } from "node:child_process"; -import { createHash, randomUUID } from "node:crypto"; -import fs from "node:fs"; -import path from "node:path"; -import { pathToFileURL } from "node:url"; - -import { type GitHubRequestOptions, githubApi } from "../advisors/github.mts"; -import * as privateFile from "./private-file.mts"; - -// Root .ts files are exposed as CommonJS under tsx and as ESM under Node's -// strip-types runtime. Normalize both forms so the workflow uses the same -// no-follow state-file helpers in either test or production execution. -const privateFileRuntime = ( - "default" in privateFile && privateFile.default ? privateFile.default : privateFile -) as typeof import("./private-file.mts"); - -export const REQUESTER_REPOSITORY = "NVIDIA/NemoClaw"; -export const PRODUCER_REPOSITORY = "brevdev/nemoclaw-image"; -export const PRODUCER_WORKFLOW_FILE = "build-qualification-image.yml"; -export const PRODUCER_WORKFLOW_PATH = `.github/workflows/${PRODUCER_WORKFLOW_FILE}`; -export const PRODUCER_REF = "main"; -export const GITHUB_API_VERSION = "2026-03-10"; -export const MANIFEST_ARTIFACT_FILE = "nemoclaw-image-manifest.v1.json"; -export const ARCHIVE_FILE = "nemoclaw-image-handoff.zip"; -export const VALIDATED_MANIFEST_FILE = "validated-manifest.v1.json"; -export const EVIDENCE_FILE = "qualification-evidence.v1.json"; -export const STATE_FILE = "controller-state.json"; -export const DISPATCH_INTENT_FILE = "dispatch-intent.v1.json"; -export const DISPATCH_RECONCILIATION_FILE = "dispatch-reconciliation.v1.json"; - -const FULL_SHA_PATTERN = /^[0-9a-f]{40}$/u; -const UUID_V4_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/u; -const DECIMAL_ID_PATTERN = /^[1-9][0-9]*$/u; -const ARTIFACT_DIGEST_PATTERN = /^sha256:[0-9a-f]{64}$/u; -const SHA256_PATTERN = /^[0-9a-f]{64}$/u; -const GITHUB_LOGIN_PATTERN = /^(?!-)[A-Za-z0-9-]{1,39}(?( - apiPath: string, - token: string, - options?: GitHubRequestOptions, -) => Promise; - -type CommandResult = { - status: number | null; - signal?: NodeJS.Signals | null; - stdout: Buffer | string; - stderr: Buffer | string; - error?: Error; -}; - -type CommandRunner = (command: string, args: readonly string[]) => CommandResult; - -export type QualificationDependencies = { - api?: GitHubApiClient; - fetch?: typeof fetch; - now?: () => number; - randomUuid?: () => string; - runCommand?: CommandRunner; - sleep?: (milliseconds: number) => Promise; - limits?: Partial<{ - artifactPropagationTimeoutMs: number; - apiRequestTimeoutMs: number; - cancelReserveMs: number; - cleanupTimeoutMs: number; - dispatchReconciliationTimeoutMs: number; - downloadTimeoutMs: number; - pollIntervalMs: number; - qualificationTimeoutMs: number; - queueTimeoutMs: number; - reconciliationPollIntervalMs: number; - warningAfterMs: number; - }>; -}; - -export type ExactImageQualificationRequest = { - actor: string; - candidateSha: string; - eventName: string; - reason: string; - ref: string; - requesterRunAttempt: number; - requesterRunId: string; - workflowSha: string; -}; - -export type DispatchDetails = { - workflowRunId: string; - runUrl: string; - htmlUrl: string; -}; - -export type ExactImageDispatchIntent = { - schemaVersion: 1; - kind: "nemoclaw-exact-image-dispatch-intent"; - requestStartedAt: string; - request: { - actor: string; - candidateSha: string; - correlationId: string; - reason: string; - requesterRunAttempt: number; - requesterRunId: string; - workflowSha: string; - }; - producer: { - repository: typeof PRODUCER_REPOSITORY; - repositorySha: string; - ref: typeof PRODUCER_REF; - workflowId: string; - workflowPath: typeof PRODUCER_WORKFLOW_PATH; - }; -}; - -export type QualificationWorkflowRun = { - id: string; - workflowId: string; - headSha: string; - runAttempt: number; - status: string; - conclusion: string | null; - url: string; - htmlUrl: string; -}; - -export type QualificationArtifact = { - id: string; - name: string; - digest: string; - sizeInBytes: number; - apiUrl: string; - archiveDownloadUrl: string; -}; - -export type ExactImageQualificationState = { - schemaVersion: 1; - status: "dispatched" | "completed" | "downloaded" | "validated"; - dispatchedAt: string; - request: { - actor: string; - candidateSha: string; - correlationId: string; - reason: string; - requesterRunAttempt: number; - requesterRunId: string; - workflowSha: string; - }; - producer: { - repository: typeof PRODUCER_REPOSITORY; - repositorySha: string; - ref: typeof PRODUCER_REF; - workflowPath: typeof PRODUCER_WORKFLOW_PATH; - runId: string; - runAttempt: 1; - workflowId?: string; - runUrl: string; - htmlUrl: string; - completedAt?: string; - }; - artifact?: QualificationArtifact & { - archiveSha256: string; - manifestSha256: string; - }; - validation?: { - acceptedAt: string; - manifestSha256: string; - normalizedManifestSha256: string; - }; -}; - -type ControllerCommand = - | { mode: "preflight"; request: ExactImageQualificationRequest } - | { - mode: "start"; - request: ExactImageQualificationRequest; - workDir: string; - } - | { mode: "wait" | "download" | "finalize" | "cancel"; workDir: string }; - -function fail(code: ExactImageQualificationFailureCode, message: string): never { - throw new ExactImageQualificationError(code, message); -} - -function record(value: unknown, label: string): Record { - if (value === null || typeof value !== "object" || Array.isArray(value)) { - fail("PROVENANCE_MISMATCH", `${label} must be an object`); - } - return value as Record; -} - -function requireExactRecordFields( - value: Record, - fields: readonly string[], - label: string, -): void { - const allowed = new Set(fields); - for (const field of fields) { - if (!Object.hasOwn(value, field)) { - fail("PROVENANCE_MISMATCH", `${label} is missing ${field}`); - } - } - const unexpected = Object.keys(value) - .filter((field) => !allowed.has(field)) - .sort(); - if (unexpected.length > 0) { - fail("PROVENANCE_MISMATCH", `${label} contains unexpected field ${unexpected[0]}`); - } -} - -function persistedString( - value: Record, - field: string, - label: string, - pattern?: RegExp, -): string { - const result = value[field]; - if (typeof result !== "string" || (pattern !== undefined && !pattern.test(result))) { - fail("PROVENANCE_MISMATCH", `${label} has an invalid format`); - } - return result; -} - -function persistedTimestamp(value: unknown, label: string): number { - if (typeof value !== "string") { - fail("PROVENANCE_MISMATCH", `${label} must be a timestamp string`); - } - const parsed = Date.parse(value); - if (!Number.isFinite(parsed) || new Date(parsed).toISOString() !== value) { - fail("PROVENANCE_MISMATCH", `${label} must be a canonical UTC timestamp`); - } - return parsed; -} - -function validatePersistedReason(value: unknown, label: string): void { - if ( - typeof value !== "string" || - value.length === 0 || - value !== value.trim() || - Buffer.byteLength(value, "utf8") > MAX_REASON_BYTES || - /[\u0000-\u001f\u007f]/u.test(value) - ) { - fail("PROVENANCE_MISMATCH", `${label} is invalid`); - } -} - -function safePositiveId(value: unknown, label: string): string { - if (!Number.isSafeInteger(value) || (value as number) < 1) { - fail("PROVENANCE_MISMATCH", `${label} must be a positive safe integer`); - } - return String(value); -} - -function positiveInteger(value: string, label: string): number { - if (!DECIMAL_ID_PATTERN.test(value)) fail("REQUEST_INVALID", `${label} must be positive`); - const parsed = Number(value); - if (!Number.isSafeInteger(parsed)) fail("REQUEST_INVALID", `${label} is outside the safe range`); - return parsed; -} - -function requireExactString(value: unknown, expected: string, label: string): void { - if (value !== expected) fail("PROVENANCE_MISMATCH", `${label} did not match the trusted request`); -} - -function sha256(bytes: Uint8Array): string { - return createHash("sha256").update(bytes).digest("hex"); -} - -function timestamp(now: () => number): string { - return new Date(now()).toISOString(); -} - -function sleep(milliseconds: number): Promise { - return new Promise((resolve) => setTimeout(resolve, milliseconds)); -} - -function limits(dependencies: QualificationDependencies) { - return { - artifactPropagationTimeoutMs: - dependencies.limits?.artifactPropagationTimeoutMs ?? ARTIFACT_PROPAGATION_TIMEOUT_MS, - apiRequestTimeoutMs: dependencies.limits?.apiRequestTimeoutMs ?? API_REQUEST_TIMEOUT_MS, - cancelReserveMs: dependencies.limits?.cancelReserveMs ?? CANCEL_RESERVE_MS, - cleanupTimeoutMs: dependencies.limits?.cleanupTimeoutMs ?? CLEANUP_TIMEOUT_MS, - dispatchReconciliationTimeoutMs: - dependencies.limits?.dispatchReconciliationTimeoutMs ?? DISPATCH_RECONCILIATION_TIMEOUT_MS, - downloadTimeoutMs: dependencies.limits?.downloadTimeoutMs ?? DOWNLOAD_TIMEOUT_MS, - pollIntervalMs: dependencies.limits?.pollIntervalMs ?? POLL_INTERVAL_MS, - qualificationTimeoutMs: dependencies.limits?.qualificationTimeoutMs ?? QUALIFICATION_TIMEOUT_MS, - queueTimeoutMs: dependencies.limits?.queueTimeoutMs ?? QUEUE_TIMEOUT_MS, - reconciliationPollIntervalMs: - dependencies.limits?.reconciliationPollIntervalMs ?? RECONCILIATION_POLL_INTERVAL_MS, - warningAfterMs: dependencies.limits?.warningAfterMs ?? WARNING_AFTER_MS, - }; -} - -function qualificationDeadline( - state: ExactImageQualificationState, - qualificationTimeoutMs: number, -): number { - const dispatchedAt = Date.parse(state.dispatchedAt); - if (!Number.isFinite(dispatchedAt)) { - fail("PROVENANCE_MISMATCH", "controller state has an invalid dispatch timestamp"); - } - return dispatchedAt + qualificationTimeoutMs; -} - -function requireQualificationTimeRemaining(deadline: number, now: () => number): number { - const remaining = deadline - now(); - if (remaining <= 0) { - fail("QUALIFICATION_TIMEOUT", "qualification exceeded the shared 45-minute budget"); - } - return remaining; -} - -function boundedApiClient( - api: GitHubApiClient, - dependencies: QualificationDependencies, - deadline?: number, -): GitHubApiClient { - const now = dependencies.now ?? Date.now; - const requestCap = limits(dependencies).apiRequestTimeoutMs; - return async (apiPath: string, token: string, options: GitHubRequestOptions = {}) => { - const remaining = deadline === undefined ? requestCap : deadline - now(); - if (remaining <= 0) { - fail("QUALIFICATION_TIMEOUT", "no time remains for the bounded GitHub API request"); - } - const controller = new AbortController(); - const timeout = Math.min(requestCap, remaining); - const timer = setTimeout(() => controller.abort(), timeout); - timer.unref?.(); - const signal = options.signal - ? AbortSignal.any([options.signal, controller.signal]) - : controller.signal; - try { - return await api(apiPath, token, { ...options, signal }); - } catch (error) { - if (controller.signal.aborted) { - fail("QUALIFICATION_TIMEOUT", `GitHub API request exceeded its ${timeout}ms budget`); - } - throw error; - } finally { - clearTimeout(timer); - } - }; -} - -function acceptanceDeadline( - state: ExactImageQualificationState, - dependencies: QualificationDependencies, -): number { - const configured = limits(dependencies); - return ( - qualificationDeadline(state, configured.qualificationTimeoutMs) - configured.cancelReserveMs - ); -} - -export function validateExactImageQualificationRequest( - request: ExactImageQualificationRequest, -): ExactImageQualificationRequest { - if (request.eventName !== "workflow_dispatch" && request.eventName !== "schedule") { - fail("REQUEST_INVALID", "qualification must be started by workflow_dispatch or schedule"); - } - if (request.ref !== "refs/heads/main") { - fail("REQUEST_INVALID", "qualification must run from the trusted main branch"); - } - if (!Number.isSafeInteger(request.requesterRunAttempt) || request.requesterRunAttempt < 1) { - fail("REQUEST_INVALID", "requester run attempt must be a positive integer"); - } - if (!FULL_SHA_PATTERN.test(request.candidateSha)) { - fail("REQUEST_INVALID", "candidate SHA must be a lowercase full commit SHA"); - } - if (!FULL_SHA_PATTERN.test(request.workflowSha)) { - fail("REQUEST_INVALID", "workflow SHA must be a lowercase full commit SHA"); - } - if (request.candidateSha !== request.workflowSha) { - fail("REQUEST_INVALID", "candidate SHA must equal the trusted main workflow SHA"); - } - if (!DECIMAL_ID_PATTERN.test(request.requesterRunId)) { - fail("REQUEST_INVALID", "requester run ID must be a positive decimal string"); - } - if (!GITHUB_LOGIN_PATTERN.test(request.actor)) { - fail("REQUEST_INVALID", "triggering actor has an invalid GitHub login"); - } - if ( - request.reason.length === 0 || - request.reason !== request.reason.trim() || - Buffer.byteLength(request.reason, "utf8") > MAX_REASON_BYTES || - /[\u0000-\u001f\u007f]/u.test(request.reason) - ) { - fail("REQUEST_INVALID", "reason must be trimmed, nonempty, bounded text without controls"); - } - return request; -} - -function validateRequesterRef(value: unknown, repository: string, expectedRef: string): string { - const payload = record(value, `${repository} requester ref`); - requireExactString(payload.ref, expectedRef, `${repository} ref`); - const object = record(payload.object, `${repository} ref object`); - requireExactString(object.type, "commit", `${repository} ref object type`); - if (typeof object.sha !== "string" || !FULL_SHA_PATTERN.test(object.sha)) { - fail("PROVENANCE_MISMATCH", `${repository} main did not resolve to a full commit SHA`); - } - return object.sha; -} - -function validateMainRef(value: unknown, repository: string): string { - return validateRequesterRef(value, repository, "refs/heads/main"); -} - -function validateProducerWorkflow(value: unknown): string { - const workflow = record(value, "producer workflow"); - const workflowId = safePositiveId(workflow.id, "producer workflow ID"); - requireExactString(workflow.path, PRODUCER_WORKFLOW_PATH, "producer workflow path"); - requireExactString(workflow.state, "active", "producer workflow state"); - return workflowId; -} - -async function authorizeRequest( - request: ExactImageQualificationRequest, - token: string, - api: GitHubApiClient, -): Promise { - validateExactImageQualificationRequest(request); - const branch = await api(`repos/${REQUESTER_REPOSITORY}/git/ref/heads/main`, token); - if (validateMainRef(branch, REQUESTER_REPOSITORY) !== request.workflowSha) { - fail("DISPATCH_FORBIDDEN", "workflow SHA is no longer the trusted main branch head"); - } - const candidate = record( - await api(`repos/${REQUESTER_REPOSITORY}/git/commits/${request.candidateSha}`, token), - "candidate commit", - ); - if (candidate.sha !== request.candidateSha) { - fail("DISPATCH_FORBIDDEN", "candidate SHA does not identify an exact NemoClaw commit"); - } - const permission = record( - await api( - `repos/${REQUESTER_REPOSITORY}/collaborators/${encodeURIComponent(request.actor)}/permission`, - token, - ), - "collaborator permission", - ); - if (permission.permission !== "admin" && permission.role_name !== "maintain") { - fail("DISPATCH_FORBIDDEN", "triggering actor must have maintain or admin permission"); - } -} - -export async function preflightExactImageQualification( - request: ExactImageQualificationRequest, - token: string, - dependencies: QualificationDependencies = {}, -): Promise { - await authorizeRequest( - request, - token, - boundedApiClient(dependencies.api ?? githubApi, dependencies), - ); -} - -export function validateWorkflowDispatchDetails(value: unknown): DispatchDetails { - const payload = record(value, "workflow dispatch response"); - const workflowRunId = safePositiveId(payload.workflow_run_id, "workflow_run_id"); - const runUrl = `https://api.github.com/repos/${PRODUCER_REPOSITORY}/actions/runs/${workflowRunId}`; - const htmlUrl = `https://github.com/${PRODUCER_REPOSITORY}/actions/runs/${workflowRunId}`; - requireExactString(payload.run_url, runUrl, "dispatch run_url"); - requireExactString(payload.html_url, htmlUrl, "dispatch html_url"); - return { workflowRunId, runUrl, htmlUrl }; -} - -export function validateQualificationWorkflowRun( - value: unknown, - expected: { - candidateSha: string; - correlationId: string; - producerSha: string; - runId: string; - runUrl: string; - htmlUrl: string; - workflowId?: string; - }, -): QualificationWorkflowRun { - const run = record(value, "producer workflow run"); - const id = safePositiveId(run.id, "producer run ID"); - if (id !== expected.runId) fail("PROVENANCE_MISMATCH", "producer run ID changed"); - const workflowId = safePositiveId(run.workflow_id, "producer workflow ID"); - if (expected.workflowId !== undefined && workflowId !== expected.workflowId) { - fail("PROVENANCE_MISMATCH", "producer workflow ID did not match the dispatch intent"); - } - if (run.run_attempt !== 1) fail("PROVENANCE_MISMATCH", "producer run attempt must equal 1"); - requireExactString(run.event, "workflow_dispatch", "producer event"); - requireExactString(run.head_branch, PRODUCER_REF, "producer head branch"); - requireExactString(run.head_sha, expected.producerSha, "producer head SHA"); - requireExactString(run.path, PRODUCER_WORKFLOW_PATH, "producer workflow path"); - requireExactString( - run.display_title, - `Qualify NemoClaw ${expected.candidateSha} (${expected.correlationId})`, - "producer display title", - ); - requireExactString(run.url, expected.runUrl, "producer run URL"); - requireExactString(run.html_url, expected.htmlUrl, "producer HTML URL"); - requireExactString( - record(run.repository, "producer repository").full_name, - PRODUCER_REPOSITORY, - "producer repository", - ); - requireExactString( - record(run.head_repository, "producer head repository").full_name, - PRODUCER_REPOSITORY, - "producer head repository", - ); - const allowedStatuses = new Set([ - "queued", - "in_progress", - "completed", - "waiting", - "requested", - "pending", - ]); - if (typeof run.status !== "string" || !allowedStatuses.has(run.status)) { - fail("PROVENANCE_MISMATCH", "producer run has an unsupported status"); - } - if (run.conclusion !== null && typeof run.conclusion !== "string") { - fail("PROVENANCE_MISMATCH", "producer conclusion must be a string or null"); - } - return { - id, - workflowId, - headSha: expected.producerSha, - runAttempt: 1, - status: run.status, - conclusion: run.conclusion as string | null, - url: expected.runUrl, - htmlUrl: expected.htmlUrl, - }; -} - -function validateCancellationWorkflowRun( - value: unknown, - state: ExactImageQualificationState, -): { status: string; conclusion: string | null } { - const run = record(value, "producer cancellation workflow run"); - const observedHeadSha = persistedString( - run, - "head_sha", - "producer cancellation head SHA", - FULL_SHA_PATTERN, - ); - const validated = validateQualificationWorkflowRun(value, { - ...expectedRun(state), - // A run whose producer ref moved after dispatch remains cleanup-owned but - // can never become qualification evidence; all other identity must bind. - producerSha: observedHeadSha, - }); - return { status: validated.status, conclusion: validated.conclusion }; -} - -function statePath(workDir: string): string { - return path.join(workDir, STATE_FILE); -} - -function intentPath(workDir: string): string { - return path.join(workDir, DISPATCH_INTENT_FILE); -} - -function writeDispatchIntent(workDir: string, intent: ExactImageDispatchIntent): void { - try { - privateFileRuntime.writePrivateRegularFile( - intentPath(workDir), - `${JSON.stringify(intent, null, 2)}\n`, - ); - } catch { - fail("OUTPUT_WRITE_FAILED", "dispatch intent could not be written safely"); - } -} - -function readDispatchIntent(workDir: string): ExactImageDispatchIntent | null { - let contents: string | null; - try { - contents = privateFileRuntime.readPrivateRegularFile(intentPath(workDir), { - allowMissing: true, - maxBytes: MAX_STATE_BYTES, - }); - } catch { - fail("PROVENANCE_MISMATCH", "dispatch intent could not be read safely"); - } - if (contents === null) return null; - let parsed: unknown; - try { - parsed = JSON.parse(contents) as unknown; - } catch { - fail("PROVENANCE_MISMATCH", "dispatch intent is not valid JSON"); - } - const intent = record(parsed, "dispatch intent"); - requireExactRecordFields( - intent, - ["schemaVersion", "kind", "requestStartedAt", "request", "producer"], - "dispatch intent", - ); - if (intent.schemaVersion !== 1 || intent.kind !== "nemoclaw-exact-image-dispatch-intent") { - fail("PROVENANCE_MISMATCH", "dispatch intent schema identity is invalid"); - } - persistedTimestamp(intent.requestStartedAt, "dispatch intent requestStartedAt"); - - const request = record(intent.request, "dispatch intent request"); - requireExactRecordFields( - request, - [ - "actor", - "candidateSha", - "correlationId", - "reason", - "requesterRunAttempt", - "requesterRunId", - "workflowSha", - ], - "dispatch intent request", - ); - persistedString(request, "actor", "dispatch intent actor", GITHUB_LOGIN_PATTERN); - persistedString(request, "candidateSha", "dispatch intent candidate SHA", FULL_SHA_PATTERN); - persistedString(request, "correlationId", "dispatch intent correlation ID", UUID_V4_PATTERN); - validatePersistedReason(request.reason, "dispatch intent reason"); - const requesterRunAttempt = request.requesterRunAttempt; - if ( - typeof requesterRunAttempt !== "number" || - !Number.isSafeInteger(requesterRunAttempt) || - requesterRunAttempt < 1 - ) { - fail("PROVENANCE_MISMATCH", "dispatch intent requester run attempt must be positive"); - } - persistedString( - request, - "requesterRunId", - "dispatch intent requester run ID", - DECIMAL_ID_PATTERN, - ); - persistedString(request, "workflowSha", "dispatch intent workflow SHA", FULL_SHA_PATTERN); - - const producer = record(intent.producer, "dispatch intent producer"); - requireExactRecordFields( - producer, - ["repository", "repositorySha", "ref", "workflowId", "workflowPath"], - "dispatch intent producer", - ); - requireExactString(producer.repository, PRODUCER_REPOSITORY, "dispatch intent repository"); - persistedString(producer, "repositorySha", "dispatch intent producer SHA", FULL_SHA_PATTERN); - requireExactString(producer.ref, PRODUCER_REF, "dispatch intent producer ref"); - persistedString(producer, "workflowId", "dispatch intent workflow ID", DECIMAL_ID_PATTERN); - requireExactString( - producer.workflowPath, - PRODUCER_WORKFLOW_PATH, - "dispatch intent workflow path", - ); - return intent as unknown as ExactImageDispatchIntent; -} - -function requiredDispatchIntent(workDir: string): ExactImageDispatchIntent { - const intent = readDispatchIntent(workDir); - if (intent === null) fail("PROVENANCE_MISMATCH", "dispatch intent is missing"); - return intent; -} - -function validateStateIntentBinding( - state: ExactImageQualificationState, - intent: ExactImageDispatchIntent, -): void { - const comparisons: Array<[unknown, unknown, string]> = [ - [state.dispatchedAt, intent.requestStartedAt, "dispatch timestamp"], - [state.request.actor, intent.request.actor, "actor"], - [state.request.candidateSha, intent.request.candidateSha, "candidate SHA"], - [state.request.correlationId, intent.request.correlationId, "correlation ID"], - [state.request.reason, intent.request.reason, "reason"], - [ - state.request.requesterRunAttempt, - intent.request.requesterRunAttempt, - "requester run attempt", - ], - [state.request.requesterRunId, intent.request.requesterRunId, "requester run ID"], - [state.request.workflowSha, intent.request.workflowSha, "workflow SHA"], - [state.producer.repository, intent.producer.repository, "producer repository"], - [state.producer.repositorySha, intent.producer.repositorySha, "producer SHA"], - [state.producer.ref, intent.producer.ref, "producer ref"], - [state.producer.workflowId, intent.producer.workflowId, "producer workflow ID"], - [state.producer.workflowPath, intent.producer.workflowPath, "producer workflow path"], - ]; - for (const [actual, expected, label] of comparisons) { - if (actual !== expected) { - fail("PROVENANCE_MISMATCH", `controller state ${label} does not match dispatch intent`); - } - } -} - -function readBoundExactImageQualificationState(workDir: string): ExactImageQualificationState { - const state = readExactImageQualificationState(workDir); - validateStateIntentBinding(state, requiredDispatchIntent(workDir)); - return state; -} - -function writeDispatchReconciliation( - workDir: string, - intent: ExactImageDispatchIntent, - runs: readonly QualificationWorkflowRun[], - outcome: "none" | "recovered-one" | "multiple", - now: () => number, -): void { - const evidence = { - schemaVersion: 1, - kind: "nemoclaw-exact-image-dispatch-reconciliation", - recordedAt: timestamp(now), - outcome, - correlationId: intent.request.correlationId, - runIds: runs.map((run) => run.id), - producerHeadShas: Object.fromEntries(runs.map((run) => [run.id, run.headSha])), - }; - try { - privateFileRuntime.writePrivateRegularFile( - path.join(workDir, DISPATCH_RECONCILIATION_FILE), - `${JSON.stringify(evidence, null, 2)}\n`, - ); - } catch { - fail("OUTPUT_WRITE_FAILED", "dispatch reconciliation evidence could not be written safely"); - } -} - -function writeAtomicPrivateRegularFile(file: string, contents: string): void { - const temporary = `${file}.${process.pid}.${randomUUID()}.tmp`; - try { - privateFileRuntime.writePrivateRegularFile(temporary, contents); - if (fs.existsSync(file)) { - const current = fs.lstatSync(file); - if (!current.isFile() || current.isSymbolicLink() || current.nlink !== 1) { - fail("OUTPUT_WRITE_FAILED", "existing controller state is not one regular file"); - } - } - fs.renameSync(temporary, file); - } finally { - try { - fs.rmSync(temporary, { force: true }); - } catch { - // A successfully renamed temporary path no longer exists; a failed - // cleanup must not hide the original atomic-write error. - } - } -} - -type PersistedQualificationStatus = ExactImageQualificationState["status"]; - -function validatePersistedRequest(value: unknown): void { - const request = record(value, "controller state request"); - requireExactRecordFields( - request, - [ - "actor", - "candidateSha", - "correlationId", - "reason", - "requesterRunAttempt", - "requesterRunId", - "workflowSha", - ], - "controller state request", - ); - persistedString(request, "actor", "controller state actor", GITHUB_LOGIN_PATTERN); - persistedString(request, "candidateSha", "controller state candidate SHA", FULL_SHA_PATTERN); - persistedString(request, "correlationId", "controller state correlation ID", UUID_V4_PATTERN); - validatePersistedReason(request.reason, "controller state reason"); - const requesterRunAttempt = request.requesterRunAttempt; - if ( - typeof requesterRunAttempt !== "number" || - !Number.isSafeInteger(requesterRunAttempt) || - requesterRunAttempt < 1 - ) { - fail("PROVENANCE_MISMATCH", "controller state requester run attempt must be positive"); - } - persistedString( - request, - "requesterRunId", - "controller state requester run ID", - DECIMAL_ID_PATTERN, - ); - persistedString(request, "workflowSha", "controller state workflow SHA", FULL_SHA_PATTERN); -} - -function validatePersistedProducer( - value: unknown, - status: PersistedQualificationStatus, - dispatchedAt: number, -): { completedAt: number | null; runId: string } { - const producer = record(value, "controller state producer"); - const completed = status !== "dispatched"; - requireExactRecordFields( - producer, - [ - "repository", - "repositorySha", - "ref", - "workflowPath", - "runId", - "runAttempt", - "workflowId", - "runUrl", - "htmlUrl", - ...(completed ? ["completedAt"] : []), - ], - "controller state producer", - ); - requireExactString(producer.repository, PRODUCER_REPOSITORY, "controller state repository"); - persistedString(producer, "repositorySha", "controller state producer SHA", FULL_SHA_PATTERN); - requireExactString(producer.ref, PRODUCER_REF, "controller state producer ref"); - requireExactString( - producer.workflowPath, - PRODUCER_WORKFLOW_PATH, - "controller state workflow path", - ); - const runId = persistedString( - producer, - "runId", - "controller state producer run ID", - DECIMAL_ID_PATTERN, - ); - if (producer.runAttempt !== 1) { - fail("PROVENANCE_MISMATCH", "controller state producer run attempt must equal 1"); - } - persistedString( - producer, - "workflowId", - "controller state producer workflow ID", - DECIMAL_ID_PATTERN, - ); - const expectedUrls = canonicalDispatchDetails(runId); - requireExactString(producer.runUrl, expectedUrls.runUrl, "controller state producer run URL"); - requireExactString(producer.htmlUrl, expectedUrls.htmlUrl, "controller state producer HTML URL"); - const completedAt = completed - ? persistedTimestamp(producer.completedAt, "controller state completedAt") - : null; - if (completedAt !== null && completedAt < dispatchedAt) { - fail("PROVENANCE_MISMATCH", "controller state completedAt precedes dispatchedAt"); - } - if (completedAt !== null && completedAt > dispatchedAt + QUALIFICATION_TIMEOUT_MS) { - fail("PROVENANCE_MISMATCH", "controller state completedAt exceeds qualification deadline"); - } - return { completedAt, runId }; -} - -function validatePersistedArtifact(value: unknown, runId: string): string { - const artifact = record(value, "controller state artifact"); - requireExactRecordFields( - artifact, - [ - "id", - "name", - "digest", - "sizeInBytes", - "apiUrl", - "archiveDownloadUrl", - "archiveSha256", - "manifestSha256", - ], - "controller state artifact", - ); - const id = persistedString(artifact, "id", "controller state artifact ID", DECIMAL_ID_PATTERN); - requireExactString( - artifact.name, - `nemoclaw-image-handoff-v1-${runId}-1`, - "controller state artifact name", - ); - const digest = persistedString( - artifact, - "digest", - "controller state artifact digest", - ARTIFACT_DIGEST_PATTERN, - ); - if ( - !Number.isSafeInteger(artifact.sizeInBytes) || - (artifact.sizeInBytes as number) < 1 || - (artifact.sizeInBytes as number) > MAX_ARCHIVE_BYTES - ) { - fail("PROVENANCE_MISMATCH", "controller state artifact size is invalid"); - } - const apiUrl = `https://api.github.com/repos/${PRODUCER_REPOSITORY}/actions/artifacts/${id}`; - requireExactString(artifact.apiUrl, apiUrl, "controller state artifact API URL"); - requireExactString( - artifact.archiveDownloadUrl, - `${apiUrl}/zip`, - "controller state artifact archive URL", - ); - const archiveSha256 = persistedString( - artifact, - "archiveSha256", - "controller state archive hash", - SHA256_PATTERN, - ); - if (digest !== `sha256:${archiveSha256}`) { - fail("PROVENANCE_MISMATCH", "controller state artifact digest does not match archive hash"); - } - return persistedString( - artifact, - "manifestSha256", - "controller state manifest hash", - SHA256_PATTERN, - ); -} - -function validatePersistedValidation( - value: unknown, - manifestSha256: string, - completedAt: number, - dispatchedAt: number, -): void { - const validation = record(value, "controller state validation"); - requireExactRecordFields( - validation, - ["acceptedAt", "manifestSha256", "normalizedManifestSha256"], - "controller state validation", - ); - const acceptedAt = persistedTimestamp(validation.acceptedAt, "controller state acceptedAt"); - if (acceptedAt < completedAt) { - fail("PROVENANCE_MISMATCH", "controller state acceptedAt precedes completedAt"); - } - if (acceptedAt > dispatchedAt + QUALIFICATION_TIMEOUT_MS) { - fail("PROVENANCE_MISMATCH", "controller state acceptedAt exceeds qualification deadline"); - } - requireExactString( - validation.manifestSha256, - manifestSha256, - "controller state accepted manifest hash", - ); - persistedString( - validation, - "normalizedManifestSha256", - "controller state normalized manifest hash", - SHA256_PATTERN, - ); -} - -function validatePersistedQualificationState(value: unknown): ExactImageQualificationState { - const state = record(value, "controller state"); - const statuses = new Set([ - "dispatched", - "completed", - "downloaded", - "validated", - ]); - if ( - typeof state.status !== "string" || - !statuses.has(state.status as PersistedQualificationStatus) - ) { - fail("PROVENANCE_MISMATCH", "controller state status is invalid"); - } - const status = state.status as PersistedQualificationStatus; - const hasArtifact = status === "downloaded" || status === "validated"; - requireExactRecordFields( - state, - [ - "schemaVersion", - "status", - "dispatchedAt", - "request", - "producer", - ...(hasArtifact ? ["artifact"] : []), - ...(status === "validated" ? ["validation"] : []), - ], - "controller state", - ); - if (state.schemaVersion !== 1) { - fail("PROVENANCE_MISMATCH", "controller state schema version must equal 1"); - } - const dispatchedAt = persistedTimestamp(state.dispatchedAt, "controller state dispatchedAt"); - validatePersistedRequest(state.request); - const producer = validatePersistedProducer(state.producer, status, dispatchedAt); - const manifestSha256 = hasArtifact - ? validatePersistedArtifact(state.artifact, producer.runId) - : null; - if (status === "validated") { - if (manifestSha256 === null || producer.completedAt === null) { - fail("PROVENANCE_MISMATCH", "validated controller state is incomplete"); - } - validatePersistedValidation( - state.validation, - manifestSha256, - producer.completedAt, - dispatchedAt, - ); - } - return state as unknown as ExactImageQualificationState; -} - -function writeState(workDir: string, state: ExactImageQualificationState): void { - try { - const validated = validatePersistedQualificationState(state); - writeAtomicPrivateRegularFile(statePath(workDir), `${JSON.stringify(validated, null, 2)}\n`); - } catch { - fail("OUTPUT_WRITE_FAILED", "controller state could not be written safely"); - } -} - -function preserveUnreadableState(workDir: string): void { - const file = statePath(workDir); - if (!fs.existsSync(file)) return; - const preserved = path.join( - workDir, - `controller-state.corrupt-${Date.now()}-${randomUUID()}.json`, - ); - try { - fs.renameSync(file, preserved); - } catch { - fail("OUTPUT_WRITE_FAILED", "unreadable controller state could not be preserved safely"); - } -} - -export function readExactImageQualificationState(workDir: string): ExactImageQualificationState { - let contents: string | null; - try { - contents = privateFileRuntime.readPrivateRegularFile(statePath(workDir), { - maxBytes: MAX_STATE_BYTES, - }); - } catch { - fail("PROVENANCE_MISMATCH", "controller state could not be read safely"); - } - if (contents === null) fail("PROVENANCE_MISMATCH", "controller state is missing"); - let parsed: unknown; - try { - parsed = JSON.parse(contents) as unknown; - } catch { - fail("PROVENANCE_MISMATCH", "controller state is not valid JSON"); - } - return validatePersistedQualificationState(parsed); -} - -function canonicalDispatchDetails(runId: string): DispatchDetails { - return { - workflowRunId: runId, - runUrl: `https://api.github.com/repos/${PRODUCER_REPOSITORY}/actions/runs/${runId}`, - htmlUrl: `https://github.com/${PRODUCER_REPOSITORY}/actions/runs/${runId}`, - }; -} - -function dispatchCreationWindow( - intent: ExactImageDispatchIntent, - configured: ReturnType, -): { earliest: number; latest: number } { - const startedAt = Date.parse(intent.requestStartedAt); - return { - earliest: startedAt - DISPATCH_CLOCK_SKEW_MS, - latest: startedAt + configured.apiRequestTimeoutMs + DISPATCH_CLOCK_SKEW_MS, - }; -} - -function reconciledWorkflowRuns( - value: unknown, - intent: ExactImageDispatchIntent, - configured: ReturnType, -): QualificationWorkflowRun[] { - const response = record(value, "workflow run reconciliation response"); - if (!Number.isSafeInteger(response.total_count) || (response.total_count as number) < 0) { - fail("DISPATCH_AMBIGUOUS", "workflow run reconciliation count is invalid"); - } - if (!Array.isArray(response.workflow_runs)) { - fail("DISPATCH_AMBIGUOUS", "workflow run reconciliation list is missing"); - } - if ((response.total_count as number) > response.workflow_runs.length) { - fail("DISPATCH_AMBIGUOUS", "workflow run reconciliation was truncated"); - } - const { earliest, latest } = dispatchCreationWindow(intent, configured); - const title = `Qualify NemoClaw ${intent.request.candidateSha} (${intent.request.correlationId})`; - const matches: QualificationWorkflowRun[] = []; - for (const candidate of response.workflow_runs) { - if (candidate === null || typeof candidate !== "object" || Array.isArray(candidate)) continue; - const run = candidate as Record; - const createdAt = typeof run.created_at === "string" ? Date.parse(run.created_at) : Number.NaN; - const id = Number.isSafeInteger(run.id) && (run.id as number) > 0 ? String(run.id) : ""; - const headSha = typeof run.head_sha === "string" ? run.head_sha : ""; - const details = id ? canonicalDispatchDetails(id) : null; - if ( - details === null || - run.workflow_id !== Number(intent.producer.workflowId) || - run.run_attempt !== 1 || - run.event !== "workflow_dispatch" || - run.head_branch !== PRODUCER_REF || - !FULL_SHA_PATTERN.test(headSha) || - run.path !== PRODUCER_WORKFLOW_PATH || - run.display_title !== title || - run.url !== details.runUrl || - run.html_url !== details.htmlUrl || - (run.repository as { full_name?: unknown } | undefined)?.full_name !== PRODUCER_REPOSITORY || - (run.head_repository as { full_name?: unknown } | undefined)?.full_name !== - PRODUCER_REPOSITORY || - !Number.isFinite(createdAt) || - createdAt < earliest || - createdAt > latest - ) { - continue; - } - matches.push( - validateQualificationWorkflowRun(candidate, { - candidateSha: intent.request.candidateSha, - correlationId: intent.request.correlationId, - producerSha: headSha, - workflowId: intent.producer.workflowId, - runId: id, - runUrl: details.runUrl, - htmlUrl: details.htmlUrl, - }), - ); - } - return matches; -} - -function stateFromIntent( - intent: ExactImageDispatchIntent, - run: QualificationWorkflowRun, -): ExactImageQualificationState { - return { - schemaVersion: 1, - status: "dispatched", - dispatchedAt: intent.requestStartedAt, - request: { - actor: intent.request.actor, - candidateSha: intent.request.candidateSha, - correlationId: intent.request.correlationId, - reason: intent.request.reason, - requesterRunAttempt: intent.request.requesterRunAttempt, - requesterRunId: intent.request.requesterRunId, - workflowSha: intent.request.workflowSha, - }, - producer: { - repository: PRODUCER_REPOSITORY, - repositorySha: run.headSha, - ref: PRODUCER_REF, - workflowPath: PRODUCER_WORKFLOW_PATH, - runId: run.id, - runAttempt: 1, - workflowId: run.workflowId, - runUrl: run.url, - htmlUrl: run.htmlUrl, - }, - }; -} - -async function cancelAndVerifyRecoveredRun( - state: ExactImageQualificationState, - initial: { status: string }, - token: string, - api: GitHubApiClient, - pause: (milliseconds: number) => Promise, - pollIntervalMs: number, - deadline: number, - now: () => number, -): Promise { - if (initial.status === "completed") return false; - let cancellationError: unknown; - try { - await cancelRun(state, token, api); - } catch (error) { - cancellationError = error; - } - for (;;) { - if (now() >= deadline) { - fail("QUALIFICATION_TIMEOUT", "recovered producer run cancellation was not verified in time"); - } - const run = validateCancellationWorkflowRun( - await api( - `repos/${PRODUCER_REPOSITORY}/actions/runs/${state.producer.runId}`, - token, - { apiVersion: GITHUB_API_VERSION, expectedStatus: 200 }, - ), - state, - ); - if (run.status === "completed") return true; - if (cancellationError !== undefined) throw cancellationError; - await pause(pollIntervalMs); - } -} - -async function reconcileAmbiguousDispatch( - options: { - intent: ExactImageDispatchIntent; - workDir: string; - producerToken: string; - mode: "start" | "cleanup"; - }, - dependencies: QualificationDependencies, -): Promise { - const now = dependencies.now ?? Date.now; - const pause = dependencies.sleep ?? sleep; - const configured = limits(dependencies); - const startedAt = now(); - const windowMs = - options.mode === "cleanup" - ? configured.cleanupTimeoutMs - : configured.dispatchReconciliationTimeoutMs; - const deadline = startedAt + windowMs; - const api = boundedApiClient(dependencies.api ?? githubApi, dependencies, deadline); - const creationWindow = dispatchCreationWindow(options.intent, configured); - const earliestQuery = new Date(Math.floor(creationWindow.earliest / 1_000) * 1_000).toISOString(); - const latestQuery = new Date(Math.ceil(creationWindow.latest / 1_000) * 1_000).toISOString(); - const created = encodeURIComponent( - `${earliestQuery.replace(".000Z", "Z")}..${latestQuery.replace(".000Z", "Z")}`, - ); - const listPath = - `repos/${PRODUCER_REPOSITORY}/actions/workflows/${PRODUCER_WORKFLOW_FILE}/runs` + - `?event=workflow_dispatch&branch=${PRODUCER_REF}&created=${created}&per_page=100`; - - for (;;) { - if (now() >= deadline) { - writeDispatchReconciliation(options.workDir, options.intent, [], "none", now); - fail( - "DISPATCH_AMBIGUOUS", - "no strict producer run match appeared before reconciliation timeout", - ); - } - const matches = reconciledWorkflowRuns( - await api(listPath, options.producerToken, { - apiVersion: GITHUB_API_VERSION, - expectedStatus: 200, - }), - options.intent, - configured, - ); - if (matches.length > 1) { - const runIds = matches.map((run) => run.id); - writeDispatchReconciliation(options.workDir, options.intent, matches, "multiple", now); - for (const run of matches) { - if (run.status !== "completed") { - try { - await api( - `repos/${PRODUCER_REPOSITORY}/actions/runs/${run.id}/cancel`, - options.producerToken, - { method: "POST", apiVersion: GITHUB_API_VERSION, expectedStatus: 202 }, - ); - } catch { - // The retained exact IDs are mandatory manual-cleanup evidence. - } - } - } - fail( - "DISPATCH_AMBIGUOUS", - `multiple strict dispatch matches require cleanup: ${runIds.join(",")}`, - ); - } - if (matches.length === 1) { - const recovered = matches[0]; - const state = stateFromIntent(options.intent, recovered); - writeState(options.workDir, state); - writeDispatchReconciliation( - options.workDir, - options.intent, - [recovered], - "recovered-one", - now, - ); - const cancelled = await cancelAndVerifyRecoveredRun( - state, - recovered, - options.producerToken, - api, - pause, - configured.reconciliationPollIntervalMs, - deadline, - now, - ); - if (options.mode === "cleanup") return cancelled; - fail( - "DISPATCH_AMBIGUOUS", - `recovered producer run ${recovered.id} was not accepted and was cleaned up`, - ); - } - await pause(configured.reconciliationPollIntervalMs); - } -} - -export async function startExactImageQualification( - options: { - request: ExactImageQualificationRequest; - coreToken: string; - producerToken: string; - workDir: string; - }, - dependencies: QualificationDependencies = {}, -): Promise { - const now = dependencies.now ?? Date.now; - const initialApi = boundedApiClient(dependencies.api ?? githubApi, dependencies); - await authorizeRequest(options.request, options.coreToken, initialApi); - - const producerRef = await initialApi( - `repos/${PRODUCER_REPOSITORY}/git/ref/heads/${PRODUCER_REF}`, - options.producerToken, - { apiVersion: GITHUB_API_VERSION, expectedStatus: 200 }, - ); - const producerSha = validateMainRef(producerRef, PRODUCER_REPOSITORY); - const workflowId = validateProducerWorkflow( - await initialApi( - `repos/${PRODUCER_REPOSITORY}/actions/workflows/${PRODUCER_WORKFLOW_FILE}`, - options.producerToken, - { apiVersion: GITHUB_API_VERSION, expectedStatus: 200 }, - ), - ); - const correlationId = (dependencies.randomUuid ?? randomUUID)(); - if (!UUID_V4_PATTERN.test(correlationId)) { - fail("REQUEST_INVALID", "generated correlation ID was not a lowercase UUIDv4"); - } - - const intent: ExactImageDispatchIntent = { - schemaVersion: 1, - kind: "nemoclaw-exact-image-dispatch-intent", - requestStartedAt: timestamp(now), - request: { - actor: options.request.actor, - candidateSha: options.request.candidateSha, - correlationId, - reason: options.request.reason, - requesterRunAttempt: options.request.requesterRunAttempt, - requesterRunId: options.request.requesterRunId, - workflowSha: options.request.workflowSha, - }, - producer: { - repository: PRODUCER_REPOSITORY, - repositorySha: producerSha, - ref: PRODUCER_REF, - workflowId, - workflowPath: PRODUCER_WORKFLOW_PATH, - }, - }; - writeDispatchIntent(options.workDir, intent); - const stateDeadline = - Date.parse(intent.requestStartedAt) + limits(dependencies).qualificationTimeoutMs; - const api = boundedApiClient(dependencies.api ?? githubApi, dependencies, stateDeadline); - - let dispatch: DispatchDetails; - try { - const dispatchResponse = await api( - `repos/${PRODUCER_REPOSITORY}/actions/workflows/${PRODUCER_WORKFLOW_FILE}/dispatches`, - options.producerToken, - { - method: "POST", - apiVersion: GITHUB_API_VERSION, - expectedStatus: 200, - body: { - ref: PRODUCER_REF, - inputs: { - nemoclaw_sha: options.request.candidateSha, - correlation_id: correlationId, - requester_workflow_run_id: options.request.requesterRunId, - requester_workflow_run_attempt: String(options.request.requesterRunAttempt), - }, - return_run_details: true, - }, - }, - ); - dispatch = validateWorkflowDispatchDetails(dispatchResponse); - } catch { - await reconcileAmbiguousDispatch( - { intent, workDir: options.workDir, producerToken: options.producerToken, mode: "start" }, - dependencies, - ); - fail("DISPATCH_AMBIGUOUS", "producer dispatch could not be bound safely"); - } - - const state: ExactImageQualificationState = { - schemaVersion: 1, - status: "dispatched", - dispatchedAt: intent.requestStartedAt, - request: { - actor: options.request.actor, - candidateSha: options.request.candidateSha, - correlationId, - reason: options.request.reason, - requesterRunAttempt: options.request.requesterRunAttempt, - requesterRunId: options.request.requesterRunId, - workflowSha: options.request.workflowSha, - }, - producer: { - repository: PRODUCER_REPOSITORY, - repositorySha: producerSha, - ref: PRODUCER_REF, - workflowPath: PRODUCER_WORKFLOW_PATH, - runId: dispatch.workflowRunId, - runAttempt: 1, - runUrl: dispatch.runUrl, - htmlUrl: dispatch.htmlUrl, - workflowId, - }, - }; - // Persist the returned ID before any further API call so the always-run - // cleanup step can cancel exactly this run if identity validation fails. - writeState(options.workDir, state); - - const run = validateQualificationWorkflowRun( - await api( - `repos/${PRODUCER_REPOSITORY}/actions/runs/${dispatch.workflowRunId}`, - options.producerToken, - { apiVersion: GITHUB_API_VERSION, expectedStatus: 200 }, - ), - { - candidateSha: options.request.candidateSha, - correlationId, - producerSha, - workflowId, - runId: dispatch.workflowRunId, - runUrl: dispatch.runUrl, - htmlUrl: dispatch.htmlUrl, - }, - ); - state.producer.workflowId = run.workflowId; - writeState(options.workDir, state); - return state; -} - -function expectedRun(state: ExactImageQualificationState) { - return { - candidateSha: state.request.candidateSha, - correlationId: state.request.correlationId, - producerSha: state.producer.repositorySha, - workflowId: state.producer.workflowId, - runId: state.producer.runId, - runUrl: state.producer.runUrl, - htmlUrl: state.producer.htmlUrl, - }; -} - -async function cancelRun( - state: ExactImageQualificationState, - token: string, - api: GitHubApiClient, -): Promise { - await api( - `repos/${PRODUCER_REPOSITORY}/actions/runs/${state.producer.runId}/cancel`, - token, - { method: "POST", apiVersion: GITHUB_API_VERSION, expectedStatus: 202 }, - ); -} - -export async function waitForExactImageQualification( - options: { workDir: string; producerToken: string }, - dependencies: QualificationDependencies = {}, -): Promise { - const now = dependencies.now ?? Date.now; - const pause = dependencies.sleep ?? sleep; - const configured = limits(dependencies); - const state = readBoundExactImageQualificationState(options.workDir); - if (state.status !== "dispatched") { - fail("PROVENANCE_MISMATCH", "producer run may only be awaited from dispatched state"); - } - const startedAt = Date.parse(state.dispatchedAt); - const hardDeadline = qualificationDeadline(state, configured.qualificationTimeoutMs); - const deadline = acceptanceDeadline(state, dependencies); - const api = boundedApiClient(dependencies.api ?? githubApi, dependencies, deadline); - const cancelApi = boundedApiClient(dependencies.api ?? githubApi, dependencies, hardDeadline); - let warned = false; - - for (;;) { - if (now() >= deadline) { - const finalRun = validateQualificationWorkflowRun( - await cancelApi( - `repos/${PRODUCER_REPOSITORY}/actions/runs/${state.producer.runId}`, - options.producerToken, - { apiVersion: GITHUB_API_VERSION, expectedStatus: 200 }, - ), - expectedRun(state), - ); - if (finalRun.status !== "completed") { - await cancelRun(state, options.producerToken, cancelApi); - } - fail("QUALIFICATION_TIMEOUT", "qualification reached its cancellation reserve"); - } - const run = validateQualificationWorkflowRun( - await api( - `repos/${PRODUCER_REPOSITORY}/actions/runs/${state.producer.runId}`, - options.producerToken, - { apiVersion: GITHUB_API_VERSION, expectedStatus: 200 }, - ), - expectedRun(state), - ); - const observedAt = now(); - const elapsed = observedAt - startedAt; - if (observedAt >= deadline) { - if (run.status !== "completed") { - await cancelRun(state, options.producerToken, cancelApi); - } - fail("QUALIFICATION_TIMEOUT", "qualification reached its cancellation reserve"); - } - if (run.status === "completed") { - if (run.conclusion !== "success") { - fail("PRODUCER_RUN_FAILED", `producer run completed with ${run.conclusion ?? "no result"}`); - } - state.status = "completed"; - state.producer.completedAt = timestamp(now); - state.producer.workflowId = run.workflowId; - writeState(options.workDir, state); - return run; - } - - if (run.status !== "in_progress" && elapsed >= configured.queueTimeoutMs) { - await cancelRun(state, options.producerToken, cancelApi); - fail("RUN_QUEUE_TIMEOUT", "producer run did not start within 10 minutes"); - } - if (!warned && elapsed >= configured.warningAfterMs) { - console.warn( - "Qualification image build has exceeded 25 minutes; continuing to the hard limit.", - ); - warned = true; - } - await pause(configured.pollIntervalMs); - } -} - -export function validateQualificationArtifactList( - value: unknown, - state: ExactImageQualificationState, -): QualificationArtifact | null { - const response = record(value, "artifact list response"); - if (!Number.isSafeInteger(response.total_count) || (response.total_count as number) < 0) { - fail("ARTIFACT_MISSING_OR_INVALID", "artifact total_count is invalid"); - } - if (!Array.isArray(response.artifacts)) { - fail("ARTIFACT_MISSING_OR_INVALID", "artifact list is missing artifacts"); - } - if (response.total_count === 0 && response.artifacts.length === 0) return null; - if (response.total_count !== 1 || response.artifacts.length !== 1) { - fail("ARTIFACT_MISSING_OR_INVALID", "producer run must publish exactly one artifact"); - } - const artifact = record(response.artifacts[0], "qualification artifact"); - const id = safePositiveId(artifact.id, "artifact ID"); - const expectedName = `nemoclaw-image-handoff-v1-${state.producer.runId}-${state.producer.runAttempt}`; - requireExactString(artifact.name, expectedName, "artifact name"); - if (artifact.expired !== false) fail("ARTIFACT_MISSING_OR_INVALID", "artifact is expired"); - if (typeof artifact.digest !== "string" || !ARTIFACT_DIGEST_PATTERN.test(artifact.digest)) { - fail("ARTIFACT_MISSING_OR_INVALID", "artifact digest must be a lowercase SHA-256 digest"); - } - if ( - !Number.isSafeInteger(artifact.size_in_bytes) || - (artifact.size_in_bytes as number) < 1 || - (artifact.size_in_bytes as number) > MAX_ARCHIVE_BYTES - ) { - fail("ARTIFACT_MISSING_OR_INVALID", "artifact archive size is outside the accepted range"); - } - const apiUrl = `https://api.github.com/repos/${PRODUCER_REPOSITORY}/actions/artifacts/${id}`; - const archiveDownloadUrl = `${apiUrl}/zip`; - requireExactString(artifact.url, apiUrl, "artifact API URL"); - requireExactString(artifact.archive_download_url, archiveDownloadUrl, "artifact archive URL"); - const workflowRun = record(artifact.workflow_run, "artifact workflow run"); - if (safePositiveId(workflowRun.id, "artifact workflow run ID") !== state.producer.runId) { - fail("PROVENANCE_MISMATCH", "artifact workflow run ID did not match the dispatched run"); - } - requireExactString( - workflowRun.head_sha, - state.producer.repositorySha, - "artifact workflow run head SHA", - ); - return { - id, - name: expectedName, - digest: artifact.digest, - sizeInBytes: artifact.size_in_bytes as number, - apiUrl, - archiveDownloadUrl, - }; -} - -async function readBoundedResponse(response: Response, maxBytes: number): Promise { - const header = response.headers.get("content-length"); - if (header !== null && (!/^[0-9]+$/u.test(header) || Number(header) > maxBytes)) { - fail("ARTIFACT_MISSING_OR_INVALID", "artifact response is larger than the accepted limit"); - } - if (!response.body) fail("ARTIFACT_DOWNLOAD_TRANSIENT", "artifact response had no body"); - const reader = response.body.getReader(); - const chunks: Buffer[] = []; - let size = 0; - for (;;) { - const { done, value } = await reader.read(); - if (done) break; - size += value.byteLength; - if (size > maxBytes) { - await reader.cancel(); - fail("ARTIFACT_MISSING_OR_INVALID", "artifact response exceeded the accepted limit"); - } - chunks.push(Buffer.from(value)); - } - return Buffer.concat(chunks, size); -} - -function writePrivateBuffer(file: string, contents: Buffer): void { - let descriptor: number; - try { - descriptor = fs.openSync( - file, - fs.constants.O_WRONLY | fs.constants.O_CREAT | fs.constants.O_EXCL | NO_FOLLOW | NON_BLOCK, - 0o600, - ); - } catch { - fail("OUTPUT_WRITE_FAILED", "artifact output could not be created safely"); - } - try { - const stat = fs.fstatSync(descriptor); - if (!stat.isFile() || stat.nlink !== 1) { - fail("OUTPUT_WRITE_FAILED", "artifact output must be one private regular file"); - } - fs.fchmodSync(descriptor, 0o600); - fs.writeFileSync(descriptor, contents); - fs.fsyncSync(descriptor); - } finally { - fs.closeSync(descriptor); - } -} - -function defaultCommandRunner(_command: string, args: readonly string[]): CommandResult { - return spawnSync("/usr/bin/unzip", [...args], { - encoding: null, - env: { - LANG: "C", - LC_ALL: "C", - }, - maxBuffer: MAX_MANIFEST_BYTES + 4096, - timeout: COMMAND_TIMEOUT_MS, - windowsHide: true, - }); -} - -function successfulCommand(result: CommandResult, label: string): Buffer { - if (result.error || result.status !== 0 || result.signal) { - fail("ARTIFACT_MISSING_OR_INVALID", `${label} rejected the artifact archive`); - } - return Buffer.isBuffer(result.stdout) ? result.stdout : Buffer.from(result.stdout); -} - -export function extractExactManifestArchive( - archivePath: string, - outputPath: string, - runCommand: CommandRunner = defaultCommandRunner, -): Buffer { - const entries = successfulCommand(runCommand("unzip", ["-Z1", archivePath]), "ZIP inventory"); - if (!entries.equals(Buffer.from(`${MANIFEST_ARTIFACT_FILE}\n`, "utf8"))) { - fail( - "ARTIFACT_MISSING_OR_INVALID", - `artifact ZIP must contain exactly ${MANIFEST_ARTIFACT_FILE} at its root`, - ); - } - const metadata = successfulCommand( - runCommand("unzip", ["-Zl", archivePath, MANIFEST_ARTIFACT_FILE]), - "ZIP metadata inspection", - ).toString("utf8"); - const matchingMetadata = metadata - .split(/\r?\n/u) - .filter((line) => line.endsWith(` ${MANIFEST_ARTIFACT_FILE}`)); - if (matchingMetadata.length !== 1 || !matchingMetadata[0]?.startsWith("-")) { - fail("ARTIFACT_MISSING_OR_INVALID", "artifact manifest ZIP entry must be a regular file"); - } - const manifest = successfulCommand( - runCommand("unzip", ["-p", archivePath, MANIFEST_ARTIFACT_FILE]), - "ZIP extraction", - ); - if (manifest.length === 0 || manifest.length > MAX_MANIFEST_BYTES) { - fail("ARTIFACT_MISSING_OR_INVALID", "artifact manifest size is outside the accepted range"); - } - writePrivateBuffer(outputPath, manifest); - const stat = fs.lstatSync(outputPath); - if ( - !stat.isFile() || - stat.isSymbolicLink() || - stat.nlink !== 1 || - stat.size !== manifest.length - ) { - fail("ARTIFACT_MISSING_OR_INVALID", "extracted manifest is not one regular direct file"); - } - return manifest; -} - -export async function downloadExactImageManifest( - options: { workDir: string; producerToken: string }, - dependencies: QualificationDependencies = {}, -): Promise { - const state = readBoundExactImageQualificationState(options.workDir); - if (state.status !== "completed") { - fail("PROVENANCE_MISMATCH", "producer run must complete before artifact download"); - } - const pause = dependencies.sleep ?? sleep; - const now = dependencies.now ?? Date.now; - const configured = limits(dependencies); - const deadline = qualificationDeadline(state, configured.qualificationTimeoutMs); - const api = boundedApiClient(dependencies.api ?? githubApi, dependencies, deadline); - requireQualificationTimeRemaining(deadline, now); - const artifactStartedAt = now(); - const artifactDeadline = Math.min( - artifactStartedAt + configured.artifactPropagationTimeoutMs, - deadline, - ); - let artifact: QualificationArtifact | null = null; - while (artifact === null) { - requireQualificationTimeRemaining(deadline, now); - artifact = validateQualificationArtifactList( - await api( - `repos/${PRODUCER_REPOSITORY}/actions/runs/${state.producer.runId}/artifacts?per_page=100`, - options.producerToken, - { apiVersion: GITHUB_API_VERSION, expectedStatus: 200 }, - ), - state, - ); - requireQualificationTimeRemaining(deadline, now); - if (artifact !== null) break; - if (now() >= artifactDeadline) { - fail("ARTIFACT_PENDING", "qualification artifact did not appear within two minutes"); - } - await pause(configured.pollIntervalMs); - } - - const controller = new AbortController(); - const downloadBudget = Math.min( - configured.downloadTimeoutMs, - requireQualificationTimeRemaining(deadline, now), - ); - const timer = setTimeout(() => controller.abort(), downloadBudget); - let archive: Buffer; - try { - const response = await (dependencies.fetch ?? fetch)(artifact.archiveDownloadUrl, { - headers: { - Accept: "application/vnd.github+json", - Authorization: `Bearer ${options.producerToken}`, - "X-GitHub-Api-Version": GITHUB_API_VERSION, - }, - redirect: "follow", - signal: controller.signal, - }); - requireQualificationTimeRemaining(deadline, now); - if (response.status !== 200) { - fail("ARTIFACT_DOWNLOAD_TRANSIENT", `artifact archive download returned ${response.status}`); - } - archive = await readBoundedResponse(response, MAX_ARCHIVE_BYTES); - } catch (error) { - if (error instanceof ExactImageQualificationError) throw error; - requireQualificationTimeRemaining(deadline, now); - fail("ARTIFACT_DOWNLOAD_TRANSIENT", "artifact archive download failed"); - } finally { - clearTimeout(timer); - } - requireQualificationTimeRemaining(deadline, now); - if (archive.length !== artifact.sizeInBytes) { - fail("ARTIFACT_MISSING_OR_INVALID", "downloaded archive size did not match artifact metadata"); - } - const archiveHash = sha256(archive); - if (`sha256:${archiveHash}` !== artifact.digest) { - fail("ARTIFACT_MISSING_OR_INVALID", "downloaded archive digest did not match GitHub metadata"); - } - - const archivePath = path.join(options.workDir, ARCHIVE_FILE); - const manifestPath = path.join(options.workDir, MANIFEST_ARTIFACT_FILE); - writePrivateBuffer(archivePath, archive); - const manifest = extractExactManifestArchive( - archivePath, - manifestPath, - dependencies.runCommand ?? defaultCommandRunner, - ); - requireQualificationTimeRemaining(deadline, now); - state.artifact = { - ...artifact, - archiveSha256: archiveHash, - manifestSha256: sha256(manifest), - }; - state.status = "downloaded"; - writeState(options.workDir, state); - return artifact; -} - -function readPrivateBuffer(file: string, maxBytes: number): Buffer { - let descriptor: number; - try { - descriptor = fs.openSync(file, fs.constants.O_RDONLY | NO_FOLLOW | NON_BLOCK); - } catch { - fail("PROVENANCE_MISMATCH", "qualification evidence file could not be opened safely"); - } - try { - const before = fs.fstatSync(descriptor); - if (!before.isFile() || before.nlink !== 1 || before.size > maxBytes) { - fail("PROVENANCE_MISMATCH", "qualification evidence file failed regular-file checks"); - } - const bytes = fs.readFileSync(descriptor); - const after = fs.fstatSync(descriptor); - if (after.size !== before.size || after.nlink !== 1) { - fail("PROVENANCE_MISMATCH", "qualification evidence file changed while it was read"); - } - return bytes; - } finally { - fs.closeSync(descriptor); - } -} - -export function finalizeExactImageQualification( - workDir: string, - dependencies: QualificationDependencies = {}, -): ExactImageQualificationState { - const state = readBoundExactImageQualificationState(workDir); - if (state.status !== "downloaded" || !state.artifact) { - fail("PROVENANCE_MISMATCH", "artifact must be digest-verified before finalization"); - } - const now = dependencies.now ?? Date.now; - const deadline = qualificationDeadline(state, limits(dependencies).qualificationTimeoutMs); - requireQualificationTimeRemaining(deadline, now); - const manifestHash = sha256( - readPrivateBuffer(path.join(workDir, MANIFEST_ARTIFACT_FILE), MAX_MANIFEST_BYTES), - ); - if (manifestHash !== state.artifact.manifestSha256) { - fail("PROVENANCE_MISMATCH", "manifest changed after archive verification"); - } - const normalizedHash = sha256( - readPrivateBuffer(path.join(workDir, VALIDATED_MANIFEST_FILE), MAX_MANIFEST_BYTES), - ); - requireQualificationTimeRemaining(deadline, now); - state.validation = { - acceptedAt: timestamp(now), - manifestSha256: manifestHash, - normalizedManifestSha256: normalizedHash, - }; - state.status = "validated"; - const evidence = { - schemaVersion: 1, - kind: "nemoclaw-exact-image-qualification-evidence", - qualificationStatus: "accepted", - request: state.request, - producer: state.producer, - artifact: state.artifact, - validation: state.validation, - }; - // The state is the durable commit point for acceptance. Validate and persist - // that transition before publishing an accepted receipt so a rejected state - // can never leave accepted evidence behind. - writeState(workDir, state); - try { - privateFileRuntime.writePrivateRegularFile( - path.join(workDir, EVIDENCE_FILE), - `${JSON.stringify(evidence, null, 2)}\n`, - ); - } catch { - fail("OUTPUT_WRITE_FAILED", "qualification evidence could not be written safely"); - } - return state; -} - -export async function cancelActiveExactImageQualification( - options: { workDir: string; producerToken: string }, - dependencies: QualificationDependencies = {}, -): Promise { - let state: ExactImageQualificationState; - try { - state = readExactImageQualificationState(options.workDir); - } catch (error) { - if (!fs.existsSync(intentPath(options.workDir))) { - if (fs.existsSync(statePath(options.workDir))) throw error; - return false; - } - // State crosses independent workflow steps and is untrusted/damaged-disk input on every read. - // Atomic replacement prevents ordinary partial writes, while this permanent defense-in-depth - // path preserves interruption/filesystem-corruption evidence and reconciles only from the - // separately validated durable intent. - preserveUnreadableState(options.workDir); - const intent = requiredDispatchIntent(options.workDir); - return reconcileAmbiguousDispatch({ ...options, intent, mode: "cleanup" }, dependencies); - } - const now = dependencies.now ?? Date.now; - const configured = limits(dependencies); - const intent = requiredDispatchIntent(options.workDir); - try { - validateStateIntentBinding(state, intent); - } catch { - preserveUnreadableState(options.workDir); - return reconcileAmbiguousDispatch({ ...options, intent, mode: "cleanup" }, dependencies); - } - const deadline = now() + configured.cleanupTimeoutMs; - const api = boundedApiClient(dependencies.api ?? githubApi, dependencies, deadline); - const initial = validateCancellationWorkflowRun( - await api( - `repos/${PRODUCER_REPOSITORY}/actions/runs/${state.producer.runId}`, - options.producerToken, - { apiVersion: GITHUB_API_VERSION, expectedStatus: 200 }, - ), - state, - ); - return cancelAndVerifyRecoveredRun( - state, - initial, - options.producerToken, - api, - dependencies.sleep ?? sleep, - configured.reconciliationPollIntervalMs, - deadline, - now, - ); -} - -function parseFlags(argv: readonly string[]): Map { - const flags = new Map(); - for (let index = 0; index < argv.length; index += 2) { - const flag = argv[index]; - const value = argv[index + 1]; - if (!flag?.startsWith("--") || value === undefined || value.startsWith("--")) { - fail("REQUEST_INVALID", `${flag ?? "argument"} requires one value`); - } - if (flags.has(flag)) fail("REQUEST_INVALID", `${flag} may be provided only once`); - flags.set(flag, value); - } - return flags; -} - -function requiredFlag(flags: Map, flag: string): string { - const value = flags.get(flag); - if (value === undefined) fail("REQUEST_INVALID", `${flag} is required`); - return value; -} - -export function parseExactImageQualificationCommand(argv: readonly string[]): ControllerCommand { - const flags = parseFlags(argv); - const mode = requiredFlag(flags, "--mode"); - if (["wait", "download", "finalize", "cancel"].includes(mode)) { - const allowed = new Set(["--mode", "--work-dir"]); - for (const flag of flags.keys()) { - if (!allowed.has(flag)) fail("REQUEST_INVALID", `unknown argument ${flag}`); - } - return { - mode: mode as "wait" | "download" | "finalize" | "cancel", - workDir: requiredFlag(flags, "--work-dir"), - }; - } - if (mode !== "preflight" && mode !== "start") { - fail("REQUEST_INVALID", `unsupported mode ${mode}`); - } - const allowed = new Set([ - "--mode", - "--actor", - "--candidate-sha", - "--event-name", - "--reason", - "--ref", - "--requester-run-attempt", - "--requester-run-id", - "--workflow-sha", - ...(mode === "start" ? ["--work-dir"] : []), - ]); - for (const flag of flags.keys()) { - if (!allowed.has(flag)) fail("REQUEST_INVALID", `unknown argument ${flag}`); - } - const request = validateExactImageQualificationRequest({ - actor: requiredFlag(flags, "--actor"), - candidateSha: requiredFlag(flags, "--candidate-sha"), - eventName: requiredFlag(flags, "--event-name"), - reason: requiredFlag(flags, "--reason"), - ref: requiredFlag(flags, "--ref"), - requesterRunAttempt: positiveInteger( - requiredFlag(flags, "--requester-run-attempt"), - "requester run attempt", - ), - requesterRunId: requiredFlag(flags, "--requester-run-id"), - workflowSha: requiredFlag(flags, "--workflow-sha"), - }); - return mode === "preflight" - ? { mode, request } - : { mode, request, workDir: requiredFlag(flags, "--work-dir") }; -} - -function requiredToken(name: string): string { - const token = process.env[name]; - if (!token) fail("REQUEST_INVALID", `${name} is required`); - return token; -} - -function writeOutput(name: string, value: string): void { - const output = process.env.GITHUB_OUTPUT; - if (!output) return; - fs.appendFileSync(output, `${name}=${value}\n`, { encoding: "utf8", mode: 0o600 }); -} - -async function main(): Promise { - const command = parseExactImageQualificationCommand(process.argv.slice(2)); - if (command.mode === "preflight") { - await preflightExactImageQualification(command.request, requiredToken("GITHUB_TOKEN")); - console.log("Draft qualification request passed preflight."); - return; - } - if (command.mode === "start") { - const state = await startExactImageQualification({ - request: command.request, - coreToken: requiredToken("GITHUB_TOKEN"), - producerToken: requiredToken("NEMOCLAW_IMAGE_QUALIFICATION_TOKEN"), - workDir: command.workDir, - }); - writeOutput("correlation_id", state.request.correlationId); - writeOutput("image_repository_sha", state.producer.repositorySha); - writeOutput("producer_run_id", state.producer.runId); - writeOutput("producer_run_attempt", String(state.producer.runAttempt)); - console.log(`Dispatched and bound producer run ${state.producer.runId}.`); - return; - } - if (command.mode === "wait") { - const run = await waitForExactImageQualification({ - workDir: command.workDir, - producerToken: requiredToken("NEMOCLAW_IMAGE_QUALIFICATION_TOKEN"), - }); - console.log(`Producer run ${run.id} completed successfully.`); - return; - } - if (command.mode === "download") { - const artifact = await downloadExactImageManifest({ - workDir: command.workDir, - producerToken: requiredToken("NEMOCLAW_IMAGE_QUALIFICATION_TOKEN"), - }); - console.log(`Verified and extracted artifact ${artifact.id}.`); - return; - } - if (command.mode === "finalize") { - const state = finalizeExactImageQualification(command.workDir); - console.log(`Recorded accepted qualification evidence for run ${state.producer.runId}.`); - return; - } - const cancelled = await cancelActiveExactImageQualification({ - workDir: command.workDir, - producerToken: requiredToken("NEMOCLAW_IMAGE_QUALIFICATION_TOKEN"), - }); - console.log(cancelled ? "Cancelled active producer run." : "No active producer run to cancel."); -} - -export function exactImageQualificationFailureCode( - error: unknown, -): ExactImageQualificationFailureCode { - return error instanceof ExactImageQualificationError ? error.code : "UNKNOWN"; -} - -if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { - main().catch((error) => { - const code = exactImageQualificationFailureCode(error); - const message = error instanceof Error ? error.message : "unexpected qualification error"; - process.stderr.write(`${code}: ${message}\n`); - process.exitCode = 1; - }); -} diff --git a/tools/e2e/operations-workflow-boundary.mts b/tools/e2e/operations-workflow-boundary.mts index 0fbec309f9..2e8fecc673 100644 --- a/tools/e2e/operations-workflow-boundary.mts +++ b/tools/e2e/operations-workflow-boundary.mts @@ -215,11 +215,16 @@ function validatePrGateDispatch(errors: string[], workflow: OperationsWorkflow): jobName === "report-to-pr" && step.name === "Check out the trusted E2E reporting helper" && step.with?.ref === "${{ github.workflow_sha }}"; + const trustedLaunchableControlCheckout = + jobName === "staging-brev-launchable" && + step.name === "Check out trusted E2E control code" && + step.with?.ref === "${{ github.workflow_sha }}"; if ( step.uses?.startsWith("actions/checkout@") && step.with?.ref !== "${{ inputs.checkout_sha || github.sha }}" && !trustedHermesFixtureCheckout && - !trustedReportHelperCheckout + !trustedReportHelperCheckout && + !trustedLaunchableControlCheckout ) { errors.push(`${jobName} checkout must use the selected PR commit`); } diff --git a/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts b/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts index b261d2fc75..33663d1abd 100644 --- a/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts +++ b/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts @@ -167,11 +167,35 @@ const EXPLICIT_UPLOAD_CONTRACTS = new Map([ path: "e2e-artifacts/live/mcp-bridge-dev/${{ matrix.agent }}/", }, ], + [ + "staging-brev-launchable", + { + name: "brev-launchable-e2e-${{ steps.request.outputs.candidate_sha }}-${{ github.run_id }}", + path: [ + "${{ steps.request.outputs.work_dir }}/producer-run.json", + "${{ steps.request.outputs.work_dir }}/image-handoff.json", + "${{ steps.request.outputs.work_dir }}/brev-deploy-request.json", + "${{ steps.request.outputs.work_dir }}/brev-workspace-ready.json", + "${{ steps.request.outputs.work_dir }}/brev-provision.json", + "${{ steps.request.outputs.work_dir }}/brev-boot-image.json", + "${{ steps.request.outputs.work_dir }}/brev-identity-evidence.json", + "${{ steps.request.outputs.work_dir }}/brev-launchable-e2e.log", + "${{ steps.request.outputs.work_dir }}/brev-launchable-e2e-evidence.json", + "${{ steps.request.outputs.work_dir }}/brev-launchable-cloud-openclaw/", + "${{ steps.request.outputs.work_dir }}/brev-cleanup-evidence.json", + "", + ].join("\n"), + }, + ], ]); const EXPLICIT_CALLER_CONDITIONS = new Map([ ["mcp-bridge", MCP_SCANNED_UPLOAD_CONDITION], ["mcp-bridge-dev", MCP_SCANNED_UPLOAD_CONDITION], + [ + "staging-brev-launchable", + "${{ always() && steps.request.outputs.work_dir != '' && steps.redact-launchable-evidence.outcome == 'success' }}", + ], ]); const EXPECTED_ACTION_INPUTS = { @@ -276,6 +300,7 @@ export function validateUploadE2eArtifactsInvocations(workflow: WorkflowRecord): const jobSteps = steps(job.steps); const env = record(job.env); return ( + EXPLICIT_UPLOAD_CONTRACTS.has(jobName) || jobName === "live" || env.E2E_JOB === "1" || env.NEMOCLAW_RUN_LIVE_E2E === "1" || diff --git a/tools/e2e/validate-exact-image-manifest.mts b/tools/e2e/validate-exact-image-manifest.mts deleted file mode 100644 index 2bbaf4186a..0000000000 --- a/tools/e2e/validate-exact-image-manifest.mts +++ /dev/null @@ -1,176 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import fs from "node:fs"; -import { pathToFileURL } from "node:url"; -import { TextDecoder } from "node:util"; -import { - ExactImageManifestError, - type ExactImageManifestExpectations, - exactImageManifestFailureCode, - normalizedExactImageManifestJson, - parseAndValidateExactImageManifest, -} from "./exact-image-manifest.mts"; -import * as privateFile from "./private-file.mts"; - -// The root TypeScript package is exposed as CommonJS under `node --import -// tsx` / `npx tsx`, but as an ESM namespace under Node's strip-types runtime -// and Vitest. Normalize both representations so every supported entrypoint -// uses the same no-follow private-file writer. -const privateFileRuntime = ( - "default" in privateFile && privateFile.default ? privateFile.default : privateFile -) as typeof import("./private-file.mts"); - -const MAX_MANIFEST_BYTES = 64 * 1024; -const NO_FOLLOW = fs.constants.O_NOFOLLOW ?? 0; -const NON_BLOCK = fs.constants.O_NONBLOCK ?? 0; - -type ExactImageManifestCliOptions = ExactImageManifestExpectations & { - manifest: string; - output: string; -}; - -const ARGUMENT_FIELDS = { - "--manifest": "manifest", - "--output": "output", - "--nemoclaw-sha": "nemoclawSha", - "--requester-run-id": "requesterWorkflowRunId", - "--requester-run-attempt": "requesterWorkflowRunAttempt", - "--correlation-id": "correlationId", - "--image-repository-sha": "imageRepositorySha", - "--producer-run-id": "workflowRunId", - "--producer-run-attempt": "workflowRunAttempt", -} as const; - -function requestInvalid(message: string): never { - throw new ExactImageManifestError("REQUEST_INVALID", message); -} - -function positiveIntegerArgument(value: string, flag: string): number { - if (!/^[1-9][0-9]*$/u.test(value)) { - requestInvalid(`${flag} must be a positive integer`); - } - const parsed = Number(value); - if (!Number.isSafeInteger(parsed)) { - requestInvalid(`${flag} exceeds the safe integer range`); - } - return parsed; -} - -export function parseExactImageManifestCliArgs( - argv: readonly string[], -): ExactImageManifestCliOptions { - const values = new Map(); - for (let index = 0; index < argv.length; index += 2) { - const flag = argv[index] as keyof typeof ARGUMENT_FIELDS | undefined; - const value = argv[index + 1]; - if (!flag || !(flag in ARGUMENT_FIELDS)) { - requestInvalid(`unknown argument ${JSON.stringify(flag ?? "")}`); - } - if (value === undefined || value.startsWith("--")) { - requestInvalid(`${flag} requires a value`); - } - const field = ARGUMENT_FIELDS[flag]; - if (values.has(field)) { - requestInvalid(`${flag} may be provided only once`); - } - values.set(field, value); - } - - const requireValue = (field: keyof ExactImageManifestCliOptions, flag: string): string => { - const value = values.get(field); - if (value === undefined) requestInvalid(`${flag} is required`); - return value; - }; - - return { - manifest: requireValue("manifest", "--manifest"), - output: requireValue("output", "--output"), - nemoclawSha: requireValue("nemoclawSha", "--nemoclaw-sha"), - requesterWorkflowRunId: requireValue("requesterWorkflowRunId", "--requester-run-id"), - requesterWorkflowRunAttempt: positiveIntegerArgument( - requireValue("requesterWorkflowRunAttempt", "--requester-run-attempt"), - "--requester-run-attempt", - ), - correlationId: requireValue("correlationId", "--correlation-id"), - imageRepositorySha: requireValue("imageRepositorySha", "--image-repository-sha"), - workflowRunId: requireValue("workflowRunId", "--producer-run-id"), - workflowRunAttempt: positiveIntegerArgument( - requireValue("workflowRunAttempt", "--producer-run-attempt"), - "--producer-run-attempt", - ), - }; -} - -function readManifestFile(file: string): string { - let descriptor: number; - try { - descriptor = fs.openSync(file, fs.constants.O_RDONLY | NO_FOLLOW | NON_BLOCK); - } catch { - throw new ExactImageManifestError( - "ARTIFACT_MISSING_OR_INVALID", - "manifest input could not be opened safely", - ); - } - - try { - const before = fs.fstatSync(descriptor); - if (!before.isFile() || before.nlink !== 1) { - throw new ExactImageManifestError( - "ARTIFACT_MISSING_OR_INVALID", - "manifest input must be one regular file", - ); - } - if (before.size > MAX_MANIFEST_BYTES) { - throw new ExactImageManifestError( - "ARTIFACT_MISSING_OR_INVALID", - `manifest input exceeds ${MAX_MANIFEST_BYTES} bytes`, - ); - } - const bytes = fs.readFileSync(descriptor); - const after = fs.fstatSync(descriptor); - if (bytes.length > MAX_MANIFEST_BYTES || after.size !== before.size || after.nlink !== 1) { - throw new ExactImageManifestError( - "ARTIFACT_MISSING_OR_INVALID", - "manifest input changed while it was read", - ); - } - try { - return new TextDecoder("utf-8", { fatal: true }).decode(bytes); - } catch { - throw new ExactImageManifestError( - "ARTIFACT_MISSING_OR_INVALID", - "manifest input must be valid UTF-8", - ); - } - } finally { - fs.closeSync(descriptor); - } -} - -export function runExactImageManifestCli(argv: readonly string[] = process.argv.slice(2)): void { - const options = parseExactImageManifestCliArgs(argv); - const accepted = parseAndValidateExactImageManifest(readManifestFile(options.manifest), options); - try { - privateFileRuntime.writePrivateRegularFile( - options.output, - normalizedExactImageManifestJson(accepted), - ); - } catch { - throw new ExactImageManifestError( - "OUTPUT_WRITE_FAILED", - "accepted manifest output could not be written safely", - ); - } -} - -if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { - try { - runExactImageManifestCli(); - } catch (error) { - const code = exactImageManifestFailureCode(error); - const message = error instanceof Error ? error.message : "unexpected manifest validation error"; - process.stderr.write(`${code}: ${message}\n`); - process.exitCode = 1; - } -} From 646a3f8d941af0c75443a2dcd0929e79e254703b Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Mon, 20 Jul 2026 18:04:07 -0400 Subject: [PATCH 24/25] fix(e2e): avoid producer contents lookup Signed-off-by: Julie Yaunches --- .github/workflows/e2e.yaml | 22 +++++++--------------- test/e2e/support/e2e-workflow.test.ts | 16 ++++++++++++++++ 2 files changed, 23 insertions(+), 15 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index fc74879899..0882421820 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -239,11 +239,6 @@ jobs: [[ "$CANDIDATE_SHA" =~ ^[0-9a-f]{40}$ ]] \ || { echo "::error::candidate SHA must be a lowercase full SHA"; exit 1; } api=https://api.github.com/repos/brevdev/nemoclaw-image - producer_sha="$(curl --fail-with-body --silent --show-error --proto '=https' \ - -H "Authorization: Bearer $IMAGE_TOKEN" \ - -H 'Accept: application/vnd.github+json' \ - -H 'X-GitHub-Api-Version: 2026-03-10' \ - "$api/git/ref/heads/main" | jq -er .object.sha)" response="$({ jq -n \ --arg ref main \ @@ -261,18 +256,15 @@ jobs: run_url="$(jq -er .html_url <<< "$response")" jq -n \ --arg candidateSha "$CANDIDATE_SHA" \ - --arg producerSha "$producer_sha" \ --arg runId "$run_id" \ --arg runUrl "$run_url" \ - '{candidateSha:$candidateSha,producerSha:$producerSha,runId:$runId,runUrl:$runUrl,status:"dispatched"}' \ + '{candidateSha:$candidateSha,runId:$runId,runUrl:$runUrl,status:"dispatched"}' \ > "$WORK_DIR/image-build.json" - printf 'producer_sha=%s\nrun_id=%s\nrun_url=%s\n' \ - "$producer_sha" "$run_id" "$run_url" >> "$GITHUB_OUTPUT" + printf 'run_id=%s\nrun_url=%s\n' "$run_id" "$run_url" >> "$GITHUB_OUTPUT" - name: Wait for the exact image build env: IMAGE_TOKEN: ${{ secrets.NEMOCLAW_IMAGE_DISPATCH_TOKEN }} - PRODUCER_SHA: ${{ steps.image-build.outputs.producer_sha }} RUN_ID: ${{ steps.image-build.outputs.run_id }} WORK_DIR: ${{ steps.workspace.outputs.work_dir }} run: | @@ -286,10 +278,9 @@ jobs: -H 'X-GitHub-Api-Version: 2026-03-10' \ "$api/actions/runs/$RUN_ID")" jq -e \ - --arg id "$RUN_ID" \ - --arg sha "$PRODUCER_SHA" ' + --arg id "$RUN_ID" ' (.id | tostring) == $id - and .head_sha == $sha + and (.head_sha | type == "string" and test("^[0-9a-f]{40}$")) and .event == "workflow_dispatch" and .path == ".github/workflows/build-on-push.yml" and .run_attempt == 1 @@ -298,8 +289,9 @@ jobs: status="$(jq -r .status <<< "$run")" if [ "$status" = completed ]; then conclusion="$(jq -r .conclusion <<< "$run")" - jq --arg status "$status" --arg conclusion "$conclusion" \ - '.status=$status | .conclusion=$conclusion' \ + producer_sha="$(jq -er .head_sha <<< "$run")" + jq --arg status "$status" --arg conclusion "$conclusion" --arg producerSha "$producer_sha" \ + '.status=$status | .conclusion=$conclusion | .producerSha=$producerSha' \ "$WORK_DIR/image-build.json" > "$WORK_DIR/image-build.tmp" mv "$WORK_DIR/image-build.tmp" "$WORK_DIR/image-build.json" [ "$conclusion" = success ] \ diff --git a/test/e2e/support/e2e-workflow.test.ts b/test/e2e/support/e2e-workflow.test.ts index abe2490463..9bcd1ce1ab 100644 --- a/test/e2e/support/e2e-workflow.test.ts +++ b/test/e2e/support/e2e-workflow.test.ts @@ -130,6 +130,22 @@ describe("e2e workflow boundary", () => { ); }); + it("binds the Launchable image build to the returned run without repository-content access (#6943)", () => { + const workflow = readWorkflow() as { + jobs: Record }>; + }; + const steps = workflow.jobs["staging-brev-launchable"]?.steps ?? []; + const dispatch = steps.find((step) => step.id === "image-build")?.run ?? ""; + const wait = steps.find((step) => step.name === "Wait for the exact image build")?.run ?? ""; + + expect(dispatch).toContain("return_run_details:true"); + expect(dispatch).toContain(".workflow_run_id"); + expect(dispatch).not.toContain("/git/ref/"); + expect(wait).toContain('"$api/actions/runs/$RUN_ID"'); + expect(wait).toContain('.path == ".github/workflows/build-on-push.yml"'); + expect(wait).toContain('.event == "workflow_dispatch"'); + }); + it("keeps controller runner selection in a trusted pre-checkout matrix (#7031)", () => { const workflow = readWorkflow() as { jobs: Record< From 4b95fa41d1848cc87b17e06efc8fa117f93a7b05 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Mon, 20 Jul 2026 18:59:08 -0400 Subject: [PATCH 25/25] fix(e2e): verify launchable image directly Signed-off-by: Julie Yaunches --- ci/source-shape-test-budget.json | 5 ++++ test/brev-launchable-runtime.test.ts | 18 +++----------- test/e2e/support/e2e-workflow.test.ts | 1 + tools/e2e/brev-launchable-runtime.sh | 35 +++++++++++---------------- 4 files changed, 24 insertions(+), 35 deletions(-) diff --git a/ci/source-shape-test-budget.json b/ci/source-shape-test-budget.json index 57d1ac7d22..fe086b9b23 100644 --- a/ci/source-shape-test-budget.json +++ b/ci/source-shape-test-budget.json @@ -231,6 +231,11 @@ "test": "derives test selectors from code and workflow jobs from workflow metadata", "category": "compatibility" }, + { + "file": "test/e2e/support/e2e-workflow.test.ts", + "test": "binds the Launchable image build to the returned run without repository-content access (#6943)", + "category": "security" + }, { "file": "test/e2e/support/e2e-workflow.test.ts", "test": "rejects channels stop/start workflow-boundary drift for secret and artifact handling", diff --git a/test/brev-launchable-runtime.test.ts b/test/brev-launchable-runtime.test.ts index a6296c77b1..04ddf41f0e 100644 --- a/test/brev-launchable-runtime.test.ts +++ b/test/brev-launchable-runtime.test.ts @@ -18,7 +18,6 @@ type FixtureOptions = { emptyWorkspacesMode?: "array" | "null"; e2eStatus?: "failed" | "passed"; imageLabelSha?: string; - provisionSha?: string; repoSha?: string; workspaceMode?: "create-failed" | "failure" | "pending" | "ready"; }; @@ -103,18 +102,6 @@ case "$1" in refresh) ;; *) exit 2 ;; esac -`, - ); - executable( - path.join(bin, "sudo"), - `#!/usr/bin/env bash -set -euo pipefail -[ "\${1:-}" != -n ] || shift -if [ "\${1:-}" = cat ] && [ "\${2:-}" = /etc/nemoclaw/provision.json ]; then - jq -cn --arg sha "$FAKE_PROVISION_SHA" '{gitSha:$sha}' - exit 0 -fi -exec "$@" `, ); executable( @@ -177,7 +164,6 @@ exit 2 FAKE_E2E_EXECUTED: e2eExecuted, FAKE_E2E_STATUS: options.e2eStatus ?? "passed", FAKE_IMAGE_LABEL_SHA: options.imageLabelSha ?? candidateSha, - FAKE_PROVISION_SHA: options.provisionSha ?? candidateSha.slice(0, 7), FAKE_REPO_SHA: options.repoSha ?? candidateSha, HOME: home, INSTANCE_NAME: "nclaw-e2e-test-1", @@ -206,7 +192,11 @@ describe("Brev Launchable E2E runtime", () => { ); expect(commands).toContain("test/e2e/live/full-e2e.test.ts"); expect(commands).toContain("NEMOCLAW_E2E_SETUP_MODE=preinstalled-launchable"); + expect(commands).not.toContain("/etc/nemoclaw/provision.json"); expect(commands).not.toMatch(/rsync|install\.sh|npm (?:ci|install)|git clone/u); + expect(commands.indexOf("instance/disks/0/device-name")).toBeLessThan( + commands.indexOf('git -C "$repo" rev-parse HEAD'), + ); expect( JSON.parse(fs.readFileSync(path.join(workDir, "brev-cleanup-evidence.json"), "utf8")), ).toMatchObject({ terminalState: "ABSENT" }); diff --git a/test/e2e/support/e2e-workflow.test.ts b/test/e2e/support/e2e-workflow.test.ts index 9bcd1ce1ab..6510f08a94 100644 --- a/test/e2e/support/e2e-workflow.test.ts +++ b/test/e2e/support/e2e-workflow.test.ts @@ -130,6 +130,7 @@ describe("e2e workflow boundary", () => { ); }); + // source-shape-contract: security -- Dispatch-only image-builder credentials must not gain repository-content access it("binds the Launchable image build to the returned run without repository-content access (#6943)", () => { const workflow = readWorkflow() as { jobs: Record }>; diff --git a/tools/e2e/brev-launchable-runtime.sh b/tools/e2e/brev-launchable-runtime.sh index 46ad3c94f0..7b423639b9 100755 --- a/tools/e2e/brev-launchable-runtime.sh +++ b/tools/e2e/brev-launchable-runtime.sh @@ -96,29 +96,10 @@ verify_identity() { [[ "$CANDIDATE_SHA" =~ ^[0-9a-f]{40}$ ]] \ || die "CANDIDATE_SHA must be a lowercase full SHA" - local provision repo_sha identity - provision="$(host_exec 'sudo -n cat /etc/nemoclaw/provision.json')" - provision="$(tail -n 1 <<<"$provision")" - jq -e 'type == "object" and (.gitSha | type == "string")' <<<"$provision" >/dev/null \ - || die "the baked provision metadata is missing or malformed" - printf '%s\n' "$provision" >"$WORK_DIR/brev-provision.json" - - # HOME is expanded by the remote host shell. - # shellcheck disable=SC2016 - repo_sha="$(host_exec 'set -euo pipefail - repo="$HOME/NemoClaw" - test -d "$repo/.git" - git -C "$repo" diff --quiet --no-ext-diff HEAD -- - git -C "$repo" diff --cached --quiet --no-ext-diff HEAD -- - git -C "$repo" rev-parse HEAD' | tail -n 1)" - [ "$repo_sha" = "$CANDIDATE_SHA" ] \ - || die "baked NemoClaw SHA $repo_sha does not match candidate $CANDIDATE_SHA" - [[ "$CANDIDATE_SHA" == "$(jq -r '.gitSha' <<<"$provision")"* ]] \ - || die "provision metadata does not identify candidate $CANDIDATE_SHA" - # Metadata variables are expanded by the remote host shell. The image labels # are the producer contract; the disk and image API responses are independent # observations of what the Launchable actually booted. + local identity repo_sha # shellcheck disable=SC2016 identity="$(host_exec 'set -euo pipefail metadata=http://metadata.google.internal/computeMetadata/v1 @@ -145,6 +126,18 @@ verify_identity() { || die "workspace boot image does not match the exact staging image for $CANDIDATE_SHA" printf '%s\n' "$identity" >"$WORK_DIR/brev-boot-image.json" + # HOME is expanded by the remote host shell. A clean exact checkout confirms + # the baked repository content without depending on a second metadata format. + # shellcheck disable=SC2016 + repo_sha="$(host_exec 'set -euo pipefail + repo="$HOME/NemoClaw" + test -d "$repo/.git" + git -C "$repo" diff --quiet --no-ext-diff HEAD -- + git -C "$repo" diff --cached --quiet --no-ext-diff HEAD -- + git -C "$repo" rev-parse HEAD' | tail -n 1)" + [ "$repo_sha" = "$CANDIDATE_SHA" ] \ + || die "baked NemoClaw SHA $repo_sha does not match candidate $CANDIDATE_SHA" + jq -n \ --arg candidateSha "$CANDIDATE_SHA" \ --arg repositorySha "$repo_sha" \ @@ -235,7 +228,7 @@ cleanup() { timeout 60s brev delete "$INSTANCE_NAME" || true fi - deadline=$((SECONDS + ${BREV_DELETE_TIMEOUT_SECONDS:-600})) + deadline=$((SECONDS + ${BREV_DELETE_TIMEOUT_SECONDS:-1200})) while [ "$SECONDS" -lt "$deadline" ]; do if record="$(workspace_record)"; then if [ -z "$record" ]; then