diff --git a/scripts/ado-script/src/executor-e2e/__tests__/common.test.ts b/scripts/ado-script/src/executor-e2e/__tests__/common.test.ts index e3431316..8af81ca0 100644 --- a/scripts/ado-script/src/executor-e2e/__tests__/common.test.ts +++ b/scripts/ado-script/src/executor-e2e/__tests__/common.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest"; -import { Teardown } from "../scenarios/common.js"; +import { stagedSafeOutputFile, Teardown } from "../scenarios/common.js"; describe("Teardown", () => { it("runs every step even when an earlier one throws", async () => { @@ -65,3 +65,41 @@ describe("Teardown", () => { expect(order).toEqual([1, 2, 3]); }); }); + +describe("stagedSafeOutputFile", () => { + it("emits the staged result fields required by staged-file executors", () => { + const staged = stagedSafeOutputFile( + "upload-build-attachment", + "ado-aw-det-123", + "build-att.txt", + "hello\n", + ); + + expect(staged.result).toEqual({ + file_path: "build-att.txt", + staged_file: "upload-build-attachment-ado-aw-det-123-e2e.txt", + file_size: 6, + staged_sha256: "5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03", + }); + expect(staged.files).toEqual({ + "build-att.txt": "hello\n", + "upload-build-attachment-ado-aw-det-123-e2e.txt": "hello\n", + }); + }); + + it("records byte length for UTF-8 content and handles extensionless files", () => { + const staged = stagedSafeOutputFile( + "upload-pipeline-artifact", + "ado-aw-det-art-123", + "artifact", + "£\n", + ); + + expect(staged.result.file_size).toBe(3); + expect(staged.result.staged_file).toBe("upload-pipeline-artifact-ado-aw-det-art-123-e2e"); + expect(staged.files).toEqual({ + artifact: "£\n", + "upload-pipeline-artifact-ado-aw-det-art-123-e2e": "£\n", + }); + }); +}); diff --git a/scripts/ado-script/src/executor-e2e/__tests__/runner.test.ts b/scripts/ado-script/src/executor-e2e/__tests__/runner.test.ts index 406dd96e..bae31b31 100644 --- a/scripts/ado-script/src/executor-e2e/__tests__/runner.test.ts +++ b/scripts/ado-script/src/executor-e2e/__tests__/runner.test.ts @@ -1,4 +1,6 @@ +import { mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; +import { join } from "node:path"; import { describe, expect, it } from "vitest"; @@ -79,3 +81,51 @@ describe("runScenario precondition handling", () => { expect(flags.cleaned).toBe(false); }); }); + +describe("runScenario expected executor failures", () => { + it("passes an expected executor rejection without running assertions", async () => { + const dir = await mkdtemp(join(tmpdir(), "ado-aw-runner-test-")); + try { + const bin = join(dir, "fake-ado-aw.js"); + await writeFile( + bin, + `#!/usr/bin/env node +const fs = require("node:fs"); +const path = require("node:path"); +const out = process.argv[process.argv.indexOf("--safe-output-dir") + 1]; +fs.writeFileSync(path.join(out, "safe-outputs-executed.ndjson"), JSON.stringify({ + name: "upload_pipeline_artifact", + status: "failed", + error: "SHA-256 mismatch: expected 0000, got abcd", +}) + "\\n"); +`, + { encoding: "utf8", mode: 0o755 }, + ); + + let asserted = false; + let cleaned = false; + const scenario: Scenario = { + id: "upload-pipeline-artifact-sha-mismatch", + tool: "upload-pipeline-artifact", + config: () => ({}), + setup: async () => ({}), + ndjson: async () => ({}), + expectedFailure: { status: "failed", error: /SHA-256 mismatch/ }, + assert: async () => { + asserted = true; + }, + cleanup: async () => { + cleaned = true; + }, + }; + + const res = await runScenario({ ...fakeCtx(), adoAwBin: bin, workDir: dir }, scenario); + + expect(res).toMatchObject({ ok: true, tool: "upload-pipeline-artifact-sha-mismatch" }); + expect(asserted).toBe(false); + expect(cleaned).toBe(true); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/scripts/ado-script/src/executor-e2e/runner.ts b/scripts/ado-script/src/executor-e2e/runner.ts index f3cfb7cb..a6d958db 100644 --- a/scripts/ado-script/src/executor-e2e/runner.ts +++ b/scripts/ado-script/src/executor-e2e/runner.ts @@ -24,13 +24,14 @@ export async function runScenario( ): Promise { const start = Date.now(); const tool = scenario.tool; - const scenarioDir = join(ctx.workDir, tool); + const scenarioId = scenario.id ?? tool; + const scenarioDir = join(ctx.workDir, scenarioId); let state: S | undefined; let setupDone = false; const finish = (partial: Omit): ScenarioResult => ({ - tool, + tool: scenarioId, durationMs: Date.now() - start, ...partial, }); @@ -46,7 +47,7 @@ export async function runScenario( } // ---- setup ---- - ctx.log(`[${tool}] setup`); + ctx.log(`[${scenarioId}] setup`); try { state = await scenario.setup(ctx); // IMPORTANT: cleanup runs ONLY after this point (guarded by setupDone in @@ -56,7 +57,7 @@ export async function runScenario( setupDone = true; } catch (err) { if (err instanceof SkipError) { - ctx.log(`[${tool}] SKIPPED: ${err.message}`); + ctx.log(`[${scenarioId}] SKIPPED: ${err.message}`); return finish({ ok: true, skipped: true, phase: "skipped", message: err.message }); } return finish({ ok: false, phase: "setup", message: errMessage(err) }); @@ -105,6 +106,16 @@ export async function runScenario( }); } if (result.record.status !== "succeeded") { + const expected = scenario.expectedFailure; + const error = result.record.error ?? ""; + if ( + expected && + (expected.status === undefined || result.record.status === expected.status) && + expected.error.test(error) + ) { + ctx.log(`[${scenarioId}] expected failure: ${error}`); + return finish({ ok: true }); + } return finish({ ok: false, phase: "execute", @@ -119,7 +130,7 @@ export async function runScenario( return finish({ ok: false, phase: "assert", message: errMessage(err) }); } - ctx.log(`[${tool}] OK`); + ctx.log(`[${scenarioId}] OK`); return finish({ ok: true }); } finally { // ---- cleanup (always, best-effort) ---- @@ -129,9 +140,9 @@ export async function runScenario( if (setupDone) { try { await scenario.cleanup(ctx, state as S); - ctx.log(`[${tool}] cleanup done`); + ctx.log(`[${scenarioId}] cleanup done`); } catch (err) { - ctx.log(`[${tool}] cleanup WARNING: ${errMessage(err)}`); + ctx.log(`[${scenarioId}] cleanup WARNING: ${errMessage(err)}`); } } } diff --git a/scripts/ado-script/src/executor-e2e/scenario.ts b/scripts/ado-script/src/executor-e2e/scenario.ts index 4fd2d81d..622a4fb1 100644 --- a/scripts/ado-script/src/executor-e2e/scenario.ts +++ b/scripts/ado-script/src/executor-e2e/scenario.ts @@ -58,6 +58,11 @@ export interface ScenarioContext { * assertion and cleanup. */ export interface Scenario { + /** + * Unique harness id for reporting and scratch directories. Defaults to + * `tool`; set this when multiple scenarios exercise the same safe-output. + */ + readonly id?: string; /** kebab-case safe-output tool name (matches the `safe-outputs:` key). */ readonly tool: string; /** @@ -97,6 +102,14 @@ export interface Scenario { * child process (e.g. BUILD_SOURCESDIRECTORY pointing at a git checkout). */ env?(ctx: ScenarioContext, state: State): Promise>; + /** + * Some scenarios intentionally submit invalid staged output and should pass + * only when the executor rejects it with the expected failure. + */ + readonly expectedFailure?: { + readonly status?: string; + readonly error: RegExp; + }; /** * Assert the ADO side-effect actually happened. Throw on failure. * diff --git a/scripts/ado-script/src/executor-e2e/scenarios/build.ts b/scripts/ado-script/src/executor-e2e/scenarios/build.ts index a453437a..edc79a38 100644 --- a/scripts/ado-script/src/executor-e2e/scenarios/build.ts +++ b/scripts/ado-script/src/executor-e2e/scenarios/build.ts @@ -10,7 +10,8 @@ */ import type { Scenario, ScenarioContext } from "../scenario.js"; import { SkipError } from "../scenario.js"; -import { numResult, requireEnv, strResult } from "./common.js"; +import { numResult, requireEnv, stagedSafeOutputFile, strResult } from "./common.js"; +import type { StagedSafeOutputFile } from "./common.js"; async function currentBuildId(ctx: ScenarioContext, tool: string): Promise { const raw = process.env.BUILD_BUILDID?.trim(); @@ -73,16 +74,27 @@ export const queueBuild: Scenario<{ pipelineId: number; queuedBuildId?: number } }, }; -export const uploadBuildAttachment: Scenario<{ buildId: number }> = { +export const uploadBuildAttachment: Scenario<{ + buildId: number; + artifactName: string; + staged: StagedSafeOutputFile; +}> = { tool: "upload-build-attachment", config: () => ({ "allowed-extensions": ["txt"], max: 1 }), - setup: async (ctx) => ({ buildId: await currentBuildId(ctx, "upload-build-attachment") }), - files: async (ctx) => ({ - "build-att.txt": `deterministic build attachment for build ${ctx.buildId}\n`, - }), - ndjson: async (ctx) => ({ - artifact_name: `ado-aw-det-${ctx.buildId}`, - file_path: "build-att.txt", + setup: async (ctx) => { + const buildId = await currentBuildId(ctx, "upload-build-attachment"); + const artifactName = `ado-aw-det-${buildId}`; + const contents = `deterministic build attachment for build ${buildId}\n`; + return { + buildId, + artifactName, + staged: stagedSafeOutputFile("upload-build-attachment", artifactName, "build-att.txt", contents), + }; + }, + files: async (_ctx, state) => state.staged.files, + ndjson: async (_ctx, state) => ({ + artifact_name: state.artifactName, + ...state.staged.result, }), assert: async (_ctx, _state, record) => { // The executor returns an attachment_url only after a successful ADO POST. @@ -93,16 +105,27 @@ export const uploadBuildAttachment: Scenario<{ buildId: number }> = { }, }; -export const uploadPipelineArtifact: Scenario<{ buildId: number }> = { +export const uploadPipelineArtifact: Scenario<{ + buildId: number; + artifactName: string; + staged: StagedSafeOutputFile; +}> = { tool: "upload-pipeline-artifact", config: () => ({ "allowed-extensions": ["txt"], max: 1 }), - setup: async (ctx) => ({ buildId: await currentBuildId(ctx, "upload-pipeline-artifact") }), - files: async (ctx) => ({ - "artifact.txt": `deterministic pipeline artifact for build ${ctx.buildId}\n`, - }), - ndjson: async (ctx) => ({ - artifact_name: `ado-aw-det-art-${ctx.buildId}`, - file_path: "artifact.txt", + setup: async (ctx) => { + const buildId = await currentBuildId(ctx, "upload-pipeline-artifact"); + const artifactName = `ado-aw-det-art-${buildId}`; + const contents = `deterministic pipeline artifact for build ${buildId}\n`; + return { + buildId, + artifactName, + staged: stagedSafeOutputFile("upload-pipeline-artifact", artifactName, "artifact.txt", contents), + }; + }, + files: async (_ctx, state) => state.staged.files, + ndjson: async (_ctx, state) => ({ + artifact_name: state.artifactName, + ...state.staged.result, }), assert: async (_ctx, _state, record) => { strResult(record, "download_url"); @@ -112,9 +135,46 @@ export const uploadPipelineArtifact: Scenario<{ buildId: number }> = { }, }; +export const uploadPipelineArtifactShaMismatch: Scenario<{ + buildId: number; + artifactName: string; + staged: StagedSafeOutputFile; +}> = { + id: "upload-pipeline-artifact-sha-mismatch", + tool: "upload-pipeline-artifact", + config: () => ({ "allowed-extensions": ["txt"], max: 1 }), + setup: async (ctx) => { + const buildId = await currentBuildId(ctx, "upload-pipeline-artifact-sha-mismatch"); + const artifactName = `ado-aw-det-art-sha-${buildId}`; + const contents = `deterministic pipeline artifact sha mismatch for build ${buildId}\n`; + return { + buildId, + artifactName, + staged: stagedSafeOutputFile("upload-pipeline-artifact", artifactName, "artifact.txt", contents), + }; + }, + files: async (_ctx, state) => state.staged.files, + ndjson: async (_ctx, state) => ({ + artifact_name: state.artifactName, + ...state.staged.result, + staged_sha256: "0".repeat(64), + }), + expectedFailure: { + status: "failed", + error: /SHA-256 mismatch/, + }, + assert: async () => { + throw new Error("expected SHA-256 mismatch to be rejected before assertion"); + }, + cleanup: async () => { + /* rejected before upload; nothing to delete */ + }, +}; + export const buildScenarios: Scenario[] = [ addBuildTag, queueBuild, uploadBuildAttachment, uploadPipelineArtifact, + uploadPipelineArtifactShaMismatch, ]; diff --git a/scripts/ado-script/src/executor-e2e/scenarios/common.ts b/scripts/ado-script/src/executor-e2e/scenarios/common.ts index f732b7f2..4dfc057a 100644 --- a/scripts/ado-script/src/executor-e2e/scenarios/common.ts +++ b/scripts/ado-script/src/executor-e2e/scenarios/common.ts @@ -2,6 +2,9 @@ * Shared helpers for executor E2E scenarios. * Test-harness module; not shipped in `ado-script.zip`. */ +import { createHash } from "node:crypto"; +import { extname } from "node:path"; + import type { ExecutedRecord, ScenarioContext } from "../scenario.js"; import { SkipError } from "../scenario.js"; @@ -52,6 +55,51 @@ export function requireEnv(name: string, tool: string): string { return value; } +export interface StagedSafeOutputFile { + readonly files: Record; + readonly result: { + readonly file_path: string; + readonly staged_file: string; + readonly file_size: number; + readonly staged_sha256: string; + }; +} + +/** + * Build the post-MCP staging shape consumed by safe-output executors that read + * a staged file (`staged_file` + integrity metadata) instead of the raw agent + * params. + */ +export function stagedSafeOutputFile( + tool: string, + artifactName: string, + filePath: string, + contents: string, +): StagedSafeOutputFile { + const extension = extname(filePath) + .slice(1) + .replace(/[^A-Za-z0-9]/g, "") + .slice(0, 16); + const stagedFile = extension + ? `${tool}-${artifactName}-e2e.${extension}` + : `${tool}-${artifactName}-e2e`; + const bytes = Buffer.from(contents, "utf8"); + const files = + stagedFile === filePath + ? { [stagedFile]: contents } + : { [filePath]: contents, [stagedFile]: contents }; + + return { + files, + result: { + file_path: filePath, + staged_file: stagedFile, + file_size: bytes.byteLength, + staged_sha256: createHash("sha256").update(bytes).digest("hex"), + }, + }; +} + /** * Runs a series of *independent* teardown steps, guaranteeing every step is * attempted even if an earlier one throws.