test(executor-e2e): stage upload artifact fixtures

scripts/ado-script/src/executor-e2e/__tests__/common.test.ts

@@ -1,6 +1,6 @@
 import { describe, expect, it } from "vitest";
 
-import { Teardown } from "../scenarios/common.js";
+import { stagedSafeOutputFile, Teardown } from "../scenarios/common.js";
 
 describe("Teardown", () => {
   it("runs every step even when an earlier one throws", async () => {
@@ -65,3 +65,41 @@ describe("Teardown", () => {
     expect(order).toEqual([1, 2, 3]);
   });
 });
+
+describe("stagedSafeOutputFile", () => {
+  it("emits the staged result fields required by staged-file executors", () => {
+    const staged = stagedSafeOutputFile(
+      "upload-build-attachment",
+      "ado-aw-det-123",
+      "build-att.txt",
+      "hello\n",
+    );
+
+    expect(staged.result).toEqual({
+      file_path: "build-att.txt",
+      staged_file: "upload-build-attachment-ado-aw-det-123-e2e.txt",
+      file_size: 6,
+      staged_sha256: "5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03",
+    });
+    expect(staged.files).toEqual({
+      "build-att.txt": "hello\n",
+      "upload-build-attachment-ado-aw-det-123-e2e.txt": "hello\n",
+    });
+  });
+
+  it("records byte length for UTF-8 content and handles extensionless files", () => {
+    const staged = stagedSafeOutputFile(
+      "upload-pipeline-artifact",
+      "ado-aw-det-art-123",
+      "artifact",
+      "£\n",
+    );
+
+    expect(staged.result.file_size).toBe(3);
+    expect(staged.result.staged_file).toBe("upload-pipeline-artifact-ado-aw-det-art-123-e2e");
+    expect(staged.files).toEqual({
+      artifact: "£\n",
+      "upload-pipeline-artifact-ado-aw-det-art-123-e2e": "£\n",
+    });
+  });
+});

scripts/ado-script/src/executor-e2e/__tests__/runner.test.ts

@@ -1,4 +1,6 @@
+import { mkdtemp, rm, writeFile } from "node:fs/promises";
 import { tmpdir } from "node:os";
+import { join } from "node:path";
 
 import { describe, expect, it } from "vitest";
 
@@ -79,3 +81,51 @@ describe("runScenario precondition handling", () => {
     expect(flags.cleaned).toBe(false);
   });
 });
+
+describe("runScenario expected executor failures", () => {
+  it("passes an expected executor rejection without running assertions", async () => {
+    const dir = await mkdtemp(join(tmpdir(), "ado-aw-runner-test-"));
+    try {
+      const bin = join(dir, "fake-ado-aw.js");
+      await writeFile(
+        bin,
+        `#!/usr/bin/env node
+const fs = require("node:fs");
+const path = require("node:path");
+const out = process.argv[process.argv.indexOf("--safe-output-dir") + 1];
+fs.writeFileSync(path.join(out, "safe-outputs-executed.ndjson"), JSON.stringify({
+  name: "upload_pipeline_artifact",
+  status: "failed",
+  error: "SHA-256 mismatch: expected 0000, got abcd",
+}) + "\\n");
+`,
+        { encoding: "utf8", mode: 0o755 },
+      );
+
+      let asserted = false;
+      let cleaned = false;
+      const scenario: Scenario<unknown> = {
+        id: "upload-pipeline-artifact-sha-mismatch",
+        tool: "upload-pipeline-artifact",
+        config: () => ({}),
+        setup: async () => ({}),
+        ndjson: async () => ({}),
+        expectedFailure: { status: "failed", error: /SHA-256 mismatch/ },
+        assert: async () => {
+          asserted = true;
+        },
+        cleanup: async () => {
+          cleaned = true;
+        },
+      };
+
+      const res = await runScenario({ ...fakeCtx(), adoAwBin: bin, workDir: dir }, scenario);
+
+      expect(res).toMatchObject({ ok: true, tool: "upload-pipeline-artifact-sha-mismatch" });
+      expect(asserted).toBe(false);
+      expect(cleaned).toBe(true);
+    } finally {
+      await rm(dir, { recursive: true, force: true });
+    }
+  });
+});

scripts/ado-script/src/executor-e2e/runner.ts

@@ -24,13 +24,14 @@ export async function runScenario<S>(
 ): Promise<ScenarioResult> {
   const start = Date.now();
   const tool = scenario.tool;
-  const scenarioDir = join(ctx.workDir, tool);
+  const scenarioId = scenario.id ?? tool;
+  const scenarioDir = join(ctx.workDir, scenarioId);
 
   let state: S | undefined;
   let setupDone = false;
 
   const finish = (partial: Omit<ScenarioResult, "tool" | "durationMs">): ScenarioResult => ({
-    tool,
+    tool: scenarioId,
     durationMs: Date.now() - start,
     ...partial,
   });
@@ -46,7 +47,7 @@ export async function runScenario<S>(
     }
 
     // ---- setup ----
-    ctx.log(`[${tool}] setup`);
+    ctx.log(`[${scenarioId}] setup`);
     try {
       state = await scenario.setup(ctx);
       // IMPORTANT: cleanup runs ONLY after this point (guarded by setupDone in
@@ -56,7 +57,7 @@ export async function runScenario<S>(
       setupDone = true;
     } catch (err) {
       if (err instanceof SkipError) {
-        ctx.log(`[${tool}] SKIPPED: ${err.message}`);
+        ctx.log(`[${scenarioId}] SKIPPED: ${err.message}`);
         return finish({ ok: true, skipped: true, phase: "skipped", message: err.message });
       }
       return finish({ ok: false, phase: "setup", message: errMessage(err) });
@@ -105,6 +106,16 @@ export async function runScenario<S>(
       });
     }
     if (result.record.status !== "succeeded") {
+      const expected = scenario.expectedFailure;
+      const error = result.record.error ?? "";
+      if (
+        expected &&
+        (expected.status === undefined || result.record.status === expected.status) &&
+        expected.error.test(error)
+      ) {
+        ctx.log(`[${scenarioId}] expected failure: ${error}`);
+        return finish({ ok: true });
+      }
       return finish({
         ok: false,
         phase: "execute",
@@ -119,7 +130,7 @@ export async function runScenario<S>(
       return finish({ ok: false, phase: "assert", message: errMessage(err) });
     }
 
-    ctx.log(`[${tool}] OK`);
+    ctx.log(`[${scenarioId}] OK`);
     return finish({ ok: true });
   } finally {
     // ---- cleanup (always, best-effort) ----
@@ -129,9 +140,9 @@ export async function runScenario<S>(
     if (setupDone) {
       try {
         await scenario.cleanup(ctx, state as S);
-        ctx.log(`[${tool}] cleanup done`);
+        ctx.log(`[${scenarioId}] cleanup done`);
       } catch (err) {
-        ctx.log(`[${tool}] cleanup WARNING: ${errMessage(err)}`);
+        ctx.log(`[${scenarioId}] cleanup WARNING: ${errMessage(err)}`);
       }
     }
   }

scripts/ado-script/src/executor-e2e/scenario.ts

@@ -58,6 +58,11 @@ export interface ScenarioContext {
  * assertion and cleanup.
  */
 export interface Scenario<State = unknown> {
+  /**
+   * Unique harness id for reporting and scratch directories. Defaults to
+   * `tool`; set this when multiple scenarios exercise the same safe-output.
+   */
+  readonly id?: string;
   /** kebab-case safe-output tool name (matches the `safe-outputs:` key). */
   readonly tool: string;
   /**
@@ -97,6 +102,14 @@ export interface Scenario<State = unknown> {
    * child process (e.g. BUILD_SOURCESDIRECTORY pointing at a git checkout).
    */
   env?(ctx: ScenarioContext, state: State): Promise<Record<string, string>>;
+  /**
+   * Some scenarios intentionally submit invalid staged output and should pass
+   * only when the executor rejects it with the expected failure.
+   */
+  readonly expectedFailure?: {
+    readonly status?: string;
+    readonly error: RegExp;
+  };
   /**
    * Assert the ADO side-effect actually happened. Throw on failure.
    *

scripts/ado-script/src/executor-e2e/scenarios/build.ts

@@ -10,7 +10,8 @@
  */
 import type { Scenario, ScenarioContext } from "../scenario.js";
 import { SkipError } from "../scenario.js";
-import { numResult, requireEnv, strResult } from "./common.js";
+import { numResult, requireEnv, stagedSafeOutputFile, strResult } from "./common.js";
+import type { StagedSafeOutputFile } from "./common.js";
 
 async function currentBuildId(ctx: ScenarioContext, tool: string): Promise<number> {
   const raw = process.env.BUILD_BUILDID?.trim();
@@ -73,16 +74,27 @@ export const queueBuild: Scenario<{ pipelineId: number; queuedBuildId?: number }
   },
 };
 
-export const uploadBuildAttachment: Scenario<{ buildId: number }> = {
+export const uploadBuildAttachment: Scenario<{
+  buildId: number;
+  artifactName: string;
+  staged: StagedSafeOutputFile;
+}> = {
   tool: "upload-build-attachment",
   config: () => ({ "allowed-extensions": ["txt"], max: 1 }),
-  setup: async (ctx) => ({ buildId: await currentBuildId(ctx, "upload-build-attachment") }),
-  files: async (ctx) => ({
-    "build-att.txt": `deterministic build attachment for build ${ctx.buildId}\n`,
-  }),
-  ndjson: async (ctx) => ({
-    artifact_name: `ado-aw-det-${ctx.buildId}`,
-    file_path: "build-att.txt",
+  setup: async (ctx) => {
+    const buildId = await currentBuildId(ctx, "upload-build-attachment");
+    const artifactName = `ado-aw-det-${buildId}`;
+    const contents = `deterministic build attachment for build ${buildId}\n`;
+    return {
+      buildId,
+      artifactName,
+      staged: stagedSafeOutputFile("upload-build-attachment", artifactName, "build-att.txt", contents),
+    };
+  },
+  files: async (_ctx, state) => state.staged.files,
+  ndjson: async (_ctx, state) => ({
+    artifact_name: state.artifactName,
+    ...state.staged.result,
   }),
   assert: async (_ctx, _state, record) => {
     // The executor returns an attachment_url only after a successful ADO POST.
@@ -93,16 +105,27 @@ export const uploadBuildAttachment: Scenario<{ buildId: number }> = {
   },
 };
 
-export const uploadPipelineArtifact: Scenario<{ buildId: number }> = {
+export const uploadPipelineArtifact: Scenario<{
+  buildId: number;
+  artifactName: string;
+  staged: StagedSafeOutputFile;
+}> = {
   tool: "upload-pipeline-artifact",
   config: () => ({ "allowed-extensions": ["txt"], max: 1 }),
-  setup: async (ctx) => ({ buildId: await currentBuildId(ctx, "upload-pipeline-artifact") }),
-  files: async (ctx) => ({
-    "artifact.txt": `deterministic pipeline artifact for build ${ctx.buildId}\n`,
-  }),
-  ndjson: async (ctx) => ({
-    artifact_name: `ado-aw-det-art-${ctx.buildId}`,
-    file_path: "artifact.txt",
+  setup: async (ctx) => {
+    const buildId = await currentBuildId(ctx, "upload-pipeline-artifact");
+    const artifactName = `ado-aw-det-art-${buildId}`;
+    const contents = `deterministic pipeline artifact for build ${buildId}\n`;
+    return {
+      buildId,
+      artifactName,
+      staged: stagedSafeOutputFile("upload-pipeline-artifact", artifactName, "artifact.txt", contents),
+    };
+  },
+  files: async (_ctx, state) => state.staged.files,
+  ndjson: async (_ctx, state) => ({
+    artifact_name: state.artifactName,
+    ...state.staged.result,
   }),
   assert: async (_ctx, _state, record) => {
     strResult(record, "download_url");
@@ -112,9 +135,46 @@ export const uploadPipelineArtifact: Scenario<{ buildId: number }> = {
   },
 };
 
+export const uploadPipelineArtifactShaMismatch: Scenario<{
+  buildId: number;
+  artifactName: string;
+  staged: StagedSafeOutputFile;
+}> = {
+  id: "upload-pipeline-artifact-sha-mismatch",
+  tool: "upload-pipeline-artifact",
+  config: () => ({ "allowed-extensions": ["txt"], max: 1 }),
+  setup: async (ctx) => {
+    const buildId = await currentBuildId(ctx, "upload-pipeline-artifact-sha-mismatch");
+    const artifactName = `ado-aw-det-art-sha-${buildId}`;
+    const contents = `deterministic pipeline artifact sha mismatch for build ${buildId}\n`;
+    return {
+      buildId,
+      artifactName,
+      staged: stagedSafeOutputFile("upload-pipeline-artifact", artifactName, "artifact.txt", contents),
+    };
+  },
+  files: async (_ctx, state) => state.staged.files,
+  ndjson: async (_ctx, state) => ({
+    artifact_name: state.artifactName,
+    ...state.staged.result,
+    staged_sha256: "0".repeat(64),
+  }),
+  expectedFailure: {
+    status: "failed",
+    error: /SHA-256 mismatch/,
+  },
+  assert: async () => {
+    throw new Error("expected SHA-256 mismatch to be rejected before assertion");
+  },
+  cleanup: async () => {
+    /* rejected before upload; nothing to delete */
+  },
+};
+
 export const buildScenarios: Scenario<unknown>[] = [
   addBuildTag,
   queueBuild,
   uploadBuildAttachment,
   uploadPipelineArtifact,
+  uploadPipelineArtifactShaMismatch,
 ];

scripts/ado-script/src/executor-e2e/scenarios/common.ts

@@ -2,6 +2,9 @@
  * Shared helpers for executor E2E scenarios.
  * Test-harness module; not shipped in `ado-script.zip`.
  */
+import { createHash } from "node:crypto";
+import { extname } from "node:path";
+
 import type { ExecutedRecord, ScenarioContext } from "../scenario.js";
 import { SkipError } from "../scenario.js";
 
@@ -52,6 +55,51 @@ export function requireEnv(name: string, tool: string): string {
   return value;
 }
 
+export interface StagedSafeOutputFile {
+  readonly files: Record<string, string>;
+  readonly result: {
+    readonly file_path: string;
+    readonly staged_file: string;
+    readonly file_size: number;
+    readonly staged_sha256: string;
+  };
+}
+
+/**
+ * Build the post-MCP staging shape consumed by safe-output executors that read
+ * a staged file (`staged_file` + integrity metadata) instead of the raw agent
+ * params.
+ */
+export function stagedSafeOutputFile(
+  tool: string,
+  artifactName: string,
+  filePath: string,
+  contents: string,
+): StagedSafeOutputFile {
+  const extension = extname(filePath)
+    .slice(1)
+    .replace(/[^A-Za-z0-9]/g, "")
+    .slice(0, 16);
+  const stagedFile = extension
+    ? `${tool}-${artifactName}-e2e.${extension}`
+    : `${tool}-${artifactName}-e2e`;
+  const bytes = Buffer.from(contents, "utf8");
+  const files =
+    stagedFile === filePath
+      ? { [stagedFile]: contents }
+      : { [filePath]: contents, [stagedFile]: contents };
+
+  return {
+    files,
+    result: {
+      file_path: filePath,
+      staged_file: stagedFile,
+      file_size: bytes.byteLength,
+      staged_sha256: createHash("sha256").update(bytes).digest("hex"),
+    },
+  };
+}
+
 /**
  * Runs a series of *independent* teardown steps, guaranteeing every step is
  * attempted even if an earlier one throws.