refactor(tests): distill safe-output smoke suite from 27 to 5 pipelines

.gitattributes

@@ -19,6 +19,7 @@ tests/fixtures/stage-agent.lock.yml linguist-generated=true merge=ours text eol=
 tests/safe-outputs/add-build-tag.lock.yml linguist-generated=true merge=ours text eol=lf
 tests/safe-outputs/add-pr-comment.lock.yml linguist-generated=true merge=ours text eol=lf
 tests/safe-outputs/azure-cli.lock.yml linguist-generated=true merge=ours text eol=lf
+tests/safe-outputs/canary.lock.yml linguist-generated=true merge=ours text eol=lf
 tests/safe-outputs/comment-on-work-item.lock.yml linguist-generated=true merge=ours text eol=lf
 tests/safe-outputs/create-branch.lock.yml linguist-generated=true merge=ours text eol=lf
 tests/safe-outputs/create-git-tag.lock.yml linguist-generated=true merge=ours text eol=lf

scripts/ado-script/src/executor-e2e/scenarios/index.ts

@@ -7,11 +7,13 @@ import { buildScenarios } from "./build.js";
 import { createPullRequestScenarios } from "./create-pull-request.js";
 import { gitScenarios } from "./git.js";
 import { prScenarios } from "./pr.js";
+import { signalScenarios } from "./signals.js";
 import { wikiScenarios } from "./wiki.js";
 import { workItemScenarios } from "./work-item.js";
 
 /** Every scenario, in a deterministic run order. */
 export const allScenarios: Scenario<unknown>[] = [
+  ...signalScenarios,
   ...workItemScenarios,
   ...wikiScenarios,
   ...prScenarios,

scripts/ado-script/src/executor-e2e/scenarios/signals.ts

@@ -0,0 +1,99 @@
+/**
+ * Signal safe-output scenarios: noop, missing-tool, missing-data,
+ * report-incomplete.
+ *
+ * These tools have no ADO write path — they emit a signal record to
+ * `safe_outputs.ndjson` and the executor writes back the result without
+ * touching any ADO API. Accordingly:
+ *   - `setup()` is trivial (no REST calls, returns an empty state).
+ *   - `assert()` is a no-op for the succeeded tools; `report-incomplete`
+ *     uses `expectedFailure` so `assert()` is never reached.
+ *   - `cleanup()` is a no-op.
+ *
+ * Adding these scenarios closes the executor-e2e coverage gap left by
+ * removing the dedicated per-tool agentic smoke pipelines.
+ *
+ * Test-harness module; not shipped in `ado-script.zip`.
+ */
+import type { Scenario } from "../scenario.js";
+
+export const noop: Scenario<unknown> = {
+  tool: "noop",
+  config: () => ({}),
+  setup: async () => ({}),
+  ndjson: async (ctx) => ({
+    context: `deterministic executor e2e noop for build ${ctx.buildId}`,
+  }),
+  assert: async () => {
+    /* no ADO side effect to verify */
+  },
+  cleanup: async () => {
+    /* nothing to tear down */
+  },
+};
+
+export const missingTool: Scenario<unknown> = {
+  id: "missing-tool",
+  tool: "missing-tool",
+  config: () => ({}),
+  setup: async () => ({}),
+  ndjson: async (ctx) => ({
+    tool_name: `ado-aw-det-${ctx.buildId}-bash`,
+    context: `deterministic executor e2e missing-tool for build ${ctx.buildId}`,
+  }),
+  assert: async () => {
+    /* no ADO side effect to verify */
+  },
+  cleanup: async () => {
+    /* nothing to tear down */
+  },
+};
+
+export const missingData: Scenario<unknown> = {
+  id: "missing-data",
+  tool: "missing-data",
+  config: () => ({}),
+  setup: async () => ({}),
+  ndjson: async (ctx) => ({
+    data_type: "deterministic-e2e-data-type",
+    reason: `deterministic executor e2e missing-data for build ${ctx.buildId}`,
+  }),
+  assert: async () => {
+    /* no ADO side effect to verify */
+  },
+  cleanup: async () => {
+    /* nothing to tear down */
+  },
+};
+
+export const reportIncomplete: Scenario<unknown> = {
+  id: "report-incomplete",
+  tool: "report-incomplete",
+  config: () => ({}),
+  setup: async () => ({}),
+  ndjson: async (ctx) => ({
+    // reason must be >= 10 characters (validated by ReportIncompleteParams).
+    reason: `deterministic executor e2e report-incomplete for build ${ctx.buildId}`,
+    context: `build ${ctx.buildId}`,
+  }),
+  // report-incomplete always returns ExecutionResult::failure(), so the
+  // executor writes status="failed". Declare that as the expected outcome so
+  // the runner treats it as a pass rather than a suite failure.
+  expectedFailure: {
+    status: "failed",
+    error: /Agent reported task incomplete/,
+  },
+  assert: async () => {
+    throw new Error("report-incomplete expected failure should have been caught before assert");
+  },
+  cleanup: async () => {
+    /* nothing to tear down */
+  },
+};
+
+export const signalScenarios: Scenario<unknown>[] = [
+  noop,
+  missingTool,
+  missingData,
+  reportIncomplete,
+];

src/audit/pipeline_graph.rs

@@ -219,7 +219,7 @@ mod tests {
         let source_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
             .join("tests")
             .join("safe-outputs")
-            .join("create-pull-request.md");
+            .join("canary.md");
         let aw_info = serde_json::json!({
             "source": source_path.display().to_string(),
             "target": "standalone"

src/inspect/lint.rs

@@ -449,8 +449,8 @@ mod tests {
     async fn create_pull_request_fixture_has_no_unused_output_inspect_lint() {
         let fixture = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
             .join("tests")
-            .join("safe-outputs")
-            .join("create-pull-request.md");
+            .join("fixtures")
+            .join("complete-agent.md");
         let (_fm, pipeline) = crate::compile::build_pipeline_ir(&fixture).await.unwrap();
         let summary = PipelineSummary::from_pipeline(&pipeline).unwrap();
         let findings = lint(&summary);

src/mcp_author/tests.rs

@@ -11,7 +11,7 @@ fn fixture_path() -> String {
     PathBuf::from(env!("CARGO_MANIFEST_DIR"))
         .join("tests")
         .join("safe-outputs")
-        .join("create-pull-request.md")
+        .join("canary.md")
         .display()
         .to_string()
 }

tests/executor-e2e/README.md

@@ -5,10 +5,11 @@ This directory holds a **deterministic**, non-agentic end-to-end test of the
 
 ## Why this exists
 
-The daily smoke suite in [`tests/safe-outputs/`](../safe-outputs/) exercises
-each safe output end-to-end, but it drives Stage 3 by having an **LLM agent
-(Stage 1) emit the safe output** — so a failure can be the model's fault and the
-suite is inherently flaky.
+The agentic smoke suite in [`tests/safe-outputs/`](../safe-outputs/) exercises
+the full Stage 1 → 2 → 3 pipeline shape, but it drives Stage 3 by having an
+**LLM agent (Stage 1) emit the safe output** — so a failure can be the model's
+fault and the suite is inherently flaky. It also only runs a small number of
+omnibus pipelines rather than one per tool.
 
 This suite removes the LLM from the loop. For every ADO-write safe output it:
 
@@ -41,8 +42,10 @@ gitignored, non-root path) and is **deliberately excluded** from the released
 ## Coverage
 
 All deterministically-assertable ADO-write safe outputs plus the flagship
-`create-pull-request`:
+`create-pull-request`, and the four signal-only tools:
 
+- **Signals:** `noop`, `missing-tool`, `missing-data`, `report-incomplete`
+  (no ADO write path; assert that the executor emits the expected status)
 - **Work items:** `create-work-item`, `update-work-item`,
   `comment-on-work-item`, `link-work-items`, `upload-workitem-attachment`
 - **Wiki:** `create-wiki-page`, `update-wiki-page`
@@ -53,9 +56,12 @@ All deterministically-assertable ADO-write safe outputs plus the flagship
   `upload-pipeline-artifact`
 - **Flagship:** `create-pull-request`
 
-Excluded (nothing to assert or out of scope): the NDJSON-only tools (`noop`,
-`missing-data`, `missing-tool`, `report-incomplete`) and the GitHub-only
-`create-issue`.
+Excluded (out of scope or GitHub-only): the GitHub-only `create-issue`.
+
+> **Coverage note.** The signal scenarios (`noop`, `missing-tool`,
+> `missing-data`, `report-incomplete`) were previously exercised only by
+> now-deleted per-tool agentic smoke pipelines. Adding them here closes
+> the coverage gap while keeping the test deterministic.
 
 ### Scenarios that skip when a precondition is missing
 

tests/inspect_integration.rs

@@ -27,7 +27,7 @@ fn fixture_copy(fixture_name: &str) -> (tempfile::TempDir, PathBuf) {
 
 #[test]
 fn inspect_emits_pipeline_summary_text() {
-    let (_workspace, src) = fixture_copy("create-pull-request.md");
+    let (_workspace, src) = fixture_copy("canary.md");
     let out = Command::new(binary_path())
         .arg("inspect")
         .arg(&src)
@@ -55,7 +55,7 @@ fn inspect_emits_pipeline_summary_text() {
 
 #[test]
 fn inspect_json_emits_schema_version_one() {
-    let (_workspace, src) = fixture_copy("create-pull-request.md");
+    let (_workspace, src) = fixture_copy("canary.md");
     let out = Command::new(binary_path())
         .arg("inspect")
         .arg(&src)
@@ -85,7 +85,7 @@ fn inspect_json_emits_schema_version_one() {
 
 #[test]
 fn graph_dot_emits_digraph_with_known_edges() {
-    let (_workspace, src) = fixture_copy("create-pull-request.md");
+    let (_workspace, src) = fixture_copy("canary.md");
     let out = Command::new(binary_path())
         .arg("graph")
         .arg("dump")
@@ -119,7 +119,7 @@ fn graph_dot_emits_digraph_with_known_edges() {
 
 #[test]
 fn graph_rejects_unknown_format() {
-    let (_workspace, src) = fixture_copy("create-pull-request.md");
+    let (_workspace, src) = fixture_copy("canary.md");
     let out = Command::new(binary_path())
         .arg("graph")
         .arg("dump")