From 1f0f34ea06b460f52435c455261e1e9063d0d871 Mon Sep 17 00:00:00 2001 From: James Devine Date: Mon, 13 Jul 2026 22:40:38 +0100 Subject: [PATCH 1/6] test(trigger-e2e): add deterministic gate/synth-PR trigger-condition E2E suite Add an ADO-native, non-agentic E2E test framework for the runtime trigger conditions managed by ado-script (gate.js PR filters and exec-context-pr-synth.js synthetic-PR promotion), modeled on the executor-e2e native bundle. A parameterized victim pipeline runs only the two bundles and reports the decision via REST-observable build tags; an orchestrator harness creates real PRs and queues the victim many ways, asserting the gate decision. Test-only bundle: excluded from ado-script.zip exactly like executor-e2e. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: f30758a5-81c8-4024-ab01-52ad22b4f877 --- scripts/ado-script/package.json | 1 + .../src/__tests__/bundle-coverage.test.ts | 6 +- .../ado-script/src/executor-e2e/ado-rest.ts | 87 +++++- .../trigger-e2e/__tests__/gate-spec.test.ts | 110 +++++++ .../src/trigger-e2e/__tests__/runner.test.ts | 117 +++++++ .../ado-script/src/trigger-e2e/gate-spec.ts | 291 ++++++++++++++++++ .../src/trigger-e2e/github-issue.ts | 150 +++++++++ scripts/ado-script/src/trigger-e2e/index.ts | 120 ++++++++ scripts/ado-script/src/trigger-e2e/queue.ts | 37 +++ scripts/ado-script/src/trigger-e2e/runner.ts | 115 +++++++ .../ado-script/src/trigger-e2e/scenario.ts | 156 ++++++++++ .../src/trigger-e2e/scenarios/bypass.ts | 49 +++ .../src/trigger-e2e/scenarios/common.ts | 149 +++++++++ .../src/trigger-e2e/scenarios/index.ts | 21 ++ .../src/trigger-e2e/scenarios/pr-filters.ts | 200 ++++++++++++ .../src/trigger-e2e/scenarios/self-cancel.ts | 55 ++++ .../src/trigger-e2e/scenarios/synth-pr.ts | 117 +++++++ tests/trigger-e2e/README.md | 158 ++++++++++ tests/trigger-e2e/azure-pipelines.yml | 85 +++++ tests/trigger-e2e/victim-pipeline.yml | 150 +++++++++ 20 files changed, 2167 insertions(+), 7 deletions(-) create mode 100644 scripts/ado-script/src/trigger-e2e/__tests__/gate-spec.test.ts create mode 100644 scripts/ado-script/src/trigger-e2e/__tests__/runner.test.ts create mode 100644 scripts/ado-script/src/trigger-e2e/gate-spec.ts create mode 100644 scripts/ado-script/src/trigger-e2e/github-issue.ts create mode 100644 scripts/ado-script/src/trigger-e2e/index.ts create mode 100644 scripts/ado-script/src/trigger-e2e/queue.ts create mode 100644 scripts/ado-script/src/trigger-e2e/runner.ts create mode 100644 scripts/ado-script/src/trigger-e2e/scenario.ts create mode 100644 scripts/ado-script/src/trigger-e2e/scenarios/bypass.ts create mode 100644 scripts/ado-script/src/trigger-e2e/scenarios/common.ts create mode 100644 scripts/ado-script/src/trigger-e2e/scenarios/index.ts create mode 100644 scripts/ado-script/src/trigger-e2e/scenarios/pr-filters.ts create mode 100644 scripts/ado-script/src/trigger-e2e/scenarios/self-cancel.ts create mode 100644 scripts/ado-script/src/trigger-e2e/scenarios/synth-pr.ts create mode 100644 tests/trigger-e2e/README.md create mode 100644 tests/trigger-e2e/azure-pipelines.yml create mode 100644 tests/trigger-e2e/victim-pipeline.yml diff --git a/scripts/ado-script/package.json b/scripts/ado-script/package.json index ab4b6768..7878102b 100644 --- a/scripts/ado-script/package.json +++ b/scripts/ado-script/package.json @@ -25,6 +25,7 @@ "build:github-app-token": "ncc build src/github-app-token/index.ts -o .ado-build/github-app-token -m -t && node -e \"const fs=require('node:fs'); fs.copyFileSync('.ado-build/github-app-token/index.js','github-app-token.js'); fs.rmSync('.ado-build/github-app-token',{recursive:true,force:true});\"", "build:prepare-pr-base": "ncc build src/prepare-pr-base/index.ts -o .ado-build/prepare-pr-base -m -t && node -e \"const fs=require('node:fs'); fs.copyFileSync('.ado-build/prepare-pr-base/index.js','prepare-pr-base.js'); fs.rmSync('.ado-build/prepare-pr-base',{recursive:true,force:true});\"", "build:executor-e2e": "ncc build src/executor-e2e/index.ts -o .ado-build/executor-e2e -m -t && node -e \"const fs=require('node:fs'); fs.mkdirSync('test-bin',{recursive:true}); fs.copyFileSync('.ado-build/executor-e2e/index.js','test-bin/executor-e2e.js'); fs.rmSync('.ado-build/executor-e2e',{recursive:true,force:true});\"", + "build:trigger-e2e": "ncc build src/trigger-e2e/index.ts -o .ado-build/trigger-e2e -m -t && node -e \"const fs=require('node:fs'); fs.mkdirSync('test-bin',{recursive:true}); fs.copyFileSync('.ado-build/trigger-e2e/index.js','test-bin/trigger-e2e.js'); fs.rmSync('.ado-build/trigger-e2e',{recursive:true,force:true});\"", "build:check": "ls -lh gate.js && wc -c gate.js", "codegen": "node -e \"require('node:fs').mkdirSync('schema', { recursive: true })\" && cargo run --quiet --manifest-path ../../Cargo.toml -- export-gate-schema --output schema/gate-spec.schema.json && npx json2ts schema/gate-spec.schema.json -o src/shared/types.gen.ts --bannerComment \"// AUTO-GENERATED from Rust IR via cargo run -- export-gate-schema. Do not edit; run npm run codegen.\"", "test": "vitest run", diff --git a/scripts/ado-script/src/__tests__/bundle-coverage.test.ts b/scripts/ado-script/src/__tests__/bundle-coverage.test.ts index b392d2cd..4e6e89ce 100644 --- a/scripts/ado-script/src/__tests__/bundle-coverage.test.ts +++ b/scripts/ado-script/src/__tests__/bundle-coverage.test.ts @@ -30,8 +30,12 @@ const packageJsonPath = join(here, "..", "..", "package.json"); * so the release glob (`ado-script/*.js`) never packages it — it is a test * harness run only by the executor-e2e pipeline, never downloaded by compiled * agentic pipelines at runtime. + * + * `trigger-e2e` is the analogous deterministic trigger-condition (gate / + * synth-PR) E2E harness. Same treatment: its own `build:trigger-e2e` emits to + * `test-bin/`, so it is never packaged in `ado-script.zip`. */ -const NON_BUNDLE_DIRS = new Set(["shared", "__tests__", "executor-e2e"]); +const NON_BUNDLE_DIRS = new Set(["shared", "__tests__", "executor-e2e", "trigger-e2e"]); function listBundleDirs(): string[] { return readdirSync(srcDir) diff --git a/scripts/ado-script/src/executor-e2e/ado-rest.ts b/scripts/ado-script/src/executor-e2e/ado-rest.ts index fb854c97..1b93155b 100644 --- a/scripts/ado-script/src/executor-e2e/ado-rest.ts +++ b/scripts/ado-script/src/executor-e2e/ado-rest.ts @@ -264,18 +264,21 @@ export class AdoRest { targetRef: string, title: string, description: string, + isDraft?: boolean, ): Promise<{ pullRequestId: number }> { const path = this.projPath( `_apis/git/repositories/${AdoRest.seg(repo)}/pullrequests?api-version=7.1`, ); + const body: Record = { + sourceRefName: sourceRef.startsWith("refs/") ? sourceRef : `refs/heads/${sourceRef}`, + targetRefName: targetRef.startsWith("refs/") ? targetRef : `refs/heads/${targetRef}`, + title, + description, + }; + if (isDraft !== undefined) body.isDraft = isDraft; const res = await this.request<{ pullRequestId: number }>(path, { method: "POST", - body: { - sourceRefName: sourceRef.startsWith("refs/") ? sourceRef : `refs/heads/${sourceRef}`, - targetRefName: targetRef.startsWith("refs/") ? targetRef : `refs/heads/${targetRef}`, - title, - description, - }, + body, }); if (!res) throw new Error("createPullRequest returned no body"); return res; @@ -365,6 +368,20 @@ export class AdoRest { await this.request(path, { method: "PATCH", body: { status: "abandoned" }, allow404: true }); } + /** + * Attach one or more labels (tags) to a PR. ADO's label endpoint takes a + * single `{ name }` per POST, so this loops. Used by the trigger E2E harness + * to exercise the gate's `label_set_match` predicate against real PR labels. + */ + async setPullRequestLabels(repo: string, prId: number, labels: string[]): Promise { + for (const name of labels) { + const path = this.projPath( + `_apis/git/repositories/${AdoRest.seg(repo)}/pullRequests/${prId}/labels?api-version=7.1`, + ); + await this.request(path, { method: "POST", body: { name } }); + } + } + // ---- Wiki ------------------------------------------------------------- async listWikis(): Promise<{ name: string; id: string; type?: string }[]> { @@ -451,4 +468,62 @@ export class AdoRest { const path = this.projPath(`_apis/build/builds/${buildId}?api-version=7.1`); await this.request(path, { method: "PATCH", body: { status: "cancelling" }, allow404: true }); } + + /** + * Queue a new build of a registered definition. `sourceBranch` selects the + * branch to build (used by the trigger E2E harness to point a queued build + * at a PR's source branch so `exec-context-pr-synth` can discover the open + * PR). `templateParameters` supplies the victim pipeline's runtime + * parameters (e.g. the base64 GATE_SPEC / PR_SYNTH_SPEC under test). + * + * Returns the new build id. Note: a build queued via this REST call has + * `Build.Reason = Manual`; the victim relies on the synthetic-PR flag (set + * by `exec-context-pr-synth` from a real open PR) — not the build reason — + * to drive full PR-gate evaluation. + */ + async queueBuild( + definitionId: number, + opts: { sourceBranch?: string; templateParameters?: Record } = {}, + ): Promise<{ id: number }> { + const path = this.projPath(`_apis/build/builds?api-version=7.1`); + const body: Record = { definition: { id: definitionId } }; + if (opts.sourceBranch) { + body.sourceBranch = opts.sourceBranch.startsWith("refs/") + ? opts.sourceBranch + : `refs/heads/${opts.sourceBranch}`; + } + if (opts.templateParameters && Object.keys(opts.templateParameters).length > 0) { + body.templateParameters = opts.templateParameters; + } + const res = await this.request<{ id: number }>(path, { method: "POST", body }); + if (!res) throw new Error(`queueBuild(${definitionId}) returned no body`); + return res; + } + + /** + * Poll a build until it reaches `status === "completed"` (or the timeout + * elapses). Returns the terminal `{ status, result }`. `result` is one of + * `succeeded` | `partiallySucceeded` | `failed` | `canceled`. + */ + async waitForBuild( + buildId: number, + opts: { timeoutMs?: number; pollMs?: number } = {}, + ): Promise<{ status: string; result?: string }> { + const timeoutMs = + opts.timeoutMs ?? (Number(process.env.TRIGGER_E2E_BUILD_TIMEOUT_MS) || 900_000); + const pollMs = opts.pollMs ?? (Number(process.env.TRIGGER_E2E_BUILD_POLL_MS) || 10_000); + const deadline = Date.now() + timeoutMs; + for (;;) { + const build = await this.getBuild(buildId); + if (build.status === "completed") { + return { status: build.status, result: build.result }; + } + if (Date.now() >= deadline) { + throw new Error( + `waitForBuild(${buildId}) timed out after ${timeoutMs}ms (last status='${build.status ?? "?"}')`, + ); + } + await new Promise((r) => setTimeout(r, pollMs)); + } + } } diff --git a/scripts/ado-script/src/trigger-e2e/__tests__/gate-spec.test.ts b/scripts/ado-script/src/trigger-e2e/__tests__/gate-spec.test.ts new file mode 100644 index 00000000..013cceb2 --- /dev/null +++ b/scripts/ado-script/src/trigger-e2e/__tests__/gate-spec.test.ts @@ -0,0 +1,110 @@ +import { describe, expect, it } from "vitest"; + +import { + buildGateSpec, + changeCountCheck, + changedFilesCheck, + draftCheck, + encodeGateSpec, + encodePrSynthSpec, + labelsCheck, + targetBranchCheck, + titleCheck, +} from "../gate-spec.js"; + +const factKinds = (checks: Parameters[1]) => + buildGateSpec("pull-request", checks).facts.map((f) => f.kind); + +describe("buildGateSpec fact derivation", () => { + it("emits no facts and no checks for an empty gate", () => { + const spec = buildGateSpec("pull-request", []); + expect(spec.facts).toEqual([]); + expect(spec.checks).toEqual([]); + expect(spec.context.tag_prefix).toBe("pr-gate"); + expect(spec.context.build_reason).toBe("PullRequest"); + expect(spec.context.bypass_label).toBe("PR"); + }); + + it("derives pr_metadata (skip_dependents) before pr_labels (fail_open) for a labels check", () => { + const spec = buildGateSpec("pull-request", [labelsCheck({ anyOf: ["run-agent"] })]); + const kinds = spec.facts.map((f) => f.kind); + expect(kinds).toEqual(["pr_metadata", "pr_labels"]); + const meta = Object.fromEntries(spec.facts.map((f) => [f.kind, f])); + expect(meta.pr_metadata!.failure_policy).toBe("skip_dependents"); + expect(meta.pr_metadata!.dependencies).toEqual([]); + expect(meta.pr_labels!.failure_policy).toBe("fail_open"); + expect(meta.pr_labels!.dependencies).toEqual(["pr_metadata"]); + }); + + it("derives pr_metadata before pr_is_draft (fail_closed) for a draft check", () => { + expect(factKinds([draftCheck(true)])).toEqual(["pr_metadata", "pr_is_draft"]); + }); + + it("derives changed_files before changed_file_count for a change-count check", () => { + expect(factKinds([changeCountCheck({ min: 5 })])).toEqual([ + "changed_files", + "changed_file_count", + ]); + }); + + it("derives only changed_files for a changed-files check", () => { + expect(factKinds([changedFilesCheck({ include: ["src/**"] })])).toEqual(["changed_files"]); + }); + + it("derives a single fail_closed pipeline-var fact for a title check", () => { + const spec = buildGateSpec("pull-request", [titleCheck("release *")]); + expect(spec.facts).toEqual([ + { kind: "pr_title", failure_policy: "fail_closed", dependencies: [] }, + ]); + }); + + it("dedupes facts shared across multiple checks", () => { + const kinds = factKinds([draftCheck(true), labelsCheck({ anyOf: ["x"] })]); + // pr_metadata appears once, before both dependents, in enum order. + expect(kinds).toEqual(["pr_metadata", "pr_is_draft", "pr_labels"]); + }); +}); + +describe("check tag suffixes match the Rust FilterCheck::build_tag_suffix", () => { + it("emits the expected suffixes", () => { + const cases: [Parameters[1][number], string][] = [ + [titleCheck("x"), "title-mismatch"], + [labelsCheck({ anyOf: ["x"] }), "labels-mismatch"], + [changedFilesCheck({ include: ["src/**"] }), "changed-files-mismatch"], + [draftCheck(true), "draft-mismatch"], + [targetBranchCheck("main"), "target-branch-mismatch"], + [changeCountCheck({ min: 1 }), "changes-mismatch"], + ]; + for (const [check, suffix] of cases) { + const spec = buildGateSpec("pull-request", [check]); + expect(spec.checks[0]!.tag_suffix).toBe(suffix); + } + }); +}); + +describe("encoding", () => { + it("round-trips a gate spec through base64", () => { + const spec = buildGateSpec("pull-request", [labelsCheck({ anyOf: ["run-agent"] })]); + const decoded = JSON.parse(Buffer.from(encodeGateSpec(spec), "base64").toString("utf8")); + expect(decoded).toEqual(spec); + }); + + it("produces the canonical empty PR_SYNTH_SPEC base64", () => { + expect(encodePrSynthSpec()).toBe( + "eyJicmFuY2hlcyI6eyJpbmNsdWRlIjpbXSwiZXhjbHVkZSI6W119LCJwYXRocyI6eyJpbmNsdWRlIjpbXSwiZXhjbHVkZSI6W119fQ==", + ); + }); + + it("encodes provided branch/path filters", () => { + const decoded = JSON.parse( + Buffer.from( + encodePrSynthSpec({ branches: { include: ["main"] }, paths: { exclude: ["docs/**"] } }), + "base64", + ).toString("utf8"), + ); + expect(decoded).toEqual({ + branches: { include: ["main"], exclude: [] }, + paths: { include: [], exclude: ["docs/**"] }, + }); + }); +}); diff --git a/scripts/ado-script/src/trigger-e2e/__tests__/runner.test.ts b/scripts/ado-script/src/trigger-e2e/__tests__/runner.test.ts new file mode 100644 index 00000000..4e26b8ea --- /dev/null +++ b/scripts/ado-script/src/trigger-e2e/__tests__/runner.test.ts @@ -0,0 +1,117 @@ +import { describe, expect, it, vi } from "vitest"; + +import type { AdoRest } from "../../executor-e2e/ado-rest.js"; +import { runScenario } from "../runner.js"; +import { SkipError } from "../scenario.js"; +import type { TriggerContext, TriggerScenario } from "../scenario.js"; + +interface FakeBuild { + result?: string; + tags?: string[]; +} + +function makeCtx(build: FakeBuild): TriggerContext { + const rest = { + queueBuild: vi.fn().mockResolvedValue({ id: 42 }), + waitForBuild: vi.fn().mockResolvedValue({ status: "completed", result: build.result }), + getBuildTags: vi.fn().mockResolvedValue(build.tags ?? []), + } as unknown as AdoRest; + return { + orgUrl: "https://dev.azure.com/org/", + project: "proj", + adoRepo: "repo", + buildId: "1", + token: "t", + victimDefinitionId: 7, + rest, + log: () => {}, + prefix: (id) => `ado-aw-trig-1-${id}`, + }; +} + +function scenario(overrides: Partial>): TriggerScenario { + return { + id: "s", + description: "test scenario", + setup: async () => "state", + queue: () => ({ templateParameters: { gateSpec: "x" } }), + expected: () => ({ result: "succeeded" }), + cleanup: async () => {}, + ...overrides, + }; +} + +describe("runScenario", () => { + it("passes when result and tags match", async () => { + const ctx = makeCtx({ result: "succeeded", tags: ["trig.should-run.true"] }); + const res = await runScenario( + ctx, + scenario({ expected: () => ({ result: "succeeded", tags: ["trig.should-run.true"] }) }), + ); + expect(res.ok).toBe(true); + }); + + it("fails (assert) when a required tag is missing", async () => { + const ctx = makeCtx({ result: "succeeded", tags: [] }); + const res = await runScenario(ctx, scenario({ expected: () => ({ tags: ["pr-gate.skipped"] }) })); + expect(res.ok).toBe(false); + expect(res.phase).toBe("assert"); + expect(res.message).toContain("pr-gate.skipped"); + }); + + it("fails (assert) on the wrong build result", async () => { + const ctx = makeCtx({ result: "canceled" }); + const res = await runScenario(ctx, scenario({ expected: () => ({ result: "succeeded" }) })); + expect(res.ok).toBe(false); + expect(res.phase).toBe("assert"); + }); + + it("fails (assert) when a forbidden tag is present", async () => { + const ctx = makeCtx({ result: "succeeded", tags: ["pr-gate.skipped"] }); + const res = await runScenario( + ctx, + scenario({ expected: () => ({ absentTags: ["pr-gate.skipped"] }) }), + ); + expect(res.ok).toBe(false); + expect(res.phase).toBe("assert"); + }); + + it("records a SkipError from setup as skipped and does NOT run cleanup", async () => { + const ctx = makeCtx({ result: "succeeded" }); + const cleanup = vi.fn().mockResolvedValue(undefined); + const res = await runScenario( + ctx, + scenario({ + setup: async () => { + throw new SkipError("no repo"); + }, + cleanup, + }), + ); + expect(res.skipped).toBe(true); + expect(res.ok).toBe(true); + expect(cleanup).not.toHaveBeenCalled(); + }); + + it("runs cleanup after a successful setup even when assertion fails", async () => { + const ctx = makeCtx({ result: "failed" }); + const cleanup = vi.fn().mockResolvedValue(undefined); + const res = await runScenario(ctx, scenario({ cleanup })); + expect(res.ok).toBe(false); + expect(cleanup).toHaveBeenCalledOnce(); + }); + + it("honours a custom assert hook", async () => { + const ctx = makeCtx({ result: "succeeded" }); + const res = await runScenario( + ctx, + scenario({ + assert: async () => { + throw new Error("custom boom"); + }, + }), + ); + expect(res.ok).toBe(false); + expect(res.message).toContain("custom boom"); + }); +}); diff --git a/scripts/ado-script/src/trigger-e2e/gate-spec.ts b/scripts/ado-script/src/trigger-e2e/gate-spec.ts new file mode 100644 index 00000000..c6d43561 --- /dev/null +++ b/scripts/ado-script/src/trigger-e2e/gate-spec.ts @@ -0,0 +1,291 @@ +/** + * Base64 `GateSpec` / `PrSynthSpec` builders for the trigger-condition E2E + * harness. + * + * These are a faithful TypeScript port of the Rust spec construction in + * `src/compile/filter_ir.rs` (`lower_pr_filters` + `build_gate_spec` + + * `build_pr_synth_spec`). They deliberately import the codegen'd `GateSpec` + * shape from `../shared/types.gen.js` so that any drift in the serialized + * schema surfaces as a TypeScript compile error rather than a silent + * wrong-answer at runtime. + * + * We do NOT run the Rust compiler here: the harness crafts the exact filter + * under test directly, so a single hand-authored victim pipeline can exercise + * every predicate by receiving a different base64 `GATE_SPEC` per queued build. + * + * Fidelity contract (must stay in lock-step with `filter_ir.rs`): + * - fact `kind` strings → `Fact::kind()` + * - fact `failure_policy` → `Fact::failure_policy()` + * - fact `dependencies` → `Fact::dependencies()` + * - canonical fact emission order → `Fact` enum declaration order + * (already topological; see + * `collect_ordered_facts`) + * - check `tag_suffix` values → `FilterCheck::build_tag_suffix` + * + * Test-harness module; not shipped in `ado-script.zip`. + */ +import type { + CheckSpec, + FactSpec, + GateContextSpec, + GateSpec, + PredicateSpec, +} from "../shared/types.gen.js"; + +// ─── Fact metadata (mirror of filter_ir.rs Fact::{kind,failure_policy,dependencies}) ── + +type FailurePolicy = "fail_closed" | "fail_open" | "skip_dependents"; + +interface FactMeta { + readonly policy: FailurePolicy; + readonly deps: readonly string[]; +} + +/** + * Fact kind → metadata. Keys are declared in `Fact` enum order, which the + * evaluator relies on being topological (a dependency always precedes its + * dependents). `buildGateSpec` emits facts in this iteration order. + */ +const FACT_META: ReadonlyMap = new Map([ + ["pr_title", { policy: "fail_closed", deps: [] }], + ["author_email", { policy: "fail_closed", deps: [] }], + ["source_branch", { policy: "fail_closed", deps: [] }], + ["target_branch", { policy: "fail_closed", deps: [] }], + ["commit_message", { policy: "fail_closed", deps: [] }], + ["build_reason", { policy: "fail_closed", deps: [] }], + ["triggered_by_pipeline", { policy: "fail_closed", deps: [] }], + ["triggering_branch", { policy: "fail_closed", deps: [] }], + ["pr_metadata", { policy: "skip_dependents", deps: [] }], + ["pr_is_draft", { policy: "fail_closed", deps: ["pr_metadata"] }], + ["pr_labels", { policy: "fail_open", deps: ["pr_metadata"] }], + ["changed_files", { policy: "fail_open", deps: [] }], + ["changed_file_count", { policy: "fail_open", deps: ["changed_files"] }], + ["current_utc_minutes", { policy: "fail_closed", deps: [] }], +]); + +/** Facts referenced directly by a predicate tree (mirror of predicates.ts collectFacts). */ +function predicateFacts(p: PredicateSpec, out: Set = new Set()): Set { + const fact = (p as { fact?: unknown }).fact; + if (typeof fact === "string") out.add(fact); + if (p.type === "time_window") out.add("current_utc_minutes"); + if (p.type === "and" || p.type === "or") { + for (const sub of p.operands) predicateFacts(sub, out); + } + if (p.type === "not") predicateFacts(p.operand, out); + return out; +} + +/** Add a fact and all its transitive dependencies to `out`. */ +function collectFactWithDeps(kind: string, out: Set): void { + if (out.has(kind)) return; + out.add(kind); + const meta = FACT_META.get(kind); + if (!meta) throw new Error(`gate-spec: unknown fact kind '${kind}'`); + for (const dep of meta.deps) collectFactWithDeps(dep, out); +} + +// ─── Check + context ───────────────────────────────────────────────────────── + +/** A filter check paired with the build tag emitted on failure. */ +export interface Check { + readonly name: string; + readonly tagSuffix: string; + readonly predicate: PredicateSpec; +} + +export type GateContextKind = "pull-request" | "pipeline-completion"; + +function contextSpec(kind: GateContextKind): GateContextSpec { + return kind === "pull-request" + ? { + build_reason: "PullRequest", + tag_prefix: "pr-gate", + step_name: "prGate", + bypass_label: "PR", + } + : { + build_reason: "ResourceTrigger", + tag_prefix: "pipeline-gate", + step_name: "pipelineGate", + bypass_label: "pipeline", + }; +} + +/** + * Assemble a `GateSpec` from a context and a set of checks, deriving the fact + * list (with transitive deps, in canonical order) exactly as `build_gate_spec` + * does in Rust. + */ +export function buildGateSpec(context: GateContextKind, checks: Check[]): GateSpec { + const required = new Set(); + for (const c of checks) { + for (const f of predicateFacts(c.predicate)) collectFactWithDeps(f, required); + } + + const facts: FactSpec[] = []; + for (const [kind, meta] of FACT_META) { + if (!required.has(kind)) continue; + facts.push({ kind, failure_policy: meta.policy, dependencies: [...meta.deps] }); + } + + const checkSpecs: CheckSpec[] = checks.map((c) => ({ + name: c.name, + predicate: c.predicate, + tag_suffix: c.tagSuffix, + })); + + return { context: contextSpec(context), facts, checks: checkSpecs }; +} + +/** Base64-encode a spec for the `GATE_SPEC` env / template parameter. */ +export function encodeGateSpec(spec: GateSpec): string { + return Buffer.from(JSON.stringify(spec), "utf8").toString("base64"); +} + +// ─── Check builders (mirror of lower_pr_filters) ───────────────────────────── + +export function titleCheck(pattern: string): Check { + return { + name: "title", + tagSuffix: "title-mismatch", + predicate: { type: "glob_match", fact: "pr_title", pattern }, + }; +} + +export function sourceBranchCheck(pattern: string): Check { + return { + name: "source-branch", + tagSuffix: "source-branch-mismatch", + predicate: { type: "glob_match", fact: "source_branch", pattern }, + }; +} + +export function targetBranchCheck(pattern: string): Check { + return { + name: "target-branch", + tagSuffix: "target-branch-mismatch", + predicate: { type: "glob_match", fact: "target_branch", pattern }, + }; +} + +export function commitMessageCheck(pattern: string): Check { + return { + name: "commit-message", + tagSuffix: "commit-message-mismatch", + predicate: { type: "glob_match", fact: "commit_message", pattern }, + }; +} + +export function authorIncludeCheck(values: string[]): Check { + return { + name: "author include", + tagSuffix: "author-mismatch", + predicate: { type: "value_in_set", fact: "author_email", values, case_insensitive: true }, + }; +} + +export function authorExcludeCheck(values: string[]): Check { + return { + name: "author exclude", + tagSuffix: "author-excluded", + predicate: { type: "value_not_in_set", fact: "author_email", values, case_insensitive: true }, + }; +} + +export function labelsCheck(opts: { + anyOf?: string[]; + allOf?: string[]; + noneOf?: string[]; +}): Check { + return { + name: "labels", + tagSuffix: "labels-mismatch", + predicate: { + type: "label_set_match", + fact: "pr_labels", + any_of: opts.anyOf ?? [], + all_of: opts.allOf ?? [], + none_of: opts.noneOf ?? [], + }, + }; +} + +export function draftCheck(expected: boolean): Check { + return { + name: "draft", + tagSuffix: "draft-mismatch", + predicate: { type: "equals", fact: "pr_is_draft", value: expected ? "true" : "false" }, + }; +} + +export function changedFilesCheck(opts: { include?: string[]; exclude?: string[] }): Check { + return { + name: "changed-files", + tagSuffix: "changed-files-mismatch", + predicate: { + type: "file_glob_match", + fact: "changed_files", + include: opts.include ?? [], + exclude: opts.exclude ?? [], + }, + }; +} + +export function timeWindowCheck(start: string, end: string): Check { + return { + name: "time-window", + tagSuffix: "time-window-mismatch", + predicate: { type: "time_window", start, end }, + }; +} + +export function changeCountCheck(opts: { min?: number; max?: number }): Check { + return { + name: "change-count", + tagSuffix: "changes-mismatch", + predicate: { + type: "numeric_range", + fact: "changed_file_count", + min: opts.min ?? null, + max: opts.max ?? null, + }, + }; +} + +export function buildReasonIncludeCheck(values: string[]): Check { + return { + name: "build-reason include", + tagSuffix: "build-reason-mismatch", + predicate: { type: "value_in_set", fact: "build_reason", values, case_insensitive: true }, + }; +} + +export function buildReasonExcludeCheck(values: string[]): Check { + return { + name: "build-reason exclude", + tagSuffix: "build-reason-excluded", + predicate: { type: "value_not_in_set", fact: "build_reason", values, case_insensitive: true }, + }; +} + +// ─── PR_SYNTH_SPEC builder (mirror of build_pr_synth_spec) ──────────────────── + +export interface PrSynthSpecInput { + branches?: { include?: string[]; exclude?: string[] }; + paths?: { include?: string[]; exclude?: string[] }; +} + +/** Base64-encode a `PR_SYNTH_SPEC` for the synthetic-PR promotion path. */ +export function encodePrSynthSpec(input: PrSynthSpecInput = {}): string { + const spec = { + branches: { + include: input.branches?.include ?? [], + exclude: input.branches?.exclude ?? [], + }, + paths: { + include: input.paths?.include ?? [], + exclude: input.paths?.exclude ?? [], + }, + }; + return Buffer.from(JSON.stringify(spec), "utf8").toString("base64"); +} diff --git a/scripts/ado-script/src/trigger-e2e/github-issue.ts b/scripts/ado-script/src/trigger-e2e/github-issue.ts new file mode 100644 index 00000000..edac0f97 --- /dev/null +++ b/scripts/ado-script/src/trigger-e2e/github-issue.ts @@ -0,0 +1,150 @@ +/** + * Failure-issue filing for the trigger-condition E2E harness. + * + * Reuses the low-level GitHub client primitives from the executor-e2e harness + * (`findOpenIssueByTitle` / `createGitHubIssue` / `diagnoseGitHubAuthFailure`) + * but frames issues with a trigger-specific title prefix + labels so the two + * suites never dedupe into each other's issues. + * + * Test-harness module; not shipped in `ado-script.zip`. + */ +import { + createGitHubIssue, + diagnoseGitHubAuthFailure, + findOpenIssueByTitle, +} from "../executor-e2e/github-issue.js"; +import type { ScenarioResult } from "./scenario.js"; + +export const ISSUE_TITLE_PREFIX = "[trigger-e2e-failure] "; +const DEFAULT_REPO = "githubnext/ado-aw"; +const DEFAULT_LABELS = ["trigger-e2e-failure", "pipeline-failure"]; +const MAX_TITLE_LEN = 200; + +export interface IssueEnv { + token?: string; + repo: string; + labels: string[]; + buildId?: string; + buildUrl?: string; + project?: string; +} + +/** Treat an UNEXPANDED ADO macro (`$(VAR)`) as absent. */ +function cleanVar(raw: string | undefined): string | undefined { + const value = raw?.trim(); + if (!value || /^\$\(.*\)$/.test(value)) return undefined; + return value; +} + +export function loadIssueEnv(env: NodeJS.ProcessEnv = process.env): IssueEnv { + const labelsRaw = env.TRIGGER_E2E_ISSUE_LABELS?.trim(); + let labels = DEFAULT_LABELS; + if (labelsRaw) { + try { + const parsed: unknown = JSON.parse(labelsRaw); + if (Array.isArray(parsed)) labels = parsed.filter((v): v is string => typeof v === "string"); + } catch { + /* keep defaults */ + } + } + return { + token: env.TRIGGER_E2E_GITHUB_TOKEN?.trim() || env.ADO_AW_DEBUG_GITHUB_TOKEN?.trim(), + repo: cleanVar(env.TRIGGER_E2E_ISSUE_REPO) || DEFAULT_REPO, + labels, + buildId: env.BUILD_BUILDID?.trim(), + buildUrl: + env.TRIGGER_E2E_BUILD_URL?.trim() || + (env.SYSTEM_COLLECTIONURI && env.SYSTEM_TEAMPROJECT && env.BUILD_BUILDID + ? `${env.SYSTEM_COLLECTIONURI.replace(/\/+$/, "")}/${encodeURIComponent(env.SYSTEM_TEAMPROJECT)}/_build/results?buildId=${env.BUILD_BUILDID}` + : undefined), + project: env.SYSTEM_TEAMPROJECT?.trim(), + }; +} + +/** Stable title keyed on the sorted set of failing scenario ids (dedupes). */ +export function buildIssueTitle(failed: ScenarioResult[]): string { + const ids = [...new Set(failed.map((r) => r.id))].sort(); + const title = `${ISSUE_TITLE_PREFIX}${ids.join(", ")}`; + return title.length <= MAX_TITLE_LEN ? title : title.slice(0, MAX_TITLE_LEN); +} + +export function renderIssueBody(results: ScenarioResult[], env: IssueEnv): string { + const failed = results.filter((r) => !r.ok); + const skipped = results.filter((r) => r.skipped); + const passed = results.filter((r) => r.ok && !r.skipped); + + const lines: string[] = [ + "The deterministic trigger-condition (gate / synth-PR) E2E suite reported failures.", + "", + "## Failed scenarios", + "", + "| Scenario | Phase | Message |", + "| --- | --- | --- |", + ...failed.map( + (r) => + `| \`${r.id}\` | ${r.phase ?? "?"} | ${(r.message ?? "").replace(/\r?\n/g, " ").replace(/\|/g, "\\|").slice(0, 400)} |`, + ), + "", + "## Run", + `- Project: ${env.project ?? "unknown"}`, + `- Build ID: ${env.buildId ?? "unknown"}`, + `- Build URL: ${env.buildUrl ?? "unknown"}`, + `- Passed: ${passed.length} | Failed: ${failed.length} | Skipped: ${skipped.length}`, + ]; + if (skipped.length > 0) { + lines.push("", "## Skipped (missing precondition)", ""); + for (const s of skipped) lines.push(`- \`${s.id}\`: ${s.message ?? ""}`); + } + lines.push( + "", + "> Filed automatically by the trigger-e2e pipeline. Re-runs with the same", + "> failing-scenario signature update this issue rather than opening a new one.", + ); + return lines.join("\n"); +} + +export interface FileIssueOutcome { + filed: boolean; + reason?: string; + url?: string; +} + +/** Extract a trailing "HTTP " code from a thrown GitHub client error. */ +function statusFromError(err: unknown): number | undefined { + const message = err instanceof Error ? err.message : String(err); + const match = message.match(/HTTP (\d{3})/); + return match ? Number(match[1]) : undefined; +} + +/** File (or dedupe) a failure issue. No-op with no failures or no token. */ +export async function fileFailureIssue( + results: ScenarioResult[], + env: IssueEnv, + log: (msg: string) => void, + fetchImpl?: typeof fetch, +): Promise { + const failed = results.filter((r) => !r.ok); + if (failed.length === 0) return { filed: false, reason: "no failures" }; + if (!env.token) { + log("no GitHub token configured (TRIGGER_E2E_GITHUB_TOKEN); skipping issue filing"); + return { filed: false, reason: "no token" }; + } + + const opts = { token: env.token, repo: env.repo, fetchImpl }; + log(`filing failure issue to '${env.repo}' (${failed.length} failed scenario(s))`); + const title = buildIssueTitle(failed); + try { + const existing = await findOpenIssueByTitle(opts, title); + if (existing !== undefined) { + log(`open issue #${existing} already tracks this failure signature; skipping`); + return { filed: false, reason: `deduped to #${existing}` }; + } + const url = await createGitHubIssue(opts, title, renderIssueBody(results, env), env.labels); + log(`filed GitHub issue: ${url}`); + return { filed: true, url }; + } catch (err) { + const status = statusFromError(err); + if (status !== undefined) await diagnoseGitHubAuthFailure(opts, status, log); + throw err; + } +} diff --git a/scripts/ado-script/src/trigger-e2e/index.ts b/scripts/ado-script/src/trigger-e2e/index.ts new file mode 100644 index 00000000..6157bea8 --- /dev/null +++ b/scripts/ado-script/src/trigger-e2e/index.ts @@ -0,0 +1,120 @@ +/** + * Entry point for the deterministic trigger-condition (gate / synth-PR) E2E + * harness. + * + * Reads its configuration from the environment (ADO org/project/token, the + * registered victim pipeline definition id, target repo), queues the victim + * pipeline under a battery of real trigger conditions, asserts the observable + * gate decision (build tags + result), files a GitHub issue on failure, and + * exits non-zero when any scenario failed so the pipeline goes red. + * + * Required env: + * - SYSTEM_COLLECTIONURI (or AZURE_DEVOPS_ORG_URL) — ADO collection URI + * - SYSTEM_TEAMPROJECT — ADO project + * - SYSTEM_ACCESSTOKEN — write-capable ADO token (queue builds, cancel, PRs) + * - TRIGGER_E2E_VICTIM_DEFINITION_ID — registered victim pipeline id + * Optional env: + * - TRIGGER_E2E_VICTIM_REPO — ADO Git repo backing the victim's `self` + * (where PRs are created). When unset, PR/synth/gate scenarios SKIP and + * only the bypass baseline runs. + * - TRIGGER_E2E_GITHUB_TOKEN — scoped PAT for issue filing + * - TRIGGER_E2E_ISSUE_REPO — GitHub repo for issues (default githubnext/ado-aw) + * - TRIGGER_E2E_BUILD_TIMEOUT_MS / TRIGGER_E2E_BUILD_POLL_MS — poll tuning + * + * Test-harness module; not shipped in `ado-script.zip`. + */ +import { AdoRest } from "../executor-e2e/ado-rest.js"; +import { fileFailureIssue, loadIssueEnv } from "./github-issue.js"; +import { runAll } from "./runner.js"; +import { allScenarios } from "./scenarios/index.js"; +import type { ScenarioResult, TriggerContext } from "./scenario.js"; + +function requireEnv(name: string, alt?: string): string { + const value = process.env[name]?.trim() || (alt ? process.env[alt]?.trim() : undefined); + if (!value) throw new Error(`required env var ${name}${alt ? ` (or ${alt})` : ""} is not set`); + return value; +} + +function log(msg: string): void { + // Percent-encode a leading '#' so a message cannot smuggle a ##vso command. + process.stdout.write(msg.replace(/^#/gm, "%23") + "\n"); +} + +export function summarise(results: ScenarioResult[]): string { + const passed = results.filter((r) => r.ok && !r.skipped).length; + const failed = results.filter((r) => !r.ok).length; + const skipped = results.filter((r) => r.skipped).length; + const lines = [ + "", + "=== Trigger E2E summary ===", + ...results.map((r) => { + const state = r.skipped ? "SKIP" : r.ok ? "PASS" : "FAIL"; + const suffix = r.ok && !r.skipped ? "" : ` (${r.phase}: ${r.message ?? ""})`; + return ` [${state}] ${r.id}${suffix}`; + }), + `Total: ${results.length} | Passed: ${passed} | Failed: ${failed} | Skipped: ${skipped}`, + ]; + return lines.join("\n"); +} + +export async function main(): Promise { + const orgUrl = requireEnv("SYSTEM_COLLECTIONURI", "AZURE_DEVOPS_ORG_URL"); + const project = requireEnv("SYSTEM_TEAMPROJECT"); + const token = requireEnv("SYSTEM_ACCESSTOKEN"); + const victimDefinitionIdRaw = requireEnv("TRIGGER_E2E_VICTIM_DEFINITION_ID"); + const victimDefinitionId = Number(victimDefinitionIdRaw); + if (!Number.isInteger(victimDefinitionId) || victimDefinitionId <= 0) { + throw new Error( + `TRIGGER_E2E_VICTIM_DEFINITION_ID must be a positive integer (got '${victimDefinitionIdRaw}')`, + ); + } + // Optional: the ADO Git repo backing the victim's `self`. Empty → PR/synth/ + // gate scenarios skip (see requirePrRepo). + const adoRepo = process.env.TRIGGER_E2E_VICTIM_REPO?.trim() || ""; + const buildId = process.env.BUILD_BUILDID?.trim() || `local-${Date.now()}`; + + const rest = new AdoRest({ orgUrl, project, token, log }); + + const ctx: TriggerContext = { + orgUrl, + project, + adoRepo, + buildId, + token, + victimDefinitionId, + rest, + log, + prefix: (id) => `ado-aw-trig-${buildId}-${id}`, + }; + + log( + `Running ${allScenarios.length} trigger E2E scenarios against ${orgUrl}${project} ` + + `(victim def #${victimDefinitionId}${adoRepo ? `, repo ${adoRepo}` : ", no PR repo — PR scenarios will skip"})`, + ); + + const results = await runAll(ctx, allScenarios); + log(summarise(results)); + + const issueEnv = loadIssueEnv(); + try { + await fileFailureIssue(results, issueEnv, log); + } catch (err) { + log(`WARNING: failed to file GitHub issue: ${(err as Error).message}`); + } + + const failed = results.filter((r) => !r.ok).length; + return failed > 0 ? 1 : 0; +} + +// Run as the bundle entry point. Skipped under Vitest so unit tests can import +// these modules without launching the whole suite. +if (process.env.VITEST !== "true") { + main().then( + (code) => process.exit(code), + (err: unknown) => { + const e = err as Error; + log(`trigger-e2e crashed: ${e.stack ?? e.message}`); + process.exit(1); + }, + ); +} diff --git a/scripts/ado-script/src/trigger-e2e/queue.ts b/scripts/ado-script/src/trigger-e2e/queue.ts new file mode 100644 index 00000000..e517eb97 --- /dev/null +++ b/scripts/ado-script/src/trigger-e2e/queue.ts @@ -0,0 +1,37 @@ +/** + * Queue one victim-pipeline build and collect its terminal outcome. + * + * Thin orchestration over the shared {@link AdoRest} client: queue the victim + * definition with the scenario's source branch + template parameters, poll to + * completion, and read the final build tags. The returned {@link BuildOutcome} + * is what the runner asserts against. + * + * Test-harness module; not shipped in `ado-script.zip`. + */ +import type { BuildOutcome, TriggerContext, VictimQueue } from "./scenario.js"; + +/** Drop `undefined` values so ADO receives a flat string→string parameter map. */ +function flattenParams(params: Record): Record { + const out: Record = {}; + for (const [k, v] of Object.entries(params)) { + if (typeof v === "string") out[k] = v; + } + return out; +} + +export async function runVictim(ctx: TriggerContext, queue: VictimQueue): Promise { + const templateParameters = flattenParams(queue.templateParameters); + const queued = await ctx.rest.queueBuild(ctx.victimDefinitionId, { + sourceBranch: queue.sourceBranch, + templateParameters, + }); + ctx.log(` queued victim build #${queued.id}${queue.sourceBranch ? ` on ${queue.sourceBranch}` : ""}`); + + const terminal = await ctx.rest.waitForBuild(queued.id); + const tags = await ctx.rest.getBuildTags(queued.id); + ctx.log( + ` victim build #${queued.id} completed: result=${terminal.result ?? "?"} tags=[${tags.join(", ")}]`, + ); + + return { status: terminal.status, result: terminal.result, tags, buildId: queued.id }; +} diff --git a/scripts/ado-script/src/trigger-e2e/runner.ts b/scripts/ado-script/src/trigger-e2e/runner.ts new file mode 100644 index 00000000..94177eea --- /dev/null +++ b/scripts/ado-script/src/trigger-e2e/runner.ts @@ -0,0 +1,115 @@ +/** + * Scenario runner for the trigger-condition E2E harness. + * + * Runs scenarios sequentially (deterministic ordering; queuing many victim + * builds in parallel would contend for the agent pool). Each scenario is fully + * isolated: a failure or skip never prevents later scenarios from running, and + * cleanup always runs once `setup` succeeded. + * + * Test-harness module; not shipped in `ado-script.zip`. + */ +import { runVictim } from "./queue.js"; +import { SkipError } from "./scenario.js"; +import type { + BuildOutcome, + Expected, + ScenarioResult, + TriggerContext, + TriggerScenario, +} from "./scenario.js"; + +function errMessage(err: unknown): string { + return err instanceof Error ? err.message : String(err); +} + +/** Default declarative assertion: build result + required/forbidden tags. */ +function assertExpected(expected: Expected, outcome: BuildOutcome): void { + const wantResult = expected.result ?? "succeeded"; + if (outcome.result !== wantResult) { + throw new Error( + `expected build result '${wantResult}' but got '${outcome.result ?? "?"}' (tags: [${outcome.tags.join(", ")}])`, + ); + } + for (const tag of expected.tags ?? []) { + if (!outcome.tags.includes(tag)) { + throw new Error(`expected build tag '${tag}' but tags were [${outcome.tags.join(", ")}]`); + } + } + for (const tag of expected.absentTags ?? []) { + if (outcome.tags.includes(tag)) { + throw new Error(`build tag '${tag}' should be absent but tags were [${outcome.tags.join(", ")}]`); + } + } +} + +export async function runScenario( + ctx: TriggerContext, + scenario: TriggerScenario, +): Promise { + const start = Date.now(); + const id = scenario.id; + let state: S | undefined; + let setupDone = false; + + const finish = (partial: Omit): ScenarioResult => ({ + id, + durationMs: Date.now() - start, + ...partial, + }); + + try { + // ---- setup ---- + ctx.log(`[${id}] setup — ${scenario.description}`); + try { + state = await scenario.setup(ctx); + setupDone = true; + } catch (err) { + if (err instanceof SkipError) { + ctx.log(`[${id}] SKIPPED: ${err.message}`); + return finish({ ok: true, skipped: true, phase: "skipped", message: err.message }); + } + return finish({ ok: false, phase: "setup", message: errMessage(err) }); + } + + // ---- queue + poll ---- + let outcome: BuildOutcome; + try { + const queue = scenario.queue(ctx, state); + outcome = await runVictim(ctx, queue); + } catch (err) { + return finish({ ok: false, phase: "queue", message: errMessage(err) }); + } + + // ---- assert ---- + try { + assertExpected(scenario.expected(ctx, state), outcome); + if (scenario.assert) await scenario.assert(ctx, state, outcome); + } catch (err) { + return finish({ ok: false, phase: "assert", message: errMessage(err) }); + } + + ctx.log(`[${id}] OK`); + return finish({ ok: true }); + } finally { + // ---- cleanup (always, best-effort) ---- + if (setupDone) { + try { + await scenario.cleanup(ctx, state as S); + ctx.log(`[${id}] cleanup done`); + } catch (err) { + ctx.log(`[${id}] cleanup WARNING: ${errMessage(err)}`); + } + } + } +} + +export async function runAll( + ctx: TriggerContext, + scenarios: TriggerScenario[], +): Promise { + const results: ScenarioResult[] = []; + for (const scenario of scenarios) { + results.push(await runScenario(ctx, scenario)); + } + return results; +} diff --git a/scripts/ado-script/src/trigger-e2e/scenario.ts b/scripts/ado-script/src/trigger-e2e/scenario.ts new file mode 100644 index 00000000..c814139d --- /dev/null +++ b/scripts/ado-script/src/trigger-e2e/scenario.ts @@ -0,0 +1,156 @@ +/** + * Deterministic trigger-condition (gate / synth-PR) E2E scenario contract. + * + * Unlike the executor E2E harness (which runs `ado-aw execute` directly), each + * trigger scenario exercises the *runtime trigger conditions managed by + * ado-script* — the `gate.js` and `exec-context-pr-synth.js` bundles — by + * queuing a real build of a hand-authored, parameterized **victim pipeline** + * and asserting the observable gate decision via **build tags + build result**. + * + * A scenario optionally sets up real ADO PR context (branch + PR + labels + + * draft state), queues the victim with a per-scenario `GATE_SPEC` / + * `PR_SYNTH_SPEC` (and pipeline-var fact overrides) as template parameters, + * polls the build to completion, then asserts the tags/result and cleans up. + * + * ## Victim observable contract (REST-visible build tags) + * - `trig.synth.promoted` | `trig.synth.skipped` | `trig.synth.real-pr` + * — emitted by the victim after `exec-context-pr-synth.js`. + * - `pr-gate.passed` — gate bypassed (not a PR/synth build). + * - `pr-gate.skipped` + `pr-gate.` — a filter failed; + * the gate self-cancels the build (`result == "canceled"`). + * - `trig.should-run.true` — emitted by a post-gate step that only runs + * when the gate did NOT self-cancel (i.e. `SHOULD_RUN == true`). + * + * Test-harness module; not shipped in `ado-script.zip`. + */ +import type { AdoRest } from "../executor-e2e/ado-rest.js"; + +/** Shared, read-only context handed to every scenario phase. */ +export interface TriggerContext { + /** ADO collection URI, e.g. https://dev.azure.com/msazuresphere/ */ + readonly orgUrl: string; + /** ADO project name, e.g. AgentPlayground. */ + readonly project: string; + /** ADO Git repo used to create real PRs (initialised `main`). */ + readonly adoRepo: string; + /** Current orchestrator build id (drives the per-run object prefix). */ + readonly buildId: string; + /** Write-capable ADO token (System.AccessToken or a PAT). */ + readonly token: string; + /** Registered definition id of the hand-authored victim pipeline. */ + readonly victimDefinitionId: number; + /** REST helper bound to {orgUrl, project, token}. */ + readonly rest: AdoRest; + /** Structured logger (writes to the pipeline log). */ + readonly log: (msg: string) => void; + /** Deterministic object-name prefix: `ado-aw-trig--`. */ + readonly prefix: (id: string) => string; +} + +/** Template parameters + source branch used to queue one victim build. */ +export interface VictimQueue { + /** Source branch the victim builds (points synth at the PR); optional. */ + sourceBranch?: string; + /** Runtime parameters passed to the victim pipeline. */ + templateParameters: VictimParameters; +} + +/** + * Victim pipeline runtime parameters. `gateSpec` is required; the rest are + * optional overrides. Empty overrides fall back to the victim's real ADO + * predefined variables. + */ +export interface VictimParameters { + /** base64 GATE_SPEC (required). */ + gateSpec: string; + /** base64 PR_SYNTH_SPEC (default: empty include/exclude). */ + prSynthSpec?: string; + /** Override for the `pr_title` fact (ADO_PR_TITLE). */ + prTitle?: string; + /** Override for the `author_email` fact (ADO_AUTHOR_EMAIL). */ + authorEmail?: string; + /** Override for the `source_branch` fact (ADO_SOURCE_BRANCH). */ + sourceBranchFact?: string; + /** Override for the `target_branch` fact (ADO_TARGET_BRANCH). */ + targetBranchFact?: string; + /** Override for the `commit_message` fact (ADO_COMMIT_MESSAGE). */ + commitMessage?: string; + /** [key: string]: string — ADO template parameters must be flat strings. */ + [k: string]: string | undefined; +} + +/** Terminal build state + tags read after the victim run completes. */ +export interface BuildOutcome { + /** ADO build status — always `completed` once polling returns. */ + status: string; + /** succeeded | partiallySucceeded | failed | canceled. */ + result?: string; + /** All build tags present at completion. */ + tags: string[]; + /** The queued victim build id (for logging / debugging). */ + buildId: number; +} + +/** Declarative expectation, checked by the runner's default assertion. */ +export interface Expected { + /** Required build result; default `succeeded`. */ + result?: string; + /** Tags that MUST all be present. */ + tags?: string[]; + /** Tags that MUST NOT be present. */ + absentTags?: string[]; +} + +/** + * A single deterministic trigger scenario. + * + * `State` is threaded from `setup` through `queue`/`assert`/`cleanup` so a + * scenario can remember the PR id / branch it created. + */ +export interface TriggerScenario { + /** Unique harness id for reporting and object naming. */ + readonly id: string; + /** One-line human description of what this scenario proves. */ + readonly description: string; + /** + * Create real ADO preconditions (branch/PR/labels); return remembered state. + * + * When this throws (a non-`SkipError`), the runner will NOT call `cleanup()`. + * Any ADO objects partially created before the throw must be torn down + * explicitly inside this function before rethrowing (see `scenarios/common.ts` + * `createPrContext`, which follows the executor-e2e pattern). + */ + setup(ctx: TriggerContext): Promise; + /** Build the victim queue request (source branch + template parameters). */ + queue(ctx: TriggerContext, state: State): VictimQueue; + /** Declarative expectation for the completed victim build. */ + expected(ctx: TriggerContext, state: State): Expected; + /** Optional extra assertion beyond `expected` (throw on failure). */ + assert?(ctx: TriggerContext, state: State, outcome: BuildOutcome): Promise; + /** Best-effort teardown of everything `setup` created. */ + cleanup(ctx: TriggerContext, state: State): Promise; +} + +/** Outcome of running one scenario. */ +export interface ScenarioResult { + id: string; + ok: boolean; + /** "setup" | "queue" | "assert" | "cleanup" | "skipped". */ + phase?: string; + message?: string; + durationMs: number; + /** True when skipped for a missing precondition (not a failure). */ + skipped?: boolean; +} + +/** + * Thrown by a scenario's `setup` when a required precondition is unavailable + * (e.g. the victim definition id was not supplied). The runner records the + * scenario as **skipped** rather than failed. + */ +export class SkipError extends Error { + constructor(reason: string) { + super(reason); + this.name = "SkipError"; + } +} diff --git a/scripts/ado-script/src/trigger-e2e/scenarios/bypass.ts b/scripts/ado-script/src/trigger-e2e/scenarios/bypass.ts new file mode 100644 index 00000000..09a443dc --- /dev/null +++ b/scripts/ado-script/src/trigger-e2e/scenarios/bypass.ts @@ -0,0 +1,49 @@ +/** + * Bypass scenario (`gate/bypass.ts`). + * + * When a build is neither a real PR nor a synth-promoted CI build, the PR gate + * auto-passes ("not a PR build — gate passes automatically") and tags the build + * `pr-gate.passed`. This scenario queues the victim on its default branch with + * NO open PR, so `exec-context-pr-synth` skips and the gate bypasses. + * + * Needs no PR context, so it runs even when `TRIGGER_E2E_VICTIM_REPO` is unset + * — a useful baseline that the victim pipeline is wired correctly. + * + * Test-harness module; not shipped in `ado-script.zip`. + */ +import { buildGateSpec, encodeGateSpec, encodePrSynthSpec, labelsCheck } from "../gate-spec.js"; +import type { TriggerScenario } from "../scenario.js"; + +const bypassManual: TriggerScenario = { + id: "bypass-no-pr", + description: "no PR + Manual build → PR gate bypasses (pr-gate.passed)", + async setup() { + return undefined; + }, + queue() { + // A non-trivial gate spec proves the bypass short-circuits BEFORE any + // filter evaluation (a label check would otherwise fail with no PR). + return { + // No sourceBranch → the victim builds its default branch; no PR exists + // for it, so synth skips and the gate bypasses. + templateParameters: { + gateSpec: encodeGateSpec( + buildGateSpec("pull-request", [labelsCheck({ anyOf: ["run-agent"] })]), + ), + prSynthSpec: encodePrSynthSpec(), + }, + }; + }, + expected() { + return { + result: "succeeded", + tags: ["pr-gate.passed", "trig.synth.skipped"], + absentTags: ["pr-gate.skipped", "trig.synth.promoted"], + }; + }, + async cleanup() { + // Nothing created. + }, +}; + +export const bypassScenarios: TriggerScenario[] = [bypassManual]; diff --git a/scripts/ado-script/src/trigger-e2e/scenarios/common.ts b/scripts/ado-script/src/trigger-e2e/scenarios/common.ts new file mode 100644 index 00000000..f5a46d5e --- /dev/null +++ b/scripts/ado-script/src/trigger-e2e/scenarios/common.ts @@ -0,0 +1,149 @@ +/** + * Shared helpers for trigger E2E scenarios: creating and tearing down real ADO + * PR context. + * + * Reuses the executor-e2e `AdoRest` client and the ctx-free {@link Teardown} + * helper so the create-branch → push-file → open-PR → set-labels pattern (and + * its guaranteed-cleanup teardown) lives in one place, mirroring + * `executor-e2e/scenarios/pr.ts`. + * + * Test-harness module; not shipped in `ado-script.zip`. + */ +import { Teardown } from "../../executor-e2e/scenarios/common.js"; +import { SkipError } from "../scenario.js"; +import type { TriggerContext } from "../scenario.js"; + +/** Remembered state for a created PR (ids needed for assertion + cleanup). */ +export interface PrContext { + readonly repo: string; + readonly prId: number; + /** Short branch name (no `refs/heads/`). */ + readonly branch: string; + /** Full source ref (`refs/heads/`). */ + readonly sourceRef: string; + /** Short target branch name (repo default, e.g. `main`). */ + readonly targetBranch: string; +} + +/** Resolve a repo's default branch short name (e.g. `main`). */ +async function defaultBranchShortName(ctx: TriggerContext, repo: string): Promise { + const info = await ctx.rest.getRepository(repo); + return (info.defaultBranch ?? "refs/heads/main").replace(/^refs\/heads\//, ""); +} + +export interface CreatePrOptions { + /** Scenario id, used for the deterministic branch/PR name prefix. */ + readonly id: string; + /** Extra files to add on the source branch (path → contents). */ + readonly files?: Record; + /** PR labels to attach after creation. */ + readonly labels?: string[]; + /** PR title (default: ` (do not merge)`). */ + readonly title?: string; + /** Open the PR in draft state (for the gate `draft` predicate). */ + readonly draft?: boolean; +} + +/** + * Create a real transient PR against the repo's default branch: pushes a new + * source branch carrying one or more files (so ADO accepts a non-empty diff), + * opens the PR, and attaches any labels. + * + * Follows the executor-e2e contract: because a `setup()` throw means the runner + * will NOT call cleanup, this tears down anything it created before rethrowing. + */ +export async function createPrContext( + ctx: TriggerContext, + opts: CreatePrOptions, +): Promise { + const repo = ctx.adoRepo; + const targetBranch = await defaultBranchShortName(ctx, repo); + const baseSha = await ctx.rest.getRefObjectId(repo, `heads/${targetBranch}`); + if (!baseSha) throw new Error(`could not resolve ${targetBranch} HEAD in repo '${repo}'`); + + const branch = `${ctx.prefix(opts.id)}-src`; + const sourceRef = `refs/heads/${branch}`; + + // Default file guarantees a real diff even when the scenario needs none. + const files = opts.files ?? { + [`/ado-aw-trig/${ctx.buildId}/${opts.id}.md`]: `trigger e2e ${opts.id} for build ${ctx.buildId}. Safe to delete.\n`, + }; + + // Push the branch with the first file, then add the rest in follow-up pushes. + const entries = Object.entries(files); + const [firstPath, firstContent] = entries[0]!; + let baseCommit = baseSha; + baseCommit = await ctx.rest.pushAddFileBranch( + repo, + branch, + baseCommit, + firstPath, + firstContent, + `trigger e2e ${opts.id}`, + ); + + // From here the source branch exists; a later throw must clean it up. + try { + for (const [path, content] of entries.slice(1)) { + baseCommit = await ctx.rest.pushAddFileBranch( + repo, + sourceRef, + baseCommit, + path, + content, + `trigger e2e ${opts.id} (${path})`, + ); + } + + const pr = await ctx.rest.createPullRequest( + repo, + branch, + targetBranch, + opts.title ?? `${ctx.prefix(opts.id)} (do not merge)`, + `Deterministic trigger E2E ${opts.id} for build ${ctx.buildId}. Safe to delete.`, + opts.draft, + ); + + if (opts.labels && opts.labels.length > 0) { + await ctx.rest.setPullRequestLabels(repo, pr.pullRequestId, opts.labels); + } + + return { repo, prId: pr.pullRequestId, branch, sourceRef, targetBranch }; + } catch (err) { + // Best-effort delete of the branch we pushed before the failure. + await ctx.rest.deleteRef(repo, sourceRef).catch(() => {}); + throw err; + } +} + +/** Abandon the PR and delete its source branch — every step always attempted. */ +export async function teardownPrContext(ctx: TriggerContext, pr: PrContext): Promise { + await new Teardown() + .add("abandon PR", () => ctx.rest.abandonPullRequest(pr.repo, pr.prId)) + .add("delete branch", () => ctx.rest.deleteRef(pr.repo, pr.sourceRef)) + .run(); +} + +/** Register cleanup of a queued victim build id (cancel if somehow still running). */ +export async function cancelVictimIfRunning(ctx: TriggerContext, buildId: number): Promise { + const build = await ctx.rest.getBuild(buildId).catch(() => undefined); + if (build && build.status !== "completed") { + await ctx.rest.cancelBuild(buildId).catch(() => {}); + } +} + +/** + * Ensure a real ADO Git repo is available for PR creation. PR/synth/gate + * scenarios need the victim pipeline's own `self` repo (where `exec-context- + * pr-synth` looks up active PRs); when it is not supplied the scenario skips + * rather than fails. + */ +export function requirePrRepo(ctx: TriggerContext): string { + const repo = ctx.adoRepo?.trim(); + if (!repo) { + throw new SkipError( + "TRIGGER_E2E_VICTIM_REPO is not set; supply the ADO Git repo backing the victim pipeline to enable PR/synth/gate scenarios", + ); + } + return repo; +} diff --git a/scripts/ado-script/src/trigger-e2e/scenarios/index.ts b/scripts/ado-script/src/trigger-e2e/scenarios/index.ts new file mode 100644 index 00000000..7aaedca3 --- /dev/null +++ b/scripts/ado-script/src/trigger-e2e/scenarios/index.ts @@ -0,0 +1,21 @@ +/** + * All trigger-condition scenarios, in run order. + * + * Test-harness module; not shipped in `ado-script.zip`. + */ +import type { TriggerScenario } from "../scenario.js"; +import { bypassScenarios } from "./bypass.js"; +import { filterScenarios } from "./pr-filters.js"; +import { selfCancelScenarios } from "./self-cancel.js"; +import { synthScenarios } from "./synth-pr.js"; + +export const allScenarios: TriggerScenario[] = [ + // Baseline that runs without a PR repo. + ...(bypassScenarios as TriggerScenario[]), + // Synthetic-PR promotion outcomes. + ...(synthScenarios as TriggerScenario[]), + // Gate filter pass/skip matrix. + ...(filterScenarios as TriggerScenario[]), + // Explicit self-cancel assertion. + ...(selfCancelScenarios as TriggerScenario[]), +]; diff --git a/scripts/ado-script/src/trigger-e2e/scenarios/pr-filters.ts b/scripts/ado-script/src/trigger-e2e/scenarios/pr-filters.ts new file mode 100644 index 00000000..eb3b43c3 --- /dev/null +++ b/scripts/ado-script/src/trigger-e2e/scenarios/pr-filters.ts @@ -0,0 +1,200 @@ +/** + * Gate filter scenarios (`gate.js`). + * + * Each creates a real PR, promotes it to synthetic-PR semantics (empty synth + * spec = match-all), and queues the victim with a single-filter `GATE_SPEC`. + * The gate then evaluates that filter against the real PR facts (labels / + * changed files / draft state / branch / build reason / change count / time) + * and either lets the run proceed or self-cancels the build. + * + * Assertion via build tags + result: + * - pass → `result=succeeded`, `trig.synth.promoted` + `trig.should-run.true`, + * no `pr-gate.skipped`. + * - skip → `result=canceled`, `trig.synth.promoted` + `pr-gate.skipped` + + * `pr-gate.`, no `trig.should-run.true`. + * + * Test-harness module; not shipped in `ado-script.zip`. + */ +import { + buildGateSpec, + buildReasonIncludeCheck, + changeCountCheck, + changedFilesCheck, + draftCheck, + encodeGateSpec, + encodePrSynthSpec, + labelsCheck, + targetBranchCheck, + timeWindowCheck, + type Check, +} from "../gate-spec.js"; +import type { Expected, TriggerScenario, VictimParameters } from "../scenario.js"; +import { createPrContext, requirePrRepo, teardownPrContext, type PrContext } from "./common.js"; +import type { CreatePrOptions } from "./common.js"; + +/** Promote-everything synth spec: any active PR on the source branch matches. */ +const PROMOTE_ALL = encodePrSynthSpec({ branches: { include: ["main"] } }); + +interface FilterScenarioSpec { + readonly id: string; + readonly description: string; + /** PR preconditions to create (labels / files / draft / title). */ + readonly pr: Omit; + /** The gate check under test. */ + readonly check: Check | ((now: Date) => Check); + /** Whether the gate should let the run proceed (`pass`) or skip it. */ + readonly outcome: "pass" | "skip"; + /** For `skip`, the `pr-gate.` tag the failing check must emit. */ + readonly failTag?: string; + /** Extra victim parameters (pipeline-var fact overrides), if any. */ + readonly params?: Omit; +} + +function makeFilterScenario(spec: FilterScenarioSpec): TriggerScenario { + return { + id: spec.id, + description: spec.description, + async setup(ctx) { + requirePrRepo(ctx); + return createPrContext(ctx, { id: spec.id, ...spec.pr }); + }, + queue(_ctx, state) { + const check = typeof spec.check === "function" ? spec.check(new Date()) : spec.check; + const gateSpec = encodeGateSpec(buildGateSpec("pull-request", [check])); + return { + sourceBranch: state.sourceRef, + templateParameters: { gateSpec, prSynthSpec: PROMOTE_ALL, ...spec.params }, + }; + }, + expected(): Expected { + if (spec.outcome === "pass") { + return { + result: "succeeded", + tags: ["trig.synth.promoted", "trig.should-run.true"], + absentTags: ["pr-gate.skipped"], + }; + } + const tags = ["trig.synth.promoted", "pr-gate.skipped"]; + if (spec.failTag) tags.push(spec.failTag); + return { result: "canceled", tags, absentTags: ["trig.should-run.true"] }; + }, + async cleanup(ctx, state) { + await teardownPrContext(ctx, state); + }, + }; +} + +/** HH:MM string for `minutes` (mod 1440) since UTC midnight. */ +function hhmm(minutes: number): string { + const m = ((minutes % 1440) + 1440) % 1440; + const h = Math.floor(m / 60); + const mm = m % 60; + return `${String(h).padStart(2, "0")}:${String(mm).padStart(2, "0")}`; +} + +const specs: FilterScenarioSpec[] = [ + // ── labels ──────────────────────────────────────────────────────────── + { + id: "labels-pass", + description: "label_set_match any-of matches the PR's labels", + pr: { labels: ["run-agent"] }, + check: labelsCheck({ anyOf: ["run-agent"] }), + outcome: "pass", + }, + { + id: "labels-skip", + description: "label_set_match any-of misses the PR's labels → self-cancel", + pr: { labels: ["do-not-run"] }, + check: labelsCheck({ anyOf: ["run-agent"] }), + outcome: "skip", + failTag: "pr-gate.labels-mismatch", + }, + // ── changed files ───────────────────────────────────────────────────── + { + id: "changed-files-pass", + description: "file_glob_match include matches a changed file", + pr: { files: { "/src/trig-changed-pass.txt": "x\n" } }, + check: changedFilesCheck({ include: ["src/**"] }), + outcome: "pass", + }, + { + id: "changed-files-skip", + description: "file_glob_match include matches no changed file → self-cancel", + pr: { files: { "/docs/trig-changed-skip.md": "x\n" } }, + check: changedFilesCheck({ include: ["src/**"] }), + outcome: "skip", + failTag: "pr-gate.changed-files-mismatch", + }, + // ── draft ───────────────────────────────────────────────────────────── + { + id: "draft-pass", + description: "draft equals true and the PR is a draft", + pr: { draft: true }, + check: draftCheck(true), + outcome: "pass", + }, + { + id: "draft-skip", + description: "draft equals true but PR is not a draft → self-cancel", + pr: {}, + check: draftCheck(true), + outcome: "skip", + failTag: "pr-gate.draft-mismatch", + }, + // ── target branch ───────────────────────────────────────────────────── + { + id: "target-branch-pass", + description: "target_branch glob matches the PR's target (main)", + pr: {}, + check: targetBranchCheck("main"), + outcome: "pass", + }, + { + id: "target-branch-skip", + description: "target_branch glob does not match the PR's target → self-cancel", + pr: {}, + check: targetBranchCheck("release/*"), + outcome: "skip", + failTag: "pr-gate.target-branch-mismatch", + }, + // ── build reason ────────────────────────────────────────────────────── + { + id: "build-reason-pass", + description: "build_reason include matches the queued build reason (Manual)", + pr: {}, + check: buildReasonIncludeCheck(["Manual"]), + outcome: "pass", + }, + { + id: "build-reason-skip", + description: "build_reason include requires PullRequest but reason is Manual → self-cancel", + pr: {}, + check: buildReasonIncludeCheck(["PullRequest"]), + outcome: "skip", + failTag: "pr-gate.build-reason-mismatch", + }, + // ── change count ────────────────────────────────────────────────────── + { + id: "change-count-skip", + description: "numeric_range min=5 but the PR changed 1 file → self-cancel", + pr: { files: { "/src/trig-count-skip.txt": "x\n" } }, + check: changeCountCheck({ min: 5 }), + outcome: "skip", + failTag: "pr-gate.changes-mismatch", + }, + // ── time window ─────────────────────────────────────────────────────── + { + id: "time-window-skip", + description: "time_window that excludes now → self-cancel", + pr: {}, + // A 1-hour window starting 2h in the future never contains the eval time. + check: (now: Date) => { + const nowMin = now.getUTCHours() * 60 + now.getUTCMinutes(); + return timeWindowCheck(hhmm(nowMin + 120), hhmm(nowMin + 180)); + }, + outcome: "skip", + failTag: "pr-gate.time-window-mismatch", + }, +]; + +export const filterScenarios: TriggerScenario[] = specs.map(makeFilterScenario); diff --git a/scripts/ado-script/src/trigger-e2e/scenarios/self-cancel.ts b/scripts/ado-script/src/trigger-e2e/scenarios/self-cancel.ts new file mode 100644 index 00000000..f8c1fdd3 --- /dev/null +++ b/scripts/ado-script/src/trigger-e2e/scenarios/self-cancel.ts @@ -0,0 +1,55 @@ +/** + * Self-cancel scenario (`gate/selfcancel.ts`). + * + * When the gate decides not to run (a filter fails on a synth-promoted build), + * it self-cancels the whole build via the ADO REST API and tags it + * `pr-gate.skipped`. This scenario explicitly asserts that the build reaches a + * terminal `canceled` result (not merely `succeeded`/`failed`), which is the + * behaviour agent pipelines rely on to avoid running the agent job. + * + * Test-harness module; not shipped in `ado-script.zip`. + */ +import { buildGateSpec, encodeGateSpec, encodePrSynthSpec, targetBranchCheck } from "../gate-spec.js"; +import type { BuildOutcome, TriggerScenario } from "../scenario.js"; +import { createPrContext, requirePrRepo, teardownPrContext, type PrContext } from "./common.js"; + +const PROMOTE_ALL = encodePrSynthSpec({ branches: { include: ["main"] } }); + +const selfCancelOnFilterFail: TriggerScenario = { + id: "self-cancel-on-filter-fail", + description: "a failing filter on a synth-promoted build self-cancels the whole build", + async setup(ctx) { + requirePrRepo(ctx); + return createPrContext(ctx, { id: "self-cancel-on-filter-fail" }); + }, + queue(_ctx, state) { + // The PR targets main; require release/* so the target-branch check fails. + return { + sourceBranch: state.sourceRef, + templateParameters: { + gateSpec: encodeGateSpec(buildGateSpec("pull-request", [targetBranchCheck("release/*")])), + prSynthSpec: PROMOTE_ALL, + }, + }; + }, + expected() { + return { + result: "canceled", + tags: ["trig.synth.promoted", "pr-gate.skipped", "pr-gate.target-branch-mismatch"], + absentTags: ["trig.should-run.true"], + }; + }, + async assert(_ctx, _state, outcome: BuildOutcome) { + // Belt-and-braces: the runner already checks result==="canceled", but make + // the intent explicit so a future refactor of the default assertion can't + // silently drop the self-cancel guarantee. + if (outcome.result !== "canceled") { + throw new Error(`self-cancel expected result 'canceled' but got '${outcome.result ?? "?"}'`); + } + }, + async cleanup(ctx, state) { + await teardownPrContext(ctx, state); + }, +}; + +export const selfCancelScenarios: TriggerScenario[] = [selfCancelOnFilterFail]; diff --git a/scripts/ado-script/src/trigger-e2e/scenarios/synth-pr.ts b/scripts/ado-script/src/trigger-e2e/scenarios/synth-pr.ts new file mode 100644 index 00000000..31e32c40 --- /dev/null +++ b/scripts/ado-script/src/trigger-e2e/scenarios/synth-pr.ts @@ -0,0 +1,117 @@ +/** + * Synthetic-PR promotion scenarios (`exec-context-pr-synth.js`). + * + * Each creates a real PR in the victim's `self` repo, queues the victim on the + * PR's source branch, and asserts the synth outcome via the victim's + * `trig.synth.*` build tag: + * - a matching PR → `trig.synth.promoted` (+ gate runs) + * - a branch mismatch → `trig.synth.skipped` (+ gate bypasses) + * - a path mismatch → `trig.synth.skipped` + * + * Test-harness module; not shipped in `ado-script.zip`. + */ +import { buildGateSpec, encodeGateSpec, encodePrSynthSpec } from "../gate-spec.js"; +import type { TriggerScenario } from "../scenario.js"; +import { createPrContext, requirePrRepo, teardownPrContext, type PrContext } from "./common.js"; + +/** Empty gate spec (no checks) — the gate passes iff it is not bypassed. */ +const EMPTY_GATE = encodeGateSpec(buildGateSpec("pull-request", [])); + +/** A matching PR promotes the CI build to synthetic-PR semantics. */ +const synthPromote: TriggerScenario = { + id: "synth-promote", + description: "matching open PR promotes the queued build to synthetic-PR", + async setup(ctx) { + requirePrRepo(ctx); + return createPrContext(ctx, { id: "synth-promote" }); + }, + queue(_ctx, state) { + return { + sourceBranch: state.sourceRef, + templateParameters: { + gateSpec: EMPTY_GATE, + // branches include-all (empty) → the PR's target branch matches. + prSynthSpec: encodePrSynthSpec({ branches: { include: ["main"] } }), + }, + }; + }, + expected() { + return { + result: "succeeded", + tags: ["trig.synth.promoted", "trig.should-run.true"], + absentTags: ["pr-gate.skipped"], + }; + }, + async cleanup(ctx, state) { + await teardownPrContext(ctx, state); + }, +}; + +/** A PR whose target branch is excluded by the synth spec is not promoted. */ +const synthBranchMismatch: TriggerScenario = { + id: "synth-branch-mismatch", + description: "PR targeting an unmatched branch is not synth-promoted (gate bypasses)", + async setup(ctx) { + requirePrRepo(ctx); + return createPrContext(ctx, { id: "synth-branch-mismatch" }); + }, + queue(_ctx, state) { + return { + sourceBranch: state.sourceRef, + templateParameters: { + gateSpec: EMPTY_GATE, + // Only release/* targets match; the PR targets main → no promotion. + prSynthSpec: encodePrSynthSpec({ branches: { include: ["release/*"] } }), + }, + }; + }, + expected() { + return { + result: "succeeded", + tags: ["trig.synth.skipped", "pr-gate.passed"], + absentTags: ["trig.synth.promoted"], + }; + }, + async cleanup(ctx, state) { + await teardownPrContext(ctx, state); + }, +}; + +/** A PR none of whose changed files match the synth path filter is not promoted. */ +const synthPathMismatch: TriggerScenario = { + id: "synth-path-mismatch", + description: "PR with no changed file matching the synth path filter is not promoted", + async setup(ctx) { + requirePrRepo(ctx); + // Change only a docs file; the synth spec will require src/**. + return createPrContext(ctx, { + id: "synth-path-mismatch", + files: { [`/docs/trig/${ctx.buildId}-synth-path.md`]: "docs only change\n" }, + }); + }, + queue(_ctx, state) { + return { + sourceBranch: state.sourceRef, + templateParameters: { + gateSpec: EMPTY_GATE, + prSynthSpec: encodePrSynthSpec({ paths: { include: ["src/**"] } }), + }, + }; + }, + expected() { + return { + result: "succeeded", + tags: ["trig.synth.skipped", "pr-gate.passed"], + absentTags: ["trig.synth.promoted"], + }; + }, + async cleanup(ctx, state) { + await teardownPrContext(ctx, state); + }, +}; + +export const synthScenarios: TriggerScenario[] = [ + synthPromote, + synthBranchMismatch, + synthPathMismatch, +]; diff --git a/tests/trigger-e2e/README.md b/tests/trigger-e2e/README.md new file mode 100644 index 00000000..3c8ffd18 --- /dev/null +++ b/tests/trigger-e2e/README.md @@ -0,0 +1,158 @@ +# Deterministic trigger-condition (gate / synth-PR) E2E suite + +This directory holds a **deterministic**, non-agentic end-to-end test of the +runtime **front-matter trigger conditions managed by ado-script** — the +`gate.js` PR filter evaluator and the `exec-context-pr-synth.js` synthetic-PR +promotion bundle. + +## Why this exists + +The executor E2E suite in [`tests/executor-e2e/`](../executor-e2e/) covers +**Stage 3 safe-output execution** (`ado-aw execute`). It does **not** cover the +trigger conditions that decide *whether the agent runs at all*: PR label/path/ +branch/draft/time-window/change-count filters and the CI→PR synthetic +promotion. Those run at build queue time in the ado-script bundles, emit build +tags + a `SHOULD_RUN` variable, and can self-cancel the build — none of which +the executor suite touches. + +This suite exercises them with **no LLM and no agent** in the loop: + +1. it creates real ADO PR context deterministically (branch + PR + labels + + draft) via the ADO REST API, +2. queues the hand-authored **victim pipeline** (`victim-pipeline.yml`) on the + PR's source branch with a per-scenario base64 `GATE_SPEC` / `PR_SYNTH_SPEC` + as template parameters, +3. polls the build to completion, +4. asserts the observable gate decision (**build tags + build result**), +5. cleans up every object it created, + +and, on any failure, files a GitHub issue on `githubnext/ado-aw` and fails the +build. + +## What's here + +| File | Purpose | +| --- | --- | +| `victim-pipeline.yml` | Hand-authored, **parameterized** victim. Runs only `exec-context-pr-synth.js` then `gate.js` from a checkout build, emits observable build tags. Registered separately. | +| `azure-pipelines.yml` | Hand-authored **orchestrator** (daily + manual). Builds the harness and runs it against AgentPlayground. | +| `README.md` | This file. | + +The harness itself lives in +[`scripts/ado-script/src/trigger-e2e/`](../../scripts/ado-script/src/trigger-e2e/). +It is a **test-only** ado-script bundle: built by `npm run build:trigger-e2e` +to `scripts/ado-script/test-bin/trigger-e2e.js` (a gitignored, non-root path) +and **deliberately excluded** from the released `ado-script.zip` (the release +glob only packages `ado-script/*.js`, and `trigger-e2e` is listed in +`NON_BUNDLE_DIRS` in `src/__tests__/bundle-coverage.test.ts`). + +## How one victim covers synth-PR + every gate filter + +A build queued via REST has `Build.Reason = Manual`. The victim runs +`exec-context-pr-synth.js` first; given a **real open PR** on the queued source +branch it sets `AW_SYNTHETIC_PR=true` + `AW_PR_ID`. The gate's bypass logic +treats `build_reason == "PullRequest" && AW_SYNTHETIC_PR == "true"` as **not +bypassed**, so the full PR filter evaluation runs against the real PR facts +(labels, changed files, draft, target branch, …) even though the build was +queued via REST — no server-side build-validation policy required. + +## Observable build tags (the assertion signal) + +| Tag | Meaning | +| --- | --- | +| `trig.synth.promoted` | synth promoted the build to synthetic-PR semantics | +| `trig.synth.skipped` | synth found no matching PR (branch/path mismatch, or none) | +| `trig.synth.real-pr` | a real PR-triggered build (not exercised by this suite) | +| `pr-gate.passed` | gate **bypassed** (not a PR/synth build) | +| `pr-gate.skipped` + `pr-gate.` | a filter failed; the gate **self-cancelled** the build | +| `trig.should-run.true` | the gate let the run proceed (`SHOULD_RUN=true`) | + +`skip` outcomes reach a terminal **`canceled`** build result (gate self-cancel); +`pass`/`bypass` outcomes reach `succeeded`. + +## Coverage + +- **Synthetic-PR promotion:** matching PR (promoted), branch mismatch (skipped), + path mismatch (skipped). +- **Gate PR filters** (pass + skip): labels, changed-files, draft, target-branch, + build-reason, change-count, time-window. +- **Bypass:** Manual build with no PR → `pr-gate.passed`. +- **Self-cancel:** a failing filter self-cancels the build (`canceled`). + +### Known gaps / future work + +- `pr_title` and `author_email` filters source from `System.PullRequest.*`, + which are empty on synth-promoted Manual builds. The victim accepts + `prTitle` / `authorEmail` template-parameter overrides so these predicates + can be tested against controlled values; scenarios for them are not yet + wired up. +- The synth **two-match refusal** path (two active PRs on one source branch) is + not tested (hard to set up deterministically). + +## Naming / cleanup convention + +Every object a scenario creates is prefixed `ado-aw-trig-$(Build.BuildId)-`. +Cleanup (abandon PR + delete branch) runs unconditionally after each scenario; +the smoke-suite janitor (which prunes `ado-aw-*` artifacts) is the backstop. + +## Running locally + +You need a write-capable ADO token (PAT) and a registered victim pipeline id: + +```bash +cd scripts/ado-script && npm ci && npm run build:trigger-e2e && cd ../.. + +export SYSTEM_COLLECTIONURI="https://dev.azure.com/msazuresphere/" +export SYSTEM_TEAMPROJECT="AgentPlayground" +export SYSTEM_ACCESSTOKEN="" +export TRIGGER_E2E_VICTIM_DEFINITION_ID="" +export TRIGGER_E2E_VICTIM_REPO="" +# Optional: +# export TRIGGER_E2E_GITHUB_TOKEN="" +# export TRIGGER_E2E_BUILD_TIMEOUT_MS=900000 # per victim build (default 900000) +# export TRIGGER_E2E_BUILD_POLL_MS=10000 # poll interval (default 10000) + +node scripts/ado-script/test-bin/trigger-e2e.js +``` + +When `TRIGGER_E2E_VICTIM_REPO` is unset, the PR/synth/gate scenarios **skip** +and only the bypass baseline runs. The harness exits non-zero if any scenario +fails. + +## Manual-handoff checklist (one-time ADO setup) + +In `https://dev.azure.com/msazuresphere/AgentPlayground`: + +1. **Register the VICTIM pipeline.** New pipeline → Azure Repos → the ado-aw + Azure Repos mirror → existing YAML → `tests/trigger-e2e/victim-pipeline.yml`. + It **must** be an ADO Git repo (not GitHub) so `exec-context-pr-synth` can + discover open PRs in the pipeline's own `self` repo. Note its **definition + id** for the next step. +2. **Register the ORCHESTRATOR pipeline.** New pipeline → existing YAML → + `tests/trigger-e2e/azure-pipelines.yml`. Place both in a `\trigger-e2e` + folder. +3. **Wire the victim id + repo** as pipeline/definition variables on the + orchestrator: + ```powershell + ado-aw secrets set TRIGGER_E2E_VICTIM_DEFINITION_ID ` + --org msazuresphere --project AgentPlayground ` + --definition-ids --value + ado-aw secrets set TRIGGER_E2E_VICTIM_REPO ` + --org msazuresphere --project AgentPlayground ` + --definition-ids --value + ``` + (These are not secrets; `secrets set` is just the variable-management path.) +4. **Grant the VICTIM build identity** Code (read) on the repo, Build (add + tags / edit build quality), and permission to cancel builds (gate + self-cancel). See [`docs/safe-output-permissions.md`](../../docs/safe-output-permissions.md) + if it hits 401/403. +5. **Grant the ORCHESTRATOR build identity** Contribute / Create branch / + Contribute to PRs on the victim repo (it creates transient PRs) and Queue + builds on the victim definition. +6. **Set the GitHub PAT secret** on the orchestrator only: + ```powershell + ado-aw secrets set TRIGGER_E2E_GITHUB_TOKEN ` + --org msazuresphere --project AgentPlayground ` + --definition-ids ` + --value + ``` +7. **Trigger one manual orchestrator run** to seed the schedule. diff --git a/tests/trigger-e2e/azure-pipelines.yml b/tests/trigger-e2e/azure-pipelines.yml new file mode 100644 index 00000000..eead6b22 --- /dev/null +++ b/tests/trigger-e2e/azure-pipelines.yml @@ -0,0 +1,85 @@ +# Deterministic trigger-condition (gate / synth-PR) E2E ORCHESTRATOR pipeline. +# +# Builds the trigger-e2e harness from this checkout and runs it against +# AgentPlayground. The harness creates real PRs and queues the registered +# VICTIM pipeline (tests/trigger-e2e/victim-pipeline.yml) under a battery of +# trigger conditions, asserting the gate decision via build tags + result. On +# any failure it files a GitHub issue on githubnext/ado-aw and fails the build. +# +# Unlike the executor-e2e orchestrator, this does NOT build the Rust `ado-aw` +# binary: trigger conditions are evaluated entirely by the ado-script bundles +# the victim runs, so the harness only needs Node. +# +# This is a hand-authored ADO pipeline (NOT compiled from an agent markdown). +# Register it in msazuresphere/AgentPlayground — see tests/trigger-e2e/README.md. + +trigger: none +pr: none + +schedules: + - cron: "0 6 * * *" + displayName: Daily deterministic trigger-condition E2E + branches: + include: + - main + always: true + +pool: + name: AZS-1ES-L-Playground-ubuntu-22.04 + +variables: + # Keep YAML defaults under private names so same-named pipeline/definition UI + # variables are not shadowed by this block. + EFFECTIVE_TRIGGER_E2E_VICTIM_DEFINITION_ID: $[ coalesce(variables['TRIGGER_E2E_VICTIM_DEFINITION_ID'], '') ] + EFFECTIVE_TRIGGER_E2E_VICTIM_REPO: $[ coalesce(variables['TRIGGER_E2E_VICTIM_REPO'], '') ] + # GitHub repo that failure issues are filed against is intentionally NOT + # defined here: a YAML `variables:` entry SHADOWS a same-named pipeline + # (definition) variable. Set TRIGGER_E2E_ISSUE_REPO as a pipeline/definition + # variable to redirect issue filing; the harness defaults to githubnext/ado-aw. + # TRIGGER_E2E_GITHUB_TOKEN must be provided as a SECRET pipeline variable + # (fine-grained PAT scoped to Issues:read/write on githubnext/ado-aw ONLY). + +steps: + - checkout: self + fetchDepth: 1 + displayName: Checkout ado-aw + + - task: UseNode@1 + inputs: + version: "20.x" + displayName: Use Node.js 20 + + - script: | + set -euo pipefail + npm ci + npm run build:trigger-e2e + workingDirectory: scripts/ado-script + displayName: Build trigger-e2e harness (not shipped in ado-script.zip) + + - task: AzureCLI@2 + displayName: Acquire ADO token (SC_WRITE_TOKEN) + inputs: + azureSubscription: agent-playground-write + scriptType: bash + scriptLocation: inlineScript + addSpnToEnvironment: true + inlineScript: | + ADO_TOKEN=$(az account get-access-token \ + --resource 499b84ac-1321-427f-aa17-267ca6975798 \ + --query accessToken -o tsv) + echo "##vso[task.setvariable variable=SC_WRITE_TOKEN;issecret=true]$ADO_TOKEN" + + - script: | + set -euo pipefail + node scripts/ado-script/test-bin/trigger-e2e.js + displayName: Run deterministic trigger-condition E2E scenarios + env: + # ADO auth + context. SYSTEM_COLLECTIONURI / SYSTEM_TEAMPROJECT / + # BUILD_BUILDID are auto-injected; SYSTEM_ACCESSTOKEN must be mapped + # explicitly so the harness can queue builds, create PRs, cancel, and tag. + SYSTEM_ACCESSTOKEN: $(SC_WRITE_TOKEN) + TRIGGER_E2E_VICTIM_DEFINITION_ID: $(EFFECTIVE_TRIGGER_E2E_VICTIM_DEFINITION_ID) + TRIGGER_E2E_VICTIM_REPO: $(EFFECTIVE_TRIGGER_E2E_VICTIM_REPO) + TRIGGER_E2E_ISSUE_REPO: $(TRIGGER_E2E_ISSUE_REPO) + # Secret PAT for filing failure issues (scoped to githubnext/ado-aw Issues). + TRIGGER_E2E_GITHUB_TOKEN: $(TRIGGER_E2E_GITHUB_TOKEN) diff --git a/tests/trigger-e2e/victim-pipeline.yml b/tests/trigger-e2e/victim-pipeline.yml new file mode 100644 index 00000000..89b86010 --- /dev/null +++ b/tests/trigger-e2e/victim-pipeline.yml @@ -0,0 +1,150 @@ +# Trigger-condition E2E VICTIM pipeline. +# +# A deterministic, parameterized pipeline that runs ONLY the runtime trigger +# bundles managed by ado-script — `exec-context-pr-synth.js` (CI→PR synthetic +# promotion) and `gate.js` (PR filter evaluation) — and reports the decision via +# REST-observable build tags. It runs NO agent workflow. +# +# The trigger-e2e harness (scripts/ado-script/src/trigger-e2e/) queues this +# pipeline many ways: it creates a real PR, then queues a build on the PR's +# source branch with a per-scenario base64 `gateSpec` / `prSynthSpec` as +# template parameters, and asserts the resulting build tags + result. +# +# ## Registration requirements (see tests/trigger-e2e/README.md) +# - MUST be registered against an **ADO Git repo** (BUILD_REPOSITORY_PROVIDER +# = TfsGit), because `exec-context-pr-synth` discovers the open PR in the +# pipeline's OWN `self` repo. A GitHub-typed repo short-circuits synth. +# Register it against the ado-aw Azure Repos mirror so the `self` checkout +# also carries `scripts/ado-script/` to build the bundles from source. +# - The build identity ($(System.AccessToken)) needs: Code (read) on the repo +# (PR/label/iteration lookups), Build (edit build quality / add tags) and +# the ability to cancel builds (gate self-cancel). See +# docs/safe-output-permissions.md if it hits 401/403. +# +# ## Observable build tags +# - trig.synth.promoted | trig.synth.skipped | trig.synth.real-pr +# - pr-gate.passed (bypass) | pr-gate.skipped + pr-gate. (filter fail) +# - trig.should-run.true (gate let the run proceed) + +trigger: none +pr: none + +pool: + name: AZS-1ES-L-Playground-ubuntu-22.04 + +parameters: + # Required per queue: base64 GateSpec produced by the harness gate-spec.ts. + - name: gateSpec + type: string + default: "" + # base64 PrSynthSpec. Default = empty include/exclude (match-all promote). + - name: prSynthSpec + type: string + default: "eyJicmFuY2hlcyI6eyJpbmNsdWRlIjpbXSwiZXhjbHVkZSI6W119LCJwYXRocyI6eyJpbmNsdWRlIjpbXSwiZXhjbHVkZSI6W119fQ==" + # Pipeline-variable fact overrides. Defaults reference the synth-resolved + # AW_PR_* vars / ADO predefined vars, so an unset override uses the real value. + - name: prTitle + type: string + default: "$(System.PullRequest.Title)" + - name: authorEmail + type: string + default: "$(Build.RequestedForEmail)" + - name: sourceBranchFact + type: string + default: "$(AW_PR_SOURCEBRANCH)" + - name: targetBranchFact + type: string + default: "$(AW_PR_TARGETBRANCH)" + - name: commitMessage + type: string + default: "$(Build.SourceVersionMessage)" + +jobs: + - job: Trigger + displayName: Evaluate trigger conditions + steps: + - checkout: self + # Full history is not needed; the bundles are built from the checkout + # and the gate/synth logic uses REST, not git. + fetchDepth: 1 + displayName: Checkout self (ado-aw source) + + - task: UseNode@1 + inputs: + version: "20.x" + displayName: Use Node.js 20 + + - script: | + set -euo pipefail + npm ci + npm run build:gate + npm run build:exec-context-pr-synth + workingDirectory: scripts/ado-script + displayName: Build gate + synth bundles from checkout + + - script: | + set -euo pipefail + node scripts/ado-script/exec-context-pr-synth.js + displayName: Run exec-context-pr-synth (synthetic PR promotion) + env: + PR_SYNTH_SPEC: ${{ parameters.prSynthSpec }} + # REST auth for active-PR / iteration lookups. + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + + - script: | + set -euo pipefail + # Classify the synth outcome from the same-job vars synth setVar's. + # AW_PR_ID is always set (empty on skip); AW_SYNTHETIC_PR is only set + # to "true" on promotion (else left as the literal macro). + SYN="$(AW_SYNTHETIC_PR)" + PRID="$(AW_PR_ID)" + case "$SYN" in '$('*) SYN="" ;; esac + case "$PRID" in '$('*) PRID="" ;; esac + if [ "$SYN" = "true" ]; then + STATE="promoted" + elif [ -n "$PRID" ]; then + STATE="real-pr" + else + STATE="skipped" + fi + echo "synth state: ${STATE} (AW_PR_ID='${PRID}' AW_SYNTHETIC_PR='${SYN}')" + echo "##vso[build.addbuildtag]trig.synth.${STATE}" + displayName: Tag synth outcome + + - script: | + set -euo pipefail + node scripts/ado-script/gate.js + name: prGate + displayName: Evaluate PR filters (gate) + env: + GATE_SPEC: ${{ parameters.gateSpec }} + # Gate context / infra facts. + ADO_BUILD_REASON: $(Build.Reason) + ADO_BUILD_ID: $(Build.BuildId) + ADO_PROJECT: $(System.TeamProject) + ADO_REPO_ID: $(Build.Repository.ID) + # PR identity: resolved by synth (real or promoted). AW_SYNTHETIC_PR + # tells the gate's bypass logic to run full evaluation on a promoted + # CI build. + ADO_PR_ID: $(AW_PR_ID) + AW_SYNTHETIC_PR: $(AW_SYNTHETIC_PR) + # Pipeline-variable facts (override-able via template parameters). + ADO_PR_TITLE: ${{ parameters.prTitle }} + ADO_AUTHOR_EMAIL: ${{ parameters.authorEmail }} + ADO_SOURCE_BRANCH: ${{ parameters.sourceBranchFact }} + ADO_TARGET_BRANCH: ${{ parameters.targetBranchFact }} + ADO_COMMIT_MESSAGE: ${{ parameters.commitMessage }} + # REST auth for API facts (labels, changed files) + self-cancel. + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + + - script: | + set -euo pipefail + # Only reached when the gate did NOT self-cancel (SHOULD_RUN=true or + # bypass). SHOULD_RUN is a step-output var of the `prGate` step. + RUN="$(prGate.SHOULD_RUN)" + echo "gate SHOULD_RUN='${RUN}'" + if [ "$RUN" = "true" ]; then + echo "##vso[build.addbuildtag]trig.should-run.true" + fi + # Default condition is succeeded(); a self-cancelled build skips this. + displayName: Reflect gate decision From abab734bd0bd9dbb5dbdcca64627949cf9ec2a73 Mon Sep 17 00:00:00 2001 From: James Devine Date: Tue, 14 Jul 2026 09:59:50 +0100 Subject: [PATCH 2/6] test(trigger-e2e): address PR review feedback - Derive the synthetic-PR spec from the PR's real target branch (promoteSynthSpec/excludeSynthSpec) instead of a hardcoded "main", so gate/synth scenarios evaluate correctly on any default branch. - Guard createPrContext against an explicitly-empty files map (avoid the entries[0]! undefined-destructuring crash; treat {} like omitted). - Cancel an orphaned victim build in the runner's finally block when the queue-phase poll throws (waitForBuild timeout), threading the queued build id via runVictim's onQueued callback; add unit tests for the cancel path. - setPullRequestLabels: name the specific failing label so a partial attach surfaces as a clear setup error, not a confusing gate mismatch. - filter_ir.rs: add a SYNC annotation pointing at gate-spec.ts::FACT_META so fact policy/dependency changes flag the hand-maintained mirror. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: f30758a5-81c8-4024-ab01-52ad22b4f877 --- .../ado-script/src/executor-e2e/ado-rest.ts | 9 +++- .../src/trigger-e2e/__tests__/runner.test.ts | 51 ++++++++++++++----- scripts/ado-script/src/trigger-e2e/queue.ts | 10 +++- scripts/ado-script/src/trigger-e2e/runner.ts | 23 ++++++++- .../src/trigger-e2e/scenarios/common.ts | 44 ++++++++++++---- .../src/trigger-e2e/scenarios/pr-filters.ts | 20 +++++--- .../src/trigger-e2e/scenarios/self-cancel.ts | 17 ++++--- .../src/trigger-e2e/scenarios/synth-pr.ts | 18 +++++-- src/compile/filter_ir.rs | 8 +++ 9 files changed, 157 insertions(+), 43 deletions(-) diff --git a/scripts/ado-script/src/executor-e2e/ado-rest.ts b/scripts/ado-script/src/executor-e2e/ado-rest.ts index 1b93155b..5400b33b 100644 --- a/scripts/ado-script/src/executor-e2e/ado-rest.ts +++ b/scripts/ado-script/src/executor-e2e/ado-rest.ts @@ -378,7 +378,14 @@ export class AdoRest { const path = this.projPath( `_apis/git/repositories/${AdoRest.seg(repo)}/pullRequests/${prId}/labels?api-version=7.1`, ); - await this.request(path, { method: "POST", body: { name } }); + try { + await this.request(path, { method: "POST", body: { name } }); + } catch (err) { + // Name the specific label that failed so a partial-attach surfaces as a + // clear setup error rather than a confusing downstream gate mismatch. + const message = err instanceof Error ? err.message : String(err); + throw new Error(`setPullRequestLabels: failed to attach label '${name}' to PR ${prId}: ${message}`); + } } } diff --git a/scripts/ado-script/src/trigger-e2e/__tests__/runner.test.ts b/scripts/ado-script/src/trigger-e2e/__tests__/runner.test.ts index 4e26b8ea..4f26028b 100644 --- a/scripts/ado-script/src/trigger-e2e/__tests__/runner.test.ts +++ b/scripts/ado-script/src/trigger-e2e/__tests__/runner.test.ts @@ -8,25 +8,34 @@ import type { TriggerContext, TriggerScenario } from "../scenario.js"; interface FakeBuild { result?: string; tags?: string[]; + /** Optional override for the finally-block getBuild status probe. */ + finalStatus?: string; + /** When set, waitForBuild rejects with this message (queue-phase failure). */ + waitError?: string; } -function makeCtx(build: FakeBuild): TriggerContext { +function makeCtx(build: FakeBuild): { ctx: TriggerContext; rest: Record> } { const rest = { queueBuild: vi.fn().mockResolvedValue({ id: 42 }), - waitForBuild: vi.fn().mockResolvedValue({ status: "completed", result: build.result }), + waitForBuild: build.waitError + ? vi.fn().mockRejectedValue(new Error(build.waitError)) + : vi.fn().mockResolvedValue({ status: "completed", result: build.result }), getBuildTags: vi.fn().mockResolvedValue(build.tags ?? []), - } as unknown as AdoRest; - return { + getBuild: vi.fn().mockResolvedValue({ id: 42, status: build.finalStatus ?? "completed" }), + cancelBuild: vi.fn().mockResolvedValue(undefined), + }; + const ctx: TriggerContext = { orgUrl: "https://dev.azure.com/org/", project: "proj", adoRepo: "repo", buildId: "1", token: "t", victimDefinitionId: 7, - rest, + rest: rest as unknown as AdoRest, log: () => {}, prefix: (id) => `ado-aw-trig-1-${id}`, }; + return { ctx, rest }; } function scenario(overrides: Partial>): TriggerScenario { @@ -43,7 +52,7 @@ function scenario(overrides: Partial>): TriggerScenario< describe("runScenario", () => { it("passes when result and tags match", async () => { - const ctx = makeCtx({ result: "succeeded", tags: ["trig.should-run.true"] }); + const { ctx } = makeCtx({ result: "succeeded", tags: ["trig.should-run.true"] }); const res = await runScenario( ctx, scenario({ expected: () => ({ result: "succeeded", tags: ["trig.should-run.true"] }) }), @@ -52,7 +61,7 @@ describe("runScenario", () => { }); it("fails (assert) when a required tag is missing", async () => { - const ctx = makeCtx({ result: "succeeded", tags: [] }); + const { ctx } = makeCtx({ result: "succeeded", tags: [] }); const res = await runScenario(ctx, scenario({ expected: () => ({ tags: ["pr-gate.skipped"] }) })); expect(res.ok).toBe(false); expect(res.phase).toBe("assert"); @@ -60,14 +69,14 @@ describe("runScenario", () => { }); it("fails (assert) on the wrong build result", async () => { - const ctx = makeCtx({ result: "canceled" }); + const { ctx } = makeCtx({ result: "canceled" }); const res = await runScenario(ctx, scenario({ expected: () => ({ result: "succeeded" }) })); expect(res.ok).toBe(false); expect(res.phase).toBe("assert"); }); it("fails (assert) when a forbidden tag is present", async () => { - const ctx = makeCtx({ result: "succeeded", tags: ["pr-gate.skipped"] }); + const { ctx } = makeCtx({ result: "succeeded", tags: ["pr-gate.skipped"] }); const res = await runScenario( ctx, scenario({ expected: () => ({ absentTags: ["pr-gate.skipped"] }) }), @@ -77,7 +86,7 @@ describe("runScenario", () => { }); it("records a SkipError from setup as skipped and does NOT run cleanup", async () => { - const ctx = makeCtx({ result: "succeeded" }); + const { ctx } = makeCtx({ result: "succeeded" }); const cleanup = vi.fn().mockResolvedValue(undefined); const res = await runScenario( ctx, @@ -94,7 +103,7 @@ describe("runScenario", () => { }); it("runs cleanup after a successful setup even when assertion fails", async () => { - const ctx = makeCtx({ result: "failed" }); + const { ctx } = makeCtx({ result: "failed" }); const cleanup = vi.fn().mockResolvedValue(undefined); const res = await runScenario(ctx, scenario({ cleanup })); expect(res.ok).toBe(false); @@ -102,7 +111,7 @@ describe("runScenario", () => { }); it("honours a custom assert hook", async () => { - const ctx = makeCtx({ result: "succeeded" }); + const { ctx } = makeCtx({ result: "succeeded" }); const res = await runScenario( ctx, scenario({ @@ -114,4 +123,22 @@ describe("runScenario", () => { expect(res.ok).toBe(false); expect(res.message).toContain("custom boom"); }); + + it("cancels an orphaned victim build when the poll times out (queue phase)", async () => { + // waitForBuild rejects (timeout); the build is still running afterwards. + const { ctx, rest } = makeCtx({ waitError: "waitForBuild timed out", finalStatus: "inProgress" }); + const res = await runScenario(ctx, scenario({})); + expect(res.ok).toBe(false); + expect(res.phase).toBe("queue"); + // The queued build id (42) must be cancelled so it is not orphaned. + expect(rest.getBuild).toHaveBeenCalledWith(42); + expect(rest.cancelBuild).toHaveBeenCalledWith(42); + }); + + it("does NOT cancel a build that already completed", async () => { + const { ctx, rest } = makeCtx({ result: "succeeded", finalStatus: "completed" }); + const res = await runScenario(ctx, scenario({ expected: () => ({ result: "succeeded" }) })); + expect(res.ok).toBe(true); + expect(rest.cancelBuild).not.toHaveBeenCalled(); + }); }); diff --git a/scripts/ado-script/src/trigger-e2e/queue.ts b/scripts/ado-script/src/trigger-e2e/queue.ts index e517eb97..600f170b 100644 --- a/scripts/ado-script/src/trigger-e2e/queue.ts +++ b/scripts/ado-script/src/trigger-e2e/queue.ts @@ -19,12 +19,20 @@ function flattenParams(params: Record): Record { +export async function runVictim( + ctx: TriggerContext, + queue: VictimQueue, + onQueued?: (buildId: number) => void, +): Promise { const templateParameters = flattenParams(queue.templateParameters); const queued = await ctx.rest.queueBuild(ctx.victimDefinitionId, { sourceBranch: queue.sourceBranch, templateParameters, }); + // Report the id as soon as the build is queued so the caller can cancel it on + // cleanup even if the subsequent `waitForBuild` poll throws (e.g. a timeout) + // and never returns a BuildOutcome. + onQueued?.(queued.id); ctx.log(` queued victim build #${queued.id}${queue.sourceBranch ? ` on ${queue.sourceBranch}` : ""}`); const terminal = await ctx.rest.waitForBuild(queued.id); diff --git a/scripts/ado-script/src/trigger-e2e/runner.ts b/scripts/ado-script/src/trigger-e2e/runner.ts index 94177eea..47806084 100644 --- a/scripts/ado-script/src/trigger-e2e/runner.ts +++ b/scripts/ado-script/src/trigger-e2e/runner.ts @@ -50,6 +50,10 @@ export async function runScenario( const id = scenario.id; let state: S | undefined; let setupDone = false; + // Captured as soon as the victim build is queued, so a queue-phase throw + // (e.g. a waitForBuild timeout) still lets the finally block cancel the + // orphaned build rather than leaking a running build for the whole suite. + let queuedBuildId: number | undefined; const finish = (partial: Omit): ScenarioResult => ({ id, @@ -75,7 +79,9 @@ export async function runScenario( let outcome: BuildOutcome; try { const queue = scenario.queue(ctx, state); - outcome = await runVictim(ctx, queue); + outcome = await runVictim(ctx, queue, (buildId) => { + queuedBuildId = buildId; + }); } catch (err) { return finish({ ok: false, phase: "queue", message: errMessage(err) }); } @@ -91,6 +97,21 @@ export async function runScenario( ctx.log(`[${id}] OK`); return finish({ ok: true }); } finally { + // ---- cancel an orphaned victim build (best-effort) ---- + // A completed build is a no-op; only a build still running after a + // queue-phase failure needs cancelling. + if (queuedBuildId !== undefined) { + try { + const build = await ctx.rest.getBuild(queuedBuildId); + if (build.status !== "completed") { + await ctx.rest.cancelBuild(queuedBuildId); + ctx.log(`[${id}] cancelled orphaned victim build #${queuedBuildId}`); + } + } catch (err) { + ctx.log(`[${id}] orphaned-build cleanup WARNING: ${errMessage(err)}`); + } + } + // ---- cleanup (always, best-effort) ---- if (setupDone) { try { diff --git a/scripts/ado-script/src/trigger-e2e/scenarios/common.ts b/scripts/ado-script/src/trigger-e2e/scenarios/common.ts index f5a46d5e..dfa4a6d3 100644 --- a/scripts/ado-script/src/trigger-e2e/scenarios/common.ts +++ b/scripts/ado-script/src/trigger-e2e/scenarios/common.ts @@ -10,6 +10,7 @@ * Test-harness module; not shipped in `ado-script.zip`. */ import { Teardown } from "../../executor-e2e/scenarios/common.js"; +import { encodePrSynthSpec } from "../gate-spec.js"; import { SkipError } from "../scenario.js"; import type { TriggerContext } from "../scenario.js"; @@ -65,9 +66,15 @@ export async function createPrContext( const sourceRef = `refs/heads/${branch}`; // Default file guarantees a real diff even when the scenario needs none. - const files = opts.files ?? { - [`/ado-aw-trig/${ctx.buildId}/${opts.id}.md`]: `trigger e2e ${opts.id} for build ${ctx.buildId}. Safe to delete.\n`, - }; + // Treat an explicitly-empty `files: {}` the same as omitted so a caller can + // never produce a PR with no diff (which ADO would reject) or crash the + // first-entry destructuring below. + const files = + opts.files && Object.keys(opts.files).length > 0 + ? opts.files + : { + [`/ado-aw-trig/${ctx.buildId}/${opts.id}.md`]: `trigger e2e ${opts.id} for build ${ctx.buildId}. Safe to delete.\n`, + }; // Push the branch with the first file, then add the rest in follow-up pushes. const entries = Object.entries(files); @@ -124,14 +131,6 @@ export async function teardownPrContext(ctx: TriggerContext, pr: PrContext): Pro .run(); } -/** Register cleanup of a queued victim build id (cancel if somehow still running). */ -export async function cancelVictimIfRunning(ctx: TriggerContext, buildId: number): Promise { - const build = await ctx.rest.getBuild(buildId).catch(() => undefined); - if (build && build.status !== "completed") { - await ctx.rest.cancelBuild(buildId).catch(() => {}); - } -} - /** * Ensure a real ADO Git repo is available for PR creation. PR/synth/gate * scenarios need the victim pipeline's own `self` repo (where `exec-context- @@ -147,3 +146,26 @@ export function requirePrRepo(ctx: TriggerContext): string { } return repo; } + +/** + * base64 PR_SYNTH_SPEC that promotes a PR targeting `targetBranch`. + * + * MUST be derived from the PR's real target branch (`PrContext.targetBranch`, + * resolved from the repo's default branch), NOT a hardcoded `"main"`: if the + * ADO repo's default branch is anything else, a constant `"main"` include would + * fail to match the real target and every gate/synth scenario would silently + * evaluate the wrong path. + */ +export function promoteSynthSpec(targetBranch: string): string { + return encodePrSynthSpec({ branches: { include: [targetBranch] } }); +} + +/** + * base64 PR_SYNTH_SPEC that deliberately does NOT promote a PR targeting + * `targetBranch` (excludes exactly that branch, include-all otherwise), so the + * "branch mismatch → not promoted" path is exercised robustly regardless of the + * repo's actual default branch name. + */ +export function excludeSynthSpec(targetBranch: string): string { + return encodePrSynthSpec({ branches: { exclude: [targetBranch] } }); +} diff --git a/scripts/ado-script/src/trigger-e2e/scenarios/pr-filters.ts b/scripts/ado-script/src/trigger-e2e/scenarios/pr-filters.ts index eb3b43c3..12e516d6 100644 --- a/scripts/ado-script/src/trigger-e2e/scenarios/pr-filters.ts +++ b/scripts/ado-script/src/trigger-e2e/scenarios/pr-filters.ts @@ -22,19 +22,21 @@ import { changedFilesCheck, draftCheck, encodeGateSpec, - encodePrSynthSpec, labelsCheck, targetBranchCheck, timeWindowCheck, type Check, } from "../gate-spec.js"; import type { Expected, TriggerScenario, VictimParameters } from "../scenario.js"; -import { createPrContext, requirePrRepo, teardownPrContext, type PrContext } from "./common.js"; +import { + createPrContext, + promoteSynthSpec, + requirePrRepo, + teardownPrContext, + type PrContext, +} from "./common.js"; import type { CreatePrOptions } from "./common.js"; -/** Promote-everything synth spec: any active PR on the source branch matches. */ -const PROMOTE_ALL = encodePrSynthSpec({ branches: { include: ["main"] } }); - interface FilterScenarioSpec { readonly id: string; readonly description: string; @@ -63,7 +65,13 @@ function makeFilterScenario(spec: FilterScenarioSpec): TriggerScenario = { id: "self-cancel-on-filter-fail", @@ -23,12 +27,13 @@ const selfCancelOnFilterFail: TriggerScenario = { return createPrContext(ctx, { id: "self-cancel-on-filter-fail" }); }, queue(_ctx, state) { - // The PR targets main; require release/* so the target-branch check fails. + // The PR targets its default branch; require release/* so the target-branch + // check fails. Promote against the real target branch so the gate runs. return { sourceBranch: state.sourceRef, templateParameters: { gateSpec: encodeGateSpec(buildGateSpec("pull-request", [targetBranchCheck("release/*")])), - prSynthSpec: PROMOTE_ALL, + prSynthSpec: promoteSynthSpec(state.targetBranch), }, }; }, diff --git a/scripts/ado-script/src/trigger-e2e/scenarios/synth-pr.ts b/scripts/ado-script/src/trigger-e2e/scenarios/synth-pr.ts index 31e32c40..6fa9bbf5 100644 --- a/scripts/ado-script/src/trigger-e2e/scenarios/synth-pr.ts +++ b/scripts/ado-script/src/trigger-e2e/scenarios/synth-pr.ts @@ -12,7 +12,14 @@ */ import { buildGateSpec, encodeGateSpec, encodePrSynthSpec } from "../gate-spec.js"; import type { TriggerScenario } from "../scenario.js"; -import { createPrContext, requirePrRepo, teardownPrContext, type PrContext } from "./common.js"; +import { + createPrContext, + excludeSynthSpec, + promoteSynthSpec, + requirePrRepo, + teardownPrContext, + type PrContext, +} from "./common.js"; /** Empty gate spec (no checks) — the gate passes iff it is not bypassed. */ const EMPTY_GATE = encodeGateSpec(buildGateSpec("pull-request", [])); @@ -30,8 +37,8 @@ const synthPromote: TriggerScenario = { sourceBranch: state.sourceRef, templateParameters: { gateSpec: EMPTY_GATE, - // branches include-all (empty) → the PR's target branch matches. - prSynthSpec: encodePrSynthSpec({ branches: { include: ["main"] } }), + // Promote against the PR's REAL target branch (not a hardcoded "main"). + prSynthSpec: promoteSynthSpec(state.targetBranch), }, }; }, @@ -60,8 +67,9 @@ const synthBranchMismatch: TriggerScenario = { sourceBranch: state.sourceRef, templateParameters: { gateSpec: EMPTY_GATE, - // Only release/* targets match; the PR targets main → no promotion. - prSynthSpec: encodePrSynthSpec({ branches: { include: ["release/*"] } }), + // Exclude the PR's REAL target branch so promotion is refused + // regardless of the repo's default branch name. + prSynthSpec: excludeSynthSpec(state.targetBranch), }, }; }, diff --git a/src/compile/filter_ir.rs b/src/compile/filter_ir.rs index 1af4ba7d..dfc4886a 100644 --- a/src/compile/filter_ir.rs +++ b/src/compile/filter_ir.rs @@ -39,6 +39,14 @@ use std::fmt; /// Each variant maps to a specific piece of data available at pipeline runtime, /// with known acquisition cost (free pipeline variable vs. REST API call vs. /// runtime computation). +/// +/// SYNC: the deterministic trigger-condition E2E harness hand-maintains a +/// mirror of `kind()` / `failure_policy()` / `dependencies()` in +/// `scripts/ado-script/src/trigger-e2e/gate-spec.ts` (the `FACT_META` table) so +/// it can craft gate specs without invoking the compiler. The `types.gen.ts` +/// codegen only guards the serialized `GateSpec` *shape*, not fact +/// policy/dependency values — so when adding a `Fact` variant, or changing a +/// `failure_policy`/`dependencies`, update `FACT_META` in that file too. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] pub enum Fact { // ── Pipeline variables (free — always available) ──────────────────── From afe3f539944ddbaa94e9a02071cdeb9f54142544 Mon Sep 17 00:00:00 2001 From: James Devine Date: Tue, 14 Jul 2026 10:55:21 +0100 Subject: [PATCH 3/6] test(trigger-e2e): address second-round PR review feedback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Fix latent multi-file push bug: pushAddFileBranch always uses new-branch semantics, so createPrContext's per-file loop would fail with a ref conflict on the second file. Add AdoRest.pushAddFilesBranch that batches all files into a single commit, and use it in createPrContext. - Decouple the shared AdoRest client from trigger-e2e env vars: waitForBuild now takes generic 15min/10s defaults; the TRIGGER_E2E_BUILD_TIMEOUT_MS / _POLL_MS knobs are read in trigger-e2e queue.ts and passed as opts, so an executor-e2e run can't be silently retimed. - Add change-count-pass scenario (two files, within range → runs), completing the numeric_range pass/skip coverage and exercising the batched push path. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: f30758a5-81c8-4024-ab01-52ad22b4f877 --- .../ado-script/src/executor-e2e/ado-rest.ts | 58 ++++++++++++++++++- scripts/ado-script/src/trigger-e2e/queue.ts | 15 ++++- .../src/trigger-e2e/scenarios/common.ts | 30 ++-------- .../src/trigger-e2e/scenarios/pr-filters.ts | 13 +++++ 4 files changed, 87 insertions(+), 29 deletions(-) diff --git a/scripts/ado-script/src/executor-e2e/ado-rest.ts b/scripts/ado-script/src/executor-e2e/ado-rest.ts index 5400b33b..78ffd453 100644 --- a/scripts/ado-script/src/executor-e2e/ado-rest.ts +++ b/scripts/ado-script/src/executor-e2e/ado-rest.ts @@ -256,6 +256,55 @@ export class AdoRest { return commitId; } + /** + * Create a NEW branch and a single commit adding one OR MORE files in one + * push. Returns the new commit id. + * + * Prefer this over calling {@link pushAddFileBranch} in a loop: that helper + * always uses new-branch semantics (`oldObjectId` = zeros), so a second call + * against the now-existing branch would be rejected by ADO with a ref + * conflict. Batching every file into a single commit sidesteps that entirely + * and still produces a real diff vs. the base for PR scenarios. + */ + async pushAddFilesBranch( + repo: string, + branchName: string, + baseCommitId: string, + files: Record, + comment: string, + ): Promise { + const entries = Object.entries(files); + if (entries.length === 0) throw new Error("pushAddFilesBranch requires at least one file"); + const path = this.projPath( + `_apis/git/repositories/${AdoRest.seg(repo)}/pushes?api-version=7.1`, + ); + const res = await this.request<{ commits?: { commitId: string }[] }>(path, { + method: "POST", + body: { + refUpdates: [ + { + name: branchName.startsWith("refs/") ? branchName : `refs/heads/${branchName}`, + oldObjectId: "0000000000000000000000000000000000000000", + }, + ], + commits: [ + { + comment, + parents: [baseCommitId], + changes: entries.map(([filePath, content]) => ({ + changeType: "add", + item: { path: filePath.startsWith("/") ? filePath : `/${filePath}` }, + newContent: { content, contentType: "rawtext" }, + })), + }, + ], + }, + }); + const commitId = res?.commits?.[0]?.commitId; + if (!commitId) throw new Error("pushAddFilesBranch returned no commit id"); + return commitId; + } + // ---- Git: pull requests & threads ------------------------------------ async createPullRequest( @@ -511,14 +560,17 @@ export class AdoRest { * Poll a build until it reaches `status === "completed"` (or the timeout * elapses). Returns the terminal `{ status, result }`. `result` is one of * `succeeded` | `partiallySucceeded` | `failed` | `canceled`. + * + * Timeout/poll defaults are generic (15 min / 10 s). Callers that need + * suite-specific tuning pass explicit `opts` — env-var knobs belong in the + * caller, not this shared client. */ async waitForBuild( buildId: number, opts: { timeoutMs?: number; pollMs?: number } = {}, ): Promise<{ status: string; result?: string }> { - const timeoutMs = - opts.timeoutMs ?? (Number(process.env.TRIGGER_E2E_BUILD_TIMEOUT_MS) || 900_000); - const pollMs = opts.pollMs ?? (Number(process.env.TRIGGER_E2E_BUILD_POLL_MS) || 10_000); + const timeoutMs = opts.timeoutMs ?? 900_000; + const pollMs = opts.pollMs ?? 10_000; const deadline = Date.now() + timeoutMs; for (;;) { const build = await this.getBuild(buildId); diff --git a/scripts/ado-script/src/trigger-e2e/queue.ts b/scripts/ado-script/src/trigger-e2e/queue.ts index 600f170b..b0b7f491 100644 --- a/scripts/ado-script/src/trigger-e2e/queue.ts +++ b/scripts/ado-script/src/trigger-e2e/queue.ts @@ -19,6 +19,19 @@ function flattenParams(params: Record): Record 0 ? opts.files @@ -76,32 +75,13 @@ export async function createPrContext( [`/ado-aw-trig/${ctx.buildId}/${opts.id}.md`]: `trigger e2e ${opts.id} for build ${ctx.buildId}. Safe to delete.\n`, }; - // Push the branch with the first file, then add the rest in follow-up pushes. - const entries = Object.entries(files); - const [firstPath, firstContent] = entries[0]!; - let baseCommit = baseSha; - baseCommit = await ctx.rest.pushAddFileBranch( - repo, - branch, - baseCommit, - firstPath, - firstContent, - `trigger e2e ${opts.id}`, - ); + // Create the source branch + ALL files in a single push. Batching avoids the + // ref-conflict a per-file loop would hit: pushAddFileBranch always uses + // new-branch semantics, so a second push to the now-existing branch fails. + await ctx.rest.pushAddFilesBranch(repo, branch, baseSha, files, `trigger e2e ${opts.id}`); // From here the source branch exists; a later throw must clean it up. try { - for (const [path, content] of entries.slice(1)) { - baseCommit = await ctx.rest.pushAddFileBranch( - repo, - sourceRef, - baseCommit, - path, - content, - `trigger e2e ${opts.id} (${path})`, - ); - } - const pr = await ctx.rest.createPullRequest( repo, branch, diff --git a/scripts/ado-script/src/trigger-e2e/scenarios/pr-filters.ts b/scripts/ado-script/src/trigger-e2e/scenarios/pr-filters.ts index 12e516d6..aa4b63d2 100644 --- a/scripts/ado-script/src/trigger-e2e/scenarios/pr-filters.ts +++ b/scripts/ado-script/src/trigger-e2e/scenarios/pr-filters.ts @@ -182,6 +182,19 @@ const specs: FilterScenarioSpec[] = [ failTag: "pr-gate.build-reason-mismatch", }, // ── change count ────────────────────────────────────────────────────── + { + id: "change-count-pass", + description: "numeric_range min=1 max=10 and the PR changed 2 files → runs", + // Two files also exercises the batched multi-file push in createPrContext. + pr: { + files: { + "/src/trig-count-pass-a.txt": "a\n", + "/src/trig-count-pass-b.txt": "b\n", + }, + }, + check: changeCountCheck({ min: 1, max: 10 }), + outcome: "pass", + }, { id: "change-count-skip", description: "numeric_range min=5 but the PR changed 1 file → self-cancel", From d689a99a3f7ec17da13fa1577b42dc21aa1c66bf Mon Sep 17 00:00:00 2001 From: James Devine Date: Tue, 14 Jul 2026 11:20:29 +0100 Subject: [PATCH 4/6] test(trigger-e2e): machine-verify FACT_META against a generated fact catalog Close the gate-spec.ts / FACT_META drift hole that types.gen.ts cannot cover: policy and dependency VALUES were invisible to any type-checker, so a Rust-side change to Fact::failure_policy()/dependencies() (or a new/removed Fact) would only surface as a wrong spec at runtime. - Rust: add Fact::ALL (completeness enforced at compile time by a wildcard-free exhaustiveness match) and generate_fact_catalog(); expose it via a hidden export-fact-catalog CLI command. - codegen: emit committed fact-catalog.gen.json alongside types.gen.ts. - CI: extend the ado-script drift-check git-diff to cover the catalog, so a Rust change forces a regen + commit. - TS: factMetaCatalog() projects FACT_META into the catalog shape and a gate-spec.test.ts test deep-compares it to fact-catalog.gen.json. Verified: corrupting the catalog fails the unit test; adding a Fact variant fails compilation until Fact::ALL is updated. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: f30758a5-81c8-4024-ab01-52ad22b4f877 --- .github/workflows/ado-script.yml | 4 +- scripts/ado-script/package.json | 2 +- .../trigger-e2e/__tests__/gate-spec.test.ts | 22 +++++ .../src/trigger-e2e/fact-catalog.gen.json | 78 ++++++++++++++++ .../ado-script/src/trigger-e2e/gate-spec.ts | 28 ++++++ src/compile/filter_ir.rs | 90 ++++++++++++++++++- src/main.rs | 25 ++++++ 7 files changed, 242 insertions(+), 7 deletions(-) create mode 100644 scripts/ado-script/src/trigger-e2e/fact-catalog.gen.json diff --git a/.github/workflows/ado-script.yml b/.github/workflows/ado-script.yml index 6590f0d0..b52d4f30 100644 --- a/.github/workflows/ado-script.yml +++ b/.github/workflows/ado-script.yml @@ -54,9 +54,9 @@ jobs: - name: Verify generated TypeScript is up to date run: | - if ! git diff --exit-code -- scripts/ado-script/src/shared/types.gen.ts; then + if ! git diff --exit-code -- scripts/ado-script/src/shared/types.gen.ts scripts/ado-script/src/trigger-e2e/fact-catalog.gen.json; then echo "" - echo "::error::types.gen.ts is out of date with the Rust IR." + echo "::error::Generated files are out of date with the Rust IR (types.gen.ts and/or fact-catalog.gen.json)." echo "Run 'cd scripts/ado-script && npm run codegen' and commit the result." exit 1 fi diff --git a/scripts/ado-script/package.json b/scripts/ado-script/package.json index 7878102b..d52b039a 100644 --- a/scripts/ado-script/package.json +++ b/scripts/ado-script/package.json @@ -27,7 +27,7 @@ "build:executor-e2e": "ncc build src/executor-e2e/index.ts -o .ado-build/executor-e2e -m -t && node -e \"const fs=require('node:fs'); fs.mkdirSync('test-bin',{recursive:true}); fs.copyFileSync('.ado-build/executor-e2e/index.js','test-bin/executor-e2e.js'); fs.rmSync('.ado-build/executor-e2e',{recursive:true,force:true});\"", "build:trigger-e2e": "ncc build src/trigger-e2e/index.ts -o .ado-build/trigger-e2e -m -t && node -e \"const fs=require('node:fs'); fs.mkdirSync('test-bin',{recursive:true}); fs.copyFileSync('.ado-build/trigger-e2e/index.js','test-bin/trigger-e2e.js'); fs.rmSync('.ado-build/trigger-e2e',{recursive:true,force:true});\"", "build:check": "ls -lh gate.js && wc -c gate.js", - "codegen": "node -e \"require('node:fs').mkdirSync('schema', { recursive: true })\" && cargo run --quiet --manifest-path ../../Cargo.toml -- export-gate-schema --output schema/gate-spec.schema.json && npx json2ts schema/gate-spec.schema.json -o src/shared/types.gen.ts --bannerComment \"// AUTO-GENERATED from Rust IR via cargo run -- export-gate-schema. Do not edit; run npm run codegen.\"", + "codegen": "node -e \"require('node:fs').mkdirSync('schema', { recursive: true })\" && cargo run --quiet --manifest-path ../../Cargo.toml -- export-gate-schema --output schema/gate-spec.schema.json && npx json2ts schema/gate-spec.schema.json -o src/shared/types.gen.ts --bannerComment \"// AUTO-GENERATED from Rust IR via cargo run -- export-gate-schema. Do not edit; run npm run codegen.\" && cargo run --quiet --manifest-path ../../Cargo.toml -- export-fact-catalog --output src/trigger-e2e/fact-catalog.gen.json", "test": "vitest run", "test:smoke": "npm run build:gate && npm run build:import && npm run build:exec-context-pr && npm run build:exec-context-pr-synth && npm run build:exec-context-manual && npm run build:exec-context-pipeline && npm run build:exec-context-ci-push && npm run build:exec-context-workitem && npm run build:exec-context-schedule && npm run build:exec-context-pr-checks && npm run build:exec-context-repo && npm run build:conclusion && npm run build:approval-summary && npm run build:github-app-token && npm run build:prepare-pr-base && vitest run -c vitest.config.smoke.ts", "lint": "echo TODO", diff --git a/scripts/ado-script/src/trigger-e2e/__tests__/gate-spec.test.ts b/scripts/ado-script/src/trigger-e2e/__tests__/gate-spec.test.ts index 013cceb2..e4583684 100644 --- a/scripts/ado-script/src/trigger-e2e/__tests__/gate-spec.test.ts +++ b/scripts/ado-script/src/trigger-e2e/__tests__/gate-spec.test.ts @@ -1,3 +1,5 @@ +import { readFileSync } from "node:fs"; + import { describe, expect, it } from "vitest"; import { @@ -7,6 +9,7 @@ import { draftCheck, encodeGateSpec, encodePrSynthSpec, + factMetaCatalog, labelsCheck, targetBranchCheck, titleCheck, @@ -108,3 +111,22 @@ describe("encoding", () => { }); }); }); + +describe("FACT_META drift guard vs Rust-generated catalog", () => { + // Deep-compares the hand-maintained FACT_META mirror against the committed + // fact-catalog.gen.json, which `npm run codegen` regenerates from + // filter_ir.rs::generate_fact_catalog and CI drift-checks (ado-script.yml). + // + // How this closes the drift hole the types.gen.ts import cannot: + // - A Rust change to a fact's failure_policy/dependencies, or a new/removed + // Fact variant, regenerates fact-catalog.gen.json → CI git-diff fails + // until committed → committing makes this test fail until FACT_META is + // updated to match. Neither the policy nor dependency VALUES are visible + // to the types.gen.ts structural codegen, so this is the only guard. + it("matches fact-catalog.gen.json exactly (kind, failure_policy, dependencies, order)", () => { + const catalog = JSON.parse( + readFileSync(new URL("../fact-catalog.gen.json", import.meta.url), "utf8"), + ); + expect(factMetaCatalog()).toEqual(catalog); + }); +}); diff --git a/scripts/ado-script/src/trigger-e2e/fact-catalog.gen.json b/scripts/ado-script/src/trigger-e2e/fact-catalog.gen.json new file mode 100644 index 00000000..fd31d464 --- /dev/null +++ b/scripts/ado-script/src/trigger-e2e/fact-catalog.gen.json @@ -0,0 +1,78 @@ +[ + { + "kind": "pr_title", + "failure_policy": "fail_closed", + "dependencies": [] + }, + { + "kind": "author_email", + "failure_policy": "fail_closed", + "dependencies": [] + }, + { + "kind": "source_branch", + "failure_policy": "fail_closed", + "dependencies": [] + }, + { + "kind": "target_branch", + "failure_policy": "fail_closed", + "dependencies": [] + }, + { + "kind": "commit_message", + "failure_policy": "fail_closed", + "dependencies": [] + }, + { + "kind": "build_reason", + "failure_policy": "fail_closed", + "dependencies": [] + }, + { + "kind": "triggered_by_pipeline", + "failure_policy": "fail_closed", + "dependencies": [] + }, + { + "kind": "triggering_branch", + "failure_policy": "fail_closed", + "dependencies": [] + }, + { + "kind": "pr_metadata", + "failure_policy": "skip_dependents", + "dependencies": [] + }, + { + "kind": "pr_is_draft", + "failure_policy": "fail_closed", + "dependencies": [ + "pr_metadata" + ] + }, + { + "kind": "pr_labels", + "failure_policy": "fail_open", + "dependencies": [ + "pr_metadata" + ] + }, + { + "kind": "changed_files", + "failure_policy": "fail_open", + "dependencies": [] + }, + { + "kind": "changed_file_count", + "failure_policy": "fail_open", + "dependencies": [ + "changed_files" + ] + }, + { + "kind": "current_utc_minutes", + "failure_policy": "fail_closed", + "dependencies": [] + } +] \ No newline at end of file diff --git a/scripts/ado-script/src/trigger-e2e/gate-spec.ts b/scripts/ado-script/src/trigger-e2e/gate-spec.ts index c6d43561..08da0b48 100644 --- a/scripts/ado-script/src/trigger-e2e/gate-spec.ts +++ b/scripts/ado-script/src/trigger-e2e/gate-spec.ts @@ -45,6 +45,11 @@ interface FactMeta { * Fact kind → metadata. Keys are declared in `Fact` enum order, which the * evaluator relies on being topological (a dependency always precedes its * dependents). `buildGateSpec` emits facts in this iteration order. + * + * DRIFT GUARD: this table is deep-compared against the Rust-generated + * `fact-catalog.gen.json` by `gate-spec.test.ts` (see `factMetaCatalog`), so a + * divergence from `filter_ir.rs::Fact` fails a unit test rather than silently + * producing wrong specs at runtime. */ const FACT_META: ReadonlyMap = new Map([ ["pr_title", { policy: "fail_closed", deps: [] }], @@ -63,6 +68,29 @@ const FACT_META: ReadonlyMap = new Map([ ["current_utc_minutes", { policy: "fail_closed", deps: [] }], ]); +/** One catalog row: the serialized view of a fact's kind/policy/dependencies. */ +export interface FactCatalogEntry { + readonly kind: string; + readonly failure_policy: FailurePolicy; + readonly dependencies: string[]; +} + +/** + * `FACT_META` projected into the exact shape of the Rust-generated + * `fact-catalog.gen.json` (array, in insertion/declaration order). The + * `gate-spec.test.ts` drift test deep-compares this against that committed + * catalog so any Rust-side change to a fact's policy/dependencies — or a + * new/removed `Fact` — fails a cheap unit test instead of silently producing + * wrong specs at runtime. + */ +export function factMetaCatalog(): FactCatalogEntry[] { + return [...FACT_META].map(([kind, meta]) => ({ + kind, + failure_policy: meta.policy, + dependencies: [...meta.deps], + })); +} + /** Facts referenced directly by a predicate tree (mirror of predicates.ts collectFacts). */ function predicateFacts(p: PredicateSpec, out: Set = new Set()): Set { const fact = (p as { fact?: unknown }).fact; diff --git a/src/compile/filter_ir.rs b/src/compile/filter_ir.rs index dfc4886a..a452f6ef 100644 --- a/src/compile/filter_ir.rs +++ b/src/compile/filter_ir.rs @@ -43,10 +43,13 @@ use std::fmt; /// SYNC: the deterministic trigger-condition E2E harness hand-maintains a /// mirror of `kind()` / `failure_policy()` / `dependencies()` in /// `scripts/ado-script/src/trigger-e2e/gate-spec.ts` (the `FACT_META` table) so -/// it can craft gate specs without invoking the compiler. The `types.gen.ts` -/// codegen only guards the serialized `GateSpec` *shape*, not fact -/// policy/dependency values — so when adding a `Fact` variant, or changing a -/// `failure_policy`/`dependencies`, update `FACT_META` in that file too. +/// it can craft gate specs without invoking the compiler. This is machine- +/// guarded: `generate_fact_catalog` emits `fact-catalog.gen.json` (regenerated +/// by `npm run codegen`, CI drift-checked like `types.gen.ts`), and a +/// `gate-spec.test.ts` test deep-compares `FACT_META` against it — so a changed +/// `failure_policy`/`dependencies` or a new/removed `Fact` fails a unit test. +/// When adding a `Fact` variant, also append it to `Fact::ALL` (the exhaustive +/// match in `_fact_all_exhaustiveness_reminder` will not compile until you do). #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] pub enum Fact { // ── Pipeline variables (free — always available) ──────────────────── @@ -898,6 +901,36 @@ pub fn generate_gate_spec_schema() -> String { serde_json::to_string_pretty(&schema).expect("schema serialization") } +/// One entry in the machine-generated fact catalog. +#[derive(serde::Serialize)] +struct FactCatalogEntry { + kind: &'static str, + failure_policy: &'static str, + dependencies: Vec<&'static str>, +} + +/// Generate the fact catalog JSON: every [`Fact`]'s `kind`, `failure_policy`, +/// and (kind-named) `dependencies`, in declaration order. +/// +/// This is the machine-verifiable source of truth for the hand-maintained +/// `FACT_META` mirror in +/// `scripts/ado-script/src/trigger-e2e/gate-spec.ts`. It is emitted to a +/// committed `fact-catalog.gen.json` by `npm run codegen` and drift-checked in +/// CI exactly like `types.gen.ts`, so a Rust-side change to a fact's policy or +/// dependencies (or a new/removed `Fact`) forces a catalog regen — which the +/// trigger-e2e unit test then flags against `FACT_META`. +pub fn generate_fact_catalog() -> String { + let entries: Vec = Fact::ALL + .iter() + .map(|f| FactCatalogEntry { + kind: f.kind(), + failure_policy: f.failure_policy().as_str(), + dependencies: f.dependencies().iter().map(|d| d.kind()).collect(), + }) + .collect(); + serde_json::to_string_pretty(&entries).expect("fact catalog serialization") +} + // ─── Codegen ──────────────────────────────────────────────────────────────── // The inline heredoc evaluator has been removed in favor of external script delivery. @@ -968,6 +1001,55 @@ impl Fact { Fact::CurrentUtcMinutes => "current_utc_minutes", } } + + /// Every `Fact` variant, in declaration order. + /// + /// Kept complete by `_fact_all_exhaustiveness_reminder` below: adding a + /// `Fact` variant is a compile error there until the variant is appended + /// here, so `ALL` (and therefore the generated `fact-catalog.gen.json`) + /// can never silently miss a fact. + pub const ALL: [Fact; 14] = [ + Fact::PrTitle, + Fact::AuthorEmail, + Fact::SourceBranch, + Fact::TargetBranch, + Fact::CommitMessage, + Fact::BuildReason, + Fact::TriggeredByPipeline, + Fact::TriggeringBranch, + Fact::PrMetadata, + Fact::PrIsDraft, + Fact::PrLabels, + Fact::ChangedFiles, + Fact::ChangedFileCount, + Fact::CurrentUtcMinutes, + ]; +} + +/// Compile-time completeness guard for [`Fact::ALL`]. +/// +/// This wildcard-free match makes adding a `Fact` variant a hard compile error +/// until the new variant is listed. When you add the arm here, ALSO append the +/// variant to `Fact::ALL` above — the generated `fact-catalog.gen.json` (and the +/// TypeScript `FACT_META` mirror it guards) is derived from `Fact::ALL`. +#[allow(dead_code)] +fn _fact_all_exhaustiveness_reminder(f: Fact) { + match f { + Fact::PrTitle + | Fact::AuthorEmail + | Fact::SourceBranch + | Fact::TargetBranch + | Fact::CommitMessage + | Fact::BuildReason + | Fact::TriggeredByPipeline + | Fact::TriggeringBranch + | Fact::PrMetadata + | Fact::PrIsDraft + | Fact::PrLabels + | Fact::ChangedFiles + | Fact::ChangedFileCount + | Fact::CurrentUtcMinutes => {} + } } impl FailurePolicy { diff --git a/src/main.rs b/src/main.rs index 42a47144..0b315be5 100644 --- a/src/main.rs +++ b/src/main.rs @@ -574,6 +574,15 @@ enum Commands { #[arg(short, long)] output: Option, }, + /// Export the fact catalog JSON (kind/failure_policy/dependencies for every + /// gate `Fact`) — build-time drift guard for the trigger-e2e `FACT_META` + /// mirror in the scripts/ado-script TypeScript workspace. + #[command(hide = true)] + ExportFactCatalog { + /// Output path; if omitted, prints to stdout. + #[arg(short, long)] + output: Option, + }, /// Inspect an agent source file's typed IR: jobs, stages, steps, outputs, derived `dependsOn`. Inspect { /// Path to the agent markdown source. @@ -999,6 +1008,7 @@ async fn main() -> Result<()> { Some(Commands::Audit { .. }) => "audit", Some(Commands::Trace { .. }) => "trace", Some(Commands::ExportGateSchema { .. }) => "export-gate-schema", + Some(Commands::ExportFactCatalog { .. }) => "export-fact-catalog", Some(Commands::Inspect { .. }) => "inspect", Some(Commands::Graph { .. }) => "graph", Some(Commands::Whatif { .. }) => "whatif", @@ -1422,6 +1432,21 @@ async fn main() -> Result<()> { None => print!("{}", schema), } } + Commands::ExportFactCatalog { output } => { + let catalog = compile::filter_ir::generate_fact_catalog(); + match output { + Some(path) => { + if let Some(parent) = path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + { + std::fs::create_dir_all(parent)?; + } + std::fs::write(&path, &catalog)?; + } + None => print!("{}", catalog), + } + } Commands::Inspect { source, json } => { inspect::dispatch_inspect(inspect::InspectOptions { source: &source, From 54d3d76b75a650818a2b8c4e36f20839172f113d Mon Sep 17 00:00:00 2001 From: James Devine Date: Tue, 14 Jul 2026 11:37:12 +0100 Subject: [PATCH 5/6] test(trigger-e2e): polish per third-round PR review - main.rs: wrap the export-gate-schema / export-fact-catalog fs writes and create_dir_all in .with_context so a CLI write failure names the target path. - filter_ir.rs: document that Fact::ALL's [Fact; 14] length literal is a second compile-time guard that must track the variant count. - gate-spec.ts: replace predicateFacts' (p as { fact? }).fact duck-typing with a discriminant switch + never-exhaustiveness default, so a renamed/added PredicateSpec field or variant is a compile error instead of a silently under-specified fact list. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: f30758a5-81c8-4024-ab01-52ad22b4f877 --- .../ado-script/src/trigger-e2e/gate-spec.ts | 39 +++++++++++++++---- src/compile/filter_ir.rs | 4 ++ src/main.rs | 14 +++++-- 3 files changed, 46 insertions(+), 11 deletions(-) diff --git a/scripts/ado-script/src/trigger-e2e/gate-spec.ts b/scripts/ado-script/src/trigger-e2e/gate-spec.ts index 08da0b48..f42e97dd 100644 --- a/scripts/ado-script/src/trigger-e2e/gate-spec.ts +++ b/scripts/ado-script/src/trigger-e2e/gate-spec.ts @@ -91,15 +91,40 @@ export function factMetaCatalog(): FactCatalogEntry[] { })); } -/** Facts referenced directly by a predicate tree (mirror of predicates.ts collectFacts). */ +/** + * Facts referenced directly by a predicate tree (mirror of predicates.ts + * collectFacts). Switches on the `type` discriminant so each variant's `fact` + * field is accessed type-safely: if a future `PredicateSpec` variant renames or + * drops `fact`, this fails to compile rather than silently returning + * `undefined` and under-specifying the fact list. The `never` default makes a + * newly-added variant a compile error too. + */ function predicateFacts(p: PredicateSpec, out: Set = new Set()): Set { - const fact = (p as { fact?: unknown }).fact; - if (typeof fact === "string") out.add(fact); - if (p.type === "time_window") out.add("current_utc_minutes"); - if (p.type === "and" || p.type === "or") { - for (const sub of p.operands) predicateFacts(sub, out); + switch (p.type) { + case "glob_match": + case "equals": + case "value_in_set": + case "value_not_in_set": + case "numeric_range": + case "label_set_match": + case "file_glob_match": + out.add(p.fact); + break; + case "time_window": + out.add("current_utc_minutes"); + break; + case "and": + case "or": + for (const sub of p.operands) predicateFacts(sub, out); + break; + case "not": + predicateFacts(p.operand, out); + break; + default: { + const _exhaustive: never = p; + throw new Error(`gate-spec: unhandled predicate type '${(_exhaustive as { type?: string }).type}'`); + } } - if (p.type === "not") predicateFacts(p.operand, out); return out; } diff --git a/src/compile/filter_ir.rs b/src/compile/filter_ir.rs index a452f6ef..a9f8b92d 100644 --- a/src/compile/filter_ir.rs +++ b/src/compile/filter_ir.rs @@ -1008,6 +1008,10 @@ impl Fact { /// `Fact` variant is a compile error there until the variant is appended /// here, so `ALL` (and therefore the generated `fact-catalog.gen.json`) /// can never silently miss a fact. + /// + /// Maintenance: the `14` length literal is itself a second compile-time + /// guard — appending a variant without bumping it is a mismatched-array- + /// length error. Keep it equal to the number of `Fact` variants. pub const ALL: [Fact; 14] = [ Fact::PrTitle, Fact::AuthorEmail, diff --git a/src/main.rs b/src/main.rs index 0b315be5..a6091677 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1425,9 +1425,12 @@ async fn main() -> Result<()> { .parent() .filter(|parent| !parent.as_os_str().is_empty()) { - std::fs::create_dir_all(parent)?; + std::fs::create_dir_all(parent).with_context(|| { + format!("creating parent dir for gate schema: {}", parent.display()) + })?; } - std::fs::write(&path, &schema)?; + std::fs::write(&path, &schema) + .with_context(|| format!("writing gate schema to {}", path.display()))?; } None => print!("{}", schema), } @@ -1440,9 +1443,12 @@ async fn main() -> Result<()> { .parent() .filter(|parent| !parent.as_os_str().is_empty()) { - std::fs::create_dir_all(parent)?; + std::fs::create_dir_all(parent).with_context(|| { + format!("creating parent dir for fact catalog: {}", parent.display()) + })?; } - std::fs::write(&path, &catalog)?; + std::fs::write(&path, &catalog) + .with_context(|| format!("writing fact catalog to {}", path.display()))?; } None => print!("{}", catalog), } From 7dd849b54fa0c5f58c655fca34e4c8a63ee15afb Mon Sep 17 00:00:00 2001 From: James Devine Date: Tue, 14 Jul 2026 11:38:31 +0100 Subject: [PATCH 6/6] docs(trigger-e2e): note multi-step setup leak risk and pool-name override Address remaining PR review doc suggestions: - createPrContext: document that a scenario doing further setup work after a successful createPrContext must teardown the PrContext itself before rethrowing, since the runner won't run cleanup until setup() returns. - README: call out that the hardcoded AZS-1ES pool name in both pipelines may need changing for the operator's ADO project. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: f30758a5-81c8-4024-ab01-52ad22b4f877 --- scripts/ado-script/src/trigger-e2e/scenarios/common.ts | 9 +++++++++ tests/trigger-e2e/README.md | 4 ++++ 2 files changed, 13 insertions(+) diff --git a/scripts/ado-script/src/trigger-e2e/scenarios/common.ts b/scripts/ado-script/src/trigger-e2e/scenarios/common.ts index ca2c483f..3b2f7867 100644 --- a/scripts/ado-script/src/trigger-e2e/scenarios/common.ts +++ b/scripts/ado-script/src/trigger-e2e/scenarios/common.ts @@ -52,6 +52,15 @@ export interface CreatePrOptions { * * Follows the executor-e2e contract: because a `setup()` throw means the runner * will NOT call cleanup, this tears down anything it created before rethrowing. + * + * IMPORTANT for multi-step setup: the runner only calls `cleanup()` once + * `setup()` returns (it gates on `setupDone`). If a scenario's `setup()` calls + * `createPrContext` successfully and then does *further* work that throws, the + * returned `PrContext` is leaked — cleanup never runs. A scenario that needs + * post-PR setup steps must wrap them in its own try/catch that calls + * `teardownPrContext` before rethrowing (mirroring executor-e2e's `setupPr`). + * All current scenarios' `setup()` is exactly `createPrContext`, so none are + * exposed today. */ export async function createPrContext( ctx: TriggerContext, diff --git a/tests/trigger-e2e/README.md b/tests/trigger-e2e/README.md index 3c8ffd18..4b558466 100644 --- a/tests/trigger-e2e/README.md +++ b/tests/trigger-e2e/README.md @@ -127,6 +127,10 @@ In `https://dev.azure.com/msazuresphere/AgentPlayground`: It **must** be an ADO Git repo (not GitHub) so `exec-context-pr-synth` can discover open PRs in the pipeline's own `self` repo. Note its **definition id** for the next step. + - **Pool name.** Both `victim-pipeline.yml` and `azure-pipelines.yml` + hardcode `pool: { name: AZS-1ES-L-Playground-ubuntu-22.04 }`. If your ADO + project uses a different agent pool, edit both files to point at a + Linux pool with Node.js 20 available before registering. 2. **Register the ORCHESTRATOR pipeline.** New pipeline → existing YAML → `tests/trigger-e2e/azure-pipelines.yml`. Place both in a `\trigger-e2e` folder.