diff --git a/.github/workflows/ado-script.yml b/.github/workflows/ado-script.yml index 6590f0d0..b52d4f30 100644 --- a/.github/workflows/ado-script.yml +++ b/.github/workflows/ado-script.yml @@ -54,9 +54,9 @@ jobs: - name: Verify generated TypeScript is up to date run: | - if ! git diff --exit-code -- scripts/ado-script/src/shared/types.gen.ts; then + if ! git diff --exit-code -- scripts/ado-script/src/shared/types.gen.ts scripts/ado-script/src/trigger-e2e/fact-catalog.gen.json; then echo "" - echo "::error::types.gen.ts is out of date with the Rust IR." + echo "::error::Generated files are out of date with the Rust IR (types.gen.ts and/or fact-catalog.gen.json)." echo "Run 'cd scripts/ado-script && npm run codegen' and commit the result." exit 1 fi diff --git a/scripts/ado-script/package.json b/scripts/ado-script/package.json index ab4b6768..d52b039a 100644 --- a/scripts/ado-script/package.json +++ b/scripts/ado-script/package.json @@ -25,8 +25,9 @@ "build:github-app-token": "ncc build src/github-app-token/index.ts -o .ado-build/github-app-token -m -t && node -e \"const fs=require('node:fs'); fs.copyFileSync('.ado-build/github-app-token/index.js','github-app-token.js'); fs.rmSync('.ado-build/github-app-token',{recursive:true,force:true});\"", "build:prepare-pr-base": "ncc build src/prepare-pr-base/index.ts -o .ado-build/prepare-pr-base -m -t && node -e \"const fs=require('node:fs'); fs.copyFileSync('.ado-build/prepare-pr-base/index.js','prepare-pr-base.js'); fs.rmSync('.ado-build/prepare-pr-base',{recursive:true,force:true});\"", "build:executor-e2e": "ncc build src/executor-e2e/index.ts -o .ado-build/executor-e2e -m -t && node -e \"const fs=require('node:fs'); fs.mkdirSync('test-bin',{recursive:true}); fs.copyFileSync('.ado-build/executor-e2e/index.js','test-bin/executor-e2e.js'); fs.rmSync('.ado-build/executor-e2e',{recursive:true,force:true});\"", + "build:trigger-e2e": "ncc build src/trigger-e2e/index.ts -o .ado-build/trigger-e2e -m -t && node -e \"const fs=require('node:fs'); fs.mkdirSync('test-bin',{recursive:true}); fs.copyFileSync('.ado-build/trigger-e2e/index.js','test-bin/trigger-e2e.js'); fs.rmSync('.ado-build/trigger-e2e',{recursive:true,force:true});\"", "build:check": "ls -lh gate.js && wc -c gate.js", - "codegen": "node -e \"require('node:fs').mkdirSync('schema', { recursive: true })\" && cargo run --quiet --manifest-path ../../Cargo.toml -- export-gate-schema --output schema/gate-spec.schema.json && npx json2ts schema/gate-spec.schema.json -o src/shared/types.gen.ts --bannerComment \"// AUTO-GENERATED from Rust IR via cargo run -- export-gate-schema. Do not edit; run npm run codegen.\"", + "codegen": "node -e \"require('node:fs').mkdirSync('schema', { recursive: true })\" && cargo run --quiet --manifest-path ../../Cargo.toml -- export-gate-schema --output schema/gate-spec.schema.json && npx json2ts schema/gate-spec.schema.json -o src/shared/types.gen.ts --bannerComment \"// AUTO-GENERATED from Rust IR via cargo run -- export-gate-schema. Do not edit; run npm run codegen.\" && cargo run --quiet --manifest-path ../../Cargo.toml -- export-fact-catalog --output src/trigger-e2e/fact-catalog.gen.json", "test": "vitest run", "test:smoke": "npm run build:gate && npm run build:import && npm run build:exec-context-pr && npm run build:exec-context-pr-synth && npm run build:exec-context-manual && npm run build:exec-context-pipeline && npm run build:exec-context-ci-push && npm run build:exec-context-workitem && npm run build:exec-context-schedule && npm run build:exec-context-pr-checks && npm run build:exec-context-repo && npm run build:conclusion && npm run build:approval-summary && npm run build:github-app-token && npm run build:prepare-pr-base && vitest run -c vitest.config.smoke.ts", "lint": "echo TODO", diff --git a/scripts/ado-script/src/__tests__/bundle-coverage.test.ts b/scripts/ado-script/src/__tests__/bundle-coverage.test.ts index b392d2cd..4e6e89ce 100644 --- a/scripts/ado-script/src/__tests__/bundle-coverage.test.ts +++ b/scripts/ado-script/src/__tests__/bundle-coverage.test.ts @@ -30,8 +30,12 @@ const packageJsonPath = join(here, "..", "..", "package.json"); * so the release glob (`ado-script/*.js`) never packages it — it is a test * harness run only by the executor-e2e pipeline, never downloaded by compiled * agentic pipelines at runtime. + * + * `trigger-e2e` is the analogous deterministic trigger-condition (gate / + * synth-PR) E2E harness. Same treatment: its own `build:trigger-e2e` emits to + * `test-bin/`, so it is never packaged in `ado-script.zip`. */ -const NON_BUNDLE_DIRS = new Set(["shared", "__tests__", "executor-e2e"]); +const NON_BUNDLE_DIRS = new Set(["shared", "__tests__", "executor-e2e", "trigger-e2e"]); function listBundleDirs(): string[] { return readdirSync(srcDir) diff --git a/scripts/ado-script/src/executor-e2e/ado-rest.ts b/scripts/ado-script/src/executor-e2e/ado-rest.ts index fb854c97..78ffd453 100644 --- a/scripts/ado-script/src/executor-e2e/ado-rest.ts +++ b/scripts/ado-script/src/executor-e2e/ado-rest.ts @@ -256,6 +256,55 @@ export class AdoRest { return commitId; } + /** + * Create a NEW branch and a single commit adding one OR MORE files in one + * push. Returns the new commit id. + * + * Prefer this over calling {@link pushAddFileBranch} in a loop: that helper + * always uses new-branch semantics (`oldObjectId` = zeros), so a second call + * against the now-existing branch would be rejected by ADO with a ref + * conflict. Batching every file into a single commit sidesteps that entirely + * and still produces a real diff vs. the base for PR scenarios. + */ + async pushAddFilesBranch( + repo: string, + branchName: string, + baseCommitId: string, + files: Record, + comment: string, + ): Promise { + const entries = Object.entries(files); + if (entries.length === 0) throw new Error("pushAddFilesBranch requires at least one file"); + const path = this.projPath( + `_apis/git/repositories/${AdoRest.seg(repo)}/pushes?api-version=7.1`, + ); + const res = await this.request<{ commits?: { commitId: string }[] }>(path, { + method: "POST", + body: { + refUpdates: [ + { + name: branchName.startsWith("refs/") ? branchName : `refs/heads/${branchName}`, + oldObjectId: "0000000000000000000000000000000000000000", + }, + ], + commits: [ + { + comment, + parents: [baseCommitId], + changes: entries.map(([filePath, content]) => ({ + changeType: "add", + item: { path: filePath.startsWith("/") ? filePath : `/${filePath}` }, + newContent: { content, contentType: "rawtext" }, + })), + }, + ], + }, + }); + const commitId = res?.commits?.[0]?.commitId; + if (!commitId) throw new Error("pushAddFilesBranch returned no commit id"); + return commitId; + } + // ---- Git: pull requests & threads ------------------------------------ async createPullRequest( @@ -264,18 +313,21 @@ export class AdoRest { targetRef: string, title: string, description: string, + isDraft?: boolean, ): Promise<{ pullRequestId: number }> { const path = this.projPath( `_apis/git/repositories/${AdoRest.seg(repo)}/pullrequests?api-version=7.1`, ); + const body: Record = { + sourceRefName: sourceRef.startsWith("refs/") ? sourceRef : `refs/heads/${sourceRef}`, + targetRefName: targetRef.startsWith("refs/") ? targetRef : `refs/heads/${targetRef}`, + title, + description, + }; + if (isDraft !== undefined) body.isDraft = isDraft; const res = await this.request<{ pullRequestId: number }>(path, { method: "POST", - body: { - sourceRefName: sourceRef.startsWith("refs/") ? sourceRef : `refs/heads/${sourceRef}`, - targetRefName: targetRef.startsWith("refs/") ? targetRef : `refs/heads/${targetRef}`, - title, - description, - }, + body, }); if (!res) throw new Error("createPullRequest returned no body"); return res; @@ -365,6 +417,27 @@ export class AdoRest { await this.request(path, { method: "PATCH", body: { status: "abandoned" }, allow404: true }); } + /** + * Attach one or more labels (tags) to a PR. ADO's label endpoint takes a + * single `{ name }` per POST, so this loops. Used by the trigger E2E harness + * to exercise the gate's `label_set_match` predicate against real PR labels. + */ + async setPullRequestLabels(repo: string, prId: number, labels: string[]): Promise { + for (const name of labels) { + const path = this.projPath( + `_apis/git/repositories/${AdoRest.seg(repo)}/pullRequests/${prId}/labels?api-version=7.1`, + ); + try { + await this.request(path, { method: "POST", body: { name } }); + } catch (err) { + // Name the specific label that failed so a partial-attach surfaces as a + // clear setup error rather than a confusing downstream gate mismatch. + const message = err instanceof Error ? err.message : String(err); + throw new Error(`setPullRequestLabels: failed to attach label '${name}' to PR ${prId}: ${message}`); + } + } + } + // ---- Wiki ------------------------------------------------------------- async listWikis(): Promise<{ name: string; id: string; type?: string }[]> { @@ -451,4 +524,65 @@ export class AdoRest { const path = this.projPath(`_apis/build/builds/${buildId}?api-version=7.1`); await this.request(path, { method: "PATCH", body: { status: "cancelling" }, allow404: true }); } + + /** + * Queue a new build of a registered definition. `sourceBranch` selects the + * branch to build (used by the trigger E2E harness to point a queued build + * at a PR's source branch so `exec-context-pr-synth` can discover the open + * PR). `templateParameters` supplies the victim pipeline's runtime + * parameters (e.g. the base64 GATE_SPEC / PR_SYNTH_SPEC under test). + * + * Returns the new build id. Note: a build queued via this REST call has + * `Build.Reason = Manual`; the victim relies on the synthetic-PR flag (set + * by `exec-context-pr-synth` from a real open PR) — not the build reason — + * to drive full PR-gate evaluation. + */ + async queueBuild( + definitionId: number, + opts: { sourceBranch?: string; templateParameters?: Record } = {}, + ): Promise<{ id: number }> { + const path = this.projPath(`_apis/build/builds?api-version=7.1`); + const body: Record = { definition: { id: definitionId } }; + if (opts.sourceBranch) { + body.sourceBranch = opts.sourceBranch.startsWith("refs/") + ? opts.sourceBranch + : `refs/heads/${opts.sourceBranch}`; + } + if (opts.templateParameters && Object.keys(opts.templateParameters).length > 0) { + body.templateParameters = opts.templateParameters; + } + const res = await this.request<{ id: number }>(path, { method: "POST", body }); + if (!res) throw new Error(`queueBuild(${definitionId}) returned no body`); + return res; + } + + /** + * Poll a build until it reaches `status === "completed"` (or the timeout + * elapses). Returns the terminal `{ status, result }`. `result` is one of + * `succeeded` | `partiallySucceeded` | `failed` | `canceled`. + * + * Timeout/poll defaults are generic (15 min / 10 s). Callers that need + * suite-specific tuning pass explicit `opts` — env-var knobs belong in the + * caller, not this shared client. + */ + async waitForBuild( + buildId: number, + opts: { timeoutMs?: number; pollMs?: number } = {}, + ): Promise<{ status: string; result?: string }> { + const timeoutMs = opts.timeoutMs ?? 900_000; + const pollMs = opts.pollMs ?? 10_000; + const deadline = Date.now() + timeoutMs; + for (;;) { + const build = await this.getBuild(buildId); + if (build.status === "completed") { + return { status: build.status, result: build.result }; + } + if (Date.now() >= deadline) { + throw new Error( + `waitForBuild(${buildId}) timed out after ${timeoutMs}ms (last status='${build.status ?? "?"}')`, + ); + } + await new Promise((r) => setTimeout(r, pollMs)); + } + } } diff --git a/scripts/ado-script/src/trigger-e2e/__tests__/gate-spec.test.ts b/scripts/ado-script/src/trigger-e2e/__tests__/gate-spec.test.ts new file mode 100644 index 00000000..e4583684 --- /dev/null +++ b/scripts/ado-script/src/trigger-e2e/__tests__/gate-spec.test.ts @@ -0,0 +1,132 @@ +import { readFileSync } from "node:fs"; + +import { describe, expect, it } from "vitest"; + +import { + buildGateSpec, + changeCountCheck, + changedFilesCheck, + draftCheck, + encodeGateSpec, + encodePrSynthSpec, + factMetaCatalog, + labelsCheck, + targetBranchCheck, + titleCheck, +} from "../gate-spec.js"; + +const factKinds = (checks: Parameters[1]) => + buildGateSpec("pull-request", checks).facts.map((f) => f.kind); + +describe("buildGateSpec fact derivation", () => { + it("emits no facts and no checks for an empty gate", () => { + const spec = buildGateSpec("pull-request", []); + expect(spec.facts).toEqual([]); + expect(spec.checks).toEqual([]); + expect(spec.context.tag_prefix).toBe("pr-gate"); + expect(spec.context.build_reason).toBe("PullRequest"); + expect(spec.context.bypass_label).toBe("PR"); + }); + + it("derives pr_metadata (skip_dependents) before pr_labels (fail_open) for a labels check", () => { + const spec = buildGateSpec("pull-request", [labelsCheck({ anyOf: ["run-agent"] })]); + const kinds = spec.facts.map((f) => f.kind); + expect(kinds).toEqual(["pr_metadata", "pr_labels"]); + const meta = Object.fromEntries(spec.facts.map((f) => [f.kind, f])); + expect(meta.pr_metadata!.failure_policy).toBe("skip_dependents"); + expect(meta.pr_metadata!.dependencies).toEqual([]); + expect(meta.pr_labels!.failure_policy).toBe("fail_open"); + expect(meta.pr_labels!.dependencies).toEqual(["pr_metadata"]); + }); + + it("derives pr_metadata before pr_is_draft (fail_closed) for a draft check", () => { + expect(factKinds([draftCheck(true)])).toEqual(["pr_metadata", "pr_is_draft"]); + }); + + it("derives changed_files before changed_file_count for a change-count check", () => { + expect(factKinds([changeCountCheck({ min: 5 })])).toEqual([ + "changed_files", + "changed_file_count", + ]); + }); + + it("derives only changed_files for a changed-files check", () => { + expect(factKinds([changedFilesCheck({ include: ["src/**"] })])).toEqual(["changed_files"]); + }); + + it("derives a single fail_closed pipeline-var fact for a title check", () => { + const spec = buildGateSpec("pull-request", [titleCheck("release *")]); + expect(spec.facts).toEqual([ + { kind: "pr_title", failure_policy: "fail_closed", dependencies: [] }, + ]); + }); + + it("dedupes facts shared across multiple checks", () => { + const kinds = factKinds([draftCheck(true), labelsCheck({ anyOf: ["x"] })]); + // pr_metadata appears once, before both dependents, in enum order. + expect(kinds).toEqual(["pr_metadata", "pr_is_draft", "pr_labels"]); + }); +}); + +describe("check tag suffixes match the Rust FilterCheck::build_tag_suffix", () => { + it("emits the expected suffixes", () => { + const cases: [Parameters[1][number], string][] = [ + [titleCheck("x"), "title-mismatch"], + [labelsCheck({ anyOf: ["x"] }), "labels-mismatch"], + [changedFilesCheck({ include: ["src/**"] }), "changed-files-mismatch"], + [draftCheck(true), "draft-mismatch"], + [targetBranchCheck("main"), "target-branch-mismatch"], + [changeCountCheck({ min: 1 }), "changes-mismatch"], + ]; + for (const [check, suffix] of cases) { + const spec = buildGateSpec("pull-request", [check]); + expect(spec.checks[0]!.tag_suffix).toBe(suffix); + } + }); +}); + +describe("encoding", () => { + it("round-trips a gate spec through base64", () => { + const spec = buildGateSpec("pull-request", [labelsCheck({ anyOf: ["run-agent"] })]); + const decoded = JSON.parse(Buffer.from(encodeGateSpec(spec), "base64").toString("utf8")); + expect(decoded).toEqual(spec); + }); + + it("produces the canonical empty PR_SYNTH_SPEC base64", () => { + expect(encodePrSynthSpec()).toBe( + "eyJicmFuY2hlcyI6eyJpbmNsdWRlIjpbXSwiZXhjbHVkZSI6W119LCJwYXRocyI6eyJpbmNsdWRlIjpbXSwiZXhjbHVkZSI6W119fQ==", + ); + }); + + it("encodes provided branch/path filters", () => { + const decoded = JSON.parse( + Buffer.from( + encodePrSynthSpec({ branches: { include: ["main"] }, paths: { exclude: ["docs/**"] } }), + "base64", + ).toString("utf8"), + ); + expect(decoded).toEqual({ + branches: { include: ["main"], exclude: [] }, + paths: { include: [], exclude: ["docs/**"] }, + }); + }); +}); + +describe("FACT_META drift guard vs Rust-generated catalog", () => { + // Deep-compares the hand-maintained FACT_META mirror against the committed + // fact-catalog.gen.json, which `npm run codegen` regenerates from + // filter_ir.rs::generate_fact_catalog and CI drift-checks (ado-script.yml). + // + // How this closes the drift hole the types.gen.ts import cannot: + // - A Rust change to a fact's failure_policy/dependencies, or a new/removed + // Fact variant, regenerates fact-catalog.gen.json → CI git-diff fails + // until committed → committing makes this test fail until FACT_META is + // updated to match. Neither the policy nor dependency VALUES are visible + // to the types.gen.ts structural codegen, so this is the only guard. + it("matches fact-catalog.gen.json exactly (kind, failure_policy, dependencies, order)", () => { + const catalog = JSON.parse( + readFileSync(new URL("../fact-catalog.gen.json", import.meta.url), "utf8"), + ); + expect(factMetaCatalog()).toEqual(catalog); + }); +}); diff --git a/scripts/ado-script/src/trigger-e2e/__tests__/runner.test.ts b/scripts/ado-script/src/trigger-e2e/__tests__/runner.test.ts new file mode 100644 index 00000000..4f26028b --- /dev/null +++ b/scripts/ado-script/src/trigger-e2e/__tests__/runner.test.ts @@ -0,0 +1,144 @@ +import { describe, expect, it, vi } from "vitest"; + +import type { AdoRest } from "../../executor-e2e/ado-rest.js"; +import { runScenario } from "../runner.js"; +import { SkipError } from "../scenario.js"; +import type { TriggerContext, TriggerScenario } from "../scenario.js"; + +interface FakeBuild { + result?: string; + tags?: string[]; + /** Optional override for the finally-block getBuild status probe. */ + finalStatus?: string; + /** When set, waitForBuild rejects with this message (queue-phase failure). */ + waitError?: string; +} + +function makeCtx(build: FakeBuild): { ctx: TriggerContext; rest: Record> } { + const rest = { + queueBuild: vi.fn().mockResolvedValue({ id: 42 }), + waitForBuild: build.waitError + ? vi.fn().mockRejectedValue(new Error(build.waitError)) + : vi.fn().mockResolvedValue({ status: "completed", result: build.result }), + getBuildTags: vi.fn().mockResolvedValue(build.tags ?? []), + getBuild: vi.fn().mockResolvedValue({ id: 42, status: build.finalStatus ?? "completed" }), + cancelBuild: vi.fn().mockResolvedValue(undefined), + }; + const ctx: TriggerContext = { + orgUrl: "https://dev.azure.com/org/", + project: "proj", + adoRepo: "repo", + buildId: "1", + token: "t", + victimDefinitionId: 7, + rest: rest as unknown as AdoRest, + log: () => {}, + prefix: (id) => `ado-aw-trig-1-${id}`, + }; + return { ctx, rest }; +} + +function scenario(overrides: Partial>): TriggerScenario { + return { + id: "s", + description: "test scenario", + setup: async () => "state", + queue: () => ({ templateParameters: { gateSpec: "x" } }), + expected: () => ({ result: "succeeded" }), + cleanup: async () => {}, + ...overrides, + }; +} + +describe("runScenario", () => { + it("passes when result and tags match", async () => { + const { ctx } = makeCtx({ result: "succeeded", tags: ["trig.should-run.true"] }); + const res = await runScenario( + ctx, + scenario({ expected: () => ({ result: "succeeded", tags: ["trig.should-run.true"] }) }), + ); + expect(res.ok).toBe(true); + }); + + it("fails (assert) when a required tag is missing", async () => { + const { ctx } = makeCtx({ result: "succeeded", tags: [] }); + const res = await runScenario(ctx, scenario({ expected: () => ({ tags: ["pr-gate.skipped"] }) })); + expect(res.ok).toBe(false); + expect(res.phase).toBe("assert"); + expect(res.message).toContain("pr-gate.skipped"); + }); + + it("fails (assert) on the wrong build result", async () => { + const { ctx } = makeCtx({ result: "canceled" }); + const res = await runScenario(ctx, scenario({ expected: () => ({ result: "succeeded" }) })); + expect(res.ok).toBe(false); + expect(res.phase).toBe("assert"); + }); + + it("fails (assert) when a forbidden tag is present", async () => { + const { ctx } = makeCtx({ result: "succeeded", tags: ["pr-gate.skipped"] }); + const res = await runScenario( + ctx, + scenario({ expected: () => ({ absentTags: ["pr-gate.skipped"] }) }), + ); + expect(res.ok).toBe(false); + expect(res.phase).toBe("assert"); + }); + + it("records a SkipError from setup as skipped and does NOT run cleanup", async () => { + const { ctx } = makeCtx({ result: "succeeded" }); + const cleanup = vi.fn().mockResolvedValue(undefined); + const res = await runScenario( + ctx, + scenario({ + setup: async () => { + throw new SkipError("no repo"); + }, + cleanup, + }), + ); + expect(res.skipped).toBe(true); + expect(res.ok).toBe(true); + expect(cleanup).not.toHaveBeenCalled(); + }); + + it("runs cleanup after a successful setup even when assertion fails", async () => { + const { ctx } = makeCtx({ result: "failed" }); + const cleanup = vi.fn().mockResolvedValue(undefined); + const res = await runScenario(ctx, scenario({ cleanup })); + expect(res.ok).toBe(false); + expect(cleanup).toHaveBeenCalledOnce(); + }); + + it("honours a custom assert hook", async () => { + const { ctx } = makeCtx({ result: "succeeded" }); + const res = await runScenario( + ctx, + scenario({ + assert: async () => { + throw new Error("custom boom"); + }, + }), + ); + expect(res.ok).toBe(false); + expect(res.message).toContain("custom boom"); + }); + + it("cancels an orphaned victim build when the poll times out (queue phase)", async () => { + // waitForBuild rejects (timeout); the build is still running afterwards. + const { ctx, rest } = makeCtx({ waitError: "waitForBuild timed out", finalStatus: "inProgress" }); + const res = await runScenario(ctx, scenario({})); + expect(res.ok).toBe(false); + expect(res.phase).toBe("queue"); + // The queued build id (42) must be cancelled so it is not orphaned. + expect(rest.getBuild).toHaveBeenCalledWith(42); + expect(rest.cancelBuild).toHaveBeenCalledWith(42); + }); + + it("does NOT cancel a build that already completed", async () => { + const { ctx, rest } = makeCtx({ result: "succeeded", finalStatus: "completed" }); + const res = await runScenario(ctx, scenario({ expected: () => ({ result: "succeeded" }) })); + expect(res.ok).toBe(true); + expect(rest.cancelBuild).not.toHaveBeenCalled(); + }); +}); diff --git a/scripts/ado-script/src/trigger-e2e/fact-catalog.gen.json b/scripts/ado-script/src/trigger-e2e/fact-catalog.gen.json new file mode 100644 index 00000000..fd31d464 --- /dev/null +++ b/scripts/ado-script/src/trigger-e2e/fact-catalog.gen.json @@ -0,0 +1,78 @@ +[ + { + "kind": "pr_title", + "failure_policy": "fail_closed", + "dependencies": [] + }, + { + "kind": "author_email", + "failure_policy": "fail_closed", + "dependencies": [] + }, + { + "kind": "source_branch", + "failure_policy": "fail_closed", + "dependencies": [] + }, + { + "kind": "target_branch", + "failure_policy": "fail_closed", + "dependencies": [] + }, + { + "kind": "commit_message", + "failure_policy": "fail_closed", + "dependencies": [] + }, + { + "kind": "build_reason", + "failure_policy": "fail_closed", + "dependencies": [] + }, + { + "kind": "triggered_by_pipeline", + "failure_policy": "fail_closed", + "dependencies": [] + }, + { + "kind": "triggering_branch", + "failure_policy": "fail_closed", + "dependencies": [] + }, + { + "kind": "pr_metadata", + "failure_policy": "skip_dependents", + "dependencies": [] + }, + { + "kind": "pr_is_draft", + "failure_policy": "fail_closed", + "dependencies": [ + "pr_metadata" + ] + }, + { + "kind": "pr_labels", + "failure_policy": "fail_open", + "dependencies": [ + "pr_metadata" + ] + }, + { + "kind": "changed_files", + "failure_policy": "fail_open", + "dependencies": [] + }, + { + "kind": "changed_file_count", + "failure_policy": "fail_open", + "dependencies": [ + "changed_files" + ] + }, + { + "kind": "current_utc_minutes", + "failure_policy": "fail_closed", + "dependencies": [] + } +] \ No newline at end of file diff --git a/scripts/ado-script/src/trigger-e2e/gate-spec.ts b/scripts/ado-script/src/trigger-e2e/gate-spec.ts new file mode 100644 index 00000000..f42e97dd --- /dev/null +++ b/scripts/ado-script/src/trigger-e2e/gate-spec.ts @@ -0,0 +1,344 @@ +/** + * Base64 `GateSpec` / `PrSynthSpec` builders for the trigger-condition E2E + * harness. + * + * These are a faithful TypeScript port of the Rust spec construction in + * `src/compile/filter_ir.rs` (`lower_pr_filters` + `build_gate_spec` + + * `build_pr_synth_spec`). They deliberately import the codegen'd `GateSpec` + * shape from `../shared/types.gen.js` so that any drift in the serialized + * schema surfaces as a TypeScript compile error rather than a silent + * wrong-answer at runtime. + * + * We do NOT run the Rust compiler here: the harness crafts the exact filter + * under test directly, so a single hand-authored victim pipeline can exercise + * every predicate by receiving a different base64 `GATE_SPEC` per queued build. + * + * Fidelity contract (must stay in lock-step with `filter_ir.rs`): + * - fact `kind` strings → `Fact::kind()` + * - fact `failure_policy` → `Fact::failure_policy()` + * - fact `dependencies` → `Fact::dependencies()` + * - canonical fact emission order → `Fact` enum declaration order + * (already topological; see + * `collect_ordered_facts`) + * - check `tag_suffix` values → `FilterCheck::build_tag_suffix` + * + * Test-harness module; not shipped in `ado-script.zip`. + */ +import type { + CheckSpec, + FactSpec, + GateContextSpec, + GateSpec, + PredicateSpec, +} from "../shared/types.gen.js"; + +// ─── Fact metadata (mirror of filter_ir.rs Fact::{kind,failure_policy,dependencies}) ── + +type FailurePolicy = "fail_closed" | "fail_open" | "skip_dependents"; + +interface FactMeta { + readonly policy: FailurePolicy; + readonly deps: readonly string[]; +} + +/** + * Fact kind → metadata. Keys are declared in `Fact` enum order, which the + * evaluator relies on being topological (a dependency always precedes its + * dependents). `buildGateSpec` emits facts in this iteration order. + * + * DRIFT GUARD: this table is deep-compared against the Rust-generated + * `fact-catalog.gen.json` by `gate-spec.test.ts` (see `factMetaCatalog`), so a + * divergence from `filter_ir.rs::Fact` fails a unit test rather than silently + * producing wrong specs at runtime. + */ +const FACT_META: ReadonlyMap = new Map([ + ["pr_title", { policy: "fail_closed", deps: [] }], + ["author_email", { policy: "fail_closed", deps: [] }], + ["source_branch", { policy: "fail_closed", deps: [] }], + ["target_branch", { policy: "fail_closed", deps: [] }], + ["commit_message", { policy: "fail_closed", deps: [] }], + ["build_reason", { policy: "fail_closed", deps: [] }], + ["triggered_by_pipeline", { policy: "fail_closed", deps: [] }], + ["triggering_branch", { policy: "fail_closed", deps: [] }], + ["pr_metadata", { policy: "skip_dependents", deps: [] }], + ["pr_is_draft", { policy: "fail_closed", deps: ["pr_metadata"] }], + ["pr_labels", { policy: "fail_open", deps: ["pr_metadata"] }], + ["changed_files", { policy: "fail_open", deps: [] }], + ["changed_file_count", { policy: "fail_open", deps: ["changed_files"] }], + ["current_utc_minutes", { policy: "fail_closed", deps: [] }], +]); + +/** One catalog row: the serialized view of a fact's kind/policy/dependencies. */ +export interface FactCatalogEntry { + readonly kind: string; + readonly failure_policy: FailurePolicy; + readonly dependencies: string[]; +} + +/** + * `FACT_META` projected into the exact shape of the Rust-generated + * `fact-catalog.gen.json` (array, in insertion/declaration order). The + * `gate-spec.test.ts` drift test deep-compares this against that committed + * catalog so any Rust-side change to a fact's policy/dependencies — or a + * new/removed `Fact` — fails a cheap unit test instead of silently producing + * wrong specs at runtime. + */ +export function factMetaCatalog(): FactCatalogEntry[] { + return [...FACT_META].map(([kind, meta]) => ({ + kind, + failure_policy: meta.policy, + dependencies: [...meta.deps], + })); +} + +/** + * Facts referenced directly by a predicate tree (mirror of predicates.ts + * collectFacts). Switches on the `type` discriminant so each variant's `fact` + * field is accessed type-safely: if a future `PredicateSpec` variant renames or + * drops `fact`, this fails to compile rather than silently returning + * `undefined` and under-specifying the fact list. The `never` default makes a + * newly-added variant a compile error too. + */ +function predicateFacts(p: PredicateSpec, out: Set = new Set()): Set { + switch (p.type) { + case "glob_match": + case "equals": + case "value_in_set": + case "value_not_in_set": + case "numeric_range": + case "label_set_match": + case "file_glob_match": + out.add(p.fact); + break; + case "time_window": + out.add("current_utc_minutes"); + break; + case "and": + case "or": + for (const sub of p.operands) predicateFacts(sub, out); + break; + case "not": + predicateFacts(p.operand, out); + break; + default: { + const _exhaustive: never = p; + throw new Error(`gate-spec: unhandled predicate type '${(_exhaustive as { type?: string }).type}'`); + } + } + return out; +} + +/** Add a fact and all its transitive dependencies to `out`. */ +function collectFactWithDeps(kind: string, out: Set): void { + if (out.has(kind)) return; + out.add(kind); + const meta = FACT_META.get(kind); + if (!meta) throw new Error(`gate-spec: unknown fact kind '${kind}'`); + for (const dep of meta.deps) collectFactWithDeps(dep, out); +} + +// ─── Check + context ───────────────────────────────────────────────────────── + +/** A filter check paired with the build tag emitted on failure. */ +export interface Check { + readonly name: string; + readonly tagSuffix: string; + readonly predicate: PredicateSpec; +} + +export type GateContextKind = "pull-request" | "pipeline-completion"; + +function contextSpec(kind: GateContextKind): GateContextSpec { + return kind === "pull-request" + ? { + build_reason: "PullRequest", + tag_prefix: "pr-gate", + step_name: "prGate", + bypass_label: "PR", + } + : { + build_reason: "ResourceTrigger", + tag_prefix: "pipeline-gate", + step_name: "pipelineGate", + bypass_label: "pipeline", + }; +} + +/** + * Assemble a `GateSpec` from a context and a set of checks, deriving the fact + * list (with transitive deps, in canonical order) exactly as `build_gate_spec` + * does in Rust. + */ +export function buildGateSpec(context: GateContextKind, checks: Check[]): GateSpec { + const required = new Set(); + for (const c of checks) { + for (const f of predicateFacts(c.predicate)) collectFactWithDeps(f, required); + } + + const facts: FactSpec[] = []; + for (const [kind, meta] of FACT_META) { + if (!required.has(kind)) continue; + facts.push({ kind, failure_policy: meta.policy, dependencies: [...meta.deps] }); + } + + const checkSpecs: CheckSpec[] = checks.map((c) => ({ + name: c.name, + predicate: c.predicate, + tag_suffix: c.tagSuffix, + })); + + return { context: contextSpec(context), facts, checks: checkSpecs }; +} + +/** Base64-encode a spec for the `GATE_SPEC` env / template parameter. */ +export function encodeGateSpec(spec: GateSpec): string { + return Buffer.from(JSON.stringify(spec), "utf8").toString("base64"); +} + +// ─── Check builders (mirror of lower_pr_filters) ───────────────────────────── + +export function titleCheck(pattern: string): Check { + return { + name: "title", + tagSuffix: "title-mismatch", + predicate: { type: "glob_match", fact: "pr_title", pattern }, + }; +} + +export function sourceBranchCheck(pattern: string): Check { + return { + name: "source-branch", + tagSuffix: "source-branch-mismatch", + predicate: { type: "glob_match", fact: "source_branch", pattern }, + }; +} + +export function targetBranchCheck(pattern: string): Check { + return { + name: "target-branch", + tagSuffix: "target-branch-mismatch", + predicate: { type: "glob_match", fact: "target_branch", pattern }, + }; +} + +export function commitMessageCheck(pattern: string): Check { + return { + name: "commit-message", + tagSuffix: "commit-message-mismatch", + predicate: { type: "glob_match", fact: "commit_message", pattern }, + }; +} + +export function authorIncludeCheck(values: string[]): Check { + return { + name: "author include", + tagSuffix: "author-mismatch", + predicate: { type: "value_in_set", fact: "author_email", values, case_insensitive: true }, + }; +} + +export function authorExcludeCheck(values: string[]): Check { + return { + name: "author exclude", + tagSuffix: "author-excluded", + predicate: { type: "value_not_in_set", fact: "author_email", values, case_insensitive: true }, + }; +} + +export function labelsCheck(opts: { + anyOf?: string[]; + allOf?: string[]; + noneOf?: string[]; +}): Check { + return { + name: "labels", + tagSuffix: "labels-mismatch", + predicate: { + type: "label_set_match", + fact: "pr_labels", + any_of: opts.anyOf ?? [], + all_of: opts.allOf ?? [], + none_of: opts.noneOf ?? [], + }, + }; +} + +export function draftCheck(expected: boolean): Check { + return { + name: "draft", + tagSuffix: "draft-mismatch", + predicate: { type: "equals", fact: "pr_is_draft", value: expected ? "true" : "false" }, + }; +} + +export function changedFilesCheck(opts: { include?: string[]; exclude?: string[] }): Check { + return { + name: "changed-files", + tagSuffix: "changed-files-mismatch", + predicate: { + type: "file_glob_match", + fact: "changed_files", + include: opts.include ?? [], + exclude: opts.exclude ?? [], + }, + }; +} + +export function timeWindowCheck(start: string, end: string): Check { + return { + name: "time-window", + tagSuffix: "time-window-mismatch", + predicate: { type: "time_window", start, end }, + }; +} + +export function changeCountCheck(opts: { min?: number; max?: number }): Check { + return { + name: "change-count", + tagSuffix: "changes-mismatch", + predicate: { + type: "numeric_range", + fact: "changed_file_count", + min: opts.min ?? null, + max: opts.max ?? null, + }, + }; +} + +export function buildReasonIncludeCheck(values: string[]): Check { + return { + name: "build-reason include", + tagSuffix: "build-reason-mismatch", + predicate: { type: "value_in_set", fact: "build_reason", values, case_insensitive: true }, + }; +} + +export function buildReasonExcludeCheck(values: string[]): Check { + return { + name: "build-reason exclude", + tagSuffix: "build-reason-excluded", + predicate: { type: "value_not_in_set", fact: "build_reason", values, case_insensitive: true }, + }; +} + +// ─── PR_SYNTH_SPEC builder (mirror of build_pr_synth_spec) ──────────────────── + +export interface PrSynthSpecInput { + branches?: { include?: string[]; exclude?: string[] }; + paths?: { include?: string[]; exclude?: string[] }; +} + +/** Base64-encode a `PR_SYNTH_SPEC` for the synthetic-PR promotion path. */ +export function encodePrSynthSpec(input: PrSynthSpecInput = {}): string { + const spec = { + branches: { + include: input.branches?.include ?? [], + exclude: input.branches?.exclude ?? [], + }, + paths: { + include: input.paths?.include ?? [], + exclude: input.paths?.exclude ?? [], + }, + }; + return Buffer.from(JSON.stringify(spec), "utf8").toString("base64"); +} diff --git a/scripts/ado-script/src/trigger-e2e/github-issue.ts b/scripts/ado-script/src/trigger-e2e/github-issue.ts new file mode 100644 index 00000000..edac0f97 --- /dev/null +++ b/scripts/ado-script/src/trigger-e2e/github-issue.ts @@ -0,0 +1,150 @@ +/** + * Failure-issue filing for the trigger-condition E2E harness. + * + * Reuses the low-level GitHub client primitives from the executor-e2e harness + * (`findOpenIssueByTitle` / `createGitHubIssue` / `diagnoseGitHubAuthFailure`) + * but frames issues with a trigger-specific title prefix + labels so the two + * suites never dedupe into each other's issues. + * + * Test-harness module; not shipped in `ado-script.zip`. + */ +import { + createGitHubIssue, + diagnoseGitHubAuthFailure, + findOpenIssueByTitle, +} from "../executor-e2e/github-issue.js"; +import type { ScenarioResult } from "./scenario.js"; + +export const ISSUE_TITLE_PREFIX = "[trigger-e2e-failure] "; +const DEFAULT_REPO = "githubnext/ado-aw"; +const DEFAULT_LABELS = ["trigger-e2e-failure", "pipeline-failure"]; +const MAX_TITLE_LEN = 200; + +export interface IssueEnv { + token?: string; + repo: string; + labels: string[]; + buildId?: string; + buildUrl?: string; + project?: string; +} + +/** Treat an UNEXPANDED ADO macro (`$(VAR)`) as absent. */ +function cleanVar(raw: string | undefined): string | undefined { + const value = raw?.trim(); + if (!value || /^\$\(.*\)$/.test(value)) return undefined; + return value; +} + +export function loadIssueEnv(env: NodeJS.ProcessEnv = process.env): IssueEnv { + const labelsRaw = env.TRIGGER_E2E_ISSUE_LABELS?.trim(); + let labels = DEFAULT_LABELS; + if (labelsRaw) { + try { + const parsed: unknown = JSON.parse(labelsRaw); + if (Array.isArray(parsed)) labels = parsed.filter((v): v is string => typeof v === "string"); + } catch { + /* keep defaults */ + } + } + return { + token: env.TRIGGER_E2E_GITHUB_TOKEN?.trim() || env.ADO_AW_DEBUG_GITHUB_TOKEN?.trim(), + repo: cleanVar(env.TRIGGER_E2E_ISSUE_REPO) || DEFAULT_REPO, + labels, + buildId: env.BUILD_BUILDID?.trim(), + buildUrl: + env.TRIGGER_E2E_BUILD_URL?.trim() || + (env.SYSTEM_COLLECTIONURI && env.SYSTEM_TEAMPROJECT && env.BUILD_BUILDID + ? `${env.SYSTEM_COLLECTIONURI.replace(/\/+$/, "")}/${encodeURIComponent(env.SYSTEM_TEAMPROJECT)}/_build/results?buildId=${env.BUILD_BUILDID}` + : undefined), + project: env.SYSTEM_TEAMPROJECT?.trim(), + }; +} + +/** Stable title keyed on the sorted set of failing scenario ids (dedupes). */ +export function buildIssueTitle(failed: ScenarioResult[]): string { + const ids = [...new Set(failed.map((r) => r.id))].sort(); + const title = `${ISSUE_TITLE_PREFIX}${ids.join(", ")}`; + return title.length <= MAX_TITLE_LEN ? title : title.slice(0, MAX_TITLE_LEN); +} + +export function renderIssueBody(results: ScenarioResult[], env: IssueEnv): string { + const failed = results.filter((r) => !r.ok); + const skipped = results.filter((r) => r.skipped); + const passed = results.filter((r) => r.ok && !r.skipped); + + const lines: string[] = [ + "The deterministic trigger-condition (gate / synth-PR) E2E suite reported failures.", + "", + "## Failed scenarios", + "", + "| Scenario | Phase | Message |", + "| --- | --- | --- |", + ...failed.map( + (r) => + `| \`${r.id}\` | ${r.phase ?? "?"} | ${(r.message ?? "").replace(/\r?\n/g, " ").replace(/\|/g, "\\|").slice(0, 400)} |`, + ), + "", + "## Run", + `- Project: ${env.project ?? "unknown"}`, + `- Build ID: ${env.buildId ?? "unknown"}`, + `- Build URL: ${env.buildUrl ?? "unknown"}`, + `- Passed: ${passed.length} | Failed: ${failed.length} | Skipped: ${skipped.length}`, + ]; + if (skipped.length > 0) { + lines.push("", "## Skipped (missing precondition)", ""); + for (const s of skipped) lines.push(`- \`${s.id}\`: ${s.message ?? ""}`); + } + lines.push( + "", + "> Filed automatically by the trigger-e2e pipeline. Re-runs with the same", + "> failing-scenario signature update this issue rather than opening a new one.", + ); + return lines.join("\n"); +} + +export interface FileIssueOutcome { + filed: boolean; + reason?: string; + url?: string; +} + +/** Extract a trailing "HTTP " code from a thrown GitHub client error. */ +function statusFromError(err: unknown): number | undefined { + const message = err instanceof Error ? err.message : String(err); + const match = message.match(/HTTP (\d{3})/); + return match ? Number(match[1]) : undefined; +} + +/** File (or dedupe) a failure issue. No-op with no failures or no token. */ +export async function fileFailureIssue( + results: ScenarioResult[], + env: IssueEnv, + log: (msg: string) => void, + fetchImpl?: typeof fetch, +): Promise { + const failed = results.filter((r) => !r.ok); + if (failed.length === 0) return { filed: false, reason: "no failures" }; + if (!env.token) { + log("no GitHub token configured (TRIGGER_E2E_GITHUB_TOKEN); skipping issue filing"); + return { filed: false, reason: "no token" }; + } + + const opts = { token: env.token, repo: env.repo, fetchImpl }; + log(`filing failure issue to '${env.repo}' (${failed.length} failed scenario(s))`); + const title = buildIssueTitle(failed); + try { + const existing = await findOpenIssueByTitle(opts, title); + if (existing !== undefined) { + log(`open issue #${existing} already tracks this failure signature; skipping`); + return { filed: false, reason: `deduped to #${existing}` }; + } + const url = await createGitHubIssue(opts, title, renderIssueBody(results, env), env.labels); + log(`filed GitHub issue: ${url}`); + return { filed: true, url }; + } catch (err) { + const status = statusFromError(err); + if (status !== undefined) await diagnoseGitHubAuthFailure(opts, status, log); + throw err; + } +} diff --git a/scripts/ado-script/src/trigger-e2e/index.ts b/scripts/ado-script/src/trigger-e2e/index.ts new file mode 100644 index 00000000..6157bea8 --- /dev/null +++ b/scripts/ado-script/src/trigger-e2e/index.ts @@ -0,0 +1,120 @@ +/** + * Entry point for the deterministic trigger-condition (gate / synth-PR) E2E + * harness. + * + * Reads its configuration from the environment (ADO org/project/token, the + * registered victim pipeline definition id, target repo), queues the victim + * pipeline under a battery of real trigger conditions, asserts the observable + * gate decision (build tags + result), files a GitHub issue on failure, and + * exits non-zero when any scenario failed so the pipeline goes red. + * + * Required env: + * - SYSTEM_COLLECTIONURI (or AZURE_DEVOPS_ORG_URL) — ADO collection URI + * - SYSTEM_TEAMPROJECT — ADO project + * - SYSTEM_ACCESSTOKEN — write-capable ADO token (queue builds, cancel, PRs) + * - TRIGGER_E2E_VICTIM_DEFINITION_ID — registered victim pipeline id + * Optional env: + * - TRIGGER_E2E_VICTIM_REPO — ADO Git repo backing the victim's `self` + * (where PRs are created). When unset, PR/synth/gate scenarios SKIP and + * only the bypass baseline runs. + * - TRIGGER_E2E_GITHUB_TOKEN — scoped PAT for issue filing + * - TRIGGER_E2E_ISSUE_REPO — GitHub repo for issues (default githubnext/ado-aw) + * - TRIGGER_E2E_BUILD_TIMEOUT_MS / TRIGGER_E2E_BUILD_POLL_MS — poll tuning + * + * Test-harness module; not shipped in `ado-script.zip`. + */ +import { AdoRest } from "../executor-e2e/ado-rest.js"; +import { fileFailureIssue, loadIssueEnv } from "./github-issue.js"; +import { runAll } from "./runner.js"; +import { allScenarios } from "./scenarios/index.js"; +import type { ScenarioResult, TriggerContext } from "./scenario.js"; + +function requireEnv(name: string, alt?: string): string { + const value = process.env[name]?.trim() || (alt ? process.env[alt]?.trim() : undefined); + if (!value) throw new Error(`required env var ${name}${alt ? ` (or ${alt})` : ""} is not set`); + return value; +} + +function log(msg: string): void { + // Percent-encode a leading '#' so a message cannot smuggle a ##vso command. + process.stdout.write(msg.replace(/^#/gm, "%23") + "\n"); +} + +export function summarise(results: ScenarioResult[]): string { + const passed = results.filter((r) => r.ok && !r.skipped).length; + const failed = results.filter((r) => !r.ok).length; + const skipped = results.filter((r) => r.skipped).length; + const lines = [ + "", + "=== Trigger E2E summary ===", + ...results.map((r) => { + const state = r.skipped ? "SKIP" : r.ok ? "PASS" : "FAIL"; + const suffix = r.ok && !r.skipped ? "" : ` (${r.phase}: ${r.message ?? ""})`; + return ` [${state}] ${r.id}${suffix}`; + }), + `Total: ${results.length} | Passed: ${passed} | Failed: ${failed} | Skipped: ${skipped}`, + ]; + return lines.join("\n"); +} + +export async function main(): Promise { + const orgUrl = requireEnv("SYSTEM_COLLECTIONURI", "AZURE_DEVOPS_ORG_URL"); + const project = requireEnv("SYSTEM_TEAMPROJECT"); + const token = requireEnv("SYSTEM_ACCESSTOKEN"); + const victimDefinitionIdRaw = requireEnv("TRIGGER_E2E_VICTIM_DEFINITION_ID"); + const victimDefinitionId = Number(victimDefinitionIdRaw); + if (!Number.isInteger(victimDefinitionId) || victimDefinitionId <= 0) { + throw new Error( + `TRIGGER_E2E_VICTIM_DEFINITION_ID must be a positive integer (got '${victimDefinitionIdRaw}')`, + ); + } + // Optional: the ADO Git repo backing the victim's `self`. Empty → PR/synth/ + // gate scenarios skip (see requirePrRepo). + const adoRepo = process.env.TRIGGER_E2E_VICTIM_REPO?.trim() || ""; + const buildId = process.env.BUILD_BUILDID?.trim() || `local-${Date.now()}`; + + const rest = new AdoRest({ orgUrl, project, token, log }); + + const ctx: TriggerContext = { + orgUrl, + project, + adoRepo, + buildId, + token, + victimDefinitionId, + rest, + log, + prefix: (id) => `ado-aw-trig-${buildId}-${id}`, + }; + + log( + `Running ${allScenarios.length} trigger E2E scenarios against ${orgUrl}${project} ` + + `(victim def #${victimDefinitionId}${adoRepo ? `, repo ${adoRepo}` : ", no PR repo — PR scenarios will skip"})`, + ); + + const results = await runAll(ctx, allScenarios); + log(summarise(results)); + + const issueEnv = loadIssueEnv(); + try { + await fileFailureIssue(results, issueEnv, log); + } catch (err) { + log(`WARNING: failed to file GitHub issue: ${(err as Error).message}`); + } + + const failed = results.filter((r) => !r.ok).length; + return failed > 0 ? 1 : 0; +} + +// Run as the bundle entry point. Skipped under Vitest so unit tests can import +// these modules without launching the whole suite. +if (process.env.VITEST !== "true") { + main().then( + (code) => process.exit(code), + (err: unknown) => { + const e = err as Error; + log(`trigger-e2e crashed: ${e.stack ?? e.message}`); + process.exit(1); + }, + ); +} diff --git a/scripts/ado-script/src/trigger-e2e/queue.ts b/scripts/ado-script/src/trigger-e2e/queue.ts new file mode 100644 index 00000000..b0b7f491 --- /dev/null +++ b/scripts/ado-script/src/trigger-e2e/queue.ts @@ -0,0 +1,58 @@ +/** + * Queue one victim-pipeline build and collect its terminal outcome. + * + * Thin orchestration over the shared {@link AdoRest} client: queue the victim + * definition with the scenario's source branch + template parameters, poll to + * completion, and read the final build tags. The returned {@link BuildOutcome} + * is what the runner asserts against. + * + * Test-harness module; not shipped in `ado-script.zip`. + */ +import type { BuildOutcome, TriggerContext, VictimQueue } from "./scenario.js"; + +/** Drop `undefined` values so ADO receives a flat string→string parameter map. */ +function flattenParams(params: Record): Record { + const out: Record = {}; + for (const [k, v] of Object.entries(params)) { + if (typeof v === "string") out[k] = v; + } + return out; +} + +/** + * Resolve the victim-build poll tuning from trigger-e2e env vars, falling back + * to `waitForBuild`'s generic defaults when unset. These knobs live HERE (in + * trigger-e2e code) rather than in the shared `AdoRest` client so an + * executor-e2e run can never be silently retimed by a `TRIGGER_E2E_*` variable + * that happens to be set in its shell. + */ +function pollOptions(): { timeoutMs?: number; pollMs?: number } { + const timeoutMs = Number(process.env.TRIGGER_E2E_BUILD_TIMEOUT_MS) || undefined; + const pollMs = Number(process.env.TRIGGER_E2E_BUILD_POLL_MS) || undefined; + return { timeoutMs, pollMs }; +} + +export async function runVictim( + ctx: TriggerContext, + queue: VictimQueue, + onQueued?: (buildId: number) => void, +): Promise { + const templateParameters = flattenParams(queue.templateParameters); + const queued = await ctx.rest.queueBuild(ctx.victimDefinitionId, { + sourceBranch: queue.sourceBranch, + templateParameters, + }); + // Report the id as soon as the build is queued so the caller can cancel it on + // cleanup even if the subsequent `waitForBuild` poll throws (e.g. a timeout) + // and never returns a BuildOutcome. + onQueued?.(queued.id); + ctx.log(` queued victim build #${queued.id}${queue.sourceBranch ? ` on ${queue.sourceBranch}` : ""}`); + + const terminal = await ctx.rest.waitForBuild(queued.id, pollOptions()); + const tags = await ctx.rest.getBuildTags(queued.id); + ctx.log( + ` victim build #${queued.id} completed: result=${terminal.result ?? "?"} tags=[${tags.join(", ")}]`, + ); + + return { status: terminal.status, result: terminal.result, tags, buildId: queued.id }; +} diff --git a/scripts/ado-script/src/trigger-e2e/runner.ts b/scripts/ado-script/src/trigger-e2e/runner.ts new file mode 100644 index 00000000..47806084 --- /dev/null +++ b/scripts/ado-script/src/trigger-e2e/runner.ts @@ -0,0 +1,136 @@ +/** + * Scenario runner for the trigger-condition E2E harness. + * + * Runs scenarios sequentially (deterministic ordering; queuing many victim + * builds in parallel would contend for the agent pool). Each scenario is fully + * isolated: a failure or skip never prevents later scenarios from running, and + * cleanup always runs once `setup` succeeded. + * + * Test-harness module; not shipped in `ado-script.zip`. + */ +import { runVictim } from "./queue.js"; +import { SkipError } from "./scenario.js"; +import type { + BuildOutcome, + Expected, + ScenarioResult, + TriggerContext, + TriggerScenario, +} from "./scenario.js"; + +function errMessage(err: unknown): string { + return err instanceof Error ? err.message : String(err); +} + +/** Default declarative assertion: build result + required/forbidden tags. */ +function assertExpected(expected: Expected, outcome: BuildOutcome): void { + const wantResult = expected.result ?? "succeeded"; + if (outcome.result !== wantResult) { + throw new Error( + `expected build result '${wantResult}' but got '${outcome.result ?? "?"}' (tags: [${outcome.tags.join(", ")}])`, + ); + } + for (const tag of expected.tags ?? []) { + if (!outcome.tags.includes(tag)) { + throw new Error(`expected build tag '${tag}' but tags were [${outcome.tags.join(", ")}]`); + } + } + for (const tag of expected.absentTags ?? []) { + if (outcome.tags.includes(tag)) { + throw new Error(`build tag '${tag}' should be absent but tags were [${outcome.tags.join(", ")}]`); + } + } +} + +export async function runScenario( + ctx: TriggerContext, + scenario: TriggerScenario, +): Promise { + const start = Date.now(); + const id = scenario.id; + let state: S | undefined; + let setupDone = false; + // Captured as soon as the victim build is queued, so a queue-phase throw + // (e.g. a waitForBuild timeout) still lets the finally block cancel the + // orphaned build rather than leaking a running build for the whole suite. + let queuedBuildId: number | undefined; + + const finish = (partial: Omit): ScenarioResult => ({ + id, + durationMs: Date.now() - start, + ...partial, + }); + + try { + // ---- setup ---- + ctx.log(`[${id}] setup — ${scenario.description}`); + try { + state = await scenario.setup(ctx); + setupDone = true; + } catch (err) { + if (err instanceof SkipError) { + ctx.log(`[${id}] SKIPPED: ${err.message}`); + return finish({ ok: true, skipped: true, phase: "skipped", message: err.message }); + } + return finish({ ok: false, phase: "setup", message: errMessage(err) }); + } + + // ---- queue + poll ---- + let outcome: BuildOutcome; + try { + const queue = scenario.queue(ctx, state); + outcome = await runVictim(ctx, queue, (buildId) => { + queuedBuildId = buildId; + }); + } catch (err) { + return finish({ ok: false, phase: "queue", message: errMessage(err) }); + } + + // ---- assert ---- + try { + assertExpected(scenario.expected(ctx, state), outcome); + if (scenario.assert) await scenario.assert(ctx, state, outcome); + } catch (err) { + return finish({ ok: false, phase: "assert", message: errMessage(err) }); + } + + ctx.log(`[${id}] OK`); + return finish({ ok: true }); + } finally { + // ---- cancel an orphaned victim build (best-effort) ---- + // A completed build is a no-op; only a build still running after a + // queue-phase failure needs cancelling. + if (queuedBuildId !== undefined) { + try { + const build = await ctx.rest.getBuild(queuedBuildId); + if (build.status !== "completed") { + await ctx.rest.cancelBuild(queuedBuildId); + ctx.log(`[${id}] cancelled orphaned victim build #${queuedBuildId}`); + } + } catch (err) { + ctx.log(`[${id}] orphaned-build cleanup WARNING: ${errMessage(err)}`); + } + } + + // ---- cleanup (always, best-effort) ---- + if (setupDone) { + try { + await scenario.cleanup(ctx, state as S); + ctx.log(`[${id}] cleanup done`); + } catch (err) { + ctx.log(`[${id}] cleanup WARNING: ${errMessage(err)}`); + } + } + } +} + +export async function runAll( + ctx: TriggerContext, + scenarios: TriggerScenario[], +): Promise { + const results: ScenarioResult[] = []; + for (const scenario of scenarios) { + results.push(await runScenario(ctx, scenario)); + } + return results; +} diff --git a/scripts/ado-script/src/trigger-e2e/scenario.ts b/scripts/ado-script/src/trigger-e2e/scenario.ts new file mode 100644 index 00000000..c814139d --- /dev/null +++ b/scripts/ado-script/src/trigger-e2e/scenario.ts @@ -0,0 +1,156 @@ +/** + * Deterministic trigger-condition (gate / synth-PR) E2E scenario contract. + * + * Unlike the executor E2E harness (which runs `ado-aw execute` directly), each + * trigger scenario exercises the *runtime trigger conditions managed by + * ado-script* — the `gate.js` and `exec-context-pr-synth.js` bundles — by + * queuing a real build of a hand-authored, parameterized **victim pipeline** + * and asserting the observable gate decision via **build tags + build result**. + * + * A scenario optionally sets up real ADO PR context (branch + PR + labels + + * draft state), queues the victim with a per-scenario `GATE_SPEC` / + * `PR_SYNTH_SPEC` (and pipeline-var fact overrides) as template parameters, + * polls the build to completion, then asserts the tags/result and cleans up. + * + * ## Victim observable contract (REST-visible build tags) + * - `trig.synth.promoted` | `trig.synth.skipped` | `trig.synth.real-pr` + * — emitted by the victim after `exec-context-pr-synth.js`. + * - `pr-gate.passed` — gate bypassed (not a PR/synth build). + * - `pr-gate.skipped` + `pr-gate.` — a filter failed; + * the gate self-cancels the build (`result == "canceled"`). + * - `trig.should-run.true` — emitted by a post-gate step that only runs + * when the gate did NOT self-cancel (i.e. `SHOULD_RUN == true`). + * + * Test-harness module; not shipped in `ado-script.zip`. + */ +import type { AdoRest } from "../executor-e2e/ado-rest.js"; + +/** Shared, read-only context handed to every scenario phase. */ +export interface TriggerContext { + /** ADO collection URI, e.g. https://dev.azure.com/msazuresphere/ */ + readonly orgUrl: string; + /** ADO project name, e.g. AgentPlayground. */ + readonly project: string; + /** ADO Git repo used to create real PRs (initialised `main`). */ + readonly adoRepo: string; + /** Current orchestrator build id (drives the per-run object prefix). */ + readonly buildId: string; + /** Write-capable ADO token (System.AccessToken or a PAT). */ + readonly token: string; + /** Registered definition id of the hand-authored victim pipeline. */ + readonly victimDefinitionId: number; + /** REST helper bound to {orgUrl, project, token}. */ + readonly rest: AdoRest; + /** Structured logger (writes to the pipeline log). */ + readonly log: (msg: string) => void; + /** Deterministic object-name prefix: `ado-aw-trig--`. */ + readonly prefix: (id: string) => string; +} + +/** Template parameters + source branch used to queue one victim build. */ +export interface VictimQueue { + /** Source branch the victim builds (points synth at the PR); optional. */ + sourceBranch?: string; + /** Runtime parameters passed to the victim pipeline. */ + templateParameters: VictimParameters; +} + +/** + * Victim pipeline runtime parameters. `gateSpec` is required; the rest are + * optional overrides. Empty overrides fall back to the victim's real ADO + * predefined variables. + */ +export interface VictimParameters { + /** base64 GATE_SPEC (required). */ + gateSpec: string; + /** base64 PR_SYNTH_SPEC (default: empty include/exclude). */ + prSynthSpec?: string; + /** Override for the `pr_title` fact (ADO_PR_TITLE). */ + prTitle?: string; + /** Override for the `author_email` fact (ADO_AUTHOR_EMAIL). */ + authorEmail?: string; + /** Override for the `source_branch` fact (ADO_SOURCE_BRANCH). */ + sourceBranchFact?: string; + /** Override for the `target_branch` fact (ADO_TARGET_BRANCH). */ + targetBranchFact?: string; + /** Override for the `commit_message` fact (ADO_COMMIT_MESSAGE). */ + commitMessage?: string; + /** [key: string]: string — ADO template parameters must be flat strings. */ + [k: string]: string | undefined; +} + +/** Terminal build state + tags read after the victim run completes. */ +export interface BuildOutcome { + /** ADO build status — always `completed` once polling returns. */ + status: string; + /** succeeded | partiallySucceeded | failed | canceled. */ + result?: string; + /** All build tags present at completion. */ + tags: string[]; + /** The queued victim build id (for logging / debugging). */ + buildId: number; +} + +/** Declarative expectation, checked by the runner's default assertion. */ +export interface Expected { + /** Required build result; default `succeeded`. */ + result?: string; + /** Tags that MUST all be present. */ + tags?: string[]; + /** Tags that MUST NOT be present. */ + absentTags?: string[]; +} + +/** + * A single deterministic trigger scenario. + * + * `State` is threaded from `setup` through `queue`/`assert`/`cleanup` so a + * scenario can remember the PR id / branch it created. + */ +export interface TriggerScenario { + /** Unique harness id for reporting and object naming. */ + readonly id: string; + /** One-line human description of what this scenario proves. */ + readonly description: string; + /** + * Create real ADO preconditions (branch/PR/labels); return remembered state. + * + * When this throws (a non-`SkipError`), the runner will NOT call `cleanup()`. + * Any ADO objects partially created before the throw must be torn down + * explicitly inside this function before rethrowing (see `scenarios/common.ts` + * `createPrContext`, which follows the executor-e2e pattern). + */ + setup(ctx: TriggerContext): Promise; + /** Build the victim queue request (source branch + template parameters). */ + queue(ctx: TriggerContext, state: State): VictimQueue; + /** Declarative expectation for the completed victim build. */ + expected(ctx: TriggerContext, state: State): Expected; + /** Optional extra assertion beyond `expected` (throw on failure). */ + assert?(ctx: TriggerContext, state: State, outcome: BuildOutcome): Promise; + /** Best-effort teardown of everything `setup` created. */ + cleanup(ctx: TriggerContext, state: State): Promise; +} + +/** Outcome of running one scenario. */ +export interface ScenarioResult { + id: string; + ok: boolean; + /** "setup" | "queue" | "assert" | "cleanup" | "skipped". */ + phase?: string; + message?: string; + durationMs: number; + /** True when skipped for a missing precondition (not a failure). */ + skipped?: boolean; +} + +/** + * Thrown by a scenario's `setup` when a required precondition is unavailable + * (e.g. the victim definition id was not supplied). The runner records the + * scenario as **skipped** rather than failed. + */ +export class SkipError extends Error { + constructor(reason: string) { + super(reason); + this.name = "SkipError"; + } +} diff --git a/scripts/ado-script/src/trigger-e2e/scenarios/bypass.ts b/scripts/ado-script/src/trigger-e2e/scenarios/bypass.ts new file mode 100644 index 00000000..09a443dc --- /dev/null +++ b/scripts/ado-script/src/trigger-e2e/scenarios/bypass.ts @@ -0,0 +1,49 @@ +/** + * Bypass scenario (`gate/bypass.ts`). + * + * When a build is neither a real PR nor a synth-promoted CI build, the PR gate + * auto-passes ("not a PR build — gate passes automatically") and tags the build + * `pr-gate.passed`. This scenario queues the victim on its default branch with + * NO open PR, so `exec-context-pr-synth` skips and the gate bypasses. + * + * Needs no PR context, so it runs even when `TRIGGER_E2E_VICTIM_REPO` is unset + * — a useful baseline that the victim pipeline is wired correctly. + * + * Test-harness module; not shipped in `ado-script.zip`. + */ +import { buildGateSpec, encodeGateSpec, encodePrSynthSpec, labelsCheck } from "../gate-spec.js"; +import type { TriggerScenario } from "../scenario.js"; + +const bypassManual: TriggerScenario = { + id: "bypass-no-pr", + description: "no PR + Manual build → PR gate bypasses (pr-gate.passed)", + async setup() { + return undefined; + }, + queue() { + // A non-trivial gate spec proves the bypass short-circuits BEFORE any + // filter evaluation (a label check would otherwise fail with no PR). + return { + // No sourceBranch → the victim builds its default branch; no PR exists + // for it, so synth skips and the gate bypasses. + templateParameters: { + gateSpec: encodeGateSpec( + buildGateSpec("pull-request", [labelsCheck({ anyOf: ["run-agent"] })]), + ), + prSynthSpec: encodePrSynthSpec(), + }, + }; + }, + expected() { + return { + result: "succeeded", + tags: ["pr-gate.passed", "trig.synth.skipped"], + absentTags: ["pr-gate.skipped", "trig.synth.promoted"], + }; + }, + async cleanup() { + // Nothing created. + }, +}; + +export const bypassScenarios: TriggerScenario[] = [bypassManual]; diff --git a/scripts/ado-script/src/trigger-e2e/scenarios/common.ts b/scripts/ado-script/src/trigger-e2e/scenarios/common.ts new file mode 100644 index 00000000..3b2f7867 --- /dev/null +++ b/scripts/ado-script/src/trigger-e2e/scenarios/common.ts @@ -0,0 +1,160 @@ +/** + * Shared helpers for trigger E2E scenarios: creating and tearing down real ADO + * PR context. + * + * Reuses the executor-e2e `AdoRest` client and the ctx-free {@link Teardown} + * helper so the create-branch → push-file → open-PR → set-labels pattern (and + * its guaranteed-cleanup teardown) lives in one place, mirroring + * `executor-e2e/scenarios/pr.ts`. + * + * Test-harness module; not shipped in `ado-script.zip`. + */ +import { Teardown } from "../../executor-e2e/scenarios/common.js"; +import { encodePrSynthSpec } from "../gate-spec.js"; +import { SkipError } from "../scenario.js"; +import type { TriggerContext } from "../scenario.js"; + +/** Remembered state for a created PR (ids needed for assertion + cleanup). */ +export interface PrContext { + readonly repo: string; + readonly prId: number; + /** Short branch name (no `refs/heads/`). */ + readonly branch: string; + /** Full source ref (`refs/heads/`). */ + readonly sourceRef: string; + /** Short target branch name (repo default, e.g. `main`). */ + readonly targetBranch: string; +} + +/** Resolve a repo's default branch short name (e.g. `main`). */ +async function defaultBranchShortName(ctx: TriggerContext, repo: string): Promise { + const info = await ctx.rest.getRepository(repo); + return (info.defaultBranch ?? "refs/heads/main").replace(/^refs\/heads\//, ""); +} + +export interface CreatePrOptions { + /** Scenario id, used for the deterministic branch/PR name prefix. */ + readonly id: string; + /** Extra files to add on the source branch (path → contents). */ + readonly files?: Record; + /** PR labels to attach after creation. */ + readonly labels?: string[]; + /** PR title (default: ` (do not merge)`). */ + readonly title?: string; + /** Open the PR in draft state (for the gate `draft` predicate). */ + readonly draft?: boolean; +} + +/** + * Create a real transient PR against the repo's default branch: pushes a new + * source branch carrying one or more files (so ADO accepts a non-empty diff), + * opens the PR, and attaches any labels. + * + * Follows the executor-e2e contract: because a `setup()` throw means the runner + * will NOT call cleanup, this tears down anything it created before rethrowing. + * + * IMPORTANT for multi-step setup: the runner only calls `cleanup()` once + * `setup()` returns (it gates on `setupDone`). If a scenario's `setup()` calls + * `createPrContext` successfully and then does *further* work that throws, the + * returned `PrContext` is leaked — cleanup never runs. A scenario that needs + * post-PR setup steps must wrap them in its own try/catch that calls + * `teardownPrContext` before rethrowing (mirroring executor-e2e's `setupPr`). + * All current scenarios' `setup()` is exactly `createPrContext`, so none are + * exposed today. + */ +export async function createPrContext( + ctx: TriggerContext, + opts: CreatePrOptions, +): Promise { + const repo = ctx.adoRepo; + const targetBranch = await defaultBranchShortName(ctx, repo); + const baseSha = await ctx.rest.getRefObjectId(repo, `heads/${targetBranch}`); + if (!baseSha) throw new Error(`could not resolve ${targetBranch} HEAD in repo '${repo}'`); + + const branch = `${ctx.prefix(opts.id)}-src`; + const sourceRef = `refs/heads/${branch}`; + + // Default file guarantees a real diff even when the scenario needs none. + // Treat an explicitly-empty `files: {}` the same as omitted so a caller can + // never produce a PR with no diff (which ADO would reject). + const files = + opts.files && Object.keys(opts.files).length > 0 + ? opts.files + : { + [`/ado-aw-trig/${ctx.buildId}/${opts.id}.md`]: `trigger e2e ${opts.id} for build ${ctx.buildId}. Safe to delete.\n`, + }; + + // Create the source branch + ALL files in a single push. Batching avoids the + // ref-conflict a per-file loop would hit: pushAddFileBranch always uses + // new-branch semantics, so a second push to the now-existing branch fails. + await ctx.rest.pushAddFilesBranch(repo, branch, baseSha, files, `trigger e2e ${opts.id}`); + + // From here the source branch exists; a later throw must clean it up. + try { + const pr = await ctx.rest.createPullRequest( + repo, + branch, + targetBranch, + opts.title ?? `${ctx.prefix(opts.id)} (do not merge)`, + `Deterministic trigger E2E ${opts.id} for build ${ctx.buildId}. Safe to delete.`, + opts.draft, + ); + + if (opts.labels && opts.labels.length > 0) { + await ctx.rest.setPullRequestLabels(repo, pr.pullRequestId, opts.labels); + } + + return { repo, prId: pr.pullRequestId, branch, sourceRef, targetBranch }; + } catch (err) { + // Best-effort delete of the branch we pushed before the failure. + await ctx.rest.deleteRef(repo, sourceRef).catch(() => {}); + throw err; + } +} + +/** Abandon the PR and delete its source branch — every step always attempted. */ +export async function teardownPrContext(ctx: TriggerContext, pr: PrContext): Promise { + await new Teardown() + .add("abandon PR", () => ctx.rest.abandonPullRequest(pr.repo, pr.prId)) + .add("delete branch", () => ctx.rest.deleteRef(pr.repo, pr.sourceRef)) + .run(); +} + +/** + * Ensure a real ADO Git repo is available for PR creation. PR/synth/gate + * scenarios need the victim pipeline's own `self` repo (where `exec-context- + * pr-synth` looks up active PRs); when it is not supplied the scenario skips + * rather than fails. + */ +export function requirePrRepo(ctx: TriggerContext): string { + const repo = ctx.adoRepo?.trim(); + if (!repo) { + throw new SkipError( + "TRIGGER_E2E_VICTIM_REPO is not set; supply the ADO Git repo backing the victim pipeline to enable PR/synth/gate scenarios", + ); + } + return repo; +} + +/** + * base64 PR_SYNTH_SPEC that promotes a PR targeting `targetBranch`. + * + * MUST be derived from the PR's real target branch (`PrContext.targetBranch`, + * resolved from the repo's default branch), NOT a hardcoded `"main"`: if the + * ADO repo's default branch is anything else, a constant `"main"` include would + * fail to match the real target and every gate/synth scenario would silently + * evaluate the wrong path. + */ +export function promoteSynthSpec(targetBranch: string): string { + return encodePrSynthSpec({ branches: { include: [targetBranch] } }); +} + +/** + * base64 PR_SYNTH_SPEC that deliberately does NOT promote a PR targeting + * `targetBranch` (excludes exactly that branch, include-all otherwise), so the + * "branch mismatch → not promoted" path is exercised robustly regardless of the + * repo's actual default branch name. + */ +export function excludeSynthSpec(targetBranch: string): string { + return encodePrSynthSpec({ branches: { exclude: [targetBranch] } }); +} diff --git a/scripts/ado-script/src/trigger-e2e/scenarios/index.ts b/scripts/ado-script/src/trigger-e2e/scenarios/index.ts new file mode 100644 index 00000000..7aaedca3 --- /dev/null +++ b/scripts/ado-script/src/trigger-e2e/scenarios/index.ts @@ -0,0 +1,21 @@ +/** + * All trigger-condition scenarios, in run order. + * + * Test-harness module; not shipped in `ado-script.zip`. + */ +import type { TriggerScenario } from "../scenario.js"; +import { bypassScenarios } from "./bypass.js"; +import { filterScenarios } from "./pr-filters.js"; +import { selfCancelScenarios } from "./self-cancel.js"; +import { synthScenarios } from "./synth-pr.js"; + +export const allScenarios: TriggerScenario[] = [ + // Baseline that runs without a PR repo. + ...(bypassScenarios as TriggerScenario[]), + // Synthetic-PR promotion outcomes. + ...(synthScenarios as TriggerScenario[]), + // Gate filter pass/skip matrix. + ...(filterScenarios as TriggerScenario[]), + // Explicit self-cancel assertion. + ...(selfCancelScenarios as TriggerScenario[]), +]; diff --git a/scripts/ado-script/src/trigger-e2e/scenarios/pr-filters.ts b/scripts/ado-script/src/trigger-e2e/scenarios/pr-filters.ts new file mode 100644 index 00000000..aa4b63d2 --- /dev/null +++ b/scripts/ado-script/src/trigger-e2e/scenarios/pr-filters.ts @@ -0,0 +1,221 @@ +/** + * Gate filter scenarios (`gate.js`). + * + * Each creates a real PR, promotes it to synthetic-PR semantics (empty synth + * spec = match-all), and queues the victim with a single-filter `GATE_SPEC`. + * The gate then evaluates that filter against the real PR facts (labels / + * changed files / draft state / branch / build reason / change count / time) + * and either lets the run proceed or self-cancels the build. + * + * Assertion via build tags + result: + * - pass → `result=succeeded`, `trig.synth.promoted` + `trig.should-run.true`, + * no `pr-gate.skipped`. + * - skip → `result=canceled`, `trig.synth.promoted` + `pr-gate.skipped` + + * `pr-gate.`, no `trig.should-run.true`. + * + * Test-harness module; not shipped in `ado-script.zip`. + */ +import { + buildGateSpec, + buildReasonIncludeCheck, + changeCountCheck, + changedFilesCheck, + draftCheck, + encodeGateSpec, + labelsCheck, + targetBranchCheck, + timeWindowCheck, + type Check, +} from "../gate-spec.js"; +import type { Expected, TriggerScenario, VictimParameters } from "../scenario.js"; +import { + createPrContext, + promoteSynthSpec, + requirePrRepo, + teardownPrContext, + type PrContext, +} from "./common.js"; +import type { CreatePrOptions } from "./common.js"; + +interface FilterScenarioSpec { + readonly id: string; + readonly description: string; + /** PR preconditions to create (labels / files / draft / title). */ + readonly pr: Omit; + /** The gate check under test. */ + readonly check: Check | ((now: Date) => Check); + /** Whether the gate should let the run proceed (`pass`) or skip it. */ + readonly outcome: "pass" | "skip"; + /** For `skip`, the `pr-gate.` tag the failing check must emit. */ + readonly failTag?: string; + /** Extra victim parameters (pipeline-var fact overrides), if any. */ + readonly params?: Omit; +} + +function makeFilterScenario(spec: FilterScenarioSpec): TriggerScenario { + return { + id: spec.id, + description: spec.description, + async setup(ctx) { + requirePrRepo(ctx); + return createPrContext(ctx, { id: spec.id, ...spec.pr }); + }, + queue(_ctx, state) { + const check = typeof spec.check === "function" ? spec.check(new Date()) : spec.check; + const gateSpec = encodeGateSpec(buildGateSpec("pull-request", [check])); + return { + sourceBranch: state.sourceRef, + // Promote against the PR's REAL target branch (not a hardcoded "main") + // so the gate evaluates the filter under test on any default branch. + templateParameters: { + gateSpec, + prSynthSpec: promoteSynthSpec(state.targetBranch), + ...spec.params, + }, + }; + }, + expected(): Expected { + if (spec.outcome === "pass") { + return { + result: "succeeded", + tags: ["trig.synth.promoted", "trig.should-run.true"], + absentTags: ["pr-gate.skipped"], + }; + } + const tags = ["trig.synth.promoted", "pr-gate.skipped"]; + if (spec.failTag) tags.push(spec.failTag); + return { result: "canceled", tags, absentTags: ["trig.should-run.true"] }; + }, + async cleanup(ctx, state) { + await teardownPrContext(ctx, state); + }, + }; +} + +/** HH:MM string for `minutes` (mod 1440) since UTC midnight. */ +function hhmm(minutes: number): string { + const m = ((minutes % 1440) + 1440) % 1440; + const h = Math.floor(m / 60); + const mm = m % 60; + return `${String(h).padStart(2, "0")}:${String(mm).padStart(2, "0")}`; +} + +const specs: FilterScenarioSpec[] = [ + // ── labels ──────────────────────────────────────────────────────────── + { + id: "labels-pass", + description: "label_set_match any-of matches the PR's labels", + pr: { labels: ["run-agent"] }, + check: labelsCheck({ anyOf: ["run-agent"] }), + outcome: "pass", + }, + { + id: "labels-skip", + description: "label_set_match any-of misses the PR's labels → self-cancel", + pr: { labels: ["do-not-run"] }, + check: labelsCheck({ anyOf: ["run-agent"] }), + outcome: "skip", + failTag: "pr-gate.labels-mismatch", + }, + // ── changed files ───────────────────────────────────────────────────── + { + id: "changed-files-pass", + description: "file_glob_match include matches a changed file", + pr: { files: { "/src/trig-changed-pass.txt": "x\n" } }, + check: changedFilesCheck({ include: ["src/**"] }), + outcome: "pass", + }, + { + id: "changed-files-skip", + description: "file_glob_match include matches no changed file → self-cancel", + pr: { files: { "/docs/trig-changed-skip.md": "x\n" } }, + check: changedFilesCheck({ include: ["src/**"] }), + outcome: "skip", + failTag: "pr-gate.changed-files-mismatch", + }, + // ── draft ───────────────────────────────────────────────────────────── + { + id: "draft-pass", + description: "draft equals true and the PR is a draft", + pr: { draft: true }, + check: draftCheck(true), + outcome: "pass", + }, + { + id: "draft-skip", + description: "draft equals true but PR is not a draft → self-cancel", + pr: {}, + check: draftCheck(true), + outcome: "skip", + failTag: "pr-gate.draft-mismatch", + }, + // ── target branch ───────────────────────────────────────────────────── + { + id: "target-branch-pass", + description: "target_branch glob matches the PR's target (main)", + pr: {}, + check: targetBranchCheck("main"), + outcome: "pass", + }, + { + id: "target-branch-skip", + description: "target_branch glob does not match the PR's target → self-cancel", + pr: {}, + check: targetBranchCheck("release/*"), + outcome: "skip", + failTag: "pr-gate.target-branch-mismatch", + }, + // ── build reason ────────────────────────────────────────────────────── + { + id: "build-reason-pass", + description: "build_reason include matches the queued build reason (Manual)", + pr: {}, + check: buildReasonIncludeCheck(["Manual"]), + outcome: "pass", + }, + { + id: "build-reason-skip", + description: "build_reason include requires PullRequest but reason is Manual → self-cancel", + pr: {}, + check: buildReasonIncludeCheck(["PullRequest"]), + outcome: "skip", + failTag: "pr-gate.build-reason-mismatch", + }, + // ── change count ────────────────────────────────────────────────────── + { + id: "change-count-pass", + description: "numeric_range min=1 max=10 and the PR changed 2 files → runs", + // Two files also exercises the batched multi-file push in createPrContext. + pr: { + files: { + "/src/trig-count-pass-a.txt": "a\n", + "/src/trig-count-pass-b.txt": "b\n", + }, + }, + check: changeCountCheck({ min: 1, max: 10 }), + outcome: "pass", + }, + { + id: "change-count-skip", + description: "numeric_range min=5 but the PR changed 1 file → self-cancel", + pr: { files: { "/src/trig-count-skip.txt": "x\n" } }, + check: changeCountCheck({ min: 5 }), + outcome: "skip", + failTag: "pr-gate.changes-mismatch", + }, + // ── time window ─────────────────────────────────────────────────────── + { + id: "time-window-skip", + description: "time_window that excludes now → self-cancel", + pr: {}, + // A 1-hour window starting 2h in the future never contains the eval time. + check: (now: Date) => { + const nowMin = now.getUTCHours() * 60 + now.getUTCMinutes(); + return timeWindowCheck(hhmm(nowMin + 120), hhmm(nowMin + 180)); + }, + outcome: "skip", + failTag: "pr-gate.time-window-mismatch", + }, +]; + +export const filterScenarios: TriggerScenario[] = specs.map(makeFilterScenario); diff --git a/scripts/ado-script/src/trigger-e2e/scenarios/self-cancel.ts b/scripts/ado-script/src/trigger-e2e/scenarios/self-cancel.ts new file mode 100644 index 00000000..ab93c61f --- /dev/null +++ b/scripts/ado-script/src/trigger-e2e/scenarios/self-cancel.ts @@ -0,0 +1,60 @@ +/** + * Self-cancel scenario (`gate/selfcancel.ts`). + * + * When the gate decides not to run (a filter fails on a synth-promoted build), + * it self-cancels the whole build via the ADO REST API and tags it + * `pr-gate.skipped`. This scenario explicitly asserts that the build reaches a + * terminal `canceled` result (not merely `succeeded`/`failed`), which is the + * behaviour agent pipelines rely on to avoid running the agent job. + * + * Test-harness module; not shipped in `ado-script.zip`. + */ +import { buildGateSpec, encodeGateSpec, targetBranchCheck } from "../gate-spec.js"; +import type { BuildOutcome, TriggerScenario } from "../scenario.js"; +import { + createPrContext, + promoteSynthSpec, + requirePrRepo, + teardownPrContext, + type PrContext, +} from "./common.js"; + +const selfCancelOnFilterFail: TriggerScenario = { + id: "self-cancel-on-filter-fail", + description: "a failing filter on a synth-promoted build self-cancels the whole build", + async setup(ctx) { + requirePrRepo(ctx); + return createPrContext(ctx, { id: "self-cancel-on-filter-fail" }); + }, + queue(_ctx, state) { + // The PR targets its default branch; require release/* so the target-branch + // check fails. Promote against the real target branch so the gate runs. + return { + sourceBranch: state.sourceRef, + templateParameters: { + gateSpec: encodeGateSpec(buildGateSpec("pull-request", [targetBranchCheck("release/*")])), + prSynthSpec: promoteSynthSpec(state.targetBranch), + }, + }; + }, + expected() { + return { + result: "canceled", + tags: ["trig.synth.promoted", "pr-gate.skipped", "pr-gate.target-branch-mismatch"], + absentTags: ["trig.should-run.true"], + }; + }, + async assert(_ctx, _state, outcome: BuildOutcome) { + // Belt-and-braces: the runner already checks result==="canceled", but make + // the intent explicit so a future refactor of the default assertion can't + // silently drop the self-cancel guarantee. + if (outcome.result !== "canceled") { + throw new Error(`self-cancel expected result 'canceled' but got '${outcome.result ?? "?"}'`); + } + }, + async cleanup(ctx, state) { + await teardownPrContext(ctx, state); + }, +}; + +export const selfCancelScenarios: TriggerScenario[] = [selfCancelOnFilterFail]; diff --git a/scripts/ado-script/src/trigger-e2e/scenarios/synth-pr.ts b/scripts/ado-script/src/trigger-e2e/scenarios/synth-pr.ts new file mode 100644 index 00000000..6fa9bbf5 --- /dev/null +++ b/scripts/ado-script/src/trigger-e2e/scenarios/synth-pr.ts @@ -0,0 +1,125 @@ +/** + * Synthetic-PR promotion scenarios (`exec-context-pr-synth.js`). + * + * Each creates a real PR in the victim's `self` repo, queues the victim on the + * PR's source branch, and asserts the synth outcome via the victim's + * `trig.synth.*` build tag: + * - a matching PR → `trig.synth.promoted` (+ gate runs) + * - a branch mismatch → `trig.synth.skipped` (+ gate bypasses) + * - a path mismatch → `trig.synth.skipped` + * + * Test-harness module; not shipped in `ado-script.zip`. + */ +import { buildGateSpec, encodeGateSpec, encodePrSynthSpec } from "../gate-spec.js"; +import type { TriggerScenario } from "../scenario.js"; +import { + createPrContext, + excludeSynthSpec, + promoteSynthSpec, + requirePrRepo, + teardownPrContext, + type PrContext, +} from "./common.js"; + +/** Empty gate spec (no checks) — the gate passes iff it is not bypassed. */ +const EMPTY_GATE = encodeGateSpec(buildGateSpec("pull-request", [])); + +/** A matching PR promotes the CI build to synthetic-PR semantics. */ +const synthPromote: TriggerScenario = { + id: "synth-promote", + description: "matching open PR promotes the queued build to synthetic-PR", + async setup(ctx) { + requirePrRepo(ctx); + return createPrContext(ctx, { id: "synth-promote" }); + }, + queue(_ctx, state) { + return { + sourceBranch: state.sourceRef, + templateParameters: { + gateSpec: EMPTY_GATE, + // Promote against the PR's REAL target branch (not a hardcoded "main"). + prSynthSpec: promoteSynthSpec(state.targetBranch), + }, + }; + }, + expected() { + return { + result: "succeeded", + tags: ["trig.synth.promoted", "trig.should-run.true"], + absentTags: ["pr-gate.skipped"], + }; + }, + async cleanup(ctx, state) { + await teardownPrContext(ctx, state); + }, +}; + +/** A PR whose target branch is excluded by the synth spec is not promoted. */ +const synthBranchMismatch: TriggerScenario = { + id: "synth-branch-mismatch", + description: "PR targeting an unmatched branch is not synth-promoted (gate bypasses)", + async setup(ctx) { + requirePrRepo(ctx); + return createPrContext(ctx, { id: "synth-branch-mismatch" }); + }, + queue(_ctx, state) { + return { + sourceBranch: state.sourceRef, + templateParameters: { + gateSpec: EMPTY_GATE, + // Exclude the PR's REAL target branch so promotion is refused + // regardless of the repo's default branch name. + prSynthSpec: excludeSynthSpec(state.targetBranch), + }, + }; + }, + expected() { + return { + result: "succeeded", + tags: ["trig.synth.skipped", "pr-gate.passed"], + absentTags: ["trig.synth.promoted"], + }; + }, + async cleanup(ctx, state) { + await teardownPrContext(ctx, state); + }, +}; + +/** A PR none of whose changed files match the synth path filter is not promoted. */ +const synthPathMismatch: TriggerScenario = { + id: "synth-path-mismatch", + description: "PR with no changed file matching the synth path filter is not promoted", + async setup(ctx) { + requirePrRepo(ctx); + // Change only a docs file; the synth spec will require src/**. + return createPrContext(ctx, { + id: "synth-path-mismatch", + files: { [`/docs/trig/${ctx.buildId}-synth-path.md`]: "docs only change\n" }, + }); + }, + queue(_ctx, state) { + return { + sourceBranch: state.sourceRef, + templateParameters: { + gateSpec: EMPTY_GATE, + prSynthSpec: encodePrSynthSpec({ paths: { include: ["src/**"] } }), + }, + }; + }, + expected() { + return { + result: "succeeded", + tags: ["trig.synth.skipped", "pr-gate.passed"], + absentTags: ["trig.synth.promoted"], + }; + }, + async cleanup(ctx, state) { + await teardownPrContext(ctx, state); + }, +}; + +export const synthScenarios: TriggerScenario[] = [ + synthPromote, + synthBranchMismatch, + synthPathMismatch, +]; diff --git a/src/compile/filter_ir.rs b/src/compile/filter_ir.rs index 1af4ba7d..a9f8b92d 100644 --- a/src/compile/filter_ir.rs +++ b/src/compile/filter_ir.rs @@ -39,6 +39,17 @@ use std::fmt; /// Each variant maps to a specific piece of data available at pipeline runtime, /// with known acquisition cost (free pipeline variable vs. REST API call vs. /// runtime computation). +/// +/// SYNC: the deterministic trigger-condition E2E harness hand-maintains a +/// mirror of `kind()` / `failure_policy()` / `dependencies()` in +/// `scripts/ado-script/src/trigger-e2e/gate-spec.ts` (the `FACT_META` table) so +/// it can craft gate specs without invoking the compiler. This is machine- +/// guarded: `generate_fact_catalog` emits `fact-catalog.gen.json` (regenerated +/// by `npm run codegen`, CI drift-checked like `types.gen.ts`), and a +/// `gate-spec.test.ts` test deep-compares `FACT_META` against it — so a changed +/// `failure_policy`/`dependencies` or a new/removed `Fact` fails a unit test. +/// When adding a `Fact` variant, also append it to `Fact::ALL` (the exhaustive +/// match in `_fact_all_exhaustiveness_reminder` will not compile until you do). #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] pub enum Fact { // ── Pipeline variables (free — always available) ──────────────────── @@ -890,6 +901,36 @@ pub fn generate_gate_spec_schema() -> String { serde_json::to_string_pretty(&schema).expect("schema serialization") } +/// One entry in the machine-generated fact catalog. +#[derive(serde::Serialize)] +struct FactCatalogEntry { + kind: &'static str, + failure_policy: &'static str, + dependencies: Vec<&'static str>, +} + +/// Generate the fact catalog JSON: every [`Fact`]'s `kind`, `failure_policy`, +/// and (kind-named) `dependencies`, in declaration order. +/// +/// This is the machine-verifiable source of truth for the hand-maintained +/// `FACT_META` mirror in +/// `scripts/ado-script/src/trigger-e2e/gate-spec.ts`. It is emitted to a +/// committed `fact-catalog.gen.json` by `npm run codegen` and drift-checked in +/// CI exactly like `types.gen.ts`, so a Rust-side change to a fact's policy or +/// dependencies (or a new/removed `Fact`) forces a catalog regen — which the +/// trigger-e2e unit test then flags against `FACT_META`. +pub fn generate_fact_catalog() -> String { + let entries: Vec = Fact::ALL + .iter() + .map(|f| FactCatalogEntry { + kind: f.kind(), + failure_policy: f.failure_policy().as_str(), + dependencies: f.dependencies().iter().map(|d| d.kind()).collect(), + }) + .collect(); + serde_json::to_string_pretty(&entries).expect("fact catalog serialization") +} + // ─── Codegen ──────────────────────────────────────────────────────────────── // The inline heredoc evaluator has been removed in favor of external script delivery. @@ -960,6 +1001,59 @@ impl Fact { Fact::CurrentUtcMinutes => "current_utc_minutes", } } + + /// Every `Fact` variant, in declaration order. + /// + /// Kept complete by `_fact_all_exhaustiveness_reminder` below: adding a + /// `Fact` variant is a compile error there until the variant is appended + /// here, so `ALL` (and therefore the generated `fact-catalog.gen.json`) + /// can never silently miss a fact. + /// + /// Maintenance: the `14` length literal is itself a second compile-time + /// guard — appending a variant without bumping it is a mismatched-array- + /// length error. Keep it equal to the number of `Fact` variants. + pub const ALL: [Fact; 14] = [ + Fact::PrTitle, + Fact::AuthorEmail, + Fact::SourceBranch, + Fact::TargetBranch, + Fact::CommitMessage, + Fact::BuildReason, + Fact::TriggeredByPipeline, + Fact::TriggeringBranch, + Fact::PrMetadata, + Fact::PrIsDraft, + Fact::PrLabels, + Fact::ChangedFiles, + Fact::ChangedFileCount, + Fact::CurrentUtcMinutes, + ]; +} + +/// Compile-time completeness guard for [`Fact::ALL`]. +/// +/// This wildcard-free match makes adding a `Fact` variant a hard compile error +/// until the new variant is listed. When you add the arm here, ALSO append the +/// variant to `Fact::ALL` above — the generated `fact-catalog.gen.json` (and the +/// TypeScript `FACT_META` mirror it guards) is derived from `Fact::ALL`. +#[allow(dead_code)] +fn _fact_all_exhaustiveness_reminder(f: Fact) { + match f { + Fact::PrTitle + | Fact::AuthorEmail + | Fact::SourceBranch + | Fact::TargetBranch + | Fact::CommitMessage + | Fact::BuildReason + | Fact::TriggeredByPipeline + | Fact::TriggeringBranch + | Fact::PrMetadata + | Fact::PrIsDraft + | Fact::PrLabels + | Fact::ChangedFiles + | Fact::ChangedFileCount + | Fact::CurrentUtcMinutes => {} + } } impl FailurePolicy { diff --git a/src/main.rs b/src/main.rs index 42a47144..a6091677 100644 --- a/src/main.rs +++ b/src/main.rs @@ -574,6 +574,15 @@ enum Commands { #[arg(short, long)] output: Option, }, + /// Export the fact catalog JSON (kind/failure_policy/dependencies for every + /// gate `Fact`) — build-time drift guard for the trigger-e2e `FACT_META` + /// mirror in the scripts/ado-script TypeScript workspace. + #[command(hide = true)] + ExportFactCatalog { + /// Output path; if omitted, prints to stdout. + #[arg(short, long)] + output: Option, + }, /// Inspect an agent source file's typed IR: jobs, stages, steps, outputs, derived `dependsOn`. Inspect { /// Path to the agent markdown source. @@ -999,6 +1008,7 @@ async fn main() -> Result<()> { Some(Commands::Audit { .. }) => "audit", Some(Commands::Trace { .. }) => "trace", Some(Commands::ExportGateSchema { .. }) => "export-gate-schema", + Some(Commands::ExportFactCatalog { .. }) => "export-fact-catalog", Some(Commands::Inspect { .. }) => "inspect", Some(Commands::Graph { .. }) => "graph", Some(Commands::Whatif { .. }) => "whatif", @@ -1415,13 +1425,34 @@ async fn main() -> Result<()> { .parent() .filter(|parent| !parent.as_os_str().is_empty()) { - std::fs::create_dir_all(parent)?; + std::fs::create_dir_all(parent).with_context(|| { + format!("creating parent dir for gate schema: {}", parent.display()) + })?; } - std::fs::write(&path, &schema)?; + std::fs::write(&path, &schema) + .with_context(|| format!("writing gate schema to {}", path.display()))?; } None => print!("{}", schema), } } + Commands::ExportFactCatalog { output } => { + let catalog = compile::filter_ir::generate_fact_catalog(); + match output { + Some(path) => { + if let Some(parent) = path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + { + std::fs::create_dir_all(parent).with_context(|| { + format!("creating parent dir for fact catalog: {}", parent.display()) + })?; + } + std::fs::write(&path, &catalog) + .with_context(|| format!("writing fact catalog to {}", path.display()))?; + } + None => print!("{}", catalog), + } + } Commands::Inspect { source, json } => { inspect::dispatch_inspect(inspect::InspectOptions { source: &source, diff --git a/tests/trigger-e2e/README.md b/tests/trigger-e2e/README.md new file mode 100644 index 00000000..4b558466 --- /dev/null +++ b/tests/trigger-e2e/README.md @@ -0,0 +1,162 @@ +# Deterministic trigger-condition (gate / synth-PR) E2E suite + +This directory holds a **deterministic**, non-agentic end-to-end test of the +runtime **front-matter trigger conditions managed by ado-script** — the +`gate.js` PR filter evaluator and the `exec-context-pr-synth.js` synthetic-PR +promotion bundle. + +## Why this exists + +The executor E2E suite in [`tests/executor-e2e/`](../executor-e2e/) covers +**Stage 3 safe-output execution** (`ado-aw execute`). It does **not** cover the +trigger conditions that decide *whether the agent runs at all*: PR label/path/ +branch/draft/time-window/change-count filters and the CI→PR synthetic +promotion. Those run at build queue time in the ado-script bundles, emit build +tags + a `SHOULD_RUN` variable, and can self-cancel the build — none of which +the executor suite touches. + +This suite exercises them with **no LLM and no agent** in the loop: + +1. it creates real ADO PR context deterministically (branch + PR + labels + + draft) via the ADO REST API, +2. queues the hand-authored **victim pipeline** (`victim-pipeline.yml`) on the + PR's source branch with a per-scenario base64 `GATE_SPEC` / `PR_SYNTH_SPEC` + as template parameters, +3. polls the build to completion, +4. asserts the observable gate decision (**build tags + build result**), +5. cleans up every object it created, + +and, on any failure, files a GitHub issue on `githubnext/ado-aw` and fails the +build. + +## What's here + +| File | Purpose | +| --- | --- | +| `victim-pipeline.yml` | Hand-authored, **parameterized** victim. Runs only `exec-context-pr-synth.js` then `gate.js` from a checkout build, emits observable build tags. Registered separately. | +| `azure-pipelines.yml` | Hand-authored **orchestrator** (daily + manual). Builds the harness and runs it against AgentPlayground. | +| `README.md` | This file. | + +The harness itself lives in +[`scripts/ado-script/src/trigger-e2e/`](../../scripts/ado-script/src/trigger-e2e/). +It is a **test-only** ado-script bundle: built by `npm run build:trigger-e2e` +to `scripts/ado-script/test-bin/trigger-e2e.js` (a gitignored, non-root path) +and **deliberately excluded** from the released `ado-script.zip` (the release +glob only packages `ado-script/*.js`, and `trigger-e2e` is listed in +`NON_BUNDLE_DIRS` in `src/__tests__/bundle-coverage.test.ts`). + +## How one victim covers synth-PR + every gate filter + +A build queued via REST has `Build.Reason = Manual`. The victim runs +`exec-context-pr-synth.js` first; given a **real open PR** on the queued source +branch it sets `AW_SYNTHETIC_PR=true` + `AW_PR_ID`. The gate's bypass logic +treats `build_reason == "PullRequest" && AW_SYNTHETIC_PR == "true"` as **not +bypassed**, so the full PR filter evaluation runs against the real PR facts +(labels, changed files, draft, target branch, …) even though the build was +queued via REST — no server-side build-validation policy required. + +## Observable build tags (the assertion signal) + +| Tag | Meaning | +| --- | --- | +| `trig.synth.promoted` | synth promoted the build to synthetic-PR semantics | +| `trig.synth.skipped` | synth found no matching PR (branch/path mismatch, or none) | +| `trig.synth.real-pr` | a real PR-triggered build (not exercised by this suite) | +| `pr-gate.passed` | gate **bypassed** (not a PR/synth build) | +| `pr-gate.skipped` + `pr-gate.` | a filter failed; the gate **self-cancelled** the build | +| `trig.should-run.true` | the gate let the run proceed (`SHOULD_RUN=true`) | + +`skip` outcomes reach a terminal **`canceled`** build result (gate self-cancel); +`pass`/`bypass` outcomes reach `succeeded`. + +## Coverage + +- **Synthetic-PR promotion:** matching PR (promoted), branch mismatch (skipped), + path mismatch (skipped). +- **Gate PR filters** (pass + skip): labels, changed-files, draft, target-branch, + build-reason, change-count, time-window. +- **Bypass:** Manual build with no PR → `pr-gate.passed`. +- **Self-cancel:** a failing filter self-cancels the build (`canceled`). + +### Known gaps / future work + +- `pr_title` and `author_email` filters source from `System.PullRequest.*`, + which are empty on synth-promoted Manual builds. The victim accepts + `prTitle` / `authorEmail` template-parameter overrides so these predicates + can be tested against controlled values; scenarios for them are not yet + wired up. +- The synth **two-match refusal** path (two active PRs on one source branch) is + not tested (hard to set up deterministically). + +## Naming / cleanup convention + +Every object a scenario creates is prefixed `ado-aw-trig-$(Build.BuildId)-`. +Cleanup (abandon PR + delete branch) runs unconditionally after each scenario; +the smoke-suite janitor (which prunes `ado-aw-*` artifacts) is the backstop. + +## Running locally + +You need a write-capable ADO token (PAT) and a registered victim pipeline id: + +```bash +cd scripts/ado-script && npm ci && npm run build:trigger-e2e && cd ../.. + +export SYSTEM_COLLECTIONURI="https://dev.azure.com/msazuresphere/" +export SYSTEM_TEAMPROJECT="AgentPlayground" +export SYSTEM_ACCESSTOKEN="" +export TRIGGER_E2E_VICTIM_DEFINITION_ID="" +export TRIGGER_E2E_VICTIM_REPO="" +# Optional: +# export TRIGGER_E2E_GITHUB_TOKEN="" +# export TRIGGER_E2E_BUILD_TIMEOUT_MS=900000 # per victim build (default 900000) +# export TRIGGER_E2E_BUILD_POLL_MS=10000 # poll interval (default 10000) + +node scripts/ado-script/test-bin/trigger-e2e.js +``` + +When `TRIGGER_E2E_VICTIM_REPO` is unset, the PR/synth/gate scenarios **skip** +and only the bypass baseline runs. The harness exits non-zero if any scenario +fails. + +## Manual-handoff checklist (one-time ADO setup) + +In `https://dev.azure.com/msazuresphere/AgentPlayground`: + +1. **Register the VICTIM pipeline.** New pipeline → Azure Repos → the ado-aw + Azure Repos mirror → existing YAML → `tests/trigger-e2e/victim-pipeline.yml`. + It **must** be an ADO Git repo (not GitHub) so `exec-context-pr-synth` can + discover open PRs in the pipeline's own `self` repo. Note its **definition + id** for the next step. + - **Pool name.** Both `victim-pipeline.yml` and `azure-pipelines.yml` + hardcode `pool: { name: AZS-1ES-L-Playground-ubuntu-22.04 }`. If your ADO + project uses a different agent pool, edit both files to point at a + Linux pool with Node.js 20 available before registering. +2. **Register the ORCHESTRATOR pipeline.** New pipeline → existing YAML → + `tests/trigger-e2e/azure-pipelines.yml`. Place both in a `\trigger-e2e` + folder. +3. **Wire the victim id + repo** as pipeline/definition variables on the + orchestrator: + ```powershell + ado-aw secrets set TRIGGER_E2E_VICTIM_DEFINITION_ID ` + --org msazuresphere --project AgentPlayground ` + --definition-ids --value + ado-aw secrets set TRIGGER_E2E_VICTIM_REPO ` + --org msazuresphere --project AgentPlayground ` + --definition-ids --value + ``` + (These are not secrets; `secrets set` is just the variable-management path.) +4. **Grant the VICTIM build identity** Code (read) on the repo, Build (add + tags / edit build quality), and permission to cancel builds (gate + self-cancel). See [`docs/safe-output-permissions.md`](../../docs/safe-output-permissions.md) + if it hits 401/403. +5. **Grant the ORCHESTRATOR build identity** Contribute / Create branch / + Contribute to PRs on the victim repo (it creates transient PRs) and Queue + builds on the victim definition. +6. **Set the GitHub PAT secret** on the orchestrator only: + ```powershell + ado-aw secrets set TRIGGER_E2E_GITHUB_TOKEN ` + --org msazuresphere --project AgentPlayground ` + --definition-ids ` + --value + ``` +7. **Trigger one manual orchestrator run** to seed the schedule. diff --git a/tests/trigger-e2e/azure-pipelines.yml b/tests/trigger-e2e/azure-pipelines.yml new file mode 100644 index 00000000..eead6b22 --- /dev/null +++ b/tests/trigger-e2e/azure-pipelines.yml @@ -0,0 +1,85 @@ +# Deterministic trigger-condition (gate / synth-PR) E2E ORCHESTRATOR pipeline. +# +# Builds the trigger-e2e harness from this checkout and runs it against +# AgentPlayground. The harness creates real PRs and queues the registered +# VICTIM pipeline (tests/trigger-e2e/victim-pipeline.yml) under a battery of +# trigger conditions, asserting the gate decision via build tags + result. On +# any failure it files a GitHub issue on githubnext/ado-aw and fails the build. +# +# Unlike the executor-e2e orchestrator, this does NOT build the Rust `ado-aw` +# binary: trigger conditions are evaluated entirely by the ado-script bundles +# the victim runs, so the harness only needs Node. +# +# This is a hand-authored ADO pipeline (NOT compiled from an agent markdown). +# Register it in msazuresphere/AgentPlayground — see tests/trigger-e2e/README.md. + +trigger: none +pr: none + +schedules: + - cron: "0 6 * * *" + displayName: Daily deterministic trigger-condition E2E + branches: + include: + - main + always: true + +pool: + name: AZS-1ES-L-Playground-ubuntu-22.04 + +variables: + # Keep YAML defaults under private names so same-named pipeline/definition UI + # variables are not shadowed by this block. + EFFECTIVE_TRIGGER_E2E_VICTIM_DEFINITION_ID: $[ coalesce(variables['TRIGGER_E2E_VICTIM_DEFINITION_ID'], '') ] + EFFECTIVE_TRIGGER_E2E_VICTIM_REPO: $[ coalesce(variables['TRIGGER_E2E_VICTIM_REPO'], '') ] + # GitHub repo that failure issues are filed against is intentionally NOT + # defined here: a YAML `variables:` entry SHADOWS a same-named pipeline + # (definition) variable. Set TRIGGER_E2E_ISSUE_REPO as a pipeline/definition + # variable to redirect issue filing; the harness defaults to githubnext/ado-aw. + # TRIGGER_E2E_GITHUB_TOKEN must be provided as a SECRET pipeline variable + # (fine-grained PAT scoped to Issues:read/write on githubnext/ado-aw ONLY). + +steps: + - checkout: self + fetchDepth: 1 + displayName: Checkout ado-aw + + - task: UseNode@1 + inputs: + version: "20.x" + displayName: Use Node.js 20 + + - script: | + set -euo pipefail + npm ci + npm run build:trigger-e2e + workingDirectory: scripts/ado-script + displayName: Build trigger-e2e harness (not shipped in ado-script.zip) + + - task: AzureCLI@2 + displayName: Acquire ADO token (SC_WRITE_TOKEN) + inputs: + azureSubscription: agent-playground-write + scriptType: bash + scriptLocation: inlineScript + addSpnToEnvironment: true + inlineScript: | + ADO_TOKEN=$(az account get-access-token \ + --resource 499b84ac-1321-427f-aa17-267ca6975798 \ + --query accessToken -o tsv) + echo "##vso[task.setvariable variable=SC_WRITE_TOKEN;issecret=true]$ADO_TOKEN" + + - script: | + set -euo pipefail + node scripts/ado-script/test-bin/trigger-e2e.js + displayName: Run deterministic trigger-condition E2E scenarios + env: + # ADO auth + context. SYSTEM_COLLECTIONURI / SYSTEM_TEAMPROJECT / + # BUILD_BUILDID are auto-injected; SYSTEM_ACCESSTOKEN must be mapped + # explicitly so the harness can queue builds, create PRs, cancel, and tag. + SYSTEM_ACCESSTOKEN: $(SC_WRITE_TOKEN) + TRIGGER_E2E_VICTIM_DEFINITION_ID: $(EFFECTIVE_TRIGGER_E2E_VICTIM_DEFINITION_ID) + TRIGGER_E2E_VICTIM_REPO: $(EFFECTIVE_TRIGGER_E2E_VICTIM_REPO) + TRIGGER_E2E_ISSUE_REPO: $(TRIGGER_E2E_ISSUE_REPO) + # Secret PAT for filing failure issues (scoped to githubnext/ado-aw Issues). + TRIGGER_E2E_GITHUB_TOKEN: $(TRIGGER_E2E_GITHUB_TOKEN) diff --git a/tests/trigger-e2e/victim-pipeline.yml b/tests/trigger-e2e/victim-pipeline.yml new file mode 100644 index 00000000..89b86010 --- /dev/null +++ b/tests/trigger-e2e/victim-pipeline.yml @@ -0,0 +1,150 @@ +# Trigger-condition E2E VICTIM pipeline. +# +# A deterministic, parameterized pipeline that runs ONLY the runtime trigger +# bundles managed by ado-script — `exec-context-pr-synth.js` (CI→PR synthetic +# promotion) and `gate.js` (PR filter evaluation) — and reports the decision via +# REST-observable build tags. It runs NO agent workflow. +# +# The trigger-e2e harness (scripts/ado-script/src/trigger-e2e/) queues this +# pipeline many ways: it creates a real PR, then queues a build on the PR's +# source branch with a per-scenario base64 `gateSpec` / `prSynthSpec` as +# template parameters, and asserts the resulting build tags + result. +# +# ## Registration requirements (see tests/trigger-e2e/README.md) +# - MUST be registered against an **ADO Git repo** (BUILD_REPOSITORY_PROVIDER +# = TfsGit), because `exec-context-pr-synth` discovers the open PR in the +# pipeline's OWN `self` repo. A GitHub-typed repo short-circuits synth. +# Register it against the ado-aw Azure Repos mirror so the `self` checkout +# also carries `scripts/ado-script/` to build the bundles from source. +# - The build identity ($(System.AccessToken)) needs: Code (read) on the repo +# (PR/label/iteration lookups), Build (edit build quality / add tags) and +# the ability to cancel builds (gate self-cancel). See +# docs/safe-output-permissions.md if it hits 401/403. +# +# ## Observable build tags +# - trig.synth.promoted | trig.synth.skipped | trig.synth.real-pr +# - pr-gate.passed (bypass) | pr-gate.skipped + pr-gate. (filter fail) +# - trig.should-run.true (gate let the run proceed) + +trigger: none +pr: none + +pool: + name: AZS-1ES-L-Playground-ubuntu-22.04 + +parameters: + # Required per queue: base64 GateSpec produced by the harness gate-spec.ts. + - name: gateSpec + type: string + default: "" + # base64 PrSynthSpec. Default = empty include/exclude (match-all promote). + - name: prSynthSpec + type: string + default: "eyJicmFuY2hlcyI6eyJpbmNsdWRlIjpbXSwiZXhjbHVkZSI6W119LCJwYXRocyI6eyJpbmNsdWRlIjpbXSwiZXhjbHVkZSI6W119fQ==" + # Pipeline-variable fact overrides. Defaults reference the synth-resolved + # AW_PR_* vars / ADO predefined vars, so an unset override uses the real value. + - name: prTitle + type: string + default: "$(System.PullRequest.Title)" + - name: authorEmail + type: string + default: "$(Build.RequestedForEmail)" + - name: sourceBranchFact + type: string + default: "$(AW_PR_SOURCEBRANCH)" + - name: targetBranchFact + type: string + default: "$(AW_PR_TARGETBRANCH)" + - name: commitMessage + type: string + default: "$(Build.SourceVersionMessage)" + +jobs: + - job: Trigger + displayName: Evaluate trigger conditions + steps: + - checkout: self + # Full history is not needed; the bundles are built from the checkout + # and the gate/synth logic uses REST, not git. + fetchDepth: 1 + displayName: Checkout self (ado-aw source) + + - task: UseNode@1 + inputs: + version: "20.x" + displayName: Use Node.js 20 + + - script: | + set -euo pipefail + npm ci + npm run build:gate + npm run build:exec-context-pr-synth + workingDirectory: scripts/ado-script + displayName: Build gate + synth bundles from checkout + + - script: | + set -euo pipefail + node scripts/ado-script/exec-context-pr-synth.js + displayName: Run exec-context-pr-synth (synthetic PR promotion) + env: + PR_SYNTH_SPEC: ${{ parameters.prSynthSpec }} + # REST auth for active-PR / iteration lookups. + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + + - script: | + set -euo pipefail + # Classify the synth outcome from the same-job vars synth setVar's. + # AW_PR_ID is always set (empty on skip); AW_SYNTHETIC_PR is only set + # to "true" on promotion (else left as the literal macro). + SYN="$(AW_SYNTHETIC_PR)" + PRID="$(AW_PR_ID)" + case "$SYN" in '$('*) SYN="" ;; esac + case "$PRID" in '$('*) PRID="" ;; esac + if [ "$SYN" = "true" ]; then + STATE="promoted" + elif [ -n "$PRID" ]; then + STATE="real-pr" + else + STATE="skipped" + fi + echo "synth state: ${STATE} (AW_PR_ID='${PRID}' AW_SYNTHETIC_PR='${SYN}')" + echo "##vso[build.addbuildtag]trig.synth.${STATE}" + displayName: Tag synth outcome + + - script: | + set -euo pipefail + node scripts/ado-script/gate.js + name: prGate + displayName: Evaluate PR filters (gate) + env: + GATE_SPEC: ${{ parameters.gateSpec }} + # Gate context / infra facts. + ADO_BUILD_REASON: $(Build.Reason) + ADO_BUILD_ID: $(Build.BuildId) + ADO_PROJECT: $(System.TeamProject) + ADO_REPO_ID: $(Build.Repository.ID) + # PR identity: resolved by synth (real or promoted). AW_SYNTHETIC_PR + # tells the gate's bypass logic to run full evaluation on a promoted + # CI build. + ADO_PR_ID: $(AW_PR_ID) + AW_SYNTHETIC_PR: $(AW_SYNTHETIC_PR) + # Pipeline-variable facts (override-able via template parameters). + ADO_PR_TITLE: ${{ parameters.prTitle }} + ADO_AUTHOR_EMAIL: ${{ parameters.authorEmail }} + ADO_SOURCE_BRANCH: ${{ parameters.sourceBranchFact }} + ADO_TARGET_BRANCH: ${{ parameters.targetBranchFact }} + ADO_COMMIT_MESSAGE: ${{ parameters.commitMessage }} + # REST auth for API facts (labels, changed files) + self-cancel. + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + + - script: | + set -euo pipefail + # Only reached when the gate did NOT self-cancel (SHOULD_RUN=true or + # bypass). SHOULD_RUN is a step-output var of the `prGate` step. + RUN="$(prGate.SHOULD_RUN)" + echo "gate SHOULD_RUN='${RUN}'" + if [ "$RUN" = "true" ]; then + echo "##vso[build.addbuildtag]trig.should-run.true" + fi + # Default condition is succeeded(); a self-cancelled build skips this. + displayName: Reflect gate decision