diff --git a/.gitattributes b/.gitattributes index 41f42ee5..93354bad 100644 --- a/.gitattributes +++ b/.gitattributes @@ -16,32 +16,9 @@ tests/fixtures/runtime_imports_author_marker_stage.lock.yml linguist-generated=t tests/fixtures/runtime_imports_job.lock.yml linguist-generated=true merge=ours text eol=lf tests/fixtures/runtime_imports_stage.lock.yml linguist-generated=true merge=ours text eol=lf tests/fixtures/stage-agent.lock.yml linguist-generated=true merge=ours text eol=lf -tests/safe-outputs/add-build-tag.lock.yml linguist-generated=true merge=ours text eol=lf -tests/safe-outputs/add-pr-comment.lock.yml linguist-generated=true merge=ours text eol=lf tests/safe-outputs/azure-cli.lock.yml linguist-generated=true merge=ours text eol=lf tests/safe-outputs/canary.lock.yml linguist-generated=true merge=ours text eol=lf -tests/safe-outputs/comment-on-work-item.lock.yml linguist-generated=true merge=ours text eol=lf -tests/safe-outputs/create-branch.lock.yml linguist-generated=true merge=ours text eol=lf -tests/safe-outputs/create-git-tag.lock.yml linguist-generated=true merge=ours text eol=lf -tests/safe-outputs/create-pull-request.lock.yml linguist-generated=true merge=ours text eol=lf -tests/safe-outputs/create-wiki-page.lock.yml linguist-generated=true merge=ours text eol=lf -tests/safe-outputs/create-work-item.lock.yml linguist-generated=true merge=ours text eol=lf tests/safe-outputs/janitor.lock.yml linguist-generated=true merge=ours text eol=lf -tests/safe-outputs/link-work-items.lock.yml linguist-generated=true merge=ours text eol=lf -tests/safe-outputs/missing-data.lock.yml linguist-generated=true merge=ours text eol=lf -tests/safe-outputs/missing-tool.lock.yml linguist-generated=true merge=ours text eol=lf tests/safe-outputs/noop-target.lock.yml linguist-generated=true merge=ours text eol=lf -tests/safe-outputs/noop.lock.yml linguist-generated=true merge=ours text eol=lf -tests/safe-outputs/queue-build.lock.yml linguist-generated=true merge=ours text eol=lf -tests/safe-outputs/reply-to-pr-comment.lock.yml linguist-generated=true merge=ours text eol=lf -tests/safe-outputs/report-incomplete.lock.yml linguist-generated=true merge=ours text eol=lf -tests/safe-outputs/resolve-pr-thread.lock.yml linguist-generated=true merge=ours text eol=lf tests/safe-outputs/smoke-failure-reporter.lock.yml linguist-generated=true merge=ours text eol=lf -tests/safe-outputs/submit-pr-review.lock.yml linguist-generated=true merge=ours text eol=lf -tests/safe-outputs/update-pr.lock.yml linguist-generated=true merge=ours text eol=lf -tests/safe-outputs/update-wiki-page.lock.yml linguist-generated=true merge=ours text eol=lf -tests/safe-outputs/update-work-item.lock.yml linguist-generated=true merge=ours text eol=lf -tests/safe-outputs/upload-build-attachment.lock.yml linguist-generated=true merge=ours text eol=lf -tests/safe-outputs/upload-pipeline-artifact.lock.yml linguist-generated=true merge=ours text eol=lf -tests/safe-outputs/upload-workitem-attachment.lock.yml linguist-generated=true merge=ours text eol=lf # END ado-aw managed diff --git a/scripts/ado-script/src/trigger-e2e/__tests__/mirror.test.ts b/scripts/ado-script/src/trigger-e2e/__tests__/mirror.test.ts new file mode 100644 index 00000000..1227814c --- /dev/null +++ b/scripts/ado-script/src/trigger-e2e/__tests__/mirror.test.ts @@ -0,0 +1,164 @@ +import { describe, expect, it, vi } from "vitest"; + +import { + loadMirrorSyncConfig, + mirrorRepoUrl, + runMirrorSyncPreflight, + type MirrorGitRequest, + type MirrorGitResult, + type MirrorGitRunner, +} from "../mirror.js"; + +const HEAD = "1111111111111111111111111111111111111111"; + +function baseEnv(): NodeJS.ProcessEnv { + return { + TRIGGER_E2E_SYNC_MIRROR: "true", + TRIGGER_E2E_VICTIM_REPO: "ado-aw-mirror", + SYSTEM_COLLECTIONURI: "https://dev.azure.com/msazuresphere/", + SYSTEM_TEAMPROJECT: "Agent Playground", + SYSTEM_ACCESSTOKEN: "secret-token", + BUILD_SOURCESDIRECTORY: "/src", + BUILD_SOURCEBRANCH: "refs/heads/main", + BUILD_SOURCEVERSION: HEAD, + }; +} + +function successRunner(calls: MirrorGitRequest[]): MirrorGitRunner { + return async (request): Promise => { + calls.push(request); + if (request.args[0] === "rev-parse" && request.args[1] === "--is-shallow-repository") { + return { status: 0, stdout: "false\n", stderr: "" }; + } + if (request.args[0] === "rev-parse") { + return { status: 0, stdout: `${HEAD}\n`, stderr: "" }; + } + if (request.args[0] === "ls-remote") { + return { status: 0, stdout: `${HEAD}\trefs/heads/main\n`, stderr: "" }; + } + return { status: 0, stdout: "", stderr: "" }; + }; +} + +describe("trigger mirror sync", () => { + it("is disabled unless explicitly opted in", async () => { + const runner = vi.fn(); + const result = await runMirrorSyncPreflight({}, () => {}, runner); + expect(result).toBeUndefined(); + expect(runner).not.toHaveBeenCalled(); + }); + + it("builds an encoded ADO Git URL", () => { + expect( + mirrorRepoUrl( + "https://dev.azure.com/msazuresphere/", + "Agent Playground", + "ado aw/mirror", + ), + ).toBe( + "https://dev.azure.com/msazuresphere/Agent%20Playground/_git/ado%20aw%2Fmirror", + ); + }); + + it("pushes main fast-forward-only and verifies the remote SHA", async () => { + const calls: MirrorGitRequest[] = []; + const result = await runMirrorSyncPreflight(baseEnv(), () => {}, successRunner(calls)); + + expect(result?.ok).toBe(true); + expect(calls.map((call) => call.args[0])).toEqual([ + "rev-parse", + "rev-parse", + "push", + "ls-remote", + ]); + const push = calls[2]; + expect(push?.args).toContain("HEAD:refs/heads/main"); + expect(push?.args).not.toContain("--force"); + expect(push?.args.join(" ")).not.toContain("secret-token"); + expect(push?.env.GIT_CONFIG_VALUE_0).toBe("Authorization: bearer secret-token"); + expect(push?.cwd).toBe("/src"); + }); + + it("rejects a non-main orchestrator checkout before invoking git", async () => { + const runner = vi.fn(); + const result = await runMirrorSyncPreflight( + { ...baseEnv(), BUILD_SOURCEBRANCH: "refs/heads/feature" }, + () => {}, + runner, + ); + + expect(result?.ok).toBe(false); + expect(result?.message).toContain("refs/heads/main"); + expect(runner).not.toHaveBeenCalled(); + }); + + it("rejects a shallow checkout", async () => { + const runner: MirrorGitRunner = async () => ({ + status: 0, + stdout: "true\n", + stderr: "", + }); + const result = await runMirrorSyncPreflight(baseEnv(), () => {}, runner); + + expect(result?.ok).toBe(false); + expect(result?.message).toContain("fetchDepth: 0"); + }); + + it("fails closed on a non-fast-forward push", async () => { + const calls: MirrorGitRequest[] = []; + const runner: MirrorGitRunner = async (request) => { + calls.push(request); + if (request.args[0] === "rev-parse" && request.args[1] === "--is-shallow-repository") { + return { status: 0, stdout: "false\n", stderr: "" }; + } + if (request.args[0] === "rev-parse") { + return { status: 0, stdout: `${HEAD}\n`, stderr: "" }; + } + return { status: 1, stdout: "", stderr: "! [rejected] non-fast-forward" }; + }; + + const result = await runMirrorSyncPreflight(baseEnv(), () => {}, runner); + expect(result?.ok).toBe(false); + expect(result?.message).toContain("non-fast-forward"); + expect(calls.some((call) => call.args[0] === "ls-remote")).toBe(false); + }); + + it("fails when the verified remote SHA differs", async () => { + const runner: MirrorGitRunner = async (request) => { + if (request.args[0] === "rev-parse" && request.args[1] === "--is-shallow-repository") { + return { status: 0, stdout: "false\n", stderr: "" }; + } + if (request.args[0] === "rev-parse") { + return { status: 0, stdout: `${HEAD}\n`, stderr: "" }; + } + if (request.args[0] === "ls-remote") { + return { + status: 0, + stdout: "2222222222222222222222222222222222222222\trefs/heads/main\n", + stderr: "", + }; + } + return { status: 0, stdout: "", stderr: "" }; + }; + + const result = await runMirrorSyncPreflight(baseEnv(), () => {}, runner); + expect(result?.ok).toBe(false); + expect(result?.message).toContain("mirror verification failed"); + }); + + it("preserves the bypass-only baseline when the victim repo is unset", async () => { + const runner = vi.fn(); + const logs: string[] = []; + const env = { + ...baseEnv(), + TRIGGER_E2E_VICTIM_REPO: "", + }; + + expect(loadMirrorSyncConfig(env)).toBeUndefined(); + expect(await runMirrorSyncPreflight(env, (message) => logs.push(message), runner)).toBeUndefined(); + expect(logs).toContain( + "[mirror-sync] skipped: TRIGGER_E2E_VICTIM_REPO is not configured", + ); + expect(runner).not.toHaveBeenCalled(); + }); +}); diff --git a/scripts/ado-script/src/trigger-e2e/index.ts b/scripts/ado-script/src/trigger-e2e/index.ts index 6157bea8..6a542fe8 100644 --- a/scripts/ado-script/src/trigger-e2e/index.ts +++ b/scripts/ado-script/src/trigger-e2e/index.ts @@ -25,6 +25,7 @@ */ import { AdoRest } from "../executor-e2e/ado-rest.js"; import { fileFailureIssue, loadIssueEnv } from "./github-issue.js"; +import { runMirrorSyncPreflight } from "./mirror.js"; import { runAll } from "./runner.js"; import { allScenarios } from "./scenarios/index.js"; import type { ScenarioResult, TriggerContext } from "./scenario.js"; @@ -87,12 +88,18 @@ export async function main(): Promise { prefix: (id) => `ado-aw-trig-${buildId}-${id}`, }; - log( - `Running ${allScenarios.length} trigger E2E scenarios against ${orgUrl}${project} ` + - `(victim def #${victimDefinitionId}${adoRepo ? `, repo ${adoRepo}` : ", no PR repo — PR scenarios will skip"})`, - ); - - const results = await runAll(ctx, allScenarios); + const preflight = await runMirrorSyncPreflight(process.env, log); + let results: ScenarioResult[]; + if (preflight && !preflight.ok) { + results = [preflight]; + } else { + log( + `Running ${allScenarios.length} trigger E2E scenarios against ${orgUrl}${project} ` + + `(victim def #${victimDefinitionId}${adoRepo ? `, repo ${adoRepo}` : ", no PR repo — PR scenarios will skip"})`, + ); + const scenarioResults = await runAll(ctx, allScenarios); + results = preflight ? [preflight, ...scenarioResults] : scenarioResults; + } log(summarise(results)); const issueEnv = loadIssueEnv(); diff --git a/scripts/ado-script/src/trigger-e2e/mirror.ts b/scripts/ado-script/src/trigger-e2e/mirror.ts new file mode 100644 index 00000000..21de5be1 --- /dev/null +++ b/scripts/ado-script/src/trigger-e2e/mirror.ts @@ -0,0 +1,238 @@ +import { spawn } from "node:child_process"; + +import { bearerEnv } from "../shared/git.js"; +import type { ScenarioResult } from "./scenario.js"; + +const DEFAULT_TIMEOUT_MS = 300_000; +const MAIN_REF = "refs/heads/main"; +const MAX_DIAGNOSTIC_CHARS = 2_000; + +export interface MirrorSyncConfig { + orgUrl: string; + project: string; + repo: string; + token: string; + cwd: string; + sourceBranch: string; + sourceVersion?: string; + timeoutMs: number; +} + +export interface MirrorGitRequest { + args: string[]; + cwd: string; + env: Record; + timeoutMs: number; +} + +export interface MirrorGitResult { + status: number | null; + stdout: string; + stderr: string; +} + +export type MirrorGitRunner = (request: MirrorGitRequest) => Promise; + +function cleanVar(raw: string | undefined): string | undefined { + const value = raw?.trim(); + if (!value || /^\$\(.*\)$/.test(value)) return undefined; + return value; +} + +function required(env: NodeJS.ProcessEnv, name: string): string { + const value = cleanVar(env[name]); + if (!value) throw new Error(`${name} is required when TRIGGER_E2E_SYNC_MIRROR=true`); + return value; +} + +function enabled(raw: string | undefined): boolean { + return cleanVar(raw)?.toLowerCase() === "true"; +} + +function timeoutMs(raw: string | undefined): number { + const parsed = Number(cleanVar(raw)); + return Number.isFinite(parsed) && parsed > 0 ? parsed : DEFAULT_TIMEOUT_MS; +} + +export function mirrorRepoUrl(orgUrl: string, project: string, repo: string): string { + return `${orgUrl.replace(/\/+$/, "")}/${encodeURIComponent(project)}/_git/${encodeURIComponent(repo)}`; +} + +export function loadMirrorSyncConfig( + env: NodeJS.ProcessEnv = process.env, +): MirrorSyncConfig | undefined { + if (!enabled(env.TRIGGER_E2E_SYNC_MIRROR)) return undefined; + // Preserve the harness's existing "bypass-only" baseline when no ADO PR + // repository is configured. Once a repo is supplied, synchronization is + // mandatory and fail-closed. + const repo = cleanVar(env.TRIGGER_E2E_VICTIM_REPO); + if (!repo) return undefined; + + return { + orgUrl: required(env, "SYSTEM_COLLECTIONURI"), + project: required(env, "SYSTEM_TEAMPROJECT"), + repo, + token: required(env, "SYSTEM_ACCESSTOKEN"), + cwd: cleanVar(env.BUILD_SOURCESDIRECTORY) ?? process.cwd(), + sourceBranch: required(env, "BUILD_SOURCEBRANCH"), + sourceVersion: cleanVar(env.BUILD_SOURCEVERSION), + timeoutMs: timeoutMs(env.TRIGGER_E2E_MIRROR_SYNC_TIMEOUT_MS), + }; +} + +function runGit(request: MirrorGitRequest): Promise { + return new Promise((resolve, reject) => { + const child = spawn("git", request.args, { + cwd: request.cwd, + env: { + ...process.env, + ...request.env, + GIT_TERMINAL_PROMPT: "0", + GIT_TRACE: "0", + GIT_TRACE_CURL: "0", + GIT_CURL_VERBOSE: "0", + }, + }); + let stdout = ""; + let stderr = ""; + let timedOut = false; + let settled = false; + + const finish = (result: MirrorGitResult): void => { + if (settled) return; + settled = true; + clearTimeout(timer); + resolve(result); + }; + + const timer = setTimeout(() => { + timedOut = true; + child.kill("SIGKILL"); + finish({ + status: null, + stdout, + stderr: `${stderr}\ngit timed out after ${request.timeoutMs}ms`, + }); + }, request.timeoutMs); + + child.stdout.on("data", (data: Buffer) => { + stdout += data.toString(); + }); + child.stderr.on("data", (data: Buffer) => { + stderr += data.toString(); + }); + child.on("error", (err) => { + if (settled) return; + settled = true; + clearTimeout(timer); + reject(err); + }); + child.on("close", (status) => { + if (timedOut) return; + finish({ status, stdout, stderr }); + }); + }); +} + +function diagnostic(result: MirrorGitResult, secret: string): string { + const text = [result.stderr.trim(), result.stdout.trim()] + .filter(Boolean) + .join("\n") + .replaceAll(secret, "***"); + if (!text) return "(no output)"; + return text.length <= MAX_DIAGNOSTIC_CHARS + ? text + : `${text.slice(0, MAX_DIAGNOSTIC_CHARS)}…`; +} + +async function git( + config: MirrorSyncConfig, + args: string[], + runner: MirrorGitRunner, +): Promise { + const result = await runner({ + args, + cwd: config.cwd, + env: bearerEnv(config.token), + timeoutMs: config.timeoutMs, + }); + if (result.status !== 0) { + throw new Error(`git ${args[0] ?? "command"} failed: ${diagnostic(result, config.token)}`); + } + return result.stdout.trim(); +} + +export async function syncMirror( + config: MirrorSyncConfig, + runner: MirrorGitRunner = runGit, +): Promise { + if (config.sourceBranch !== MAIN_REF) { + throw new Error( + `mirror sync requires BUILD_SOURCEBRANCH=${MAIN_REF} (got '${config.sourceBranch}')`, + ); + } + + const shallow = await git(config, ["rev-parse", "--is-shallow-repository"], runner); + if (shallow !== "false") { + throw new Error( + `mirror sync requires a full checkout (git reported is-shallow-repository='${shallow}'); set fetchDepth: 0`, + ); + } + + const head = await git(config, ["rev-parse", "HEAD"], runner); + if (config.sourceVersion && head.toLowerCase() !== config.sourceVersion.toLowerCase()) { + throw new Error( + `checked-out HEAD ${head} does not match BUILD_SOURCEVERSION ${config.sourceVersion}`, + ); + } + + const url = mirrorRepoUrl(config.orgUrl, config.project, config.repo); + await git(config, ["push", "--porcelain", url, `HEAD:${MAIN_REF}`], runner); + + const remote = await git(config, ["ls-remote", url, MAIN_REF], runner); + const remoteHead = remote.split(/\s+/)[0]?.trim(); + if (!remoteHead || remoteHead.toLowerCase() !== head.toLowerCase()) { + throw new Error( + `mirror verification failed: expected ${head} at ${MAIN_REF}, got '${remoteHead ?? ""}'`, + ); + } + return head; +} + +export async function runMirrorSyncPreflight( + env: NodeJS.ProcessEnv, + log: (message: string) => void, + runner: MirrorGitRunner = runGit, +): Promise { + const start = Date.now(); + try { + if ( + enabled(env.TRIGGER_E2E_SYNC_MIRROR) && + !cleanVar(env.TRIGGER_E2E_VICTIM_REPO) + ) { + log("[mirror-sync] skipped: TRIGGER_E2E_VICTIM_REPO is not configured"); + return undefined; + } + const config = loadMirrorSyncConfig(env); + if (!config) return undefined; + + log(`[mirror-sync] syncing checked-out main to ADO repo '${config.repo}'`); + const head = await syncMirror(config, runner); + log(`[mirror-sync] OK: '${config.repo}' ${MAIN_REF} is ${head}`); + return { + id: "mirror-sync", + ok: true, + durationMs: Date.now() - start, + }; + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + log(`[mirror-sync] FAILED: ${message}`); + return { + id: "mirror-sync", + ok: false, + phase: "setup", + message, + durationMs: Date.now() - start, + }; + } +} diff --git a/tests/compiler_tests.rs b/tests/compiler_tests.rs index 5c324335..0b046b51 100644 --- a/tests/compiler_tests.rs +++ b/tests/compiler_tests.rs @@ -8547,3 +8547,32 @@ fn test_no_create_pull_request_omits_prepare_pr_base_step() { "prepare step display name must be absent:\n{compiled}" ); } + +#[test] +fn test_smoke_failure_reporter_uses_registered_ado_names_and_staging_repo() { + let reporter_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests") + .join("safe-outputs") + .join("smoke-failure-reporter.md"); + let reporter = fs::read_to_string(reporter_path).expect("read smoke-failure-reporter fixture"); + + for definition_name in [ + "Daily safe-output smoke canary", + "Daily smoke az CLI access", + ] { + assert!( + reporter.contains(&format!("- `{definition_name}`")), + "reporter must query the ADO-safe definition name '{definition_name}'" + ); + } + assert!( + !reporter.contains("Daily safe-output smoke: canary") + && !reporter.contains("Daily smoke: az CLI access"), + "reporter must not query colon-containing front-matter names" + ); + assert!( + reporter.contains("target-repo: jamesadevine/ado-aw-issues") + && reporter.contains("Search open issues on `jamesadevine/ado-aw-issues`"), + "front matter and prompt must agree on the staging issue repository" + ); +} diff --git a/tests/executor-e2e/README.md b/tests/executor-e2e/README.md index f6613e2d..05026b75 100644 --- a/tests/executor-e2e/README.md +++ b/tests/executor-e2e/README.md @@ -20,8 +20,10 @@ This suite removes the LLM from the loop. For every ADO-write safe output it: 4. asserts the effect via the ADO REST API, 5. cleans up every object it created, -and, on any failure, files a GitHub issue on `githubnext/ado-aw` and fails the -build. +and, on any failure, files a GitHub issue on the configured issue repository +and fails the build. AgentPlayground currently uses +`jamesadevine/ado-aw-issues` because a canonical-repository credential is not +available. ## What's here @@ -96,7 +98,8 @@ export SYSTEM_ACCESSTOKEN="" export EXECUTOR_E2E_ADO_AW_BIN="$PWD/target/release/ado-aw" export EXECUTOR_E2E_ADO_REPO="agent-definitions" # Optional: -# export EXECUTOR_E2E_GITHUB_TOKEN="" +# export EXECUTOR_E2E_GITHUB_TOKEN="" +# export EXECUTOR_E2E_ISSUE_REPO="jamesadevine/ado-aw-issues" # export E2E_QUEUE_PIPELINE_ID="" # Optional timeout tuning (milliseconds) for slow environments: # export EXECUTOR_E2E_REST_TIMEOUT_MS=30000 # per ADO REST call (default 30000) @@ -113,12 +116,14 @@ no current build. The harness exits non-zero if any scenario fails. In `https://dev.azure.com/msazuresphere/AgentPlayground`: -1. **Register the pipeline.** New pipeline → Azure Repos/GitHub → existing YAML - → `tests/executor-e2e/azure-pipelines.yml`. Place it in a `\executor-e2e` - folder. -2. **Grant the build identity write access** on the `agent-definitions` repo - (Contribute, Create branch, Contribute to PRs) and on Build (add tags) — the - pipeline uses `$(System.AccessToken)`. See +1. **Register the pipeline.** New pipeline → GitHub through the + `github.com_githubnext` service connection → existing YAML → + `tests/executor-e2e/azure-pipelines.yml`. Place it in a `\executor-e2e` + folder and skip the first run until variables are configured. +2. **Grant the principal behind `agent-playground-write` write access** on the + `agent-definitions` repo (Contribute, Create branch, Contribute to PRs) and + on Build (add tags). The YAML maps its AAD token to + `SYSTEM_ACCESSTOKEN` through `SC_WRITE_TOKEN`. See [`docs/safe-output-permissions.md`](../../docs/safe-output-permissions.md) if Stage 3 hits 401/403. 3. **Set the GitHub PAT secret** on this pipeline only: @@ -126,9 +131,12 @@ In `https://dev.azure.com/msazuresphere/AgentPlayground`: ado-aw secrets set EXECUTOR_E2E_GITHUB_TOKEN ` --org msazuresphere --project AgentPlayground ` --definition-ids ` - --value + --value ``` Do **not** place this token in a shared variable group. -4. *(Optional)* Set `E2E_QUEUE_PIPELINE_ID` (the `noop-target` pipeline id) and - `E2E_WIKI_NAME` to enable the queue-build and wiki scenarios. -5. **Trigger one manual run** to seed the schedule. +4. Set `EXECUTOR_E2E_ISSUE_REPO=jamesadevine/ado-aw-issues`. + Confirm the target repository has `executor-e2e-failure` and + `pipeline-failure` labels. +5. Set `E2E_QUEUE_PIPELINE_ID` to the replacement `noop-target` definition ID. + *(Optional)* Set `E2E_WIKI_NAME` to enable the wiki scenarios. +6. **Trigger one manual run** to seed the schedule. diff --git a/tests/executor-e2e/azure-pipelines.yml b/tests/executor-e2e/azure-pipelines.yml index 5d16ce63..207f7869 100644 --- a/tests/executor-e2e/azure-pipelines.yml +++ b/tests/executor-e2e/azure-pipelines.yml @@ -6,8 +6,8 @@ # (scripts/ado-script/src/executor-e2e/) sets up preconditions via the ADO REST # API, crafts the executor's NDJSON input directly, runs the real `ado-aw # execute` binary built from this checkout, asserts the effect via REST, and -# cleans up. On any failure it files a GitHub issue on githubnext/ado-aw and -# fails the build. +# cleans up. On any failure it files a GitHub issue on the configured issue +# repository and fails the build. # # This is a hand-authored ADO pipeline (NOT compiled from an agent markdown). # Register it in msazuresphere/AgentPlayground — see tests/executor-e2e/README.md. @@ -54,7 +54,7 @@ variables: # Optional-precondition scenarios: empty by default so queue-build / wiki # skip gracefully. Override at queue time (or via a variable group) to enable. # EXECUTOR_E2E_GITHUB_TOKEN must be provided as a SECRET pipeline variable - # (fine-grained PAT scoped to Issues:read/write on githubnext/ado-aw ONLY). + # scoped to Issues:read/write on the configured issue repository. steps: - checkout: self @@ -133,7 +133,7 @@ steps: EXECUTOR_E2E_ADO_AW_BIN: $(ADO_AW_BIN) EXECUTOR_E2E_ADO_REPO: $(EFFECTIVE_EXECUTOR_E2E_ADO_REPO) EXECUTOR_E2E_ISSUE_REPO: $(EXECUTOR_E2E_ISSUE_REPO) - # Secret PAT for filing failure issues (scoped to githubnext/ado-aw Issues). + # Secret PAT for filing failure issues on the configured repository. EXECUTOR_E2E_GITHUB_TOKEN: $(EXECUTOR_E2E_GITHUB_TOKEN) # Optional-precondition scenarios (skipped when unset): E2E_QUEUE_PIPELINE_ID: $(EFFECTIVE_E2E_QUEUE_PIPELINE_ID) diff --git a/tests/safe-outputs/README.md b/tests/safe-outputs/README.md index 3b5ccf83..152248e0 100644 --- a/tests/safe-outputs/README.md +++ b/tests/safe-outputs/README.md @@ -30,7 +30,7 @@ five pipelines: | `azure-cli.md` / `azure-cli.lock.yml` | Daily: verifies the AWF az CLI extension is mounted, the `az devops` subcommand authenticates via `AZURE_DEVOPS_EXT_PAT`, and the sandbox can reach the ADO control plane. | | `noop-target.md` / `noop-target.lock.yml` | No-schedule target pipeline queued by the `queue-build` executor-e2e scenario (its ID feeds `E2E_QUEUE_PIPELINE_ID`). | | `janitor.md` / `janitor.lock.yml` | Weekly: prunes `ado-aw-smoke-*` artifacts (work items, branches, wiki pages, tags, PRs) older than 30 days from AgentPlayground. | -| `smoke-failure-reporter.md` / `smoke-failure-reporter.lock.yml` | Daily ~04:30: queries the canary and azure-cli pipelines for failures and files `[smoke-failure] …` issues on `githubnext/ado-aw`. | +| `smoke-failure-reporter.md` / `smoke-failure-reporter.lock.yml` | Daily ~04:30: queries the canary and azure-cli pipelines for failures and files `[smoke-failure] …` issues on `jamesadevine/ado-aw-issues` while canonical-repo credentials are unavailable. | | `REGISTERED.md` | Contributor-maintained `fixture → ADO pipeline ID` mapping. | > **Deterministic complement.** For a flake-free regression check of @@ -89,18 +89,27 @@ In `https://dev.azure.com/msazuresphere/AgentPlayground`: 2. Bulk-register the smoke pipelines with `ado-aw enable`: ```powershell - ado-aw enable ` + cargo run -- enable ` --org msazuresphere --project AgentPlayground ` - --service-connection ado-aw-github ` + --service-connection github.com_githubnext ` + --also-set-token ` --folder '\smoke' ` tests/safe-outputs/ ``` 3. Capture each Pipeline ID and update `REGISTERED.md`. 4. Provision the `ADO_AW_DEBUG_GITHUB_TOKEN` secret (fine-grained PAT, - Issues: read/write on `githubnext/ado-aw`) on the - `smoke-failure-reporter` pipeline **only**. -5. Set `EXECUTOR_E2E_ISSUE_REPO` / `E2E_QUEUE_PIPELINE_ID` on the + Issues: read/write on `jamesadevine/ado-aw-issues`) on the + `smoke-failure-reporter` pipeline **only**. Confirm the staging repository + has the `pipeline-failure` and `ado-aw-smoke` labels. +5. Set `EXECUTOR_E2E_ISSUE_REPO=jamesadevine/ado-aw-issues` and + `E2E_QUEUE_PIPELINE_ID` on the executor-e2e pipeline using the `noop-target` pipeline ID from step 3. 6. Trigger one manual run per pipeline to seed the schedule. + +> **Existing-definition cutover.** `ado-aw enable` matches definitions by YAML +> path and will reuse the four legacy registrations. To create side-by-side +> replacements before deleting the old definitions, register the five YAML +> paths explicitly with `az pipelines create --skip-run true`, configure and +> validate the returned IDs, then retire the legacy IDs. diff --git a/tests/safe-outputs/REGISTERED.md b/tests/safe-outputs/REGISTERED.md index 0ff52f8b..400bc18f 100644 --- a/tests/safe-outputs/REGISTERED.md +++ b/tests/safe-outputs/REGISTERED.md @@ -16,7 +16,7 @@ After the manual-handoff registration step is complete, fill in the | `azure-cli.md` | `daily around 03:00` | `TBD` | Verifies AWF az CLI mount + ADO auth via `AZURE_DEVOPS_EXT_PAT`. | | `noop-target.md` | _no schedule_ | `TBD` | Target of the `queue-build` executor-e2e scenario. Its Pipeline ID must be set as `E2E_QUEUE_PIPELINE_ID` on the executor-e2e pipeline. | | `janitor.md` | `weekly on monday around 02:00` | `TBD` | Prunes `ado-aw-smoke-*` artifacts older than 30 days. | -| `smoke-failure-reporter.md` | `daily around 04:30` | `TBD` | Files `[smoke-failure] …` issues on `githubnext/ado-aw`. Requires the `ADO_AW_DEBUG_GITHUB_TOKEN` secret pipeline variable, **only on this pipeline**. | +| `smoke-failure-reporter.md` | `daily around 04:30` | `TBD` | Files `[smoke-failure] …` issues on `jamesadevine/ado-aw-issues`. Requires the `ADO_AW_DEBUG_GITHUB_TOKEN` secret pipeline variable, **only on this pipeline**. | ## Manual-handoff checklist @@ -30,9 +30,10 @@ the following one-time setup in `githubnext/ado-aw` checkout: ```powershell - ado-aw enable ` + cargo run -- enable ` --org msazuresphere --project AgentPlayground ` - --service-connection ado-aw-github ` + --service-connection github.com_githubnext ` + --also-set-token ` --folder '\smoke' ` tests/safe-outputs/ ``` @@ -49,7 +50,8 @@ the following one-time setup in 5. Provision pipeline variable `ADO_AW_DEBUG_GITHUB_TOKEN` (secret) on the `smoke-failure-reporter` pipeline **only**. Use a GitHub fine-grained PAT scoped to `Issues: Read and write` on - `githubnext/ado-aw` only. + `jamesadevine/ado-aw-issues` only. Confirm the target repository has the + `pipeline-failure` and `ado-aw-smoke` labels. ```powershell ado-aw secrets set ADO_AW_DEBUG_GITHUB_TOKEN ` @@ -65,3 +67,8 @@ the following one-time setup in ```powershell ado-aw run --org msazuresphere --project AgentPlayground tests/safe-outputs/ ``` + +For the AgentPlayground replacement-first migration, create side-by-side +definitions explicitly with `az pipelines create --skip-run true`. +`ado-aw enable` intentionally reuses an existing definition with the same YAML +path, so it cannot create replacements while the legacy definitions remain. diff --git a/tests/safe-outputs/smoke-failure-reporter.lock.yml b/tests/safe-outputs/smoke-failure-reporter.lock.yml index 7a52fade..05659148 100644 --- a/tests/safe-outputs/smoke-failure-reporter.lock.yml +++ b/tests/safe-outputs/smoke-failure-reporter.lock.yml @@ -1,5 +1,5 @@ # This file is auto-generated by ado-aw. Do not edit manually. -# @ado-aw source="tests/safe-outputs/smoke-failure-reporter.md" version=0.44.1 +# @ado-aw source="tests/safe-outputs/smoke-failure-reporter.md" version=0.45.1 name: ado-aw smoke failure reporter-$(BuildID) resources: @@ -39,7 +39,7 @@ jobs: - bash: | set -euo pipefail TARBALL_NAME="copilot-linux-x64.tar.gz" - BASE_URL="https://github.com/github/copilot-cli/releases/download/v1.0.69" + BASE_URL="https://github.com/github/copilot-cli/releases/download/v1.0.70" TARBALL_URL="$BASE_URL/$TARBALL_NAME" CHECKSUMS_URL="$BASE_URL/SHA256SUMS.txt" TOOLS_DIR="$(Agent.TempDirectory)/tools" @@ -77,14 +77,14 @@ jobs: echo "##vso[task.prependpath]$TOOLS_DIR" cp "$TOOLS_DIR/copilot" /tmp/awf-tools/copilot chmod +x /tmp/awf-tools/copilot - displayName: Install Copilot CLI (v1.0.69) + displayName: Install Copilot CLI (v1.0.70) - bash: | copilot --version copilot -h displayName: Output copilot version - bash: | set -eo pipefail - COMPILER_VERSION="0.44.1" + COMPILER_VERSION="0.45.1" DOWNLOAD_DIR="$(Pipeline.Workspace)/agentic-pipeline-compiler" DOWNLOAD_URL="https://github.com/githubnext/ado-aw/releases/download/v${COMPILER_VERSION}/ado-aw-linux-x64" CHECKSUM_URL="https://github.com/githubnext/ado-aw/releases/download/v${COMPILER_VERSION}/checksums.txt" @@ -99,7 +99,7 @@ jobs: grep "ado-aw-linux-x64" checksums.txt | sha256sum -c - mv ado-aw-linux-x64 ado-aw chmod +x ado-aw - displayName: Download agentic pipeline compiler (v0.44.1) + displayName: Download agentic pipeline compiler (v0.45.1) - bash: | AGENTIC_PIPELINES_PATH="$(Pipeline.Workspace)/agentic-pipeline-compiler/ado-aw" chmod +x "$AGENTIC_PIPELINES_PATH" @@ -185,7 +185,7 @@ jobs: - bash: | set -eo pipefail - AWF_VERSION="0.27.28" + AWF_VERSION="0.27.9" DOWNLOAD_DIR="$(Pipeline.Workspace)/awf" DOWNLOAD_URL="https://github.com/github/gh-aw-firewall/releases/download/v${AWF_VERSION}/awf-linux-x64" CHECKSUM_URL="https://github.com/github/gh-aw-firewall/releases/download/v${AWF_VERSION}/checksums.txt" @@ -202,16 +202,16 @@ jobs: chmod +x awf echo "##vso[task.prependpath]$(Pipeline.Workspace)/awf" ./awf --version - displayName: Download AWF (Agentic Workflow Firewall) v0.27.28 + displayName: Download AWF (Agentic Workflow Firewall) v0.27.9 - bash: | set -eo pipefail - docker pull ghcr.io/github/gh-aw-firewall/squid:0.27.28 - docker pull ghcr.io/github/gh-aw-firewall/agent:0.27.28 - docker tag ghcr.io/github/gh-aw-firewall/squid:0.27.28 ghcr.io/github/gh-aw-firewall/squid:latest - docker tag ghcr.io/github/gh-aw-firewall/agent:0.27.28 ghcr.io/github/gh-aw-firewall/agent:latest + docker pull ghcr.io/github/gh-aw-firewall/squid:0.27.9 + docker pull ghcr.io/github/gh-aw-firewall/agent:0.27.9 + docker tag ghcr.io/github/gh-aw-firewall/squid:0.27.9 ghcr.io/github/gh-aw-firewall/squid:latest + docker tag ghcr.io/github/gh-aw-firewall/agent:0.27.9 ghcr.io/github/gh-aw-firewall/agent:latest docker pull ghcr.io/github/gh-aw-mcpg:v0.4.1 - displayName: Pre-pull AWF and MCPG container images (v0.27.28) + displayName: Pre-pull AWF and MCPG container images (v0.27.9) - task: UseNode@1 inputs: version: 22.x @@ -221,11 +221,11 @@ jobs: - bash: | set -eo pipefail mkdir -p /tmp/ado-aw-scripts - curl -fsSL "https://github.com/githubnext/ado-aw/releases/download/v0.44.1/checksums.txt" -o /tmp/ado-aw-scripts/checksums.txt - curl -fsSL "https://github.com/githubnext/ado-aw/releases/download/v0.44.1/ado-script.zip" -o /tmp/ado-aw-scripts/ado-script.zip + curl -fsSL "https://github.com/githubnext/ado-aw/releases/download/v0.45.1/checksums.txt" -o /tmp/ado-aw-scripts/checksums.txt + curl -fsSL "https://github.com/githubnext/ado-aw/releases/download/v0.45.1/ado-script.zip" -o /tmp/ado-aw-scripts/ado-script.zip cd /tmp/ado-aw-scripts && grep "ado-script.zip" checksums.txt | sha256sum -c - unzip -o /tmp/ado-aw-scripts/ado-script.zip -d /tmp/ado-aw-scripts/ - displayName: Download ado-aw scripts (v0.44.1) + displayName: Download ado-aw scripts (v0.45.1) timeoutInMinutes: 5 condition: succeeded() - bash: | @@ -234,15 +234,15 @@ jobs: displayName: Resolve runtime imports (agent prompt) condition: succeeded() - bash: | - # ado-aw-metadata: {"org":"","repo":"","schema":1,"source":"tests/safe-outputs/smoke-failure-reporter.md","target":"standalone","version":"0.44.1"} - echo 'ado-aw metadata: source=tests/safe-outputs/smoke-failure-reporter.md org= repo= version=0.44.1 target=standalone' + # ado-aw-metadata: {"org":"","repo":"","schema":1,"source":"tests/safe-outputs/smoke-failure-reporter.md","target":"standalone","version":"0.45.1"} + echo 'ado-aw metadata: source=tests/safe-outputs/smoke-failure-reporter.md org= repo= version=0.45.1 target=standalone' displayName: ado-aw - bash: | set -eo pipefail mkdir -p "$(Agent.TempDirectory)/staging" cat >"$(Agent.TempDirectory)/staging/aw_info.json" <<'AW_INFO_EOF' - {"agent_name":"ado-aw smoke failure reporter","build_definition_id":"$(System.DefinitionId)","build_id":"$(Build.BuildId)","compiler_version":"0.44.1","engine":"copilot","model":"gpt-5-mini","org":"","repo":"","schema":"ado-aw/aw_info/1","source":"tests/safe-outputs/smoke-failure-reporter.md","source_branch":"$(Build.SourceBranch)","source_version":"$(Build.SourceVersion)","target":"standalone"} + {"agent_name":"ado-aw smoke failure reporter","build_definition_id":"$(System.DefinitionId)","build_id":"$(Build.BuildId)","compiler_version":"0.45.1","engine":"copilot","model":"gpt-5-mini","org":"","repo":"","schema":"ado-aw/aw_info/1","source":"tests/safe-outputs/smoke-failure-reporter.md","source_branch":"$(Build.SourceBranch)","source_version":"$(Build.SourceVersion)","target":"standalone"} AW_INFO_EOF displayName: Emit aw_info.json condition: always() @@ -533,7 +533,7 @@ jobs: - bash: | set -euo pipefail TARBALL_NAME="copilot-linux-x64.tar.gz" - BASE_URL="https://github.com/github/copilot-cli/releases/download/v1.0.69" + BASE_URL="https://github.com/github/copilot-cli/releases/download/v1.0.70" TARBALL_URL="$BASE_URL/$TARBALL_NAME" CHECKSUMS_URL="$BASE_URL/SHA256SUMS.txt" TOOLS_DIR="$(Agent.TempDirectory)/tools" @@ -571,14 +571,14 @@ jobs: echo "##vso[task.prependpath]$TOOLS_DIR" cp "$TOOLS_DIR/copilot" /tmp/awf-tools/copilot chmod +x /tmp/awf-tools/copilot - displayName: Install Copilot CLI (v1.0.69) + displayName: Install Copilot CLI (v1.0.70) - bash: | copilot --version copilot -h displayName: Output copilot version - bash: | set -eo pipefail - COMPILER_VERSION="0.44.1" + COMPILER_VERSION="0.45.1" DOWNLOAD_DIR="$(Pipeline.Workspace)/agentic-pipeline-compiler" DOWNLOAD_URL="https://github.com/githubnext/ado-aw/releases/download/v${COMPILER_VERSION}/ado-aw-linux-x64" CHECKSUM_URL="https://github.com/githubnext/ado-aw/releases/download/v${COMPILER_VERSION}/checksums.txt" @@ -593,7 +593,7 @@ jobs: grep "ado-aw-linux-x64" checksums.txt | sha256sum -c - mv ado-aw-linux-x64 ado-aw chmod +x ado-aw - displayName: Download agentic pipeline compiler (v0.44.1) + displayName: Download agentic pipeline compiler (v0.45.1) - task: DockerInstaller@0 inputs: dockerVersion: 26.1.4 @@ -601,7 +601,7 @@ jobs: - bash: | set -eo pipefail - AWF_VERSION="0.27.28" + AWF_VERSION="0.27.9" DOWNLOAD_DIR="$(Pipeline.Workspace)/awf" DOWNLOAD_URL="https://github.com/github/gh-aw-firewall/releases/download/v${AWF_VERSION}/awf-linux-x64" CHECKSUM_URL="https://github.com/github/gh-aw-firewall/releases/download/v${AWF_VERSION}/checksums.txt" @@ -618,22 +618,22 @@ jobs: chmod +x awf echo "##vso[task.prependpath]$(Pipeline.Workspace)/awf" ./awf --version - displayName: Download AWF (Agentic Workflow Firewall) v0.27.28 + displayName: Download AWF (Agentic Workflow Firewall) v0.27.9 - bash: | set -eo pipefail - docker pull ghcr.io/github/gh-aw-firewall/squid:0.27.28 - docker pull ghcr.io/github/gh-aw-firewall/agent:0.27.28 - docker tag ghcr.io/github/gh-aw-firewall/squid:0.27.28 ghcr.io/github/gh-aw-firewall/squid:latest - docker tag ghcr.io/github/gh-aw-firewall/agent:0.27.28 ghcr.io/github/gh-aw-firewall/agent:latest - displayName: Pre-pull AWF container images (v0.27.28) + docker pull ghcr.io/github/gh-aw-firewall/squid:0.27.9 + docker pull ghcr.io/github/gh-aw-firewall/agent:0.27.9 + docker tag ghcr.io/github/gh-aw-firewall/squid:0.27.9 ghcr.io/github/gh-aw-firewall/squid:latest + docker tag ghcr.io/github/gh-aw-firewall/agent:0.27.9 ghcr.io/github/gh-aw-firewall/agent:latest + displayName: Pre-pull AWF container images (v0.27.9) - bash: | mkdir -p "$(Build.SourcesDirectory)/safe_outputs" cp -a "$(Pipeline.Workspace)/agent_outputs_$(Build.BuildId)/." "$(Build.SourcesDirectory)/safe_outputs" displayName: Prepare safe outputs for analysis - bash: | # Write threat analysis prompt to /tmp (accessible inside AWF container) - cat > "/tmp/awf-tools/threat-analysis-prompt.md" << 'THREAT_ANALYSIS_EOF_6e6028ecc807' + cat > "/tmp/awf-tools/threat-analysis-prompt.md" << 'THREAT_ANALYSIS_EOF_13f2a688652c' # Threat Detection Analysis You are a security analyst tasked with analyzing agent output and code changes for potential security threats. @@ -643,7 +643,7 @@ jobs: The pipeline prompt file is available at: $(Build.SourcesDirectory)/tests/safe-outputs/smoke-failure-reporter.md Load and read this file to understand the intent and context of the pipeline. The pipeline information includes: - pipeline name: ado-aw smoke failure reporter - - pipeline description: Files [smoke-failure] issues on githubnext/ado-aw for failed daily smoke pipelines + - pipeline description: Files [smoke-failure] issues on jamesadevine/ado-aw-issues for failed daily smoke pipelines - Full pipeline instructions and context in the prompt file Use this information to understand the pipeline's intended purpose and legitimate use cases. @@ -671,7 +671,7 @@ jobs: - Focus on actual security risks rather than style issues - If you're uncertain about a potential threat, err on the side of caution - Provide clear, actionable reasons for any threats detected - THREAT_ANALYSIS_EOF_6e6028ecc807 + THREAT_ANALYSIS_EOF_13f2a688652c echo "Threat analysis prompt:" cat "/tmp/awf-tools/threat-analysis-prompt.md" @@ -811,7 +811,7 @@ jobs: artifact: analyzed_outputs_$(Build.BuildId) - bash: | set -eo pipefail - COMPILER_VERSION="0.44.1" + COMPILER_VERSION="0.45.1" DOWNLOAD_DIR="$(Pipeline.Workspace)/agentic-pipeline-compiler" DOWNLOAD_URL="https://github.com/githubnext/ado-aw/releases/download/v${COMPILER_VERSION}/ado-aw-linux-x64" CHECKSUM_URL="https://github.com/githubnext/ado-aw/releases/download/v${COMPILER_VERSION}/checksums.txt" @@ -826,7 +826,7 @@ jobs: grep "ado-aw-linux-x64" checksums.txt | sha256sum -c - mv ado-aw-linux-x64 ado-aw chmod +x ado-aw - displayName: Download agentic pipeline compiler (v0.44.1) + displayName: Download agentic pipeline compiler (v0.45.1) - bash: | ls -la "$(Pipeline.Workspace)/agentic-pipeline-compiler" chmod +x "$(Pipeline.Workspace)/agentic-pipeline-compiler/ado-aw" diff --git a/tests/safe-outputs/smoke-failure-reporter.md b/tests/safe-outputs/smoke-failure-reporter.md index 60bf5b1f..362de409 100644 --- a/tests/safe-outputs/smoke-failure-reporter.md +++ b/tests/safe-outputs/smoke-failure-reporter.md @@ -1,6 +1,6 @@ --- name: "ado-aw smoke failure reporter" -description: "Files [smoke-failure] issues on githubnext/ado-aw for failed daily smoke pipelines" +description: "Files [smoke-failure] issues on jamesadevine/ado-aw-issues for failed daily smoke pipelines" on: schedule: daily around 04:30 target: standalone @@ -15,7 +15,7 @@ permissions: write: agent-playground-write ado-aw-debug: create-issue: - target-repo: githubnext/ado-aw + target-repo: jamesadevine/ado-aw-issues title-prefix: "[smoke-failure] " labels: - pipeline-failure @@ -33,8 +33,8 @@ suite running in the AgentPlayground ADO project. Query only these two pipelines (matched by exact `definition.name`): -- `Daily safe-output smoke: canary` -- `Daily smoke: az CLI access` +- `Daily safe-output smoke canary` +- `Daily smoke az CLI access` ### Tasks @@ -44,7 +44,7 @@ Query only these two pipelines (matched by exact `definition.name`): `SYSTEM_ACCESSTOKEN`-equivalent bearer token already available to you in the agent environment. 2. For every run with `result != "succeeded"`: - 1. Search open issues on `githubnext/ado-aw` for one whose title + 1. Search open issues on `jamesadevine/ado-aw-issues` for one whose title starts with `[smoke-failure] `. If one already exists, skip this pipeline. 2. Otherwise, call the `create-issue` safe output **exactly once @@ -68,7 +68,7 @@ Query only these two pipelines (matched by exact `definition.name`): `report-incomplete` for the remainder. - Do **not** call `create-issue` with a `target_repo` parameter. The agent has no override; the target is fixed by the operator at - `githubnext/ado-aw`. + `jamesadevine/ado-aw-issues`. - The `ADO_AW_DEBUG_GITHUB_TOKEN` PAT is not visible to you. Stage 3 uses it to authenticate against GitHub. diff --git a/tests/trigger-e2e/README.md b/tests/trigger-e2e/README.md index 4b558466..d4b4e6ea 100644 --- a/tests/trigger-e2e/README.md +++ b/tests/trigger-e2e/README.md @@ -26,8 +26,10 @@ This suite exercises them with **no LLM and no agent** in the loop: 4. asserts the observable gate decision (**build tags + build result**), 5. cleans up every object it created, -and, on any failure, files a GitHub issue on `githubnext/ado-aw` and fails the -build. +and, on any failure, files a GitHub issue on the configured issue repository +and fails the build. AgentPlayground currently uses +`jamesadevine/ado-aw-issues` because a canonical-repository credential is not +available. ## What's here @@ -94,6 +96,19 @@ Every object a scenario creates is prefixed `ado-aw-trig-$(Build.BuildId)-`. Cleanup (abandon PR + delete branch) runs unconditionally after each scenario; the smoke-suite janitor (which prunes `ado-aw-*` artifacts) is the backstop. +## Mirror synchronization + +The victim's `self` repository must be Azure Repos, while the orchestrator is +registered against GitHub. Before running scenarios, the orchestrator pushes +its checked-out `main` commit to `ado-aw-mirror` and verifies the remote SHA. +The push is authenticated with the `agent-playground-write` AAD token through +an environment-only git bearer header. + +Synchronization is fast-forward-only and fail-closed: a divergent mirror stops +the suite before any victim build is queued. No force push is used. When +`TRIGGER_E2E_VICTIM_REPO` is unset, synchronization is skipped and the +existing bypass-only baseline still runs. + ## Running locally You need a write-capable ADO token (PAT) and a registered victim pipeline id: @@ -107,7 +122,9 @@ export SYSTEM_ACCESSTOKEN="" export TRIGGER_E2E_VICTIM_DEFINITION_ID="" export TRIGGER_E2E_VICTIM_REPO="" # Optional: -# export TRIGGER_E2E_GITHUB_TOKEN="" +# export TRIGGER_E2E_GITHUB_TOKEN="" +# export TRIGGER_E2E_ISSUE_REPO="jamesadevine/ado-aw-issues" +# export TRIGGER_E2E_SYNC_MIRROR="true" # requires a full main checkout # export TRIGGER_E2E_BUILD_TIMEOUT_MS=900000 # per victim build (default 900000) # export TRIGGER_E2E_BUILD_POLL_MS=10000 # poll interval (default 10000) @@ -122,8 +139,12 @@ fails. In `https://dev.azure.com/msazuresphere/AgentPlayground`: -1. **Register the VICTIM pipeline.** New pipeline → Azure Repos → the ado-aw - Azure Repos mirror → existing YAML → `tests/trigger-e2e/victim-pipeline.yml`. +1. **Create and seed `ado-aw-mirror`.** Import or push the merged + `githubnext/ado-aw` `main` commit into a dedicated Azure Repo. Do not reuse + `agent-definitions`. +2. **Register the VICTIM pipeline.** New pipeline → Azure Repos → + `ado-aw-mirror` → existing YAML → + `tests/trigger-e2e/victim-pipeline.yml`. It **must** be an ADO Git repo (not GitHub) so `exec-context-pr-synth` can discover open PRs in the pipeline's own `self` repo. Note its **definition id** for the next step. @@ -131,32 +152,29 @@ In `https://dev.azure.com/msazuresphere/AgentPlayground`: hardcode `pool: { name: AZS-1ES-L-Playground-ubuntu-22.04 }`. If your ADO project uses a different agent pool, edit both files to point at a Linux pool with Node.js 20 available before registering. -2. **Register the ORCHESTRATOR pipeline.** New pipeline → existing YAML → - `tests/trigger-e2e/azure-pipelines.yml`. Place both in a `\trigger-e2e` - folder. -3. **Wire the victim id + repo** as pipeline/definition variables on the +3. **Register the ORCHESTRATOR pipeline.** New pipeline → GitHub through + `github.com_githubnext` → existing YAML → + `tests/trigger-e2e/azure-pipelines.yml`. Place both definitions in a + `\trigger-e2e` folder and skip the first run until variables are configured. +4. **Wire the victim id + repo** as non-secret definition variables on the orchestrator: - ```powershell - ado-aw secrets set TRIGGER_E2E_VICTIM_DEFINITION_ID ` - --org msazuresphere --project AgentPlayground ` - --definition-ids --value - ado-aw secrets set TRIGGER_E2E_VICTIM_REPO ` - --org msazuresphere --project AgentPlayground ` - --definition-ids --value - ``` - (These are not secrets; `secrets set` is just the variable-management path.) -4. **Grant the VICTIM build identity** Code (read) on the repo, Build (add + - `TRIGGER_E2E_VICTIM_DEFINITION_ID=` + - `TRIGGER_E2E_VICTIM_REPO=ado-aw-mirror` +5. **Grant the VICTIM build identity** Code (read) on the repo, Build (add tags / edit build quality), and permission to cancel builds (gate self-cancel). See [`docs/safe-output-permissions.md`](../../docs/safe-output-permissions.md) if it hits 401/403. -5. **Grant the ORCHESTRATOR build identity** Contribute / Create branch / - Contribute to PRs on the victim repo (it creates transient PRs) and Queue - builds on the victim definition. -6. **Set the GitHub PAT secret** on the orchestrator only: +6. **Grant the principal behind `agent-playground-write`** Contribute, Create + branch, delete refs, and Contribute to PRs on `ado-aw-mirror`, plus Queue + builds on the victim definition. This same principal performs mirror sync + and creates the transient PR context. +7. **Set the GitHub PAT secret** on the orchestrator only: ```powershell ado-aw secrets set TRIGGER_E2E_GITHUB_TOKEN ` --org msazuresphere --project AgentPlayground ` --definition-ids ` - --value + --value ``` -7. **Trigger one manual orchestrator run** to seed the schedule. +8. Set `TRIGGER_E2E_ISSUE_REPO=jamesadevine/ado-aw-issues`. Confirm the target + has `trigger-e2e-failure` and `pipeline-failure` labels. +9. **Trigger one manual orchestrator run from `main`** to seed the schedule. diff --git a/tests/trigger-e2e/azure-pipelines.yml b/tests/trigger-e2e/azure-pipelines.yml index eead6b22..bdd4ff83 100644 --- a/tests/trigger-e2e/azure-pipelines.yml +++ b/tests/trigger-e2e/azure-pipelines.yml @@ -4,7 +4,8 @@ # AgentPlayground. The harness creates real PRs and queues the registered # VICTIM pipeline (tests/trigger-e2e/victim-pipeline.yml) under a battery of # trigger conditions, asserting the gate decision via build tags + result. On -# any failure it files a GitHub issue on githubnext/ado-aw and fails the build. +# any failure it files a GitHub issue on the configured issue repository and +# fails the build. # # Unlike the executor-e2e orchestrator, this does NOT build the Rust `ado-aw` # binary: trigger conditions are evaluated entirely by the ado-script bundles @@ -37,11 +38,13 @@ variables: # (definition) variable. Set TRIGGER_E2E_ISSUE_REPO as a pipeline/definition # variable to redirect issue filing; the harness defaults to githubnext/ado-aw. # TRIGGER_E2E_GITHUB_TOKEN must be provided as a SECRET pipeline variable - # (fine-grained PAT scoped to Issues:read/write on githubnext/ado-aw ONLY). + # scoped to Issues:read/write on the configured issue repository. steps: - checkout: self - fetchDepth: 1 + # Mirror synchronization is fast-forward-only and therefore needs enough + # history to prove ancestry against the current ADO mirror main. + fetchDepth: 0 displayName: Checkout ado-aw - task: UseNode@1 @@ -80,6 +83,7 @@ steps: SYSTEM_ACCESSTOKEN: $(SC_WRITE_TOKEN) TRIGGER_E2E_VICTIM_DEFINITION_ID: $(EFFECTIVE_TRIGGER_E2E_VICTIM_DEFINITION_ID) TRIGGER_E2E_VICTIM_REPO: $(EFFECTIVE_TRIGGER_E2E_VICTIM_REPO) + TRIGGER_E2E_SYNC_MIRROR: "true" TRIGGER_E2E_ISSUE_REPO: $(TRIGGER_E2E_ISSUE_REPO) - # Secret PAT for filing failure issues (scoped to githubnext/ado-aw Issues). + # Secret PAT for filing failure issues on the configured repository. TRIGGER_E2E_GITHUB_TOKEN: $(TRIGGER_E2E_GITHUB_TOKEN)