diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 68a9ea5875..93001165e3 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -225,26 +225,31 @@ jobs: expected_run_url="https://github.com/${GITHUB_REPOSITORY}/actions/runs/${RUN_ID}" jobs_summary="${JOBS//,/, }" targets_summary="${TARGETS//,/, }" - expected_summary="Risk plan ${PLAN_HASH} selected jobs: ${jobs_summary:-none}; targets: ${targets_summary:-none}." + expected_summary="Risk plan ${PLAN_HASH} selected jobs: ${jobs_summary:-none}; targets: ${targets_summary:-none}. Child run: ${expected_run_url}." + controller_auth_max_attempts=45 + controller_auth_poll_seconds=2 check_json="" - for _attempt in {1..15}; do + for ((_attempt = 1; _attempt <= controller_auth_max_attempts; _attempt++)); do if check_json="$(curl --fail --silent --show-error --proto '=https' \ --header "Authorization: Bearer ${GITHUB_TOKEN}" \ --header "Accept: application/vnd.github+json" \ + --header "Cache-Control: no-cache" \ + --header "Pragma: no-cache" \ --header "X-GitHub-Api-Version: 2022-11-28" \ "https://api.github.com/repos/${GITHUB_REPOSITORY}/check-runs/${CONTROLLER_CHECK_ID}")" && - [[ "$(jq -r '.details_url // ""' <<< "$check_json")" == "$expected_run_url" ]]; then + [[ "$(jq -r '.output.summary // ""' <<< "$check_json")" == "$expected_summary" ]]; then break fi check_json="" - sleep 2 + if (( _attempt < controller_auth_max_attempts )); then + sleep "$controller_auth_poll_seconds" + fi done [[ -n "$check_json" ]] || { echo "::error::trusted controller authorization was not published for this run"; exit 1; } jq -e \ --argjson check_id "$CONTROLLER_CHECK_ID" \ --arg external_id "$expected_external_id" \ --arg head_sha "$CHECKOUT_SHA" \ - --arg run_url "$expected_run_url" \ --arg summary "$expected_summary" \ ' .id == $check_id and @@ -255,7 +260,6 @@ jobs: .external_id == $external_id and .status == "in_progress" and .conclusion == null and - .details_url == $run_url and .output.summary == $summary ' <<< "$check_json" >/dev/null || { echo "::error::controller check does not authorize this exact E2E run"; exit 1; } @@ -1457,6 +1461,11 @@ jobs: - *dockerhub-auth - name: Prepare E2E workspace uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75 + - name: Initialize runner comparison telemetry + if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && inputs.checkout_sha == '' }} + continue-on-error: true + shell: bash + run: npx tsx tools/e2e/runner-comparison.mts initialize - name: Install OpenShell CLI run: bash scripts/install-openshell.sh - name: Run agent turn latency live Vitest test @@ -1479,6 +1488,11 @@ jobs: echo "Using OPENSHELL_BIN=$OPENSHELL_BIN" "$OPENSHELL_BIN" --version npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/agent-turn-latency.test.ts + - name: Finalize runner comparison telemetry + if: ${{ always() && github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && inputs.checkout_sha == '' }} + continue-on-error: true + shell: bash + run: npx tsx tools/e2e/runner-comparison.mts finalize - name: Upload agent turn latency artifacts if: always() uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 @@ -5157,6 +5171,12 @@ jobs: - name: Prepare E2E workspace uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75 + - name: Initialize runner comparison telemetry + if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && inputs.checkout_sha == '' && matrix.agent == 'hermes' }} + continue-on-error: true + shell: bash + run: npx tsx tools/e2e/runner-comparison.mts initialize + - name: Run Bedrock Runtime compatible Anthropic live test # Direct Vitest coverage for # Preserves the @@ -5167,6 +5187,12 @@ jobs: set -euo pipefail npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/bedrock-runtime-compatible-anthropic.test.ts + - name: Finalize runner comparison telemetry + if: ${{ always() && github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && inputs.checkout_sha == '' && matrix.agent == 'hermes' }} + continue-on-error: true + shell: bash + run: npx tsx tools/e2e/runner-comparison.mts finalize + - name: Upload Bedrock Runtime compatible Anthropic artifacts if: always() uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 diff --git a/test/e2e/README.md b/test/e2e/README.md index 9546fed156..eefa4f5445 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -181,9 +181,11 @@ graph as the live targets: ### Runner comparison telemetry Trusted `main` runs without an alternate checkout SHA record runner-comparison -telemetry for the #7145 contract: 12 routed workflow lane identities / 15 +telemetry for 14 routed workflow lane identities / 17 concrete job executions. +- `agent-turn-latency`, spanning its sequential OpenClaw and Hermes setup +- `bedrock-runtime-compatible-anthropic` with the `hermes` shard - `common-egress-agent` with the `openclaw-balanced-weather`, `openclaw-open-reference`, and `hermes-open-reference` shards - `rebuild-hermes` @@ -201,7 +203,12 @@ concrete job executions. The three extra executions come from `common-egress-agent`, which runs three scenario shards, and `hermes-inference-switch`, which runs both listed modes. The OpenClaw matrix entries for `mcp-bridge`, -`channels-stop-start`, and `security-posture` are not instrumented. +`channels-stop-start`, `security-posture`, and +`bedrock-runtime-compatible-anthropic` are not instrumented. +The #7145 standard-versus-larger-runner cohort compares the same lane and +equivalent workload while varying the runner class. The newly instrumented +`agent-turn-latency` and Bedrock Hermes lanes extend diagnostic coverage; this +change does not route them to a larger runner. Each execution writes one bounded, ordered v2 time series to the canonical `runner-comparison.jsonl` ledger. It contains: @@ -520,9 +527,13 @@ and dispatches the selected jobs and targets in one workflow run. The child workflow receives the controller-owned coordination check ID. Before checking out the PR revision, it requires a GitHub Actions dispatch and verifies that the exact check is owned by the GitHub Actions app, matches the PR head and base -identity, names the selected plan, and links to the current child run. A direct -manual dispatch that supplies otherwise-valid PR inputs cannot forge that -one-run authorization and fails before checkout. +identity, names the selected plan, and binds the exact current child Actions run +in the controller-owned output summary. The child requests uncached check-run +state up to 45 times with two-second delays, which spans GitHub's 60-second +check cache, before failing closed. It does not use the check details URL for +this binding because GitHub may canonicalize that field to the check's own +`/runs/` URL. A direct manual dispatch that supplies otherwise-valid +PR inputs cannot forge that one-run authorization and fails before checkout. The manual maintainer path remains available as a fallback. A repository maintainer or administrator chooses **Run workflow** on `main`, selects diff --git a/test/e2e/support/e2e-operations-workflow-boundary.test.ts b/test/e2e/support/e2e-operations-workflow-boundary.test.ts index 8c2e4c633c..fcb3f52a72 100644 --- a/test/e2e/support/e2e-operations-workflow-boundary.test.ts +++ b/test/e2e/support/e2e-operations-workflow-boundary.test.ts @@ -146,7 +146,6 @@ describe("E2E operations workflow boundary", () => { "Controller authentication must bind CONTROLLER_CHECK_ID", 'Controller authentication must retain "$ACTOR" == "github-actions[bot]"', "Controller authentication must retain .app.id == 15368", - "Controller authentication must retain .details_url == $run_url", "Controller validation must be activated only by checkout_sha", "Controller validation must bind BASE_SHA", "Controller validation must bind CHECKOUT_REPOSITORY", @@ -179,6 +178,48 @@ describe("E2E operations workflow boundary", () => { ); }); + it("requires fresh controller state for longer than GitHub's check cache (#7140)", () => { + const workflow = readE2eOperationsWorkflow(); + const authentication = workflow.jobs["generate-matrix"].steps!.find( + (step) => step.name === "Authenticate controller dispatch", + )!; + authentication.run = String(authentication.run) + .replace('--header "Cache-Control: no-cache"', '--header "Cache-Control: max-age=60"') + .replace(" Child run: ${expected_run_url}.", "") + .replace("controller_auth_max_attempts=45", "controller_auth_max_attempts=15"); + + expect(validateE2eOperationsWorkflow(workflow)).toEqual( + expect.arrayContaining([ + 'Controller authentication must retain --header "Cache-Control: no-cache"', + "Controller authentication must retain Child run: ${expected_run_url}.", + "Controller authentication polling window must exceed GitHub's 60-second check cache", + ]), + ); + }); + + it.each([ + [ + "attempt count", + "controller_auth_max_attempts=45", + "controller_auth_max_attempts=46", + "Controller authentication must use exactly 45 polling attempts", + ], + [ + "poll interval", + "controller_auth_poll_seconds=2", + "controller_auth_poll_seconds=3", + "Controller authentication must use two-second polling intervals", + ], + ])("rejects controller %s drift (#7140)", (_name, current, replacement, expectedError) => { + const workflow = readE2eOperationsWorkflow(); + const authentication = workflow.jobs["generate-matrix"].steps!.find( + (step) => step.name === "Authenticate controller dispatch", + )!; + authentication.run = String(authentication.run).replace(current, replacement); + + expect(validateE2eOperationsWorkflow(workflow)).toContain(expectedError); + }); + it("fails a valid-looking manual fork dispatch before controller API authentication", () => { const workflow = readE2eOperationsWorkflow(); const authentication = workflow.jobs["generate-matrix"].steps!.find( @@ -214,7 +255,7 @@ describe("E2E operations workflow boundary", () => { expect(result.stderr).not.toContain("curl:"); }); - it("accepts a controller check bound to the exact child run and selected plan", () => { + it("accepts a summary-bound child when GitHub canonicalizes the details URL (#7140)", () => { const workflow = readE2eOperationsWorkflow(); const authentication = workflow.jobs["generate-matrix"].steps!.find( (step) => step.name === "Authenticate controller dispatch", @@ -230,9 +271,9 @@ describe("E2E operations workflow boundary", () => { external_id: `nemoclaw-pr-e2e:v2:42:${headSha}:${baseSha}`, status: "in_progress", conclusion: null, - details_url: "https://github.com/NVIDIA/NemoClaw/actions/runs/23", + details_url: "https://github.com/NVIDIA/NemoClaw/runs/17", output: { - summary: `Risk plan ${planHash} selected jobs: credential-sanitization; targets: none.`, + summary: `Risk plan ${planHash} selected jobs: credential-sanitization; targets: none. Child run: https://github.com/NVIDIA/NemoClaw/actions/runs/23.`, }, }); const result = spawnSync( diff --git a/test/e2e/support/runner-comparison-workflow-boundary.test.ts b/test/e2e/support/runner-comparison-workflow-boundary.test.ts index beb3a75e5e..a6b8dcb568 100644 --- a/test/e2e/support/runner-comparison-workflow-boundary.test.ts +++ b/test/e2e/support/runner-comparison-workflow-boundary.test.ts @@ -29,6 +29,8 @@ type Workflow = { }; const JOBS = [ + "agent-turn-latency", + "bedrock-runtime-compatible-anthropic", "channels-stop-start", "common-egress-agent", "hermes-dashboard", @@ -59,8 +61,8 @@ function telemetrySteps(workflow: Workflow, jobId: string): WorkflowStep[] { ); } -describe("runner comparison E2E workflow boundary (#7145)", () => { - it("accepts 12 routed workflow lane identities / 15 concrete job executions", () => { +describe("runner comparison E2E workflow boundary (#7140)", () => { + it("accepts 14 routed workflow lane identities / 17 concrete job executions", () => { const workflow = loadWorkflow(); expect(validateRunnerComparisonWorkflowBoundary(workflow)).toEqual([]); @@ -86,12 +88,16 @@ describe("runner comparison E2E workflow boundary (#7145)", () => { const routedLanes = JOBS.length - 1 + mcpLanes; const concreteExecutions = routedLanes + commonEgressScenarios!.length - 1 + inferenceSwitchModes!.length - 1; - expect(routedLanes).toBe(12); - expect(concreteExecutions).toBe(15); + expect(routedLanes).toBe(14); + expect(concreteExecutions).toBe(17); }); - it("locks the matrix topology that produces fifteen concrete executions", () => { + it("locks the matrix topology that produces seventeen concrete executions", () => { const workflow = loadWorkflow(); + workflow.jobs["bedrock-runtime-compatible-anthropic"]!.strategy!.matrix!.agent = [ + "openclaw", + "openclaw", + ]; workflow.jobs["mcp-bridge"]!.strategy!.matrix!.agent = ["openclaw", "hermes", "hermes"]; workflow.jobs["channels-stop-start"]!.strategy!.matrix!.agent = ["openclaw", "openclaw"]; workflow.jobs["common-egress-agent"]!.strategy!.matrix!.include = [ @@ -110,6 +116,7 @@ describe("runner comparison E2E workflow boundary (#7145)", () => { expect(validateRunnerComparisonWorkflow(workflow)).toEqual( expect.arrayContaining([ + "bedrock-runtime-compatible-anthropic matrix must contain exactly openclaw, hermes for runner comparison telemetry", "channels-stop-start matrix must contain exactly openclaw, hermes for runner comparison telemetry", "common-egress-agent matrix must contain exactly openclaw-balanced-weather, openclaw-open-reference, hermes-open-reference for runner comparison telemetry", "mcp-bridge matrix must contain exactly openclaw, hermes, deepagents for runner comparison telemetry", @@ -119,7 +126,7 @@ describe("runner comparison E2E workflow boundary (#7145)", () => { ); }); - it("rejects runner comparison consumers outside the eleven comparison jobs", () => { + it("rejects runner comparison consumers outside the thirteen comparison jobs", () => { const workflow = loadWorkflow(); workflow.jobs["shields-config"]!.steps.push( structuredClone(telemetrySteps(workflow, "common-egress-agent")[0]!), @@ -236,6 +243,11 @@ describe("runner comparison E2E workflow boundary (#7145)", () => { "(matrix.agent == 'openclaw' || matrix.agent == 'hermes')", ); } + const bedrockComparison = step(workflow, "bedrock-runtime-compatible-anthropic", name); + bedrockComparison.if = bedrockComparison.if!.replace( + "matrix.agent == 'hermes'", + "(matrix.agent == 'openclaw' || matrix.agent == 'hermes')", + ); } expect(validateRunnerComparisonWorkflow(workflow)).toEqual( @@ -246,6 +258,8 @@ describe("runner comparison E2E workflow boundary (#7145)", () => { "channels-stop-start must use the exact always-run trusted finalize telemetry step", "security-posture must use the exact trusted initialize telemetry step", "security-posture must use the exact always-run trusted finalize telemetry step", + "bedrock-runtime-compatible-anthropic must use the exact trusted initialize telemetry step", + "bedrock-runtime-compatible-anthropic must use the exact always-run trusted finalize telemetry step", ]), ); }); diff --git a/test/pr-e2e-gate.test.ts b/test/pr-e2e-gate.test.ts index 7834ad251f..f7e7cddbd1 100644 --- a/test/pr-e2e-gate.test.ts +++ b/test/pr-e2e-gate.test.ts @@ -1250,6 +1250,21 @@ describe("PR E2E controller", () => { expect(outputs).toContain("dispatched=true"); expect(outputs).not.toContain("approval_mode="); expect(outputs).not.toContain("finalized=true"); + const runningUpdate = requests.find( + (request) => + request.url.endsWith("/check-runs/17") && + request.method === "PATCH" && + (request.body as { output?: { title?: string } }).output?.title === + "Running 3 E2E checks", + ); + expect(runningUpdate?.body).toMatchObject({ + details_url: "https://github.com/NVIDIA/NemoClaw/actions/runs/23", + output: { + summary: expect.stringContaining( + "Child run: https://github.com/NVIDIA/NemoClaw/actions/runs/23.", + ), + }, + }); } finally { fs.rmSync(workDir, { recursive: true, force: true }); } diff --git a/tools/e2e/operations-workflow-boundary.mts b/tools/e2e/operations-workflow-boundary.mts index c76acb4cbd..cbdf49ce5a 100644 --- a/tools/e2e/operations-workflow-boundary.mts +++ b/tools/e2e/operations-workflow-boundary.mts @@ -188,20 +188,44 @@ function validateControllerAuthorization( "nemoclaw-pr-e2e:v2:${PR_NUMBER}:${CHECKOUT_SHA}:${BASE_SHA}", "https://github.com/${GITHUB_REPOSITORY}/actions/runs/${RUN_ID}", "https://api.github.com/repos/${GITHUB_REPOSITORY}/check-runs/${CONTROLLER_CHECK_ID}", - `[[ "$(jq -r '.details_url // ""' <<< "$check_json")" == "$expected_run_url" ]]`, + '--header "Cache-Control: no-cache"', + "Child run: ${expected_run_url}.", + `[[ "$(jq -r '.output.summary // ""' <<< "$check_json")" == "$expected_summary" ]]`, '.name == "E2E / PR Gate Coordination"', ".app.id == 15368", '.app.slug == "github-actions"', ".external_id == $external_id", '.status == "in_progress"', ".conclusion == null", - ".details_url == $run_url", ".output.summary == $summary", ]) { if (!source.includes(fragment)) { errors.push(`Controller authentication must retain ${fragment}`); } } + const maxAttempts = Number.parseInt( + source.match(/controller_auth_max_attempts=(\d+)/u)?.[1] ?? "", + 10, + ); + const pollSeconds = Number.parseInt( + source.match(/controller_auth_poll_seconds=(\d+)/u)?.[1] ?? "", + 10, + ); + if (maxAttempts !== 45) { + errors.push("Controller authentication must use exactly 45 polling attempts"); + } + if (pollSeconds !== 2) { + errors.push("Controller authentication must use two-second polling intervals"); + } + if ( + !Number.isSafeInteger(maxAttempts) || + !Number.isSafeInteger(pollSeconds) || + (maxAttempts - 1) * pollSeconds <= 60 + ) { + errors.push( + "Controller authentication polling window must exceed GitHub's 60-second check cache", + ); + } } function validatePrGateDispatch(errors: string[], workflow: OperationsWorkflow): void { diff --git a/tools/e2e/pr-e2e-gate.mts b/tools/e2e/pr-e2e-gate.mts index 5f40bfbad5..b365bef56c 100755 --- a/tools/e2e/pr-e2e-gate.mts +++ b/tools/e2e/pr-e2e-gate.mts @@ -1467,7 +1467,7 @@ async function updateRunningCheck( const childRunUrl = `https://github.com/${context.repository}/actions/runs/${options.childRunId}`; const selectionCount = options.jobs.length + options.targets.length; const title = `Running ${selectionCount} E2E ${selectionCount === 1 ? "check" : "checks"}`; - const summary = `Risk plan ${options.planHash} selected jobs: ${options.jobs.join(", ") || "none"}; targets: ${options.targets.join(", ") || "none"}.`; + const summary = `Risk plan ${options.planHash} selected jobs: ${options.jobs.join(", ") || "none"}; targets: ${options.targets.join(", ") || "none"}. Child run: ${childRunUrl}.`; const check = await githubApi( `repos/${context.repository}/check-runs/${context.checkRunId}`, token, diff --git a/tools/e2e/runner-comparison-workflow-boundary.mts b/tools/e2e/runner-comparison-workflow-boundary.mts index df2e87d308..356054a4d2 100644 --- a/tools/e2e/runner-comparison-workflow-boundary.mts +++ b/tools/e2e/runner-comparison-workflow-boundary.mts @@ -22,6 +22,14 @@ const MCP_INITIALIZE_GUARD = `\${{ ${TRUSTED_MAIN_GUARD} && ${MCP_AGENT_GUARD} } const MCP_FINALIZE_GUARD = `\${{ always() && ${TRUSTED_MAIN_GUARD} && ${MCP_AGENT_GUARD} }}`; const COMPARISON_JOBS: ReadonlyMap = new Map([ + [ + "agent-turn-latency", + { initializeIf: ORDINARY_INITIALIZE_GUARD, finalizeIf: ORDINARY_FINALIZE_GUARD }, + ], + [ + "bedrock-runtime-compatible-anthropic", + { initializeIf: HERMES_INITIALIZE_GUARD, finalizeIf: HERMES_FINALIZE_GUARD }, + ], [ "channels-stop-start", { initializeIf: HERMES_INITIALIZE_GUARD, finalizeIf: HERMES_FINALIZE_GUARD }, @@ -136,16 +144,20 @@ function publicationIndex(jobSteps: readonly WorkflowStep[]): number { } /** - * Keep the #7145 comparison to 12 routed workflow lane identities / 15 - * concrete trusted-main job executions. Telemetry is best-effort, but it must - * span the complete stable-capacity job and finish before evidence is scanned - * or uploaded. Rebuild jobs establish their fixed swap capacity first because - * the v2 ledger rejects capacity changes after initialization. + * Keep runner diagnostics to 14 routed workflow lane identities / 17 concrete + * trusted-main job executions. Telemetry is best-effort, but it must span the + * complete stable-capacity job and finish before evidence is scanned or + * uploaded. Rebuild jobs establish their fixed swap capacity first because the + * v2 ledger rejects capacity changes after initialization. */ export function validateRunnerComparisonWorkflow(workflowValue: unknown): string[] { const jobs = record(record(workflowValue).jobs); const errors: string[] = []; + requireExactMatrixValues(errors, jobs, "bedrock-runtime-compatible-anthropic", "agent", [ + "openclaw", + "hermes", + ]); requireExactMatrixValues(errors, jobs, "channels-stop-start", "agent", ["openclaw", "hermes"]); requireExactMatrixValues(errors, jobs, "common-egress-agent", "scenario", [ "openclaw-balanced-weather",