ci(e2e): extend Hermes runner telemetry

.github/workflows/e2e.yaml

@@ -225,26 +225,31 @@ jobs:
           expected_run_url="https://github.com/${GITHUB_REPOSITORY}/actions/runs/${RUN_ID}"
           jobs_summary="${JOBS//,/, }"
           targets_summary="${TARGETS//,/, }"
-          expected_summary="Risk plan ${PLAN_HASH} selected jobs: ${jobs_summary:-none}; targets: ${targets_summary:-none}."
+          expected_summary="Risk plan ${PLAN_HASH} selected jobs: ${jobs_summary:-none}; targets: ${targets_summary:-none}. Child run: ${expected_run_url}."
+          controller_auth_max_attempts=45
+          controller_auth_poll_seconds=2
           check_json=""
-          for _attempt in {1..15}; do
+          for ((_attempt = 1; _attempt <= controller_auth_max_attempts; _attempt++)); do
             if check_json="$(curl --fail --silent --show-error --proto '=https' \
               --header "Authorization: Bearer ${GITHUB_TOKEN}" \
               --header "Accept: application/vnd.github+json" \
+              --header "Cache-Control: no-cache" \
+              --header "Pragma: no-cache" \
               --header "X-GitHub-Api-Version: 2022-11-28" \
               "https://api.github.com/repos/${GITHUB_REPOSITORY}/check-runs/${CONTROLLER_CHECK_ID}")" &&
-              [[ "$(jq -r '.details_url // ""' <<< "$check_json")" == "$expected_run_url" ]]; then
+              [[ "$(jq -r '.output.summary // ""' <<< "$check_json")" == "$expected_summary" ]]; then
               break
             fi
             check_json=""
-            sleep 2
+            if (( _attempt < controller_auth_max_attempts )); then
+              sleep "$controller_auth_poll_seconds"
+            fi
           done
           [[ -n "$check_json" ]] || { echo "::error::trusted controller authorization was not published for this run"; exit 1; }
           jq -e \
             --argjson check_id "$CONTROLLER_CHECK_ID" \
             --arg external_id "$expected_external_id" \
             --arg head_sha "$CHECKOUT_SHA" \
-            --arg run_url "$expected_run_url" \
             --arg summary "$expected_summary" \
             '
               .id == $check_id and
@@ -255,7 +260,6 @@ jobs:
               .external_id == $external_id and
               .status == "in_progress" and
               .conclusion == null and
-              .details_url == $run_url and
               .output.summary == $summary
             ' <<< "$check_json" >/dev/null ||
             { echo "::error::controller check does not authorize this exact E2E run"; exit 1; }
@@ -1457,6 +1461,11 @@ jobs:
       - *dockerhub-auth
       - name: Prepare E2E workspace
         uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75
+      - name: Initialize runner comparison telemetry
+        if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && inputs.checkout_sha == '' }}
+        continue-on-error: true
+        shell: bash
+        run: npx tsx tools/e2e/runner-comparison.mts initialize
       - name: Install OpenShell CLI
         run: bash scripts/install-openshell.sh
       - name: Run agent turn latency live Vitest test
@@ -1479,6 +1488,11 @@ jobs:
           echo "Using OPENSHELL_BIN=$OPENSHELL_BIN"
           "$OPENSHELL_BIN" --version
           npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/agent-turn-latency.test.ts
+      - name: Finalize runner comparison telemetry
+        if: ${{ always() && github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && inputs.checkout_sha == '' }}
+        continue-on-error: true
+        shell: bash
+        run: npx tsx tools/e2e/runner-comparison.mts finalize
       - name: Upload agent turn latency artifacts
         if: always()
         uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57
@@ -5157,6 +5171,12 @@ jobs:
       - name: Prepare E2E workspace
         uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75
 
+      - name: Initialize runner comparison telemetry
+        if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && inputs.checkout_sha == '' && matrix.agent == 'hermes' }}
+        continue-on-error: true
+        shell: bash
+        run: npx tsx tools/e2e/runner-comparison.mts initialize
+
       - name: Run Bedrock Runtime compatible Anthropic live test
         # Direct Vitest coverage for
         # Preserves the
@@ -5167,6 +5187,12 @@ jobs:
           set -euo pipefail
           npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/bedrock-runtime-compatible-anthropic.test.ts
 
+      - name: Finalize runner comparison telemetry
+        if: ${{ always() && github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && inputs.checkout_sha == '' && matrix.agent == 'hermes' }}
+        continue-on-error: true
+        shell: bash
+        run: npx tsx tools/e2e/runner-comparison.mts finalize
+
       - name: Upload Bedrock Runtime compatible Anthropic artifacts
         if: always()
         uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57

test/e2e/README.md

@@ -181,9 +181,11 @@ graph as the live targets:
 ### Runner comparison telemetry
 
 Trusted `main` runs without an alternate checkout SHA record runner-comparison
-telemetry for the #7145 contract: 12 routed workflow lane identities / 15
+telemetry for 14 routed workflow lane identities / 17
 concrete job executions.
 
+- `agent-turn-latency`, spanning its sequential OpenClaw and Hermes setup
+- `bedrock-runtime-compatible-anthropic` with the `hermes` shard
 - `common-egress-agent` with the `openclaw-balanced-weather`,
   `openclaw-open-reference`, and `hermes-open-reference` shards
 - `rebuild-hermes`
@@ -201,7 +203,12 @@ concrete job executions.
 The three extra executions come from `common-egress-agent`, which runs three
 scenario shards, and `hermes-inference-switch`, which runs both listed modes.
 The OpenClaw matrix entries for `mcp-bridge`,
-`channels-stop-start`, and `security-posture` are not instrumented.
+`channels-stop-start`, `security-posture`, and
+`bedrock-runtime-compatible-anthropic` are not instrumented.
+The #7145 standard-versus-larger-runner cohort compares the same lane and
+equivalent workload while varying the runner class. The newly instrumented
+`agent-turn-latency` and Bedrock Hermes lanes extend diagnostic coverage; this
+change does not route them to a larger runner.
 
 Each execution writes one bounded, ordered v2 time series to the canonical
 `runner-comparison.jsonl` ledger. It contains:
@@ -520,9 +527,13 @@ and dispatches the selected jobs and targets in one workflow run. The child
 workflow receives the controller-owned coordination check ID. Before checking
 out the PR revision, it requires a GitHub Actions dispatch and verifies that
 the exact check is owned by the GitHub Actions app, matches the PR head and base
-identity, names the selected plan, and links to the current child run. A direct
-manual dispatch that supplies otherwise-valid PR inputs cannot forge that
-one-run authorization and fails before checkout.
+identity, names the selected plan, and binds the exact current child Actions run
+in the controller-owned output summary. The child requests uncached check-run
+state up to 45 times with two-second delays, which spans GitHub's 60-second
+check cache, before failing closed. It does not use the check details URL for
+this binding because GitHub may canonicalize that field to the check's own
+`/runs/<check-id>` URL. A direct manual dispatch that supplies otherwise-valid
+PR inputs cannot forge that one-run authorization and fails before checkout.
 
 The manual maintainer path remains available as a fallback. A repository
 maintainer or administrator chooses **Run workflow** on `main`, selects

test/e2e/support/e2e-operations-workflow-boundary.test.ts

@@ -146,7 +146,6 @@ describe("E2E operations workflow boundary", () => {
         "Controller authentication must bind CONTROLLER_CHECK_ID",
         'Controller authentication must retain "$ACTOR" == "github-actions[bot]"',
         "Controller authentication must retain .app.id == 15368",
-        "Controller authentication must retain .details_url == $run_url",
         "Controller validation must be activated only by checkout_sha",
         "Controller validation must bind BASE_SHA",
         "Controller validation must bind CHECKOUT_REPOSITORY",
@@ -179,6 +178,48 @@ describe("E2E operations workflow boundary", () => {
     );
   });
 
+  it("requires fresh controller state for longer than GitHub's check cache (#7140)", () => {
+    const workflow = readE2eOperationsWorkflow();
+    const authentication = workflow.jobs["generate-matrix"].steps!.find(
+      (step) => step.name === "Authenticate controller dispatch",
+    )!;
+    authentication.run = String(authentication.run)
+      .replace('--header "Cache-Control: no-cache"', '--header "Cache-Control: max-age=60"')
+      .replace(" Child run: ${expected_run_url}.", "")
+      .replace("controller_auth_max_attempts=45", "controller_auth_max_attempts=15");
+
+    expect(validateE2eOperationsWorkflow(workflow)).toEqual(
+      expect.arrayContaining([
+        'Controller authentication must retain --header "Cache-Control: no-cache"',
+        "Controller authentication must retain Child run: ${expected_run_url}.",
+        "Controller authentication polling window must exceed GitHub's 60-second check cache",
+      ]),
+    );
+  });
+
+  it.each([
+    [
+      "attempt count",
+      "controller_auth_max_attempts=45",
+      "controller_auth_max_attempts=46",
+      "Controller authentication must use exactly 45 polling attempts",
+    ],
+    [
+      "poll interval",
+      "controller_auth_poll_seconds=2",
+      "controller_auth_poll_seconds=3",
+      "Controller authentication must use two-second polling intervals",
+    ],
+  ])("rejects controller %s drift (#7140)", (_name, current, replacement, expectedError) => {
+    const workflow = readE2eOperationsWorkflow();
+    const authentication = workflow.jobs["generate-matrix"].steps!.find(
+      (step) => step.name === "Authenticate controller dispatch",
+    )!;
+    authentication.run = String(authentication.run).replace(current, replacement);
+
+    expect(validateE2eOperationsWorkflow(workflow)).toContain(expectedError);
+  });
+
   it("fails a valid-looking manual fork dispatch before controller API authentication", () => {
     const workflow = readE2eOperationsWorkflow();
     const authentication = workflow.jobs["generate-matrix"].steps!.find(
@@ -214,7 +255,7 @@ describe("E2E operations workflow boundary", () => {
     expect(result.stderr).not.toContain("curl:");
   });
 
-  it("accepts a controller check bound to the exact child run and selected plan", () => {
+  it("accepts a summary-bound child when GitHub canonicalizes the details URL (#7140)", () => {
     const workflow = readE2eOperationsWorkflow();
     const authentication = workflow.jobs["generate-matrix"].steps!.find(
       (step) => step.name === "Authenticate controller dispatch",
@@ -230,9 +271,9 @@ describe("E2E operations workflow boundary", () => {
       external_id: `nemoclaw-pr-e2e:v2:42:${headSha}:${baseSha}`,
       status: "in_progress",
       conclusion: null,
-      details_url: "https://github.com/NVIDIA/NemoClaw/actions/runs/23",
+      details_url: "https://github.com/NVIDIA/NemoClaw/runs/17",
       output: {
-        summary: `Risk plan ${planHash} selected jobs: credential-sanitization; targets: none.`,
+        summary: `Risk plan ${planHash} selected jobs: credential-sanitization; targets: none. Child run: https://github.com/NVIDIA/NemoClaw/actions/runs/23.`,
       },
     });
     const result = spawnSync(

test/e2e/support/runner-comparison-workflow-boundary.test.ts

@@ -29,6 +29,8 @@ type Workflow = {
 };
 
 const JOBS = [
+  "agent-turn-latency",
+  "bedrock-runtime-compatible-anthropic",
   "channels-stop-start",
   "common-egress-agent",
   "hermes-dashboard",
@@ -59,8 +61,8 @@ function telemetrySteps(workflow: Workflow, jobId: string): WorkflowStep[] {
   );
 }
 
-describe("runner comparison E2E workflow boundary (#7145)", () => {
-  it("accepts 12 routed workflow lane identities / 15 concrete job executions", () => {
+describe("runner comparison E2E workflow boundary (#7140)", () => {
+  it("accepts 14 routed workflow lane identities / 17 concrete job executions", () => {
     const workflow = loadWorkflow();
 
     expect(validateRunnerComparisonWorkflowBoundary(workflow)).toEqual([]);
@@ -86,12 +88,16 @@ describe("runner comparison E2E workflow boundary (#7145)", () => {
     const routedLanes = JOBS.length - 1 + mcpLanes;
     const concreteExecutions =
       routedLanes + commonEgressScenarios!.length - 1 + inferenceSwitchModes!.length - 1;
-    expect(routedLanes).toBe(12);
-    expect(concreteExecutions).toBe(15);
+    expect(routedLanes).toBe(14);
+    expect(concreteExecutions).toBe(17);
   });
 
-  it("locks the matrix topology that produces fifteen concrete executions", () => {
+  it("locks the matrix topology that produces seventeen concrete executions", () => {
     const workflow = loadWorkflow();
+    workflow.jobs["bedrock-runtime-compatible-anthropic"]!.strategy!.matrix!.agent = [
+      "openclaw",
+      "openclaw",
+    ];
     workflow.jobs["mcp-bridge"]!.strategy!.matrix!.agent = ["openclaw", "hermes", "hermes"];
     workflow.jobs["channels-stop-start"]!.strategy!.matrix!.agent = ["openclaw", "openclaw"];
     workflow.jobs["common-egress-agent"]!.strategy!.matrix!.include = [
@@ -110,6 +116,7 @@ describe("runner comparison E2E workflow boundary (#7145)", () => {
 
     expect(validateRunnerComparisonWorkflow(workflow)).toEqual(
       expect.arrayContaining([
+        "bedrock-runtime-compatible-anthropic matrix must contain exactly openclaw, hermes for runner comparison telemetry",
         "channels-stop-start matrix must contain exactly openclaw, hermes for runner comparison telemetry",
         "common-egress-agent matrix must contain exactly openclaw-balanced-weather, openclaw-open-reference, hermes-open-reference for runner comparison telemetry",
         "mcp-bridge matrix must contain exactly openclaw, hermes, deepagents for runner comparison telemetry",
@@ -119,7 +126,7 @@ describe("runner comparison E2E workflow boundary (#7145)", () => {
     );
   });
 
-  it("rejects runner comparison consumers outside the eleven comparison jobs", () => {
+  it("rejects runner comparison consumers outside the thirteen comparison jobs", () => {
     const workflow = loadWorkflow();
     workflow.jobs["shields-config"]!.steps.push(
       structuredClone(telemetrySteps(workflow, "common-egress-agent")[0]!),
@@ -236,6 +243,11 @@ describe("runner comparison E2E workflow boundary (#7145)", () => {
           "(matrix.agent == 'openclaw' || matrix.agent == 'hermes')",
         );
       }
+      const bedrockComparison = step(workflow, "bedrock-runtime-compatible-anthropic", name);
+      bedrockComparison.if = bedrockComparison.if!.replace(
+        "matrix.agent == 'hermes'",
+        "(matrix.agent == 'openclaw' || matrix.agent == 'hermes')",
+      );
     }
 
     expect(validateRunnerComparisonWorkflow(workflow)).toEqual(
@@ -246,6 +258,8 @@ describe("runner comparison E2E workflow boundary (#7145)", () => {
         "channels-stop-start must use the exact always-run trusted finalize telemetry step",
         "security-posture must use the exact trusted initialize telemetry step",
         "security-posture must use the exact always-run trusted finalize telemetry step",
+        "bedrock-runtime-compatible-anthropic must use the exact trusted initialize telemetry step",
+        "bedrock-runtime-compatible-anthropic must use the exact always-run trusted finalize telemetry step",
       ]),
     );
   });

test/pr-e2e-gate.test.ts

@@ -1250,6 +1250,21 @@ describe("PR E2E controller", () => {
       expect(outputs).toContain("dispatched=true");
       expect(outputs).not.toContain("approval_mode=");
       expect(outputs).not.toContain("finalized=true");
+      const runningUpdate = requests.find(
+        (request) =>
+          request.url.endsWith("/check-runs/17") &&
+          request.method === "PATCH" &&
+          (request.body as { output?: { title?: string } }).output?.title ===
+            "Running 3 E2E checks",
+      );
+      expect(runningUpdate?.body).toMatchObject({
+        details_url: "https://github.com/NVIDIA/NemoClaw/actions/runs/23",
+        output: {
+          summary: expect.stringContaining(
+            "Child run: https://github.com/NVIDIA/NemoClaw/actions/runs/23.",
+          ),
+        },
+      });
     } finally {
       fs.rmSync(workDir, { recursive: true, force: true });
     }

tools/e2e/operations-workflow-boundary.mts

@@ -188,20 +188,44 @@ function validateControllerAuthorization(
     "nemoclaw-pr-e2e:v2:${PR_NUMBER}:${CHECKOUT_SHA}:${BASE_SHA}",
     "https://github.com/${GITHUB_REPOSITORY}/actions/runs/${RUN_ID}",
     "https://api.github.com/repos/${GITHUB_REPOSITORY}/check-runs/${CONTROLLER_CHECK_ID}",
-    `[[ "$(jq -r '.details_url // ""' <<< "$check_json")" == "$expected_run_url" ]]`,
+    '--header "Cache-Control: no-cache"',
+    "Child run: ${expected_run_url}.",
+    `[[ "$(jq -r '.output.summary // ""' <<< "$check_json")" == "$expected_summary" ]]`,
     '.name == "E2E / PR Gate Coordination"',
     ".app.id == 15368",
     '.app.slug == "github-actions"',
     ".external_id == $external_id",
     '.status == "in_progress"',
     ".conclusion == null",
-    ".details_url == $run_url",
     ".output.summary == $summary",
   ]) {
     if (!source.includes(fragment)) {
       errors.push(`Controller authentication must retain ${fragment}`);
     }
   }
+  const maxAttempts = Number.parseInt(
+    source.match(/controller_auth_max_attempts=(\d+)/u)?.[1] ?? "",
+    10,
+  );
+  const pollSeconds = Number.parseInt(
+    source.match(/controller_auth_poll_seconds=(\d+)/u)?.[1] ?? "",
+    10,
+  );
+  if (maxAttempts !== 45) {
+    errors.push("Controller authentication must use exactly 45 polling attempts");
+  }
+  if (pollSeconds !== 2) {
+    errors.push("Controller authentication must use two-second polling intervals");
+  }
+  if (
+    !Number.isSafeInteger(maxAttempts) ||
+    !Number.isSafeInteger(pollSeconds) ||
+    (maxAttempts - 1) * pollSeconds <= 60
+  ) {
+    errors.push(
+      "Controller authentication polling window must exceed GitHub's 60-second check cache",
+    );
+  }
 }
 
 function validatePrGateDispatch(errors: string[], workflow: OperationsWorkflow): void {

tools/e2e/pr-e2e-gate.mts

@@ -1467,7 +1467,7 @@ async function updateRunningCheck(
   const childRunUrl = `https://github.com/${context.repository}/actions/runs/${options.childRunId}`;
   const selectionCount = options.jobs.length + options.targets.length;
   const title = `Running ${selectionCount} E2E ${selectionCount === 1 ? "check" : "checks"}`;
-  const summary = `Risk plan ${options.planHash} selected jobs: ${options.jobs.join(", ") || "none"}; targets: ${options.targets.join(", ") || "none"}.`;
+  const summary = `Risk plan ${options.planHash} selected jobs: ${options.jobs.join(", ") || "none"}; targets: ${options.targets.join(", ") || "none"}. Child run: ${childRunUrl}.`;
   const check = await githubApi<unknown>(
     `repos/${context.repository}/check-runs/${context.checkRunId}`,
     token,