fix(ci): recover hosted runner loss once

.github/workflows/hosted-runner-recovery.yaml

@@ -0,0 +1,84 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+name: Hosted Runner Recovery
+
+run-name: Hosted runner recovery for source run ${{ github.event.workflow_run.id }}
+
+on:
+  workflow_run:
+    workflows:
+      - E2E
+      - E2E / WSL
+      - E2E / macOS
+      - CI / Platform Vitest Main Watch
+    types:
+      - completed
+
+permissions: {}
+
+jobs:
+  recover:
+    if: >-
+      ${{
+      github.run_attempt == 1 &&
+      github.repository == 'NVIDIA/NemoClaw' &&
+      github.event.workflow_run.run_attempt == 1 &&
+      github.event.workflow_run.status == 'completed' &&
+      github.event.workflow_run.conclusion == 'failure' &&
+      github.event.workflow_run.head_branch == 'main' &&
+      github.event.workflow_run.head_repository.full_name == 'NVIDIA/NemoClaw' &&
+      (
+      (
+      github.event.workflow_run.path == '.github/workflows/e2e.yaml' &&
+      (github.event.workflow_run.event == 'schedule' ||
+      github.event.workflow_run.event == 'workflow_dispatch') &&
+      github.event.workflow_run.display_title == 'E2E main'
+      ) ||
+      (
+      github.event.workflow_run.event == 'push' &&
+      (
+      github.event.workflow_run.path == '.github/workflows/wsl-e2e.yaml' ||
+      github.event.workflow_run.path == '.github/workflows/macos-e2e.yaml' ||
+      github.event.workflow_run.path == '.github/workflows/platform-vitest-main.yaml'
+      )
+      )
+      )
+      }}
+    concurrency:
+      group: hosted-runner-recovery-${{ github.event.workflow_run.workflow_id }}
+      queue: max
+      cancel-in-progress: false
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    permissions:
+      actions: write
+      checks: read
+      contents: read
+    steps:
+      - name: Checkout trusted recovery controller
+        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+        with:
+          ref: ${{ github.workflow_sha }}
+          persist-credentials: false
+
+      - name: Setup Node.js
+        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version: "22"
+
+      - name: Evaluate exact hosted-runner-loss evidence
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+          SOURCE_RUN_ID: ${{ github.event.workflow_run.id }}
+        run: >-
+          node --experimental-strip-types --no-warnings
+          tools/e2e/hosted-runner-recovery.mts
+
+      - name: Record static recovery policy
+        if: ${{ always() }}
+        shell: bash
+        run: >-
+          printf '%s\n'
+          'Hosted-runner recovery evaluated the fail-closed latest-eligible main-run policy.'
+          >> "$GITHUB_STEP_SUMMARY"

ci/source-shape-test-budget.json

@@ -301,6 +301,11 @@
       "test": "restores exact locked posture after root-separated repair and later failure (#7033)",
       "category": "security"
     },
+    {
+      "file": "test/hosted-runner-recovery-workflow.test.ts",
+      "test": "locks recovery identities to the source workflows' runtime names (#7140)",
+      "category": "security"
+    },
     {
       "file": "test/inference-options-docs.test.ts",
       "test": "keeps a per-model task-fit comparison table for curated onboarding models",

test/e2e/README.md

@@ -16,6 +16,9 @@ before those targets run; local runners must provide it themselves.
   GitHub Actions check suite.
 - `.github/workflows/e2e-branch-validation.yaml` provisions Brev instances and
   runs focused E2E targets from source on a clean machine.
+- `.github/workflows/hosted-runner-recovery.yaml` evaluates first-attempt
+  failures from approved `main` workflows and requests one full rerun only when
+  every non-passing job has authenticated GitHub-hosted runner-loss evidence.
 - Platform workflows such as macOS, WSL, sandbox image, and regression E2E
   call their target E2E tests directly. The Ollama auth proxy target is
   selected through `.github/workflows/e2e.yaml`.
@@ -201,6 +204,52 @@ graph as the live targets:
   `post_to_slack=true`, which uses the preview Slack route. Branch-dispatched
   runs never receive Slack webhook secrets.
 
+### Hosted-runner recovery
+
+The trusted recovery workflow can request one full rerun of the latest eligible
+first attempt for these source workflows:
+
+- scheduled or manually dispatched `E2E main`;
+- `E2E / WSL` pushes to `main`;
+- `E2E / macOS` pushes to `main`; and
+- `CI / Platform Vitest Main Watch` pushes to `main`.
+
+The controller authenticates the active workflow name and path, repository,
+head repository, branch, run name, event, run URL, and first-attempt failure.
+It ignores E2E PR child runs and an eligible source run that has a newer
+eligible run for the same workflow.
+
+The complete non-passing job listing must contain only exact hosted-runner-loss
+markers for that workflow's approved runner labels.
+The E2E policy accepts only `ubuntu-latest`.
+The WSL and macOS policies accept only `cancelled` jobs with their exact
+platform label and exact GitHub internal-error annotation.
+The platform-watch policy applies those platform contracts to Windows and
+macOS jobs and the authenticated hosted-runner-loss contract to Ubuntu jobs.
+An ordinary assertion failure, mixed failure set, incomplete listing, custom or
+self-hosted label, changed evidence, or ambiguous pagination prevents recovery.
+
+The controller collects and fingerprints the complete source, workflow,
+latest-run, job, check, annotation, and optional log evidence twice.
+It requests the full GitHub Actions rerun only when both snapshots match.
+The rerun executes every job in attempt two, not only the failed jobs.
+Neither a source attempt two nor a recovery-controller rerun can request
+another source rerun.
+
+The recovery job checks out only the trusted default-branch controller and does
+not check out source-run code.
+It receives no repository secrets and holds `actions: write` only for the
+bounded rerun request.
+
+The runner-allocation and internal-error failures originate in the
+GitHub-hosted Actions service, outside repository-controlled workflow code, so
+this controller contains the failure without claiming to repair its source.
+Remove the recovery workflow and controller after the supported source
+workflows record 30 consecutive days with no first-attempt failure accepted by
+the exact recovery classifier, or when those workflows stop using
+GitHub-hosted runners. Any accepted recovery request resets that observation
+window.
+
 ### Runner comparison telemetry
 
 Trusted `main` runs without an alternate checkout SHA record runner-comparison
@@ -703,8 +752,8 @@ exactly one failure annotation. Its message must be
 `The operation was canceled.` for one completed `cancelled` workload step or
 `Process completed with exit code 143.` for one completed `failure` workload
 step. The annotation must use `.github`, equal start and end lines, null
-columns, and empty title and detail fields. Every annotation must use a blob URL
-bound to the same workflow commit. The controller accepts at most 20
+columns, and null or empty title and detail fields. Every annotation must use a
+blob URL bound to the same workflow commit. The controller accepts at most 20
 annotations, bounds each text field, and limits the normalized annotation
 evidence to 64 KiB. This permits trusted bounded non-failure notices beside the
 sole failure annotation without allowing annotation output to exhaust the

test/helpers/e2e-workflow-contract.ts

@@ -9,7 +9,7 @@ import YAML from "yaml";
 const REPO_ROOT = join(dirname(fileURLToPath(import.meta.url)), "..", "..");
 
 export type WorkflowJob = {
-  concurrency?: { group: string; "cancel-in-progress": boolean };
+  concurrency?: { group: string; queue?: "max"; "cancel-in-progress": boolean };
   environment?: string | { name: string; deployment?: boolean };
   if?: string;
   name?: string;

test/helpers/vitest-watch-triggers.ts

@@ -18,6 +18,7 @@ const E2E_WORKFLOW_CONTRACTS = [
   "test/e2e/support/gateway-guard-workflow-boundary.test.ts",
   "test/e2e/support/hermes-dashboard-workflow-boundary.test.ts",
   "test/e2e/support/hermes-workflow-boundary.test.ts",
+  "test/hosted-runner-recovery-workflow.test.ts",
   "test/e2e/support/inference-switch-workflow-boundary.test.ts",
   "test/e2e/support/jetson-workflow-boundary.test.ts",
   "test/e2e/support/mcp-workflow-boundary.test.ts",
@@ -97,6 +98,11 @@ export const vitestWatchTriggerPatterns: VitestWatchTriggerPattern[] = [
     pattern: /(?:^|\/)\.github\/workflows\/pr-e2e-gate\.yaml$/,
     testsToRun: runTests("test/pr-e2e-gate-workflow.test.ts", "test/pr-e2e-required.test.ts"),
   },
+  {
+    pattern:
+      /(?:^|\/)\.github\/workflows\/(?:hosted-runner-recovery|wsl-e2e|macos-e2e|platform-vitest-main)\.yaml$/,
+    testsToRun: runTests("test/hosted-runner-recovery-workflow.test.ts"),
+  },
   {
     pattern:
       /(?:^|\/)(?:\.github\/workflows\/platform-vitest-main\.yaml|ci\/platform-vitest-macos-requirements\.lock)$/,