fix(ci): retry confirmed hosted runner loss once

.github/workflows/pr-e2e-gate.yaml

@@ -53,15 +53,16 @@ permissions: {}
 
 jobs:
   initialize:
-    if: ${{ github.event_name == 'pull_request_target' && github.repository == 'NVIDIA/NemoClaw' && github.event.action != 'closed' && (github.event.action != 'edited' || github.event.changes.base != null) }}
+    if: ${{ github.run_attempt == 1 && github.event_name == 'pull_request_target' && github.repository == 'NVIDIA/NemoClaw' && github.event.action != 'closed' && (github.event.action != 'edited' || github.event.changes.base != null) }}
     runs-on: ubuntu-latest
     timeout-minutes: 10
     permissions:
       checks: write
       contents: read
       pull-requests: read
     concurrency:
-      group: pr-e2e-gate-${{ github.event.pull_request.head.repo.full_name }}-${{ github.event.pull_request.head.ref }}
+      group: pr-e2e-gate-${{ github.repository }}-${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}-${{ github.event.pull_request.base.sha }}
+      queue: max
       cancel-in-progress: false
     steps:
       - name: Checkout controller
@@ -95,7 +96,7 @@ jobs:
     name: E2E / PR Gate
     if: ${{ github.event_name == 'pull_request_target' && github.repository == 'NVIDIA/NemoClaw' && github.event.action != 'closed' && (github.event.action != 'edited' || github.event.changes.base != null) }}
     runs-on: ubuntu-latest
-    timeout-minutes: 170
+    timeout-minutes: 360
     permissions:
       checks: read
       contents: read
@@ -126,10 +127,10 @@ jobs:
           --pr "$PR_NUMBER"
           --head "$HEAD_SHA"
           --base "$BASE_SHA"
-          --timeout-seconds 9900
+          --timeout-seconds 21480
 
   cancel-superseded:
-    if: ${{ github.event_name == 'pull_request_target' && github.repository == 'NVIDIA/NemoClaw' && github.event.pull_request.head.repo.full_name == github.repository && (github.event.action != 'edited' || github.event.changes.base != null) }}
+    if: ${{ github.run_attempt == 1 && github.event_name == 'pull_request_target' && github.repository == 'NVIDIA/NemoClaw' && github.event.pull_request.head.repo.full_name == github.repository && (github.event.action != 'edited' || github.event.changes.base != null) }}
     runs-on: ubuntu-latest
     timeout-minutes: 10
     permissions:
@@ -166,9 +167,9 @@ jobs:
           --superseded-head "$SUPERSEDED_HEAD_SHA"
 
   coordinate:
-    if: ${{ github.repository == 'NVIDIA/NemoClaw' && ((github.event_name == 'workflow_run' && github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.path == '.github/workflows/pr.yaml' && endsWith(github.event.workflow_run.display_title, ' gate true')) || (github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' && inputs.operation == 'run-control-plane' && github.run_attempt == 1)) }}
+    if: ${{ github.run_attempt == 1 && github.repository == 'NVIDIA/NemoClaw' && ((github.event_name == 'workflow_run' && github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.path == '.github/workflows/pr.yaml' && endsWith(github.event.workflow_run.display_title, ' gate true')) || (github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' && inputs.operation == 'run-control-plane')) }}
     runs-on: ubuntu-latest
-    timeout-minutes: 180
+    timeout-minutes: 330
     outputs:
       control_plane_approval_mode: ${{ steps.start.outputs.control_plane_approval_mode }}
       control_plane_approval_pr_number: ${{ steps.start.outputs.control_plane_approval_pr_number }}
@@ -184,7 +185,8 @@ jobs:
       contents: read
       pull-requests: read
     concurrency:
-      group: pr-e2e-gate-${{ github.repository }}-${{ github.event_name == 'workflow_run' && github.event.workflow_run.head_sha || inputs.expected_head_sha }}
+      group: pr-e2e-gate-${{ github.repository }}-${{ github.event_name == 'workflow_run' && github.event.workflow_run.pull_requests[0].number || inputs.pr_number }}-${{ github.event_name == 'workflow_run' && github.event.workflow_run.head_sha || inputs.expected_head_sha }}-${{ github.event_name == 'workflow_run' && github.event.workflow_run.pull_requests[0].base.sha || inputs.expected_base_sha }}
+      queue: max
       # Let the previous coordinator observe E2E cancellation and close its check.
       cancel-in-progress: false
     steps:
@@ -310,6 +312,81 @@ jobs:
           --run-id "${{ steps.start.outputs.run_id }}"
           --evidence-outcome "${{ steps.evidence.outcome }}"
 
+      - id: retry
+        name: Retry after hosted runner loss
+        if: ${{ always() && steps.finish.outputs.runner_loss_retry_authorized == 'true' && github.run_attempt == 1 }}
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        run: >-
+          node --experimental-strip-types tools/e2e/pr-e2e-gate.mts
+          --mode retry-runner-loss
+          --work-dir "${{ steps.workspace.outputs.work_dir }}"
+          --state-hash "${{ steps.start.outputs.state_hash }}"
+          --check-id "${{ steps.start.outputs.check_id }}"
+          --run-id "${{ steps.start.outputs.run_id }}"
+          --workflow-run-attempt "${{ github.run_attempt }}"
+
+      - id: retry_wait
+        name: Wait for retry E2E run
+        if: ${{ steps.retry.outputs.dispatched == 'true' }}
+        continue-on-error: true
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        run: >-
+          node --experimental-strip-types tools/e2e/pr-e2e-gate.mts
+          --mode wait
+          --run-id "${{ steps.retry.outputs.run_id }}"
+
+      - id: retry_evidence
+        name: Download retry evidence
+        if: ${{ always() && steps.retry.outputs.dispatched == 'true' }}
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ github.token }}
+          GITHUB_TOKEN: ${{ github.token }}
+        run: >-
+          node --experimental-strip-types tools/e2e/pr-e2e-gate.mts
+          --mode download
+          --slot runner-loss-retry
+          --work-dir "${{ steps.workspace.outputs.work_dir }}"
+          --run-id "${{ steps.retry.outputs.run_id }}"
+
+      - id: retry_finish
+        name: Verify retry evidence
+        if: ${{ always() && steps.retry.outputs.dispatched == 'true' }}
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        run: >-
+          node --experimental-strip-types tools/e2e/pr-e2e-gate.mts
+          --mode finish
+          --slot runner-loss-retry
+          --work-dir "${{ steps.workspace.outputs.work_dir }}"
+          --state-hash "${{ steps.retry.outputs.state_hash }}"
+          --check-id "${{ steps.retry.outputs.check_id }}"
+          --run-id "${{ steps.retry.outputs.run_id }}"
+          --evidence-outcome "${{ steps.retry_evidence.outcome }}"
+
+      - name: Close incomplete retry check
+        if: ${{ always() && steps.retry.outputs.check_id != '' && steps.retry.outputs.finalized != 'true' && steps.retry_finish.outputs.finalized != 'true' }}
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        run: >-
+          node --experimental-strip-types tools/e2e/pr-e2e-gate.mts
+          --mode abandon
+          --check-id "${{ steps.retry.outputs.check_id }}"
+          --run-id "${{ steps.retry.outputs.run_id }}"
+
+      - name: Terminalize interrupted retry setup
+        if: ${{ always() && steps.finish.outputs.runner_loss_retry_authorized == 'true' && steps.retry.outcome != 'success' && steps.retry.outputs.check_id == '' }}
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        run: >-
+          node --experimental-strip-types tools/e2e/pr-e2e-gate.mts
+          --mode abandon-runner-loss-retry
+          --check-id "${{ steps.start.outputs.check_id }}"
+          --run-id "${{ steps.start.outputs.run_id }}"
+          --workflow-run-attempt "${{ github.run_attempt }}"
+
       - name: Close incomplete check
         if: ${{ always() && steps.start.outputs.check_id != '' && steps.start.outputs.finalized != 'true' && steps.finish.outputs.finalized != 'true' }}
         env:
@@ -329,7 +406,7 @@ jobs:
     needs: coordinate
     if: ${{ needs.coordinate.result == 'success' && needs.coordinate.outputs.control_plane_approval_mode != '' && github.run_attempt == 1 }}
     runs-on: ubuntu-latest
-    timeout-minutes: 180
+    timeout-minutes: 330
     environment:
       name: approve-credentialed-e2e-for-internal-pr
       deployment: false
@@ -339,8 +416,9 @@ jobs:
       contents: read
       pull-requests: read
     concurrency:
-      group: pr-e2e-gate-approve-internal-${{ needs.coordinate.outputs.control_plane_approval_pr_number }}
-      cancel-in-progress: true
+      group: pr-e2e-gate-${{ github.repository }}-${{ needs.coordinate.outputs.control_plane_approval_pr_number }}-${{ needs.coordinate.outputs.control_plane_approval_head_sha }}-${{ needs.coordinate.outputs.control_plane_approval_base_sha }}
+      queue: max
+      cancel-in-progress: false
     steps:
       - name: Checkout controller
         uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
@@ -438,6 +516,81 @@ jobs:
           --run-id "${{ steps.start.outputs.run_id }}"
           --evidence-outcome "${{ steps.evidence.outcome }}"
 
+      - id: retry
+        name: Retry approved E2E after hosted runner loss
+        if: ${{ always() && steps.finish.outputs.runner_loss_retry_authorized == 'true' && github.run_attempt == 1 }}
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        run: >-
+          node --experimental-strip-types tools/e2e/pr-e2e-gate.mts
+          --mode retry-runner-loss
+          --work-dir "${{ steps.workspace.outputs.work_dir }}"
+          --state-hash "${{ steps.start.outputs.state_hash }}"
+          --check-id "${{ steps.start.outputs.check_id }}"
+          --run-id "${{ steps.start.outputs.run_id }}"
+          --workflow-run-attempt "${{ github.run_attempt }}"
+
+      - id: retry_wait
+        name: Wait for approved retry E2E run
+        if: ${{ steps.retry.outputs.dispatched == 'true' }}
+        continue-on-error: true
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        run: >-
+          node --experimental-strip-types tools/e2e/pr-e2e-gate.mts
+          --mode wait
+          --run-id "${{ steps.retry.outputs.run_id }}"
+
+      - id: retry_evidence
+        name: Download approved retry evidence
+        if: ${{ always() && steps.retry.outputs.dispatched == 'true' }}
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ github.token }}
+          GITHUB_TOKEN: ${{ github.token }}
+        run: >-
+          node --experimental-strip-types tools/e2e/pr-e2e-gate.mts
+          --mode download
+          --slot runner-loss-retry
+          --work-dir "${{ steps.workspace.outputs.work_dir }}"
+          --run-id "${{ steps.retry.outputs.run_id }}"
+
+      - id: retry_finish
+        name: Verify approved retry evidence
+        if: ${{ always() && steps.retry.outputs.dispatched == 'true' }}
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        run: >-
+          node --experimental-strip-types tools/e2e/pr-e2e-gate.mts
+          --mode finish
+          --slot runner-loss-retry
+          --work-dir "${{ steps.workspace.outputs.work_dir }}"
+          --state-hash "${{ steps.retry.outputs.state_hash }}"
+          --check-id "${{ steps.retry.outputs.check_id }}"
+          --run-id "${{ steps.retry.outputs.run_id }}"
+          --evidence-outcome "${{ steps.retry_evidence.outcome }}"
+
+      - name: Close incomplete approved retry check
+        if: ${{ always() && steps.retry.outputs.check_id != '' && steps.retry.outputs.finalized != 'true' && steps.retry_finish.outputs.finalized != 'true' }}
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        run: >-
+          node --experimental-strip-types tools/e2e/pr-e2e-gate.mts
+          --mode abandon
+          --check-id "${{ steps.retry.outputs.check_id }}"
+          --run-id "${{ steps.retry.outputs.run_id }}"
+
+      - name: Terminalize interrupted approved retry setup
+        if: ${{ always() && steps.finish.outputs.runner_loss_retry_authorized == 'true' && steps.retry.outcome != 'success' && steps.retry.outputs.check_id == '' }}
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        run: >-
+          node --experimental-strip-types tools/e2e/pr-e2e-gate.mts
+          --mode abandon-runner-loss-retry
+          --check-id "${{ steps.start.outputs.check_id }}"
+          --run-id "${{ steps.start.outputs.run_id }}"
+          --workflow-run-attempt "${{ github.run_attempt }}"
+
       - name: Close incomplete approved check
         if: ${{ always() && steps.start.outputs.check_id != '' && steps.start.outputs.finalized != 'true' && steps.finish.outputs.finalized != 'true' }}
         env:
@@ -467,8 +620,9 @@ jobs:
       contents: read
       pull-requests: read
     concurrency:
-      group: pr-e2e-gate-approve-fork-skip-${{ needs.coordinate.outputs.fork_skip_pr_number }}
-      cancel-in-progress: true
+      group: pr-e2e-gate-${{ github.repository }}-${{ needs.coordinate.outputs.fork_skip_pr_number }}-${{ needs.coordinate.outputs.fork_skip_head_sha }}-${{ needs.coordinate.outputs.fork_skip_base_sha }}
+      queue: max
+      cancel-in-progress: false
     steps:
       - name: Checkout controller
         uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
@@ -505,15 +659,16 @@ jobs:
 
   record-fork-e2e-skip:
     name: Record credentialed E2E skip for fork PR
-    if: ${{ github.event_name == 'workflow_dispatch' && github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && inputs.operation == 'approve-fork-e2e-skip' }}
+    if: ${{ github.run_attempt == 1 && github.event_name == 'workflow_dispatch' && github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && inputs.operation == 'approve-fork-e2e-skip' }}
     runs-on: ubuntu-latest
     timeout-minutes: 10
     permissions:
       checks: write
       contents: read
       pull-requests: read
     concurrency:
-      group: pr-e2e-gate-record-fork-skip-${{ inputs.pr_number }}
+      group: pr-e2e-gate-${{ github.repository }}-${{ inputs.pr_number }}-${{ inputs.expected_head_sha }}-${{ inputs.expected_base_sha }}
+      queue: max
       cancel-in-progress: false
     steps:
       - name: Checkout controller