From 33d488e32c8a1f47255e659471d9ac828fe723c9 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Sat, 25 Jul 2026 07:59:15 -0700 Subject: [PATCH 01/10] fix(ci): harden hosted runner-loss evidence Signed-off-by: Apurv Kumaria --- .../hosted-runner-loss-internal-error.test.ts | 410 ++++++++++++ ...pr-e2e-gate-runner-loss-classifier.test.ts | 2 +- test/pr-e2e-gate-runner-loss-retry.test.ts | 2 +- tools/e2e/hosted-runner-loss.mts | 600 ++++++++++++++++++ tools/e2e/pr-e2e-gate.mts | 471 ++------------ 5 files changed, 1066 insertions(+), 419 deletions(-) create mode 100644 test/hosted-runner-loss-internal-error.test.ts create mode 100644 tools/e2e/hosted-runner-loss.mts diff --git a/test/hosted-runner-loss-internal-error.test.ts b/test/hosted-runner-loss-internal-error.test.ts new file mode 100644 index 0000000000..324f1d9bf7 --- /dev/null +++ b/test/hosted-runner-loss-internal-error.test.ts @@ -0,0 +1,410 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; +import { + type HostedRunnerLossPolicy, + verifiedRunnerLossEvidence, + type WorkflowJob, + type WorkflowJobAnnotation, +} from "../tools/e2e/hosted-runner-loss.mts"; +import { detectRunnerLoss } from "../tools/e2e/runner-pressure-core.mts"; + +const REPOSITORY = "NVIDIA/NemoClaw"; +const WORKFLOW_SHA = "d".repeat(40); +const RUN_ID = 29_897_237_525; +const JOB_ID = 89_074_697_099; +const API_REPOSITORY = `https://api.github.com/repos/${REPOSITORY}`; +const WEB_REPOSITORY = `https://github.com/${REPOSITORY}`; +const RUN_URL = `${API_REPOSITORY}/actions/runs/${RUN_ID}`; +const JOB_API_URL = `${API_REPOSITORY}/actions/jobs/${JOB_ID}`; +const CHECK_URL = `${API_REPOSITORY}/check-runs/${JOB_ID}`; +const JOB_HTML_URL = `${WEB_REPOSITORY}/actions/runs/${RUN_ID}/job/${JOB_ID}`; +const INTERNAL_ERROR_MESSAGE = + "GitHub Actions has encountered an internal error when running your job."; + +const WINDOWS_POLICY: HostedRunnerLossPolicy = { + githubInternalError: { + approvedRunnerLabels: ["windows-latest"], + approvedJobConclusions: ["cancelled"], + }, +}; + +function internalErrorAnnotation( + overrides: Partial = {}, +): WorkflowJobAnnotation { + return { + path: ".github", + blobHref: `${WEB_REPOSITORY}/blob/${WORKFLOW_SHA}/.github`, + startLine: 1, + startColumn: null, + endLine: 1, + endColumn: null, + annotationLevel: "failure", + title: "", + message: INTERNAL_ERROR_MESSAGE, + rawDetails: "", + ...overrides, + }; +} + +function internalErrorJob(overrides: Partial = {}): WorkflowJob { + const job: WorkflowJob = { + id: JOB_ID, + name: "MCP bridge (windows-latest)", + runId: RUN_ID, + runAttempt: 1, + headSha: WORKFLOW_SHA, + runUrl: RUN_URL, + apiUrl: JOB_API_URL, + htmlUrl: JOB_HTML_URL, + checkRunUrl: CHECK_URL, + status: "completed", + conclusion: "cancelled", + runnerId: 1_021_277_393, + runnerName: "GitHub Actions 1021277393", + runnerGroupId: 0, + runnerGroupName: "GitHub Actions", + labels: ["windows-latest"], + annotations: [internalErrorAnnotation()], + steps: [ + { name: "Set up job", status: "completed", conclusion: "success" }, + { name: "Run MCP bridge test", status: "in_progress", conclusion: null }, + { name: "Upload artifacts", status: "pending", conclusion: null }, + ], + }; + job.checkEvidence = { + id: job.id, + name: job.name, + headSha: WORKFLOW_SHA, + apiUrl: CHECK_URL, + htmlUrl: JOB_HTML_URL, + detailsUrl: JOB_HTML_URL, + status: "completed", + conclusion: job.conclusion, + appId: 15368, + appSlug: "github-actions", + annotationsCount: 1, + annotationsUrl: `${CHECK_URL}/annotations`, + }; + return { ...job, ...overrides }; +} + +function withCheck( + job: WorkflowJob, + overrides: Partial>, +): WorkflowJob { + return { + ...job, + checkEvidence: { ...job.checkEvidence!, ...overrides }, + }; +} + +function confirmsInternalError( + job: WorkflowJob, + policy: HostedRunnerLossPolicy = WINDOWS_POLICY, + workflowConclusion = "failure", +): boolean { + const evidence = verifiedRunnerLossEvidence({ + repository: REPOSITORY, + workflowSha: WORKFLOW_SHA, + workflowConclusion, + jobs: [job], + jobDetailsAvailable: true, + jobDetailsComplete: true, + policy, + }); + return evidence !== null && detectRunnerLoss(evidence); +} + +describe("hosted-runner GitHub internal-error evidence", () => { + it("accepts an exact assigned Windows runner only after explicit opt-in (#7140)", () => { + expect(confirmsInternalError(internalErrorJob())).toBe(true); + expect(confirmsInternalError(internalErrorJob(), {})).toBe(false); + }); + + it("accepts an exact assigned macOS runner under its own label policy (#7140)", () => { + const job = internalErrorJob({ + name: "MCP bridge (macos-26)", + labels: ["macos-26"], + runnerId: 1_021_277_394, + runnerName: "GitHub Actions 1021277394", + }); + const exactJob = withCheck(job, { name: job.name }); + expect( + confirmsInternalError(exactJob, { + githubInternalError: { + approvedRunnerLabels: ["macos-26"], + approvedJobConclusions: ["cancelled"], + }, + }), + ).toBe(true); + }); + + it("requires a failed workflow even for an authenticated cancelled job (#7140)", () => { + expect(confirmsInternalError(internalErrorJob(), WINDOWS_POLICY, "cancelled")).toBe(false); + }); + + it("supports an explicitly approved failed internal-error job (#7140)", () => { + const job = internalErrorJob({ conclusion: "failure" }); + const exactJob = withCheck(job, { conclusion: "failure" }); + expect( + confirmsInternalError(exactJob, { + githubInternalError: { + approvedRunnerLabels: ["windows-latest"], + approvedJobConclusions: ["failure"], + }, + }), + ).toBe(true); + }); + + it("rejects a job conclusion that its caller did not approve (#7140)", () => { + expect( + confirmsInternalError(internalErrorJob(), { + githubInternalError: { + approvedRunnerLabels: ["windows-latest"], + approvedJobConclusions: ["failure"], + }, + }), + ).toBe(false); + }); + + it.each([ + ["runner ID", { runnerId: null }], + ["runner name", { runnerName: "self-hosted-windows" }], + ["runner group ID", { runnerGroupId: 1 }], + ["runner group name", { runnerGroupName: "Custom runners" }], + ["self-hosted label", { labels: ["windows-latest", "self-hosted"] }], + ["extra unapproved label", { labels: ["windows-latest", "X64"] }], + ["unapproved label", { labels: ["windows-2025"] }], + ["zero steps", { steps: [] }], + ])("rejects an invalid assigned-runner %s (#7140)", (_label, overrides) => { + expect(confirmsInternalError(internalErrorJob(overrides))).toBe(false); + }); + + it("requires exactly one approved label match (#7140)", () => { + const policy: HostedRunnerLossPolicy = { + githubInternalError: { + approvedRunnerLabels: ["windows-latest", "macos-26"], + approvedJobConclusions: ["cancelled"], + }, + }; + expect( + confirmsInternalError(internalErrorJob({ labels: ["windows-latest", "macos-26"] }), policy), + ).toBe(false); + }); + + it.each([ + ["missing prior step", [{ name: "Run", status: "in_progress", conclusion: null }]], + [ + "missing later step", + [ + { name: "Set up job", status: "completed", conclusion: "success" }, + { name: "Run", status: "in_progress", conclusion: null }, + ], + ], + [ + "failed prior step", + [ + { name: "Set up job", status: "completed", conclusion: "failure" }, + { name: "Run", status: "in_progress", conclusion: null }, + { name: "Upload", status: "pending", conclusion: null }, + ], + ], + [ + "completed later step", + [ + { name: "Set up job", status: "completed", conclusion: "success" }, + { name: "Run", status: "in_progress", conclusion: null }, + { name: "Upload", status: "completed", conclusion: "skipped" }, + ], + ], + [ + "multiple stranded steps", + [ + { name: "Set up job", status: "completed", conclusion: "success" }, + { name: "Run", status: "in_progress", conclusion: null }, + { name: "Upload", status: "in_progress", conclusion: null }, + { name: "Cleanup", status: "pending", conclusion: null }, + ], + ], + ])("rejects a non-exact stranded-step shape: %s (#7140)", (_label, steps) => { + expect(confirmsInternalError(internalErrorJob({ steps }))).toBe(false); + }); + + it.each([ + ["path", { path: "src/index.ts" }], + ["blob SHA", { blobHref: `${WEB_REPOSITORY}/blob/${"e".repeat(40)}/.github` }], + ["start line", { startLine: 2 }], + ["start column", { startColumn: 1 }], + ["end line", { endLine: 2 }], + ["end column", { endColumn: 1 }], + ["level", { annotationLevel: "warning" }], + ["title", { title: "Internal error" }], + ["message", { message: "The operation was canceled." }], + ["raw details", { rawDetails: "untrusted" }], + ])("rejects an internal-error annotation with the wrong %s (#7140)", (_label, annotation) => { + expect( + confirmsInternalError( + internalErrorJob({ annotations: [internalErrorAnnotation(annotation)] }), + ), + ).toBe(false); + }); + + it("accepts one exact failure annotation alongside an authenticated notice (#7140)", () => { + const notice = internalErrorAnnotation({ + path: ".github/workflows/e2e.yaml", + blobHref: `${WEB_REPOSITORY}/blob/${WORKFLOW_SHA}/.github/workflows/e2e.yaml`, + startLine: 42, + endLine: 42, + annotationLevel: "notice", + message: "Docker Hub credentials are withheld for this ref.", + }); + const job = internalErrorJob({ + annotations: [internalErrorAnnotation(), notice], + }); + expect(confirmsInternalError(withCheck(job, { annotationsCount: 2 }))).toBe(true); + }); + + it("rejects multiple failure annotations and authenticated count drift (#7140)", () => { + const annotations = [internalErrorAnnotation(), internalErrorAnnotation()]; + const multipleFailures = internalErrorJob({ annotations }); + expect(confirmsInternalError(withCheck(multipleFailures, { annotationsCount: 2 }))).toBe(false); + expect(confirmsInternalError(withCheck(internalErrorJob(), { annotationsCount: 2 }))).toBe( + false, + ); + }); + + it("rejects internal-error evidence without its authenticated check run (#7140)", () => { + expect(confirmsInternalError(internalErrorJob({ checkEvidence: undefined }))).toBe(false); + }); + + it.each([ + ["check ID", { id: JOB_ID + 1 }], + ["check name", { name: "different job" }], + ["check SHA", { headSha: "e".repeat(40) }], + ["check API URL", { apiUrl: `${CHECK_URL}-other` }], + ["check HTML URL", { htmlUrl: `${JOB_HTML_URL}-other` }], + ["check details URL", { detailsUrl: `${JOB_HTML_URL}-other` }], + ["check status", { status: "in_progress" }], + ["check conclusion", { conclusion: "failure" }], + ["app ID", { appId: 7 }], + ["app slug", { appSlug: "github-actions-enterprise" }], + ["annotation URL", { annotationsUrl: `${CHECK_URL}/other` }], + ])("rejects internal-error check evidence with the wrong %s (#7140)", (_label, check) => { + expect(confirmsInternalError(withCheck(internalErrorJob(), check))).toBe(false); + }); + + it.each([ + ["job SHA", { headSha: "e".repeat(40) }], + ["run URL", { runUrl: `${RUN_URL}-other` }], + ["job API URL", { apiUrl: `${JOB_API_URL}-other` }], + ["job HTML URL", { htmlUrl: `${JOB_HTML_URL}-other` }], + ["check-run URL", { checkRunUrl: `${CHECK_URL}-other` }], + ["run ID", { runId: RUN_ID + 1 }], + ["run attempt", { runAttempt: 0 }], + ])("rejects internal-error job evidence with the wrong %s (#7140)", (_label, job) => { + expect(confirmsInternalError(internalErrorJob(job))).toBe(false); + }); + + it("rejects a mixed run with any ordinary failure (#7140)", () => { + const evidence = verifiedRunnerLossEvidence({ + repository: REPOSITORY, + workflowSha: WORKFLOW_SHA, + workflowConclusion: "failure", + jobs: [ + internalErrorJob(), + { + ...internalErrorJob(), + id: JOB_ID + 1, + name: "ordinary assertion", + conclusion: "failure", + runnerId: null, + runnerName: null, + annotations: [], + checkEvidence: undefined, + steps: [{ name: "Run tests", status: "completed", conclusion: "failure" }], + }, + ], + jobDetailsAvailable: true, + jobDetailsComplete: true, + policy: WINDOWS_POLICY, + }); + expect(evidence).not.toBeNull(); + expect(detectRunnerLoss(evidence!)).toBe(false); + }); + + it("rejects zero-job and incomplete evidence (#7140)", () => { + const base = { + repository: REPOSITORY, + workflowSha: WORKFLOW_SHA, + workflowConclusion: "failure", + jobs: [] as WorkflowJob[], + jobDetailsAvailable: true, + jobDetailsComplete: true, + policy: WINDOWS_POLICY, + }; + expect(verifiedRunnerLossEvidence(base)).toBeNull(); + expect( + verifiedRunnerLossEvidence({ + ...base, + jobs: [internalErrorJob()], + jobDetailsComplete: false, + }), + ).toBeNull(); + }); + + it.each([ + {}, + { githubInternalError: { approvedRunnerLabels: [], approvedJobConclusions: ["cancelled"] } }, + { + githubInternalError: { + approvedRunnerLabels: ["windows-*"], + approvedJobConclusions: ["cancelled"], + }, + }, + { + githubInternalError: { + approvedRunnerLabels: ["self-hosted"], + approvedJobConclusions: ["cancelled"], + }, + }, + { + githubInternalError: { + approvedRunnerLabels: ["windows-latest", "windows-latest"], + approvedJobConclusions: ["cancelled"], + }, + }, + { + githubInternalError: { + approvedRunnerLabels: ["windows-latest"], + approvedJobConclusions: [], + }, + }, + { + githubInternalError: { + approvedRunnerLabels: ["windows-latest"], + approvedJobConclusions: ["cancelled", "cancelled"], + }, + }, + { + githubInternalError: { + approvedRunnerLabels: ["windows-latest"], + approvedJobConclusions: ["success"], + }, + }, + { unsupported: true }, + ])("rejects malformed internal-error policy %# (#7140)", (policy) => { + if (Object.keys(policy).length === 0) { + expect(() => + confirmsInternalError(internalErrorJob(), { + githubInternalError: policy as never, + }), + ).toThrow(/policy/u); + return; + } + expect(() => + confirmsInternalError(internalErrorJob(), policy as HostedRunnerLossPolicy), + ).toThrow(/policy/u); + }); +}); diff --git a/test/pr-e2e-gate-runner-loss-classifier.test.ts b/test/pr-e2e-gate-runner-loss-classifier.test.ts index 9d9b096b7f..10b22f21ce 100644 --- a/test/pr-e2e-gate-runner-loss-classifier.test.ts +++ b/test/pr-e2e-gate-runner-loss-classifier.test.ts @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { describe, expect, it } from "vitest"; -import { verifiedRunnerLossEvidence } from "../tools/e2e/pr-e2e-gate.mts"; +import { verifiedRunnerLossEvidence } from "../tools/e2e/hosted-runner-loss.mts"; import { detectRunnerLoss } from "../tools/e2e/runner-pressure-core.mts"; const WORKFLOW_SHA = "d".repeat(40); diff --git a/test/pr-e2e-gate-runner-loss-retry.test.ts b/test/pr-e2e-gate-runner-loss-retry.test.ts index 7caac32f81..f646284ace 100644 --- a/test/pr-e2e-gate-runner-loss-retry.test.ts +++ b/test/pr-e2e-gate-runner-loss-retry.test.ts @@ -305,7 +305,7 @@ function workflowJobCheckRun(job: ReturnType) { details_url: job.html_url, status: "completed", conclusion: "failure", - app: { id: 15368 }, + app: { id: 15368, slug: "github-actions" }, output: { annotations_count: 1, annotations_url: annotationsUrl }, }; } diff --git a/tools/e2e/hosted-runner-loss.mts b/tools/e2e/hosted-runner-loss.mts new file mode 100644 index 0000000000..b0554acb46 --- /dev/null +++ b/tools/e2e/hosted-runner-loss.mts @@ -0,0 +1,600 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { WorkflowAttemptEvidence } from "./runner-pressure-core.mts"; + +export const MAX_RUNNER_LOSS_JOB_ANNOTATIONS = 20; +export const MAX_RUNNER_LOSS_JOB_LOG_TAIL_BYTES = 64 * 1024; + +const GITHUB_ACTIONS_APP_ID = 15368; +const GITHUB_HOSTED_RUNNER_NAME_PATTERN = /^GitHub Actions [1-9][0-9]*$/u; +const APPROVED_RUNNER_LABEL_PATTERN = /^[A-Za-z0-9](?:[A-Za-z0-9._-]{0,62}[A-Za-z0-9])?$/u; +const HOSTED_RUNNER_LOST_COMMUNICATION_MESSAGE = + "The hosted runner lost communication with the server. Anything in your workflow that terminates the runner process, starves it for CPU/Memory, or blocks its network access can cause this error."; +const GITHUB_INTERNAL_ERROR_MESSAGE = + "GitHub Actions has encountered an internal error when running your job."; +const HOSTED_RUNNER_SHUTDOWN_MESSAGE = + "The runner has received a shutdown signal. This can happen when the runner service is stopped, or a manually started runner is canceled."; +const HOSTED_RUNNER_OPERATION_CANCELLED_MESSAGE = "The operation was canceled."; +const HOSTED_RUNNER_EXIT_143_MESSAGE = "Process completed with exit code 143."; +const HOSTED_RUNNER_ORPHAN_CLEANUP_MESSAGE = "Cleaning up orphan processes"; +const MAX_RUNNER_LOSS_ORPHAN_PROCESSES = 64; +const JOB_LOG_TIMESTAMPED_LINE_PATTERN = + /^([0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{7}Z) (.*)$/u; +const JOB_LOG_ORPHAN_PROCESS_PATTERN = + /^Terminate orphan process: pid \(([1-9][0-9]*)\) \(([A-Za-z0-9._+ -]{1,128})\)$/u; + +type ApprovedInternalErrorConclusion = "failure" | "cancelled"; + +export type HostedRunnerLossPolicy = { + githubInternalError?: { + approvedRunnerLabels: readonly string[]; + approvedJobConclusions: readonly ApprovedInternalErrorConclusion[]; + }; +}; + +export type WorkflowJobAnnotation = { + path: string; + blobHref: string; + startLine: number; + startColumn: number | null; + endLine: number; + endColumn: number | null; + annotationLevel: string; + title: string; + message: string; + rawDetails: string; +}; + +export type WorkflowJobCheckEvidence = { + id: number; + name: string; + headSha: string; + apiUrl: string; + htmlUrl: string; + detailsUrl: string; + status: string; + conclusion: string | null; + appId: number; + appSlug: string; + annotationsCount: number; + annotationsUrl: string; +}; + +export type WorkflowJobLogEvidence = { + etag: string; + totalBytes: number; + tail: string; +}; + +export type WorkflowJob = { + id: number; + name: string; + runId?: number; + runAttempt?: number; + headSha?: string; + runUrl?: string; + apiUrl?: string; + htmlUrl?: string; + checkRunUrl?: string; + status?: string; + conclusion: string | null; + runnerId?: number | null; + runnerName?: string | null; + runnerGroupId?: number | null; + runnerGroupName?: string | null; + labels?: string[]; + annotations?: WorkflowJobAnnotation[]; + checkEvidence?: WorkflowJobCheckEvidence; + logEvidence?: WorkflowJobLogEvidence; + startedAt?: string | null; + completedAt?: string | null; + steps: Array<{ + name: string; + status?: string; + conclusion: string | null; + startedAt?: string | null; + completedAt?: string | null; + }>; +}; + +type HostedRunnerShutdownLogMarker = { + shutdownTimestamp: string; + terminalTimestamp: string; + cleanupTimestamp: string; + lastTimestamp: string; + annotationMessage: string; + interruptedStepConclusion: "cancelled" | "failure"; +}; + +type NormalizedInternalErrorPolicy = { + approvedRunnerLabels: ReadonlySet; + approvedJobConclusions: ReadonlySet; +}; + +function isObjectRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function normalizedInternalErrorPolicy( + policy: HostedRunnerLossPolicy, +): NormalizedInternalErrorPolicy | null { + if (!isObjectRecord(policy) || Object.keys(policy).some((key) => key !== "githubInternalError")) { + throw new Error("hosted-runner-loss policy has an unsupported shape"); + } + if (policy.githubInternalError === undefined) return null; + const internal = policy.githubInternalError; + if ( + !isObjectRecord(internal) || + Object.keys(internal).sort().join(",") !== "approvedJobConclusions,approvedRunnerLabels" || + !Array.isArray(internal.approvedRunnerLabels) || + internal.approvedRunnerLabels.length === 0 || + internal.approvedRunnerLabels.some( + (label) => + typeof label !== "string" || + !APPROVED_RUNNER_LABEL_PATTERN.test(label) || + label === "self-hosted", + ) || + new Set(internal.approvedRunnerLabels).size !== internal.approvedRunnerLabels.length || + !Array.isArray(internal.approvedJobConclusions) || + internal.approvedJobConclusions.length === 0 || + internal.approvedJobConclusions.some( + (conclusion) => conclusion !== "failure" && conclusion !== "cancelled", + ) || + new Set(internal.approvedJobConclusions).size !== internal.approvedJobConclusions.length + ) { + throw new Error("GitHub internal-error policy must use unique exact labels and conclusions"); + } + return { + approvedRunnerLabels: new Set(internal.approvedRunnerLabels), + approvedJobConclusions: new Set(internal.approvedJobConclusions), + }; +} + +function hasAssignedGitHubHostedRunner(job: WorkflowJob): boolean { + return ( + Number.isSafeInteger(job.runnerId) && + (job.runnerId ?? 0) > 0 && + typeof job.runnerName === "string" && + GITHUB_HOSTED_RUNNER_NAME_PATTERN.test(job.runnerName) && + job.runnerGroupId === 0 && + job.runnerGroupName === "GitHub Actions" && + Array.isArray(job.labels) && + !job.labels.includes("self-hosted") + ); +} + +function hasLegacyStrandedStepShape(job: WorkflowJob, requireNeighbors: boolean): boolean { + const strandedIndexes = job.steps.flatMap((step, index) => + step.status === "in_progress" && step.conclusion === null ? [index] : [], + ); + if (strandedIndexes.length !== 1) return false; + const strandedIndex = strandedIndexes[0]!; + if (requireNeighbors && (strandedIndex === 0 || strandedIndex === job.steps.length - 1)) { + return false; + } + return ( + job.steps + .slice(0, strandedIndex) + .every( + (step) => + step.status === "completed" && ["success", "skipped"].includes(step.conclusion ?? ""), + ) && + job.steps + .slice(strandedIndex + 1) + .every((step) => step.status === "pending" && step.conclusion === null) + ); +} + +function hasTrustedHostedRunnerLossStepShapeForConclusion( + job: WorkflowJob, + interruptedStepConclusion: "cancelled" | "failure", + options: { allowLegacyStrandedStep: boolean }, +): boolean { + if ( + job.status !== "completed" || + job.conclusion !== "failure" || + !hasAssignedGitHubHostedRunner(job) || + !job.labels?.includes("ubuntu-latest") + ) { + return false; + } + if ( + options.allowLegacyStrandedStep && + interruptedStepConclusion === "cancelled" && + hasLegacyStrandedStepShape(job, false) + ) { + return true; + } + + const interruptedStepIndexes = job.steps.flatMap((step, index) => + step.status === "completed" && step.conclusion === interruptedStepConclusion ? [index] : [], + ); + if (interruptedStepIndexes.length !== 1) return false; + const interruptedIndex = interruptedStepIndexes[0]!; + if (job.steps[interruptedIndex]?.name === "Complete job") return false; + const beforeInterruption = job.steps.slice(0, interruptedIndex); + const afterInterruption = job.steps.slice(interruptedIndex + 1); + const syntheticCompletion = afterInterruption.at(-1); + const skippedCleanup = afterInterruption.slice(0, -1); + return ( + beforeInterruption.every( + (step) => + step.status === "completed" && ["success", "skipped"].includes(step.conclusion ?? ""), + ) && + skippedCleanup.length > 0 && + skippedCleanup.every( + (step) => + step.name !== "Complete job" && + step.status === "completed" && + step.conclusion === "skipped", + ) && + syntheticCompletion?.name === "Complete job" && + syntheticCompletion.status === "completed" && + syntheticCompletion.conclusion === "success" + ); +} + +function hasTrustedHostedRunnerLossStepShape(job: WorkflowJob): boolean { + return hasTrustedHostedRunnerLossStepShapeForConclusion(job, "cancelled", { + allowLegacyStrandedStep: true, + }); +} + +function hasGitHubInternalErrorInspectionShape( + job: WorkflowJob, + policy: NormalizedInternalErrorPolicy | null, +): boolean { + return ( + policy !== null && + job.status === "completed" && + (job.conclusion === "failure" || job.conclusion === "cancelled") && + policy.approvedJobConclusions.has(job.conclusion) && + hasAssignedGitHubHostedRunner(job) && + job.labels?.length === 1 && + policy.approvedRunnerLabels.has(job.labels[0]!) && + hasLegacyStrandedStepShape(job, true) + ); +} + +function isHostedRunnerLossInspectionCandidateWithPolicy( + job: WorkflowJob, + internalPolicy: NormalizedInternalErrorPolicy | null, +): boolean { + return ( + hasTrustedHostedRunnerLossStepShape(job) || + hasTrustedHostedRunnerLossStepShapeForConclusion(job, "failure", { + allowLegacyStrandedStep: false, + }) || + hasGitHubInternalErrorInspectionShape(job, internalPolicy) + ); +} + +export function isHostedRunnerLossInspectionCandidate( + job: WorkflowJob, + policy: HostedRunnerLossPolicy = {}, +): boolean { + return isHostedRunnerLossInspectionCandidateWithPolicy( + job, + normalizedInternalErrorPolicy(policy), + ); +} + +function trustedWorkflowJobAnnotations( + job: WorkflowJob, + repository: string, + workflowSha: string, +): WorkflowJobAnnotation[] | null { + if (job.headSha !== workflowSha || !Array.isArray(job.annotations)) return null; + const blobPrefix = `https://github.com/${repository}/blob/${workflowSha}/`; + if ( + job.annotations.some((annotation) => annotation.blobHref !== `${blobPrefix}${annotation.path}`) + ) { + return null; + } + return job.annotations; +} + +function hasExactFailureAnnotation( + job: WorkflowJob, + repository: string, + workflowSha: string, + message: string, + exactFirstLine: boolean, +): boolean { + const annotations = trustedWorkflowJobAnnotations(job, repository, workflowSha); + if (!annotations) return false; + const failures = annotations.filter((annotation) => annotation.annotationLevel === "failure"); + const failure = failures[0]; + return ( + failures.length === 1 && + failure?.path === ".github" && + (exactFirstLine + ? failure.startLine === 1 && failure.endLine === 1 + : failure.startLine === failure.endLine) && + failure.startColumn === null && + failure.endColumn === null && + failure.title === "" && + failure.rawDetails === "" && + failure.message === message + ); +} + +function hasTrustedHostedRunnerLossAnnotation( + job: WorkflowJob, + repository: string, + workflowSha: string, +): boolean { + return hasExactFailureAnnotation( + job, + repository, + workflowSha, + HOSTED_RUNNER_LOST_COMMUNICATION_MESSAGE, + true, + ); +} + +function hasCompatibleHostedRunnerShutdownAnnotations( + job: WorkflowJob, + repository: string, + workflowSha: string, + expectedMessage: string, +): boolean { + return hasExactFailureAnnotation(job, repository, workflowSha, expectedMessage, false); +} + +function hasExactGitHubCheckEvidence( + job: WorkflowJob, + repository: string, + workflowSha: string, +): boolean { + const check = job.checkEvidence; + if ( + !check || + !Number.isSafeInteger(job.runId) || + (job.runId ?? 0) < 1 || + !Number.isSafeInteger(job.runAttempt) || + (job.runAttempt ?? 0) < 1 + ) { + return false; + } + const apiRepository = `https://api.github.com/repos/${repository}`; + const webRepository = `https://github.com/${repository}`; + const expectedRunUrl = `${apiRepository}/actions/runs/${job.runId}`; + const expectedJobUrl = `${apiRepository}/actions/jobs/${job.id}`; + const expectedCheckRunUrl = `${apiRepository}/check-runs/${job.id}`; + const expectedHtmlUrl = `${webRepository}/actions/runs/${job.runId}/job/${job.id}`; + return ( + job.headSha === workflowSha && + job.runUrl === expectedRunUrl && + job.apiUrl === expectedJobUrl && + job.htmlUrl === expectedHtmlUrl && + job.checkRunUrl === expectedCheckRunUrl && + check.id === job.id && + check.name === job.name && + check.headSha === workflowSha && + check.apiUrl === expectedCheckRunUrl && + check.htmlUrl === expectedHtmlUrl && + check.detailsUrl === expectedHtmlUrl && + check.status === "completed" && + check.conclusion === job.conclusion && + check.appId === GITHUB_ACTIONS_APP_ID && + check.appSlug === "github-actions" && + Number.isSafeInteger(check.annotationsCount) && + check.annotationsCount >= 0 && + check.annotationsCount <= MAX_RUNNER_LOSS_JOB_ANNOTATIONS && + check.annotationsCount === job.annotations?.length && + check.annotationsUrl === `${expectedCheckRunUrl}/annotations` + ); +} + +function jobLogTimestampSecond(timestamp: string): string | null { + const second = `${timestamp.slice(0, 19)}Z`; + const milliseconds = Date.parse(second); + return Number.isFinite(milliseconds) && + new Date(milliseconds).toISOString().slice(0, 19) === timestamp.slice(0, 19) + ? second + : null; +} + +function parseHostedRunnerShutdownLogTail(logTail: string): HostedRunnerShutdownLogMarker | null { + if (!logTail.endsWith("\n") || logTail.endsWith("\n\n")) return null; + const lines = logTail.slice(0, -1).split("\n"); + const shutdownMessage = `##[error]${HOSTED_RUNNER_SHUTDOWN_MESSAGE}`; + const shutdownIndex = lines + .map((line) => JOB_LOG_TIMESTAMPED_LINE_PATTERN.exec(line)?.[2] ?? "") + .lastIndexOf(shutdownMessage); + if (shutdownIndex < 0) return null; + const terminalLines = lines.slice(shutdownIndex); + if (terminalLines.length < 3 || terminalLines.length > 3 + MAX_RUNNER_LOSS_ORPHAN_PROCESSES) { + return null; + } + if (terminalLines.some((line) => line.includes("\r"))) return null; + const parsed = terminalLines.map((line) => JOB_LOG_TIMESTAMPED_LINE_PATTERN.exec(line)); + if (parsed.some((line) => line === null)) return null; + const timestamps = parsed.map((line) => line?.[1] ?? ""); + const timestampSeconds = timestamps.map(jobLogTimestampSecond); + const messages = parsed.map((line) => line?.[2] ?? ""); + const terminalMessage = messages[1]; + const interruptedStepConclusion = + terminalMessage === `##[error]${HOSTED_RUNNER_OPERATION_CANCELLED_MESSAGE}` + ? "cancelled" + : terminalMessage === `##[error]${HOSTED_RUNNER_EXIT_143_MESSAGE}` + ? "failure" + : null; + if ( + timestampSeconds.some((timestamp) => timestamp === null) || + messages[0] !== shutdownMessage || + interruptedStepConclusion === null || + messages[2] !== HOSTED_RUNNER_ORPHAN_CLEANUP_MESSAGE || + timestamps[0]! >= timestamps[1]! || + timestamps.slice(1).some((timestamp, index) => timestamp < timestamps[index]!) + ) { + return null; + } + const orphanProcesses = messages + .slice(3) + .map((message) => JOB_LOG_ORPHAN_PROCESS_PATTERN.exec(message)); + const orphanProcessIds = orphanProcesses.map((process) => process?.[1] ?? ""); + if ( + orphanProcesses.some((process) => process === null) || + new Set(orphanProcessIds).size !== orphanProcessIds.length + ) { + return null; + } + return { + shutdownTimestamp: timestamps[0]!, + terminalTimestamp: timestamps[1]!, + cleanupTimestamp: timestamps[2]!, + lastTimestamp: timestamps.at(-1)!, + annotationMessage: terminalMessage!.slice("##[error]".length), + interruptedStepConclusion, + }; +} + +function isBoundedWorkflowJobLogEvidence(evidence: WorkflowJobLogEvidence): boolean { + const tailBytes = Buffer.byteLength(evidence.tail, "utf8"); + return ( + /^"[^"\r\n]{1,128}"$/u.test(evidence.etag) && + Number.isSafeInteger(evidence.totalBytes) && + evidence.totalBytes > 0 && + tailBytes > 0 && + tailBytes <= evidence.totalBytes && + tailBytes <= MAX_RUNNER_LOSS_JOB_LOG_TAIL_BYTES + ); +} + +function hasTrustedHostedRunnerShutdownLog( + job: WorkflowJob, + repository: string, + workflowSha: string, +): boolean { + const evidence = job.logEvidence; + if (!evidence || !isBoundedWorkflowJobLogEvidence(evidence)) return false; + const marker = parseHostedRunnerShutdownLogTail(evidence.tail); + if ( + !marker || + !hasCompatibleHostedRunnerShutdownAnnotations( + job, + repository, + workflowSha, + marker.annotationMessage, + ) || + !hasTrustedHostedRunnerLossStepShapeForConclusion(job, marker.interruptedStepConclusion, { + allowLegacyStrandedStep: false, + }) + ) { + return false; + } + const interruptedSteps = job.steps.filter( + (step) => step.status === "completed" && step.conclusion === marker.interruptedStepConclusion, + ); + const interruptedStep = interruptedSteps[0]; + if ( + interruptedSteps.length !== 1 || + !job.startedAt || + !job.completedAt || + !interruptedStep?.startedAt || + !interruptedStep.completedAt + ) { + return false; + } + const shutdownSecond = jobLogTimestampSecond(marker.shutdownTimestamp); + const terminalSecond = jobLogTimestampSecond(marker.terminalTimestamp); + const cleanupSecond = jobLogTimestampSecond(marker.cleanupTimestamp); + const lastSecond = jobLogTimestampSecond(marker.lastTimestamp); + return ( + shutdownSecond !== null && + terminalSecond !== null && + cleanupSecond !== null && + lastSecond !== null && + job.startedAt <= interruptedStep.startedAt && + interruptedStep.startedAt <= shutdownSecond && + terminalSecond === interruptedStep.completedAt && + interruptedStep.completedAt <= cleanupSecond && + cleanupSecond <= lastSecond && + lastSecond <= job.completedAt + ); +} + +function hasGitHubInternalErrorMarker( + job: WorkflowJob, + repository: string, + workflowSha: string, + policy: NormalizedInternalErrorPolicy | null, +): boolean { + return ( + hasGitHubInternalErrorInspectionShape(job, policy) && + hasExactGitHubCheckEvidence(job, repository, workflowSha) && + hasExactFailureAnnotation(job, repository, workflowSha, GITHUB_INTERNAL_ERROR_MESSAGE, true) + ); +} + +function hasTrustedHostedRunnerLossMarker( + job: WorkflowJob, + repository: string, + workflowSha: string, + internalPolicy: NormalizedInternalErrorPolicy | null, +): boolean { + return ( + (hasTrustedHostedRunnerLossStepShape(job) && + hasTrustedHostedRunnerLossAnnotation(job, repository, workflowSha)) || + hasTrustedHostedRunnerShutdownLog(job, repository, workflowSha) || + hasGitHubInternalErrorMarker(job, repository, workflowSha, internalPolicy) + ); +} + +export function needsHostedRunnerShutdownLog( + job: WorkflowJob, + repository: string, + workflowSha: string, + policy: HostedRunnerLossPolicy = {}, +): boolean { + const internalPolicy = normalizedInternalErrorPolicy(policy); + if (!isHostedRunnerLossInspectionCandidateWithPolicy(job, internalPolicy)) return false; + if (hasGitHubInternalErrorInspectionShape(job, internalPolicy)) return false; + return ( + !hasTrustedHostedRunnerLossAnnotation(job, repository, workflowSha) && + (hasCompatibleHostedRunnerShutdownAnnotations( + job, + repository, + workflowSha, + HOSTED_RUNNER_OPERATION_CANCELLED_MESSAGE, + ) || + hasCompatibleHostedRunnerShutdownAnnotations( + job, + repository, + workflowSha, + HOSTED_RUNNER_EXIT_143_MESSAGE, + )) + ); +} + +export function verifiedRunnerLossEvidence(options: { + repository: string; + workflowSha: string; + workflowConclusion: string | null; + jobs: readonly WorkflowJob[]; + jobDetailsAvailable: boolean; + jobDetailsComplete: boolean; + policy?: HostedRunnerLossPolicy; +}): WorkflowAttemptEvidence | null { + const internalPolicy = normalizedInternalErrorPolicy(options.policy ?? {}); + if ( + !options.jobDetailsAvailable || + !options.jobDetailsComplete || + options.jobs.length === 0 || + options.workflowConclusion !== "failure" + ) { + return null; + } + const hasTrustedMarker = (job: WorkflowJob): boolean => + hasTrustedHostedRunnerLossMarker(job, options.repository, options.workflowSha, internalPolicy); + const runnerLostMarkerCount = options.jobs.filter(hasTrustedMarker).length; + const otherNonPassingEvidencePresent = options.jobs.some((job) => !hasTrustedMarker(job)); + return { + terminalClassificationPresent: otherNonPassingEvidencePresent, + jobConclusion: "failure", + runnerLostMarkerCount, + }; +} diff --git a/tools/e2e/pr-e2e-gate.mts b/tools/e2e/pr-e2e-gate.mts index b365bef56c..322009b7a9 100755 --- a/tools/e2e/pr-e2e-gate.mts +++ b/tools/e2e/pr-e2e-gate.mts @@ -23,6 +23,18 @@ import { riskPlanRequiredTargetIds, } from "../advisors/risk-plan.mts"; import { SHARED_E2E_JOB_ID } from "./credential-free-tests.mts"; +import { + type HostedRunnerLossPolicy, + isHostedRunnerLossInspectionCandidate, + MAX_RUNNER_LOSS_JOB_ANNOTATIONS, + MAX_RUNNER_LOSS_JOB_LOG_TAIL_BYTES, + needsHostedRunnerShutdownLog, + verifiedRunnerLossEvidence, + type WorkflowJob, + type WorkflowJobAnnotation, + type WorkflowJobCheckEvidence, + type WorkflowJobLogEvidence, +} from "./hosted-runner-loss.mts"; import { readPrivateRegularFile, writePrivateRegularFile } from "./private-file.mts"; import type { E2eRiskSignal } from "./risk-signal.ts"; import { @@ -86,26 +98,12 @@ const MAX_COMPATIBILITY_FILES = 300; const MAX_ACTIVE_RUN_PAGES_PER_STATUS = 10; const MAX_WORKFLOW_JOB_PAGES = 10; const MAX_JOB_ANNOTATION_PAGES = 1; -const MAX_RUNNER_LOSS_JOB_ANNOTATIONS = 20; const MAX_JOB_ANNOTATION_IDENTITY_BYTES = 8 * 1024; const MAX_JOB_ANNOTATION_TEXT_BYTES = 16 * 1024; const MAX_RUNNER_LOSS_JOB_ANNOTATION_BYTES = 64 * 1024; -const HOSTED_RUNNER_LOST_COMMUNICATION_MESSAGE = - "The hosted runner lost communication with the server. Anything in your workflow that terminates the runner process, starves it for CPU/Memory, or blocks its network access can cause this error."; -const HOSTED_RUNNER_SHUTDOWN_MESSAGE = - "The runner has received a shutdown signal. This can happen when the runner service is stopped, or a manually started runner is canceled."; -const HOSTED_RUNNER_OPERATION_CANCELLED_MESSAGE = "The operation was canceled."; -const HOSTED_RUNNER_EXIT_143_MESSAGE = "Process completed with exit code 143."; -const HOSTED_RUNNER_ORPHAN_CLEANUP_MESSAGE = "Cleaning up orphan processes"; const MAX_RUNNER_LOSS_JOB_INSPECTIONS = 20; -const MAX_RUNNER_LOSS_JOB_LOG_TAIL_BYTES = 64 * 1024; -const MAX_RUNNER_LOSS_ORPHAN_PROCESSES = 64; const RUNNER_LOSS_JOB_LOG_TIMEOUT_MS = 30_000; const JOB_LOG_DOWNLOAD_HOST_PATTERN = /^productionresultssa[0-9]+\.blob\.core\.windows\.net$/u; -const JOB_LOG_TIMESTAMPED_LINE_PATTERN = - /^([0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{7}Z) (.*)$/u; -const JOB_LOG_ORPHAN_PROCESS_PATTERN = - /^Terminate orphan process: pid \(([1-9][0-9]*)\) \(([A-Za-z0-9._+ -]{1,128})\)$/u; const GITHUB_TIMESTAMP_PATTERN = /^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$/u; const MAX_REPORTED_WORKFLOW_JOBS = 10; const MAX_WAIVER_REASON_CHARS = 500; @@ -119,6 +117,7 @@ const ACTIVE_WORKFLOW_RUN_STATUSES = [ "in_progress", ] as const; const ACTIVE_WORKFLOW_RUN_STATUS_SET = new Set(ACTIVE_WORKFLOW_RUN_STATUSES); +const PR_E2E_HOSTED_RUNNER_LOSS_POLICY: HostedRunnerLossPolicy = {}; const TERMINAL_WORKFLOW_RUN_CONCLUSIONS = [ "success", "failure", @@ -261,60 +260,6 @@ type WorkflowRun = { }; type WorkflowRunsResponse = { workflow_runs: WorkflowRun[] }; -type WorkflowJobAnnotation = { - path: string; - blobHref: string; - startLine: number; - startColumn: number | null; - endLine: number; - endColumn: number | null; - annotationLevel: string; - title: string; - message: string; - rawDetails: string; -}; -type WorkflowJobLogEvidence = { - etag: string; - totalBytes: number; - tail: string; -}; -type HostedRunnerShutdownLogMarker = { - shutdownTimestamp: string; - terminalTimestamp: string; - cleanupTimestamp: string; - lastTimestamp: string; - annotationMessage: string; - interruptedStepConclusion: "cancelled" | "failure"; -}; -type WorkflowJob = { - id: number; - name: string; - runId?: number; - runAttempt?: number; - headSha?: string; - runUrl?: string; - apiUrl?: string; - htmlUrl?: string; - checkRunUrl?: string; - status?: string; - conclusion: string | null; - runnerId?: number | null; - runnerName?: string | null; - runnerGroupId?: number | null; - runnerGroupName?: string | null; - labels?: string[]; - annotations?: WorkflowJobAnnotation[]; - logEvidence?: WorkflowJobLogEvidence; - startedAt?: string | null; - completedAt?: string | null; - steps: Array<{ - name: string; - status?: string; - conclusion: string | null; - startedAt?: string | null; - completedAt?: string | null; - }>; -}; type WorkflowJobsPage = { totalCount: number; jobs: WorkflowJob[] }; type CheckRun = { id: number; @@ -1835,7 +1780,10 @@ async function listWorkflowJobAnnotations( job: WorkflowJob, runId: number, runAttempt: number, -): Promise { +): Promise<{ + annotations: WorkflowJobAnnotation[]; + checkEvidence: WorkflowJobCheckEvidence; +}> { const apiRepository = `https://api.github.com/repos/${repository}`; const webRepository = `https://github.com/${repository}`; const expectedRunUrl = `${apiRepository}/actions/runs/${runId}`; @@ -1866,9 +1814,10 @@ async function listWorkflowJobAnnotations( check.html_url !== expectedHtmlUrl || check.details_url !== expectedHtmlUrl || check.status !== "completed" || - check.conclusion !== "failure" || + check.conclusion !== job.conclusion || !isObjectRecord(check.app) || check.app.id !== GITHUB_ACTIONS_APP_ID || + check.app.slug !== "github-actions" || !isObjectRecord(check.output) || !Number.isSafeInteger(check.output.annotations_count) || (check.output.annotations_count as number) < 0 || @@ -1877,6 +1826,20 @@ async function listWorkflowJobAnnotations( throw new Error("workflow job check run does not match the exact failed job"); } const expectedCount = check.output.annotations_count as number; + const checkEvidence: WorkflowJobCheckEvidence = { + id: check.id as number, + name: check.name as string, + headSha: check.head_sha as string, + apiUrl: check.url as string, + htmlUrl: check.html_url as string, + detailsUrl: check.details_url as string, + status: check.status as string, + conclusion: check.conclusion as string, + appId: check.app.id as number, + appSlug: check.app.slug as string, + annotationsCount: expectedCount, + annotationsUrl: check.output.annotations_url as string, + }; if (expectedCount > MAX_RUNNER_LOSS_JOB_ANNOTATIONS) { throw new Error("workflow job annotation count exceeds the hosted-runner-loss limit"); } @@ -1908,7 +1871,7 @@ async function listWorkflowJobAnnotations( if (annotations.length > expectedCount) { throw new Error("workflow job annotation listing exceeds the trusted annotation count"); } - if (annotations.length === expectedCount) return annotations; + if (annotations.length === expectedCount) return { annotations, checkEvidence }; if (value.length < MAX_RUNNER_LOSS_JOB_ANNOTATIONS) { throw new Error("workflow job annotation listing is incomplete"); } @@ -2127,7 +2090,10 @@ async function listNonPassingWorkflowJobs( token: string, runId: number, runAttempt: number, - options: { includeAnnotations?: boolean } = {}, + options: { + includeAnnotations?: boolean; + hostedRunnerLossPolicy?: HostedRunnerLossPolicy; + } = {}, ): Promise<{ jobs: WorkflowJob[]; complete: boolean }> { if ( !Number.isSafeInteger(runId) || @@ -2164,36 +2130,25 @@ async function listNonPassingWorkflowJobs( (job) => !["success", "skipped", "neutral"].includes(job.conclusion ?? ""), ); if (options.includeAnnotations) { - const runnerLossCandidates = nonPassingJobs.filter( - hasTrustedHostedRunnerLossInspectionStepShape, + const hostedRunnerLossPolicy = options.hostedRunnerLossPolicy ?? {}; + const runnerLossCandidates = nonPassingJobs.filter((job) => + isHostedRunnerLossInspectionCandidate(job, hostedRunnerLossPolicy), ); if (runnerLossCandidates.length > MAX_RUNNER_LOSS_JOB_INSPECTIONS) { throw new Error("workflow run exceeded the hosted-runner-loss inspection limit"); } for (const job of runnerLossCandidates) { - job.annotations = await listWorkflowJobAnnotations( + const evidence = await listWorkflowJobAnnotations( repository, token, job, runId, runAttempt, ); + job.annotations = evidence.annotations; + job.checkEvidence = evidence.checkEvidence; const workflowSha = job.headSha ?? ""; - if ( - !hasTrustedHostedRunnerLossAnnotation(job, repository, workflowSha) && - (hasCompatibleHostedRunnerShutdownAnnotations( - job, - repository, - workflowSha, - HOSTED_RUNNER_OPERATION_CANCELLED_MESSAGE, - ) || - hasCompatibleHostedRunnerShutdownAnnotations( - job, - repository, - workflowSha, - HOSTED_RUNNER_EXIT_143_MESSAGE, - )) - ) { + if (needsHostedRunnerShutdownLog(job, repository, workflowSha, hostedRunnerLossPolicy)) { try { job.logEvidence = await downloadWorkflowJobLogTail(repository, token, job.id); } catch { @@ -2339,332 +2294,6 @@ function ciFailureReport(options: { }; } -const GITHUB_HOSTED_RUNNER_NAME_PATTERN = /^GitHub Actions [1-9][0-9]*$/u; - -function trustedWorkflowJobAnnotations( - job: WorkflowJob, - repository: string, - workflowSha: string, -): WorkflowJobAnnotation[] | null { - if (job.headSha !== workflowSha || !Array.isArray(job.annotations)) return null; - const blobPrefix = `https://github.com/${repository}/blob/${workflowSha}/`; - if ( - job.annotations.some((annotation) => annotation.blobHref !== `${blobPrefix}${annotation.path}`) - ) { - return null; - } - return job.annotations; -} - -/** - * GitHub records a lost hosted runner as a completed failed job with no - * ordinary failed step. Older Jobs API responses left the interrupted step - * `in_progress`; current responses can terminalize it as `cancelled`, skip the - * remaining cleanup, and append the synthetic successful `Complete job` step. - * A user or concurrency cancellation concludes the job itself as `cancelled`, - * while an ordinary assertion records a failed step. The step shape must be - * paired with either a canonical GitHub runner-loss annotation or an - * authenticated exact terminal shutdown log. - */ -function hasTrustedHostedRunnerLossAnnotation( - job: WorkflowJob, - repository: string, - workflowSha: string, -): boolean { - const annotations = trustedWorkflowJobAnnotations(job, repository, workflowSha); - if (!annotations) return false; - const failures = annotations.filter((annotation) => annotation.annotationLevel === "failure"); - return ( - failures.length === 1 && - failures[0]?.path === ".github" && - failures[0].startLine === 1 && - failures[0].startColumn === null && - failures[0].endLine === 1 && - failures[0].endColumn === null && - failures[0].title === "" && - failures[0].rawDetails === "" && - failures[0].message === HOSTED_RUNNER_LOST_COMMUNICATION_MESSAGE - ); -} - -function hasCompatibleHostedRunnerShutdownAnnotations( - job: WorkflowJob, - repository: string, - workflowSha: string, - expectedMessage: string, -): boolean { - const annotations = trustedWorkflowJobAnnotations(job, repository, workflowSha); - if (!annotations) return false; - const failures = annotations.filter((annotation) => annotation.annotationLevel === "failure"); - const failure = failures[0]; - return ( - failures.length === 1 && - failure?.path === ".github" && - failure.startLine === failure.endLine && - failure.startColumn === null && - failure.endColumn === null && - failure.title === "" && - failure.rawDetails === "" && - failure.message === expectedMessage - ); -} - -function jobLogTimestampSecond(timestamp: string): string | null { - const second = `${timestamp.slice(0, 19)}Z`; - const milliseconds = Date.parse(second); - return Number.isFinite(milliseconds) && - new Date(milliseconds).toISOString().slice(0, 19) === timestamp.slice(0, 19) - ? second - : null; -} - -function parseHostedRunnerShutdownLogTail(logTail: string): HostedRunnerShutdownLogMarker | null { - if (!logTail.endsWith("\n") || logTail.endsWith("\n\n")) return null; - const lines = logTail.slice(0, -1).split("\n"); - const shutdownMessage = `##[error]${HOSTED_RUNNER_SHUTDOWN_MESSAGE}`; - const shutdownIndex = lines - .map((line) => JOB_LOG_TIMESTAMPED_LINE_PATTERN.exec(line)?.[2] ?? "") - .lastIndexOf(shutdownMessage); - if (shutdownIndex < 0) return null; - const terminalLines = lines.slice(shutdownIndex); - if (terminalLines.length < 3 || terminalLines.length > 3 + MAX_RUNNER_LOSS_ORPHAN_PROCESSES) { - return null; - } - if (terminalLines.some((line) => line.includes("\r"))) return null; - const parsed = terminalLines.map((line) => JOB_LOG_TIMESTAMPED_LINE_PATTERN.exec(line)); - if (parsed.some((line) => line === null)) return null; - const timestamps = parsed.map((line) => line?.[1] ?? ""); - const timestampSeconds = timestamps.map(jobLogTimestampSecond); - const messages = parsed.map((line) => line?.[2] ?? ""); - const terminalMessage = messages[1]; - const interruptedStepConclusion = - terminalMessage === `##[error]${HOSTED_RUNNER_OPERATION_CANCELLED_MESSAGE}` - ? "cancelled" - : terminalMessage === `##[error]${HOSTED_RUNNER_EXIT_143_MESSAGE}` - ? "failure" - : null; - if ( - timestampSeconds.some((timestamp) => timestamp === null) || - messages[0] !== shutdownMessage || - interruptedStepConclusion === null || - messages[2] !== HOSTED_RUNNER_ORPHAN_CLEANUP_MESSAGE || - timestamps[0]! >= timestamps[1]! || - timestamps.slice(1).some((timestamp, index) => timestamp < timestamps[index]!) - ) { - return null; - } - const orphanProcesses = messages - .slice(3) - .map((message) => JOB_LOG_ORPHAN_PROCESS_PATTERN.exec(message)); - const orphanProcessIds = orphanProcesses.map((process) => process?.[1] ?? ""); - if ( - orphanProcesses.some((process) => process === null) || - new Set(orphanProcessIds).size !== orphanProcessIds.length - ) { - return null; - } - return { - shutdownTimestamp: timestamps[0]!, - terminalTimestamp: timestamps[1]!, - cleanupTimestamp: timestamps[2]!, - lastTimestamp: timestamps.at(-1)!, - annotationMessage: terminalMessage!.slice("##[error]".length), - interruptedStepConclusion, - }; -} - -function isBoundedWorkflowJobLogEvidence(evidence: WorkflowJobLogEvidence): boolean { - const tailBytes = Buffer.byteLength(evidence.tail, "utf8"); - return ( - /^"[^"\r\n]{1,128}"$/u.test(evidence.etag) && - Number.isSafeInteger(evidence.totalBytes) && - evidence.totalBytes > 0 && - tailBytes > 0 && - tailBytes <= evidence.totalBytes && - tailBytes <= MAX_RUNNER_LOSS_JOB_LOG_TAIL_BYTES - ); -} - -function hasTrustedHostedRunnerShutdownLog( - job: WorkflowJob, - repository: string, - workflowSha: string, -): boolean { - const evidence = job.logEvidence; - if (!evidence || !isBoundedWorkflowJobLogEvidence(evidence)) return false; - const marker = parseHostedRunnerShutdownLogTail(evidence.tail); - if ( - !marker || - !hasCompatibleHostedRunnerShutdownAnnotations( - job, - repository, - workflowSha, - marker.annotationMessage, - ) || - !hasTrustedHostedRunnerLossStepShapeForConclusion(job, marker.interruptedStepConclusion, { - allowLegacyStrandedStep: false, - }) - ) { - return false; - } - const interruptedSteps = job.steps.filter( - (step) => step.status === "completed" && step.conclusion === marker.interruptedStepConclusion, - ); - const interruptedStep = interruptedSteps[0]; - if ( - interruptedSteps.length !== 1 || - !job.startedAt || - !job.completedAt || - !interruptedStep?.startedAt || - !interruptedStep.completedAt - ) { - return false; - } - const shutdownSecond = jobLogTimestampSecond(marker.shutdownTimestamp); - const terminalSecond = jobLogTimestampSecond(marker.terminalTimestamp); - const cleanupSecond = jobLogTimestampSecond(marker.cleanupTimestamp); - const lastSecond = jobLogTimestampSecond(marker.lastTimestamp); - return ( - shutdownSecond !== null && - terminalSecond !== null && - cleanupSecond !== null && - lastSecond !== null && - job.startedAt <= interruptedStep.startedAt && - interruptedStep.startedAt <= shutdownSecond && - terminalSecond === interruptedStep.completedAt && - interruptedStep.completedAt <= cleanupSecond && - cleanupSecond <= lastSecond && - lastSecond <= job.completedAt - ); -} - -function hasTrustedHostedRunnerLossStepShapeForConclusion( - job: WorkflowJob, - interruptedStepConclusion: "cancelled" | "failure", - options: { allowLegacyStrandedStep: boolean }, -): boolean { - if ( - job.status !== "completed" || - job.conclusion !== "failure" || - !Number.isSafeInteger(job.runnerId) || - (job.runnerId ?? 0) < 1 || - typeof job.runnerName !== "string" || - !GITHUB_HOSTED_RUNNER_NAME_PATTERN.test(job.runnerName) || - job.runnerGroupId !== 0 || - job.runnerGroupName !== "GitHub Actions" || - !Array.isArray(job.labels) || - !job.labels.includes("ubuntu-latest") || - job.labels.includes("self-hosted") - ) { - return false; - } - const strandedSteps = job.steps.filter( - (step) => step.status === "in_progress" && step.conclusion === null, - ); - const strandedIndex = job.steps.findIndex( - (step) => step.status === "in_progress" && step.conclusion === null, - ); - const legacyStrandedStep = - strandedSteps.length === 1 && - job.steps - .slice(0, strandedIndex) - .every( - (step) => - step.status === "completed" && ["success", "skipped"].includes(step.conclusion ?? ""), - ) && - job.steps - .slice(strandedIndex + 1) - .every((step) => step.status === "pending" && step.conclusion === null); - if ( - options.allowLegacyStrandedStep && - interruptedStepConclusion === "cancelled" && - legacyStrandedStep - ) { - return true; - } - - const interruptedStepIndexes = job.steps.flatMap((step, index) => - step.status === "completed" && step.conclusion === interruptedStepConclusion ? [index] : [], - ); - if (interruptedStepIndexes.length !== 1) return false; - const interruptedIndex = interruptedStepIndexes[0]!; - if (job.steps[interruptedIndex]?.name === "Complete job") return false; - const beforeInterruption = job.steps.slice(0, interruptedIndex); - const afterInterruption = job.steps.slice(interruptedIndex + 1); - const syntheticCompletion = afterInterruption.at(-1); - const skippedCleanup = afterInterruption.slice(0, -1); - return ( - beforeInterruption.every( - (step) => - step.status === "completed" && ["success", "skipped"].includes(step.conclusion ?? ""), - ) && - skippedCleanup.length > 0 && - skippedCleanup.every( - (step) => - step.name !== "Complete job" && - step.status === "completed" && - step.conclusion === "skipped", - ) && - syntheticCompletion?.name === "Complete job" && - syntheticCompletion.status === "completed" && - syntheticCompletion.conclusion === "success" - ); -} - -function hasTrustedHostedRunnerLossStepShape(job: WorkflowJob): boolean { - return hasTrustedHostedRunnerLossStepShapeForConclusion(job, "cancelled", { - allowLegacyStrandedStep: true, - }); -} - -function hasTrustedHostedRunnerLossInspectionStepShape(job: WorkflowJob): boolean { - return ( - hasTrustedHostedRunnerLossStepShape(job) || - hasTrustedHostedRunnerLossStepShapeForConclusion(job, "failure", { - allowLegacyStrandedStep: false, - }) - ); -} - -function hasTrustedHostedRunnerLossMarker( - job: WorkflowJob, - repository: string, - workflowSha: string, -): boolean { - return ( - (hasTrustedHostedRunnerLossStepShape(job) && - hasTrustedHostedRunnerLossAnnotation(job, repository, workflowSha)) || - hasTrustedHostedRunnerShutdownLog(job, repository, workflowSha) - ); -} - -export function verifiedRunnerLossEvidence(options: { - repository: string; - workflowSha: string; - workflowConclusion: string | null; - jobs: readonly WorkflowJob[]; - jobDetailsAvailable: boolean; - jobDetailsComplete: boolean; -}): WorkflowAttemptEvidence | null { - if ( - !options.jobDetailsAvailable || - !options.jobDetailsComplete || - options.jobs.length === 0 || - options.workflowConclusion !== "failure" - ) { - return null; - } - const hasTrustedMarker = (job: WorkflowJob): boolean => - hasTrustedHostedRunnerLossMarker(job, options.repository, options.workflowSha); - const runnerLostMarkerCount = options.jobs.filter(hasTrustedMarker).length; - const otherNonPassingEvidencePresent = options.jobs.some((job) => !hasTrustedMarker(job)); - return { - terminalClassificationPresent: otherNonPassingEvidencePresent, - jobConclusion: "failure", - runnerLostMarkerCount, - }; -} - export function e2eFailureReport(options: { repository: string; runId: number; @@ -3520,6 +3149,7 @@ export async function retryRunnerLossPrGate( const jobDetails = await listNonPassingWorkflowJobs(repository, token, command.childRunId, 1, { includeAnnotations: true, + hostedRunnerLossPolicy: PR_E2E_HOSTED_RUNNER_LOSS_POLICY, }); await requireUnchangedCompletedWorkflowRun(repository, token, child, { childRunId: command.childRunId, @@ -3536,6 +3166,7 @@ export async function retryRunnerLossPrGate( jobs: jobDetails.jobs, jobDetailsAvailable: true, jobDetailsComplete: jobDetails.complete, + policy: PR_E2E_HOSTED_RUNNER_LOSS_POLICY, }); const retryDecision = runnerLossEvidence ? decideRetry({ @@ -3564,7 +3195,10 @@ export async function retryRunnerLossPrGate( token, command.childRunId, 1, - { includeAnnotations: true }, + { + includeAnnotations: true, + hostedRunnerLossPolicy: PR_E2E_HOSTED_RUNNER_LOSS_POLICY, + }, ); await requireUnchangedCompletedWorkflowRun(repository, token, child, { childRunId: command.childRunId, @@ -3583,6 +3217,7 @@ export async function retryRunnerLossPrGate( jobs: confirmedJobDetails.jobs, jobDetailsAvailable: true, jobDetailsComplete: confirmedJobDetails.complete, + policy: PR_E2E_HOSTED_RUNNER_LOSS_POLICY, }); const confirmedRetryDecision = confirmedRunnerLossEvidence ? decideRetry({ @@ -4368,6 +4003,7 @@ export async function finishPrGate(options: { try { const details = await listNonPassingWorkflowJobs(repository, token, options.childRunId, 1, { includeAnnotations: true, + hostedRunnerLossPolicy: PR_E2E_HOSTED_RUNNER_LOSS_POLICY, }); await requireUnchangedCompletedWorkflowRun(repository, token, child, { childRunId: options.childRunId, @@ -4397,6 +4033,7 @@ export async function finishPrGate(options: { jobs, jobDetailsAvailable, jobDetailsComplete, + policy: PR_E2E_HOSTED_RUNNER_LOSS_POLICY, }), }); } From 74a78a30d2e776c61edbdca42cfdfc4ccc6fc6f5 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Sat, 25 Jul 2026 08:05:01 -0700 Subject: [PATCH 02/10] refactor(ci): expose runner-loss evidence collector Signed-off-by: Apurv Kumaria --- tools/e2e/pr-e2e-gate.mts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/e2e/pr-e2e-gate.mts b/tools/e2e/pr-e2e-gate.mts index 322009b7a9..6afbe61072 100755 --- a/tools/e2e/pr-e2e-gate.mts +++ b/tools/e2e/pr-e2e-gate.mts @@ -2085,7 +2085,7 @@ function validateWorkflowJobsPage(value: unknown): WorkflowJobsPage { }; } -async function listNonPassingWorkflowJobs( +export async function listNonPassingWorkflowJobs( repository: string, token: string, runId: number, @@ -2655,7 +2655,7 @@ async function requireUnchangedCompletedWorkflowRun( } } -function workflowJobEvidenceFingerprint(details: { +export function workflowJobEvidenceFingerprint(details: { jobs: readonly WorkflowJob[]; complete: boolean; }): string { From b0cbeade2183752520a6b95279844af6f68811f8 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Sat, 25 Jul 2026 08:13:56 -0700 Subject: [PATCH 03/10] refactor(ci): isolate runner-loss GitHub evidence Signed-off-by: Apurv Kumaria --- tools/e2e/hosted-runner-loss-github.mts | 602 ++++++++++++++++++++++++ tools/e2e/pr-e2e-gate.mts | 588 +---------------------- 2 files changed, 611 insertions(+), 579 deletions(-) create mode 100644 tools/e2e/hosted-runner-loss-github.mts diff --git a/tools/e2e/hosted-runner-loss-github.mts b/tools/e2e/hosted-runner-loss-github.mts new file mode 100644 index 0000000000..0328121117 --- /dev/null +++ b/tools/e2e/hosted-runner-loss-github.mts @@ -0,0 +1,602 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; + +import { githubApi } from "../advisors/github.mts"; +import { + type HostedRunnerLossPolicy, + isHostedRunnerLossInspectionCandidate, + MAX_RUNNER_LOSS_JOB_ANNOTATIONS, + MAX_RUNNER_LOSS_JOB_LOG_TAIL_BYTES, + needsHostedRunnerShutdownLog, + type WorkflowJob, + type WorkflowJobAnnotation, + type WorkflowJobCheckEvidence, + type WorkflowJobLogEvidence, +} from "./hosted-runner-loss.mts"; + +const GITHUB_ACTIONS_APP_ID = 15368; +const USER_AGENT = "nemoclaw-pr-e2e-gate"; +const SHA_PATTERN = /^[a-f0-9]{40}$/u; +const MAX_WORKFLOW_JOB_PAGES = 10; +const MAX_JOB_ANNOTATION_PAGES = 1; +const MAX_JOB_ANNOTATION_IDENTITY_BYTES = 8 * 1024; +const MAX_JOB_ANNOTATION_TEXT_BYTES = 16 * 1024; +const MAX_RUNNER_LOSS_JOB_ANNOTATION_BYTES = 64 * 1024; +const MAX_RUNNER_LOSS_JOB_INSPECTIONS = 20; +const RUNNER_LOSS_JOB_LOG_TIMEOUT_MS = 30_000; +const JOB_LOG_DOWNLOAD_HOST_PATTERN = /^productionresultssa[0-9]+\.blob\.core\.windows\.net$/u; +const GITHUB_TIMESTAMP_PATTERN = /^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$/u; + +type WorkflowJobsPage = { totalCount: number; jobs: WorkflowJob[] }; + +function isObjectRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function sha256(value: string): string { + return createHash("sha256").update(value).digest("hex"); +} + +function isOptionalGitHubTimestamp(value: unknown): boolean { + return ( + value === undefined || + value === null || + (typeof value === "string" && GITHUB_TIMESTAMP_PATTERN.test(value)) + ); +} + +function validateWorkflowJob(value: unknown): WorkflowJob { + if ( + !isObjectRecord(value) || + !Number.isSafeInteger(value.id) || + (value.id as number) < 1 || + typeof value.name !== "string" || + value.name.length === 0 || + (value.run_id !== undefined && + (!Number.isSafeInteger(value.run_id) || (value.run_id as number) < 1)) || + (value.run_attempt !== undefined && + (!Number.isSafeInteger(value.run_attempt) || (value.run_attempt as number) < 1)) || + (value.head_sha !== undefined && + (typeof value.head_sha !== "string" || !SHA_PATTERN.test(value.head_sha))) || + (value.run_url !== undefined && typeof value.run_url !== "string") || + (value.url !== undefined && typeof value.url !== "string") || + (value.html_url !== undefined && typeof value.html_url !== "string") || + (value.check_run_url !== undefined && typeof value.check_run_url !== "string") || + (value.status !== undefined && typeof value.status !== "string") || + (value.conclusion !== null && typeof value.conclusion !== "string") || + !isOptionalGitHubTimestamp(value.started_at) || + !isOptionalGitHubTimestamp(value.completed_at) || + (value.runner_id !== undefined && + value.runner_id !== null && + (!Number.isSafeInteger(value.runner_id) || (value.runner_id as number) < 1)) || + (value.runner_name !== undefined && + value.runner_name !== null && + typeof value.runner_name !== "string") || + (value.runner_group_id !== undefined && + value.runner_group_id !== null && + (!Number.isSafeInteger(value.runner_group_id) || (value.runner_group_id as number) < 0)) || + (value.runner_group_name !== undefined && + value.runner_group_name !== null && + typeof value.runner_group_name !== "string") || + (value.labels !== undefined && + (!Array.isArray(value.labels) || value.labels.some((label) => typeof label !== "string"))) || + (value.steps !== undefined && !Array.isArray(value.steps)) + ) { + throw new Error("GitHub returned an invalid workflow job"); + } + const steps = (value.steps ?? []).map((step) => { + if ( + !isObjectRecord(step) || + typeof step.name !== "string" || + step.name.length === 0 || + (step.status !== undefined && typeof step.status !== "string") || + (step.conclusion !== null && typeof step.conclusion !== "string") || + !isOptionalGitHubTimestamp(step.started_at) || + !isOptionalGitHubTimestamp(step.completed_at) + ) { + throw new Error("GitHub returned an invalid workflow job step"); + } + return { + name: step.name, + ...(step.status === undefined ? {} : { status: step.status }), + conclusion: step.conclusion, + ...(step.started_at === undefined ? {} : { startedAt: step.started_at as string | null }), + ...(step.completed_at === undefined + ? {} + : { completedAt: step.completed_at as string | null }), + }; + }); + return { + id: value.id as number, + name: value.name, + ...(value.run_id === undefined ? {} : { runId: value.run_id as number }), + ...(value.run_attempt === undefined ? {} : { runAttempt: value.run_attempt as number }), + ...(value.head_sha === undefined ? {} : { headSha: value.head_sha }), + ...(value.run_url === undefined ? {} : { runUrl: value.run_url }), + ...(value.url === undefined ? {} : { apiUrl: value.url }), + ...(value.html_url === undefined ? {} : { htmlUrl: value.html_url }), + ...(value.check_run_url === undefined ? {} : { checkRunUrl: value.check_run_url }), + ...(value.status === undefined ? {} : { status: value.status }), + conclusion: value.conclusion, + ...(value.runner_id === undefined ? {} : { runnerId: value.runner_id as number | null }), + ...(value.runner_name === undefined ? {} : { runnerName: value.runner_name }), + ...(value.runner_group_id === undefined + ? {} + : { runnerGroupId: value.runner_group_id as number | null }), + ...(value.runner_group_name === undefined ? {} : { runnerGroupName: value.runner_group_name }), + ...(value.labels === undefined ? {} : { labels: value.labels as string[] }), + ...(value.started_at === undefined ? {} : { startedAt: value.started_at as string | null }), + ...(value.completed_at === undefined + ? {} + : { completedAt: value.completed_at as string | null }), + steps, + }; +} + +function validateWorkflowJobAnnotation(value: unknown): WorkflowJobAnnotation { + if ( + !isObjectRecord(value) || + typeof value.path !== "string" || + value.path.length === 0 || + Buffer.byteLength(value.path, "utf8") > MAX_JOB_ANNOTATION_IDENTITY_BYTES || + typeof value.blob_href !== "string" || + Buffer.byteLength(value.blob_href, "utf8") > MAX_JOB_ANNOTATION_IDENTITY_BYTES || + !Number.isSafeInteger(value.start_line) || + (value.start_line as number) < 1 || + (value.start_column !== null && + (!Number.isSafeInteger(value.start_column) || (value.start_column as number) < 1)) || + !Number.isSafeInteger(value.end_line) || + (value.end_line as number) < (value.start_line as number) || + (value.end_column !== null && + (!Number.isSafeInteger(value.end_column) || (value.end_column as number) < 1)) || + typeof value.annotation_level !== "string" || + Buffer.byteLength(value.annotation_level, "utf8") > MAX_JOB_ANNOTATION_IDENTITY_BYTES || + typeof value.title !== "string" || + Buffer.byteLength(value.title, "utf8") > MAX_JOB_ANNOTATION_TEXT_BYTES || + typeof value.message !== "string" || + Buffer.byteLength(value.message, "utf8") > MAX_JOB_ANNOTATION_TEXT_BYTES || + typeof value.raw_details !== "string" || + Buffer.byteLength(value.raw_details, "utf8") > MAX_JOB_ANNOTATION_TEXT_BYTES + ) { + throw new Error("GitHub returned an invalid workflow job annotation"); + } + return { + path: value.path, + blobHref: value.blob_href, + startLine: value.start_line as number, + startColumn: value.start_column as number | null, + endLine: value.end_line as number, + endColumn: value.end_column as number | null, + annotationLevel: value.annotation_level, + title: value.title, + message: value.message, + rawDetails: value.raw_details, + }; +} + +async function listWorkflowJobAnnotations( + repository: string, + token: string, + job: WorkflowJob, + runId: number, + runAttempt: number, +): Promise<{ + annotations: WorkflowJobAnnotation[]; + checkEvidence: WorkflowJobCheckEvidence; +}> { + const apiRepository = `https://api.github.com/repos/${repository}`; + const webRepository = `https://github.com/${repository}`; + const expectedRunUrl = `${apiRepository}/actions/runs/${runId}`; + const expectedJobUrl = `${apiRepository}/actions/jobs/${job.id}`; + const expectedCheckRunUrl = `${apiRepository}/check-runs/${job.id}`; + const expectedHtmlUrl = `${webRepository}/actions/runs/${runId}/job/${job.id}`; + if ( + !job.headSha || + job.runId !== runId || + job.runAttempt !== runAttempt || + job.runUrl !== expectedRunUrl || + job.apiUrl !== expectedJobUrl || + job.htmlUrl !== expectedHtmlUrl || + job.checkRunUrl !== expectedCheckRunUrl + ) { + throw new Error("workflow job identity does not match its exact run attempt"); + } + const check = await githubApi(`repos/${repository}/check-runs/${job.id}`, token, { + userAgent: USER_AGENT, + }); + const expectedAnnotationsUrl = `${expectedCheckRunUrl}/annotations`; + if ( + !isObjectRecord(check) || + check.id !== job.id || + check.name !== job.name || + check.head_sha !== job.headSha || + check.url !== expectedCheckRunUrl || + check.html_url !== expectedHtmlUrl || + check.details_url !== expectedHtmlUrl || + check.status !== "completed" || + check.conclusion !== job.conclusion || + !isObjectRecord(check.app) || + check.app.id !== GITHUB_ACTIONS_APP_ID || + check.app.slug !== "github-actions" || + !isObjectRecord(check.output) || + !Number.isSafeInteger(check.output.annotations_count) || + (check.output.annotations_count as number) < 0 || + check.output.annotations_url !== expectedAnnotationsUrl + ) { + throw new Error("workflow job check run does not match the exact failed job"); + } + const expectedCount = check.output.annotations_count as number; + const checkEvidence: WorkflowJobCheckEvidence = { + id: check.id as number, + name: check.name as string, + headSha: check.head_sha as string, + apiUrl: check.url as string, + htmlUrl: check.html_url as string, + detailsUrl: check.details_url as string, + status: check.status as string, + conclusion: check.conclusion as string, + appId: check.app.id as number, + appSlug: check.app.slug as string, + annotationsCount: expectedCount, + annotationsUrl: check.output.annotations_url as string, + }; + if (expectedCount > MAX_RUNNER_LOSS_JOB_ANNOTATIONS) { + throw new Error("workflow job annotation count exceeds the hosted-runner-loss limit"); + } + const annotations: WorkflowJobAnnotation[] = []; + const fingerprints = new Set(); + let annotationBytes = 0; + for (let page = 1; page <= MAX_JOB_ANNOTATION_PAGES; page += 1) { + const value = await githubApi( + `repos/${repository}/check-runs/${job.id}/annotations?per_page=${MAX_RUNNER_LOSS_JOB_ANNOTATIONS}&page=${page}`, + token, + { userAgent: USER_AGENT }, + ); + if (!Array.isArray(value) || value.length > MAX_RUNNER_LOSS_JOB_ANNOTATIONS) { + throw new Error("GitHub returned an invalid workflow job annotation listing"); + } + const pageAnnotations = value.map(validateWorkflowJobAnnotation); + for (const annotation of pageAnnotations) { + const fingerprint = JSON.stringify(annotation); + if (fingerprints.has(fingerprint)) { + throw new Error("GitHub returned duplicate workflow job annotations"); + } + fingerprints.add(fingerprint); + annotationBytes += Buffer.byteLength(fingerprint, "utf8"); + if (annotationBytes > MAX_RUNNER_LOSS_JOB_ANNOTATION_BYTES) { + throw new Error("workflow job annotation evidence exceeds its byte limit"); + } + annotations.push(annotation); + } + if (annotations.length > expectedCount) { + throw new Error("workflow job annotation listing exceeds the trusted annotation count"); + } + if (annotations.length === expectedCount) return { annotations, checkEvidence }; + if (value.length < MAX_RUNNER_LOSS_JOB_ANNOTATIONS) { + throw new Error("workflow job annotation listing is incomplete"); + } + } + throw new Error("workflow job annotation listing exceeded its page limit"); +} + +function parseJobLogContentLength(value: string | null, label: string): number { + if (!value || !/^(?:0|[1-9][0-9]*)$/u.test(value)) { + throw new Error(`${label} did not provide a valid content length`); + } + const length = Number(value); + if (!Number.isSafeInteger(length) || length < 0) { + throw new Error(`${label} content length is outside the safe integer range`); + } + return length; +} + +function validateJobLogEtag(value: string | null): string { + if (!value || value.length > 130 || !/^"[^"\r\n]{1,128}"$/u.test(value)) { + throw new Error("job log download did not provide a strong bounded ETag"); + } + return value; +} + +function validateJobLogDownloadUrl(value: string | null): URL { + let url: URL; + try { + url = new URL(value ?? ""); + } catch { + throw new Error("job log API returned an invalid signed download URL"); + } + if ( + url.protocol !== "https:" || + url.username !== "" || + url.password !== "" || + url.port !== "" || + !JOB_LOG_DOWNLOAD_HOST_PATTERN.test(url.hostname) || + !url.pathname.startsWith("/actions-results/") || + url.search.length < 2 || + url.hash !== "" + ) { + throw new Error("job log API returned an untrusted signed download URL"); + } + return url; +} + +function assertPlainUnencodedJobLog(response: Response, label: string): void { + const contentType = response.headers.get("content-type")?.split(";", 1)[0]?.trim(); + if (contentType !== "text/plain" || response.headers.get("content-encoding") !== null) { + throw new Error(`${label} did not return unencoded plain text`); + } +} + +async function cancelJobLogResponseBody(response: Response): Promise { + await response.body?.cancel().catch(() => undefined); +} + +async function readExactJobLogRange( + response: Response, + expectedBytes: number, + discardPartialFirstLine: boolean, +): Promise { + if (!response.body) throw new Error("job log range response did not include a body"); + const reader = response.body.getReader(); + const chunks: Uint8Array[] = []; + let receivedBytes = 0; + try { + for (;;) { + const chunk = await reader.read(); + if (chunk.done) break; + receivedBytes += chunk.value.byteLength; + if (receivedBytes > expectedBytes || receivedBytes > MAX_RUNNER_LOSS_JOB_LOG_TAIL_BYTES) { + throw new Error("job log range response exceeded its authenticated byte bound"); + } + chunks.push(chunk.value); + } + } catch (error) { + await reader.cancel().catch(() => undefined); + throw error; + } finally { + reader.releaseLock(); + } + if (receivedBytes !== expectedBytes) { + throw new Error("job log range response was incomplete"); + } + const bytes = new Uint8Array(receivedBytes); + let offset = 0; + for (const chunk of chunks) { + bytes.set(chunk, offset); + offset += chunk.byteLength; + } + const firstLineFeed = discardPartialFirstLine ? bytes.indexOf(0x0a) : -1; + if (discardPartialFirstLine && firstLineFeed < 0) { + throw new Error("job log range did not contain a complete record"); + } + const completeRecords = firstLineFeed < 0 ? bytes : bytes.subarray(firstLineFeed + 1); + return new TextDecoder("utf-8", { fatal: true }).decode(completeRecords); +} + +async function downloadWorkflowJobLogTail( + repository: string, + token: string, + jobId: number, +): Promise { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), RUNNER_LOSS_JOB_LOG_TIMEOUT_MS); + const apiUrl = `https://api.github.com/repos/${repository}/actions/jobs/${jobId}/logs`; + try { + const redirect = await fetch(apiUrl, { + headers: { + Accept: "application/vnd.github+json", + Authorization: `Bearer ${token}`, + "User-Agent": USER_AGENT, + "X-GitHub-Api-Version": "2022-11-28", + }, + redirect: "manual", + signal: controller.signal, + }); + if (redirect.status !== 302) { + await cancelJobLogResponseBody(redirect); + throw new Error(`job log API returned unexpected status ${redirect.status}`); + } + const location = redirect.headers.get("location"); + await cancelJobLogResponseBody(redirect); + const downloadUrl = validateJobLogDownloadUrl(location); + + const downloadHeaders = { + Accept: "text/plain", + "Accept-Encoding": "identity", + "User-Agent": USER_AGENT, + }; + const metadata = await fetch(downloadUrl, { + method: "HEAD", + headers: downloadHeaders, + redirect: "error", + signal: controller.signal, + }); + if (metadata.status !== 200) { + await cancelJobLogResponseBody(metadata); + throw new Error(`job log metadata returned unexpected status ${metadata.status}`); + } + let totalBytes: number; + let etag: string; + try { + assertPlainUnencodedJobLog(metadata, "job log metadata"); + totalBytes = parseJobLogContentLength( + metadata.headers.get("content-length"), + "job log metadata", + ); + if (totalBytes < 1) throw new Error("job log is empty"); + etag = validateJobLogEtag(metadata.headers.get("etag")); + } catch (error) { + await cancelJobLogResponseBody(metadata); + throw error; + } + await cancelJobLogResponseBody(metadata); + + const rangeStart = Math.max(0, totalBytes - MAX_RUNNER_LOSS_JOB_LOG_TAIL_BYTES); + const rangeEnd = totalBytes - 1; + const expectedBytes = rangeEnd - rangeStart + 1; + const range = await fetch(downloadUrl, { + headers: { + ...downloadHeaders, + "If-Match": etag, + Range: `bytes=${rangeStart}-${rangeEnd}`, + }, + redirect: "error", + signal: controller.signal, + }); + if (range.status !== 206) { + await cancelJobLogResponseBody(range); + throw new Error(`job log range returned unexpected status ${range.status}`); + } + try { + assertPlainUnencodedJobLog(range, "job log range"); + if ( + range.headers.get("etag") !== etag || + range.headers.get("content-range") !== `bytes ${rangeStart}-${rangeEnd}/${totalBytes}` || + parseJobLogContentLength(range.headers.get("content-length"), "job log range") !== + expectedBytes + ) { + throw new Error("job log range did not match its authenticated metadata"); + } + } catch (error) { + await cancelJobLogResponseBody(range); + throw error; + } + return { + etag, + totalBytes, + tail: await readExactJobLogRange(range, expectedBytes, rangeStart > 0), + }; + } finally { + clearTimeout(timeout); + } +} + +function validateWorkflowJobsPage(value: unknown): WorkflowJobsPage { + if ( + !isObjectRecord(value) || + !Number.isSafeInteger(value.total_count) || + (value.total_count as number) < 0 || + !Array.isArray(value.jobs) + ) { + throw new Error("GitHub returned an invalid workflow job listing"); + } + return { + totalCount: value.total_count as number, + jobs: value.jobs.map(validateWorkflowJob), + }; +} + +export async function listNonPassingWorkflowJobs( + repository: string, + token: string, + runId: number, + runAttempt: number, + options: { + includeAnnotations?: boolean; + hostedRunnerLossPolicy?: HostedRunnerLossPolicy; + } = {}, +): Promise<{ jobs: WorkflowJob[]; complete: boolean }> { + if ( + !Number.isSafeInteger(runId) || + runId < 1 || + !Number.isSafeInteger(runAttempt) || + runAttempt < 1 + ) { + throw new Error("workflow run and attempt IDs must be positive safe integers"); + } + const jobs: WorkflowJob[] = []; + const jobIds = new Set(); + let totalCount: number | undefined; + for (let page = 1; page <= MAX_WORKFLOW_JOB_PAGES; page += 1) { + const response = validateWorkflowJobsPage( + await githubApi( + `repos/${repository}/actions/runs/${runId}/attempts/${runAttempt}/jobs?per_page=100&page=${page}`, + token, + { userAgent: USER_AGENT }, + ), + ); + totalCount ??= response.totalCount; + if (response.totalCount !== totalCount || jobs.length + response.jobs.length > totalCount) { + throw new Error("GitHub returned an invalid workflow job count"); + } + for (const job of response.jobs) { + if (jobIds.has(job.id)) { + throw new Error("GitHub returned duplicate workflow job IDs across the job listing"); + } + jobIds.add(job.id); + } + jobs.push(...response.jobs); + if (jobs.length === totalCount) { + const nonPassingJobs = jobs.filter( + (job) => !["success", "skipped", "neutral"].includes(job.conclusion ?? ""), + ); + if (options.includeAnnotations) { + const hostedRunnerLossPolicy = options.hostedRunnerLossPolicy ?? {}; + const runnerLossCandidates = nonPassingJobs.filter((job) => + isHostedRunnerLossInspectionCandidate(job, hostedRunnerLossPolicy), + ); + if (runnerLossCandidates.length > MAX_RUNNER_LOSS_JOB_INSPECTIONS) { + throw new Error("workflow run exceeded the hosted-runner-loss inspection limit"); + } + for (const job of runnerLossCandidates) { + const evidence = await listWorkflowJobAnnotations( + repository, + token, + job, + runId, + runAttempt, + ); + job.annotations = evidence.annotations; + job.checkEvidence = evidence.checkEvidence; + const workflowSha = job.headSha ?? ""; + if (needsHostedRunnerShutdownLog(job, repository, workflowSha, hostedRunnerLossPolicy)) { + try { + job.logEvidence = await downloadWorkflowJobLogTail(repository, token, job.id); + } catch { + console.warn( + `Could not authenticate hosted-runner shutdown log for job ${job.id}; automatic retry remains disabled`, + ); + } + } + } + } + return { + jobs: nonPassingJobs, + complete: true, + }; + } + if (response.jobs.length < 100) break; + } + return { + jobs: jobs.filter((job) => !["success", "skipped", "neutral"].includes(job.conclusion ?? "")), + complete: jobs.length === totalCount, + }; +} + +export function workflowJobEvidenceFingerprint(details: { + jobs: readonly WorkflowJob[]; + complete: boolean; +}): string { + const jobs = [...details.jobs] + .sort((left, right) => left.id - right.id) + .map((job) => { + const { annotations, logEvidence, ...metadata } = job; + return { + ...metadata, + ...(annotations === undefined + ? {} + : { annotations: annotations.map((annotation) => JSON.stringify(annotation)).sort() }), + ...(logEvidence === undefined + ? {} + : { + logEvidence: { + etag: logEvidence.etag, + totalBytes: logEvidence.totalBytes, + tailHash: sha256(logEvidence.tail), + }, + }), + }; + }); + return sha256(JSON.stringify({ complete: details.complete, jobs })); +} diff --git a/tools/e2e/pr-e2e-gate.mts b/tools/e2e/pr-e2e-gate.mts index 6afbe61072..2ad2081b78 100755 --- a/tools/e2e/pr-e2e-gate.mts +++ b/tools/e2e/pr-e2e-gate.mts @@ -25,16 +25,13 @@ import { import { SHARED_E2E_JOB_ID } from "./credential-free-tests.mts"; import { type HostedRunnerLossPolicy, - isHostedRunnerLossInspectionCandidate, - MAX_RUNNER_LOSS_JOB_ANNOTATIONS, - MAX_RUNNER_LOSS_JOB_LOG_TAIL_BYTES, - needsHostedRunnerShutdownLog, verifiedRunnerLossEvidence, type WorkflowJob, - type WorkflowJobAnnotation, - type WorkflowJobCheckEvidence, - type WorkflowJobLogEvidence, } from "./hosted-runner-loss.mts"; +import { + listNonPassingWorkflowJobs, + workflowJobEvidenceFingerprint, +} from "./hosted-runner-loss-github.mts"; import { readPrivateRegularFile, writePrivateRegularFile } from "./private-file.mts"; import type { E2eRiskSignal } from "./risk-signal.ts"; import { @@ -47,6 +44,11 @@ import { readFreeStandingJobsInventory, } from "./workflow-boundary.mts"; +export { + listNonPassingWorkflowJobs, + workflowJobEvidenceFingerprint, +} from "./hosted-runner-loss-github.mts"; + const E2E_WORKFLOW = "e2e.yaml"; const E2E_WORKFLOW_PATH = `.github/workflows/${E2E_WORKFLOW}`; const PR_GATE_WORKFLOW_PATH = ".github/workflows/pr-e2e-gate.yaml"; @@ -96,15 +98,6 @@ const MAX_CONTROLLER_ERROR_CHARS = 512; const MAX_PR_FILES = 3000; const MAX_COMPATIBILITY_FILES = 300; const MAX_ACTIVE_RUN_PAGES_PER_STATUS = 10; -const MAX_WORKFLOW_JOB_PAGES = 10; -const MAX_JOB_ANNOTATION_PAGES = 1; -const MAX_JOB_ANNOTATION_IDENTITY_BYTES = 8 * 1024; -const MAX_JOB_ANNOTATION_TEXT_BYTES = 16 * 1024; -const MAX_RUNNER_LOSS_JOB_ANNOTATION_BYTES = 64 * 1024; -const MAX_RUNNER_LOSS_JOB_INSPECTIONS = 20; -const RUNNER_LOSS_JOB_LOG_TIMEOUT_MS = 30_000; -const JOB_LOG_DOWNLOAD_HOST_PATTERN = /^productionresultssa[0-9]+\.blob\.core\.windows\.net$/u; -const GITHUB_TIMESTAMP_PATTERN = /^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$/u; const MAX_REPORTED_WORKFLOW_JOBS = 10; const MAX_WAIVER_REASON_CHARS = 500; const MAX_APPROVAL_REVIEWS = 20; @@ -260,7 +253,6 @@ type WorkflowRun = { }; type WorkflowRunsResponse = { workflow_runs: WorkflowRun[] }; -type WorkflowJobsPage = { totalCount: number; jobs: WorkflowJob[] }; type CheckRun = { id: number; name?: string; @@ -1637,541 +1629,6 @@ export async function resolvePullRequest(options: { return detail; } -function isOptionalGitHubTimestamp(value: unknown): boolean { - return ( - value === undefined || - value === null || - (typeof value === "string" && GITHUB_TIMESTAMP_PATTERN.test(value)) - ); -} - -function validateWorkflowJob(value: unknown): WorkflowJob { - if ( - !isObjectRecord(value) || - !Number.isSafeInteger(value.id) || - (value.id as number) < 1 || - typeof value.name !== "string" || - value.name.length === 0 || - (value.run_id !== undefined && - (!Number.isSafeInteger(value.run_id) || (value.run_id as number) < 1)) || - (value.run_attempt !== undefined && - (!Number.isSafeInteger(value.run_attempt) || (value.run_attempt as number) < 1)) || - (value.head_sha !== undefined && - (typeof value.head_sha !== "string" || !SHA_PATTERN.test(value.head_sha))) || - (value.run_url !== undefined && typeof value.run_url !== "string") || - (value.url !== undefined && typeof value.url !== "string") || - (value.html_url !== undefined && typeof value.html_url !== "string") || - (value.check_run_url !== undefined && typeof value.check_run_url !== "string") || - (value.status !== undefined && typeof value.status !== "string") || - (value.conclusion !== null && typeof value.conclusion !== "string") || - !isOptionalGitHubTimestamp(value.started_at) || - !isOptionalGitHubTimestamp(value.completed_at) || - (value.runner_id !== undefined && - value.runner_id !== null && - (!Number.isSafeInteger(value.runner_id) || (value.runner_id as number) < 1)) || - (value.runner_name !== undefined && - value.runner_name !== null && - typeof value.runner_name !== "string") || - (value.runner_group_id !== undefined && - value.runner_group_id !== null && - (!Number.isSafeInteger(value.runner_group_id) || (value.runner_group_id as number) < 0)) || - (value.runner_group_name !== undefined && - value.runner_group_name !== null && - typeof value.runner_group_name !== "string") || - (value.labels !== undefined && - (!Array.isArray(value.labels) || value.labels.some((label) => typeof label !== "string"))) || - (value.steps !== undefined && !Array.isArray(value.steps)) - ) { - throw new Error("GitHub returned an invalid workflow job"); - } - const steps = (value.steps ?? []).map((step) => { - if ( - !isObjectRecord(step) || - typeof step.name !== "string" || - step.name.length === 0 || - (step.status !== undefined && typeof step.status !== "string") || - (step.conclusion !== null && typeof step.conclusion !== "string") || - !isOptionalGitHubTimestamp(step.started_at) || - !isOptionalGitHubTimestamp(step.completed_at) - ) { - throw new Error("GitHub returned an invalid workflow job step"); - } - return { - name: step.name, - ...(step.status === undefined ? {} : { status: step.status }), - conclusion: step.conclusion, - ...(step.started_at === undefined ? {} : { startedAt: step.started_at as string | null }), - ...(step.completed_at === undefined - ? {} - : { completedAt: step.completed_at as string | null }), - }; - }); - return { - id: value.id as number, - name: value.name, - ...(value.run_id === undefined ? {} : { runId: value.run_id as number }), - ...(value.run_attempt === undefined ? {} : { runAttempt: value.run_attempt as number }), - ...(value.head_sha === undefined ? {} : { headSha: value.head_sha }), - ...(value.run_url === undefined ? {} : { runUrl: value.run_url }), - ...(value.url === undefined ? {} : { apiUrl: value.url }), - ...(value.html_url === undefined ? {} : { htmlUrl: value.html_url }), - ...(value.check_run_url === undefined ? {} : { checkRunUrl: value.check_run_url }), - ...(value.status === undefined ? {} : { status: value.status }), - conclusion: value.conclusion, - ...(value.runner_id === undefined ? {} : { runnerId: value.runner_id as number | null }), - ...(value.runner_name === undefined ? {} : { runnerName: value.runner_name }), - ...(value.runner_group_id === undefined - ? {} - : { runnerGroupId: value.runner_group_id as number | null }), - ...(value.runner_group_name === undefined ? {} : { runnerGroupName: value.runner_group_name }), - ...(value.labels === undefined ? {} : { labels: value.labels as string[] }), - ...(value.started_at === undefined ? {} : { startedAt: value.started_at as string | null }), - ...(value.completed_at === undefined - ? {} - : { completedAt: value.completed_at as string | null }), - steps, - }; -} - -function validateWorkflowJobAnnotation(value: unknown): WorkflowJobAnnotation { - if ( - !isObjectRecord(value) || - typeof value.path !== "string" || - value.path.length === 0 || - Buffer.byteLength(value.path, "utf8") > MAX_JOB_ANNOTATION_IDENTITY_BYTES || - typeof value.blob_href !== "string" || - Buffer.byteLength(value.blob_href, "utf8") > MAX_JOB_ANNOTATION_IDENTITY_BYTES || - !Number.isSafeInteger(value.start_line) || - (value.start_line as number) < 1 || - (value.start_column !== null && - (!Number.isSafeInteger(value.start_column) || (value.start_column as number) < 1)) || - !Number.isSafeInteger(value.end_line) || - (value.end_line as number) < (value.start_line as number) || - (value.end_column !== null && - (!Number.isSafeInteger(value.end_column) || (value.end_column as number) < 1)) || - typeof value.annotation_level !== "string" || - Buffer.byteLength(value.annotation_level, "utf8") > MAX_JOB_ANNOTATION_IDENTITY_BYTES || - typeof value.title !== "string" || - Buffer.byteLength(value.title, "utf8") > MAX_JOB_ANNOTATION_TEXT_BYTES || - typeof value.message !== "string" || - Buffer.byteLength(value.message, "utf8") > MAX_JOB_ANNOTATION_TEXT_BYTES || - typeof value.raw_details !== "string" || - Buffer.byteLength(value.raw_details, "utf8") > MAX_JOB_ANNOTATION_TEXT_BYTES - ) { - throw new Error("GitHub returned an invalid workflow job annotation"); - } - return { - path: value.path, - blobHref: value.blob_href, - startLine: value.start_line as number, - startColumn: value.start_column as number | null, - endLine: value.end_line as number, - endColumn: value.end_column as number | null, - annotationLevel: value.annotation_level, - title: value.title, - message: value.message, - rawDetails: value.raw_details, - }; -} - -async function listWorkflowJobAnnotations( - repository: string, - token: string, - job: WorkflowJob, - runId: number, - runAttempt: number, -): Promise<{ - annotations: WorkflowJobAnnotation[]; - checkEvidence: WorkflowJobCheckEvidence; -}> { - const apiRepository = `https://api.github.com/repos/${repository}`; - const webRepository = `https://github.com/${repository}`; - const expectedRunUrl = `${apiRepository}/actions/runs/${runId}`; - const expectedJobUrl = `${apiRepository}/actions/jobs/${job.id}`; - const expectedCheckRunUrl = `${apiRepository}/check-runs/${job.id}`; - const expectedHtmlUrl = `${webRepository}/actions/runs/${runId}/job/${job.id}`; - if ( - !job.headSha || - job.runId !== runId || - job.runAttempt !== runAttempt || - job.runUrl !== expectedRunUrl || - job.apiUrl !== expectedJobUrl || - job.htmlUrl !== expectedHtmlUrl || - job.checkRunUrl !== expectedCheckRunUrl - ) { - throw new Error("workflow job identity does not match its exact run attempt"); - } - const check = await githubApi(`repos/${repository}/check-runs/${job.id}`, token, { - userAgent: USER_AGENT, - }); - const expectedAnnotationsUrl = `${expectedCheckRunUrl}/annotations`; - if ( - !isObjectRecord(check) || - check.id !== job.id || - check.name !== job.name || - check.head_sha !== job.headSha || - check.url !== expectedCheckRunUrl || - check.html_url !== expectedHtmlUrl || - check.details_url !== expectedHtmlUrl || - check.status !== "completed" || - check.conclusion !== job.conclusion || - !isObjectRecord(check.app) || - check.app.id !== GITHUB_ACTIONS_APP_ID || - check.app.slug !== "github-actions" || - !isObjectRecord(check.output) || - !Number.isSafeInteger(check.output.annotations_count) || - (check.output.annotations_count as number) < 0 || - check.output.annotations_url !== expectedAnnotationsUrl - ) { - throw new Error("workflow job check run does not match the exact failed job"); - } - const expectedCount = check.output.annotations_count as number; - const checkEvidence: WorkflowJobCheckEvidence = { - id: check.id as number, - name: check.name as string, - headSha: check.head_sha as string, - apiUrl: check.url as string, - htmlUrl: check.html_url as string, - detailsUrl: check.details_url as string, - status: check.status as string, - conclusion: check.conclusion as string, - appId: check.app.id as number, - appSlug: check.app.slug as string, - annotationsCount: expectedCount, - annotationsUrl: check.output.annotations_url as string, - }; - if (expectedCount > MAX_RUNNER_LOSS_JOB_ANNOTATIONS) { - throw new Error("workflow job annotation count exceeds the hosted-runner-loss limit"); - } - const annotations: WorkflowJobAnnotation[] = []; - const fingerprints = new Set(); - let annotationBytes = 0; - for (let page = 1; page <= MAX_JOB_ANNOTATION_PAGES; page += 1) { - const value = await githubApi( - `repos/${repository}/check-runs/${job.id}/annotations?per_page=${MAX_RUNNER_LOSS_JOB_ANNOTATIONS}&page=${page}`, - token, - { userAgent: USER_AGENT }, - ); - if (!Array.isArray(value) || value.length > MAX_RUNNER_LOSS_JOB_ANNOTATIONS) { - throw new Error("GitHub returned an invalid workflow job annotation listing"); - } - const pageAnnotations = value.map(validateWorkflowJobAnnotation); - for (const annotation of pageAnnotations) { - const fingerprint = JSON.stringify(annotation); - if (fingerprints.has(fingerprint)) { - throw new Error("GitHub returned duplicate workflow job annotations"); - } - fingerprints.add(fingerprint); - annotationBytes += Buffer.byteLength(fingerprint, "utf8"); - if (annotationBytes > MAX_RUNNER_LOSS_JOB_ANNOTATION_BYTES) { - throw new Error("workflow job annotation evidence exceeds its byte limit"); - } - annotations.push(annotation); - } - if (annotations.length > expectedCount) { - throw new Error("workflow job annotation listing exceeds the trusted annotation count"); - } - if (annotations.length === expectedCount) return { annotations, checkEvidence }; - if (value.length < MAX_RUNNER_LOSS_JOB_ANNOTATIONS) { - throw new Error("workflow job annotation listing is incomplete"); - } - } - throw new Error("workflow job annotation listing exceeded its page limit"); -} - -function parseJobLogContentLength(value: string | null, label: string): number { - if (!value || !/^(?:0|[1-9][0-9]*)$/u.test(value)) { - throw new Error(`${label} did not provide a valid content length`); - } - const length = Number(value); - if (!Number.isSafeInteger(length) || length < 0) { - throw new Error(`${label} content length is outside the safe integer range`); - } - return length; -} - -function validateJobLogEtag(value: string | null): string { - if (!value || value.length > 130 || !/^"[^"\r\n]{1,128}"$/u.test(value)) { - throw new Error("job log download did not provide a strong bounded ETag"); - } - return value; -} - -function validateJobLogDownloadUrl(value: string | null): URL { - let url: URL; - try { - url = new URL(value ?? ""); - } catch { - throw new Error("job log API returned an invalid signed download URL"); - } - if ( - url.protocol !== "https:" || - url.username !== "" || - url.password !== "" || - url.port !== "" || - !JOB_LOG_DOWNLOAD_HOST_PATTERN.test(url.hostname) || - !url.pathname.startsWith("/actions-results/") || - url.search.length < 2 || - url.hash !== "" - ) { - throw new Error("job log API returned an untrusted signed download URL"); - } - return url; -} - -function assertPlainUnencodedJobLog(response: Response, label: string): void { - const contentType = response.headers.get("content-type")?.split(";", 1)[0]?.trim(); - if (contentType !== "text/plain" || response.headers.get("content-encoding") !== null) { - throw new Error(`${label} did not return unencoded plain text`); - } -} - -async function cancelJobLogResponseBody(response: Response): Promise { - await response.body?.cancel().catch(() => undefined); -} - -async function readExactJobLogRange( - response: Response, - expectedBytes: number, - discardPartialFirstLine: boolean, -): Promise { - if (!response.body) throw new Error("job log range response did not include a body"); - const reader = response.body.getReader(); - const chunks: Uint8Array[] = []; - let receivedBytes = 0; - try { - for (;;) { - const chunk = await reader.read(); - if (chunk.done) break; - receivedBytes += chunk.value.byteLength; - if (receivedBytes > expectedBytes || receivedBytes > MAX_RUNNER_LOSS_JOB_LOG_TAIL_BYTES) { - throw new Error("job log range response exceeded its authenticated byte bound"); - } - chunks.push(chunk.value); - } - } catch (error) { - await reader.cancel().catch(() => undefined); - throw error; - } finally { - reader.releaseLock(); - } - if (receivedBytes !== expectedBytes) { - throw new Error("job log range response was incomplete"); - } - const bytes = new Uint8Array(receivedBytes); - let offset = 0; - for (const chunk of chunks) { - bytes.set(chunk, offset); - offset += chunk.byteLength; - } - const firstLineFeed = discardPartialFirstLine ? bytes.indexOf(0x0a) : -1; - if (discardPartialFirstLine && firstLineFeed < 0) { - throw new Error("job log range did not contain a complete record"); - } - const completeRecords = firstLineFeed < 0 ? bytes : bytes.subarray(firstLineFeed + 1); - return new TextDecoder("utf-8", { fatal: true }).decode(completeRecords); -} - -async function downloadWorkflowJobLogTail( - repository: string, - token: string, - jobId: number, -): Promise { - const controller = new AbortController(); - const timeout = setTimeout(() => controller.abort(), RUNNER_LOSS_JOB_LOG_TIMEOUT_MS); - const apiUrl = `https://api.github.com/repos/${repository}/actions/jobs/${jobId}/logs`; - try { - const redirect = await fetch(apiUrl, { - headers: { - Accept: "application/vnd.github+json", - Authorization: `Bearer ${token}`, - "User-Agent": USER_AGENT, - "X-GitHub-Api-Version": "2022-11-28", - }, - redirect: "manual", - signal: controller.signal, - }); - if (redirect.status !== 302) { - await cancelJobLogResponseBody(redirect); - throw new Error(`job log API returned unexpected status ${redirect.status}`); - } - const location = redirect.headers.get("location"); - await cancelJobLogResponseBody(redirect); - const downloadUrl = validateJobLogDownloadUrl(location); - - const downloadHeaders = { - Accept: "text/plain", - "Accept-Encoding": "identity", - "User-Agent": USER_AGENT, - }; - const metadata = await fetch(downloadUrl, { - method: "HEAD", - headers: downloadHeaders, - redirect: "error", - signal: controller.signal, - }); - if (metadata.status !== 200) { - await cancelJobLogResponseBody(metadata); - throw new Error(`job log metadata returned unexpected status ${metadata.status}`); - } - let totalBytes: number; - let etag: string; - try { - assertPlainUnencodedJobLog(metadata, "job log metadata"); - totalBytes = parseJobLogContentLength( - metadata.headers.get("content-length"), - "job log metadata", - ); - if (totalBytes < 1) throw new Error("job log is empty"); - etag = validateJobLogEtag(metadata.headers.get("etag")); - } catch (error) { - await cancelJobLogResponseBody(metadata); - throw error; - } - await cancelJobLogResponseBody(metadata); - - const rangeStart = Math.max(0, totalBytes - MAX_RUNNER_LOSS_JOB_LOG_TAIL_BYTES); - const rangeEnd = totalBytes - 1; - const expectedBytes = rangeEnd - rangeStart + 1; - const range = await fetch(downloadUrl, { - headers: { - ...downloadHeaders, - "If-Match": etag, - Range: `bytes=${rangeStart}-${rangeEnd}`, - }, - redirect: "error", - signal: controller.signal, - }); - if (range.status !== 206) { - await cancelJobLogResponseBody(range); - throw new Error(`job log range returned unexpected status ${range.status}`); - } - try { - assertPlainUnencodedJobLog(range, "job log range"); - if ( - range.headers.get("etag") !== etag || - range.headers.get("content-range") !== `bytes ${rangeStart}-${rangeEnd}/${totalBytes}` || - parseJobLogContentLength(range.headers.get("content-length"), "job log range") !== - expectedBytes - ) { - throw new Error("job log range did not match its authenticated metadata"); - } - } catch (error) { - await cancelJobLogResponseBody(range); - throw error; - } - return { - etag, - totalBytes, - tail: await readExactJobLogRange(range, expectedBytes, rangeStart > 0), - }; - } finally { - clearTimeout(timeout); - } -} - -function validateWorkflowJobsPage(value: unknown): WorkflowJobsPage { - if ( - !isObjectRecord(value) || - !Number.isSafeInteger(value.total_count) || - (value.total_count as number) < 0 || - !Array.isArray(value.jobs) - ) { - throw new Error("GitHub returned an invalid workflow job listing"); - } - return { - totalCount: value.total_count as number, - jobs: value.jobs.map(validateWorkflowJob), - }; -} - -export async function listNonPassingWorkflowJobs( - repository: string, - token: string, - runId: number, - runAttempt: number, - options: { - includeAnnotations?: boolean; - hostedRunnerLossPolicy?: HostedRunnerLossPolicy; - } = {}, -): Promise<{ jobs: WorkflowJob[]; complete: boolean }> { - if ( - !Number.isSafeInteger(runId) || - runId < 1 || - !Number.isSafeInteger(runAttempt) || - runAttempt < 1 - ) { - throw new Error("workflow run and attempt IDs must be positive safe integers"); - } - const jobs: WorkflowJob[] = []; - const jobIds = new Set(); - let totalCount: number | undefined; - for (let page = 1; page <= MAX_WORKFLOW_JOB_PAGES; page += 1) { - const response = validateWorkflowJobsPage( - await githubApi( - `repos/${repository}/actions/runs/${runId}/attempts/${runAttempt}/jobs?per_page=100&page=${page}`, - token, - { userAgent: USER_AGENT }, - ), - ); - totalCount ??= response.totalCount; - if (response.totalCount !== totalCount || jobs.length + response.jobs.length > totalCount) { - throw new Error("GitHub returned an invalid workflow job count"); - } - for (const job of response.jobs) { - if (jobIds.has(job.id)) { - throw new Error("GitHub returned duplicate workflow job IDs across the job listing"); - } - jobIds.add(job.id); - } - jobs.push(...response.jobs); - if (jobs.length === totalCount) { - const nonPassingJobs = jobs.filter( - (job) => !["success", "skipped", "neutral"].includes(job.conclusion ?? ""), - ); - if (options.includeAnnotations) { - const hostedRunnerLossPolicy = options.hostedRunnerLossPolicy ?? {}; - const runnerLossCandidates = nonPassingJobs.filter((job) => - isHostedRunnerLossInspectionCandidate(job, hostedRunnerLossPolicy), - ); - if (runnerLossCandidates.length > MAX_RUNNER_LOSS_JOB_INSPECTIONS) { - throw new Error("workflow run exceeded the hosted-runner-loss inspection limit"); - } - for (const job of runnerLossCandidates) { - const evidence = await listWorkflowJobAnnotations( - repository, - token, - job, - runId, - runAttempt, - ); - job.annotations = evidence.annotations; - job.checkEvidence = evidence.checkEvidence; - const workflowSha = job.headSha ?? ""; - if (needsHostedRunnerShutdownLog(job, repository, workflowSha, hostedRunnerLossPolicy)) { - try { - job.logEvidence = await downloadWorkflowJobLogTail(repository, token, job.id); - } catch { - console.warn( - `Could not authenticate hosted-runner shutdown log for job ${job.id}; automatic retry remains disabled`, - ); - } - } - } - } - return { - jobs: nonPassingJobs, - complete: true, - }; - } - if (response.jobs.length < 100) break; - } - return { - jobs: jobs.filter((job) => !["success", "skipped", "neutral"].includes(job.conclusion ?? "")), - complete: jobs.length === totalCount, - }; -} - function normalizedCiMetadata(value: string, fallback: string): string { const normalized = value .replace(/[\u0000-\u001f\u007f]+/gu, " ") @@ -2655,33 +2112,6 @@ async function requireUnchangedCompletedWorkflowRun( } } -export function workflowJobEvidenceFingerprint(details: { - jobs: readonly WorkflowJob[]; - complete: boolean; -}): string { - const jobs = [...details.jobs] - .sort((left, right) => left.id - right.id) - .map((job) => { - const { annotations, logEvidence, ...metadata } = job; - return { - ...metadata, - ...(annotations === undefined - ? {} - : { annotations: annotations.map((annotation) => JSON.stringify(annotation)).sort() }), - ...(logEvidence === undefined - ? {} - : { - logEvidence: { - etag: logEvidence.etag, - totalBytes: logEvidence.totalBytes, - tailHash: sha256(logEvidence.tail), - }, - }), - }; - }); - return sha256(JSON.stringify({ complete: details.complete, jobs })); -} - export async function dispatchPrGate(options: { repository: string; checkoutRepository: string; From 4b9eab2ca1ba641039c3bf63c02c46540c01679c Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Sat, 25 Jul 2026 08:43:13 -0700 Subject: [PATCH 04/10] fix(ci): recover latest hosted runner loss Signed-off-by: Apurv Kumaria --- test/hosted-runner-recovery.test.ts | 753 +++++++++++++++++++++++++++ tools/e2e/hosted-runner-recovery.mts | 534 +++++++++++++++++++ 2 files changed, 1287 insertions(+) create mode 100644 test/hosted-runner-recovery.test.ts create mode 100755 tools/e2e/hosted-runner-recovery.mts diff --git a/test/hosted-runner-recovery.test.ts b/test/hosted-runner-recovery.test.ts new file mode 100644 index 0000000000..27e65c73a8 --- /dev/null +++ b/test/hosted-runner-recovery.test.ts @@ -0,0 +1,753 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { afterEach, describe, expect, it, vi } from "vitest"; +import { recoverHostedRunnerLoss } from "../tools/e2e/hosted-runner-recovery.mts"; +import { + createGitHubFetchRouter, + githubFetchRoute, + type RecordedGitHubRequest, +} from "./support/github-fetch-router.ts"; + +const REPOSITORY = "NVIDIA/NemoClaw"; +const MAIN_SHA = "d".repeat(40); +const SOURCE_RUN_ID = 29_897_237_525; +const JOB_ID = 89_074_697_099; +const RUN_API_URL = `https://api.github.com/repos/${REPOSITORY}/actions/runs/${SOURCE_RUN_ID}`; +const RUN_HTML_URL = `https://github.com/${REPOSITORY}/actions/runs/${SOURCE_RUN_ID}`; +const JOB_API_URL = `https://api.github.com/repos/${REPOSITORY}/actions/jobs/${JOB_ID}`; +const JOB_HTML_URL = `${RUN_HTML_URL}/job/${JOB_ID}`; +const CHECK_URL = `https://api.github.com/repos/${REPOSITORY}/check-runs/${JOB_ID}`; +const INTERNAL_ERROR_MESSAGE = + "GitHub Actions has encountered an internal error when running your job."; +const RUNNER_LOSS_MESSAGE = + "The hosted runner lost communication with the server. Anything in your workflow that terminates the runner process, starves it for CPU/Memory, or blocks its network access can cause this error."; + +afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllGlobals(); +}); + +function githubResponse(value?: unknown, status = 200): Response { + return { + ok: status >= 200 && status < 300, + status, + text: async () => (value === undefined ? "" : JSON.stringify(value)), + } as Response; +} + +type SourceOverrides = { + conclusion?: string | null; + created_at?: string; + display_title?: string; + event?: string; + head_branch?: string; + head_repository?: { full_name: string }; + head_sha?: string; + html_url?: string; + id?: number; + name?: string; + path?: string; + repository?: { full_name: string }; + run_attempt?: number; + status?: string; + workflow_id?: number; +}; + +function sourceRun(overrides: SourceOverrides = {}) { + const id = overrides.id ?? SOURCE_RUN_ID; + return { + id, + name: "E2E main", + path: ".github/workflows/e2e.yaml", + workflow_id: 304_268_429, + created_at: "2026-07-25T10:00:00Z", + event: "schedule", + head_branch: "main", + head_sha: MAIN_SHA, + run_attempt: 1, + status: "completed", + conclusion: "failure", + display_title: "E2E main", + html_url: `https://github.com/${REPOSITORY}/actions/runs/${id}`, + repository: { full_name: REPOSITORY }, + head_repository: { full_name: REPOSITORY }, + ...overrides, + }; +} + +function workflowForSource(source: ReturnType) { + const names = new Map([ + [".github/workflows/e2e.yaml", "E2E"], + [".github/workflows/wsl-e2e.yaml", "E2E / WSL"], + [".github/workflows/macos-e2e.yaml", "E2E / macOS"], + [".github/workflows/platform-vitest-main.yaml", "CI / Platform Vitest Main Watch"], + ]); + return { + id: source.workflow_id, + name: names.get(source.path) ?? "unknown", + path: source.path, + state: "active", + }; +} + +function runnerLossAnnotation(message = RUNNER_LOSS_MESSAGE, sha = MAIN_SHA) { + return { + path: ".github", + blob_href: `https://github.com/${REPOSITORY}/blob/${sha}/.github`, + start_line: 1, + start_column: null, + end_line: 1, + end_column: null, + annotation_level: "failure", + title: "", + message, + raw_details: "", + }; +} + +function hostedRunnerLossJob(overrides: Record = {}) { + return { + id: JOB_ID, + run_id: SOURCE_RUN_ID, + run_attempt: 1, + head_sha: MAIN_SHA, + run_url: RUN_API_URL, + url: JOB_API_URL, + html_url: JOB_HTML_URL, + check_run_url: CHECK_URL, + name: "Hermes security-posture", + status: "completed", + conclusion: "failure", + runner_id: 1_021_277_393, + runner_name: "GitHub Actions 1021277393", + runner_group_id: 0, + runner_group_name: "GitHub Actions", + labels: ["ubuntu-latest"], + steps: [ + { name: "Set up job", status: "completed", conclusion: "success" }, + { name: "Run live test", status: "completed", conclusion: "cancelled" }, + { name: "Upload artifacts", status: "completed", conclusion: "skipped" }, + { name: "Complete job", status: "completed", conclusion: "success" }, + ], + ...overrides, + }; +} + +function internalErrorJob(label: "windows-latest" | "macos-26") { + const runnerId = label === "windows-latest" ? 1_021_277_394 : 1_021_277_395; + return hostedRunnerLossJob({ + name: `Platform E2E (${label})`, + conclusion: "cancelled", + runner_id: runnerId, + runner_name: `GitHub Actions ${runnerId}`, + labels: [label], + steps: [ + { name: "Set up job", status: "completed", conclusion: "success" }, + { name: "Run platform test", status: "in_progress", conclusion: null }, + { name: "Upload artifacts", status: "pending", conclusion: null }, + ], + }); +} + +function ordinaryFailureJob() { + return { + ...hostedRunnerLossJob(), + id: JOB_ID + 1, + url: `https://api.github.com/repos/${REPOSITORY}/actions/jobs/${JOB_ID + 1}`, + html_url: `${RUN_HTML_URL}/job/${JOB_ID + 1}`, + check_run_url: `https://api.github.com/repos/${REPOSITORY}/check-runs/${JOB_ID + 1}`, + name: "ordinary assertion", + runner_id: null, + runner_name: null, + annotations: undefined, + steps: [{ name: "Run tests", status: "completed", conclusion: "failure" }], + }; +} + +function checkRun(job: ReturnType) { + return { + id: job.id, + name: job.name, + head_sha: job.head_sha, + url: job.check_run_url, + html_url: job.html_url, + details_url: job.html_url, + status: "completed", + conclusion: job.conclusion, + app: { id: 15368, slug: "github-actions" }, + output: { + annotations_count: 1, + annotations_url: `${job.check_run_url}/annotations`, + }, + }; +} + +type RouteOptions = { + annotations?: unknown[][]; + jobListings?: Array<{ total_count: number; jobs: unknown[] }>; + rerunStatus?: number; + runListings?: Array<{ total_count: number; workflow_runs: unknown[] }>; + sources?: unknown[]; + workflows?: unknown[]; +}; + +function setupRoutes(options: RouteOptions = {}) { + const requests: RecordedGitHubRequest[] = []; + let sourceRead = 0; + let jobsRead = 0; + let annotationRead = 0; + let runListingRead = 0; + let workflowRead = 0; + let lastSource = sourceRun(); + const lastJobs = new Map>(); + const defaultJob = hostedRunnerLossJob(); + const defaultListing = { total_count: 1, jobs: [defaultJob] }; + const sources = options.sources ?? [sourceRun(), sourceRun()]; + const jobListings = options.jobListings ?? [defaultListing, defaultListing]; + + vi.stubGlobal( + "fetch", + createGitHubFetchRouter( + [ + githubFetchRoute( + ({ url, method }) => url.endsWith(`/actions/runs/${SOURCE_RUN_ID}`) && method === "GET", + () => { + lastSource = sources[Math.min(sourceRead++, sources.length - 1)] as ReturnType< + typeof sourceRun + >; + return githubResponse(lastSource); + }, + ), + githubFetchRoute( + ({ url, method }) => + url.endsWith(`/actions/workflows/${lastSource.workflow_id}`) && method === "GET", + () => { + const workflow = + options.workflows?.[Math.min(workflowRead, (options.workflows?.length ?? 1) - 1)] ?? + workflowForSource(lastSource); + workflowRead += 1; + return githubResponse(workflow); + }, + ), + githubFetchRoute( + ({ url, method }) => + url.includes(`/actions/workflows/${lastSource.workflow_id}/runs?`) && method === "GET", + () => { + const listing = options.runListings + ? options.runListings[Math.min(runListingRead, options.runListings.length - 1)] + : { total_count: 1, workflow_runs: [lastSource] }; + runListingRead += 1; + return githubResponse(listing); + }, + ), + githubFetchRoute( + ({ url, method }) => + url.includes(`/actions/runs/${SOURCE_RUN_ID}/attempts/1/jobs?`) && method === "GET", + (request) => { + const page = Number(new URL(request.url).searchParams.get("page")); + const listing = + page === 1 + ? jobListings[Math.min(jobsRead++, jobListings.length - 1)] + : { total_count: 0, jobs: [] }; + for (const candidate of listing?.jobs ?? []) { + const job = candidate as ReturnType; + if (Number.isSafeInteger(job.id)) lastJobs.set(job.id, job); + } + return githubResponse(listing); + }, + ), + githubFetchRoute( + ({ url, method }) => /\/check-runs\/[1-9][0-9]*$/u.test(url) && method === "GET", + (request) => { + const id = Number(request.url.split("/").at(-1)); + const job = lastJobs.get(id); + if (!job) throw new Error(`missing job fixture for check ${id}`); + return githubResponse(checkRun(job)); + }, + ), + githubFetchRoute( + ({ url, method }) => url.includes("/annotations?") && method === "GET", + (request) => { + const id = Number(/\/check-runs\/([1-9][0-9]*)\/annotations/u.exec(request.url)?.[1]); + const job = lastJobs.get(id); + if (!job) throw new Error(`missing job fixture for annotations ${id}`); + const defaultAnnotation = + job.conclusion === "cancelled" + ? runnerLossAnnotation(INTERNAL_ERROR_MESSAGE, job.head_sha) + : runnerLossAnnotation(RUNNER_LOSS_MESSAGE, job.head_sha); + const annotations = options.annotations?.[ + Math.min(annotationRead, (options.annotations?.length ?? 1) - 1) + ] ?? [defaultAnnotation]; + annotationRead += 1; + return githubResponse(annotations); + }, + ), + githubFetchRoute( + ({ url, method }) => + url.endsWith(`/actions/runs/${SOURCE_RUN_ID}/rerun`) && method === "POST", + () => + options.rerunStatus && options.rerunStatus !== 201 + ? githubResponse({ message: "rerun unavailable" }, options.rerunStatus) + : githubResponse(undefined, 201), + ), + ], + requests, + ), + ); + return requests; +} + +function recoveryRequest() { + return { + repository: REPOSITORY, + token: "token", + controllerRunAttempt: 1, + sourceRunId: SOURCE_RUN_ID, + }; +} + +function mutationRequests(requests: RecordedGitHubRequest[]) { + return requests.filter((request) => request.method !== "GET"); +} + +describe("hosted-runner recovery controller", () => { + it.each([ + { + label: "scheduled E2E", + source: sourceRun(), + job: hostedRunnerLossJob(), + }, + { + label: "manually dispatched main E2E", + source: sourceRun({ event: "workflow_dispatch" }), + job: hostedRunnerLossJob(), + }, + { + label: "WSL main push", + source: sourceRun({ + name: "E2E / WSL", + path: ".github/workflows/wsl-e2e.yaml", + event: "push", + display_title: "main", + }), + job: internalErrorJob("windows-latest"), + }, + { + label: "macOS main push", + source: sourceRun({ + name: "E2E / macOS", + path: ".github/workflows/macos-e2e.yaml", + event: "push", + display_title: "main", + }), + job: internalErrorJob("macos-26"), + }, + { + label: "platform Ubuntu main push", + source: sourceRun({ + name: "CI / Platform Vitest Main Watch", + path: ".github/workflows/platform-vitest-main.yaml", + event: "push", + display_title: "main", + }), + job: hostedRunnerLossJob(), + }, + { + label: "platform Windows main push", + source: sourceRun({ + name: "CI / Platform Vitest Main Watch", + path: ".github/workflows/platform-vitest-main.yaml", + event: "push", + display_title: "main", + }), + job: internalErrorJob("windows-latest"), + }, + { + label: "platform macOS main push", + source: sourceRun({ + name: "CI / Platform Vitest Main Watch", + path: ".github/workflows/platform-vitest-main.yaml", + event: "push", + display_title: "main", + }), + job: internalErrorJob("macos-26"), + }, + ])("requests one full rerun for exact $label runner loss (#7140)", async ({ source, job }) => { + const listing = { total_count: 1, jobs: [job] }; + const requests = setupRoutes({ + sources: [source, source], + jobListings: [listing, listing], + }); + + await expect(recoverHostedRunnerLoss(recoveryRequest())).resolves.toEqual({ + action: "rerun-requested", + reason: "exact hosted-runner-loss evidence remained stable across both reads", + }); + expect(mutationRequests(requests)).toEqual([ + { + method: "POST", + url: `${RUN_API_URL}/rerun`, + }, + ]); + expect(requests.some((request) => request.url.endsWith("/rerun-failed-jobs"))).toBe(false); + }); + + it.each([ + { + label: "E2E on Windows", + source: sourceRun(), + job: hostedRunnerLossJob({ labels: ["windows-latest"] }), + }, + { + label: "E2E on macOS", + source: sourceRun(), + job: hostedRunnerLossJob({ labels: ["macos-26"] }), + }, + { + label: "E2E with multiple runner labels", + source: sourceRun(), + job: hostedRunnerLossJob({ labels: ["ubuntu-latest", "self-hosted"] }), + }, + { + label: "WSL on Ubuntu", + source: sourceRun({ + name: "E2E / WSL", + path: ".github/workflows/wsl-e2e.yaml", + event: "push", + }), + job: hostedRunnerLossJob(), + }, + { + label: "WSL with a macOS internal error", + source: sourceRun({ + name: "E2E / WSL", + path: ".github/workflows/wsl-e2e.yaml", + event: "push", + }), + job: internalErrorJob("macos-26"), + }, + { + label: "macOS on Ubuntu", + source: sourceRun({ + name: "E2E / macOS", + path: ".github/workflows/macos-e2e.yaml", + event: "push", + }), + job: hostedRunnerLossJob(), + }, + { + label: "macOS with a Windows internal error", + source: sourceRun({ + name: "E2E / macOS", + path: ".github/workflows/macos-e2e.yaml", + event: "push", + }), + job: internalErrorJob("windows-latest"), + }, + ])("does not broaden allowed runner labels for $label (#7140)", async ({ source, job }) => { + const listing = { total_count: 1, jobs: [job] }; + const requests = setupRoutes({ + sources: [source], + jobListings: [listing], + }); + await expect(recoverHostedRunnerLoss(recoveryRequest())).resolves.toMatchObject({ + action: "ignored", + }); + expect(mutationRequests(requests)).toEqual([]); + }); + + it("reruns the latest eligible source even after main advances (#7140)", async () => { + const olderSha = "c".repeat(40); + const source = sourceRun({ head_sha: olderSha }); + const job = hostedRunnerLossJob({ head_sha: olderSha }); + const requests = setupRoutes({ + sources: [source], + jobListings: [{ total_count: 1, jobs: [job] }], + }); + await expect(recoverHostedRunnerLoss(recoveryRequest())).resolves.toMatchObject({ + action: "rerun-requested", + }); + expect(mutationRequests(requests)).toHaveLength(1); + expect(requests.some((request) => request.url.includes("/git/ref/heads/main"))).toBe(false); + }); + + it.each([ + { + label: "WSL pull request", + source: sourceRun({ + name: "E2E / WSL", + path: ".github/workflows/wsl-e2e.yaml", + event: "pull_request", + }), + }, + { + label: "macOS manual dispatch", + source: sourceRun({ + name: "E2E / macOS", + path: ".github/workflows/macos-e2e.yaml", + event: "workflow_dispatch", + }), + }, + { + label: "platform manual dispatch", + source: sourceRun({ + name: "CI / Platform Vitest Main Watch", + path: ".github/workflows/platform-vitest-main.yaml", + event: "workflow_dispatch", + }), + }, + ])("ignores non-push platform source: $label (#7140)", async ({ source }) => { + const requests = setupRoutes({ sources: [source] }); + await expect(recoverHostedRunnerLoss(recoveryRequest())).resolves.toMatchObject({ + action: "ignored", + }); + expect(mutationRequests(requests)).toEqual([]); + }); + + it("ignores an E2E PR child run title (#7140)", async () => { + const requests = setupRoutes({ + sources: [ + sourceRun({ + event: "workflow_dispatch", + display_title: "E2E PR #42 (12345678-1234-4123-8123-123456789abc)", + }), + ], + }); + await expect(recoverHostedRunnerLoss(recoveryRequest())).resolves.toMatchObject({ + action: "ignored", + }); + expect(mutationRequests(requests)).toEqual([]); + }); + + it("does not rerun when a newer eligible main run exists (#7140)", async () => { + const source = sourceRun(); + const newer = sourceRun({ + id: SOURCE_RUN_ID + 100, + created_at: "2026-07-25T11:00:00Z", + head_sha: "e".repeat(40), + conclusion: "success", + }); + const requests = setupRoutes({ + sources: [source], + runListings: [{ total_count: 2, workflow_runs: [newer, source] }], + }); + await expect(recoverHostedRunnerLoss(recoveryRequest())).resolves.toMatchObject({ + action: "ignored", + }); + expect(mutationRequests(requests)).toEqual([]); + }); + + it("ignores a newer E2E PR child when selecting the latest eligible main run (#7140)", async () => { + const source = sourceRun(); + const newerPrChild = sourceRun({ + id: SOURCE_RUN_ID + 100, + name: "E2E PR #42", + created_at: "2026-07-25T11:00:00Z", + event: "workflow_dispatch", + head_sha: "e".repeat(40), + conclusion: "success", + display_title: "E2E PR #42 (12345678-1234-4123-8123-123456789abc)", + }); + const requests = setupRoutes({ + sources: [source], + runListings: [{ total_count: 2, workflow_runs: [newerPrChild, source] }], + }); + await expect(recoverHostedRunnerLoss(recoveryRequest())).resolves.toMatchObject({ + action: "rerun-requested", + }); + expect(mutationRequests(requests)).toHaveLength(1); + }); + + it.each([ + { + label: "incomplete listing", + listing: { total_count: 2, workflow_runs: [sourceRun()] }, + message: /listing was incomplete/u, + }, + { + label: "duplicate run IDs", + listing: { total_count: 2, workflow_runs: [sourceRun(), sourceRun()] }, + message: /duplicate workflow run IDs/u, + }, + { + label: "source run absent", + listing: { + total_count: 1, + workflow_runs: [ + sourceRun({ + id: SOURCE_RUN_ID + 100, + created_at: "2026-07-25T11:00:00Z", + }), + ], + }, + message: /source run was not found/u, + }, + { + label: "ambiguous ordering", + listing: { + total_count: 2, + workflow_runs: [ + sourceRun({ + id: SOURCE_RUN_ID - 100, + created_at: "2026-07-25T09:00:00Z", + }), + sourceRun(), + ], + }, + message: /ambiguously ordered/u, + }, + ])("rejects an unsafe latest-run $label (#7140)", async ({ listing, message }) => { + const requests = setupRoutes({ runListings: [listing] }); + await expect(recoverHostedRunnerLoss(recoveryRequest())).rejects.toThrow(message); + expect(mutationRequests(requests)).toEqual([]); + }); + + it("rejects mixed runner loss and ordinary failure evidence (#7140)", async () => { + const jobs = [hostedRunnerLossJob(), ordinaryFailureJob()]; + const requests = setupRoutes({ + jobListings: [{ total_count: jobs.length, jobs }], + }); + await expect(recoverHostedRunnerLoss(recoveryRequest())).resolves.toMatchObject({ + action: "ignored", + }); + expect(mutationRequests(requests)).toEqual([]); + }); + + it.each([ + { + label: "zero jobs", + listing: { total_count: 0, jobs: [] }, + rejects: false, + }, + { + label: "incomplete listing", + listing: { total_count: 2, jobs: [hostedRunnerLossJob()] }, + rejects: false, + }, + { + label: "duplicate job IDs", + listing: { + total_count: 2, + jobs: [hostedRunnerLossJob(), hostedRunnerLossJob({ name: "duplicate" })], + }, + rejects: true, + }, + ])("never reruns with $label (#7140)", async ({ listing, rejects }) => { + const requests = setupRoutes({ jobListings: [listing] }); + const recovery = recoverHostedRunnerLoss(recoveryRequest()); + if (rejects) { + await expect(recovery).rejects.toThrow(/duplicate workflow job IDs/u); + } else { + await expect(recovery).resolves.toMatchObject({ action: "ignored" }); + } + expect(mutationRequests(requests)).toEqual([]); + }); + + it("rejects source evidence that changes on the confirmation read (#7140)", async () => { + const requests = setupRoutes({ + sources: [sourceRun(), sourceRun({ display_title: "E2E changed" })], + }); + await expect(recoverHostedRunnerLoss(recoveryRequest())).rejects.toThrow( + /identity changed during runner-loss evidence collection/u, + ); + expect(mutationRequests(requests)).toEqual([]); + }); + + it("rejects latest-run evidence that changes after job collection (#7140)", async () => { + const source = sourceRun(); + const olderPrChild = sourceRun({ + id: SOURCE_RUN_ID - 100, + name: "E2E PR #41", + created_at: "2026-07-25T09:00:00Z", + event: "workflow_dispatch", + conclusion: "success", + display_title: "E2E PR #41 (12345678-1234-4123-8123-123456789abc)", + }); + const baseline = { total_count: 1, workflow_runs: [source] }; + const changed = { total_count: 2, workflow_runs: [source, olderPrChild] }; + const requests = setupRoutes({ + sources: [source], + runListings: [baseline, changed], + }); + await expect(recoverHostedRunnerLoss(recoveryRequest())).rejects.toThrow( + /evidence changed during runner-loss evidence collection/u, + ); + expect(mutationRequests(requests)).toEqual([]); + }); + + it("rejects latest-run evidence that changes between snapshots (#7140)", async () => { + const source = sourceRun(); + const olderPrChild = sourceRun({ + id: SOURCE_RUN_ID - 100, + name: "E2E PR #41", + created_at: "2026-07-25T09:00:00Z", + event: "workflow_dispatch", + conclusion: "success", + display_title: "E2E PR #41 (12345678-1234-4123-8123-123456789abc)", + }); + const baseline = { total_count: 1, workflow_runs: [source] }; + const changed = { total_count: 2, workflow_runs: [source, olderPrChild] }; + const requests = setupRoutes({ + sources: [source], + runListings: [baseline, baseline, changed, changed], + }); + await expect(recoverHostedRunnerLoss(recoveryRequest())).rejects.toThrow( + /evidence changed before rerun/u, + ); + expect(mutationRequests(requests)).toEqual([]); + }); + + it("rejects job evidence that changes on the confirmation read (#7140)", async () => { + const firstJob = hostedRunnerLossJob(); + const secondJob = hostedRunnerLossJob({ name: "Hermes security-posture changed" }); + const requests = setupRoutes({ + jobListings: [ + { total_count: 1, jobs: [firstJob] }, + { total_count: 1, jobs: [secondJob] }, + ], + }); + await expect(recoverHostedRunnerLoss(recoveryRequest())).rejects.toThrow( + /evidence changed before rerun/u, + ); + expect(mutationRequests(requests)).toEqual([]); + }); + + it.each([ + ["name", { ...workflowForSource(sourceRun()), name: "E2E renamed" }], + ["path", { ...workflowForSource(sourceRun()), path: ".github/workflows/other.yaml" }], + ["state", { ...workflowForSource(sourceRun()), state: "disabled_manually" }], + ])("ignores source workflow metadata with the wrong %s (#7140)", async (_field, workflow) => { + const requests = setupRoutes({ workflows: [workflow] }); + await expect(recoverHostedRunnerLoss(recoveryRequest())).resolves.toMatchObject({ + action: "ignored", + }); + expect(mutationRequests(requests)).toEqual([]); + }); + + it("never mutates from controller attempt two (#7140)", async () => { + const requests = setupRoutes(); + await expect( + recoverHostedRunnerLoss({ ...recoveryRequest(), controllerRunAttempt: 2 }), + ).resolves.toMatchObject({ action: "ignored" }); + expect(requests).toEqual([]); + }); + + it("never mutates source attempt two (#7140)", async () => { + const requests = setupRoutes({ + sources: [sourceRun({ run_attempt: 2 })], + }); + await expect(recoverHostedRunnerLoss(recoveryRequest())).resolves.toMatchObject({ + action: "ignored", + }); + expect(mutationRequests(requests)).toEqual([]); + }); + + it("never repeats an ambiguous rerun mutation (#7140)", async () => { + const requests = setupRoutes({ rerunStatus: 500 }); + await expect(recoverHostedRunnerLoss(recoveryRequest())).rejects.toThrow( + /actions\/runs\/29897237525\/rerun failed: 500/u, + ); + expect(mutationRequests(requests)).toHaveLength(1); + }); +}); diff --git a/tools/e2e/hosted-runner-recovery.mts b/tools/e2e/hosted-runner-recovery.mts new file mode 100755 index 0000000000..de899273aa --- /dev/null +++ b/tools/e2e/hosted-runner-recovery.mts @@ -0,0 +1,534 @@ +#!/usr/bin/env node + +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; +import { pathToFileURL } from "node:url"; + +import { githubApi } from "../advisors/github.mts"; +import { type HostedRunnerLossPolicy, verifiedRunnerLossEvidence } from "./hosted-runner-loss.mts"; +import { + listNonPassingWorkflowJobs, + workflowJobEvidenceFingerprint, +} from "./hosted-runner-loss-github.mts"; +import { detectRunnerLoss } from "./runner-pressure-core.mts"; + +const TRUSTED_REPOSITORY = "NVIDIA/NemoClaw"; +const USER_AGENT = "nemoclaw-hosted-runner-recovery"; +const SHA_PATTERN = /^[a-f0-9]{40}$/u; +const GITHUB_TIMESTAMP_PATTERN = /^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$/u; +const MAX_WORKFLOW_RUN_PAGES = 10; +const WINDOWS_INTERNAL_ERROR_POLICY: HostedRunnerLossPolicy = { + githubInternalError: { + approvedRunnerLabels: ["windows-latest"], + approvedJobConclusions: ["cancelled"], + }, +}; +const MACOS_INTERNAL_ERROR_POLICY: HostedRunnerLossPolicy = { + githubInternalError: { + approvedRunnerLabels: ["macos-26"], + approvedJobConclusions: ["cancelled"], + }, +}; +const PLATFORM_INTERNAL_ERROR_POLICY: HostedRunnerLossPolicy = { + githubInternalError: { + approvedRunnerLabels: ["windows-latest", "macos-26"], + approvedJobConclusions: ["cancelled"], + }, +}; + +const SOURCE_WORKFLOWS = [ + { + workflowName: "E2E", + path: ".github/workflows/e2e.yaml", + runName: "E2E main", + events: ["schedule", "workflow_dispatch"], + displayTitle: "E2E main", + policy: {}, + allowedRunnerLabels: ["ubuntu-latest"], + }, + { + workflowName: "E2E / WSL", + path: ".github/workflows/wsl-e2e.yaml", + runName: "E2E / WSL", + events: ["push"], + policy: WINDOWS_INTERNAL_ERROR_POLICY, + allowedRunnerLabels: ["windows-latest"], + }, + { + workflowName: "E2E / macOS", + path: ".github/workflows/macos-e2e.yaml", + runName: "E2E / macOS", + events: ["push"], + policy: MACOS_INTERNAL_ERROR_POLICY, + allowedRunnerLabels: ["macos-26"], + }, + { + workflowName: "CI / Platform Vitest Main Watch", + path: ".github/workflows/platform-vitest-main.yaml", + runName: "CI / Platform Vitest Main Watch", + events: ["push"], + policy: PLATFORM_INTERNAL_ERROR_POLICY, + allowedRunnerLabels: ["ubuntu-latest", "windows-latest", "macos-26"], + }, +] as const; + +type SourceWorkflowRun = { + id: number; + runName: string; + path: string; + workflowId: number; + createdAt: string; + event: string; + headBranch: string; + headSha: string; + runAttempt: number; + status: string; + conclusion: string | null; + displayTitle: string; + htmlUrl: string; + repository: string; + headRepository: string; +}; + +type SourceWorkflow = { + id: number; + name: string; + path: string; + state: string; +}; + +type RecoverySnapshot = { + fingerprint: string; +}; + +type WorkflowRunsPage = { + totalCount: number; + runs: SourceWorkflowRun[]; +}; + +type LatestEligibleRunEvidence = { + runs: SourceWorkflowRun[]; + totalCount: number; +}; + +export type HostedRunnerRecoveryResult = + | { action: "ignored"; reason: string } + | { action: "rerun-requested"; reason: string }; + +function isObjectRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function sha256(value: string): string { + return createHash("sha256").update(value).digest("hex"); +} + +function validateSourceWorkflowRun(value: unknown): SourceWorkflowRun { + if ( + !isObjectRecord(value) || + !Number.isSafeInteger(value.id) || + (value.id as number) < 1 || + typeof value.name !== "string" || + typeof value.path !== "string" || + !Number.isSafeInteger(value.workflow_id) || + (value.workflow_id as number) < 1 || + typeof value.created_at !== "string" || + !GITHUB_TIMESTAMP_PATTERN.test(value.created_at) || + typeof value.event !== "string" || + typeof value.head_branch !== "string" || + typeof value.head_sha !== "string" || + !SHA_PATTERN.test(value.head_sha) || + !Number.isSafeInteger(value.run_attempt) || + (value.run_attempt as number) < 1 || + typeof value.status !== "string" || + (value.conclusion !== null && typeof value.conclusion !== "string") || + typeof value.display_title !== "string" || + typeof value.html_url !== "string" || + !isObjectRecord(value.repository) || + typeof value.repository.full_name !== "string" || + !isObjectRecord(value.head_repository) || + typeof value.head_repository.full_name !== "string" + ) { + throw new Error("GitHub returned an invalid source workflow run"); + } + return { + id: value.id as number, + runName: value.name, + path: value.path, + workflowId: value.workflow_id as number, + createdAt: value.created_at, + event: value.event, + headBranch: value.head_branch, + headSha: value.head_sha, + runAttempt: value.run_attempt as number, + status: value.status, + conclusion: value.conclusion, + displayTitle: value.display_title, + htmlUrl: value.html_url, + repository: value.repository.full_name, + headRepository: value.head_repository.full_name, + }; +} + +function validateSourceWorkflow(value: unknown): SourceWorkflow { + if ( + !isObjectRecord(value) || + !Number.isSafeInteger(value.id) || + (value.id as number) < 1 || + typeof value.name !== "string" || + typeof value.path !== "string" || + typeof value.state !== "string" + ) { + throw new Error("GitHub returned an invalid source workflow"); + } + return { + id: value.id as number, + name: value.name, + path: value.path, + state: value.state, + }; +} + +async function readSourceWorkflowRun( + repository: string, + token: string, + sourceRunId: number, +): Promise { + return validateSourceWorkflowRun( + await githubApi(`repos/${repository}/actions/runs/${sourceRunId}`, token, { + userAgent: USER_AGENT, + }), + ); +} + +async function readSourceWorkflow( + repository: string, + token: string, + workflowId: number, +): Promise { + return validateSourceWorkflow( + await githubApi(`repos/${repository}/actions/workflows/${workflowId}`, token, { + userAgent: USER_AGENT, + }), + ); +} + +function eligibleRunDefinition( + source: SourceWorkflowRun, + workflow: SourceWorkflow, + repository: string, +): (typeof SOURCE_WORKFLOWS)[number] | null { + const definition = SOURCE_WORKFLOWS.find( + (candidate) => + candidate.workflowName === workflow.name && + candidate.path === workflow.path && + candidate.path === source.path && + candidate.runName === source.runName, + ); + if (!definition) return null; + const exactRunUrl = `https://github.com/${repository}/actions/runs/${source.id}`; + if ( + workflow.id !== source.workflowId || + workflow.state !== "active" || + source.repository !== repository || + source.headRepository !== repository || + source.headBranch !== "main" || + source.htmlUrl !== exactRunUrl || + !(definition.events as readonly string[]).includes(source.event) + ) { + return null; + } + if ("displayTitle" in definition && source.displayTitle !== definition.displayTitle) { + return null; + } + return definition; +} + +function eligibleSourceDefinition( + source: SourceWorkflowRun, + workflow: SourceWorkflow, + repository: string, + sourceRunId: number, +): (typeof SOURCE_WORKFLOWS)[number] | null { + const definition = eligibleRunDefinition(source, workflow, repository); + return definition && + source.id === sourceRunId && + source.runAttempt === 1 && + source.status === "completed" && + source.conclusion === "failure" + ? definition + : null; +} + +function validateWorkflowRunsPage(value: unknown): WorkflowRunsPage { + if ( + !isObjectRecord(value) || + !Number.isSafeInteger(value.total_count) || + (value.total_count as number) < 0 || + !Array.isArray(value.workflow_runs) || + value.workflow_runs.length > 100 + ) { + throw new Error("GitHub returned an invalid workflow run listing"); + } + return { + totalCount: value.total_count as number, + runs: value.workflow_runs.map(validateSourceWorkflowRun), + }; +} + +function isStrictlyOlderRun(previous: SourceWorkflowRun, current: SourceWorkflowRun): boolean { + return ( + current.createdAt < previous.createdAt || + (current.createdAt === previous.createdAt && current.id < previous.id) + ); +} + +async function latestEligibleRunEvidence(options: { + repository: string; + token: string; + source: SourceWorkflowRun; + workflow: SourceWorkflow; +}): Promise { + const runs: SourceWorkflowRun[] = []; + const runIds = new Set(); + let totalCount: number | undefined; + for (let page = 1; page <= MAX_WORKFLOW_RUN_PAGES; page += 1) { + const response = validateWorkflowRunsPage( + await githubApi( + `repos/${options.repository}/actions/workflows/${options.workflow.id}/runs?branch=main&per_page=100&page=${page}`, + options.token, + { userAgent: USER_AGENT }, + ), + ); + totalCount ??= response.totalCount; + if (response.totalCount !== totalCount || runs.length + response.runs.length > totalCount) { + throw new Error("GitHub returned an unstable workflow run count"); + } + for (const run of response.runs) { + if (run.workflowId !== options.workflow.id) { + throw new Error("workflow run listing crossed its authenticated workflow boundary"); + } + if (runIds.has(run.id)) { + throw new Error("GitHub returned duplicate workflow run IDs"); + } + if (runs.length > 0 && !isStrictlyOlderRun(runs.at(-1)!, run)) { + throw new Error("GitHub returned an ambiguously ordered workflow run listing"); + } + runIds.add(run.id); + runs.push(run); + } + + const sourceIndex = runs.findIndex((run) => run.id === options.source.id); + if (sourceIndex >= 0) { + if (JSON.stringify(runs[sourceIndex]) !== JSON.stringify(options.source)) { + throw new Error("source run did not match its workflow run listing"); + } + if (response.runs.length < 100 && runs.length < totalCount) { + throw new Error("workflow run listing was incomplete before its reported total"); + } + const newerEligibleRun = runs + .slice(0, sourceIndex) + .find((run) => eligibleRunDefinition(run, options.workflow, options.repository) !== null); + return newerEligibleRun ? null : { runs, totalCount }; + } + if (runs.length === totalCount) { + throw new Error("source run was not found in its workflow run listing"); + } + if (response.runs.length < 100) { + throw new Error("workflow run listing was incomplete before the source run"); + } + } + throw new Error("workflow run listing exceeded its page limit before the source run"); +} + +async function collectRecoverySnapshot(options: { + repository: string; + token: string; + sourceRunId: number; +}): Promise { + const source = await readSourceWorkflowRun( + options.repository, + options.token, + options.sourceRunId, + ); + const workflow = await readSourceWorkflow(options.repository, options.token, source.workflowId); + const definition = eligibleSourceDefinition( + source, + workflow, + options.repository, + options.sourceRunId, + ); + if (!definition) return null; + const latestRuns = await latestEligibleRunEvidence({ + repository: options.repository, + token: options.token, + source, + workflow, + }); + if (!latestRuns) return null; + const jobDetails = await listNonPassingWorkflowJobs( + options.repository, + options.token, + options.sourceRunId, + 1, + { + includeAnnotations: true, + hostedRunnerLossPolicy: definition.policy, + }, + ); + if (!jobDetails.complete || jobDetails.jobs.length === 0) return null; + const exactAllowedLabels = jobDetails.jobs.every( + (job) => + job.labels?.length === 1 && + (definition.allowedRunnerLabels as readonly string[]).includes(job.labels[0]!), + ); + if (!exactAllowedLabels) return null; + const evidence = verifiedRunnerLossEvidence({ + repository: options.repository, + workflowSha: source.headSha, + workflowConclusion: source.conclusion, + jobs: jobDetails.jobs, + jobDetailsAvailable: true, + jobDetailsComplete: jobDetails.complete, + policy: definition.policy, + }); + if ( + !evidence || + !detectRunnerLoss(evidence) || + evidence.runnerLostMarkerCount !== jobDetails.jobs.length + ) { + return null; + } + const confirmedSource = await readSourceWorkflowRun( + options.repository, + options.token, + options.sourceRunId, + ); + const confirmedWorkflow = await readSourceWorkflow( + options.repository, + options.token, + confirmedSource.workflowId, + ); + const confirmedDefinition = eligibleSourceDefinition( + confirmedSource, + confirmedWorkflow, + options.repository, + options.sourceRunId, + ); + if (!confirmedDefinition) { + throw new Error("source workflow identity changed during runner-loss evidence collection"); + } + const confirmedLatestRuns = await latestEligibleRunEvidence({ + repository: options.repository, + token: options.token, + source: confirmedSource, + workflow: confirmedWorkflow, + }); + if ( + !confirmedLatestRuns || + JSON.stringify(confirmedDefinition) !== JSON.stringify(definition) || + JSON.stringify(confirmedSource) !== JSON.stringify(source) || + JSON.stringify(confirmedWorkflow) !== JSON.stringify(workflow) || + JSON.stringify(confirmedLatestRuns) !== JSON.stringify(latestRuns) + ) { + throw new Error("source workflow evidence changed during runner-loss evidence collection"); + } + return { + fingerprint: sha256( + JSON.stringify({ + jobs: workflowJobEvidenceFingerprint(jobDetails), + latestRuns: confirmedLatestRuns, + source: confirmedSource, + workflow: confirmedWorkflow, + }), + ), + }; +} + +export async function recoverHostedRunnerLoss(options: { + repository: string; + token: string; + controllerRunAttempt: number; + sourceRunId: number; +}): Promise { + if (options.repository !== TRUSTED_REPOSITORY) { + throw new Error(`hosted-runner recovery is restricted to ${TRUSTED_REPOSITORY}`); + } + if (!options.token) throw new Error("GitHub token is required"); + if ( + !Number.isSafeInteger(options.controllerRunAttempt) || + options.controllerRunAttempt < 1 || + !Number.isSafeInteger(options.sourceRunId) || + options.sourceRunId < 1 + ) { + throw new Error("controller attempt and source run ID must be positive safe integers"); + } + if (options.controllerRunAttempt !== 1) { + return { + action: "ignored", + reason: "controller reruns cannot request source workflow reruns", + }; + } + + const initial = await collectRecoverySnapshot(options); + if (!initial) { + return { + action: "ignored", + reason: "source run did not provide exact latest-eligible hosted-runner-loss evidence", + }; + } + const confirmed = await collectRecoverySnapshot(options); + if (!confirmed || confirmed.fingerprint !== initial.fingerprint) { + throw new Error("hosted-runner-loss evidence changed before rerun"); + } + + await githubApi( + `repos/${options.repository}/actions/runs/${options.sourceRunId}/rerun`, + options.token, + { + method: "POST", + userAgent: USER_AGENT, + }, + ); + return { + action: "rerun-requested", + reason: "exact hosted-runner-loss evidence remained stable across both reads", + }; +} + +function requiredEnvironment(name: string): string { + const value = process.env[name]; + if (!value) throw new Error(`${name} is required`); + return value; +} + +function positiveIntegerEnvironment(name: string): number { + const value = requiredEnvironment(name); + if (!/^[1-9][0-9]*$/u.test(value)) throw new Error(`${name} must be a positive integer`); + const parsed = Number(value); + if (!Number.isSafeInteger(parsed)) throw new Error(`${name} exceeds the safe integer range`); + return parsed; +} + +async function main(): Promise { + const result = await recoverHostedRunnerLoss({ + repository: requiredEnvironment("GITHUB_REPOSITORY"), + token: requiredEnvironment("GITHUB_TOKEN"), + controllerRunAttempt: positiveIntegerEnvironment("GITHUB_RUN_ATTEMPT"), + sourceRunId: positiveIntegerEnvironment("SOURCE_RUN_ID"), + }); + console.log( + result.action === "rerun-requested" + ? "Requested one full hosted-runner recovery rerun." + : "No hosted-runner recovery rerun was requested.", + ); +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((error: unknown) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); + }); +} From ee95c420d11e43a1846069f65eddc35109edb77e Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Sat, 25 Jul 2026 08:44:02 -0700 Subject: [PATCH 05/10] ci: run hosted runner recovery controller Signed-off-by: Apurv Kumaria --- .github/workflows/hosted-runner-recovery.yaml | 84 ++++++++ ci/source-shape-test-budget.json | 5 + test/helpers/e2e-workflow-contract.ts | 2 +- test/helpers/vitest-watch-triggers.ts | 6 + test/hosted-runner-recovery-workflow.test.ts | 179 ++++++++++++++++++ test/vitest-watch-triggers.test.ts | 14 ++ 6 files changed, 289 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/hosted-runner-recovery.yaml create mode 100644 test/hosted-runner-recovery-workflow.test.ts diff --git a/.github/workflows/hosted-runner-recovery.yaml b/.github/workflows/hosted-runner-recovery.yaml new file mode 100644 index 0000000000..8a0c0b1110 --- /dev/null +++ b/.github/workflows/hosted-runner-recovery.yaml @@ -0,0 +1,84 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +name: Hosted Runner Recovery + +run-name: Hosted runner recovery for source run ${{ github.event.workflow_run.id }} + +on: + workflow_run: + workflows: + - E2E + - E2E / WSL + - E2E / macOS + - CI / Platform Vitest Main Watch + types: + - completed + +permissions: {} + +jobs: + recover: + if: >- + ${{ + github.run_attempt == 1 && + github.repository == 'NVIDIA/NemoClaw' && + github.event.workflow_run.run_attempt == 1 && + github.event.workflow_run.status == 'completed' && + github.event.workflow_run.conclusion == 'failure' && + github.event.workflow_run.head_branch == 'main' && + github.event.workflow_run.head_repository.full_name == 'NVIDIA/NemoClaw' && + ( + ( + github.event.workflow_run.path == '.github/workflows/e2e.yaml' && + (github.event.workflow_run.event == 'schedule' || + github.event.workflow_run.event == 'workflow_dispatch') && + github.event.workflow_run.display_title == 'E2E main' + ) || + ( + github.event.workflow_run.event == 'push' && + ( + github.event.workflow_run.path == '.github/workflows/wsl-e2e.yaml' || + github.event.workflow_run.path == '.github/workflows/macos-e2e.yaml' || + github.event.workflow_run.path == '.github/workflows/platform-vitest-main.yaml' + ) + ) + ) + }} + concurrency: + group: hosted-runner-recovery-${{ github.event.workflow_run.workflow_id }} + queue: max + cancel-in-progress: false + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + actions: write + checks: read + contents: read + steps: + - name: Checkout trusted recovery controller + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + ref: ${{ github.workflow_sha }} + persist-credentials: false + + - name: Setup Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: "22" + + - name: Evaluate exact hosted-runner-loss evidence + env: + GITHUB_TOKEN: ${{ github.token }} + SOURCE_RUN_ID: ${{ github.event.workflow_run.id }} + run: >- + node --experimental-strip-types --no-warnings + tools/e2e/hosted-runner-recovery.mts + + - name: Record static recovery policy + if: ${{ always() }} + shell: bash + run: >- + printf '%s\n' + 'Hosted-runner recovery evaluated the fail-closed latest-eligible main-run policy.' + >> "$GITHUB_STEP_SUMMARY" diff --git a/ci/source-shape-test-budget.json b/ci/source-shape-test-budget.json index fe88cdbc92..7ea600a074 100644 --- a/ci/source-shape-test-budget.json +++ b/ci/source-shape-test-budget.json @@ -301,6 +301,11 @@ "test": "restores exact locked posture after root-separated repair and later failure (#7033)", "category": "security" }, + { + "file": "test/hosted-runner-recovery-workflow.test.ts", + "test": "locks recovery identities to the source workflows' runtime names (#7140)", + "category": "security" + }, { "file": "test/inference-options-docs.test.ts", "test": "keeps a per-model task-fit comparison table for curated onboarding models", diff --git a/test/helpers/e2e-workflow-contract.ts b/test/helpers/e2e-workflow-contract.ts index 6fb76de99f..a8d799cca8 100644 --- a/test/helpers/e2e-workflow-contract.ts +++ b/test/helpers/e2e-workflow-contract.ts @@ -9,7 +9,7 @@ import YAML from "yaml"; const REPO_ROOT = join(dirname(fileURLToPath(import.meta.url)), "..", ".."); export type WorkflowJob = { - concurrency?: { group: string; "cancel-in-progress": boolean }; + concurrency?: { group: string; queue?: "max"; "cancel-in-progress": boolean }; environment?: string | { name: string; deployment?: boolean }; if?: string; name?: string; diff --git a/test/helpers/vitest-watch-triggers.ts b/test/helpers/vitest-watch-triggers.ts index 080f7390bd..67e7f07263 100644 --- a/test/helpers/vitest-watch-triggers.ts +++ b/test/helpers/vitest-watch-triggers.ts @@ -18,6 +18,7 @@ const E2E_WORKFLOW_CONTRACTS = [ "test/e2e/support/gateway-guard-workflow-boundary.test.ts", "test/e2e/support/hermes-dashboard-workflow-boundary.test.ts", "test/e2e/support/hermes-workflow-boundary.test.ts", + "test/hosted-runner-recovery-workflow.test.ts", "test/e2e/support/inference-switch-workflow-boundary.test.ts", "test/e2e/support/jetson-workflow-boundary.test.ts", "test/e2e/support/mcp-workflow-boundary.test.ts", @@ -97,6 +98,11 @@ export const vitestWatchTriggerPatterns: VitestWatchTriggerPattern[] = [ pattern: /(?:^|\/)\.github\/workflows\/pr-e2e-gate\.yaml$/, testsToRun: runTests("test/pr-e2e-gate-workflow.test.ts", "test/pr-e2e-required.test.ts"), }, + { + pattern: + /(?:^|\/)\.github\/workflows\/(?:hosted-runner-recovery|wsl-e2e|macos-e2e|platform-vitest-main)\.yaml$/, + testsToRun: runTests("test/hosted-runner-recovery-workflow.test.ts"), + }, { pattern: /(?:^|\/)(?:\.github\/workflows\/platform-vitest-main\.yaml|ci\/platform-vitest-macos-requirements\.lock)$/, diff --git a/test/hosted-runner-recovery-workflow.test.ts b/test/hosted-runner-recovery-workflow.test.ts new file mode 100644 index 0000000000..2f3f75be07 --- /dev/null +++ b/test/hosted-runner-recovery-workflow.test.ts @@ -0,0 +1,179 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; +import { readYaml, type WorkflowJob, type WorkflowStep } from "./helpers/e2e-workflow-contract.ts"; + +const WORKFLOW_PATH = ".github/workflows/hosted-runner-recovery.yaml"; +const E2E_WORKFLOW_PATH = ".github/workflows/e2e.yaml"; +const WSL_WORKFLOW_PATH = ".github/workflows/wsl-e2e.yaml"; +const MACOS_WORKFLOW_PATH = ".github/workflows/macos-e2e.yaml"; +const PLATFORM_WORKFLOW_PATH = ".github/workflows/platform-vitest-main.yaml"; +const TRUSTED_CHECKOUT = "actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10"; +const TRUSTED_SETUP_NODE = "actions/setup-node@820762786026740c76f36085b0efc47a31fe5020"; +const E2E_RUN_NAME = + "${{ inputs.checkout_sha != '' && format('E2E PR #{0} ({1})', inputs.pr_number, inputs.correlation_id) || format('E2E {0}', github.ref_name) }}"; + +type RecoveryWorkflow = { + name: string; + "run-name": string; + on: { + workflow_run: { + workflows: string[]; + types: string[]; + }; + }; + permissions: Record; + jobs: { + recover: WorkflowJob; + }; +}; + +type SourceWorkflowIdentity = { + name: string; + "run-name"?: string; +}; + +function workflow(): RecoveryWorkflow { + return readYaml(WORKFLOW_PATH); +} + +function sourceWorkflow(path: string): SourceWorkflowIdentity { + return readYaml(path); +} + +function step(job: WorkflowJob, name: string): WorkflowStep { + const match = job.steps?.find((candidate) => candidate.name === name); + expect(match, `missing workflow step ${name}`).toBeDefined(); + return match!; +} + +function collectStrings(value: unknown): string[] { + return typeof value === "string" + ? [value] + : Array.isArray(value) + ? value.flatMap(collectStrings) + : value && typeof value === "object" + ? Object.values(value).flatMap(collectStrings) + : []; +} + +describe("hosted-runner recovery workflow boundary", () => { + it("subscribes only to completed runs from the four approved workflows (#7140)", () => { + const value = workflow(); + expect(value.name).toBe("Hosted Runner Recovery"); + expect(value.on).toEqual({ + workflow_run: { + workflows: ["E2E", "E2E / WSL", "E2E / macOS", "CI / Platform Vitest Main Watch"], + types: ["completed"], + }, + }); + expect(value.permissions).toEqual({}); + expect(value).not.toHaveProperty("concurrency"); + expect(value.jobs.recover.concurrency).toEqual({ + group: "hosted-runner-recovery-${{ github.event.workflow_run.workflow_id }}", + queue: "max", + "cancel-in-progress": false, + }); + expect(Object.keys(value.jobs)).toEqual(["recover"]); + }); + + // source-shape-contract: security -- Exact source workflow names and run-name expressions keep the write-capable recovery subscription bound to reviewed trusted-main identities + it("locks recovery identities to the source workflows' runtime names (#7140)", () => { + const e2e = sourceWorkflow(E2E_WORKFLOW_PATH); + const wsl = sourceWorkflow(WSL_WORKFLOW_PATH); + const macos = sourceWorkflow(MACOS_WORKFLOW_PATH); + const platform = sourceWorkflow(PLATFORM_WORKFLOW_PATH); + + expect(e2e).toMatchObject({ name: "E2E", "run-name": E2E_RUN_NAME }); + expect(E2E_RUN_NAME).toContain("format('E2E {0}', github.ref_name)"); + expect([wsl.name, macos.name, platform.name]).toEqual([ + "E2E / WSL", + "E2E / macOS", + "CI / Platform Vitest Main Watch", + ]); + expect(wsl).not.toHaveProperty("run-name"); + expect(macos).not.toHaveProperty("run-name"); + expect(platform).not.toHaveProperty("run-name"); + expect(workflow().on.workflow_run.workflows).toEqual([ + e2e.name, + wsl.name, + macos.name, + platform.name, + ]); + }); + + it("fails closed on controller, source, repository, branch, event, path, and title (#7140)", () => { + const guard = workflow().jobs.recover.if ?? ""; + for (const fragment of [ + "github.run_attempt == 1", + "github.repository == 'NVIDIA/NemoClaw'", + "github.event.workflow_run.run_attempt == 1", + "github.event.workflow_run.status == 'completed'", + "github.event.workflow_run.conclusion == 'failure'", + "github.event.workflow_run.head_branch == 'main'", + "github.event.workflow_run.head_repository.full_name == 'NVIDIA/NemoClaw'", + "github.event.workflow_run.path == '.github/workflows/e2e.yaml'", + "github.event.workflow_run.event == 'schedule'", + "github.event.workflow_run.event == 'workflow_dispatch'", + "github.event.workflow_run.display_title == 'E2E main'", + "github.event.workflow_run.event == 'push'", + "github.event.workflow_run.path == '.github/workflows/wsl-e2e.yaml'", + "github.event.workflow_run.path == '.github/workflows/macos-e2e.yaml'", + "github.event.workflow_run.path == '.github/workflows/platform-vitest-main.yaml'", + ]) { + expect(guard).toContain(fragment); + } + expect(guard).not.toContain("pull_request"); + }); + + it("uses only the least privileges and trusted default-branch controller (#7140)", () => { + const job = workflow().jobs.recover; + expect(job["runs-on"]).toBe("ubuntu-latest"); + expect(job["timeout-minutes"]).toBe(15); + expect(job.permissions).toEqual({ + actions: "write", + checks: "read", + contents: "read", + }); + + const checkout = step(job, "Checkout trusted recovery controller"); + expect(checkout.uses).toBe(TRUSTED_CHECKOUT); + expect(checkout.with).toEqual({ + ref: "${{ github.workflow_sha }}", + "persist-credentials": false, + }); + const setupNode = step(job, "Setup Node.js"); + expect(setupNode.uses).toBe(TRUSTED_SETUP_NODE); + expect(setupNode.with).toEqual({ "node-version": "22" }); + expect( + job.steps?.filter((candidate) => candidate.uses?.startsWith("actions/checkout@")), + ).toHaveLength(1); + expect(collectStrings(job).some((value) => value.includes("secrets."))).toBe(false); + expect(collectStrings(job).some((value) => value.includes("npm ci"))).toBe(false); + expect( + collectStrings(job).some((value) => value.includes("github.event.workflow_run.head_sha")), + ).toBe(false); + }); + + it("runs native TypeScript with bounded metadata and no source checkout (#7140)", () => { + const evaluate = step(workflow().jobs.recover, "Evaluate exact hosted-runner-loss evidence"); + expect(evaluate.env).toEqual({ + GITHUB_TOKEN: "${{ github.token }}", + SOURCE_RUN_ID: "${{ github.event.workflow_run.id }}", + }); + expect(evaluate.run).toBe( + "node --experimental-strip-types --no-warnings tools/e2e/hosted-runner-recovery.mts", + ); + }); + + it("writes only a static policy sentence to the job summary (#7140)", () => { + const summary = step(workflow().jobs.recover, "Record static recovery policy"); + expect(summary.if).toBe("${{ always() }}"); + expect(summary.run).toBe( + "printf '%s\\n' 'Hosted-runner recovery evaluated the fail-closed latest-eligible main-run policy.' >> \"$GITHUB_STEP_SUMMARY\"", + ); + expect(summary.run).not.toContain("${{"); + expect(summary.run).not.toContain("SOURCE_RUN"); + }); +}); diff --git a/test/vitest-watch-triggers.test.ts b/test/vitest-watch-triggers.test.ts index 0e67b043dc..8ce6f3b349 100644 --- a/test/vitest-watch-triggers.test.ts +++ b/test/vitest-watch-triggers.test.ts @@ -24,6 +24,7 @@ const E2E_WORKFLOW_CONTRACTS = [ "test/e2e/support/gateway-guard-workflow-boundary.test.ts", "test/e2e/support/hermes-dashboard-workflow-boundary.test.ts", "test/e2e/support/hermes-workflow-boundary.test.ts", + "test/hosted-runner-recovery-workflow.test.ts", "test/e2e/support/inference-switch-workflow-boundary.test.ts", "test/e2e/support/jetson-workflow-boundary.test.ts", "test/e2e/support/mcp-workflow-boundary.test.ts", @@ -59,6 +60,9 @@ const OPAQUE_INPUTS = [ ".github/workflows/e2e.yaml", ".github/workflows/code-scanning.yaml", ".github/workflows/pr-e2e-gate.yaml", + ".github/workflows/hosted-runner-recovery.yaml", + ".github/workflows/wsl-e2e.yaml", + ".github/workflows/macos-e2e.yaml", ".github/workflows/platform-vitest-main.yaml", "ci/platform-vitest-macos-requirements.lock", ] as const; @@ -112,7 +116,17 @@ describe("Vitest opaque-input watch triggers", () => { "test/pr-e2e-gate-workflow.test.ts", "test/pr-e2e-required.test.ts", ]); + expect(triggeredBy(".github/workflows/hosted-runner-recovery.yaml")).toEqual([ + "test/hosted-runner-recovery-workflow.test.ts", + ]); + expect(triggeredBy(".github/workflows/wsl-e2e.yaml")).toEqual([ + "test/hosted-runner-recovery-workflow.test.ts", + ]); + expect(triggeredBy(".github/workflows/macos-e2e.yaml")).toEqual([ + "test/hosted-runner-recovery-workflow.test.ts", + ]); expect(triggeredBy(".github/workflows/platform-vitest-main.yaml")).toEqual([ + "test/hosted-runner-recovery-workflow.test.ts", "test/platform-vitest-main-workflow.test.ts", ]); expect(triggeredBy("ci/platform-vitest-macos-requirements.lock")).toEqual([ From 7a4c3762282cc2cc8a0f7ec0e4d433b6ff69bd10 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Sat, 25 Jul 2026 08:48:14 -0700 Subject: [PATCH 06/10] docs(e2e): document hosted runner recovery Signed-off-by: Apurv Kumaria --- test/e2e/README.md | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/test/e2e/README.md b/test/e2e/README.md index 527992f12e..b156dec0a4 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -16,6 +16,9 @@ before those targets run; local runners must provide it themselves. GitHub Actions check suite. - `.github/workflows/e2e-branch-validation.yaml` provisions Brev instances and runs focused E2E targets from source on a clean machine. +- `.github/workflows/hosted-runner-recovery.yaml` evaluates first-attempt + failures from approved `main` workflows and requests one full rerun only when + every non-passing job has authenticated GitHub-hosted runner-loss evidence. - Platform workflows such as macOS, WSL, sandbox image, and regression E2E call their target E2E tests directly. The Ollama auth proxy target is selected through `.github/workflows/e2e.yaml`. @@ -178,6 +181,43 @@ graph as the live targets: `post_to_slack=true`, which uses the preview Slack route. Branch-dispatched runs never receive Slack webhook secrets. +### Hosted-runner recovery + +The trusted recovery workflow can request one full rerun of the latest eligible +first attempt for these source workflows: + +- scheduled or manually dispatched `E2E main`; +- `E2E / WSL` pushes to `main`; +- `E2E / macOS` pushes to `main`; and +- `CI / Platform Vitest Main Watch` pushes to `main`. + +The controller authenticates the active workflow name and path, repository, +head repository, branch, run name, event, run URL, and first-attempt failure. +It ignores E2E PR child runs and an eligible source run that has a newer +eligible run for the same workflow. + +The complete non-passing job listing must contain only exact hosted-runner-loss +markers for that workflow's approved runner labels. +The E2E policy accepts only `ubuntu-latest`. +The WSL and macOS policies accept only `cancelled` jobs with their exact +platform label and exact GitHub internal-error annotation. +The platform-watch policy applies those platform contracts to Windows and +macOS jobs and the authenticated hosted-runner-loss contract to Ubuntu jobs. +An ordinary assertion failure, mixed failure set, incomplete listing, custom or +self-hosted label, changed evidence, or ambiguous pagination prevents recovery. + +The controller collects and fingerprints the complete source, workflow, +latest-run, job, check, annotation, and optional log evidence twice. +It requests the full GitHub Actions rerun only when both snapshots match. +The rerun executes every job in attempt two, not only the failed jobs. +Neither a source attempt two nor a recovery-controller rerun can request +another source rerun. + +The recovery job checks out only the trusted default-branch controller and does +not check out source-run code. +It receives no repository secrets and holds `actions: write` only for the +bounded rerun request. + ### Runner comparison telemetry Trusted `main` runs without an alternate checkout SHA record runner-comparison From 29224ac332714168b2ec1734d9a2a88db5ca249d Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Sat, 25 Jul 2026 13:42:02 -0700 Subject: [PATCH 07/10] test(ci): keep recovery fixtures linear Signed-off-by: Apurv Kumaria --- .../hosted-runner-loss-internal-error.test.ts | 17 +++---- test/hosted-runner-recovery.test.ts | 51 ++++++++++--------- 2 files changed, 35 insertions(+), 33 deletions(-) diff --git a/test/hosted-runner-loss-internal-error.test.ts b/test/hosted-runner-loss-internal-error.test.ts index 324f1d9bf7..24ff133ae2 100644 --- a/test/hosted-runner-loss-internal-error.test.ts +++ b/test/hosted-runner-loss-internal-error.test.ts @@ -354,8 +354,15 @@ describe("hosted-runner GitHub internal-error evidence", () => { ).toBeNull(); }); + it("rejects an empty internal-error policy (#7140)", () => { + expect(() => + confirmsInternalError(internalErrorJob(), { + githubInternalError: {} as never, + }), + ).toThrow(/policy/u); + }); + it.each([ - {}, { githubInternalError: { approvedRunnerLabels: [], approvedJobConclusions: ["cancelled"] } }, { githubInternalError: { @@ -395,14 +402,6 @@ describe("hosted-runner GitHub internal-error evidence", () => { }, { unsupported: true }, ])("rejects malformed internal-error policy %# (#7140)", (policy) => { - if (Object.keys(policy).length === 0) { - expect(() => - confirmsInternalError(internalErrorJob(), { - githubInternalError: policy as never, - }), - ).toThrow(/policy/u); - return; - } expect(() => confirmsInternalError(internalErrorJob(), policy as HostedRunnerLossPolicy), ).toThrow(/policy/u); diff --git a/test/hosted-runner-recovery.test.ts b/test/hosted-runner-recovery.test.ts index 27e65c73a8..607b79c335 100644 --- a/test/hosted-runner-recovery.test.ts +++ b/test/hosted-runner-recovery.test.ts @@ -252,7 +252,8 @@ function setupRoutes(options: RouteOptions = {}) { : { total_count: 0, jobs: [] }; for (const candidate of listing?.jobs ?? []) { const job = candidate as ReturnType; - if (Number.isSafeInteger(job.id)) lastJobs.set(job.id, job); + expect(Number.isSafeInteger(job.id)).toBe(true); + lastJobs.set(job.id, job); } return githubResponse(listing); }, @@ -262,8 +263,8 @@ function setupRoutes(options: RouteOptions = {}) { (request) => { const id = Number(request.url.split("/").at(-1)); const job = lastJobs.get(id); - if (!job) throw new Error(`missing job fixture for check ${id}`); - return githubResponse(checkRun(job)); + expect(job, `missing job fixture for check ${id}`).toBeDefined(); + return githubResponse(checkRun(job!)); }, ), githubFetchRoute( @@ -271,11 +272,11 @@ function setupRoutes(options: RouteOptions = {}) { (request) => { const id = Number(/\/check-runs\/([1-9][0-9]*)\/annotations/u.exec(request.url)?.[1]); const job = lastJobs.get(id); - if (!job) throw new Error(`missing job fixture for annotations ${id}`); + expect(job, `missing job fixture for annotations ${id}`).toBeDefined(); const defaultAnnotation = - job.conclusion === "cancelled" - ? runnerLossAnnotation(INTERNAL_ERROR_MESSAGE, job.head_sha) - : runnerLossAnnotation(RUNNER_LOSS_MESSAGE, job.head_sha); + job!.conclusion === "cancelled" + ? runnerLossAnnotation(INTERNAL_ERROR_MESSAGE, job!.head_sha) + : runnerLossAnnotation(RUNNER_LOSS_MESSAGE, job!.head_sha); const annotations = options.annotations?.[ Math.min(annotationRead, (options.annotations?.length ?? 1) - 1) ] ?? [defaultAnnotation]; @@ -618,29 +619,31 @@ describe("hosted-runner recovery controller", () => { { label: "zero jobs", listing: { total_count: 0, jobs: [] }, - rejects: false, }, { label: "incomplete listing", listing: { total_count: 2, jobs: [hostedRunnerLossJob()] }, - rejects: false, }, - { - label: "duplicate job IDs", - listing: { - total_count: 2, - jobs: [hostedRunnerLossJob(), hostedRunnerLossJob({ name: "duplicate" })], - }, - rejects: true, - }, - ])("never reruns with $label (#7140)", async ({ listing, rejects }) => { + ])("never reruns with $label (#7140)", async ({ listing }) => { const requests = setupRoutes({ jobListings: [listing] }); - const recovery = recoverHostedRunnerLoss(recoveryRequest()); - if (rejects) { - await expect(recovery).rejects.toThrow(/duplicate workflow job IDs/u); - } else { - await expect(recovery).resolves.toMatchObject({ action: "ignored" }); - } + await expect(recoverHostedRunnerLoss(recoveryRequest())).resolves.toMatchObject({ + action: "ignored", + }); + expect(mutationRequests(requests)).toEqual([]); + }); + + it("never reruns a listing with duplicate job IDs (#7140)", async () => { + const requests = setupRoutes({ + jobListings: [ + { + total_count: 2, + jobs: [hostedRunnerLossJob(), hostedRunnerLossJob({ name: "duplicate" })], + }, + ], + }); + await expect(recoverHostedRunnerLoss(recoveryRequest())).rejects.toThrow( + /duplicate workflow job IDs/u, + ); expect(mutationRequests(requests)).toEqual([]); }); From 8a5cfdca1a5fa94e8e2d434c682c9e43ed1088b1 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Sat, 25 Jul 2026 14:12:42 -0700 Subject: [PATCH 08/10] docs(e2e): record recovery retirement condition Signed-off-by: Apurv Kumaria --- test/e2e/README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/e2e/README.md b/test/e2e/README.md index b156dec0a4..daf09357ec 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -218,6 +218,15 @@ not check out source-run code. It receives no repository secrets and holds `actions: write` only for the bounded rerun request. +The runner-allocation and internal-error failures originate in the +GitHub-hosted Actions service, outside repository-controlled workflow code, so +this controller contains the failure without claiming to repair its source. +Remove the recovery workflow and controller after the supported source +workflows record 30 consecutive days with no first-attempt failure accepted by +the exact recovery classifier, or when those workflows stop using +GitHub-hosted runners. Any accepted recovery request resets that observation +window. + ### Runner comparison telemetry Trusted `main` runs without an alternate checkout SHA record runner-comparison From a9d642d664c6ce47ff49fd891426d3d500638089 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Sat, 25 Jul 2026 15:11:10 -0700 Subject: [PATCH 09/10] fix(ci): accept nullable check annotations Signed-off-by: Apurv Kumaria --- test/hosted-runner-recovery.test.ts | 26 +++++++++++++++++++++++++ tools/e2e/hosted-runner-loss-github.mts | 18 ++++++++++------- 2 files changed, 37 insertions(+), 7 deletions(-) diff --git a/test/hosted-runner-recovery.test.ts b/test/hosted-runner-recovery.test.ts index 607b79c335..32f3689bdb 100644 --- a/test/hosted-runner-recovery.test.ts +++ b/test/hosted-runner-recovery.test.ts @@ -394,6 +394,32 @@ describe("hosted-runner recovery controller", () => { expect(requests.some((request) => request.url.endsWith("/rerun-failed-jobs"))).toBe(false); }); + it("requests a rerun when GitHub returns null annotation title and raw details (#7140)", async () => { + const annotation = { ...runnerLossAnnotation(), title: null, raw_details: null }; + const requests = setupRoutes({ annotations: [[annotation], [annotation]] }); + + await expect(recoverHostedRunnerLoss(recoveryRequest())).resolves.toMatchObject({ + action: "rerun-requested", + }); + expect(mutationRequests(requests)).toHaveLength(1); + }); + + it.each([ + ["a non-string title", { title: 7 }], + ["non-string raw details", { raw_details: false }], + ["an oversized title", { title: "x".repeat(16 * 1024 + 1) }], + ["oversized raw details", { raw_details: "x".repeat(16 * 1024 + 1) }], + ])("rejects an annotation with $label (#7140)", async (_label, override) => { + const requests = setupRoutes({ + annotations: [[{ ...runnerLossAnnotation(), ...override }]], + }); + + await expect(recoverHostedRunnerLoss(recoveryRequest())).rejects.toThrow( + /invalid workflow job annotation/u, + ); + expect(mutationRequests(requests)).toEqual([]); + }); + it.each([ { label: "E2E on Windows", diff --git a/tools/e2e/hosted-runner-loss-github.mts b/tools/e2e/hosted-runner-loss-github.mts index 0328121117..5bdb6f1f46 100644 --- a/tools/e2e/hosted-runner-loss-github.mts +++ b/tools/e2e/hosted-runner-loss-github.mts @@ -136,8 +136,12 @@ function validateWorkflowJob(value: unknown): WorkflowJob { } function validateWorkflowJobAnnotation(value: unknown): WorkflowJobAnnotation { + if (!isObjectRecord(value)) { + throw new Error("GitHub returned an invalid workflow job annotation"); + } + const title = value.title === null ? "" : value.title; + const rawDetails = value.raw_details === null ? "" : value.raw_details; if ( - !isObjectRecord(value) || typeof value.path !== "string" || value.path.length === 0 || Buffer.byteLength(value.path, "utf8") > MAX_JOB_ANNOTATION_IDENTITY_BYTES || @@ -153,12 +157,12 @@ function validateWorkflowJobAnnotation(value: unknown): WorkflowJobAnnotation { (!Number.isSafeInteger(value.end_column) || (value.end_column as number) < 1)) || typeof value.annotation_level !== "string" || Buffer.byteLength(value.annotation_level, "utf8") > MAX_JOB_ANNOTATION_IDENTITY_BYTES || - typeof value.title !== "string" || - Buffer.byteLength(value.title, "utf8") > MAX_JOB_ANNOTATION_TEXT_BYTES || + typeof title !== "string" || + Buffer.byteLength(title, "utf8") > MAX_JOB_ANNOTATION_TEXT_BYTES || typeof value.message !== "string" || Buffer.byteLength(value.message, "utf8") > MAX_JOB_ANNOTATION_TEXT_BYTES || - typeof value.raw_details !== "string" || - Buffer.byteLength(value.raw_details, "utf8") > MAX_JOB_ANNOTATION_TEXT_BYTES + typeof rawDetails !== "string" || + Buffer.byteLength(rawDetails, "utf8") > MAX_JOB_ANNOTATION_TEXT_BYTES ) { throw new Error("GitHub returned an invalid workflow job annotation"); } @@ -170,9 +174,9 @@ function validateWorkflowJobAnnotation(value: unknown): WorkflowJobAnnotation { endLine: value.end_line as number, endColumn: value.end_column as number | null, annotationLevel: value.annotation_level, - title: value.title, + title, message: value.message, - rawDetails: value.raw_details, + rawDetails, }; } From 4a2165725af815373f26d66778a78a13a4730bba Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Sat, 25 Jul 2026 15:13:53 -0700 Subject: [PATCH 10/10] docs(e2e): document nullable runner annotations Signed-off-by: Apurv Kumaria --- test/e2e/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/e2e/README.md b/test/e2e/README.md index daf09357ec..4dd1fb430e 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -729,8 +729,8 @@ exactly one failure annotation. Its message must be `The operation was canceled.` for one completed `cancelled` workload step or `Process completed with exit code 143.` for one completed `failure` workload step. The annotation must use `.github`, equal start and end lines, null -columns, and empty title and detail fields. Every annotation must use a blob URL -bound to the same workflow commit. The controller accepts at most 20 +columns, and null or empty title and detail fields. Every annotation must use a +blob URL bound to the same workflow commit. The controller accepts at most 20 annotations, bounds each text field, and limits the normalized annotation evidence to 64 KiB. This permits trusted bounded non-failure notices beside the sole failure annotation without allowing annotation output to exhaust the