From d26fb2c0f41117140446d87d603e14dc3383e661 Mon Sep 17 00:00:00 2001 From: Charan Jagwani Date: Wed, 22 Jul 2026 10:37:34 -0700 Subject: [PATCH 1/5] ci(e2e): publish phase runtime summary Signed-off-by: Charan Jagwani --- .github/workflows/e2e.yaml | 27 ++++++++++++++++- scripts/audit-test-runtime.mts | 21 +++++++++++--- test/e2e/README.md | 11 ++++--- test/e2e/docs/README.md | 4 ++- test/e2e/live/onboard-resume.test.ts | 2 -- .../e2e-operations-workflow-boundary.test.ts | 26 ++++++++++++++++- test/e2e/support/e2e-scorecard.test.ts | 2 ++ test/e2e/support/test-runtime-audit.test.ts | 18 ++++++++++-- tools/e2e/operations-workflow-boundary.mts | 29 +++++++++++++++++-- 9 files changed, 123 insertions(+), 17 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 44f29301f0..9147a392c2 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -5076,15 +5076,24 @@ jobs: persist-credentials: false sparse-checkout: | ci/onboard-performance-budget.json + scripts/audit-test-runtime.mts scripts/scorecard sparse-checkout-cone-mode: false + - name: Download E2E progress artifacts + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + path: ${{ runner.temp }}/e2e-runtime-audit + pattern: e2e-* + - name: Generate E2E scorecard id: scorecard uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: EXPLICIT_ONLY_JOBS: ${{ needs.generate-matrix.outputs.explicit_only_jobs }} JOBS: ${{ inputs.jobs }} + RUNTIME_ARTIFACTS: ${{ runner.temp }}/e2e-runtime-audit TARGETS: ${{ inputs.targets }} with: script: | @@ -5098,6 +5107,9 @@ jobs: const traceTiming = require( path.join(process.env.GITHUB_WORKSPACE, 'scripts/scorecard/analyze-trace-timing.mts'), ); + const runtimeAudit = require( + path.join(process.env.GITHUB_WORKSPACE, 'scripts/audit-test-runtime.mts'), + ); const needs = ${{ toJSON(needs) }}; // GitHub's jobs API is the canonical source because `needs.live` @@ -5106,6 +5118,19 @@ jobs: const apiJobs = await scorecardJobs.loadWorkflowRunJobs({ github, context, core }); const trace = await traceTiming.buildTraceTimingResult({ github, context, core }); if (trace.budgetWarningMessage) core.warning(trace.budgetWarningMessage); + let runtimeSummaryMarkdown; + try { + const runtimeRows = runtimeAudit.auditTestRuntime([process.env.RUNTIME_ARTIFACTS]); + runtimeSummaryMarkdown = runtimeAudit.formatRuntimeAuditSummary(runtimeRows); + } catch { + core.warning('E2E test phase runtime summary unavailable: invalid progress artifact'); + runtimeSummaryMarkdown = [ + '## E2E Test Phase Runtime', + '', + 'The summary is unavailable because a `test-progress.json` artifact was invalid.', + '', + ].join('\n'); + } const { summaryMarkdown, scorecardData, slackData } = buildScorecard({ eventName: context.eventName, @@ -5122,7 +5147,7 @@ jobs: today: new Date().toLocaleDateString('en-US', { month: 'short', day: 'numeric' }), }); - await core.summary.addRaw(summaryMarkdown).write(); + await core.summary.addRaw(`${summaryMarkdown}\n\n${runtimeSummaryMarkdown}`).write(); core.setOutput('scorecardData', JSON.stringify(scorecardData)); core.setOutput('slackData', JSON.stringify(slackData)); diff --git a/scripts/audit-test-runtime.mts b/scripts/audit-test-runtime.mts index fe3647d010..c6611a7bf2 100644 --- a/scripts/audit-test-runtime.mts +++ b/scripts/audit-test-runtime.mts @@ -5,7 +5,7 @@ import fs from "node:fs"; import path from "node:path"; import { pathToFileURL } from "node:url"; -import type { ProgressSummary } from "../test/e2e/fixtures/progress.ts"; +import type { ProgressPhase, ProgressSummary } from "../test/e2e/fixtures/progress.ts"; export interface RuntimeAuditRow { target: string; @@ -17,6 +17,7 @@ export interface RuntimeAuditRow { variabilityMs: number; slowestPhase: string; slowestPhaseMs: number; + slowestPhaseOutcome: "passed" | "failed" | "skipped"; } function isProgressSummary(value: unknown): value is ProgressSummary { @@ -36,6 +37,7 @@ function isProgressSummary(value: unknown): value is ProgressSummary { (phase) => phase && typeof phase.label === "string" && + (phase.outcome === "passed" || phase.outcome === "failed" || phase.outcome === "skipped") && typeof phase.durationMs === "number" && Number.isFinite(phase.durationMs) && phase.durationMs >= 0, @@ -100,9 +102,9 @@ export function auditTestRuntime(roots: readonly string[]): RuntimeAuditRow[] { if (!first) throw new Error("runtime audit group is unexpectedly empty"); const durations = runs.map((run) => run.durationMs as number).sort((a, b) => a - b); const phases = runs.flatMap((run) => run.phases); - const slowestPhase = phases.reduce( + const slowestPhase = phases.reduce>( (slowest, phase) => (phase.durationMs > slowest.durationMs ? phase : slowest), - { label: "n/a", durationMs: 0 }, + { label: "n/a", durationMs: 0, outcome: "skipped" as const }, ); const medianMs = median(durations); const p95Ms = percentile(durations, 0.95); @@ -116,6 +118,7 @@ export function auditTestRuntime(roots: readonly string[]): RuntimeAuditRow[] { variabilityMs: Math.max(0, p95Ms - medianMs), slowestPhase: slowestPhase.label, slowestPhaseMs: slowestPhase.durationMs, + slowestPhaseOutcome: slowestPhase.outcome, }; }) .sort((a, b) => b.p95Ms - a.p95Ms || b.variabilityMs - a.variabilityMs); @@ -132,12 +135,22 @@ export function formatRuntimeAudit(rows: readonly RuntimeAuditRow[]): string { ]; for (const row of rows) { lines.push( - `| ${row.target.replaceAll("|", "\\|")} | ${row.scenario.replaceAll("|", "\\|")} | ${row.runs} | ${seconds(row.medianMs)}s | ${seconds(row.p95Ms)}s | ${seconds(row.maxMs)}s | ${seconds(row.variabilityMs)}s | ${row.slowestPhase.replaceAll("|", "\\|")} (${seconds(row.slowestPhaseMs)}s) |`, + `| ${row.target.replaceAll("|", "\\|")} | ${row.scenario.replaceAll("|", "\\|")} | ${row.runs} | ${seconds(row.medianMs)}s | ${seconds(row.p95Ms)}s | ${seconds(row.maxMs)}s | ${seconds(row.variabilityMs)}s | ${row.slowestPhase.replaceAll("|", "\\|")} (${seconds(row.slowestPhaseMs)}s, ${row.slowestPhaseOutcome}) |`, ); } return `${lines.join("\n")}\n`; } +export function formatRuntimeAuditSummary(rows: readonly RuntimeAuditRow[]): string { + const lines = ["## E2E Test Phase Runtime", "", "This run's semantic phase timing summary.", ""]; + if (rows.length === 0) { + lines.push("No `test-progress.json` artifacts were available for this run."); + } else { + lines.push(formatRuntimeAudit(rows).trimEnd()); + } + return `${lines.join("\n")}\n`; +} + function main(argv: readonly string[]): void { const roots = argv.length > 0 ? argv : [".e2e/live"]; const rows = auditTestRuntime(roots); diff --git a/test/e2e/README.md b/test/e2e/README.md index 0c3c7d98a2..c21b61c3ce 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -62,9 +62,10 @@ graph as the live targets: retired. Any future issue escalation should use a separately reviewed exceptional threshold, such as the same lane failing twice consecutively or remaining broken for 24 hours, rather than posting on every failed schedule. -- `scorecard` writes the scheduled/manual result summary, compares the trusted - cloud-onboard timing summary with the latest prior-release `e2e.yaml` run, - and posts to the daily or full-run Slack route. +- `scorecard` writes the scheduled/manual result summary, adds this run's + semantic phase runtime table, compares the trusted cloud-onboard timing + summary with the latest prior-release `e2e.yaml` run, and posts to the daily + or full-run Slack route. - Selective dispatches remain silent unless they run on `main` with `post_to_slack=true`, which uses the preview Slack route. Branch-dispatched runs never receive Slack webhook secrets. @@ -120,7 +121,9 @@ npm run test:runtime-audit -- path/to/run-1 path/to/run-2 ``` The audit groups each test by target and optional shard, ranks the groups by -p95 runtime, and reports variability and the slowest observed phase. Keep phase +p95 runtime, and reports variability plus the slowest observed phase's duration +and outcome. Scheduled and ordinary manual runs include the same table for that +run in the GitHub Actions scorecard summary. Keep phase labels specific to test behavior, call `progress.phase("literal phase label")` at the declared boundaries in order, and transition through the final test-declared phase on every passing path. The fixture rejects a passing live diff --git a/test/e2e/docs/README.md b/test/e2e/docs/README.md index c29f8606eb..a0c49abe32 100644 --- a/test/e2e/docs/README.md +++ b/test/e2e/docs/README.md @@ -101,7 +101,9 @@ During fixture teardown, every passing or failing live test writes `test-progress.json` beside its other target artifacts. The runtime audit groups those files by target, optional shard, and test name, then reports median, p95, maximum, p95-minus-median variability, and the slowest observed -phase. The summary reads the matrix identity from `E2E_TARGET_ID` and +phase with its duration and outcome. Scheduled and ordinary manual workflows +publish the current run's table in the GitHub Actions scorecard summary. The +summary reads the matrix identity from `E2E_TARGET_ID` and `NEMOCLAW_E2E_SHARD` when set. It retains overall start, finish, and duration, and records each declared or harness-owned phase's start, finish, duration, outcome, output-event count, and last-output timestamp. Use several recent diff --git a/test/e2e/live/onboard-resume.test.ts b/test/e2e/live/onboard-resume.test.ts index 559ef251a0..bb4fe761b8 100644 --- a/test/e2e/live/onboard-resume.test.ts +++ b/test/e2e/live/onboard-resume.test.ts @@ -163,7 +163,6 @@ test("onboard-resume: interrupted onboard then --resume can recreate with cached "validate resumed sandbox state and corporate trust", "retry final verification after route repair", "compare implicit resume with fresh onboard", - "record the completed resume contract", ], }, }, async ({ artifacts, cleanup, host, progress, sandbox }) => { @@ -673,6 +672,5 @@ test("onboard-resume: interrupted onboard then --resume can recreate with cached expect(freshRun.exitCode, freshText).not.toBe(0); expect(freshText).toContain("[e2e] Forced onboarding failure at step 'preflight'."); expect(freshText).not.toContain("(resume mode)"); - progress.phase("record the completed resume contract"); await artifacts.target.complete({ id: "onboard-resume", status: "passed" }); }); diff --git a/test/e2e/support/e2e-operations-workflow-boundary.test.ts b/test/e2e/support/e2e-operations-workflow-boundary.test.ts index 4150280665..4a024b04ff 100644 --- a/test/e2e/support/e2e-operations-workflow-boundary.test.ts +++ b/test/e2e/support/e2e-operations-workflow-boundary.test.ts @@ -64,6 +64,21 @@ describe("E2E operations workflow boundary", () => { ); }); + it("pins the scorecard's current-run progress artifact download", () => { + const workflow = readE2eOperationsWorkflow(); + const download = workflow.jobs.scorecard.steps!.find( + (step) => step.name === "Download E2E progress artifacts", + )!; + download.uses = "actions/download-artifact@0000000000000000000000000000000000000000"; + download.with!.pattern = "*"; + + expect(validateE2eOperationsWorkflow(workflow)).toEqual( + expect.arrayContaining([ + "scorecard must download this run's E2E artifacts into the runtime audit directory", + ]), + ); + }); + it("rejects controller protocol and PR validation drift", () => { const workflow = readE2eOperationsWorkflow(); delete workflow.on?.workflow_dispatch?.inputs?.base_sha; @@ -375,8 +390,15 @@ describe("E2E operations workflow boundary", () => { summaryMarkdown: "## 🌅 NemoClaw E2E Scorecard\n\n### Onboard Performance Budget", }), }; + const runtimeAudit = { + auditTestRuntime: vi.fn().mockReturnValue([{ target: "full-e2e" }]), + formatRuntimeAuditSummary: vi + .fn() + .mockReturnValue("## E2E Test Phase Runtime\n\n| Target | Slowest observed phase |"), + }; const runtimeModules = new Map([ ["path", { join: (...parts: string[]) => parts.join("/") }], + ["/workspace/scripts/audit-test-runtime.mts", runtimeAudit], ["/workspace/scripts/scorecard/coordinate-scorecard.mts", coordinator], ["/workspace/scripts/scorecard/analyze-trace-timing.mts", traceTiming], ["/workspace/scripts/scorecard/summarize-jobs.mts", scorecardJobs], @@ -391,6 +413,7 @@ describe("E2E operations workflow boundary", () => { EXPLICIT_ONLY_JOBS: "", GITHUB_WORKSPACE: "/workspace", JOBS: "", + RUNTIME_ARTIFACTS: "/runner/e2e-runtime-audit", TARGETS: "", }, }; @@ -412,6 +435,7 @@ describe("E2E operations workflow boundary", () => { ); expect(traceTiming.buildTraceTimingResult).toHaveBeenCalledWith({ github: {}, context, core }); + expect(runtimeAudit.auditTestRuntime).toHaveBeenCalledWith(["/runner/e2e-runtime-audit"]); expect(warning).toHaveBeenCalledWith("Cloud onboard advisory performance budget exceeded"); expect(coordinator.buildScorecard).toHaveBeenCalledWith( expect.objectContaining({ @@ -427,7 +451,7 @@ describe("E2E operations workflow boundary", () => { }), ); expect(summary.addRaw).toHaveBeenCalledWith( - expect.stringContaining("### Onboard Performance Budget"), + expect.stringMatching(/### Onboard Performance Budget[\s\S]*## E2E Test Phase Runtime/u), ); expect(summary.write).toHaveBeenCalledOnce(); expect(setOutput).toHaveBeenCalledWith("scorecardData", expect.any(String)); diff --git a/test/e2e/support/e2e-scorecard.test.ts b/test/e2e/support/e2e-scorecard.test.ts index ba76a81629..d6c3fe5701 100644 --- a/test/e2e/support/e2e-scorecard.test.ts +++ b/test/e2e/support/e2e-scorecard.test.ts @@ -188,6 +188,8 @@ describe("E2E scorecard", () => { const loaded = require(path.join(process.env.GITHUB_WORKSPACE, 'scripts/scorecard', file)); if (Object.keys(loaded).length === 0) process.exit(2); } + const runtimeAudit = require(path.join(process.env.GITHUB_WORKSPACE, 'scripts/audit-test-runtime.mts')); + if (Object.keys(runtimeAudit).length === 0) process.exit(2); `; const result = spawnSync(process.execPath, ["--experimental-strip-types", "-e", script], { cwd: process.cwd(), diff --git a/test/e2e/support/test-runtime-audit.test.ts b/test/e2e/support/test-runtime-audit.test.ts index a4b5245631..dbec382fbf 100644 --- a/test/e2e/support/test-runtime-audit.test.ts +++ b/test/e2e/support/test-runtime-audit.test.ts @@ -7,7 +7,11 @@ import path from "node:path"; import { describe, expect, it } from "vitest"; -import { auditTestRuntime, formatRuntimeAudit } from "../../../scripts/audit-test-runtime.mts"; +import { + auditTestRuntime, + formatRuntimeAudit, + formatRuntimeAuditSummary, +} from "../../../scripts/audit-test-runtime.mts"; function writeProgress( root: string, @@ -31,6 +35,7 @@ function writeProgress( phases: [ { label: phase, + outcome: phase === "inference" ? "failed" : "passed", startedAtMs: 1_000, finishedAtMs: 1_000 + phaseDurationMs, durationMs: phaseDurationMs, @@ -64,6 +69,7 @@ describe("test runtime audit", () => { variabilityMs: 20_000, slowestPhase: "inference", slowestPhaseMs: 40_000, + slowestPhaseOutcome: "failed", }, { target: "steady test-target", @@ -75,16 +81,24 @@ describe("test runtime audit", () => { variabilityMs: 0, slowestPhase: "sandbox", slowestPhaseMs: 15_000, + slowestPhaseOutcome: "passed", }, ]); expect(formatRuntimeAudit(rows)).toContain( - "| variable test-target | variable test | 2 | 30.0s | 50.0s | 50.0s | 20.0s | inference (40.0s) |", + "| variable test-target | variable test | 2 | 30.0s | 50.0s | 50.0s | 20.0s | inference (40.0s, failed) |", ); + expect(formatRuntimeAuditSummary(rows)).toContain("## E2E Test Phase Runtime"); } finally { fs.rmSync(root, { recursive: true, force: true }); } }); + it("renders an explicit no-artifact summary", () => { + expect(formatRuntimeAuditSummary([])).toContain( + "No `test-progress.json` artifacts were available for this run.", + ); + }); + it("rejects malformed progress artifacts instead of producing a misleading ranking", () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-runtime-audit-invalid-")); try { diff --git a/tools/e2e/operations-workflow-boundary.mts b/tools/e2e/operations-workflow-boundary.mts index 0fbec309f9..36e44bf7b4 100644 --- a/tools/e2e/operations-workflow-boundary.mts +++ b/tools/e2e/operations-workflow-boundary.mts @@ -15,6 +15,8 @@ const META_JOBS = new Set(["report-to-pr", "scorecard"]); const FULL_SHA_ACTION = /^[^\s@]+@[0-9a-f]{40}$/u; const GITHUB_SCRIPT_NODE24_ACTION = "actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3"; +const DOWNLOAD_ARTIFACT_ACTION = + "actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c"; const PR_GATE_REPORTER = "test/e2e/risk-signal-reporter.ts"; const LIVE_VITEST_HELPER = "tools/e2e/live-vitest-invocation.mts run --test-path"; const E2E_ARTIFACT_ACTION = "NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@"; @@ -29,6 +31,7 @@ const GENERIC_ISSUE_GRAPHQL_MUTATION = /github\.graphql\s*\(\s*["'`]\s*mutation\b[\s\S]*?\b(?:addComment|closeIssue|createIssue|reopenIssue|updateIssue)\b/u; type WorkflowStep = { + "continue-on-error"?: boolean; env?: Record; if?: string; name?: string; @@ -420,11 +423,27 @@ function validateScorecard(errors: string[], workflow: OperationsWorkflow): void .map((entry) => entry.trim()) .filter(Boolean); if ( - sparseCheckout.length !== 2 || + sparseCheckout.length !== 3 || !sparseCheckout.includes("ci/onboard-performance-budget.json") || + !sparseCheckout.includes("scripts/audit-test-runtime.mts") || !sparseCheckout.includes("scripts/scorecard") ) { - errors.push("scorecard checkout must be limited to scorecard builders and budget config"); + errors.push( + "scorecard checkout must be limited to runtime and scorecard builders and budget config", + ); + } + + const download = findStep(job, "Download E2E progress artifacts"); + requirePinnedAction(errors, download, "scorecard artifact download"); + if ( + download.uses !== DOWNLOAD_ARTIFACT_ACTION || + download["continue-on-error"] !== true || + download.with?.pattern !== "e2e-*" || + download.with?.path !== "${{ runner.temp }}/e2e-runtime-audit" + ) { + errors.push( + "scorecard must download this run's E2E artifacts into the runtime audit directory", + ); } const generate = findStep(job, "Generate E2E scorecard"); @@ -440,6 +459,9 @@ function validateScorecard(errors: string[], workflow: OperationsWorkflow): void "core.warning(trace.budgetWarningMessage)", "scripts/scorecard/summarize-jobs.mts", "scorecardJobs.loadWorkflowRunJobs", + "scripts/audit-test-runtime.mts", + "runtimeAudit.auditTestRuntime", + "runtimeAudit.formatRuntimeAuditSummary", "core.summary", "scorecardData", "slackData", @@ -452,6 +474,9 @@ function validateScorecard(errors: string[], workflow: OperationsWorkflow): void ) { errors.push("scorecard generator must derive explicit-only jobs from workflow inventory"); } + if (generate.env?.RUNTIME_ARTIFACTS !== "${{ runner.temp }}/e2e-runtime-audit") { + errors.push("scorecard generator must read the downloaded runtime audit directory"); + } const slack = findStep(job, "Post scorecard to Slack"); requirePinnedAction(errors, slack, "scorecard Slack publisher"); From b35b68fa3d34a6a25f949fa56c9c12374aa6f094 Mon Sep 17 00:00:00 2001 From: Charan Jagwani Date: Wed, 22 Jul 2026 10:49:40 -0700 Subject: [PATCH 2/5] test(e2e): tighten runtime scorecard contract Signed-off-by: Charan Jagwani --- .github/workflows/e2e.yaml | 4 ++-- .../e2e-operations-workflow-boundary.test.ts | 22 ++++++++++++++----- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 9147a392c2..041633b6cc 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -5116,8 +5116,6 @@ jobs: // collapses every matrix target into one result and has no job URL. // The typed helper owns and tests the degraded `needs` fallback. const apiJobs = await scorecardJobs.loadWorkflowRunJobs({ github, context, core }); - const trace = await traceTiming.buildTraceTimingResult({ github, context, core }); - if (trace.budgetWarningMessage) core.warning(trace.budgetWarningMessage); let runtimeSummaryMarkdown; try { const runtimeRows = runtimeAudit.auditTestRuntime([process.env.RUNTIME_ARTIFACTS]); @@ -5131,6 +5129,8 @@ jobs: '', ].join('\n'); } + const trace = await traceTiming.buildTraceTimingResult({ github, context, core }); + if (trace.budgetWarningMessage) core.warning(trace.budgetWarningMessage); const { summaryMarkdown, scorecardData, slackData } = buildScorecard({ eventName: context.eventName, diff --git a/test/e2e/support/e2e-operations-workflow-boundary.test.ts b/test/e2e/support/e2e-operations-workflow-boundary.test.ts index 4a024b04ff..3f3a8cc596 100644 --- a/test/e2e/support/e2e-operations-workflow-boundary.test.ts +++ b/test/e2e/support/e2e-operations-workflow-boundary.test.ts @@ -64,18 +64,27 @@ describe("E2E operations workflow boundary", () => { ); }); - it("pins the scorecard's current-run progress artifact download", () => { + it("pins the scorecard's current-run progress artifact action", () => { const workflow = readE2eOperationsWorkflow(); const download = workflow.jobs.scorecard.steps!.find( (step) => step.name === "Download E2E progress artifacts", )!; download.uses = "actions/download-artifact@0000000000000000000000000000000000000000"; + + expect(validateE2eOperationsWorkflow(workflow)).toContain( + "scorecard must download this run's E2E artifacts into the runtime audit directory", + ); + }); + + it("limits the scorecard artifact download to E2E progress sources", () => { + const workflow = readE2eOperationsWorkflow(); + const download = workflow.jobs.scorecard.steps!.find( + (step) => step.name === "Download E2E progress artifacts", + )!; download.with!.pattern = "*"; - expect(validateE2eOperationsWorkflow(workflow)).toEqual( - expect.arrayContaining([ - "scorecard must download this run's E2E artifacts into the runtime audit directory", - ]), + expect(validateE2eOperationsWorkflow(workflow)).toContain( + "scorecard must download this run's E2E artifacts into the runtime audit directory", ); }); @@ -436,6 +445,9 @@ describe("E2E operations workflow boundary", () => { expect(traceTiming.buildTraceTimingResult).toHaveBeenCalledWith({ github: {}, context, core }); expect(runtimeAudit.auditTestRuntime).toHaveBeenCalledWith(["/runner/e2e-runtime-audit"]); + expect(runtimeAudit.auditTestRuntime.mock.invocationCallOrder[0]).toBeLessThan( + traceTiming.buildTraceTimingResult.mock.invocationCallOrder[0], + ); expect(warning).toHaveBeenCalledWith("Cloud onboard advisory performance budget exceeded"); expect(coordinator.buildScorecard).toHaveBeenCalledWith( expect.objectContaining({ From 25a1775ec85d1e8d5749dfe91686572ad5f4a215 Mon Sep 17 00:00:00 2001 From: Charan Jagwani Date: Wed, 22 Jul 2026 11:02:43 -0700 Subject: [PATCH 3/5] test(e2e): cover invalid progress fallback Signed-off-by: Charan Jagwani --- .../e2e-operations-workflow-boundary.test.ts | 90 +++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/test/e2e/support/e2e-operations-workflow-boundary.test.ts b/test/e2e/support/e2e-operations-workflow-boundary.test.ts index 3f3a8cc596..4f2d77d8b1 100644 --- a/test/e2e/support/e2e-operations-workflow-boundary.test.ts +++ b/test/e2e/support/e2e-operations-workflow-boundary.test.ts @@ -470,6 +470,96 @@ describe("E2E operations workflow boundary", () => { expect(setOutput).toHaveBeenCalledWith("slackData", expect.any(String)); }); + it("keeps scorecard outputs available when a progress artifact is invalid", async () => { + const script = workflowScript("scorecard", "Generate E2E scorecard").replace( + "${{ toJSON(needs) }}", + JSON.stringify({ "generate-matrix": { result: "success" } }), + ); + const warning = vi.fn(); + const setOutput = vi.fn(); + const summary = { + addRaw: vi.fn(), + write: vi.fn().mockResolvedValue(undefined), + }; + summary.addRaw.mockReturnValue(summary); + const runtimeAudit = { + auditTestRuntime: vi.fn(() => { + throw new Error("invalid progress artifact"); + }), + formatRuntimeAuditSummary: vi.fn(), + }; + const runtimeModules = new Map([ + ["path", { join: (...parts: string[]) => parts.join("/") }], + ["/workspace/scripts/audit-test-runtime.mts", runtimeAudit], + [ + "/workspace/scripts/scorecard/coordinate-scorecard.mts", + { + buildScorecard: vi.fn().mockReturnValue({ + scorecardData: { ran: 0, runMode: "Scheduled E2E", total: 0 }, + slackData: { channel: "daily", payload: { attachments: [], text: "scorecard" } }, + summaryMarkdown: "## 🌅 NemoClaw E2E Scorecard", + }), + }, + ], + [ + "/workspace/scripts/scorecard/analyze-trace-timing.mts", + { + buildTraceTimingResult: vi.fn().mockResolvedValue({ + budgetWarningMessage: undefined, + traceSummaryLines: [], + traceTimingLine: "Trace: unavailable", + }), + }, + ], + [ + "/workspace/scripts/scorecard/summarize-jobs.mts", + { loadWorkflowRunJobs: vi.fn().mockResolvedValue([]) }, + ], + ]); + const runtimeRequire = (specifier: string) => { + const runtimeModule = runtimeModules.get(specifier); + expect(runtimeModule, `Unexpected scorecard require: ${specifier}`).toBeDefined(); + return runtimeModule; + }; + const processMock = { + env: { + EXPLICIT_ONLY_JOBS: "", + GITHUB_WORKSPACE: "/workspace", + JOBS: "", + RUNTIME_ARTIFACTS: "/runner/e2e-runtime-audit", + TARGETS: "", + }, + }; + const context = { + actor: "scorecard-test", + eventName: "schedule", + repo: { owner: "NVIDIA", repo: "NemoClaw" }, + runId: 123, + serverUrl: "https://github.com", + }; + + await new AsyncFunction("require", "process", "github", "context", "core", script)( + runtimeRequire, + processMock, + {}, + context, + { setOutput, summary, warning }, + ); + + expect(warning).toHaveBeenCalledWith( + "E2E test phase runtime summary unavailable: invalid progress artifact", + ); + expect(runtimeAudit.formatRuntimeAuditSummary).not.toHaveBeenCalled(); + expect(summary.addRaw).toHaveBeenCalledWith( + expect.stringContaining( + "The summary is unavailable because a `test-progress.json` artifact was invalid.", + ), + ); + expect(summary.write).toHaveBeenCalledOnce(); + expect(setOutput).toHaveBeenCalledWith("scorecardData", expect.any(String)); + expect(setOutput).toHaveBeenCalledWith("slackData", expect.any(String)); + }); + it("keeps selective scorecards silent unless Slack posting is explicitly enabled", async () => { const script = workflowScript("scorecard", "Post scorecard to Slack"); const info = vi.fn(); From 867f6735fbc1a3d2b03ebe6e66b0931124b99d2b Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Wed, 22 Jul 2026 15:20:50 -0700 Subject: [PATCH 4/5] test(e2e): reject invalid phase outcomes Signed-off-by: Carlos Villela --- test/e2e/support/test-runtime-audit.test.ts | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/test/e2e/support/test-runtime-audit.test.ts b/test/e2e/support/test-runtime-audit.test.ts index dbec382fbf..b0c113bd76 100644 --- a/test/e2e/support/test-runtime-audit.test.ts +++ b/test/e2e/support/test-runtime-audit.test.ts @@ -20,6 +20,7 @@ function writeProgress( durationMs: number, phase: string, phaseDurationMs: number, + phaseOutcome: string | undefined = phase === "inference" ? "failed" : "passed", ): void { const directory = path.join(root, run, scenario); fs.mkdirSync(directory, { recursive: true }); @@ -35,7 +36,7 @@ function writeProgress( phases: [ { label: phase, - outcome: phase === "inference" ? "failed" : "passed", + ...(phaseOutcome === undefined ? {} : { outcome: phaseOutcome }), startedAtMs: 1_000, finishedAtMs: 1_000 + phaseDurationMs, durationMs: phaseDurationMs, @@ -108,4 +109,17 @@ describe("test runtime audit", () => { fs.rmSync(root, { recursive: true, force: true }); } }); + + it.each([ + ["missing", undefined], + ["unknown", "unexpected"], + ])("rejects a valid progress artifact with a %s phase outcome", (_case, outcome) => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-runtime-audit-outcome-")); + try { + writeProgress(root, "run-1", "invalid outcome", 10_000, "install", 8_000, outcome); + expect(() => auditTestRuntime([root])).toThrow(/invalid test progress summary/); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } + }); }); From 5d9be88955c0e206f527bcdae3e23171640ed38b Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Wed, 22 Jul 2026 15:48:47 -0700 Subject: [PATCH 5/5] test(e2e): exercise missing phase outcomes Signed-off-by: Carlos Villela --- test/e2e/support/test-runtime-audit.test.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/e2e/support/test-runtime-audit.test.ts b/test/e2e/support/test-runtime-audit.test.ts index b0c113bd76..b03e40fa6a 100644 --- a/test/e2e/support/test-runtime-audit.test.ts +++ b/test/e2e/support/test-runtime-audit.test.ts @@ -20,7 +20,7 @@ function writeProgress( durationMs: number, phase: string, phaseDurationMs: number, - phaseOutcome: string | undefined = phase === "inference" ? "failed" : "passed", + phaseOutcome: string | undefined, ): void { const directory = path.join(root, run, scenario); fs.mkdirSync(directory, { recursive: true }); @@ -53,9 +53,9 @@ describe("test runtime audit", () => { it("ranks repeated artifact summaries by p95 and identifies the slowest phase", () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-runtime-audit-")); try { - writeProgress(root, "run-1", "variable test", 10_000, "install", 8_000); - writeProgress(root, "run-2", "variable test", 50_000, "inference", 40_000); - writeProgress(root, "run-1", "steady test", 20_000, "sandbox", 15_000); + writeProgress(root, "run-1", "variable test", 10_000, "install", 8_000, "passed"); + writeProgress(root, "run-2", "variable test", 50_000, "inference", 40_000, "failed"); + writeProgress(root, "run-1", "steady test", 20_000, "sandbox", 15_000, "passed"); const rows = auditTestRuntime([root]);