ci(e2e): publish phase runtime summary

.github/workflows/e2e.yaml

@@ -5130,15 +5130,24 @@ jobs:
           persist-credentials: false
           sparse-checkout: |
             ci/onboard-performance-budget.json
+            scripts/audit-test-runtime.mts
             scripts/scorecard
           sparse-checkout-cone-mode: false
 
+      - name: Download E2E progress artifacts
+        continue-on-error: true
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          path: ${{ runner.temp }}/e2e-runtime-audit
+          pattern: e2e-*
+
       - name: Generate E2E scorecard
         id: scorecard
         uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
         env:
           EXPLICIT_ONLY_JOBS: ${{ needs.generate-matrix.outputs.explicit_only_jobs }}
           JOBS: ${{ inputs.jobs }}
+          RUNTIME_ARTIFACTS: ${{ runner.temp }}/e2e-runtime-audit
           TARGETS: ${{ inputs.targets }}
         with:
           script: |
@@ -5152,12 +5161,28 @@ jobs:
             const traceTiming = require(
               path.join(process.env.GITHUB_WORKSPACE, 'scripts/scorecard/analyze-trace-timing.mts'),
             );
+            const runtimeAudit = require(
+              path.join(process.env.GITHUB_WORKSPACE, 'scripts/audit-test-runtime.mts'),
+            );
             const needs = ${{ toJSON(needs) }};
 
             // GitHub's jobs API is the canonical source because `needs.live`
             // collapses every matrix target into one result and has no job URL.
             // The typed helper owns and tests the degraded `needs` fallback.
             const apiJobs = await scorecardJobs.loadWorkflowRunJobs({ github, context, core });
+            let runtimeSummaryMarkdown;
+            try {
+              const runtimeRows = runtimeAudit.auditTestRuntime([process.env.RUNTIME_ARTIFACTS]);
+              runtimeSummaryMarkdown = runtimeAudit.formatRuntimeAuditSummary(runtimeRows);
+            } catch {
+              core.warning('E2E test phase runtime summary unavailable: invalid progress artifact');
+              runtimeSummaryMarkdown = [
+                '## E2E Test Phase Runtime',
+                '',
+                'The summary is unavailable because a `test-progress.json` artifact was invalid.',
+                '',
+              ].join('\n');
+            }
             const trace = await traceTiming.buildTraceTimingResult({ github, context, core });
             if (trace.budgetWarningMessage) core.warning(trace.budgetWarningMessage);
 
@@ -5176,7 +5201,7 @@ jobs:
               today: new Date().toLocaleDateString('en-US', { month: 'short', day: 'numeric' }),
             });
 
-            await core.summary.addRaw(summaryMarkdown).write();
+            await core.summary.addRaw(`${summaryMarkdown}\n\n${runtimeSummaryMarkdown}`).write();
             core.setOutput('scorecardData', JSON.stringify(scorecardData));
             core.setOutput('slackData', JSON.stringify(slackData));
 

scripts/audit-test-runtime.mts

@@ -5,7 +5,7 @@ import fs from "node:fs";
 import path from "node:path";
 import { pathToFileURL } from "node:url";
 
-import type { ProgressSummary } from "../test/e2e/fixtures/progress.ts";
+import type { ProgressPhase, ProgressSummary } from "../test/e2e/fixtures/progress.ts";
 
 export interface RuntimeAuditRow {
   target: string;
@@ -17,6 +17,7 @@ export interface RuntimeAuditRow {
   variabilityMs: number;
   slowestPhase: string;
   slowestPhaseMs: number;
+  slowestPhaseOutcome: "passed" | "failed" | "skipped";
 }
 
 function isProgressSummary(value: unknown): value is ProgressSummary {
@@ -36,6 +37,7 @@ function isProgressSummary(value: unknown): value is ProgressSummary {
       (phase) =>
         phase &&
         typeof phase.label === "string" &&
+        (phase.outcome === "passed" || phase.outcome === "failed" || phase.outcome === "skipped") &&
         typeof phase.durationMs === "number" &&
         Number.isFinite(phase.durationMs) &&
         phase.durationMs >= 0,
@@ -100,9 +102,9 @@ export function auditTestRuntime(roots: readonly string[]): RuntimeAuditRow[] {
       if (!first) throw new Error("runtime audit group is unexpectedly empty");
       const durations = runs.map((run) => run.durationMs as number).sort((a, b) => a - b);
       const phases = runs.flatMap((run) => run.phases);
-      const slowestPhase = phases.reduce(
+      const slowestPhase = phases.reduce<Pick<ProgressPhase, "label" | "durationMs" | "outcome">>(
         (slowest, phase) => (phase.durationMs > slowest.durationMs ? phase : slowest),
-        { label: "n/a", durationMs: 0 },
+        { label: "n/a", durationMs: 0, outcome: "skipped" as const },
       );
       const medianMs = median(durations);
       const p95Ms = percentile(durations, 0.95);
@@ -116,6 +118,7 @@ export function auditTestRuntime(roots: readonly string[]): RuntimeAuditRow[] {
         variabilityMs: Math.max(0, p95Ms - medianMs),
         slowestPhase: slowestPhase.label,
         slowestPhaseMs: slowestPhase.durationMs,
+        slowestPhaseOutcome: slowestPhase.outcome,
       };
     })
     .sort((a, b) => b.p95Ms - a.p95Ms || b.variabilityMs - a.variabilityMs);
@@ -132,12 +135,22 @@ export function formatRuntimeAudit(rows: readonly RuntimeAuditRow[]): string {
   ];
   for (const row of rows) {
     lines.push(
-      `| ${row.target.replaceAll("|", "\\|")} | ${row.scenario.replaceAll("|", "\\|")} | ${row.runs} | ${seconds(row.medianMs)}s | ${seconds(row.p95Ms)}s | ${seconds(row.maxMs)}s | ${seconds(row.variabilityMs)}s | ${row.slowestPhase.replaceAll("|", "\\|")} (${seconds(row.slowestPhaseMs)}s) |`,
+      `| ${row.target.replaceAll("|", "\\|")} | ${row.scenario.replaceAll("|", "\\|")} | ${row.runs} | ${seconds(row.medianMs)}s | ${seconds(row.p95Ms)}s | ${seconds(row.maxMs)}s | ${seconds(row.variabilityMs)}s | ${row.slowestPhase.replaceAll("|", "\\|")} (${seconds(row.slowestPhaseMs)}s, ${row.slowestPhaseOutcome}) |`,
     );
   }
   return `${lines.join("\n")}\n`;
 }
 
+export function formatRuntimeAuditSummary(rows: readonly RuntimeAuditRow[]): string {
+  const lines = ["## E2E Test Phase Runtime", "", "This run's semantic phase timing summary.", ""];
+  if (rows.length === 0) {
+    lines.push("No `test-progress.json` artifacts were available for this run.");
+  } else {
+    lines.push(formatRuntimeAudit(rows).trimEnd());
+  }
+  return `${lines.join("\n")}\n`;
+}
+
 function main(argv: readonly string[]): void {
   const roots = argv.length > 0 ? argv : [".e2e/live"];
   const rows = auditTestRuntime(roots);

test/e2e/README.md

@@ -62,9 +62,10 @@ graph as the live targets:
   retired. Any future issue escalation should use a separately reviewed
   exceptional threshold, such as the same lane failing twice consecutively or
   remaining broken for 24 hours, rather than posting on every failed schedule.
-- `scorecard` writes the scheduled/manual result summary, compares the trusted
-  cloud-onboard timing summary with the latest prior-release `e2e.yaml` run,
-  and posts to the daily or full-run Slack route.
+- `scorecard` writes the scheduled/manual result summary, adds this run's
+  semantic phase runtime table, compares the trusted cloud-onboard timing
+  summary with the latest prior-release `e2e.yaml` run, and posts to the daily
+  or full-run Slack route.
 - Selective dispatches remain silent unless they run on `main` with
   `post_to_slack=true`, which uses the preview Slack route. Branch-dispatched
   runs never receive Slack webhook secrets.
@@ -120,7 +121,9 @@ npm run test:runtime-audit -- path/to/run-1 path/to/run-2
 ```
 
 The audit groups each test by target and optional shard, ranks the groups by
-p95 runtime, and reports variability and the slowest observed phase. Keep phase
+p95 runtime, and reports variability plus the slowest observed phase's duration
+and outcome. Scheduled and ordinary manual runs include the same table for that
+run in the GitHub Actions scorecard summary. Keep phase
 labels specific to test behavior, call `progress.phase("literal phase label")`
 at the declared boundaries in order, and transition through the final
 test-declared phase on every passing path. The fixture rejects a passing live

test/e2e/docs/README.md

@@ -101,7 +101,9 @@ During fixture teardown, every passing or failing live test writes
 `test-progress.json` beside its other target artifacts. The runtime audit
 groups those files by target, optional shard, and test name, then reports
 median, p95, maximum, p95-minus-median variability, and the slowest observed
-phase. The summary reads the matrix identity from `E2E_TARGET_ID` and
+phase with its duration and outcome. Scheduled and ordinary manual workflows
+publish the current run's table in the GitHub Actions scorecard summary. The
+summary reads the matrix identity from `E2E_TARGET_ID` and
 `NEMOCLAW_E2E_SHARD` when set. It retains overall start, finish, and duration,
 and records each declared or harness-owned phase's start, finish, duration,
 outcome, output-event count, and last-output timestamp. Use several recent

test/e2e/live/onboard-resume.test.ts

@@ -163,7 +163,6 @@ test("onboard-resume: interrupted onboard then --resume can recreate with cached
       "validate resumed sandbox state and corporate trust",
       "retry final verification after route repair",
       "compare implicit resume with fresh onboard",
-      "record the completed resume contract",
     ],
   },
 }, async ({ artifacts, cleanup, host, progress, sandbox }) => {
@@ -673,6 +672,5 @@ test("onboard-resume: interrupted onboard then --resume can recreate with cached
   expect(freshRun.exitCode, freshText).not.toBe(0);
   expect(freshText).toContain("[e2e] Forced onboarding failure at step 'preflight'.");
   expect(freshText).not.toContain("(resume mode)");
-  progress.phase("record the completed resume contract");
   await artifacts.target.complete({ id: "onboard-resume", status: "passed" });
 });

test/e2e/support/e2e-operations-workflow-boundary.test.ts

@@ -64,6 +64,30 @@ describe("E2E operations workflow boundary", () => {
     );
   });
 
+  it("pins the scorecard's current-run progress artifact action", () => {
+    const workflow = readE2eOperationsWorkflow();
+    const download = workflow.jobs.scorecard.steps!.find(
+      (step) => step.name === "Download E2E progress artifacts",
+    )!;
+    download.uses = "actions/download-artifact@0000000000000000000000000000000000000000";
+
+    expect(validateE2eOperationsWorkflow(workflow)).toContain(
+      "scorecard must download this run's E2E artifacts into the runtime audit directory",
+    );
+  });
+
+  it("limits the scorecard artifact download to E2E progress sources", () => {
+    const workflow = readE2eOperationsWorkflow();
+    const download = workflow.jobs.scorecard.steps!.find(
+      (step) => step.name === "Download E2E progress artifacts",
+    )!;
+    download.with!.pattern = "*";
+
+    expect(validateE2eOperationsWorkflow(workflow)).toContain(
+      "scorecard must download this run's E2E artifacts into the runtime audit directory",
+    );
+  });
+
   it("rejects controller protocol and PR validation drift", () => {
     const workflow = readE2eOperationsWorkflow();
     delete workflow.on?.workflow_dispatch?.inputs?.base_sha;
@@ -375,8 +399,15 @@ describe("E2E operations workflow boundary", () => {
         summaryMarkdown: "## 🌅 NemoClaw E2E Scorecard\n\n### Onboard Performance Budget",
       }),
     };
+    const runtimeAudit = {
+      auditTestRuntime: vi.fn().mockReturnValue([{ target: "full-e2e" }]),
+      formatRuntimeAuditSummary: vi
+        .fn()
+        .mockReturnValue("## E2E Test Phase Runtime\n\n| Target | Slowest observed phase |"),
+    };
     const runtimeModules = new Map<string, unknown>([
       ["path", { join: (...parts: string[]) => parts.join("/") }],
+      ["/workspace/scripts/audit-test-runtime.mts", runtimeAudit],
       ["/workspace/scripts/scorecard/coordinate-scorecard.mts", coordinator],
       ["/workspace/scripts/scorecard/analyze-trace-timing.mts", traceTiming],
       ["/workspace/scripts/scorecard/summarize-jobs.mts", scorecardJobs],
@@ -391,6 +422,7 @@ describe("E2E operations workflow boundary", () => {
         EXPLICIT_ONLY_JOBS: "",
         GITHUB_WORKSPACE: "/workspace",
         JOBS: "",
+        RUNTIME_ARTIFACTS: "/runner/e2e-runtime-audit",
         TARGETS: "",
       },
     };
@@ -412,6 +444,10 @@ describe("E2E operations workflow boundary", () => {
     );
 
     expect(traceTiming.buildTraceTimingResult).toHaveBeenCalledWith({ github: {}, context, core });
+    expect(runtimeAudit.auditTestRuntime).toHaveBeenCalledWith(["/runner/e2e-runtime-audit"]);
+    expect(runtimeAudit.auditTestRuntime.mock.invocationCallOrder[0]).toBeLessThan(
+      traceTiming.buildTraceTimingResult.mock.invocationCallOrder[0],
+    );
     expect(warning).toHaveBeenCalledWith("Cloud onboard advisory performance budget exceeded");
     expect(coordinator.buildScorecard).toHaveBeenCalledWith(
       expect.objectContaining({
@@ -427,7 +463,97 @@ describe("E2E operations workflow boundary", () => {
       }),
     );
     expect(summary.addRaw).toHaveBeenCalledWith(
-      expect.stringContaining("### Onboard Performance Budget"),
+      expect.stringMatching(/### Onboard Performance Budget[\s\S]*## E2E Test Phase Runtime/u),
+    );
+    expect(summary.write).toHaveBeenCalledOnce();
+    expect(setOutput).toHaveBeenCalledWith("scorecardData", expect.any(String));
+    expect(setOutput).toHaveBeenCalledWith("slackData", expect.any(String));
+  });
+
+  it("keeps scorecard outputs available when a progress artifact is invalid", async () => {
+    const script = workflowScript("scorecard", "Generate E2E scorecard").replace(
+      "${{ toJSON(needs) }}",
+      JSON.stringify({ "generate-matrix": { result: "success" } }),
+    );
+    const warning = vi.fn();
+    const setOutput = vi.fn();
+    const summary = {
+      addRaw: vi.fn(),
+      write: vi.fn().mockResolvedValue(undefined),
+    };
+    summary.addRaw.mockReturnValue(summary);
+    const runtimeAudit = {
+      auditTestRuntime: vi.fn(() => {
+        throw new Error("invalid progress artifact");
+      }),
+      formatRuntimeAuditSummary: vi.fn(),
+    };
+    const runtimeModules = new Map<string, unknown>([
+      ["path", { join: (...parts: string[]) => parts.join("/") }],
+      ["/workspace/scripts/audit-test-runtime.mts", runtimeAudit],
+      [
+        "/workspace/scripts/scorecard/coordinate-scorecard.mts",
+        {
+          buildScorecard: vi.fn().mockReturnValue({
+            scorecardData: { ran: 0, runMode: "Scheduled E2E", total: 0 },
+            slackData: { channel: "daily", payload: { attachments: [], text: "scorecard" } },
+            summaryMarkdown: "## 🌅 NemoClaw E2E Scorecard",
+          }),
+        },
+      ],
+      [
+        "/workspace/scripts/scorecard/analyze-trace-timing.mts",
+        {
+          buildTraceTimingResult: vi.fn().mockResolvedValue({
+            budgetWarningMessage: undefined,
+            traceSummaryLines: [],
+            traceTimingLine: "Trace: unavailable",
+          }),
+        },
+      ],
+      [
+        "/workspace/scripts/scorecard/summarize-jobs.mts",
+        { loadWorkflowRunJobs: vi.fn().mockResolvedValue([]) },
+      ],
+    ]);
+    const runtimeRequire = (specifier: string) => {
+      const runtimeModule = runtimeModules.get(specifier);
+      expect(runtimeModule, `Unexpected scorecard require: ${specifier}`).toBeDefined();
+      return runtimeModule;
+    };
+    const processMock = {
+      env: {
+        EXPLICIT_ONLY_JOBS: "",
+        GITHUB_WORKSPACE: "/workspace",
+        JOBS: "",
+        RUNTIME_ARTIFACTS: "/runner/e2e-runtime-audit",
+        TARGETS: "",
+      },
+    };
+    const context = {
+      actor: "scorecard-test",
+      eventName: "schedule",
+      repo: { owner: "NVIDIA", repo: "NemoClaw" },
+      runId: 123,
+      serverUrl: "https://github.com",
+    };
+
+    await new AsyncFunction("require", "process", "github", "context", "core", script)(
+      runtimeRequire,
+      processMock,
+      {},
+      context,
+      { setOutput, summary, warning },
+    );
+
+    expect(warning).toHaveBeenCalledWith(
+      "E2E test phase runtime summary unavailable: invalid progress artifact",
+    );
+    expect(runtimeAudit.formatRuntimeAuditSummary).not.toHaveBeenCalled();
+    expect(summary.addRaw).toHaveBeenCalledWith(
+      expect.stringContaining(
+        "The summary is unavailable because a `test-progress.json` artifact was invalid.",
+      ),
     );
     expect(summary.write).toHaveBeenCalledOnce();
     expect(setOutput).toHaveBeenCalledWith("scorecardData", expect.any(String));

test/e2e/support/e2e-scorecard.test.ts

@@ -188,6 +188,8 @@ describe("E2E scorecard", () => {
         const loaded = require(path.join(process.env.GITHUB_WORKSPACE, 'scripts/scorecard', file));
         if (Object.keys(loaded).length === 0) process.exit(2);
       }
+      const runtimeAudit = require(path.join(process.env.GITHUB_WORKSPACE, 'scripts/audit-test-runtime.mts'));
+      if (Object.keys(runtimeAudit).length === 0) process.exit(2);
     `;
     const result = spawnSync(process.execPath, ["--experimental-strip-types", "-e", script], {
       cwd: process.cwd(),