perf(e2e): parallelize plugin EXDEV coverage

.github/workflows/e2e.yaml

@@ -356,6 +356,7 @@ jobs:
             e2e-artifacts/live/${{ matrix.id }}/run-plan.json
             e2e-artifacts/live/${{ matrix.id }}/target.json
             e2e-artifacts/live/${{ matrix.id }}/target-result.json
+            e2e-artifacts/live/${{ matrix.id }}/test-progress.json
             e2e-artifacts/live/${{ matrix.id }}/environment.result.json
             e2e-artifacts/live/${{ matrix.id }}/onboarding.result.json
             e2e-artifacts/live/${{ matrix.id }}/state-validation.result.json
@@ -3875,18 +3876,83 @@ jobs:
         shell: bash
         run: bash .github/scripts/docker-auth-cleanup.sh
 
-  # Scheduled coverage for #6108 / #3513 / #3127. This exercises the complete
-  # managed v0.0.71 runtime with a custom plugin, then proves restart/rebuild
-  # persistence and target-side runtime-dependency replacement across devices.
+  # Scheduled release-baseline coverage for #6108. This runs independently of
+  # the current lifecycle lane so the exact v0.0.71 build does not extend that
+  # lane's critical path.
+  openclaw-plugin-runtime-exdev-release:
+    needs: generate-matrix
+    if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',openclaw-plugin-runtime-exdev-release,') || contains(format(',{0},', inputs.targets), ',openclaw-plugin-runtime-exdev-release,') }}
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    timeout-minutes: 55
+    env:
+      E2E_JOB: "1"
+      E2E_TARGET_ID: "openclaw-plugin-runtime-exdev-release"
+      E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/openclaw-plugin-runtime-exdev-release
+      NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js
+      NEMOCLAW_RUN_LIVE_E2E: "1"
+      NEMOCLAW_NON_INTERACTIVE: "1"
+      NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1"
+      NEMOCLAW_SANDBOX_NAME: "e2e-openclaw-plugin-exdev-release"
+      OPENSHELL_GATEWAY: "nemoclaw"
+    steps:
+      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+        with:
+          ref: ${{ inputs.checkout_sha || github.sha }}
+          persist-credentials: false
+
+      - *dockerhub-auth
+
+      - name: Pre-pull release-matched Docker Hub builder image
+        shell: bash
+        run: |
+          set -euo pipefail
+          docker pull node:22-trixie-slim@sha256:2d9f5c76c8f4dd36e8f253bee5d828a83a6c09f36188f0b0414325232e0b175d
+
+      - name: Remove Docker auth before release-pinned fixture
+        if: always()
+        shell: bash
+        run: |
+          set -euo pipefail
+          bash .github/scripts/docker-auth-cleanup.sh
+
+      - name: Prepare E2E workspace
+        uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@50281ee84c4a6fc759da95ea28fc0b7d9c378a28
+
+      - name: Run OpenClaw custom-plugin release baseline live test
+        run: |
+          set -euo pipefail
+          test -n "${DOCKER_CONFIG:-}"
+          test ! -e "${DOCKER_CONFIG}"
+          test -z "${DOCKERHUB_USERNAME:-}"
+          test -z "${DOCKERHUB_TOKEN:-}"
+          env -u DOCKER_CONFIG -u DOCKERHUB_USERNAME -u DOCKERHUB_TOKEN \
+            npx tsx tools/e2e/live-vitest-invocation.mts run \
+              --test-path test/e2e/live/openclaw-plugin-runtime-exdev.test.ts \
+              --selector release-baseline
+
+      - name: Upload OpenClaw plugin release baseline artifacts
+        if: always()
+        uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57
+
+      - name: Clean up Docker auth
+        if: always()
+        shell: bash
+        run: bash .github/scripts/docker-auth-cleanup.sh
+
+  # Scheduled current-lifecycle coverage for #6108 / #3513 / #3127. This
+  # proves restart/rebuild persistence and target-side runtime-dependency
+  # replacement across devices without waiting for the release baseline.
   openclaw-plugin-runtime-exdev:
     needs: generate-matrix
     if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',openclaw-plugin-runtime-exdev,') || contains(format(',{0},', inputs.targets), ',openclaw-plugin-runtime-exdev,') }}
     runs-on: ubuntu-latest
     permissions:
       contents: read
-    # Three bounded 25-minute onboards plus the 20-minute rebuild and 15-minute
-    # Vitest buffer need 110 minutes; allow 20 more for setup and teardown.
-    timeout-minutes: 130
+    # Two bounded 25-minute onboards plus the 20-minute rebuild and 15-minute
+    # Vitest buffer need 85 minutes; allow 20 more for setup and teardown.
+    timeout-minutes: 105
     env:
       E2E_JOB: "1"
       E2E_TARGET_ID: "openclaw-plugin-runtime-exdev"
@@ -3929,7 +3995,9 @@ jobs:
           test -z "${DOCKERHUB_USERNAME:-}"
           test -z "${DOCKERHUB_TOKEN:-}"
           env -u DOCKER_CONFIG -u DOCKERHUB_USERNAME -u DOCKERHUB_TOKEN \
-            npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/openclaw-plugin-runtime-exdev.test.ts
+            npx tsx tools/e2e/live-vitest-invocation.mts run \
+              --test-path test/e2e/live/openclaw-plugin-runtime-exdev.test.ts \
+              --selector current-lifecycle
 
       - name: Upload OpenClaw plugin runtime-deps EXDEV artifacts
         if: always()
@@ -4913,6 +4981,7 @@ jobs:
         sandbox-survival,
         diagnostics,
         snapshot-commands,
+        openclaw-plugin-runtime-exdev-release,
         openclaw-plugin-runtime-exdev,
         openclaw-tui-chat-correlation,
         gateway-guard-recovery,

.github/workflows/regression-e2e.yaml

@@ -288,11 +288,63 @@ jobs:
             /tmp/nemoclaw-e2e-model-router-response.log
           if-no-files-found: ignore
 
-  # ── OpenClaw custom-plugin lifecycle and runtime-deps EXDEV E2E ─
-  # Coverage guard for #6108 / #3513 / #3127. On Ubuntu/OpenShell sandbox layouts
-  # where /tmp and /sandbox can live on different filesystems, OpenClaw's
-  # first CLI bootstrap must not fail plugin runtime dependency installation
-  # with EXDEV cross-device rename errors.
+  # ── OpenClaw release-baseline custom-plugin E2E ────────────────
+  # The exact v0.0.71 baseline runs in parallel with the current lifecycle
+  # contract so release provenance does not extend the EXDEV critical path.
+  openclaw-plugin-runtime-exdev-release-e2e:
+    needs: select_regression_jobs
+    if: >-
+      github.repository == 'NVIDIA/NemoClaw' &&
+      needs.select_regression_jobs.outputs.openclaw_plugin_runtime_exdev == 'true'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    timeout-minutes: 55
+    steps:
+      - name: Checkout
+        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+        with:
+          persist-credentials: false
+
+      - name: Setup Node
+        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0
+        with:
+          node-version: "22"
+          cache: npm
+
+      - name: Install root dependencies
+        run: npm ci --ignore-scripts
+
+      - name: Build CLI
+        run: npm run build:cli
+
+      - name: Run OpenClaw custom-plugin release baseline Vitest test
+        env:
+          E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/openclaw-plugin-runtime-exdev-release
+          E2E_TARGET_ID: openclaw-plugin-runtime-exdev-release
+          NEMOCLAW_RUN_LIVE_E2E: "1"
+          NEMOCLAW_SANDBOX_NAME: e2e-openclaw-plugin-exdev-release
+        run: |
+          set -euo pipefail
+          npx vitest run --project e2e-live \
+            test/e2e/live/openclaw-plugin-runtime-exdev.test.ts \
+            -t release-baseline \
+            --silent=false --reporter=default
+
+      - name: Upload OpenClaw plugin release baseline artifacts
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: openclaw-plugin-runtime-exdev-release-artifacts
+          path: e2e-artifacts/live/openclaw-plugin-runtime-exdev-release/
+          include-hidden-files: false
+          if-no-files-found: ignore
+          retention-days: 14
+
+  # ── OpenClaw current lifecycle and runtime-deps EXDEV E2E ─────
+  # Coverage guard for #6108 / #3513 / #3127. On Ubuntu/OpenShell sandbox
+  # layouts where /tmp and /sandbox can live on different filesystems, the
+  # runtime dependency replacement must complete without EXDEV failures.
   openclaw-plugin-runtime-exdev-e2e:
     needs: select_regression_jobs
     if: >-
@@ -301,9 +353,9 @@ jobs:
     runs-on: ubuntu-latest
     permissions:
       contents: read
-    # Three bounded 25-minute onboards plus the 20-minute rebuild and 15-minute
-    # Vitest buffer need 110 minutes; allow 20 more for setup and teardown.
-    timeout-minutes: 130
+    # Two bounded 25-minute onboards plus the 20-minute rebuild and 15-minute
+    # Vitest buffer need 85 minutes; allow 20 more for setup and teardown.
+    timeout-minutes: 105
     steps:
       - name: Checkout
         uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
@@ -325,11 +377,13 @@ jobs:
       - name: Run OpenClaw custom-plugin lifecycle and runtime-deps EXDEV Vitest test
         env:
           E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/openclaw-plugin-runtime-exdev
+          E2E_TARGET_ID: openclaw-plugin-runtime-exdev
           NEMOCLAW_RUN_LIVE_E2E: "1"
         run: |
           set -euo pipefail
           npx vitest run --project e2e-live \
             test/e2e/live/openclaw-plugin-runtime-exdev.test.ts \
+            -t current-lifecycle \
             --silent=false --reporter=default
 
       - name: Upload OpenClaw plugin runtime-deps EXDEV artifacts

CONTRIBUTING.md

@@ -206,6 +206,7 @@ These are the primary npm scripts for day-to-day development:
 | `npm run test:watch` | Watch the CLI, plugin, and E2E-support projects and rerun affected tests |
 | `npm run test:shuffle` | Shuffle test order in the focused source projects without collecting coverage |
 | `npm run test:diagnose:leaks` | Report async-resource leaks and diagnose a Vitest process that hangs during shutdown |
+| `npm run test:runtime-audit -- <artifact-dir> [...]` | Rank captured live E2E runs by median, p95, variability, and slowest phase |
 | `npm run test:integration` | Clean-build the CLI and run root integration and installer tests |
 | `npm run test:package` | Clean-build CLI/plugin artifacts and run compiled-package contracts |
 | `npm run test:live-e2e` | Opt into live E2E scenarios (mutates real external state) |

package.json

@@ -30,6 +30,7 @@
     "test:watch": "vitest watch --project cli --project plugin --project e2e-support",
     "test:shuffle": "vitest run --project cli --project plugin --project e2e-support --sequence.shuffle.tests --coverage=false",
     "test:diagnose:leaks": "vitest run --project cli --project plugin --project e2e-support --detectAsyncLeaks --coverage=false --reporter=default --reporter=hanging-process",
+    "test:runtime-audit": "tsx scripts/audit-test-runtime.mts",
     "test:integration": "npm run clean:cli && npm run build:cli && vitest run --project integration --project installer-integration",
     "test:package": "npm run clean:cli && npm --prefix nemoclaw run clean && npm run build:cli && npm --prefix nemoclaw run build && vitest run --project package-contract",
     "test:coverage:cli": "npm run clean:cli && npm run build:cli && tsx scripts/check-dist-sourcemaps.mts dist && vitest run --project cli --project integration --coverage --coverage.reporter=text-summary --coverage.reporter=json-summary --coverage.reportsDirectory=coverage/cli --coverage.include=\"bin/**/*.js\" --coverage.include=\"src/**/*.ts\" --coverage.exclude=\"test/**/*.js\" --coverage.exclude=\"test/**/*.ts\" && tsx scripts/check-coverage-ratchet.mts coverage/cli/coverage-summary.json ci/coverage-threshold-cli.json \"CLI coverage\"",

scripts/audit-test-runtime.mts

@@ -0,0 +1,152 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import fs from "node:fs";
+import path from "node:path";
+import { pathToFileURL } from "node:url";
+
+import type { ProgressSummary } from "../test/e2e/fixtures/progress.ts";
+
+export interface RuntimeAuditRow {
+  target: string;
+  scenario: string;
+  runs: number;
+  medianMs: number;
+  p95Ms: number;
+  maxMs: number;
+  variabilityMs: number;
+  slowestPhase: string;
+  slowestPhaseMs: number;
+}
+
+function isProgressSummary(value: unknown): value is ProgressSummary {
+  if (!value || typeof value !== "object" || Array.isArray(value)) return false;
+  const summary = value as Partial<ProgressSummary>;
+  return (
+    summary.version === 1 &&
+    typeof summary.scenario === "string" &&
+    summary.scenario.length > 0 &&
+    (summary.targetId === undefined || typeof summary.targetId === "string") &&
+    (summary.shardId === undefined || typeof summary.shardId === "string") &&
+    typeof summary.durationMs === "number" &&
+    Number.isFinite(summary.durationMs) &&
+    summary.durationMs >= 0 &&
+    Array.isArray(summary.phases) &&
+    summary.phases.every(
+      (phase) =>
+        phase &&
+        typeof phase.label === "string" &&
+        typeof phase.durationMs === "number" &&
+        Number.isFinite(phase.durationMs) &&
+        phase.durationMs >= 0,
+    )
+  );
+}
+
+function progressFiles(root: string): string[] {
+  const result: string[] = [];
+  const pending = [path.resolve(root)];
+  while (pending.length > 0) {
+    const current = pending.pop();
+    if (!current || !fs.existsSync(current)) continue;
+    const stat = fs.lstatSync(current);
+    if (stat.isSymbolicLink()) continue;
+    if (stat.isFile()) {
+      if (path.basename(current) === "test-progress.json") result.push(current);
+      continue;
+    }
+    if (!stat.isDirectory()) continue;
+    for (const entry of fs.readdirSync(current, { withFileTypes: true })) {
+      if (!entry.isSymbolicLink()) pending.push(path.join(current, entry.name));
+    }
+  }
+  return result.sort();
+}
+
+function percentile(sorted: readonly number[], fraction: number): number {
+  const index = Math.max(0, Math.ceil(sorted.length * fraction) - 1);
+  return sorted[index] ?? 0;
+}
+
+function median(sorted: readonly number[]): number {
+  const middle = Math.floor(sorted.length / 2);
+  if (sorted.length % 2 === 0) {
+    return ((sorted[middle - 1] ?? 0) + (sorted[middle] ?? 0)) / 2;
+  }
+  return sorted[middle] ?? 0;
+}
+
+export function auditTestRuntime(roots: readonly string[]): RuntimeAuditRow[] {
+  const summaries = roots.flatMap(progressFiles).map((file) => {
+    const parsed: unknown = JSON.parse(fs.readFileSync(file, "utf8"));
+    if (!isProgressSummary(parsed)) throw new Error(`${file}: invalid test progress summary`);
+    return parsed;
+  });
+  const grouped = new Map<string, ProgressSummary[]>();
+  for (const summary of summaries) {
+    const key = JSON.stringify([
+      summary.targetId ?? "unlabeled",
+      summary.shardId,
+      summary.scenario,
+    ]);
+    const group = grouped.get(key) ?? [];
+    group.push(summary);
+    grouped.set(key, group);
+  }
+
+  return [...grouped.entries()]
+    .map(([, runs]): RuntimeAuditRow => {
+      const first = runs[0];
+      if (!first) throw new Error("runtime audit group is unexpectedly empty");
+      const durations = runs.map((run) => run.durationMs as number).sort((a, b) => a - b);
+      const phases = runs.flatMap((run) => run.phases);
+      const slowestPhase = phases.reduce(
+        (slowest, phase) => (phase.durationMs > slowest.durationMs ? phase : slowest),
+        { label: "n/a", durationMs: 0 },
+      );
+      const medianMs = median(durations);
+      const p95Ms = percentile(durations, 0.95);
+      return {
+        target: [first.targetId ?? "unlabeled", first.shardId].filter(Boolean).join("/"),
+        scenario: first.scenario,
+        runs: runs.length,
+        medianMs,
+        p95Ms,
+        maxMs: durations.at(-1) ?? 0,
+        variabilityMs: Math.max(0, p95Ms - medianMs),
+        slowestPhase: slowestPhase.label,
+        slowestPhaseMs: slowestPhase.durationMs,
+      };
+    })
+    .sort((a, b) => b.p95Ms - a.p95Ms || b.variabilityMs - a.variabilityMs);
+}
+
+function seconds(milliseconds: number): string {
+  return (milliseconds / 1_000).toFixed(1);
+}
+
+export function formatRuntimeAudit(rows: readonly RuntimeAuditRow[]): string {
+  const lines = [
+    "| Target | Scenario | Runs | Median | p95 | Max | p95 - median | Slowest observed phase |",
+    "| --- | --- | ---: | ---: | ---: | ---: | ---: | --- |",
+  ];
+  for (const row of rows) {
+    lines.push(
+      `| ${row.target.replaceAll("|", "\\|")} | ${row.scenario.replaceAll("|", "\\|")} | ${row.runs} | ${seconds(row.medianMs)}s | ${seconds(row.p95Ms)}s | ${seconds(row.maxMs)}s | ${seconds(row.variabilityMs)}s | ${row.slowestPhase.replaceAll("|", "\\|")} (${seconds(row.slowestPhaseMs)}s) |`,
+    );
+  }
+  return `${lines.join("\n")}\n`;
+}
+
+function main(argv: readonly string[]): void {
+  const roots = argv.length > 0 ? argv : [".e2e/live"];
+  const rows = auditTestRuntime(roots);
+  if (rows.length === 0) {
+    throw new Error(`no test-progress.json files found under: ${roots.join(", ")}`);
+  }
+  process.stdout.write(formatRuntimeAudit(rows));
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+  main(process.argv.slice(2));
+}