diff --git a/test/README.md b/test/README.md new file mode 100644 index 0000000000..b8956efb3c --- /dev/null +++ b/test/README.md @@ -0,0 +1,100 @@ + + +# NemoClaw test suite + +## Local full-suite performance reference (#6245) + +This is the dated acceptance snapshot for the work that restored the complete +non-live local test loop to the two-to-five-minute range. It records a +host-specific result, not a permanent performance budget. Product growth, +dependency changes, and different hardware can change the absolute times. + +### Measurement method + +The final measurement was taken on 2026-07-07 with: + +- Linux 6.17.0-1014-nvidia on arm64; +- 20 logical CPUs available to Node; +- Node.js 22.23.1 and npm 10.9.8; +- the normal contributor and CI file-creation mask, `umask 022`; +- code head `45800b3318079d59524239e0f05026c17a5ea0ac` from PR #6420. + +The final record-only Markdown commit does not change the measured test code. +`npm test` removed and rebuilt the CLI and plugin artifacts before Vitest ran. +Dependencies were already installed and operating-system page caches were not +dropped, so this is a clean-build measurement rather than a cold-machine +benchmark. + +```bash +umask 022 +/usr/bin/time -v npm test -- --reporter=blob +/usr/bin/time -p npx prek run --from-ref origin/main --to-ref HEAD --stage pre-commit +``` + +### Results + +| Measurement | Revision | Wall time | Result | +|---|---|---:|---| +| Original full-suite baseline from #6245 | `54f90a023` plus the #6214 candidate | 14:19.65 | 980 files passed, 2 skipped; 11,109 tests passed, 35 skipped | +| Pre-PR full-suite control | `5552a5ba5` | 13:21.17 | Passed | +| Final complete non-live suite | `45800b331` | **3:52.03** | 1,249 files passed, 2 skipped; 13,879 tests passed, 39 skipped, 1 todo | +| Original static/format/policy/security pre-commit phase | #6245 baseline | about 0:30 | Passed before the old coverage hook | +| Original `test-cli` pre-commit hook | #6245 baseline | 11:43 before exit | Failed on a late unrelated flake; not a completed pass | +| Final diff-scoped routine pre-commit stage | `45800b331` | **0:13.95** | Passed | + +The final full suite is 73.0% faster than the issue baseline and 71.0% faster +than the same-machine pre-PR control. Over the longer baseline interval, the +passing-test count grew 24.9% and the measured file count grew 27.4%. + +PR #6270 intentionally moved full CLI and plugin coverage from routine +pre-commit to the explicit manual gate while retaining authoritative coverage +in CI. The final pre-commit number therefore describes the current routine +contributor gate; `npm test` above remains the complete non-live behavioral +signal. + +### Highest-cost file disposition + +The initial profile in #6245 identified these six measured hotspots. The +classification distinguishes process launches used only for test isolation +from launches that are themselves part of the contract. + +| Initial hotspot | Initial time | Classification | Final disposition | +|---|---:|---|---| +| `test/onboard-selection.test.ts` | 24.2s | Mixed, dominated by unit-shaped subprocesses | #6276, #6336, and #6383 moved decision branches to typed in-process seams while retaining boundary-sensitive witnesses | +| `test/sandbox-connect-inference/route-swap-repair.test.ts` | 21.4s | Mixed lifecycle decisions and real command boundaries | #6280 moved route-state branches in-process and retained CLI, listener, timeout, security, and cross-command contracts | +| `test/gateway-state-reconcile-2276.test.ts` | 19.8s | Mixed reconciliation logic and process contracts | #6280 moved repeated reconciliation decisions in-process and kept representative gateway process behavior | +| `test/generate-openclaw-config.test.ts` | 18.6s | Unit-shaped configuration generation | #6276 and #6339 replaced process isolation with direct typed configuration and policy seams | +| `test/onboard.test.ts` | 18.0s | Mixed orchestration and genuine CLI/shell boundaries | #6276 moved internal decisions in-process and retained fail-closed, Bash, credential, and argv witnesses | +| `test/nemoclaw-start.test.ts` | 16.4s | Genuine shell and startup process contract | Kept process-shaped; focused direct tests cover internal decisions without replacing the real startup boundary | + +The later batches applied the same rule beyond the pilot. PRs #6279 and #6282, +along with #6285, #6339, #6342, #6345, #6360, #6369, #6373, #6383, #6411, +and #6417, removed unit-shaped launches or shared safe harnesses while +preserving the boundary cases. + +### Representative retained process contracts + +The remaining real-process layer includes at least one representative for each +boundary required by #6245: + +| Boundary | Representative coverage | +|---|---| +| CLI argv, exit status, and executable discovery | `test/exit-code-user-error-surfaces.test.ts`; `test/package-contract/cli/command-registry.test.ts`; `test/package-contract/cli/public-argv-translation.test.ts` | +| Environment inheritance and secret isolation | `test/rebuild-credential-preflight.test.ts`; `test/secret-redaction.test.ts`; `test/clean-runtime-shell-env-shim.test.ts` | +| Shell, signal, process-group, and supervisor behavior | `test/nemoclaw-start.test.ts`; `test/runner.test.ts`; `test/gateway-supervisor-control.test.ts` | +| Permissions, symlinks, and atomic filesystem replacement | `test/state-dir-guard.test.ts`; `test/nemoclaw-start-perms.test.ts`; `src/lib/adapters/fs/build-context-fingerprint.test.ts` | +| Security and fail-closed behavior | `test/exit-code-user-error-surfaces.test.ts`; `test/package-contract/ssrf-parity.test.ts` | +| Compiled artifact and package behavior | the disjoint `test/package-contract/` project | + +The optimization chain did not add blanket retries, timeout increases, or +stale compiled-output reuse. Process-removal PRs #6276, #6279, #6280, #6282, +and #6285 landed before #6286 addressed #6237's hottest loader graph. The +loader work in #6299 and #6388, followed by #6415, then ratcheted remaining +CommonJS loader and cache seams. + +PR #6420 is the final scheduling step. Only the canonical local `npm test` +route runs integration files with bounded parallelism. CI, coverage, focused +integration runs, and direct Vitest invocations remain serialized. diff --git a/test/exit-code-user-error-surfaces.test.ts b/test/exit-code-user-error-surfaces.test.ts index 89259abf86..bffc6c6336 100644 --- a/test/exit-code-user-error-surfaces.test.ts +++ b/test/exit-code-user-error-surfaces.test.ts @@ -27,12 +27,12 @@ * This matrix locks the nonexistent-sandbox share/upload surfaces instead. * * Issue instance 3 (onboard dashboard-port exhaustion) is locked by its own - * hermetic `onboard` spawn in the second describe below: it binds the whole - * dashboard port range and drives the real `onboard` preflight to the - * fail-fast "All dashboard ports in range … are occupied" exit, asserting a - * non-zero code. (That preflight exits via an explicit `exitFn(1)`, so it never - * rode the `oclif.exit === 0` catch-all this PR hardens — the spawn simply - * proves the surface stays non-zero end-to-end.) + * hermetic `onboard` spawn in the second describe below: a fake `lsof` reports + * the whole dashboard port range as occupied and drives the real `onboard` + * preflight to the fail-fast "All dashboard ports in range … are occupied" + * exit, asserting a non-zero code. (That preflight exits via an explicit + * `exitFn(1)`, so it never rode the `oclif.exit === 0` catch-all this PR + * hardens — the spawn simply proves the surface stays non-zero end-to-end.) * * Issue instance 5 (Model Router Python preflight) is the one surface left to * unit tests: `reconcileModelRouter` runs only deep in `onboard`, behind live @@ -50,7 +50,6 @@ import { spawnSync } from "node:child_process"; import fs from "node:fs"; -import net from "node:net"; import os from "node:os"; import path from "node:path"; @@ -241,11 +240,14 @@ describe("user-error/startup surfaces return non-zero exit (#5974)", () => { describe("onboard dashboard-port exhaustion exits non-zero (#5974)", () => { const PORT_RANGE_START = 18789; const PORT_RANGE_END = 18799; + const OCCUPIED_PORT_CASES = Array.from( + { length: PORT_RANGE_END - PORT_RANGE_START + 1 }, + (_unused, index) => PORT_RANGE_START + index, + ).join("|"); let home: string; let binDir: string; - let servers: net.Server[]; - beforeEach(async () => { + beforeEach(() => { home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-5974-onboard-")); binDir = path.join(home, "bin"); fs.mkdirSync(binDir, { recursive: true }); @@ -278,29 +280,26 @@ describe("onboard dashboard-port exhaustion exits non-zero (#5974)", () => { { mode: 0o755 }, ); - // Occupy the entire dashboard port range so the preflight has no free port. - servers = []; - const ports = Array.from( - { length: PORT_RANGE_END - PORT_RANGE_START + 1 }, - (_unused, i) => PORT_RANGE_START + i, - ); - await Promise.all( - ports.map( - (port) => - new Promise((resolve) => { - const server = net.createServer(); - server.once("error", () => resolve()); - server.listen(port, "127.0.0.1", () => { - servers.push(server); - resolve(); - }); - }), - ), + // Report the whole dashboard range as occupied without binding host-global + // ports that can interfere with other integration workers. + fs.writeFileSync( + path.join(binDir, "lsof"), + [ + "#!/usr/bin/env bash", + 'port=""', + 'for arg in "$@"; do', + ' case "$arg" in :*) port="${arg#:}";; esac', + "done", + 'case "$port" in', + ` ${OCCUPIED_PORT_CASES}) echo "node 1 user 1u IPv4 TCP 127.0.0.1:\${port} (LISTEN)"; exit 0;;`, + "esac", + "exit 1", + ].join("\n"), + { mode: 0o755 }, ); }); afterEach(() => { - for (const server of servers) server.close(); fs.rmSync(home, { recursive: true, force: true }); }); diff --git a/test/helpers/integration-project-scheduling.ts b/test/helpers/integration-project-scheduling.ts new file mode 100644 index 0000000000..d7b3f279af --- /dev/null +++ b/test/helpers/integration-project-scheduling.ts @@ -0,0 +1,63 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +const LOCAL_INTEGRATION_WORKER_CAP = 4; + +interface IntegrationProjectSchedulingContext { + isCi: boolean; + npmLifecycleEvent: string | undefined; + argv: readonly string[]; + availableParallelism?: number; +} + +function parseWorkerCount(rawValue: string, availableWorkers: number): number { + if (/^\d+$/.test(rawValue)) { + const parsed = Number(rawValue); + if (parsed >= 1 && Number.isSafeInteger(parsed)) return parsed; + } else if (/^\d+%$/.test(rawValue)) { + const percentage = Number(rawValue.slice(0, -1)); + if (percentage >= 1 && Number.isSafeInteger(percentage)) { + return Math.max(1, Math.round((percentage / 100) * availableWorkers)); + } + } + throw new Error(`Invalid --maxWorkers value: "${rawValue}"`); +} + +function resolveWorkerCap(argv: readonly string[], availableWorkers: number): number { + let requested: number | null = null; + for (let index = 0; index < argv.length; index += 1) { + const argument = argv[index] ?? ""; + let rawValue: string | undefined; + if (argument.startsWith("--maxWorkers=")) { + rawValue = argument.slice("--maxWorkers=".length); + } else if (argument === "--maxWorkers") { + rawValue = argv[index + 1]; + index += 1; + } else { + continue; + } + if (rawValue === undefined) throw new Error("--maxWorkers requires a number or percentage"); + requested = parseWorkerCount(rawValue, availableWorkers); + } + return Math.min(requested ?? LOCAL_INTEGRATION_WORKER_CAP, LOCAL_INTEGRATION_WORKER_CAP); +} + +export function resolveIntegrationProjectScheduling({ + isCi, + npmLifecycleEvent, + argv, + availableParallelism = LOCAL_INTEGRATION_WORKER_CAP, +}: IntegrationProjectSchedulingContext) { + const parallelize = + !isCi && + npmLifecycleEvent === "test" && + !argv.some((argument) => argument.startsWith("--coverage")); + + return parallelize + ? { + fileParallelism: true, + maxWorkers: resolveWorkerCap(argv, availableParallelism), + sequence: { groupOrder: 1 }, + } + : { fileParallelism: false }; +} diff --git a/test/onboard-installer-restore-intent.test.ts b/test/onboard-installer-restore-intent.test.ts index b80e9af1d6..c3db7c1f78 100644 --- a/test/onboard-installer-restore-intent.test.ts +++ b/test/onboard-installer-restore-intent.test.ts @@ -265,6 +265,11 @@ runner.run = (command) => { runner.runCapture = (command) => { if (_n(command).includes("sandbox get my-assistant")) return "my-assistant"; if (_n(command).includes("sandbox list")) return "my-assistant NotReady"; + // Keep dashboard allocation inside this restore-intent fixture; host port + // occupancy is unrelated to the not-ready decision under test. + if (_n(command).includes("forward list")) { + return "my-assistant 127.0.0.1 18789 12345 running"; + } return ""; }; registry.getSandbox = () => ({ diff --git a/test/vitest-integration-scheduling.test.ts b/test/vitest-integration-scheduling.test.ts new file mode 100644 index 0000000000..f5fba07801 --- /dev/null +++ b/test/vitest-integration-scheduling.test.ts @@ -0,0 +1,76 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { resolveIntegrationProjectScheduling } from "./helpers/integration-project-scheduling"; + +describe("integration project scheduling", () => { + it("parallelizes the canonical local full-suite run (#6245)", () => { + expect( + resolveIntegrationProjectScheduling({ + isCi: false, + npmLifecycleEvent: "test", + argv: [], + }), + ).toEqual({ + fileParallelism: true, + maxWorkers: 4, + sequence: { groupOrder: 1 }, + }); + }); + + it.each([ + [["--maxWorkers=1"], 1, 8], + [["--maxWorkers", "2"], 2, 8], + [["--maxWorkers=10%"], 2, 20], + [["--maxWorkers=8"], 4, 8], + ])("honors the explicit local worker cap in %j (#6245)", (argv, maxWorkers, workers) => { + expect( + resolveIntegrationProjectScheduling({ + isCi: false, + npmLifecycleEvent: "test", + argv, + availableParallelism: workers, + }), + ).toEqual({ + fileParallelism: true, + maxWorkers, + sequence: { groupOrder: 1 }, + }); + }); + + it.each([ + "--maxWorkers=invalid", + "--maxWorkers", + ])("rejects the invalid local worker cap %s (#6245)", (argument) => { + expect(() => + resolveIntegrationProjectScheduling({ + isCi: false, + npmLifecycleEvent: "test", + argv: [argument], + }), + ).toThrow(/--maxWorkers/); + }); + + it.each([ + ["CI full suite", { isCi: true, npmLifecycleEvent: "test", argv: [] }], + ["coverage shorthand", { isCi: false, npmLifecycleEvent: "test", argv: ["--coverage"] }], + [ + "coverage property", + { isCi: false, npmLifecycleEvent: "test", argv: ["--coverage.enabled=true"] }, + ], + [ + "coverage sub-option", + { isCi: false, npmLifecycleEvent: "test", argv: ["--coverage.provider=v8"] }, + ], + ["coverage script", { isCi: false, npmLifecycleEvent: "test:coverage:cli", argv: [] }], + [ + "focused integration script", + { isCi: false, npmLifecycleEvent: "test:integration", argv: [] }, + ], + ["direct Vitest", { isCi: false, npmLifecycleEvent: undefined, argv: [] }], + ])("keeps $0 serialized (#6245)", (_name, context) => { + expect(resolveIntegrationProjectScheduling(context)).toEqual({ fileParallelism: false }); + }); +}); diff --git a/vitest.config.ts b/vitest.config.ts index 5cbe8c26a6..214b0d9156 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { availableParallelism } from "node:os"; import path from "node:path"; import { defineConfig } from "vitest/config"; @@ -9,6 +10,7 @@ import { shouldRunBranchValidationE2E, shouldRunLiveE2E, } from "./test/e2e/fixtures/live-project-gate.ts"; +import { resolveIntegrationProjectScheduling } from "./test/helpers/integration-project-scheduling"; import { sourceLoaderNodeOptions } from "./test/helpers/source-loader-options"; import { testTimeout } from "./test/helpers/timeouts"; @@ -32,6 +34,12 @@ const typedSourceTransform = { }, }; const sourceNodeOptions = sourceLoaderNodeOptions(process.env.NODE_OPTIONS); +const integrationProjectScheduling = resolveIntegrationProjectScheduling({ + isCi, + npmLifecycleEvent: process.env.npm_lifecycle_event, + argv: process.argv.slice(2), + availableParallelism: availableParallelism(), +}); export default defineConfig({ test: { @@ -64,16 +72,11 @@ export default defineConfig({ // when several coverage shards transpile and spawn them concurrently. testTimeout: testTimeout(15_000), setupFiles: ["test/helpers/onboard-script-mocks.cjs"], - // Integration fixtures often spawn short Node programs. Keep those - // programs on the same source graph as their parent test process. - // The integration suite shells out heavily, and stacking multiple - // forks of the require-hook transpile cache on the 7 GiB ubuntu - // runner reliably exhausts physical RAM when coverage is on. - // Disable file parallelism for the integration project so the test - // files run serially against a single worker (vitest 4 dropped - // poolOptions.forks.singleFork; fileParallelism: false is the - // documented replacement). - fileParallelism: false, + // Integration fixtures often spawn short Node programs. Coverage + // stays serial because concurrent source-loader forks exhaust the + // 7 GiB CI runner. The canonical local full suite instead runs this + // project as a bounded four-worker phase after the other projects. + ...integrationProjectScheduling, env: { NODE_OPTIONS: sourceNodeOptions }, include: ["test/**/*.test.{js,ts}"], exclude: [