From e2f338815600bcc4371562be1f775f41d9eece34 Mon Sep 17 00:00:00 2001 From: Chengjie Wang Date: Fri, 10 Jul 2026 17:20:35 +0800 Subject: [PATCH 01/14] fix(onboard): scope live inference recovery to sandbox Signed-off-by: Chengjie Wang --- src/lib/onboard/provider-recovery.test.ts | 52 +++++++++++++++++++++++ src/lib/onboard/provider-recovery.ts | 34 +++++++++------ 2 files changed, 72 insertions(+), 14 deletions(-) diff --git a/src/lib/onboard/provider-recovery.test.ts b/src/lib/onboard/provider-recovery.test.ts index 527996c606..d2862b6ffc 100644 --- a/src/lib/onboard/provider-recovery.test.ts +++ b/src/lib/onboard/provider-recovery.test.ts @@ -50,6 +50,58 @@ describe("provider recovery persisted routing state", () => { expect(helpers().readLiveInference("alpha")).toBeNull(); }); + it("does not recover a stale live route for a brand-new sandbox name (#6630)", () => { + vi.spyOn(registry, "getSandbox").mockReturnValue(null); + vi.spyOn(registry, "listSandboxes").mockReturnValue({ + defaultSandbox: null, + sandboxes: [], + }); + vi.spyOn(onboardSession, "loadSession").mockReturnValue( + onboardSession.createSession({ + sandboxName: null, + provider: "nvidia-prod", + model: "nvidia/nemotron-3-super-120b-a12b", + }), + ); + const parseGatewayInference = vi.fn(() => ({ + provider: "nvidia-prod", + model: "nvidia/nemotron-3-super-120b-a12b", + })); + const runCaptureOpenshell = vi.fn(() => "Gateway inference:"); + const recovery = createProviderRecoveryHelpers({ + parseGatewayInference, + runCaptureOpenshell, + }); + + expect(recovery.readRecordedProvider("dc-after")).toBeNull(); + expect(recovery.readRecordedModel("dc-after")).toBeNull(); + expect(runCaptureOpenshell).not.toHaveBeenCalled(); + expect(parseGatewayInference).not.toHaveBeenCalled(); + }); + + it("recovers a live route when a matching session proves the sandbox identity", () => { + vi.spyOn(registry, "getSandbox").mockReturnValue(null); + vi.spyOn(registry, "listSandboxes").mockReturnValue({ + defaultSandbox: null, + sandboxes: [], + }); + vi.spyOn(onboardSession, "loadSession").mockReturnValue( + onboardSession.createSession({ sandboxName: "rebuild-box" }), + ); + const runCaptureOpenshell = vi.fn(() => "Gateway inference:"); + const recovery = createProviderRecoveryHelpers({ + parseGatewayInference: () => ({ + provider: "nvidia-prod", + model: "nvidia/nemotron-3-super-120b-a12b", + }), + runCaptureOpenshell, + }); + + expect(recovery.readRecordedProvider("rebuild-box")).toBe("nvidia-prod"); + expect(recovery.readRecordedModel("rebuild-box")).toBe("nvidia/nemotron-3-super-120b-a12b"); + expect(runCaptureOpenshell).toHaveBeenCalledTimes(2); + }); + it("prefers the selected sandbox registry endpoint over session state", () => { vi.spyOn(registry, "getSandbox").mockReturnValue({ name: "alpha", diff --git a/src/lib/onboard/provider-recovery.ts b/src/lib/onboard/provider-recovery.ts index 6814835f0a..25280ccaff 100644 --- a/src/lib/onboard/provider-recovery.ts +++ b/src/lib/onboard/provider-recovery.ts @@ -133,8 +133,10 @@ export function createProviderRecoveryHelpers(deps: ProviderRecoveryDeps): Provi function readRecordedProvider(sandboxName: string | null | undefined): string | null { if (!sandboxName) return null; + let hasRecordedSandboxIdentity = false; try { const entry = registry.getSandbox(sandboxName); + hasRecordedSandboxIdentity = Boolean(entry); if (entry && typeof entry.provider === "string" && entry.provider) { return entry.provider; } @@ -143,17 +145,20 @@ export function createProviderRecoveryHelpers(deps: ProviderRecoveryDeps): Provi } try { const session = onboardSession.loadSession(); - if ( - session && - session.sandboxName === sandboxName && - typeof session.provider === "string" && - session.provider - ) { - return session.provider; + if (session && session.sandboxName === sandboxName) { + hasRecordedSandboxIdentity = true; + if (typeof session.provider === "string" && session.provider) { + return session.provider; + } } } catch { // fall through to live gateway } + // An empty registry does not prove that the gateway's residual route + // belongs to this requested sandbox name. Require a matching registry or + // session identity before using that route as rebuild recovery state; + // otherwise a brand-new sandbox inherits the previous sandbox's model. + if (!hasRecordedSandboxIdentity) return null; const live = readLiveInference(sandboxName); if (live && typeof live.provider === "string" && live.provider) { return live.provider; @@ -189,8 +194,10 @@ export function createProviderRecoveryHelpers(deps: ProviderRecoveryDeps): Provi function readRecordedModel(sandboxName: string | null | undefined): string | null { if (!sandboxName) return null; + let hasRecordedSandboxIdentity = false; try { const entry = registry.getSandbox(sandboxName); + hasRecordedSandboxIdentity = Boolean(entry); if (entry && typeof entry.model === "string" && entry.model) { return entry.model; } @@ -199,17 +206,16 @@ export function createProviderRecoveryHelpers(deps: ProviderRecoveryDeps): Provi } try { const session = onboardSession.loadSession(); - if ( - session && - session.sandboxName === sandboxName && - typeof session.model === "string" && - session.model - ) { - return session.model; + if (session && session.sandboxName === sandboxName) { + hasRecordedSandboxIdentity = true; + if (typeof session.model === "string" && session.model) { + return session.model; + } } } catch { // fall through to live gateway } + if (!hasRecordedSandboxIdentity) return null; const live = readLiveInference(sandboxName); if (live && typeof live.model === "string" && live.model) { return live.model; From df8fc9380f2deb587ef69fefdf936edcda9ba728 Mon Sep 17 00:00:00 2001 From: Chengjie Wang Date: Fri, 10 Jul 2026 17:36:17 +0800 Subject: [PATCH 02/14] fix(onboard): preserve explicit provider recovery Signed-off-by: Chengjie Wang --- src/lib/onboard.ts | 14 +-- src/lib/onboard/machine/core-flow-phases.ts | 2 + .../handlers/provider-inference.test.ts | 21 ++++ .../machine/handlers/provider-inference.ts | 5 +- src/lib/onboard/provider-recovery.test.ts | 113 ++++++++++-------- src/lib/onboard/provider-recovery.ts | 51 ++++---- 6 files changed, 123 insertions(+), 83 deletions(-) diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 68403a1dc2..04ab0c2158 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -4417,21 +4417,17 @@ async function runOnboard(opts: OnboardOptions = {}): Promise { process.exit(1); } - const coreFlowContext: InitialOnboardFlowContext = { - ...initialContext, - session, - sandboxName, - selectedMessagingChannels, - gpu, - sandboxGpuConfig, - gpuPassthrough, - }; + // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. + const coreFlowContext: InitialOnboardFlowContext = { ...initialContext, session, sandboxName, selectedMessagingChannels, gpu, sandboxGpuConfig, gpuPassthrough }; + // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. + const allowRecordedProviderRecovery = providerRecovery.shouldRecoverRecordedProvider({ fresh, resume, sandboxName, hasRegisteredSandbox: Boolean(sandboxName && registry.getSandbox(sandboxName)), sessionSandboxName: session?.sandboxName ?? null }); // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. const runCoreGatewayOpenshell = setupInferenceFactory.createGatewayScopedOpenshellRunner(runOpenshell, GATEWAY_NAME); const [providerInferencePhase, sandboxPhase] = createCoreOnboardFlowPhases({ gatewayName: GATEWAY_NAME, forceProviderSelection: forceProviderSelectionForAgentChange, + allowRecordedProviderRecovery, ...authoritativeRebuildTarget.rebuildProviderFlowOptions(opts, coreFlowContext), env: process.env, constants: { diff --git a/src/lib/onboard/machine/core-flow-phases.ts b/src/lib/onboard/machine/core-flow-phases.ts index 957807b4d3..6a73d15ae8 100644 --- a/src/lib/onboard/machine/core-flow-phases.ts +++ b/src/lib/onboard/machine/core-flow-phases.ts @@ -28,6 +28,7 @@ export interface CoreOnboardFlowPhaseOptions< > { gatewayName: string; forceProviderSelection: boolean; + allowRecordedProviderRecovery?: boolean; forceInferenceSetup?: boolean; authoritativeResumeConfig?: boolean; env: NodeJS.ProcessEnv; @@ -69,6 +70,7 @@ export function createCoreOnboardFlowPhases< sandboxName: context.sandboxName, agent: context.agent, forceProviderSelection: options.forceProviderSelection, + allowRecordedProviderRecovery: options.allowRecordedProviderRecovery, forceInferenceSetup: options.forceInferenceSetup, authoritativeResumeConfig: options.authoritativeResumeConfig, initial: { diff --git a/src/lib/onboard/machine/handlers/provider-inference.test.ts b/src/lib/onboard/machine/handlers/provider-inference.test.ts index 4c70a01e50..8d78241604 100644 --- a/src/lib/onboard/machine/handlers/provider-inference.test.ts +++ b/src/lib/onboard/machine/handlers/provider-inference.test.ts @@ -313,6 +313,27 @@ describe("handleProviderInferenceState", () => { ); }); + it("disables recorded provider recovery for a brand-new sandbox identity (#6630)", async () => { + const { deps, calls } = createDeps(); + + await handleProviderInferenceState({ + ...baseOptions(deps), + fresh: false, + sandboxName: "dc-after", + allowRecordedProviderRecovery: false, + }); + + expect(calls.setupNim).toHaveBeenCalledWith( + { type: "nvidia" }, + "dc-after", + null, + false, + "nemoclaw", + expect.any(Function), + expect.any(Function), + ); + }); + it("does not use resume shortcuts when fresh is also set", async () => { const session = createSession({ provider: "ollama-local", model: "llama3.1" }); session.steps.provider_selection.status = "complete"; diff --git a/src/lib/onboard/machine/handlers/provider-inference.ts b/src/lib/onboard/machine/handlers/provider-inference.ts index 37e1ec4020..043d3ef7d8 100644 --- a/src/lib/onboard/machine/handlers/provider-inference.ts +++ b/src/lib/onboard/machine/handlers/provider-inference.ts @@ -61,6 +61,7 @@ export interface ProviderInferenceStateOptions { sandboxName: string | null; agent: Agent; forceProviderSelection?: boolean; + allowRecordedProviderRecovery?: boolean; /** Force setup for a provider that authoritative rebuild preflight observed missing. */ forceInferenceSetup?: boolean; /** Trust the rebuild-preflighted session selection even if its old step marker is incomplete. */ @@ -281,6 +282,7 @@ export async function handleProviderInferenceState({ sandboxName, agent, forceProviderSelection: initialForceProviderSelection = false, + allowRecordedProviderRecovery, forceInferenceSetup: initialForceInferenceSetup = false, authoritativeResumeConfig = false, initial, @@ -319,6 +321,7 @@ export async function handleProviderInferenceState({ let reuseGatewayCredentialWithoutLocalKey = false; let endpointPinnedAddresses: string[] | undefined; const effectiveResume = resume && !fresh; + const recoverRecordedProvider = allowRecordedProviderRecovery ?? !fresh; const stateResults: OnboardStateTransitionResult[] = []; const retryStateResults: OnboardStateTransitionResult[] = []; @@ -447,7 +450,7 @@ export async function handleProviderInferenceState({ gpu, sandboxName, agent, - !fresh, + recoverRecordedProvider, gatewayName, (route) => guardProviderInferenceRouteSelection(deps, gatewayName, sandboxName, route), (provider) => diff --git a/src/lib/onboard/provider-recovery.test.ts b/src/lib/onboard/provider-recovery.test.ts index d2862b6ffc..321b40ecb5 100644 --- a/src/lib/onboard/provider-recovery.test.ts +++ b/src/lib/onboard/provider-recovery.test.ts @@ -5,7 +5,11 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import * as onboardSession from "../state/onboard-session"; import * as registry from "../state/registry"; -import { createProviderRecoveryHelpers, validateLiveGatewayInference } from "./provider-recovery"; +import { + createProviderRecoveryHelpers, + shouldRecoverRecordedProvider, + validateLiveGatewayInference, +} from "./provider-recovery"; afterEach(() => { vi.restoreAllMocks(); @@ -33,6 +37,61 @@ describe("validateLiveGatewayInference", () => { }); }); +describe("shouldRecoverRecordedProvider", () => { + it.each([ + { + label: "rejects gateway recovery for a brand-new sandbox", + fresh: false, + resume: false, + hasRegisteredSandbox: false, + sessionSandboxName: null, + expected: false, + }, + { + label: "allows gateway recovery while resuming", + fresh: false, + resume: true, + hasRegisteredSandbox: false, + sessionSandboxName: null, + expected: true, + }, + { + label: "allows gateway recovery for a registered sandbox", + fresh: false, + resume: false, + hasRegisteredSandbox: true, + sessionSandboxName: null, + expected: true, + }, + { + label: "allows gateway recovery for a matching session", + fresh: false, + resume: false, + hasRegisteredSandbox: false, + sessionSandboxName: "dc-after", + expected: true, + }, + { + label: "rejects gateway recovery when fresh overrides existing identity", + fresh: true, + resume: true, + hasRegisteredSandbox: true, + sessionSandboxName: "dc-after", + expected: false, + }, + ])("$label", ({ fresh, resume, hasRegisteredSandbox, sessionSandboxName, expected }) => { + expect( + shouldRecoverRecordedProvider({ + fresh, + resume, + sandboxName: "dc-after", + hasRegisteredSandbox, + sessionSandboxName, + }), + ).toBe(expected); + }); +}); + describe("provider recovery persisted routing state", () => { function helpers() { return createProviderRecoveryHelpers({ @@ -50,58 +109,6 @@ describe("provider recovery persisted routing state", () => { expect(helpers().readLiveInference("alpha")).toBeNull(); }); - it("does not recover a stale live route for a brand-new sandbox name (#6630)", () => { - vi.spyOn(registry, "getSandbox").mockReturnValue(null); - vi.spyOn(registry, "listSandboxes").mockReturnValue({ - defaultSandbox: null, - sandboxes: [], - }); - vi.spyOn(onboardSession, "loadSession").mockReturnValue( - onboardSession.createSession({ - sandboxName: null, - provider: "nvidia-prod", - model: "nvidia/nemotron-3-super-120b-a12b", - }), - ); - const parseGatewayInference = vi.fn(() => ({ - provider: "nvidia-prod", - model: "nvidia/nemotron-3-super-120b-a12b", - })); - const runCaptureOpenshell = vi.fn(() => "Gateway inference:"); - const recovery = createProviderRecoveryHelpers({ - parseGatewayInference, - runCaptureOpenshell, - }); - - expect(recovery.readRecordedProvider("dc-after")).toBeNull(); - expect(recovery.readRecordedModel("dc-after")).toBeNull(); - expect(runCaptureOpenshell).not.toHaveBeenCalled(); - expect(parseGatewayInference).not.toHaveBeenCalled(); - }); - - it("recovers a live route when a matching session proves the sandbox identity", () => { - vi.spyOn(registry, "getSandbox").mockReturnValue(null); - vi.spyOn(registry, "listSandboxes").mockReturnValue({ - defaultSandbox: null, - sandboxes: [], - }); - vi.spyOn(onboardSession, "loadSession").mockReturnValue( - onboardSession.createSession({ sandboxName: "rebuild-box" }), - ); - const runCaptureOpenshell = vi.fn(() => "Gateway inference:"); - const recovery = createProviderRecoveryHelpers({ - parseGatewayInference: () => ({ - provider: "nvidia-prod", - model: "nvidia/nemotron-3-super-120b-a12b", - }), - runCaptureOpenshell, - }); - - expect(recovery.readRecordedProvider("rebuild-box")).toBe("nvidia-prod"); - expect(recovery.readRecordedModel("rebuild-box")).toBe("nvidia/nemotron-3-super-120b-a12b"); - expect(runCaptureOpenshell).toHaveBeenCalledTimes(2); - }); - it("prefers the selected sandbox registry endpoint over session state", () => { vi.spyOn(registry, "getSandbox").mockReturnValue({ name: "alpha", diff --git a/src/lib/onboard/provider-recovery.ts b/src/lib/onboard/provider-recovery.ts index 25280ccaff..4d418489ba 100644 --- a/src/lib/onboard/provider-recovery.ts +++ b/src/lib/onboard/provider-recovery.ts @@ -66,6 +66,23 @@ const MAX_LIVE_PROVIDER_LENGTH = 128; const MAX_LIVE_MODEL_LENGTH = 512; const SAFE_LIVE_PROVIDER = /^[A-Za-z0-9._:-]+$/; +export function shouldRecoverRecordedProvider(input: { + fresh: boolean; + resume: boolean; + sandboxName: string | null; + hasRegisteredSandbox: boolean; + sessionSandboxName: string | null; +}): boolean { + return ( + !input.fresh && + (input.resume || + Boolean( + input.sandboxName && + (input.hasRegisteredSandbox || input.sessionSandboxName === input.sandboxName), + )) + ); +} + export function validateLiveGatewayInference( value: { provider: string | null; model: string | null } | null, ): { provider: string; model: string } | null { @@ -133,10 +150,8 @@ export function createProviderRecoveryHelpers(deps: ProviderRecoveryDeps): Provi function readRecordedProvider(sandboxName: string | null | undefined): string | null { if (!sandboxName) return null; - let hasRecordedSandboxIdentity = false; try { const entry = registry.getSandbox(sandboxName); - hasRecordedSandboxIdentity = Boolean(entry); if (entry && typeof entry.provider === "string" && entry.provider) { return entry.provider; } @@ -145,20 +160,17 @@ export function createProviderRecoveryHelpers(deps: ProviderRecoveryDeps): Provi } try { const session = onboardSession.loadSession(); - if (session && session.sandboxName === sandboxName) { - hasRecordedSandboxIdentity = true; - if (typeof session.provider === "string" && session.provider) { - return session.provider; - } + if ( + session && + session.sandboxName === sandboxName && + typeof session.provider === "string" && + session.provider + ) { + return session.provider; } } catch { // fall through to live gateway } - // An empty registry does not prove that the gateway's residual route - // belongs to this requested sandbox name. Require a matching registry or - // session identity before using that route as rebuild recovery state; - // otherwise a brand-new sandbox inherits the previous sandbox's model. - if (!hasRecordedSandboxIdentity) return null; const live = readLiveInference(sandboxName); if (live && typeof live.provider === "string" && live.provider) { return live.provider; @@ -194,10 +206,8 @@ export function createProviderRecoveryHelpers(deps: ProviderRecoveryDeps): Provi function readRecordedModel(sandboxName: string | null | undefined): string | null { if (!sandboxName) return null; - let hasRecordedSandboxIdentity = false; try { const entry = registry.getSandbox(sandboxName); - hasRecordedSandboxIdentity = Boolean(entry); if (entry && typeof entry.model === "string" && entry.model) { return entry.model; } @@ -206,16 +216,17 @@ export function createProviderRecoveryHelpers(deps: ProviderRecoveryDeps): Provi } try { const session = onboardSession.loadSession(); - if (session && session.sandboxName === sandboxName) { - hasRecordedSandboxIdentity = true; - if (typeof session.model === "string" && session.model) { - return session.model; - } + if ( + session && + session.sandboxName === sandboxName && + typeof session.model === "string" && + session.model + ) { + return session.model; } } catch { // fall through to live gateway } - if (!hasRecordedSandboxIdentity) return null; const live = readLiveInference(sandboxName); if (live && typeof live.model === "string" && live.model) { return live.model; From cddddd246c684f4230e0c7d8cf2b3548260f7e9f Mon Sep 17 00:00:00 2001 From: Chengjie Wang Date: Fri, 10 Jul 2026 17:51:08 +0800 Subject: [PATCH 03/14] refactor(onboard): keep recovery decision in handler Signed-off-by: Chengjie Wang --- src/lib/onboard.ts | 4 +-- .../onboard/machine/core-flow-phases.test.ts | 1 + src/lib/onboard/machine/core-flow-phases.ts | 2 -- ...ovider-inference-route-containment.test.ts | 1 + .../provider-inference.test-support.ts | 1 + .../handlers/provider-inference.test.ts | 1 - .../machine/handlers/provider-inference.ts | 12 ++++++--- src/lib/onboard/provider-recovery.test.ts | 25 +++++++++++++++++-- src/lib/onboard/provider-recovery.ts | 1 + 9 files changed, 37 insertions(+), 11 deletions(-) diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 04ab0c2158..1e83c913d8 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -4420,14 +4420,11 @@ async function runOnboard(opts: OnboardOptions = {}): Promise { // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. const coreFlowContext: InitialOnboardFlowContext = { ...initialContext, session, sandboxName, selectedMessagingChannels, gpu, sandboxGpuConfig, gpuPassthrough }; // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. - const allowRecordedProviderRecovery = providerRecovery.shouldRecoverRecordedProvider({ fresh, resume, sandboxName, hasRegisteredSandbox: Boolean(sandboxName && registry.getSandbox(sandboxName)), sessionSandboxName: session?.sandboxName ?? null }); - // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. const runCoreGatewayOpenshell = setupInferenceFactory.createGatewayScopedOpenshellRunner(runOpenshell, GATEWAY_NAME); const [providerInferencePhase, sandboxPhase] = createCoreOnboardFlowPhases({ gatewayName: GATEWAY_NAME, forceProviderSelection: forceProviderSelectionForAgentChange, - allowRecordedProviderRecovery, ...authoritativeRebuildTarget.rebuildProviderFlowOptions(opts, coreFlowContext), env: process.env, constants: { @@ -4438,6 +4435,7 @@ async function runOnboard(opts: OnboardOptions = {}): Promise { providerDeps: { checkGatewayRouteCompatibility, preflightGatewayRouteDiscovery, + hasRegisteredSandbox: (name) => Boolean(registry.getSandbox(name)), withGatewayRouteMutationLock: gatewayRouteMutationLock.withGatewayRouteMutationLock, normalizeHermesAuthMethod, setupNim: (g, s, a, recover, gateway, assertRouteCompatible, canProbeRoute) => diff --git a/src/lib/onboard/machine/core-flow-phases.test.ts b/src/lib/onboard/machine/core-flow-phases.test.ts index 7fc4b99ee4..60377f53e5 100644 --- a/src/lib/onboard/machine/core-flow-phases.test.ts +++ b/src/lib/onboard/machine/core-flow-phases.test.ts @@ -91,6 +91,7 @@ function createPhases( requiredEndpointUrl: null, requiredInferenceApi: null, }), + hasRegisteredSandbox: () => false, withGatewayRouteMutationLock: async ( _gatewayName: string, operation: () => Promise | T, diff --git a/src/lib/onboard/machine/core-flow-phases.ts b/src/lib/onboard/machine/core-flow-phases.ts index 6a73d15ae8..957807b4d3 100644 --- a/src/lib/onboard/machine/core-flow-phases.ts +++ b/src/lib/onboard/machine/core-flow-phases.ts @@ -28,7 +28,6 @@ export interface CoreOnboardFlowPhaseOptions< > { gatewayName: string; forceProviderSelection: boolean; - allowRecordedProviderRecovery?: boolean; forceInferenceSetup?: boolean; authoritativeResumeConfig?: boolean; env: NodeJS.ProcessEnv; @@ -70,7 +69,6 @@ export function createCoreOnboardFlowPhases< sandboxName: context.sandboxName, agent: context.agent, forceProviderSelection: options.forceProviderSelection, - allowRecordedProviderRecovery: options.allowRecordedProviderRecovery, forceInferenceSetup: options.forceInferenceSetup, authoritativeResumeConfig: options.authoritativeResumeConfig, initial: { diff --git a/src/lib/onboard/machine/handlers/provider-inference-route-containment.test.ts b/src/lib/onboard/machine/handlers/provider-inference-route-containment.test.ts index 994add89ef..e713934c85 100644 --- a/src/lib/onboard/machine/handlers/provider-inference-route-containment.test.ts +++ b/src/lib/onboard/machine/handlers/provider-inference-route-containment.test.ts @@ -73,6 +73,7 @@ function createDeps() { const deps: Options["deps"] = { checkGatewayRouteCompatibility: calls.checkGatewayRouteCompatibility, preflightGatewayRouteDiscovery: calls.preflightGatewayRouteDiscovery, + hasRegisteredSandbox: () => false, withGatewayRouteMutationLock: async (_gatewayName, operation) => await operation(), normalizeHermesAuthMethod: () => null, setupNim: calls.setupNim, diff --git a/src/lib/onboard/machine/handlers/provider-inference.test-support.ts b/src/lib/onboard/machine/handlers/provider-inference.test-support.ts index dbb9a29488..8529432837 100644 --- a/src/lib/onboard/machine/handlers/provider-inference.test-support.ts +++ b/src/lib/onboard/machine/handlers/provider-inference.test-support.ts @@ -93,6 +93,7 @@ export function createDeps( deps: { checkGatewayRouteCompatibility: calls.checkGatewayRouteCompatibility, preflightGatewayRouteDiscovery: calls.preflightGatewayRouteDiscovery, + hasRegisteredSandbox: () => false, withGatewayRouteMutationLock: async ( _gatewayName: string, operation: () => Promise | T, diff --git a/src/lib/onboard/machine/handlers/provider-inference.test.ts b/src/lib/onboard/machine/handlers/provider-inference.test.ts index 8d78241604..c083258df3 100644 --- a/src/lib/onboard/machine/handlers/provider-inference.test.ts +++ b/src/lib/onboard/machine/handlers/provider-inference.test.ts @@ -320,7 +320,6 @@ describe("handleProviderInferenceState", () => { ...baseOptions(deps), fresh: false, sandboxName: "dc-after", - allowRecordedProviderRecovery: false, }); expect(calls.setupNim).toHaveBeenCalledWith( diff --git a/src/lib/onboard/machine/handlers/provider-inference.ts b/src/lib/onboard/machine/handlers/provider-inference.ts index 043d3ef7d8..a5a537f700 100644 --- a/src/lib/onboard/machine/handlers/provider-inference.ts +++ b/src/lib/onboard/machine/handlers/provider-inference.ts @@ -10,6 +10,7 @@ import type { } from "../../../inference/gateway-route-compatibility"; import type { WebSearchConfig } from "../../../inference/web-search"; import type { HermesAuthMethod, Session, SessionUpdates } from "../../../state/onboard-session"; +import { shouldRecoverRecordedProvider } from "../../provider-recovery"; import { withInferenceTrace, withProviderSelectionTrace } from "../../tracing"; import { advanceTo, type OnboardStateTransitionResult, retryTo } from "../result"; import { @@ -61,7 +62,6 @@ export interface ProviderInferenceStateOptions { sandboxName: string | null; agent: Agent; forceProviderSelection?: boolean; - allowRecordedProviderRecovery?: boolean; /** Force setup for a provider that authoritative rebuild preflight observed missing. */ forceInferenceSetup?: boolean; /** Trust the rebuild-preflighted session selection even if its old step marker is incomplete. */ @@ -88,6 +88,7 @@ export interface ProviderInferenceStateOptions { deps: { checkGatewayRouteCompatibility: CurrentGatewayRouteCompatibilityCheck; preflightGatewayRouteDiscovery: CurrentGatewayRouteDiscoveryPreflight; + hasRegisteredSandbox(sandboxName: string): boolean; withGatewayRouteMutationLock( gatewayName: string, operation: () => Promise | T, @@ -282,7 +283,6 @@ export async function handleProviderInferenceState({ sandboxName, agent, forceProviderSelection: initialForceProviderSelection = false, - allowRecordedProviderRecovery, forceInferenceSetup: initialForceInferenceSetup = false, authoritativeResumeConfig = false, initial, @@ -321,7 +321,13 @@ export async function handleProviderInferenceState({ let reuseGatewayCredentialWithoutLocalKey = false; let endpointPinnedAddresses: string[] | undefined; const effectiveResume = resume && !fresh; - const recoverRecordedProvider = allowRecordedProviderRecovery ?? !fresh; + const recoverRecordedProvider = shouldRecoverRecordedProvider({ + fresh, + resume, + sandboxName, + hasRegisteredSandbox: Boolean(sandboxName && deps.hasRegisteredSandbox(sandboxName)), + sessionSandboxName: session?.sandboxName ?? null, + }); const stateResults: OnboardStateTransitionResult[] = []; const retryStateResults: OnboardStateTransitionResult[] = []; diff --git a/src/lib/onboard/provider-recovery.test.ts b/src/lib/onboard/provider-recovery.test.ts index 321b40ecb5..274451f0bb 100644 --- a/src/lib/onboard/provider-recovery.test.ts +++ b/src/lib/onboard/provider-recovery.test.ts @@ -43,14 +43,25 @@ describe("shouldRecoverRecordedProvider", () => { label: "rejects gateway recovery for a brand-new sandbox", fresh: false, resume: false, + sandboxName: "dc-after", hasRegisteredSandbox: false, sessionSandboxName: null, expected: false, }, + { + label: "allows gateway recovery before an interactive sandbox name is selected", + fresh: false, + resume: false, + sandboxName: null, + hasRegisteredSandbox: false, + sessionSandboxName: null, + expected: true, + }, { label: "allows gateway recovery while resuming", fresh: false, resume: true, + sandboxName: "dc-after", hasRegisteredSandbox: false, sessionSandboxName: null, expected: true, @@ -59,6 +70,7 @@ describe("shouldRecoverRecordedProvider", () => { label: "allows gateway recovery for a registered sandbox", fresh: false, resume: false, + sandboxName: "dc-after", hasRegisteredSandbox: true, sessionSandboxName: null, expected: true, @@ -67,6 +79,7 @@ describe("shouldRecoverRecordedProvider", () => { label: "allows gateway recovery for a matching session", fresh: false, resume: false, + sandboxName: "dc-after", hasRegisteredSandbox: false, sessionSandboxName: "dc-after", expected: true, @@ -75,16 +88,24 @@ describe("shouldRecoverRecordedProvider", () => { label: "rejects gateway recovery when fresh overrides existing identity", fresh: true, resume: true, + sandboxName: "dc-after", hasRegisteredSandbox: true, sessionSandboxName: "dc-after", expected: false, }, - ])("$label", ({ fresh, resume, hasRegisteredSandbox, sessionSandboxName, expected }) => { + ])("$label", ({ + fresh, + resume, + sandboxName, + hasRegisteredSandbox, + sessionSandboxName, + expected, + }) => { expect( shouldRecoverRecordedProvider({ fresh, resume, - sandboxName: "dc-after", + sandboxName, hasRegisteredSandbox, sessionSandboxName, }), diff --git a/src/lib/onboard/provider-recovery.ts b/src/lib/onboard/provider-recovery.ts index 4d418489ba..47d202e042 100644 --- a/src/lib/onboard/provider-recovery.ts +++ b/src/lib/onboard/provider-recovery.ts @@ -76,6 +76,7 @@ export function shouldRecoverRecordedProvider(input: { return ( !input.fresh && (input.resume || + !input.sandboxName || Boolean( input.sandboxName && (input.hasRegisteredSandbox || input.sessionSandboxName === input.sandboxName), From 81f37c9848349a3ca99c5c0ec4e9541102e96d58 Mon Sep 17 00:00:00 2001 From: Chengjie Wang Date: Fri, 10 Jul 2026 18:06:28 +0800 Subject: [PATCH 04/14] fix(onboard): recompute provider recovery after name selection Signed-off-by: Chengjie Wang --- ...provider-inference-recovery-gating.test.ts | 54 +++++++++++++++++++ .../handlers/provider-inference.test.ts | 43 ++------------- .../machine/handlers/provider-inference.ts | 15 +++--- src/lib/onboard/provider-recovery.test.ts | 9 ++++ 4 files changed, 74 insertions(+), 47 deletions(-) create mode 100644 src/lib/onboard/machine/handlers/provider-inference-recovery-gating.test.ts diff --git a/src/lib/onboard/machine/handlers/provider-inference-recovery-gating.test.ts b/src/lib/onboard/machine/handlers/provider-inference-recovery-gating.test.ts new file mode 100644 index 0000000000..08059ffd1a --- /dev/null +++ b/src/lib/onboard/machine/handlers/provider-inference-recovery-gating.test.ts @@ -0,0 +1,54 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { createSession } from "../../../state/onboard-session"; +import { handleProviderInferenceState } from "./provider-inference"; +import { baseOptions, createDeps } from "./provider-inference.test-support"; + +describe("provider inference recovery gating", () => { + it.each([ + { label: "fresh provider selection", fresh: true, sandboxName: "dcode-station" }, + { label: "brand-new sandbox identity (#6630)", fresh: false, sandboxName: "dc-after" }, + ])("disables recorded provider recovery for $label", async ({ fresh, sandboxName }) => { + const { deps, calls } = createDeps(); + + await handleProviderInferenceState({ + ...baseOptions(deps), + fresh, + sandboxName, + }); + + expect(calls.setupNim).toHaveBeenCalledWith( + { type: "nvidia" }, + sandboxName, + null, + false, + "nemoclaw", + expect.any(Function), + expect.any(Function), + ); + }); + + it("rejects a matching session whose sandbox step is incomplete (#6630)", async () => { + const session = createSession(); + session.sandboxName = "dc-after"; + const { deps, calls } = createDeps(); + + await handleProviderInferenceState({ + ...baseOptions(deps, session), + sandboxName: "dc-after", + }); + + expect(calls.setupNim).toHaveBeenCalledWith( + { type: "nvidia" }, + "dc-after", + null, + false, + "nemoclaw", + expect.any(Function), + expect.any(Function), + ); + }); +}); diff --git a/src/lib/onboard/machine/handlers/provider-inference.test.ts b/src/lib/onboard/machine/handlers/provider-inference.test.ts index c083258df3..f85426161c 100644 --- a/src/lib/onboard/machine/handlers/provider-inference.test.ts +++ b/src/lib/onboard/machine/handlers/provider-inference.test.ts @@ -293,46 +293,6 @@ describe("handleProviderInferenceState", () => { }); }); - it("disables recorded provider recovery during fresh provider selection", async () => { - const { deps, calls } = createDeps(); - - await handleProviderInferenceState({ - ...baseOptions(deps), - fresh: true, - sandboxName: "dcode-station", - }); - - expect(calls.setupNim).toHaveBeenCalledWith( - { type: "nvidia" }, - "dcode-station", - null, - false, - "nemoclaw", - expect.any(Function), - expect.any(Function), - ); - }); - - it("disables recorded provider recovery for a brand-new sandbox identity (#6630)", async () => { - const { deps, calls } = createDeps(); - - await handleProviderInferenceState({ - ...baseOptions(deps), - fresh: false, - sandboxName: "dc-after", - }); - - expect(calls.setupNim).toHaveBeenCalledWith( - { type: "nvidia" }, - "dc-after", - null, - false, - "nemoclaw", - expect.any(Function), - expect.any(Function), - ); - }); - it("does not use resume shortcuts when fresh is also set", async () => { const session = createSession({ provider: "ollama-local", model: "llama3.1" }); session.steps.provider_selection.status = "complete"; @@ -1121,6 +1081,9 @@ describe("handleProviderInferenceState", () => { const result = await handleProviderInferenceState(baseOptions(deps)); expect(setupNim).toHaveBeenCalledTimes(2); + expect(setupNim.mock.calls[0]?.[3]).toBe(true); + expect(setupNim.mock.calls[1]?.[1]).toBe("my-assistant"); + expect(setupNim.mock.calls[1]?.[3]).toBe(false); expect(setupInference).toHaveBeenCalledTimes(2); expect(result.model).toBe("good"); expect(calls.startStep).toHaveBeenCalledWith("provider_selection"); diff --git a/src/lib/onboard/machine/handlers/provider-inference.ts b/src/lib/onboard/machine/handlers/provider-inference.ts index a5a537f700..f78ffb709d 100644 --- a/src/lib/onboard/machine/handlers/provider-inference.ts +++ b/src/lib/onboard/machine/handlers/provider-inference.ts @@ -321,13 +321,6 @@ export async function handleProviderInferenceState({ let reuseGatewayCredentialWithoutLocalKey = false; let endpointPinnedAddresses: string[] | undefined; const effectiveResume = resume && !fresh; - const recoverRecordedProvider = shouldRecoverRecordedProvider({ - fresh, - resume, - sandboxName, - hasRegisteredSandbox: Boolean(sandboxName && deps.hasRegisteredSandbox(sandboxName)), - sessionSandboxName: session?.sandboxName ?? null, - }); const stateResults: OnboardStateTransitionResult[] = []; const retryStateResults: OnboardStateTransitionResult[] = []; @@ -448,6 +441,14 @@ export async function handleProviderInferenceState({ } } else { await deps.startRecordedStep("provider_selection"); + const recoverRecordedProvider = shouldRecoverRecordedProvider({ + fresh, + resume, + sandboxName, + hasRegisteredSandbox: Boolean(sandboxName && deps.hasRegisteredSandbox(sandboxName)), + sessionSandboxName: + session?.steps?.sandbox?.status === "complete" ? (session.sandboxName ?? null) : null, + }); const selection = await withProviderSelectionTrace( sandboxName, (agent as { name?: string } | null)?.name, diff --git a/src/lib/onboard/provider-recovery.test.ts b/src/lib/onboard/provider-recovery.test.ts index 274451f0bb..f0e73c7d32 100644 --- a/src/lib/onboard/provider-recovery.test.ts +++ b/src/lib/onboard/provider-recovery.test.ts @@ -84,6 +84,15 @@ describe("shouldRecoverRecordedProvider", () => { sessionSandboxName: "dc-after", expected: true, }, + { + label: "rejects gateway recovery when an incomplete session identity is not supplied", + fresh: false, + resume: false, + sandboxName: "dc-after", + hasRegisteredSandbox: false, + sessionSandboxName: null, + expected: false, + }, { label: "rejects gateway recovery when fresh overrides existing identity", fresh: true, From c86c42fcad3fdef776406805c130097b6438b598 Mon Sep 17 00:00:00 2001 From: Chengjie Wang Date: Fri, 10 Jul 2026 18:18:08 +0800 Subject: [PATCH 05/14] test(onboard): cover trusted recovery identities Signed-off-by: Chengjie Wang --- src/lib/onboard.ts | 12 ++++-- ...provider-inference-recovery-gating.test.ts | 41 +++++++++++++++++++ src/lib/onboard/provider-recovery.test.ts | 4 +- 3 files changed, 52 insertions(+), 5 deletions(-) diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 1e83c913d8..4b1a40b576 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -4416,9 +4416,15 @@ async function runOnboard(opts: OnboardOptions = {}): Promise { console.error(" Start a fresh onboard with --name to choose a different name."); process.exit(1); } - - // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. - const coreFlowContext: InitialOnboardFlowContext = { ...initialContext, session, sandboxName, selectedMessagingChannels, gpu, sandboxGpuConfig, gpuPassthrough }; + const coreFlowContext: InitialOnboardFlowContext = { + ...initialContext, + session, + sandboxName, + selectedMessagingChannels, + gpu, + sandboxGpuConfig, + gpuPassthrough, + }; // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. const runCoreGatewayOpenshell = setupInferenceFactory.createGatewayScopedOpenshellRunner(runOpenshell, GATEWAY_NAME); const [providerInferencePhase, sandboxPhase] = diff --git a/src/lib/onboard/machine/handlers/provider-inference-recovery-gating.test.ts b/src/lib/onboard/machine/handlers/provider-inference-recovery-gating.test.ts index 08059ffd1a..ae71de06c4 100644 --- a/src/lib/onboard/machine/handlers/provider-inference-recovery-gating.test.ts +++ b/src/lib/onboard/machine/handlers/provider-inference-recovery-gating.test.ts @@ -51,4 +51,45 @@ describe("provider inference recovery gating", () => { expect.any(Function), ); }); + + it("allows recovery for a registered sandbox", async () => { + const { deps, calls } = createDeps({ hasRegisteredSandbox: () => true }); + + await handleProviderInferenceState({ + ...baseOptions(deps), + sandboxName: "dc-after", + }); + + expect(calls.setupNim).toHaveBeenCalledWith( + { type: "nvidia" }, + "dc-after", + null, + true, + "nemoclaw", + expect.any(Function), + expect.any(Function), + ); + }); + + it("allows recovery for a matching completed session sandbox", async () => { + const session = createSession(); + session.sandboxName = "dc-after"; + session.steps.sandbox.status = "complete"; + const { deps, calls } = createDeps(); + + await handleProviderInferenceState({ + ...baseOptions(deps, session), + sandboxName: "dc-after", + }); + + expect(calls.setupNim).toHaveBeenCalledWith( + { type: "nvidia" }, + "dc-after", + null, + true, + "nemoclaw", + expect.any(Function), + expect.any(Function), + ); + }); }); diff --git a/src/lib/onboard/provider-recovery.test.ts b/src/lib/onboard/provider-recovery.test.ts index f0e73c7d32..9287b4ccab 100644 --- a/src/lib/onboard/provider-recovery.test.ts +++ b/src/lib/onboard/provider-recovery.test.ts @@ -85,12 +85,12 @@ describe("shouldRecoverRecordedProvider", () => { expected: true, }, { - label: "rejects gateway recovery when an incomplete session identity is not supplied", + label: "rejects gateway recovery for a different session sandbox", fresh: false, resume: false, sandboxName: "dc-after", hasRegisteredSandbox: false, - sessionSandboxName: null, + sessionSandboxName: "dc-before", expected: false, }, { From ce957f1297782e7efc9f6127585e57e44b388aba Mon Sep 17 00:00:00 2001 From: Chengjie Wang Date: Fri, 10 Jul 2026 18:29:34 +0800 Subject: [PATCH 06/14] test(onboard): assert registered sandbox identity Signed-off-by: Chengjie Wang --- .../handlers/provider-inference-recovery-gating.test.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/lib/onboard/machine/handlers/provider-inference-recovery-gating.test.ts b/src/lib/onboard/machine/handlers/provider-inference-recovery-gating.test.ts index ae71de06c4..457d6eacdc 100644 --- a/src/lib/onboard/machine/handlers/provider-inference-recovery-gating.test.ts +++ b/src/lib/onboard/machine/handlers/provider-inference-recovery-gating.test.ts @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import { createSession } from "../../../state/onboard-session"; import { handleProviderInferenceState } from "./provider-inference"; @@ -53,7 +53,8 @@ describe("provider inference recovery gating", () => { }); it("allows recovery for a registered sandbox", async () => { - const { deps, calls } = createDeps({ hasRegisteredSandbox: () => true }); + const hasRegisteredSandbox = vi.fn((name: string) => name === "dc-after"); + const { deps, calls } = createDeps({ hasRegisteredSandbox }); await handleProviderInferenceState({ ...baseOptions(deps), @@ -69,6 +70,7 @@ describe("provider inference recovery gating", () => { expect.any(Function), expect.any(Function), ); + expect(hasRegisteredSandbox).toHaveBeenCalledWith("dc-after"); }); it("allows recovery for a matching completed session sandbox", async () => { From 6fafe76d654d87c1c6e65cb232e915a9f8852ba2 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Fri, 10 Jul 2026 12:16:48 -0700 Subject: [PATCH 07/14] fix(onboard): scope recovery to owned reservations Signed-off-by: Carlos Villela --- src/lib/onboard.ts | 3 +- .../onboard/machine/core-flow-phases.test.ts | 2 +- ...provider-inference-recovery-gating.test.ts | 75 ++++++++++++++++++- ...ovider-inference-route-containment.test.ts | 2 +- .../provider-inference.test-support.ts | 2 +- .../machine/handlers/provider-inference.ts | 10 ++- src/lib/onboard/provider-recovery.test.ts | 72 +++++++++++------- src/lib/onboard/provider-recovery.ts | 19 +++-- 8 files changed, 145 insertions(+), 40 deletions(-) diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index dfbcc8ffac..c1f05b029e 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -4441,7 +4441,8 @@ async function runOnboard(opts: OnboardOptions = {}): Promise { providerDeps: { checkGatewayRouteCompatibility, preflightGatewayRouteDiscovery, - hasRegisteredSandbox: (name) => Boolean(registry.getSandbox(name)), + hasRecoverableSandboxIdentity: (name, sessionId) => + providerRecovery.hasRecoverableSandboxIdentity(registry.getSandbox(name), sessionId), withGatewayRouteMutationLock: gatewayRouteMutationLock.withGatewayRouteMutationLock, normalizeHermesAuthMethod, setupNim: (g, s, a, recover, gateway, assertRouteCompatible, canProbeRoute) => diff --git a/src/lib/onboard/machine/core-flow-phases.test.ts b/src/lib/onboard/machine/core-flow-phases.test.ts index a713fe2a30..ff7f55b95e 100644 --- a/src/lib/onboard/machine/core-flow-phases.test.ts +++ b/src/lib/onboard/machine/core-flow-phases.test.ts @@ -91,7 +91,7 @@ function createPhases( requiredEndpointUrl: null, requiredInferenceApi: null, }), - hasRegisteredSandbox: () => false, + hasRecoverableSandboxIdentity: () => false, withGatewayRouteMutationLock: async ( _gatewayName: string, operation: () => Promise | T, diff --git a/src/lib/onboard/machine/handlers/provider-inference-recovery-gating.test.ts b/src/lib/onboard/machine/handlers/provider-inference-recovery-gating.test.ts index 457d6eacdc..5db3c43b7e 100644 --- a/src/lib/onboard/machine/handlers/provider-inference-recovery-gating.test.ts +++ b/src/lib/onboard/machine/handlers/provider-inference-recovery-gating.test.ts @@ -4,6 +4,8 @@ import { describe, expect, it, vi } from "vitest"; import { createSession } from "../../../state/onboard-session"; +import type { SandboxEntry } from "../../../state/registry"; +import { hasRecoverableSandboxIdentity } from "../../provider-recovery"; import { handleProviderInferenceState } from "./provider-inference"; import { baseOptions, createDeps } from "./provider-inference.test-support"; @@ -53,8 +55,8 @@ describe("provider inference recovery gating", () => { }); it("allows recovery for a registered sandbox", async () => { - const hasRegisteredSandbox = vi.fn((name: string) => name === "dc-after"); - const { deps, calls } = createDeps({ hasRegisteredSandbox }); + const hasIdentity = vi.fn((name: string) => name === "dc-after"); + const { deps, calls } = createDeps({ hasRecoverableSandboxIdentity: hasIdentity }); await handleProviderInferenceState({ ...baseOptions(deps), @@ -70,7 +72,74 @@ describe("provider inference recovery gating", () => { expect.any(Function), expect.any(Function), ); - expect(hasRegisteredSandbox).toHaveBeenCalledWith("dc-after"); + expect(hasIdentity).toHaveBeenCalledWith("dc-after", expect.any(String)); + }); + + it.each([ + { label: "orphaned", reservationSessionId: undefined }, + { label: "owned by another session", reservationSessionId: "session-other" }, + ])("rejects an $label pending reservation during resume (#6630)", async ({ + reservationSessionId, + }) => { + const session = createSession(); + session.sandboxName = "dc-after"; + const entry: SandboxEntry = { + name: "dc-after", + pendingRouteReservation: true, + ...(reservationSessionId ? { reservationSessionId } : {}), + }; + const hasIdentity = vi.fn((_name: string, sessionId: string | null | undefined) => + hasRecoverableSandboxIdentity(entry, sessionId), + ); + const { deps, calls } = createDeps({ hasRecoverableSandboxIdentity: hasIdentity }); + + await handleProviderInferenceState({ + ...baseOptions(deps, session), + resume: true, + sandboxName: "dc-after", + }); + + expect(calls.setupNim).toHaveBeenCalledWith( + { type: "nvidia" }, + "dc-after", + null, + false, + "nemoclaw", + expect.any(Function), + expect.any(Function), + ); + expect(hasIdentity).toHaveBeenCalledWith("dc-after", session.sessionId); + }); + + it("allows recovery for the current session's pending reservation (#6630)", async () => { + const session = createSession(); + session.sandboxName = "dc-after"; + const entry: SandboxEntry = { + name: "dc-after", + pendingRouteReservation: true, + reservationSessionId: session.sessionId, + }; + const hasIdentity = vi.fn((_name: string, sessionId: string | null | undefined) => + hasRecoverableSandboxIdentity(entry, sessionId), + ); + const { deps, calls } = createDeps({ hasRecoverableSandboxIdentity: hasIdentity }); + + await handleProviderInferenceState({ + ...baseOptions(deps, session), + resume: true, + sandboxName: "dc-after", + }); + + expect(calls.setupNim).toHaveBeenCalledWith( + { type: "nvidia" }, + "dc-after", + null, + true, + "nemoclaw", + expect.any(Function), + expect.any(Function), + ); + expect(hasIdentity).toHaveBeenCalledWith("dc-after", session.sessionId); }); it("allows recovery for a matching completed session sandbox", async () => { diff --git a/src/lib/onboard/machine/handlers/provider-inference-route-containment.test.ts b/src/lib/onboard/machine/handlers/provider-inference-route-containment.test.ts index e713934c85..21917eff2e 100644 --- a/src/lib/onboard/machine/handlers/provider-inference-route-containment.test.ts +++ b/src/lib/onboard/machine/handlers/provider-inference-route-containment.test.ts @@ -73,7 +73,7 @@ function createDeps() { const deps: Options["deps"] = { checkGatewayRouteCompatibility: calls.checkGatewayRouteCompatibility, preflightGatewayRouteDiscovery: calls.preflightGatewayRouteDiscovery, - hasRegisteredSandbox: () => false, + hasRecoverableSandboxIdentity: () => false, withGatewayRouteMutationLock: async (_gatewayName, operation) => await operation(), normalizeHermesAuthMethod: () => null, setupNim: calls.setupNim, diff --git a/src/lib/onboard/machine/handlers/provider-inference.test-support.ts b/src/lib/onboard/machine/handlers/provider-inference.test-support.ts index 8529432837..87214cef6c 100644 --- a/src/lib/onboard/machine/handlers/provider-inference.test-support.ts +++ b/src/lib/onboard/machine/handlers/provider-inference.test-support.ts @@ -93,7 +93,7 @@ export function createDeps( deps: { checkGatewayRouteCompatibility: calls.checkGatewayRouteCompatibility, preflightGatewayRouteDiscovery: calls.preflightGatewayRouteDiscovery, - hasRegisteredSandbox: () => false, + hasRecoverableSandboxIdentity: () => false, withGatewayRouteMutationLock: async ( _gatewayName: string, operation: () => Promise | T, diff --git a/src/lib/onboard/machine/handlers/provider-inference.ts b/src/lib/onboard/machine/handlers/provider-inference.ts index 727383054c..bd4545b09d 100644 --- a/src/lib/onboard/machine/handlers/provider-inference.ts +++ b/src/lib/onboard/machine/handlers/provider-inference.ts @@ -90,7 +90,10 @@ export interface ProviderInferenceStateOptions { deps: { checkGatewayRouteCompatibility: CurrentGatewayRouteCompatibilityCheck; preflightGatewayRouteDiscovery: CurrentGatewayRouteDiscoveryPreflight; - hasRegisteredSandbox(sandboxName: string): boolean; + hasRecoverableSandboxIdentity( + sandboxName: string, + sessionId: string | null | undefined, + ): boolean; withGatewayRouteMutationLock( gatewayName: string, operation: () => Promise | T, @@ -446,9 +449,10 @@ export async function handleProviderInferenceState({ await deps.startRecordedStep("provider_selection"); const recoverRecordedProvider = shouldRecoverRecordedProvider({ fresh, - resume, sandboxName, - hasRegisteredSandbox: Boolean(sandboxName && deps.hasRegisteredSandbox(sandboxName)), + hasRecoverableSandboxIdentity: Boolean( + sandboxName && deps.hasRecoverableSandboxIdentity(sandboxName, session?.sessionId), + ), sessionSandboxName: session?.steps?.sandbox?.status === "complete" ? (session.sandboxName ?? null) : null, }); diff --git a/src/lib/onboard/provider-recovery.test.ts b/src/lib/onboard/provider-recovery.test.ts index 9287b4ccab..4e735af2da 100644 --- a/src/lib/onboard/provider-recovery.test.ts +++ b/src/lib/onboard/provider-recovery.test.ts @@ -7,6 +7,7 @@ import * as onboardSession from "../state/onboard-session"; import * as registry from "../state/registry"; import { createProviderRecoveryHelpers, + hasRecoverableSandboxIdentity, shouldRecoverRecordedProvider, validateLiveGatewayInference, } from "./provider-recovery"; @@ -42,86 +43,107 @@ describe("shouldRecoverRecordedProvider", () => { { label: "rejects gateway recovery for a brand-new sandbox", fresh: false, - resume: false, sandboxName: "dc-after", - hasRegisteredSandbox: false, + hasRecoverableSandboxIdentity: false, sessionSandboxName: null, expected: false, }, { label: "allows gateway recovery before an interactive sandbox name is selected", fresh: false, - resume: false, sandboxName: null, - hasRegisteredSandbox: false, - sessionSandboxName: null, - expected: true, - }, - { - label: "allows gateway recovery while resuming", - fresh: false, - resume: true, - sandboxName: "dc-after", - hasRegisteredSandbox: false, + hasRecoverableSandboxIdentity: false, sessionSandboxName: null, expected: true, }, { label: "allows gateway recovery for a registered sandbox", fresh: false, - resume: false, sandboxName: "dc-after", - hasRegisteredSandbox: true, + hasRecoverableSandboxIdentity: true, sessionSandboxName: null, expected: true, }, { label: "allows gateway recovery for a matching session", fresh: false, - resume: false, sandboxName: "dc-after", - hasRegisteredSandbox: false, + hasRecoverableSandboxIdentity: false, sessionSandboxName: "dc-after", expected: true, }, { label: "rejects gateway recovery for a different session sandbox", fresh: false, - resume: false, sandboxName: "dc-after", - hasRegisteredSandbox: false, + hasRecoverableSandboxIdentity: false, sessionSandboxName: "dc-before", expected: false, }, { label: "rejects gateway recovery when fresh overrides existing identity", fresh: true, - resume: true, sandboxName: "dc-after", - hasRegisteredSandbox: true, + hasRecoverableSandboxIdentity: true, sessionSandboxName: "dc-after", expected: false, }, ])("$label", ({ fresh, - resume, sandboxName, - hasRegisteredSandbox, + hasRecoverableSandboxIdentity, sessionSandboxName, expected, }) => { expect( shouldRecoverRecordedProvider({ fresh, - resume, sandboxName, - hasRegisteredSandbox, + hasRecoverableSandboxIdentity, sessionSandboxName, }), ).toBe(expected); }); }); +describe("hasRecoverableSandboxIdentity", () => { + const pending = (reservationSessionId?: string): registry.SandboxEntry => ({ + name: "dc-after", + pendingRouteReservation: true, + ...(reservationSessionId ? { reservationSessionId } : {}), + }); + + it.each([ + { label: "missing registry row", entry: null, sessionId: "session-current", expected: false }, + { + label: "orphaned pending reservation", + entry: pending(), + sessionId: "session-current", + expected: false, + }, + { + label: "another session's pending reservation", + entry: pending("session-other"), + sessionId: "session-current", + expected: false, + }, + { + label: "the current session's pending reservation", + entry: pending("session-current"), + sessionId: "session-current", + expected: true, + }, + { + label: "fully registered sandbox", + entry: { name: "dc-after" }, + sessionId: null, + expected: true, + }, + ])("classifies $label (#6630)", ({ entry, sessionId, expected }) => { + expect(hasRecoverableSandboxIdentity(entry, sessionId)).toBe(expected); + }); +}); + describe("provider recovery persisted routing state", () => { function helpers() { return createProviderRecoveryHelpers({ diff --git a/src/lib/onboard/provider-recovery.ts b/src/lib/onboard/provider-recovery.ts index 47d202e042..d1ffeda4af 100644 --- a/src/lib/onboard/provider-recovery.ts +++ b/src/lib/onboard/provider-recovery.ts @@ -66,20 +66,29 @@ const MAX_LIVE_PROVIDER_LENGTH = 128; const MAX_LIVE_MODEL_LENGTH = 512; const SAFE_LIVE_PROVIDER = /^[A-Za-z0-9._:-]+$/; +export function hasRecoverableSandboxIdentity( + entry: registry.SandboxEntry | null, + sessionId: string | null | undefined, +): boolean { + return Boolean( + entry && + (entry.pendingRouteReservation !== true || + registry.isPendingReservationForSession(entry, sessionId)), + ); +} + export function shouldRecoverRecordedProvider(input: { fresh: boolean; - resume: boolean; sandboxName: string | null; - hasRegisteredSandbox: boolean; + hasRecoverableSandboxIdentity: boolean; sessionSandboxName: string | null; }): boolean { return ( !input.fresh && - (input.resume || - !input.sandboxName || + (!input.sandboxName || Boolean( input.sandboxName && - (input.hasRegisteredSandbox || input.sessionSandboxName === input.sandboxName), + (input.hasRecoverableSandboxIdentity || input.sessionSandboxName === input.sandboxName), )) ); } From e5fb9f5d798f21de4ae1437ba9c4abb2fce716d1 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Fri, 10 Jul 2026 12:19:48 -0700 Subject: [PATCH 08/14] refactor(onboard): isolate recovery identity lookup Signed-off-by: Carlos Villela --- src/lib/onboard.ts | 3 +-- .../provider-inference-recovery-gating.test.ts | 6 +++--- src/lib/onboard/provider-recovery.test.ts | 12 ++++++++++-- src/lib/onboard/provider-recovery.ts | 9 ++++++++- 4 files changed, 22 insertions(+), 8 deletions(-) diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index c1f05b029e..7d9fe843b9 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -4441,8 +4441,7 @@ async function runOnboard(opts: OnboardOptions = {}): Promise { providerDeps: { checkGatewayRouteCompatibility, preflightGatewayRouteDiscovery, - hasRecoverableSandboxIdentity: (name, sessionId) => - providerRecovery.hasRecoverableSandboxIdentity(registry.getSandbox(name), sessionId), + hasRecoverableSandboxIdentity: providerRecovery.hasRecoverableSandboxIdentity, withGatewayRouteMutationLock: gatewayRouteMutationLock.withGatewayRouteMutationLock, normalizeHermesAuthMethod, setupNim: (g, s, a, recover, gateway, assertRouteCompatible, canProbeRoute) => diff --git a/src/lib/onboard/machine/handlers/provider-inference-recovery-gating.test.ts b/src/lib/onboard/machine/handlers/provider-inference-recovery-gating.test.ts index 5db3c43b7e..73cf61a8c1 100644 --- a/src/lib/onboard/machine/handlers/provider-inference-recovery-gating.test.ts +++ b/src/lib/onboard/machine/handlers/provider-inference-recovery-gating.test.ts @@ -5,7 +5,7 @@ import { describe, expect, it, vi } from "vitest"; import { createSession } from "../../../state/onboard-session"; import type { SandboxEntry } from "../../../state/registry"; -import { hasRecoverableSandboxIdentity } from "../../provider-recovery"; +import { isRecoverableSandboxIdentity } from "../../provider-recovery"; import { handleProviderInferenceState } from "./provider-inference"; import { baseOptions, createDeps } from "./provider-inference.test-support"; @@ -89,7 +89,7 @@ describe("provider inference recovery gating", () => { ...(reservationSessionId ? { reservationSessionId } : {}), }; const hasIdentity = vi.fn((_name: string, sessionId: string | null | undefined) => - hasRecoverableSandboxIdentity(entry, sessionId), + isRecoverableSandboxIdentity(entry, sessionId), ); const { deps, calls } = createDeps({ hasRecoverableSandboxIdentity: hasIdentity }); @@ -120,7 +120,7 @@ describe("provider inference recovery gating", () => { reservationSessionId: session.sessionId, }; const hasIdentity = vi.fn((_name: string, sessionId: string | null | undefined) => - hasRecoverableSandboxIdentity(entry, sessionId), + isRecoverableSandboxIdentity(entry, sessionId), ); const { deps, calls } = createDeps({ hasRecoverableSandboxIdentity: hasIdentity }); diff --git a/src/lib/onboard/provider-recovery.test.ts b/src/lib/onboard/provider-recovery.test.ts index 4e735af2da..0dbfab2c5a 100644 --- a/src/lib/onboard/provider-recovery.test.ts +++ b/src/lib/onboard/provider-recovery.test.ts @@ -8,6 +8,7 @@ import * as registry from "../state/registry"; import { createProviderRecoveryHelpers, hasRecoverableSandboxIdentity, + isRecoverableSandboxIdentity, shouldRecoverRecordedProvider, validateLiveGatewayInference, } from "./provider-recovery"; @@ -106,7 +107,7 @@ describe("shouldRecoverRecordedProvider", () => { }); }); -describe("hasRecoverableSandboxIdentity", () => { +describe("recoverable sandbox identity", () => { const pending = (reservationSessionId?: string): registry.SandboxEntry => ({ name: "dc-after", pendingRouteReservation: true, @@ -140,7 +141,14 @@ describe("hasRecoverableSandboxIdentity", () => { expected: true, }, ])("classifies $label (#6630)", ({ entry, sessionId, expected }) => { - expect(hasRecoverableSandboxIdentity(entry, sessionId)).toBe(expected); + expect(isRecoverableSandboxIdentity(entry, sessionId)).toBe(expected); + }); + + it("loads the named registry row before applying session ownership (#6630)", () => { + vi.spyOn(registry, "getSandbox").mockReturnValue(pending("session-current")); + + expect(hasRecoverableSandboxIdentity("dc-after", "session-current")).toBe(true); + expect(registry.getSandbox).toHaveBeenCalledWith("dc-after"); }); }); diff --git a/src/lib/onboard/provider-recovery.ts b/src/lib/onboard/provider-recovery.ts index d1ffeda4af..c501c24ae8 100644 --- a/src/lib/onboard/provider-recovery.ts +++ b/src/lib/onboard/provider-recovery.ts @@ -66,7 +66,7 @@ const MAX_LIVE_PROVIDER_LENGTH = 128; const MAX_LIVE_MODEL_LENGTH = 512; const SAFE_LIVE_PROVIDER = /^[A-Za-z0-9._:-]+$/; -export function hasRecoverableSandboxIdentity( +export function isRecoverableSandboxIdentity( entry: registry.SandboxEntry | null, sessionId: string | null | undefined, ): boolean { @@ -77,6 +77,13 @@ export function hasRecoverableSandboxIdentity( ); } +export function hasRecoverableSandboxIdentity( + sandboxName: string, + sessionId: string | null | undefined, +): boolean { + return isRecoverableSandboxIdentity(registry.getSandbox(sandboxName), sessionId); +} + export function shouldRecoverRecordedProvider(input: { fresh: boolean; sandboxName: string | null; From 152286d039e47143148b1b02c18ef97f075a03e9 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Fri, 10 Jul 2026 12:29:35 -0700 Subject: [PATCH 09/14] test(onboard): compose reservation recovery gating Signed-off-by: Carlos Villela --- ...provider-inference-recovery-gating.test.ts | 70 +++++++++++++++++++ .../handlers/provider-inference.test.ts | 23 +++++- src/lib/onboard/provider-recovery.test.ts | 9 +++ 3 files changed, 99 insertions(+), 3 deletions(-) diff --git a/src/lib/onboard/machine/handlers/provider-inference-recovery-gating.test.ts b/src/lib/onboard/machine/handlers/provider-inference-recovery-gating.test.ts index 73cf61a8c1..8a6264a6b3 100644 --- a/src/lib/onboard/machine/handlers/provider-inference-recovery-gating.test.ts +++ b/src/lib/onboard/machine/handlers/provider-inference-recovery-gating.test.ts @@ -1,6 +1,9 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; import { describe, expect, it, vi } from "vitest"; import { createSession } from "../../../state/onboard-session"; @@ -142,6 +145,73 @@ describe("provider inference recovery gating", () => { expect(hasIdentity).toHaveBeenCalledWith("dc-after", session.sessionId); }); + it("composes persisted route reservation ownership with resume recovery (#6626, #6630)", async () => { + const home = await fs.mkdtemp(path.join(os.tmpdir(), "nemoclaw-recovery-reservation-")); + vi.stubEnv("HOME", home); + vi.resetModules(); + try { + const registry = await import("../../../state/registry"); + const recovery = await import("../../provider-recovery"); + const sessionId = "session-current"; + const route = { + provider: "compatible-endpoint", + model: "model-a", + endpointUrl: "https://api.example.test/v1", + credentialEnv: "CUSTOM_API_KEY", + preferredInferenceApi: "openai-responses", + gatewayName: "nemoclaw", + }; + + for (const { sandboxName, reservationSessionId, expectedRecovery } of [ + { + sandboxName: "owned-reservation", + reservationSessionId: sessionId, + expectedRecovery: true, + }, + { + sandboxName: "foreign-reservation", + reservationSessionId: "session-other", + expectedRecovery: false, + }, + ]) { + registry.reserveSandboxInferenceRoute(sandboxName, { + ...route, + reservationSessionId, + }); + expect(registry.getSandbox(sandboxName)).toMatchObject({ + pendingRouteReservation: true, + reservationSessionId, + }); + + const session = createSession({ sessionId }); + session.sandboxName = sandboxName; + const { deps, calls } = createDeps({ + hasRecoverableSandboxIdentity: recovery.hasRecoverableSandboxIdentity, + }); + + await handleProviderInferenceState({ + ...baseOptions(deps, session), + resume: true, + sandboxName, + }); + + expect(calls.setupNim).toHaveBeenCalledWith( + { type: "nvidia" }, + sandboxName, + null, + expectedRecovery, + "nemoclaw", + expect.any(Function), + expect.any(Function), + ); + } + } finally { + vi.unstubAllEnvs(); + vi.resetModules(); + await fs.rm(home, { recursive: true, force: true }); + } + }); + it("allows recovery for a matching completed session sandbox", async () => { const session = createSession(); session.sandboxName = "dc-after"; diff --git a/src/lib/onboard/machine/handlers/provider-inference.test.ts b/src/lib/onboard/machine/handlers/provider-inference.test.ts index 215fc2fd46..52dde72fb7 100644 --- a/src/lib/onboard/machine/handlers/provider-inference.test.ts +++ b/src/lib/onboard/machine/handlers/provider-inference.test.ts @@ -1119,9 +1119,26 @@ describe("handleProviderInferenceState", () => { const result = await handleProviderInferenceState(baseOptions(deps)); expect(setupNim).toHaveBeenCalledTimes(2); - expect(setupNim.mock.calls[0]?.[3]).toBe(true); - expect(setupNim.mock.calls[1]?.[1]).toBe("my-assistant"); - expect(setupNim.mock.calls[1]?.[3]).toBe(false); + expect(setupNim).toHaveBeenNthCalledWith( + 1, + { type: "nvidia" }, + null, + null, + true, + "nemoclaw", + expect.any(Function), + expect.any(Function), + ); + expect(setupNim).toHaveBeenNthCalledWith( + 2, + { type: "nvidia" }, + "my-assistant", + null, + false, + "nemoclaw", + expect.any(Function), + expect.any(Function), + ); expect(setupInference).toHaveBeenCalledTimes(2); expect(result.model).toBe("good"); expect(calls.startStep).toHaveBeenCalledWith("provider_selection"); diff --git a/src/lib/onboard/provider-recovery.test.ts b/src/lib/onboard/provider-recovery.test.ts index 0dbfab2c5a..5ff2a5aa6d 100644 --- a/src/lib/onboard/provider-recovery.test.ts +++ b/src/lib/onboard/provider-recovery.test.ts @@ -150,6 +150,15 @@ describe("recoverable sandbox identity", () => { expect(hasRecoverableSandboxIdentity("dc-after", "session-current")).toBe(true); expect(registry.getSandbox).toHaveBeenCalledWith("dc-after"); }); + + it("short-circuits ownership checks when the named registry row is missing (#6630)", () => { + vi.spyOn(registry, "getSandbox").mockReturnValue(null); + const isOwned = vi.spyOn(registry, "isPendingReservationForSession"); + + expect(hasRecoverableSandboxIdentity("missing-sandbox", "session-current")).toBe(false); + expect(registry.getSandbox).toHaveBeenCalledWith("missing-sandbox"); + expect(isOwned).not.toHaveBeenCalled(); + }); }); describe("provider recovery persisted routing state", () => { From 73dc06f59cc74f09be2bcf26ddaf1ed77de0da10 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Fri, 10 Jul 2026 13:59:47 -0700 Subject: [PATCH 10/14] fix(onboard): enforce reservation recovery ownership Signed-off-by: Carlos Villela --- src/lib/onboard.ts | 2 +- .../onboard/machine/core-flow-phases.test.ts | 2 +- ...provider-inference-recovery-gating.test.ts | 40 +++--- ...ovider-inference-route-containment.test.ts | 2 +- .../provider-inference.test-support.ts | 2 +- .../machine/handlers/provider-inference.ts | 15 ++- src/lib/onboard/provider-recovery.test.ts | 114 ++++++++++++++---- src/lib/onboard/provider-recovery.ts | 64 +++++++--- 8 files changed, 175 insertions(+), 66 deletions(-) diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 7d9fe843b9..55ac0a3b07 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -4441,7 +4441,7 @@ async function runOnboard(opts: OnboardOptions = {}): Promise { providerDeps: { checkGatewayRouteCompatibility, preflightGatewayRouteDiscovery, - hasRecoverableSandboxIdentity: providerRecovery.hasRecoverableSandboxIdentity, + getSandboxRecoveryAuthority: providerRecovery.getSandboxRecoveryAuthority, withGatewayRouteMutationLock: gatewayRouteMutationLock.withGatewayRouteMutationLock, normalizeHermesAuthMethod, setupNim: (g, s, a, recover, gateway, assertRouteCompatible, canProbeRoute) => diff --git a/src/lib/onboard/machine/core-flow-phases.test.ts b/src/lib/onboard/machine/core-flow-phases.test.ts index ff7f55b95e..8a3d100b4b 100644 --- a/src/lib/onboard/machine/core-flow-phases.test.ts +++ b/src/lib/onboard/machine/core-flow-phases.test.ts @@ -91,7 +91,7 @@ function createPhases( requiredEndpointUrl: null, requiredInferenceApi: null, }), - hasRecoverableSandboxIdentity: () => false, + getSandboxRecoveryAuthority: (): "missing" => "missing", withGatewayRouteMutationLock: async ( _gatewayName: string, operation: () => Promise | T, diff --git a/src/lib/onboard/machine/handlers/provider-inference-recovery-gating.test.ts b/src/lib/onboard/machine/handlers/provider-inference-recovery-gating.test.ts index 8a6264a6b3..a28ca4b9d7 100644 --- a/src/lib/onboard/machine/handlers/provider-inference-recovery-gating.test.ts +++ b/src/lib/onboard/machine/handlers/provider-inference-recovery-gating.test.ts @@ -8,7 +8,7 @@ import { describe, expect, it, vi } from "vitest"; import { createSession } from "../../../state/onboard-session"; import type { SandboxEntry } from "../../../state/registry"; -import { isRecoverableSandboxIdentity } from "../../provider-recovery"; +import { classifySandboxRecoveryAuthority } from "../../provider-recovery"; import { handleProviderInferenceState } from "./provider-inference"; import { baseOptions, createDeps } from "./provider-inference.test-support"; @@ -58,8 +58,10 @@ describe("provider inference recovery gating", () => { }); it("allows recovery for a registered sandbox", async () => { - const hasIdentity = vi.fn((name: string) => name === "dc-after"); - const { deps, calls } = createDeps({ hasRecoverableSandboxIdentity: hasIdentity }); + const getAuthority = vi.fn((name: string) => + name === "dc-after" ? ("authorized" as const) : ("missing" as const), + ); + const { deps, calls } = createDeps({ getSandboxRecoveryAuthority: getAuthority }); await handleProviderInferenceState({ ...baseOptions(deps), @@ -75,26 +77,27 @@ describe("provider inference recovery gating", () => { expect.any(Function), expect.any(Function), ); - expect(hasIdentity).toHaveBeenCalledWith("dc-after", expect.any(String)); + expect(getAuthority).toHaveBeenCalledWith("dc-after", expect.any(String)); }); it.each([ { label: "orphaned", reservationSessionId: undefined }, { label: "owned by another session", reservationSessionId: "session-other" }, - ])("rejects an $label pending reservation during resume (#6630)", async ({ + ])("rejects an $label pending reservation despite a completed session (#6630)", async ({ reservationSessionId, }) => { const session = createSession(); session.sandboxName = "dc-after"; + session.steps.sandbox.status = "complete"; const entry: SandboxEntry = { name: "dc-after", pendingRouteReservation: true, ...(reservationSessionId ? { reservationSessionId } : {}), }; - const hasIdentity = vi.fn((_name: string, sessionId: string | null | undefined) => - isRecoverableSandboxIdentity(entry, sessionId), + const getAuthority = vi.fn((_name: string, sessionId: string | null | undefined) => + classifySandboxRecoveryAuthority(entry, sessionId), ); - const { deps, calls } = createDeps({ hasRecoverableSandboxIdentity: hasIdentity }); + const { deps, calls } = createDeps({ getSandboxRecoveryAuthority: getAuthority }); await handleProviderInferenceState({ ...baseOptions(deps, session), @@ -111,21 +114,22 @@ describe("provider inference recovery gating", () => { expect.any(Function), expect.any(Function), ); - expect(hasIdentity).toHaveBeenCalledWith("dc-after", session.sessionId); + expect(getAuthority).toHaveBeenCalledWith("dc-after", session.sessionId); }); it("allows recovery for the current session's pending reservation (#6630)", async () => { const session = createSession(); session.sandboxName = "dc-after"; + session.steps.sandbox.status = "complete"; const entry: SandboxEntry = { name: "dc-after", pendingRouteReservation: true, reservationSessionId: session.sessionId, }; - const hasIdentity = vi.fn((_name: string, sessionId: string | null | undefined) => - isRecoverableSandboxIdentity(entry, sessionId), + const getAuthority = vi.fn((_name: string, sessionId: string | null | undefined) => + classifySandboxRecoveryAuthority(entry, sessionId), ); - const { deps, calls } = createDeps({ hasRecoverableSandboxIdentity: hasIdentity }); + const { deps, calls } = createDeps({ getSandboxRecoveryAuthority: getAuthority }); await handleProviderInferenceState({ ...baseOptions(deps, session), @@ -142,7 +146,7 @@ describe("provider inference recovery gating", () => { expect.any(Function), expect.any(Function), ); - expect(hasIdentity).toHaveBeenCalledWith("dc-after", session.sessionId); + expect(getAuthority).toHaveBeenCalledWith("dc-after", session.sessionId); }); it("composes persisted route reservation ownership with resume recovery (#6626, #6630)", async () => { @@ -173,6 +177,11 @@ describe("provider inference recovery gating", () => { reservationSessionId: "session-other", expectedRecovery: false, }, + { + sandboxName: "ownerless-reservation", + reservationSessionId: undefined, + expectedRecovery: false, + }, ]) { registry.reserveSandboxInferenceRoute(sandboxName, { ...route, @@ -180,13 +189,14 @@ describe("provider inference recovery gating", () => { }); expect(registry.getSandbox(sandboxName)).toMatchObject({ pendingRouteReservation: true, - reservationSessionId, + ...(reservationSessionId ? { reservationSessionId } : {}), }); const session = createSession({ sessionId }); session.sandboxName = sandboxName; + session.steps.sandbox.status = "complete"; const { deps, calls } = createDeps({ - hasRecoverableSandboxIdentity: recovery.hasRecoverableSandboxIdentity, + getSandboxRecoveryAuthority: recovery.getSandboxRecoveryAuthority, }); await handleProviderInferenceState({ diff --git a/src/lib/onboard/machine/handlers/provider-inference-route-containment.test.ts b/src/lib/onboard/machine/handlers/provider-inference-route-containment.test.ts index 21917eff2e..3c330b4c19 100644 --- a/src/lib/onboard/machine/handlers/provider-inference-route-containment.test.ts +++ b/src/lib/onboard/machine/handlers/provider-inference-route-containment.test.ts @@ -73,7 +73,7 @@ function createDeps() { const deps: Options["deps"] = { checkGatewayRouteCompatibility: calls.checkGatewayRouteCompatibility, preflightGatewayRouteDiscovery: calls.preflightGatewayRouteDiscovery, - hasRecoverableSandboxIdentity: () => false, + getSandboxRecoveryAuthority: (): "missing" => "missing", withGatewayRouteMutationLock: async (_gatewayName, operation) => await operation(), normalizeHermesAuthMethod: () => null, setupNim: calls.setupNim, diff --git a/src/lib/onboard/machine/handlers/provider-inference.test-support.ts b/src/lib/onboard/machine/handlers/provider-inference.test-support.ts index 87214cef6c..97274c5122 100644 --- a/src/lib/onboard/machine/handlers/provider-inference.test-support.ts +++ b/src/lib/onboard/machine/handlers/provider-inference.test-support.ts @@ -93,7 +93,7 @@ export function createDeps( deps: { checkGatewayRouteCompatibility: calls.checkGatewayRouteCompatibility, preflightGatewayRouteDiscovery: calls.preflightGatewayRouteDiscovery, - hasRecoverableSandboxIdentity: () => false, + getSandboxRecoveryAuthority: (): "missing" => "missing", withGatewayRouteMutationLock: async ( _gatewayName: string, operation: () => Promise | T, diff --git a/src/lib/onboard/machine/handlers/provider-inference.ts b/src/lib/onboard/machine/handlers/provider-inference.ts index bd4545b09d..143d62ace8 100644 --- a/src/lib/onboard/machine/handlers/provider-inference.ts +++ b/src/lib/onboard/machine/handlers/provider-inference.ts @@ -10,7 +10,10 @@ import type { } from "../../../inference/gateway-route-compatibility"; import type { WebSearchConfig } from "../../../inference/web-search"; import type { HermesAuthMethod, Session, SessionUpdates } from "../../../state/onboard-session"; -import { shouldRecoverRecordedProvider } from "../../provider-recovery"; +import { + type SandboxRecoveryAuthority, + shouldRecoverRecordedProvider, +} from "../../provider-recovery"; import { withInferenceTrace, withProviderSelectionTrace } from "../../tracing"; import { advanceTo, type OnboardStateTransitionResult, retryTo } from "../result"; import { @@ -90,10 +93,10 @@ export interface ProviderInferenceStateOptions { deps: { checkGatewayRouteCompatibility: CurrentGatewayRouteCompatibilityCheck; preflightGatewayRouteDiscovery: CurrentGatewayRouteDiscoveryPreflight; - hasRecoverableSandboxIdentity( + getSandboxRecoveryAuthority( sandboxName: string, sessionId: string | null | undefined, - ): boolean; + ): SandboxRecoveryAuthority; withGatewayRouteMutationLock( gatewayName: string, operation: () => Promise | T, @@ -450,9 +453,9 @@ export async function handleProviderInferenceState({ const recoverRecordedProvider = shouldRecoverRecordedProvider({ fresh, sandboxName, - hasRecoverableSandboxIdentity: Boolean( - sandboxName && deps.hasRecoverableSandboxIdentity(sandboxName, session?.sessionId), - ), + sandboxRecoveryAuthority: sandboxName + ? deps.getSandboxRecoveryAuthority(sandboxName, session?.sessionId) + : "missing", sessionSandboxName: session?.steps?.sandbox?.status === "complete" ? (session.sandboxName ?? null) : null, }); diff --git a/src/lib/onboard/provider-recovery.test.ts b/src/lib/onboard/provider-recovery.test.ts index 5ff2a5aa6d..ac8da604d2 100644 --- a/src/lib/onboard/provider-recovery.test.ts +++ b/src/lib/onboard/provider-recovery.test.ts @@ -6,9 +6,9 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import * as onboardSession from "../state/onboard-session"; import * as registry from "../state/registry"; import { + classifySandboxRecoveryAuthority, createProviderRecoveryHelpers, - hasRecoverableSandboxIdentity, - isRecoverableSandboxIdentity, + getSandboxRecoveryAuthority, shouldRecoverRecordedProvider, validateLiveGatewayInference, } from "./provider-recovery"; @@ -45,7 +45,7 @@ describe("shouldRecoverRecordedProvider", () => { label: "rejects gateway recovery for a brand-new sandbox", fresh: false, sandboxName: "dc-after", - hasRecoverableSandboxIdentity: false, + sandboxRecoveryAuthority: "missing", sessionSandboxName: null, expected: false, }, @@ -53,7 +53,7 @@ describe("shouldRecoverRecordedProvider", () => { label: "allows gateway recovery before an interactive sandbox name is selected", fresh: false, sandboxName: null, - hasRecoverableSandboxIdentity: false, + sandboxRecoveryAuthority: "missing", sessionSandboxName: null, expected: true, }, @@ -61,23 +61,31 @@ describe("shouldRecoverRecordedProvider", () => { label: "allows gateway recovery for a registered sandbox", fresh: false, sandboxName: "dc-after", - hasRecoverableSandboxIdentity: true, + sandboxRecoveryAuthority: "authorized", sessionSandboxName: null, expected: true, }, { - label: "allows gateway recovery for a matching session", + label: "allows gateway recovery for a matching session when the registry row is missing", fresh: false, sandboxName: "dc-after", - hasRecoverableSandboxIdentity: false, + sandboxRecoveryAuthority: "missing", sessionSandboxName: "dc-after", expected: true, }, + { + label: "rejects a matching session when the present registry row is unauthorized", + fresh: false, + sandboxName: "dc-after", + sandboxRecoveryAuthority: "unauthorized", + sessionSandboxName: "dc-after", + expected: false, + }, { label: "rejects gateway recovery for a different session sandbox", fresh: false, sandboxName: "dc-after", - hasRecoverableSandboxIdentity: false, + sandboxRecoveryAuthority: "missing", sessionSandboxName: "dc-before", expected: false, }, @@ -85,14 +93,14 @@ describe("shouldRecoverRecordedProvider", () => { label: "rejects gateway recovery when fresh overrides existing identity", fresh: true, sandboxName: "dc-after", - hasRecoverableSandboxIdentity: true, + sandboxRecoveryAuthority: "authorized", sessionSandboxName: "dc-after", expected: false, }, - ])("$label", ({ + ] as const)("$label", ({ fresh, sandboxName, - hasRecoverableSandboxIdentity, + sandboxRecoveryAuthority, sessionSandboxName, expected, }) => { @@ -100,14 +108,14 @@ describe("shouldRecoverRecordedProvider", () => { shouldRecoverRecordedProvider({ fresh, sandboxName, - hasRecoverableSandboxIdentity, + sandboxRecoveryAuthority, sessionSandboxName, }), ).toBe(expected); }); }); -describe("recoverable sandbox identity", () => { +describe("sandbox recovery authority", () => { const pending = (reservationSessionId?: string): registry.SandboxEntry => ({ name: "dc-after", pendingRouteReservation: true, @@ -115,39 +123,44 @@ describe("recoverable sandbox identity", () => { }); it.each([ - { label: "missing registry row", entry: null, sessionId: "session-current", expected: false }, + { + label: "missing registry row", + entry: null, + sessionId: "session-current", + expected: "missing", + }, { label: "orphaned pending reservation", entry: pending(), sessionId: "session-current", - expected: false, + expected: "unauthorized", }, { label: "another session's pending reservation", entry: pending("session-other"), sessionId: "session-current", - expected: false, + expected: "unauthorized", }, { label: "the current session's pending reservation", entry: pending("session-current"), sessionId: "session-current", - expected: true, + expected: "authorized", }, { label: "fully registered sandbox", entry: { name: "dc-after" }, sessionId: null, - expected: true, + expected: "authorized", }, ])("classifies $label (#6630)", ({ entry, sessionId, expected }) => { - expect(isRecoverableSandboxIdentity(entry, sessionId)).toBe(expected); + expect(classifySandboxRecoveryAuthority(entry, sessionId)).toBe(expected); }); it("loads the named registry row before applying session ownership (#6630)", () => { vi.spyOn(registry, "getSandbox").mockReturnValue(pending("session-current")); - expect(hasRecoverableSandboxIdentity("dc-after", "session-current")).toBe(true); + expect(getSandboxRecoveryAuthority("dc-after", "session-current")).toBe("authorized"); expect(registry.getSandbox).toHaveBeenCalledWith("dc-after"); }); @@ -155,7 +168,7 @@ describe("recoverable sandbox identity", () => { vi.spyOn(registry, "getSandbox").mockReturnValue(null); const isOwned = vi.spyOn(registry, "isPendingReservationForSession"); - expect(hasRecoverableSandboxIdentity("missing-sandbox", "session-current")).toBe(false); + expect(getSandboxRecoveryAuthority("missing-sandbox", "session-current")).toBe("missing"); expect(registry.getSandbox).toHaveBeenCalledWith("missing-sandbox"); expect(isOwned).not.toHaveBeenCalled(); }); @@ -230,6 +243,65 @@ describe("provider recovery persisted routing state", () => { }); }); + it.each([ + { label: "ownerless", reservationSessionId: undefined }, + { label: "foreign-owned", reservationSessionId: "session-other" }, + ])("rejects every $label pending route reader before session fallback", ({ + reservationSessionId, + }) => { + vi.spyOn(registry, "getSandbox").mockReturnValue({ + name: "alpha", + pendingRouteReservation: true, + ...(reservationSessionId ? { reservationSessionId } : {}), + provider: "compatible-endpoint", + model: "registry-model", + endpointUrl: "https://registry.example/v1", + preferredInferenceApi: "openai-completions", + nimContainer: "registry-container", + }); + vi.spyOn(onboardSession, "loadSession").mockReturnValue( + onboardSession.createSession({ + sessionId: "session-current", + sandboxName: "alpha", + provider: "compatible-endpoint", + model: "session-model", + endpointUrl: "https://session.example/v1", + preferredInferenceApi: "openai-responses", + nimContainer: "session-container", + }), + ); + const recovery = helpers(); + + expect(recovery.readRecordedProvider("alpha")).toBeNull(); + expect(recovery.readRecordedModel("alpha")).toBeNull(); + expect(recovery.readRecordedEndpointUrl("alpha")).toBeNull(); + expect(recovery.readRecordedNimContainer("alpha")).toBeNull(); + expect(recovery.readRecordedInferenceRoute("alpha")).toBeNull(); + }); + + it("allows the current session to read its pending route", () => { + vi.spyOn(registry, "getSandbox").mockReturnValue({ + name: "alpha", + pendingRouteReservation: true, + reservationSessionId: "session-current", + provider: "compatible-endpoint", + model: "registry-model", + endpointUrl: "https://registry.example/v1", + preferredInferenceApi: "openai-completions", + }); + vi.spyOn(onboardSession, "loadSession").mockReturnValue( + onboardSession.createSession({ sessionId: "session-current", sandboxName: "alpha" }), + ); + + expect(helpers().readRecordedInferenceRoute("alpha")).toEqual({ + provider: "compatible-endpoint", + model: "registry-model", + endpointUrl: "https://registry.example/v1", + preferredInferenceApi: "openai-completions", + source: "registry", + }); + }); + it("rejects a partial current registry route instead of mixing in stale session fields", () => { vi.spyOn(registry, "getSandbox").mockReturnValue({ name: "alpha", diff --git a/src/lib/onboard/provider-recovery.ts b/src/lib/onboard/provider-recovery.ts index c501c24ae8..c20c6c41fc 100644 --- a/src/lib/onboard/provider-recovery.ts +++ b/src/lib/onboard/provider-recovery.ts @@ -66,38 +66,57 @@ const MAX_LIVE_PROVIDER_LENGTH = 128; const MAX_LIVE_MODEL_LENGTH = 512; const SAFE_LIVE_PROVIDER = /^[A-Za-z0-9._:-]+$/; -export function isRecoverableSandboxIdentity( +export type SandboxRecoveryAuthority = "missing" | "authorized" | "unauthorized"; + +export function classifySandboxRecoveryAuthority( entry: registry.SandboxEntry | null, sessionId: string | null | undefined, -): boolean { - return Boolean( - entry && - (entry.pendingRouteReservation !== true || - registry.isPendingReservationForSession(entry, sessionId)), - ); +): SandboxRecoveryAuthority { + if (!entry) return "missing"; + if (entry.pendingRouteReservation !== true) return "authorized"; + return registry.isPendingReservationForSession(entry, sessionId) ? "authorized" : "unauthorized"; } -export function hasRecoverableSandboxIdentity( +export function getSandboxRecoveryAuthority( sandboxName: string, sessionId: string | null | undefined, -): boolean { - return isRecoverableSandboxIdentity(registry.getSandbox(sandboxName), sessionId); +): SandboxRecoveryAuthority { + return classifySandboxRecoveryAuthority(registry.getSandbox(sandboxName), sessionId); } export function shouldRecoverRecordedProvider(input: { fresh: boolean; sandboxName: string | null; - hasRecoverableSandboxIdentity: boolean; + sandboxRecoveryAuthority: SandboxRecoveryAuthority; sessionSandboxName: string | null; }): boolean { return ( !input.fresh && (!input.sandboxName || - Boolean( - input.sandboxName && - (input.hasRecoverableSandboxIdentity || input.sessionSandboxName === input.sandboxName), - )) + input.sandboxRecoveryAuthority === "authorized" || + (input.sandboxRecoveryAuthority === "missing" && + input.sessionSandboxName === input.sandboxName)) + ); +} + +function currentSessionId(): string | null { + try { + return onboardSession.loadSession()?.sessionId ?? null; + } catch { + return null; + } +} + +function readRegistryRecoveryState(sandboxName: string): { + authority: SandboxRecoveryAuthority; + entry: registry.SandboxEntry | null; +} { + const entry = registry.getSandbox(sandboxName); + const authority = classifySandboxRecoveryAuthority( + entry, + entry?.pendingRouteReservation === true ? currentSessionId() : null, ); + return { authority, entry }; } export function validateLiveGatewayInference( @@ -168,7 +187,8 @@ export function createProviderRecoveryHelpers(deps: ProviderRecoveryDeps): Provi function readRecordedProvider(sandboxName: string | null | undefined): string | null { if (!sandboxName) return null; try { - const entry = registry.getSandbox(sandboxName); + const { authority, entry } = readRegistryRecoveryState(sandboxName); + if (authority === "unauthorized") return null; if (entry && typeof entry.provider === "string" && entry.provider) { return entry.provider; } @@ -198,7 +218,8 @@ export function createProviderRecoveryHelpers(deps: ProviderRecoveryDeps): Provi function readRecordedNimContainer(sandboxName: string | null | undefined): string | null { if (!sandboxName) return null; try { - const entry = registry.getSandbox(sandboxName); + const { authority, entry } = readRegistryRecoveryState(sandboxName); + if (authority === "unauthorized") return null; if (entry && typeof entry.nimContainer === "string" && entry.nimContainer) { return entry.nimContainer; } @@ -224,7 +245,8 @@ export function createProviderRecoveryHelpers(deps: ProviderRecoveryDeps): Provi function readRecordedModel(sandboxName: string | null | undefined): string | null { if (!sandboxName) return null; try { - const entry = registry.getSandbox(sandboxName); + const { authority, entry } = readRegistryRecoveryState(sandboxName); + if (authority === "unauthorized") return null; if (entry && typeof entry.model === "string" && entry.model) { return entry.model; } @@ -254,7 +276,8 @@ export function createProviderRecoveryHelpers(deps: ProviderRecoveryDeps): Provi function readRecordedEndpointUrl(sandboxName: string | null | undefined): string | null { if (!sandboxName) return null; try { - const entry = registry.getSandbox(sandboxName); + const { authority, entry } = readRegistryRecoveryState(sandboxName); + if (authority === "unauthorized") return null; if (entry && typeof entry.endpointUrl === "string" && entry.endpointUrl) { return entry.endpointUrl; } @@ -282,7 +305,8 @@ export function createProviderRecoveryHelpers(deps: ProviderRecoveryDeps): Provi ): RecordedInferenceRoute | null { if (!sandboxName) return null; try { - const entry = registry.getSandbox(sandboxName); + const { authority, entry } = readRegistryRecoveryState(sandboxName); + if (authority === "unauthorized") return null; // A present registry row is authoritative. If it is incomplete, fail // closed instead of filling its gaps from an older onboard session. if (entry) return completeRecordedInferenceRoute(entry, "registry"); From 288947ed8126c3f44dbb181c4a58e179deb43265 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Fri, 10 Jul 2026 14:13:44 -0700 Subject: [PATCH 11/14] test(onboard): cover ownerless recovery without session Signed-off-by: Carlos Villela --- src/lib/onboard/provider-recovery.test.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/lib/onboard/provider-recovery.test.ts b/src/lib/onboard/provider-recovery.test.ts index ac8da604d2..e2d8e6a721 100644 --- a/src/lib/onboard/provider-recovery.test.ts +++ b/src/lib/onboard/provider-recovery.test.ts @@ -135,6 +135,12 @@ describe("sandbox recovery authority", () => { sessionId: "session-current", expected: "unauthorized", }, + { + label: "orphaned pending reservation without an active session", + entry: pending(), + sessionId: null, + expected: "unauthorized", + }, { label: "another session's pending reservation", entry: pending("session-other"), From 7e04b4feaaf509ed26225ac96b74bb2b8ae6a299 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Fri, 10 Jul 2026 14:35:25 -0700 Subject: [PATCH 12/14] fix(onboard): bind recovery to caller session Signed-off-by: Carlos Villela --- src/lib/onboard.ts | 22 +-- .../onboard/machine/core-flow-phases.test.ts | 1 + ...provider-inference-recovery-gating.test.ts | 68 ++++++++++ .../handlers/provider-inference.test.ts | 4 + .../machine/handlers/provider-inference.ts | 24 +++- src/lib/onboard/provider-recovery.test.ts | 64 ++++++++- src/lib/onboard/provider-recovery.ts | 126 +++++++++++------- .../setup-inference-route-containment.test.ts | 53 ++++++++ src/lib/onboard/setup-inference.ts | 9 ++ src/lib/onboard/setup-nim-flow.test.ts | 6 +- src/lib/onboard/setup-nim-flow.ts | 46 +++++-- 11 files changed, 338 insertions(+), 85 deletions(-) diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 55ac0a3b07..a4483633b8 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -3069,7 +3069,7 @@ type RebuildRouteHandoff = import("./onboard/rebuild-route-handoff").RebuildRout // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. const { readRecordedProvider, readRecordedNimContainer, readRecordedModel, readRecordedEndpointUrl, - readRecordedInferenceRoute, readRecordedProviderEndpoints } = providerRecovery.createProviderRecoveryHelpers({ parseGatewayInference, runCaptureOpenshell }); + readRecordedInferenceRoute, readRecordedProviderEndpoints } = providerRecovery.createProviderRecoveryHelpers({ parseGatewayInference, runCaptureOpenshell, warn: (message) => console.warn(message) }); type OllamaModelSelectionOutcome = | { outcome: "selected"; model: string; allowToolsIncompatible: boolean } @@ -3180,7 +3180,7 @@ type SetupNimSelectionState = type SetupNimSelectionResult = "selected" | "retry-selection"; // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. -type RemoteProviderSelectionArgs = { selected: ProviderChoice; requestedModel: string | null; recoveredFromSandbox: boolean; recoveredModel: string | null; sandboxName: string | null; gatewayName: string | null; intendedInferenceApi: string | null }; +type RemoteProviderSelectionArgs = { selected: ProviderChoice; requestedModel: string | null; recoveredFromSandbox: boolean; recoveredModel: string | null; sandboxName: string | null; gatewayName: string | null; intendedInferenceApi: string | null; recoverySessionId: string | null | undefined }; async function handleRoutedSelection( state: SetupNimSelectionState, @@ -3394,7 +3394,8 @@ async function handleRemoteProviderSelection(args: RemoteProviderSelectionArgs, kind, envUrl: process.env.NEMOCLAW_ENDPOINT_URL, recoveredEndpointUrl: recoveredFromSandbox - ? (recoveredRegistryRoute?.endpointUrl ?? readRecordedEndpointUrl(sandboxName)) + ? (recoveredRegistryRoute?.endpointUrl ?? + readRecordedEndpointUrl(sandboxName, args.recoverySessionId)) : null, nonInteractive: isNonInteractive(), prompt, @@ -3582,7 +3583,7 @@ async function handleRemoteProviderSelection(args: RemoteProviderSelectionArgs, // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. recoveredProviderReuse.resolveRecoveredProviderCredentialReuse( { selected, remoteConfig, state, selectedCredentialEnv, recoveredFromSandbox, selectedModel: defaultModel, sandboxName, recoveredRegistryRoute }, - { resolveProviderCredential, readRecordedInferenceRoute, readRecordedProviderEndpoints, readGatewayProviderMetadata: (provider) => onboardProviders.readGatewayProviderMetadata(provider, runOpenshell, args.gatewayName ?? GATEWAY_NAME), note }, + { resolveProviderCredential, readRecordedInferenceRoute: (name) => readRecordedInferenceRoute(name, args.recoverySessionId), readRecordedProviderEndpoints, readGatewayProviderMetadata: (provider) => onboardProviders.readGatewayProviderMetadata(provider, runOpenshell, args.gatewayName ?? GATEWAY_NAME), note }, ); } else { const credentialResult = await credentialPrompt.ensureNamedCredential( @@ -4444,17 +4445,8 @@ async function runOnboard(opts: OnboardOptions = {}): Promise { getSandboxRecoveryAuthority: providerRecovery.getSandboxRecoveryAuthority, withGatewayRouteMutationLock: gatewayRouteMutationLock.withGatewayRouteMutationLock, normalizeHermesAuthMethod, - setupNim: (g, s, a, recover, gateway, assertRouteCompatible, canProbeRoute) => - setupNim( - g, - s, - a, - recover, - opts.rebuildRegistryInferenceRoute, - gateway, - assertRouteCompatible, - canProbeRoute, - ), + // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. + setupNim: (g, s, a, recover, gateway, assertRouteCompatible, canProbeRoute, recoverySessionId) => setupNim(g, s, a, recover, opts.rebuildRegistryInferenceRoute, gateway, assertRouteCompatible, canProbeRoute, recoverySessionId), setupInference, startRecordedStep, recordStepComplete, diff --git a/src/lib/onboard/machine/core-flow-phases.test.ts b/src/lib/onboard/machine/core-flow-phases.test.ts index 8a3d100b4b..70910afa6b 100644 --- a/src/lib/onboard/machine/core-flow-phases.test.ts +++ b/src/lib/onboard/machine/core-flow-phases.test.ts @@ -302,6 +302,7 @@ describe("core onboard flow phases", () => { "nemoclaw", expect.any(Function), expect.any(Function), + expect.any(String), ); }); diff --git a/src/lib/onboard/machine/handlers/provider-inference-recovery-gating.test.ts b/src/lib/onboard/machine/handlers/provider-inference-recovery-gating.test.ts index a28ca4b9d7..b56bf39d51 100644 --- a/src/lib/onboard/machine/handlers/provider-inference-recovery-gating.test.ts +++ b/src/lib/onboard/machine/handlers/provider-inference-recovery-gating.test.ts @@ -33,6 +33,7 @@ describe("provider inference recovery gating", () => { "nemoclaw", expect.any(Function), expect.any(Function), + expect.any(String), ); }); @@ -54,6 +55,7 @@ describe("provider inference recovery gating", () => { "nemoclaw", expect.any(Function), expect.any(Function), + session.sessionId, ); }); @@ -76,6 +78,7 @@ describe("provider inference recovery gating", () => { "nemoclaw", expect.any(Function), expect.any(Function), + expect.any(String), ); expect(getAuthority).toHaveBeenCalledWith("dc-after", expect.any(String)); }); @@ -113,6 +116,7 @@ describe("provider inference recovery gating", () => { "nemoclaw", expect.any(Function), expect.any(Function), + session.sessionId, ); expect(getAuthority).toHaveBeenCalledWith("dc-after", session.sessionId); }); @@ -145,10 +149,72 @@ describe("provider inference recovery gating", () => { "nemoclaw", expect.any(Function), expect.any(Function), + session.sessionId, ); expect(getAuthority).toHaveBeenCalledWith("dc-after", session.sessionId); }); + it("forces route setup and rechecks ownership after recorded selection (#6630)", async () => { + const session = createSession(); + session.sandboxName = "dc-after"; + session.steps.sandbox.status = "complete"; + const persistedAfterSelection = createSession({ sessionId: "session-after-selection" }); + let authority: "authorized" | "unauthorized" = "authorized"; + const getAuthority = vi.fn((_name: string, sessionId: string | null | undefined) => + sessionId === session.sessionId ? authority : "unauthorized", + ); + const setupNim = vi.fn(async () => { + authority = "unauthorized"; + return { + model: "nvidia/test", + provider: "nvidia-prod", + endpointUrl: "https://integrate.api.nvidia.com/v1", + credentialEnv: "NVIDIA_INFERENCE_API_KEY", + hermesAuthMethod: null, + hermesToolGateways: [], + preferredInferenceApi: "openai-responses", + compatibleEndpointReasoning: null, + nimContainer: null, + recoveredFromSandbox: true, + }; + }); + let setupInferenceCalls = 0; + const { deps } = createDeps({ + getSandboxRecoveryAuthority: getAuthority, + setupNim, + setupInference: async (...args) => { + setupInferenceCalls += 1; + const options = args[7]; + expect(options?.reservationSessionId).toBe(session.sessionId); + expect(options?.isRecordedProviderRecoveryAuthorized).toBeTypeOf("function"); + expect(options?.isRecordedProviderRecoveryAuthorized?.()).toBe(false); + return { ok: true as const }; + }, + recordStepComplete: async () => persistedAfterSelection, + isInferenceRouteReady: () => true, + }); + + await handleProviderInferenceState({ + ...baseOptions(deps, session), + resume: true, + sandboxName: "dc-after", + }); + + expect(setupNim).toHaveBeenCalledWith( + { type: "nvidia" }, + "dc-after", + null, + true, + "nemoclaw", + expect.any(Function), + expect.any(Function), + session.sessionId, + ); + expect(getAuthority).toHaveBeenNthCalledWith(1, "dc-after", session.sessionId); + expect(getAuthority).toHaveBeenLastCalledWith("dc-after", session.sessionId); + expect(setupInferenceCalls).toBe(1); + }); + it("composes persisted route reservation ownership with resume recovery (#6626, #6630)", async () => { const home = await fs.mkdtemp(path.join(os.tmpdir(), "nemoclaw-recovery-reservation-")); vi.stubEnv("HOME", home); @@ -213,6 +279,7 @@ describe("provider inference recovery gating", () => { "nemoclaw", expect.any(Function), expect.any(Function), + sessionId, ); } } finally { @@ -241,6 +308,7 @@ describe("provider inference recovery gating", () => { "nemoclaw", expect.any(Function), expect.any(Function), + session.sessionId, ); }); }); diff --git a/src/lib/onboard/machine/handlers/provider-inference.test.ts b/src/lib/onboard/machine/handlers/provider-inference.test.ts index 52dde72fb7..16eb98a5a8 100644 --- a/src/lib/onboard/machine/handlers/provider-inference.test.ts +++ b/src/lib/onboard/machine/handlers/provider-inference.test.ts @@ -40,6 +40,7 @@ describe("handleProviderInferenceState", () => { "nemoclaw", expect.any(Function), expect.any(Function), + session.sessionId, ); expect(calls.promptName).toHaveBeenCalledWith(null); expect(calls.log).toHaveBeenCalledWith("summary:nvidia-prod/nvidia/test/my-assistant"); @@ -325,6 +326,7 @@ describe("handleProviderInferenceState", () => { "nemoclaw", expect.any(Function), expect.any(Function), + session.sessionId, ); expect(calls.setupInference).toHaveBeenCalled(); }); @@ -1128,6 +1130,7 @@ describe("handleProviderInferenceState", () => { "nemoclaw", expect.any(Function), expect.any(Function), + expect.any(String), ); expect(setupNim).toHaveBeenNthCalledWith( 2, @@ -1138,6 +1141,7 @@ describe("handleProviderInferenceState", () => { "nemoclaw", expect.any(Function), expect.any(Function), + expect.any(String), ); expect(setupInference).toHaveBeenCalledTimes(2); expect(result.model).toBe("good"); diff --git a/src/lib/onboard/machine/handlers/provider-inference.ts b/src/lib/onboard/machine/handlers/provider-inference.ts index 143d62ace8..8674b81225 100644 --- a/src/lib/onboard/machine/handlers/provider-inference.ts +++ b/src/lib/onboard/machine/handlers/provider-inference.ts @@ -40,6 +40,8 @@ export interface ProviderInferenceSetupOptions { endpointPinnedAddresses?: readonly string[]; /** Onboard session that owns the route reservation this setup creates. */ reservationSessionId?: string; + /** Recheck recorded-route ownership after acquiring route mutation locks. */ + isRecordedProviderRecoveryAuthorized?: () => boolean; } export interface ProviderSelectionResult { @@ -55,6 +57,7 @@ export interface ProviderSelectionResult { allowToolsIncompatible?: boolean; skipHostInferenceSmoke?: boolean; reuseGatewayCredentialWithoutLocalKey?: boolean; + recoveredFromSandbox?: boolean; endpointPinnedAddresses?: string[]; } @@ -112,6 +115,7 @@ export interface ProviderInferenceStateOptions { route: ProviderInferenceProbeRoute, ) => GatewayRouteDiscoveryConstraints, canProbeRoute?: (provider: string) => boolean, + recoverySessionId?: string | null, ): Promise; setupInference( sandboxName: string | null, @@ -342,6 +346,8 @@ export async function handleProviderInferenceState({ // (#6177; resume/repair coverage per PR #6293 PRA-3). clearAutoDetectedCompatibleContextWindow(process.env); let forceInferenceSetup = initialForceInferenceSetup; + let recoveredRecordedProvider = false; + const providerRecoverySessionId = session?.sessionId ?? null; const resumeProviderSelection = !forceProviderSelection && effectiveResume && @@ -454,7 +460,7 @@ export async function handleProviderInferenceState({ fresh, sandboxName, sandboxRecoveryAuthority: sandboxName - ? deps.getSandboxRecoveryAuthority(sandboxName, session?.sessionId) + ? deps.getSandboxRecoveryAuthority(sandboxName, providerRecoverySessionId) : "missing", sessionSandboxName: session?.steps?.sandbox?.status === "complete" ? (session.sandboxName ?? null) : null, @@ -482,6 +488,7 @@ export async function handleProviderInferenceState({ credentialEnv: null, }, }).ok, + providerRecoverySessionId, ), ); model = selection.model; @@ -497,6 +504,8 @@ export async function handleProviderInferenceState({ skipHostInferenceSmoke = selection.skipHostInferenceSmoke === true; reuseGatewayCredentialWithoutLocalKey = selection.reuseGatewayCredentialWithoutLocalKey === true; + recoveredRecordedProvider = selection.recoveredFromSandbox === true; + forceInferenceSetup ||= recoveredRecordedProvider; endpointPinnedAddresses = selection.endpointPinnedAddresses; shouldRecordProviderSelection = true; } @@ -761,7 +770,18 @@ export async function handleProviderInferenceState({ ...(reuseGatewayCredentialWithoutLocalKey ? { reuseGatewayCredentialWithoutLocalKey } : {}), ...(preferredInferenceApi ? { preferredInferenceApi } : {}), ...(endpointPinnedAddresses ? { endpointPinnedAddresses } : {}), - reservationSessionId: session?.sessionId, + reservationSessionId: recoveredRecordedProvider + ? (providerRecoverySessionId ?? undefined) + : session?.sessionId, + ...(recoveredRecordedProvider + ? { + isRecordedProviderRecoveryAuthorized: () => + deps.getSandboxRecoveryAuthority( + confirmedSandboxName, + providerRecoverySessionId, + ) !== "unauthorized", + } + : {}), }; await deps.startRecordedStep("inference", { provider, model }); inferenceResult = await withInferenceTrace( diff --git a/src/lib/onboard/provider-recovery.test.ts b/src/lib/onboard/provider-recovery.test.ts index e2d8e6a721..cc37fc19ed 100644 --- a/src/lib/onboard/provider-recovery.test.ts +++ b/src/lib/onboard/provider-recovery.test.ts @@ -278,11 +278,11 @@ describe("provider recovery persisted routing state", () => { ); const recovery = helpers(); - expect(recovery.readRecordedProvider("alpha")).toBeNull(); - expect(recovery.readRecordedModel("alpha")).toBeNull(); - expect(recovery.readRecordedEndpointUrl("alpha")).toBeNull(); - expect(recovery.readRecordedNimContainer("alpha")).toBeNull(); - expect(recovery.readRecordedInferenceRoute("alpha")).toBeNull(); + expect(recovery.readRecordedProvider("alpha", "session-current")).toBeNull(); + expect(recovery.readRecordedModel("alpha", "session-current")).toBeNull(); + expect(recovery.readRecordedEndpointUrl("alpha", "session-current")).toBeNull(); + expect(recovery.readRecordedNimContainer("alpha", "session-current")).toBeNull(); + expect(recovery.readRecordedInferenceRoute("alpha", "session-current")).toBeNull(); }); it("allows the current session to read its pending route", () => { @@ -299,13 +299,65 @@ describe("provider recovery persisted routing state", () => { onboardSession.createSession({ sessionId: "session-current", sandboxName: "alpha" }), ); - expect(helpers().readRecordedInferenceRoute("alpha")).toEqual({ + expect(helpers().readRecordedInferenceRoute("alpha", "session-current")).toEqual({ + provider: "compatible-endpoint", + model: "registry-model", + endpointUrl: "https://registry.example/v1", + preferredInferenceApi: "openai-completions", + source: "registry", + }); + }); + + it("uses the caller session identity instead of ambient on-disk session state", () => { + vi.spyOn(registry, "getSandbox").mockReturnValue({ + name: "alpha", + pendingRouteReservation: true, + reservationSessionId: "session-caller", provider: "compatible-endpoint", model: "registry-model", endpointUrl: "https://registry.example/v1", preferredInferenceApi: "openai-completions", + }); + const loadSession = vi + .spyOn(onboardSession, "loadSession") + .mockReturnValue( + onboardSession.createSession({ sessionId: "session-ambient", sandboxName: "alpha" }), + ); + const recovery = helpers(); + + expect(recovery.readRecordedInferenceRoute("alpha", "session-caller")).toMatchObject({ + model: "registry-model", source: "registry", }); + expect(recovery.readRecordedInferenceRoute("alpha", "session-ambient")).toBeNull(); + expect(loadSession).not.toHaveBeenCalled(); + }); + + it("fails closed and warns when registry ownership cannot be read", () => { + const failure = new Error("registry unreadable"); + vi.spyOn(registry, "getSandbox").mockImplementation(() => { + throw failure; + }); + const loadSession = vi.spyOn(onboardSession, "loadSession").mockReturnValue( + onboardSession.createSession({ + sandboxName: "alpha", + provider: "compatible-endpoint", + model: "stale-session-model", + }), + ); + const warn = vi.fn(); + const recovery = createProviderRecoveryHelpers({ + parseGatewayInference: () => ({ provider: "compatible-endpoint", model: "live-model" }), + runCaptureOpenshell: () => "Gateway inference:", + warn, + }); + + expect(recovery.readRecordedProvider("alpha", "session-current")).toBeNull(); + expect(recovery.readRecordedModel("alpha", "session-current")).toBeNull(); + expect(recovery.readRecordedEndpointUrl("alpha", "session-current")).toBeNull(); + expect(loadSession).not.toHaveBeenCalled(); + expect(warn).toHaveBeenCalledTimes(3); + expect(warn).toHaveBeenCalledWith(expect.stringContaining("refusing recovery")); }); it("rejects a partial current registry route instead of mixing in stale session fields", () => { diff --git a/src/lib/onboard/provider-recovery.ts b/src/lib/onboard/provider-recovery.ts index c20c6c41fc..48c60e98bc 100644 --- a/src/lib/onboard/provider-recovery.ts +++ b/src/lib/onboard/provider-recovery.ts @@ -37,17 +37,33 @@ export interface ProviderRecoveryDeps { output: string | null, ): { provider: string | null; model: string | null } | null; runCaptureOpenshell(args: string[], opts?: Record): string | null; + warn?(message: string): void; } export interface ProviderRecoveryHelpers { readLiveInference( sandboxName: string | null | undefined, ): { provider: string | null; model: string | null } | null; - readRecordedProvider(sandboxName: string | null | undefined): string | null; - readRecordedNimContainer(sandboxName: string | null | undefined): string | null; - readRecordedModel(sandboxName: string | null | undefined): string | null; - readRecordedEndpointUrl(sandboxName: string | null | undefined): string | null; - readRecordedInferenceRoute(sandboxName: string | null | undefined): RecordedInferenceRoute | null; + readRecordedProvider( + sandboxName: string | null | undefined, + recoverySessionId?: string | null, + ): string | null; + readRecordedNimContainer( + sandboxName: string | null | undefined, + recoverySessionId?: string | null, + ): string | null; + readRecordedModel( + sandboxName: string | null | undefined, + recoverySessionId?: string | null, + ): string | null; + readRecordedEndpointUrl( + sandboxName: string | null | undefined, + recoverySessionId?: string | null, + ): string | null; + readRecordedInferenceRoute( + sandboxName: string | null | undefined, + recoverySessionId?: string | null, + ): RecordedInferenceRoute | null; readRecordedProviderEndpoints( provider: string, excludeSandboxName: string | null | undefined, @@ -99,23 +115,15 @@ export function shouldRecoverRecordedProvider(input: { ); } -function currentSessionId(): string | null { - try { - return onboardSession.loadSession()?.sessionId ?? null; - } catch { - return null; - } -} - -function readRegistryRecoveryState(sandboxName: string): { +function readRegistryRecoveryState( + sandboxName: string, + recoverySessionId: string | null | undefined, +): { authority: SandboxRecoveryAuthority; entry: registry.SandboxEntry | null; } { const entry = registry.getSandbox(sandboxName); - const authority = classifySandboxRecoveryAuthority( - entry, - entry?.pendingRouteReservation === true ? currentSessionId() : null, - ); + const authority = classifySandboxRecoveryAuthority(entry, recoverySessionId); return { authority, entry }; } @@ -161,6 +169,14 @@ function completeRecordedInferenceRoute( } export function createProviderRecoveryHelpers(deps: ProviderRecoveryDeps): ProviderRecoveryHelpers { + function refuseRecoveryAfterRegistryError(sandboxName: string, error: unknown): null { + const detail = error instanceof Error ? error.message : String(error); + deps.warn?.( + ` Warning: could not verify recorded inference ownership for sandbox '${sandboxName}'; refusing recovery (${detail}).`, + ); + return null; + } + function readLiveInference( sandboxName: string | null | undefined, ): { provider: string | null; model: string | null } | null { @@ -184,16 +200,18 @@ export function createProviderRecoveryHelpers(deps: ProviderRecoveryDeps): Provi } } - function readRecordedProvider(sandboxName: string | null | undefined): string | null { + function readRecordedProvider( + sandboxName: string | null | undefined, + recoverySessionId?: string | null, + ): string | null { if (!sandboxName) return null; try { - const { authority, entry } = readRegistryRecoveryState(sandboxName); + const { authority, entry } = readRegistryRecoveryState(sandboxName, recoverySessionId); if (authority === "unauthorized") return null; - if (entry && typeof entry.provider === "string" && entry.provider) { - return entry.provider; - } - } catch { - // fall through to session + if (entry) + return typeof entry.provider === "string" && entry.provider ? entry.provider : null; + } catch (error) { + return refuseRecoveryAfterRegistryError(sandboxName, error); } try { const session = onboardSession.loadSession(); @@ -215,16 +233,20 @@ export function createProviderRecoveryHelpers(deps: ProviderRecoveryDeps): Provi return null; } - function readRecordedNimContainer(sandboxName: string | null | undefined): string | null { + function readRecordedNimContainer( + sandboxName: string | null | undefined, + recoverySessionId?: string | null, + ): string | null { if (!sandboxName) return null; try { - const { authority, entry } = readRegistryRecoveryState(sandboxName); + const { authority, entry } = readRegistryRecoveryState(sandboxName, recoverySessionId); if (authority === "unauthorized") return null; - if (entry && typeof entry.nimContainer === "string" && entry.nimContainer) { - return entry.nimContainer; - } - } catch { - // fall through to session + if (entry) + return typeof entry.nimContainer === "string" && entry.nimContainer + ? entry.nimContainer + : null; + } catch (error) { + return refuseRecoveryAfterRegistryError(sandboxName, error); } try { const session = onboardSession.loadSession(); @@ -242,16 +264,17 @@ export function createProviderRecoveryHelpers(deps: ProviderRecoveryDeps): Provi return null; } - function readRecordedModel(sandboxName: string | null | undefined): string | null { + function readRecordedModel( + sandboxName: string | null | undefined, + recoverySessionId?: string | null, + ): string | null { if (!sandboxName) return null; try { - const { authority, entry } = readRegistryRecoveryState(sandboxName); + const { authority, entry } = readRegistryRecoveryState(sandboxName, recoverySessionId); if (authority === "unauthorized") return null; - if (entry && typeof entry.model === "string" && entry.model) { - return entry.model; - } - } catch { - // fall through to session + if (entry) return typeof entry.model === "string" && entry.model ? entry.model : null; + } catch (error) { + return refuseRecoveryAfterRegistryError(sandboxName, error); } try { const session = onboardSession.loadSession(); @@ -273,16 +296,20 @@ export function createProviderRecoveryHelpers(deps: ProviderRecoveryDeps): Provi return null; } - function readRecordedEndpointUrl(sandboxName: string | null | undefined): string | null { + function readRecordedEndpointUrl( + sandboxName: string | null | undefined, + recoverySessionId?: string | null, + ): string | null { if (!sandboxName) return null; try { - const { authority, entry } = readRegistryRecoveryState(sandboxName); + const { authority, entry } = readRegistryRecoveryState(sandboxName, recoverySessionId); if (authority === "unauthorized") return null; - if (entry && typeof entry.endpointUrl === "string" && entry.endpointUrl) { - return entry.endpointUrl; - } - } catch { - // fall through to the matching session + if (entry) + return typeof entry.endpointUrl === "string" && entry.endpointUrl + ? entry.endpointUrl + : null; + } catch (error) { + return refuseRecoveryAfterRegistryError(sandboxName, error); } try { const session = onboardSession.loadSession(); @@ -302,16 +329,17 @@ export function createProviderRecoveryHelpers(deps: ProviderRecoveryDeps): Provi function readRecordedInferenceRoute( sandboxName: string | null | undefined, + recoverySessionId?: string | null, ): RecordedInferenceRoute | null { if (!sandboxName) return null; try { - const { authority, entry } = readRegistryRecoveryState(sandboxName); + const { authority, entry } = readRegistryRecoveryState(sandboxName, recoverySessionId); if (authority === "unauthorized") return null; // A present registry row is authoritative. If it is incomplete, fail // closed instead of filling its gaps from an older onboard session. if (entry) return completeRecordedInferenceRoute(entry, "registry"); - } catch { - return null; + } catch (error) { + return refuseRecoveryAfterRegistryError(sandboxName, error); } try { const session = onboardSession.loadSession(); diff --git a/src/lib/onboard/setup-inference-route-containment.test.ts b/src/lib/onboard/setup-inference-route-containment.test.ts index f61c4468dd..0b719fe4fb 100644 --- a/src/lib/onboard/setup-inference-route-containment.test.ts +++ b/src/lib/onboard/setup-inference-route-containment.test.ts @@ -152,6 +152,59 @@ describe("onboard shared gateway route containment", () => { expect(exitProcess).toHaveBeenCalledWith(1); }); + it("rechecks recovered-route ownership inside both mutation locks before setup (#6630)", async () => { + const events: string[] = []; + const checkGatewayRouteCompatibility = vi.fn(() => ({ ok: true as const })); + const updateSandbox = vi.fn(() => true); + const runOpenshell = vi.fn(() => ({ status: 0 })); + const exitProcess = vi.fn((code: number): never => { + events.push(`exit:${code}`); + throw new Error(`exit ${code}`); + }); + const setupInference = createSetupInference({ + checkGatewayRouteCompatibility, + withSandboxMutationLock: async (_name: string, operation: () => Promise | T) => { + events.push("sandbox-lock"); + return await operation(); + }, + withGatewayRouteMutationLock: async (_name: string, operation: () => Promise | T) => { + events.push("gateway-lock"); + return await operation(); + }, + getGatewayName: () => "nemoclaw", + error: (message: string) => events.push(`error:${message}`), + exitProcess, + updateSandbox, + runOpenshell, + } as unknown as SetupInferenceDeps); + + await expect( + setupInference( + "alpha", + "model-a", + "anthropic-prod", + "https://api.anthropic.com", + "ANTHROPIC_API_KEY", + null, + [], + { + reservationSessionId: "session-current", + isRecordedProviderRecoveryAuthorized: () => { + events.push("recovery-authority"); + return false; + }, + }, + ), + ).rejects.toThrow("exit 1"); + + expect(events.slice(0, 3)).toEqual(["sandbox-lock", "gateway-lock", "recovery-authority"]); + expect(events).toContainEqual(expect.stringContaining("lost reservation ownership")); + expect(checkGatewayRouteCompatibility).not.toHaveBeenCalled(); + expect(updateSandbox).not.toHaveBeenCalled(); + expect(runOpenshell).not.toHaveBeenCalled(); + expect(exitProcess).toHaveBeenCalledWith(1); + }); + it("reserves a fresh route before smoke failure lets another setup mutate it (#6315)", async () => { const reservations: SandboxEntry[] = []; let lockTail = Promise.resolve(); diff --git a/src/lib/onboard/setup-inference.ts b/src/lib/onboard/setup-inference.ts index 7ddc40ed06..485c106233 100644 --- a/src/lib/onboard/setup-inference.ts +++ b/src/lib/onboard/setup-inference.ts @@ -228,6 +228,15 @@ export function createSetupInference( const gatewayName = options.gatewayName ?? deps.getGatewayName(); const mutateGatewayRoute = (): Promise => deps.withGatewayRouteMutationLock(gatewayName, async () => { + if ( + options.isRecordedProviderRecoveryAuthorized && + !options.isRecordedProviderRecoveryAuthorized() + ) { + deps.error( + ` Error: recorded inference recovery for sandbox '${sandboxName}' lost reservation ownership before route setup.`, + ); + return deps.exitProcess(1); + } const compatibility = deps.checkGatewayRouteCompatibility({ gatewayName, sandboxName, diff --git a/src/lib/onboard/setup-nim-flow.test.ts b/src/lib/onboard/setup-nim-flow.test.ts index 0d5926e4f0..347c57c8a3 100644 --- a/src/lib/onboard/setup-nim-flow.test.ts +++ b/src/lib/onboard/setup-nim-flow.test.ts @@ -527,9 +527,9 @@ describe("createSetupNim", () => { const result = await setupNim(null, "existing-sandbox"); expect(prompt).not.toHaveBeenCalled(); - expect(readRecordedProvider).toHaveBeenCalledWith("existing-sandbox"); - expect(readRecordedNimContainer).toHaveBeenCalledWith("existing-sandbox"); - expect(readRecordedModel).toHaveBeenCalledWith("existing-sandbox"); + expect(readRecordedProvider).toHaveBeenCalledWith("existing-sandbox", undefined); + expect(readRecordedNimContainer).toHaveBeenCalledWith("existing-sandbox", undefined); + expect(readRecordedModel).toHaveBeenCalledWith("existing-sandbox", undefined); expect(note).toHaveBeenCalledWith( " [non-interactive] Provider: openai (recovered from sandbox 'existing-sandbox')", ); diff --git a/src/lib/onboard/setup-nim-flow.ts b/src/lib/onboard/setup-nim-flow.ts index 6f24c6587c..40bb5c34a8 100644 --- a/src/lib/onboard/setup-nim-flow.ts +++ b/src/lib/onboard/setup-nim-flow.ts @@ -44,6 +44,7 @@ export interface SetupNimRemoteSelectionArgs { recoveredModel: string | null; sandboxName: string | null; intendedInferenceApi: string | null; + recoverySessionId: string | null | undefined; } export type SetupNim = ( @@ -55,6 +56,7 @@ export type SetupNim = ( gatewayName?: string | null, assertRouteCompatible?: (route: ProviderInferenceProbeRoute) => GatewayRouteDiscoveryConstraints, canProbeRoute?: (provider: string) => boolean, + recoverySessionId?: string | null, ) => Promise; export interface SetupNimFlowDeps { @@ -77,9 +79,18 @@ export interface SetupNimFlowDeps { }): InferenceProviderHostState; getAgentInferenceProviderOptions(agent: AgentDefinition | null | undefined): string[]; loadRoutedProfile(): { router?: { enabled?: boolean } } | null | undefined; - readRecordedProvider(sandboxName: string | null | undefined): string | null; - readRecordedNimContainer(sandboxName: string | null | undefined): string | null; - readRecordedModel(sandboxName: string | null | undefined): string | null; + readRecordedProvider( + sandboxName: string | null | undefined, + recoverySessionId?: string | null, + ): string | null; + readRecordedNimContainer( + sandboxName: string | null | undefined, + recoverySessionId?: string | null, + ): string | null; + readRecordedModel( + sandboxName: string | null | undefined, + recoverySessionId?: string | null, + ): string | null; rejectWindowsHostOllama( requirement: InferenceProviderHostState["windowsHostOllamaDockerRequirement"], providerKey: string, @@ -238,6 +249,7 @@ function prepareProviderDiscovery(options: { rebuildRegistryInferenceRoute: RebuildRouteHandoff | null; assertRouteCompatible?: (route: ProviderInferenceProbeRoute) => GatewayRouteDiscoveryConstraints; canProbeRoute?: (provider: string) => boolean; + recoverySessionId: string | null | undefined; }): { requestedProvider: string | null; requestedModel: string | null; @@ -252,6 +264,7 @@ function prepareProviderDiscovery(options: { rebuildRegistryInferenceRoute, assertRouteCompatible, canProbeRoute, + recoverySessionId, } = options; const nonInteractive = deps.isNonInteractive(); const requestedProvider = deps.getNonInteractiveProvider(); @@ -265,7 +278,8 @@ function prepareProviderDiscovery(options: { : null; const recoveredProbeProvider = nonInteractive && !requestedProvider && recoverProvider - ? (recoveredRegistryRoute?.provider ?? deps.readRecordedProvider(sandboxName)) + ? (recoveredRegistryRoute?.provider ?? + deps.readRecordedProvider(sandboxName, recoverySessionId)) : null; const recoveredProbeKey = providerNameToOptionKey( deps.remoteProviderConfig, @@ -273,7 +287,7 @@ function prepareProviderDiscovery(options: { { hasNimContainer: recoveredProbeProvider === "vllm-local" && - Boolean(deps.readRecordedNimContainer(sandboxName)), + Boolean(deps.readRecordedNimContainer(sandboxName, recoverySessionId)), }, ); const providerIntentKey = @@ -283,7 +297,9 @@ function prepareProviderDiscovery(options: { if (guardedProvider && assertRouteCompatible) { const recoveredModel = recoveredRegistryRoute?.model ?? - (!requestedProvider && recoverProvider ? deps.readRecordedModel(sandboxName) : null); + (!requestedProvider && recoverProvider + ? deps.readRecordedModel(sandboxName, recoverySessionId) + : null); assertRouteCompatible({ provider: guardedProvider, model: requestedModel || recoveredModel, @@ -322,6 +338,7 @@ export function createSetupNim( route: ProviderInferenceProbeRoute, ) => GatewayRouteDiscoveryConstraints, canProbeRoute?: (provider: string) => boolean, + recoverySessionId?: string | null, ): Promise { deps.step(3, 8, "Configuring inference provider"); @@ -392,6 +409,7 @@ export function createSetupNim( rebuildRegistryInferenceRoute, assertRouteCompatible, canProbeRoute, + recoverySessionId, }); const providerHostState = deps.detectInferenceProviderHostState({ gpu, @@ -452,10 +470,11 @@ export function createSetupNim( ); } + let recoveredFromSandbox = false; if (options.length > 1) { selectionLoop: while (true) { let selected: ProviderMenuChoice | undefined; - let recoveredFromSandbox = false; + recoveredFromSandbox = false; let recoveredModel: string | null = null; let preparedVllmState: SetupNimSelectionState | null = null; hermesAuthMethod = null; @@ -471,11 +490,16 @@ export function createSetupNim( windowsHostOllamaSupported: windowsHostOllamaDockerRequirement.supported, hermesProviderAvailable, readRecordedProvider: recoverProvider - ? (name) => recoveredRegistryRoute?.provider ?? deps.readRecordedProvider(name) + ? (name) => + recoveredRegistryRoute?.provider ?? + deps.readRecordedProvider(name, recoverySessionId) + : () => null, + readRecordedNimContainer: recoverProvider + ? (name) => deps.readRecordedNimContainer(name, recoverySessionId) : () => null, - readRecordedNimContainer: recoverProvider ? deps.readRecordedNimContainer : () => null, readRecordedModel: recoverProvider - ? (name) => recoveredRegistryRoute?.model ?? deps.readRecordedModel(name) + ? (name) => + recoveredRegistryRoute?.model ?? deps.readRecordedModel(name, recoverySessionId) : () => null, }); if (providerSelection.kind === "failure") { @@ -522,6 +546,7 @@ export function createSetupNim( recoveredModel, sandboxName, gatewayName, + recoverySessionId, intendedInferenceApi: resolveValidationInferenceApi( selected.key, deps.remoteProviderConfig[selected.key].providerName, @@ -718,6 +743,7 @@ export function createSetupNim( allowToolsIncompatible, skipHostInferenceSmoke: reuseGatewayCredential, reuseGatewayCredentialWithoutLocalKey: reuseGatewayCredential, + ...(recoveredFromSandbox ? { recoveredFromSandbox: true } : {}), ...(endpointPinnedAddresses ? { endpointPinnedAddresses } : {}), }; }; From 7834b5f9ca05cc1150576f8f4898724e6d8a1e02 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Fri, 10 Jul 2026 14:45:19 -0700 Subject: [PATCH 13/14] test(onboard): assert fail-closed recovery Signed-off-by: Carlos Villela --- test/onboard-resume-provider-recovery.test.ts | 22 +++++++++++++------ 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/test/onboard-resume-provider-recovery.test.ts b/test/onboard-resume-provider-recovery.test.ts index d64fd535b0..c9325d588e 100644 --- a/test/onboard-resume-provider-recovery.test.ts +++ b/test/onboard-resume-provider-recovery.test.ts @@ -5,7 +5,7 @@ import { spawnSync } from "node:child_process"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { afterEach, describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; type ProviderRecoveryInternals = { providerNameToOptionKey: ( @@ -101,6 +101,7 @@ describe("readRecordedProvider", () => { registry.getSandbox = originalGetSandbox; registry.listSandboxes = originalListSandboxes; onboardSession.loadSession = originalLoadSession; + vi.restoreAllMocks(); }); it("returns the provider stored in sandboxes.json", () => { @@ -161,16 +162,23 @@ describe("readRecordedProvider", () => { expect(readRecordedProvider("")).toBeNull(); }); - it("falls back to the session when the registry read throws", () => { + it("fails closed instead of trusting session state when the registry read throws (#6630)", () => { registry.getSandbox = () => { throw new Error("registry unreadable"); }; - onboardSession.loadSession = () => - ({ sandboxName: "spark-1", provider: "ollama-local" }) as ReturnType< - typeof onboardSession.loadSession - >; + const loadSession = vi.fn( + () => + ({ sandboxName: "spark-1", provider: "ollama-local" }) as ReturnType< + typeof onboardSession.loadSession + >, + ); + onboardSession.loadSession = loadSession; + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); stubLiveGatewayUntrusted(); - expect(readRecordedProvider("spark-1")).toBe("ollama-local"); + + expect(readRecordedProvider("spark-1")).toBeNull(); + expect(loadSession).not.toHaveBeenCalled(); + expect(warn).toHaveBeenCalledWith(expect.stringContaining("refusing recovery")); }); it("returns null when registry, session, and live-gateway lookups all throw", () => { From 50efbbec99720b2a5eaa35885cbd1a0c7e4a594f Mon Sep 17 00:00:00 2001 From: Charan Jagwani Date: Fri, 10 Jul 2026 16:04:38 -0700 Subject: [PATCH 14/14] refactor(onboard): isolate recovery authorization Signed-off-by: Charan Jagwani --- .../handlers/provider-inference-recovery.ts | 60 +++++++ .../machine/handlers/provider-inference.ts | 38 ++-- src/lib/onboard/setup-nim-flow.test.ts | 20 ++- src/lib/onboard/setup-nim-flow.ts | 146 ++-------------- .../onboard/setup-nim-provider-discovery.ts | 164 ++++++++++++++++++ 5 files changed, 268 insertions(+), 160 deletions(-) create mode 100644 src/lib/onboard/machine/handlers/provider-inference-recovery.ts create mode 100644 src/lib/onboard/setup-nim-provider-discovery.ts diff --git a/src/lib/onboard/machine/handlers/provider-inference-recovery.ts b/src/lib/onboard/machine/handlers/provider-inference-recovery.ts new file mode 100644 index 0000000000..d12bd1acae --- /dev/null +++ b/src/lib/onboard/machine/handlers/provider-inference-recovery.ts @@ -0,0 +1,60 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { Session } from "../../../state/onboard-session"; +import { + type SandboxRecoveryAuthority, + shouldRecoverRecordedProvider, +} from "../../provider-recovery"; + +export type RecoveryAuthority = SandboxRecoveryAuthority; + +interface ProviderRecoveryDeps { + getSandboxRecoveryAuthority( + sandboxName: string, + sessionId: string | null | undefined, + ): SandboxRecoveryAuthority; +} + +interface ProviderRecoverySetupOptions { + reservationSessionId?: string; + isRecordedProviderRecoveryAuthorized?: () => boolean; +} + +export function createRecovery( + fresh: boolean, + sandboxName: string | null, + session: Session | null, + deps: ProviderRecoveryDeps, +): { + sessionId: string | null; + shouldRecover(): boolean; + setupOptions( + recoveredRecordedProvider: boolean, + selectedSandboxName: string, + currentSessionId: string | undefined, + ): ProviderRecoverySetupOptions; +} { + const sessionId = session?.sessionId ?? null; + return { + sessionId, + shouldRecover: () => + shouldRecoverRecordedProvider({ + fresh, + sandboxName, + sandboxRecoveryAuthority: sandboxName + ? deps.getSandboxRecoveryAuthority(sandboxName, sessionId) + : "missing", + sessionSandboxName: + session?.steps?.sandbox?.status === "complete" ? (session.sandboxName ?? null) : null, + }), + setupOptions(recoveredRecordedProvider, selectedSandboxName, currentSessionId) { + if (!recoveredRecordedProvider) return { reservationSessionId: currentSessionId }; + return { + reservationSessionId: sessionId ?? undefined, + isRecordedProviderRecoveryAuthorized: () => + deps.getSandboxRecoveryAuthority(selectedSandboxName, sessionId) !== "unauthorized", + }; + }, + }; +} diff --git a/src/lib/onboard/machine/handlers/provider-inference.ts b/src/lib/onboard/machine/handlers/provider-inference.ts index 8674b81225..254304e9e5 100644 --- a/src/lib/onboard/machine/handlers/provider-inference.ts +++ b/src/lib/onboard/machine/handlers/provider-inference.ts @@ -10,12 +10,9 @@ import type { } from "../../../inference/gateway-route-compatibility"; import type { WebSearchConfig } from "../../../inference/web-search"; import type { HermesAuthMethod, Session, SessionUpdates } from "../../../state/onboard-session"; -import { - type SandboxRecoveryAuthority, - shouldRecoverRecordedProvider, -} from "../../provider-recovery"; import { withInferenceTrace, withProviderSelectionTrace } from "../../tracing"; import { advanceTo, type OnboardStateTransitionResult, retryTo } from "../result"; +import { createRecovery, type RecoveryAuthority } from "./provider-inference-recovery"; import { assertProviderInferenceRouteCompatible, guardProviderInferenceRouteSelection, @@ -99,7 +96,7 @@ export interface ProviderInferenceStateOptions { getSandboxRecoveryAuthority( sandboxName: string, sessionId: string | null | undefined, - ): SandboxRecoveryAuthority; + ): RecoveryAuthority; withGatewayRouteMutationLock( gatewayName: string, operation: () => Promise | T, @@ -347,7 +344,7 @@ export async function handleProviderInferenceState({ clearAutoDetectedCompatibleContextWindow(process.env); let forceInferenceSetup = initialForceInferenceSetup; let recoveredRecordedProvider = false; - const providerRecoverySessionId = session?.sessionId ?? null; + const providerRecovery = createRecovery(fresh, sandboxName, session, deps); const resumeProviderSelection = !forceProviderSelection && effectiveResume && @@ -456,15 +453,7 @@ export async function handleProviderInferenceState({ } } else { await deps.startRecordedStep("provider_selection"); - const recoverRecordedProvider = shouldRecoverRecordedProvider({ - fresh, - sandboxName, - sandboxRecoveryAuthority: sandboxName - ? deps.getSandboxRecoveryAuthority(sandboxName, providerRecoverySessionId) - : "missing", - sessionSandboxName: - session?.steps?.sandbox?.status === "complete" ? (session.sandboxName ?? null) : null, - }); + const recoverRecordedProvider = providerRecovery.shouldRecover(); const selection = await withProviderSelectionTrace( sandboxName, (agent as { name?: string } | null)?.name, @@ -488,7 +477,7 @@ export async function handleProviderInferenceState({ credentialEnv: null, }, }).ok, - providerRecoverySessionId, + providerRecovery.sessionId, ), ); model = selection.model; @@ -770,18 +759,11 @@ export async function handleProviderInferenceState({ ...(reuseGatewayCredentialWithoutLocalKey ? { reuseGatewayCredentialWithoutLocalKey } : {}), ...(preferredInferenceApi ? { preferredInferenceApi } : {}), ...(endpointPinnedAddresses ? { endpointPinnedAddresses } : {}), - reservationSessionId: recoveredRecordedProvider - ? (providerRecoverySessionId ?? undefined) - : session?.sessionId, - ...(recoveredRecordedProvider - ? { - isRecordedProviderRecoveryAuthorized: () => - deps.getSandboxRecoveryAuthority( - confirmedSandboxName, - providerRecoverySessionId, - ) !== "unauthorized", - } - : {}), + ...providerRecovery.setupOptions( + recoveredRecordedProvider, + confirmedSandboxName, + session?.sessionId, + ), }; await deps.startRecordedStep("inference", { provider, model }); inferenceResult = await withInferenceTrace( diff --git a/src/lib/onboard/setup-nim-flow.test.ts b/src/lib/onboard/setup-nim-flow.test.ts index 347c57c8a3..b6c0961237 100644 --- a/src/lib/onboard/setup-nim-flow.test.ts +++ b/src/lib/onboard/setup-nim-flow.test.ts @@ -490,6 +490,7 @@ describe("createSetupNim", () => { }); it("recovers a recorded provider and model without prompting in non-interactive mode (#6245)", async () => { + const recoverySessionId = "session-recovery"; const prompt = vi.fn(async () => unexpected("interactive provider prompt")); const note = vi.fn(); const readRecordedProvider = vi.fn(() => "openai-api"); @@ -503,6 +504,7 @@ describe("createSetupNim", () => { recoveredFromSandbox: true, recoveredModel: "gpt-4.1", sandboxName: "existing-sandbox", + recoverySessionId, }); state.model = args.recoveredModel; state.provider = "openai-api"; @@ -524,12 +526,22 @@ describe("createSetupNim", () => { }), ); - const result = await setupNim(null, "existing-sandbox"); + const result = await setupNim( + null, + "existing-sandbox", + null, + true, + null, + null, + undefined, + undefined, + recoverySessionId, + ); expect(prompt).not.toHaveBeenCalled(); - expect(readRecordedProvider).toHaveBeenCalledWith("existing-sandbox", undefined); - expect(readRecordedNimContainer).toHaveBeenCalledWith("existing-sandbox", undefined); - expect(readRecordedModel).toHaveBeenCalledWith("existing-sandbox", undefined); + expect(readRecordedProvider).toHaveBeenCalledWith("existing-sandbox", recoverySessionId); + expect(readRecordedNimContainer).toHaveBeenCalledWith("existing-sandbox", recoverySessionId); + expect(readRecordedModel).toHaveBeenCalledWith("existing-sandbox", recoverySessionId); expect(note).toHaveBeenCalledWith( " [non-interactive] Provider: openai (recovered from sandbox 'existing-sandbox')", ); diff --git a/src/lib/onboard/setup-nim-flow.ts b/src/lib/onboard/setup-nim-flow.ts index 40bb5c34a8..9003aa77a7 100644 --- a/src/lib/onboard/setup-nim-flow.ts +++ b/src/lib/onboard/setup-nim-flow.ts @@ -18,11 +18,11 @@ import type { } from "./nvidia-featured-model-selection"; import type { InferenceProviderHostGpu, InferenceProviderHostState } from "./provider-host-state"; import { buildInferenceProviderMenu, type ProviderMenuChoice } from "./provider-menu"; -import { providerNameToOptionKey } from "./provider-recovery"; import { resolveRequestedProviderSelection } from "./provider-selection"; import { reportProviderSelectionFailure } from "./provider-selection-failure"; import { promptForInferenceProviderSelection } from "./provider-selection-prompt"; import type { RebuildRouteHandoff, RegistryInferenceRoute } from "./rebuild-route-handoff"; +import { prepareProviderDiscovery } from "./setup-nim-provider-discovery"; import type { SetupNimSelectionState as BaseSetupNimSelectionState } from "./setup-nim-selection"; export type SetupNimGpu = ReturnType; @@ -216,111 +216,6 @@ function applyGatewayRouteDiscoveryConstraints( } } -const OLLAMA_PROBE_PROVIDER_KEYS = new Set([ - "ollama", - "install-ollama", - "start-windows-ollama", - "install-windows-ollama", -]); -const VLLM_ROUTE_PROVIDER_KEYS = new Set(["vllm", "install-vllm"]); -const VLLM_PROBE_PROVIDER_KEYS = new Set(["vllm", "install-vllm"]); - -function localProviderProbeIntent(providerKey: string | null): { - ollama: boolean; - vllm: boolean; -} { - if (!providerKey) return { ollama: true, vllm: true }; - return { - ollama: OLLAMA_PROBE_PROVIDER_KEYS.has(providerKey), - vllm: VLLM_PROBE_PROVIDER_KEYS.has(providerKey), - }; -} - -function localProbeRouteProvider(providerKey: string | null): string | null { - if (providerKey && OLLAMA_PROBE_PROVIDER_KEYS.has(providerKey)) return "ollama-local"; - if (providerKey && VLLM_ROUTE_PROVIDER_KEYS.has(providerKey)) return "vllm-local"; - return null; -} - -function prepareProviderDiscovery(options: { - deps: SetupNimFlowDeps; - sandboxName: string | null; - recoverProvider: boolean; - rebuildRegistryInferenceRoute: RebuildRouteHandoff | null; - assertRouteCompatible?: (route: ProviderInferenceProbeRoute) => GatewayRouteDiscoveryConstraints; - canProbeRoute?: (provider: string) => boolean; - recoverySessionId: string | null | undefined; -}): { - requestedProvider: string | null; - requestedModel: string | null; - recoveredRegistryRoute: RegistryInferenceRoute | null; - probeOllama: boolean; - probeVllm: boolean; -} { - const { - deps, - sandboxName, - recoverProvider, - rebuildRegistryInferenceRoute, - assertRouteCompatible, - canProbeRoute, - recoverySessionId, - } = options; - const nonInteractive = deps.isNonInteractive(); - const requestedProvider = deps.getNonInteractiveProvider(); - const requestedModel = nonInteractive - ? deps.getNonInteractiveModel(requestedProvider || "build") - : null; - const recoveredRegistryRoute = - rebuildRegistryInferenceRoute?.sandboxName === sandboxName && - rebuildRegistryInferenceRoute.route.source === "registry" - ? rebuildRegistryInferenceRoute.route - : null; - const recoveredProbeProvider = - nonInteractive && !requestedProvider && recoverProvider - ? (recoveredRegistryRoute?.provider ?? - deps.readRecordedProvider(sandboxName, recoverySessionId)) - : null; - const recoveredProbeKey = providerNameToOptionKey( - deps.remoteProviderConfig, - recoveredProbeProvider, - { - hasNimContainer: - recoveredProbeProvider === "vllm-local" && - Boolean(deps.readRecordedNimContainer(sandboxName, recoverySessionId)), - }, - ); - const providerIntentKey = - requestedProvider || recoveredProbeKey || (nonInteractive ? "build" : null); - const intent = localProviderProbeIntent(providerIntentKey); - const guardedProvider = localProbeRouteProvider(providerIntentKey); - if (guardedProvider && assertRouteCompatible) { - const recoveredModel = - recoveredRegistryRoute?.model ?? - (!requestedProvider && recoverProvider - ? deps.readRecordedModel(sandboxName, recoverySessionId) - : null); - assertRouteCompatible({ - provider: guardedProvider, - model: requestedModel || recoveredModel, - endpointUrl: null, - preferredInferenceApi: null, - credentialEnv: null, - }); - } - const ollamaPreflightPassed = - guardedProvider === "ollama-local" && Boolean(assertRouteCompatible); - const vllmPreflightPassed = guardedProvider === "vllm-local" && Boolean(assertRouteCompatible); - return { - requestedProvider, - requestedModel, - recoveredRegistryRoute, - probeOllama: - intent.ollama && (ollamaPreflightPassed || (canProbeRoute?.("ollama-local") ?? true)), - probeVllm: intent.vllm && (vllmPreflightPassed || (canProbeRoute?.("vllm-local") ?? true)), - }; -} - export function createSetupNim( defaults: SetupNimFlowDeps, overrides: Partial = {}, @@ -401,16 +296,22 @@ export function createSetupNim( return state; }; - const { requestedProvider, requestedModel, recoveredRegistryRoute, probeOllama, probeVllm } = - prepareProviderDiscovery({ - deps, - sandboxName, - recoverProvider, - rebuildRegistryInferenceRoute, - assertRouteCompatible, - canProbeRoute, - recoverySessionId, - }); + const { + requestedProvider, + requestedModel, + recoveredRegistryRoute, + recordedProviderReaders, + probeOllama, + probeVllm, + } = prepareProviderDiscovery({ + deps, + sandboxName, + recoverProvider, + rebuildRegistryInferenceRoute, + assertRouteCompatible, + canProbeRoute, + recoverySessionId, + }); const providerHostState = deps.detectInferenceProviderHostState({ gpu, experimental: deps.experimental, @@ -489,18 +390,7 @@ export function createSetupNim( isWindowsHostOllama, windowsHostOllamaSupported: windowsHostOllamaDockerRequirement.supported, hermesProviderAvailable, - readRecordedProvider: recoverProvider - ? (name) => - recoveredRegistryRoute?.provider ?? - deps.readRecordedProvider(name, recoverySessionId) - : () => null, - readRecordedNimContainer: recoverProvider - ? (name) => deps.readRecordedNimContainer(name, recoverySessionId) - : () => null, - readRecordedModel: recoverProvider - ? (name) => - recoveredRegistryRoute?.model ?? deps.readRecordedModel(name, recoverySessionId) - : () => null, + ...recordedProviderReaders, }); if (providerSelection.kind === "failure") { reportProviderSelectionFailure({ diff --git a/src/lib/onboard/setup-nim-provider-discovery.ts b/src/lib/onboard/setup-nim-provider-discovery.ts new file mode 100644 index 0000000000..54914b4e5c --- /dev/null +++ b/src/lib/onboard/setup-nim-provider-discovery.ts @@ -0,0 +1,164 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { GatewayRouteDiscoveryConstraints } from "../inference/gateway-route-compatibility"; +import type { ProviderInferenceProbeRoute } from "./machine/handlers/provider-inference-route-containment"; +import { providerNameToOptionKey } from "./provider-recovery"; +import type { RebuildRouteHandoff, RegistryInferenceRoute } from "./rebuild-route-handoff"; + +interface ProviderDiscoveryDeps { + remoteProviderConfig: Record; + isNonInteractive(): boolean; + getNonInteractiveProvider(): string | null; + getNonInteractiveModel(providerKey: string): string | null; + readRecordedProvider( + sandboxName: string | null | undefined, + recoverySessionId?: string | null, + ): string | null; + readRecordedNimContainer( + sandboxName: string | null | undefined, + recoverySessionId?: string | null, + ): string | null; + readRecordedModel( + sandboxName: string | null | undefined, + recoverySessionId?: string | null, + ): string | null; +} + +interface RecordedProviderReaders { + readRecordedProvider(sandboxName: string | null | undefined): string | null; + readRecordedNimContainer(sandboxName: string | null | undefined): string | null; + readRecordedModel(sandboxName: string | null | undefined): string | null; +} + +const OLLAMA_PROBE_PROVIDER_KEYS = new Set([ + "ollama", + "install-ollama", + "start-windows-ollama", + "install-windows-ollama", +]); +const VLLM_ROUTE_PROVIDER_KEYS = new Set(["vllm", "install-vllm"]); +const VLLM_PROBE_PROVIDER_KEYS = new Set(["vllm", "install-vllm"]); + +function localProviderProbeIntent(providerKey: string | null): { + ollama: boolean; + vllm: boolean; +} { + if (!providerKey) return { ollama: true, vllm: true }; + return { + ollama: OLLAMA_PROBE_PROVIDER_KEYS.has(providerKey), + vllm: VLLM_PROBE_PROVIDER_KEYS.has(providerKey), + }; +} + +function localProbeRouteProvider(providerKey: string | null): string | null { + if (providerKey && OLLAMA_PROBE_PROVIDER_KEYS.has(providerKey)) return "ollama-local"; + if (providerKey && VLLM_ROUTE_PROVIDER_KEYS.has(providerKey)) return "vllm-local"; + return null; +} + +function bindRecordedProviderReaders( + deps: ProviderDiscoveryDeps, + recoverProvider: boolean, + recoveredRegistryRoute: RegistryInferenceRoute | null, + recoverySessionId: string | null | undefined, +): RecordedProviderReaders { + if (!recoverProvider) { + return { + readRecordedProvider: () => null, + readRecordedNimContainer: () => null, + readRecordedModel: () => null, + }; + } + return { + readRecordedProvider: (name) => + recoveredRegistryRoute?.provider ?? deps.readRecordedProvider(name, recoverySessionId), + readRecordedNimContainer: (name) => deps.readRecordedNimContainer(name, recoverySessionId), + readRecordedModel: (name) => + recoveredRegistryRoute?.model ?? deps.readRecordedModel(name, recoverySessionId), + }; +} + +export function prepareProviderDiscovery(options: { + deps: ProviderDiscoveryDeps; + sandboxName: string | null; + recoverProvider: boolean; + rebuildRegistryInferenceRoute: RebuildRouteHandoff | null; + assertRouteCompatible?: (route: ProviderInferenceProbeRoute) => GatewayRouteDiscoveryConstraints; + canProbeRoute?: (provider: string) => boolean; + recoverySessionId: string | null | undefined; +}): { + requestedProvider: string | null; + requestedModel: string | null; + recoveredRegistryRoute: RegistryInferenceRoute | null; + recordedProviderReaders: RecordedProviderReaders; + probeOllama: boolean; + probeVllm: boolean; +} { + const { + deps, + sandboxName, + recoverProvider, + rebuildRegistryInferenceRoute, + assertRouteCompatible, + canProbeRoute, + recoverySessionId, + } = options; + const nonInteractive = deps.isNonInteractive(); + const requestedProvider = deps.getNonInteractiveProvider(); + const requestedModel = nonInteractive + ? deps.getNonInteractiveModel(requestedProvider || "build") + : null; + const recoveredRegistryRoute = + rebuildRegistryInferenceRoute?.sandboxName === sandboxName && + rebuildRegistryInferenceRoute.route.source === "registry" + ? rebuildRegistryInferenceRoute.route + : null; + const recordedProviderReaders = bindRecordedProviderReaders( + deps, + recoverProvider, + recoveredRegistryRoute, + recoverySessionId, + ); + const recoveredProbeProvider = + nonInteractive && !requestedProvider + ? recordedProviderReaders.readRecordedProvider(sandboxName) + : null; + const recoveredProbeKey = providerNameToOptionKey( + deps.remoteProviderConfig, + recoveredProbeProvider, + { + hasNimContainer: + recoveredProbeProvider === "vllm-local" && + Boolean(recordedProviderReaders.readRecordedNimContainer(sandboxName)), + }, + ); + const providerIntentKey = + requestedProvider || recoveredProbeKey || (nonInteractive ? "build" : null); + const intent = localProviderProbeIntent(providerIntentKey); + const guardedProvider = localProbeRouteProvider(providerIntentKey); + if (guardedProvider && assertRouteCompatible) { + const recoveredModel = + recoveredRegistryRoute?.model ?? + (!requestedProvider ? recordedProviderReaders.readRecordedModel(sandboxName) : null); + assertRouteCompatible({ + provider: guardedProvider, + model: requestedModel || recoveredModel, + endpointUrl: null, + preferredInferenceApi: null, + credentialEnv: null, + }); + } + const ollamaPreflightPassed = + guardedProvider === "ollama-local" && Boolean(assertRouteCompatible); + const vllmPreflightPassed = guardedProvider === "vllm-local" && Boolean(assertRouteCompatible); + return { + requestedProvider, + requestedModel, + recoveredRegistryRoute, + recordedProviderReaders, + probeOllama: + intent.ollama && (ollamaPreflightPassed || (canProbeRoute?.("ollama-local") ?? true)), + probeVllm: intent.vllm && (vllmPreflightPassed || (canProbeRoute?.("vllm-local") ?? true)), + }; +}