diff --git a/src/lib/onboard/authoritative-rebuild-target.test.ts b/src/lib/onboard/authoritative-rebuild-target.test.ts index 4f00be3d1d..d8ff4f1bac 100644 --- a/src/lib/onboard/authoritative-rebuild-target.test.ts +++ b/src/lib/onboard/authoritative-rebuild-target.test.ts @@ -101,6 +101,8 @@ describe("prepared provider reconfiguration handoff", () => { resume: true, recreateSandbox: true, onboardLockAlreadyHeld: true, + targetGatewayName: "nemoclaw-8081", + targetGatewayPort: 8081, rebuildProviderReconfigure: providerTarget, }; @@ -115,13 +117,30 @@ describe("prepared provider reconfiguration handoff", () => { }); }); + it("authorizes incomplete-session recovery only for the locked rebuild context", () => { + const recoveryOptions = { ...authorizedOptions, rebuildProviderReconfigure: undefined }; + expect(rebuildProviderFlowOptions(recoveryOptions, providerTarget)).toEqual({ + authoritativeResumeConfig: true, + forceInferenceSetup: false, + }); + for (const options of [ + { ...recoveryOptions, resume: false }, + { ...recoveryOptions, recreateSandbox: false }, + { ...recoveryOptions, onboardLockAlreadyHeld: false }, + ]) { + expect(() => rebuildProviderFlowOptions(options, providerTarget)).toThrow( + "requires a preflighted locked rebuild resume", + ); + } + }); + it("rejects an unauthorized or mismatched handoff (#6114)", () => { expect(() => rebuildProviderFlowOptions( { ...authorizedOptions, onboardLockAlreadyHeld: false }, providerTarget, ), - ).toThrow("requires an authoritative locked rebuild resume"); + ).toThrow("requires a preflighted locked rebuild resume"); expect(() => rebuildProviderFlowOptions(authorizedOptions, { ...providerTarget, diff --git a/src/lib/onboard/authoritative-rebuild-target.ts b/src/lib/onboard/authoritative-rebuild-target.ts index 9b562f582d..6dca8bdc68 100644 --- a/src/lib/onboard/authoritative-rebuild-target.ts +++ b/src/lib/onboard/authoritative-rebuild-target.ts @@ -112,8 +112,25 @@ export function rebuildProviderFlowOptions( opts: OnboardOptions, target: Parameters[1], ): { authoritativeResumeConfig: boolean; forceInferenceSetup: boolean } { + const authoritativeResumeConfig = opts.authoritativeResumeConfig === true; + if (authoritativeResumeConfig) { + const gateway = resolveAuthoritativeOnboardGatewayBinding(opts); + if ( + opts.resume !== true || + opts.recreateSandbox !== true || + opts.onboardLockAlreadyHeld !== true || + !gateway || + !target.sandboxName || + !target.provider || + !target.model + ) { + throw new Error( + "Authoritative provider recovery requires a preflighted locked rebuild resume.", + ); + } + } return { - authoritativeResumeConfig: opts.authoritativeResumeConfig === true, + authoritativeResumeConfig, forceInferenceSetup: validateRebuildHandoff(opts, target), }; } diff --git a/src/lib/onboard/machine/handlers/provider-inference-authoritative-recovery.test.ts b/src/lib/onboard/machine/handlers/provider-inference-authoritative-recovery.test.ts new file mode 100644 index 0000000000..b9b746d512 --- /dev/null +++ b/src/lib/onboard/machine/handlers/provider-inference-authoritative-recovery.test.ts @@ -0,0 +1,76 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import { createSession } from "../../../state/onboard-session"; +import { handleProviderInferenceState } from "./provider-inference"; +import { baseOptions, baseSelection, createDeps } from "./provider-inference.test-support"; + +describe("authoritative provider inference recovery", () => { + it("stays enabled across messaging revalidation", async () => { + const session = createSession({ + sandboxName: "my-assistant", + provider: "compatible-endpoint", + model: "mock/channels-rebuild", + endpointUrl: "https://compatible.example.test/v1", + credentialEnv: "COMPATIBLE_API_KEY", + preferredInferenceApi: "openai-completions", + }); + const recoveredSelection = { + ...baseSelection, + model: "mock/channels-rebuild", + provider: "compatible-endpoint", + endpointUrl: "https://compatible.example.test/v1", + credentialEnv: "COMPATIBLE_API_KEY", + preferredInferenceApi: "openai-completions", + recoveredFromSandbox: true, + skipHostInferenceSmoke: true, + reuseGatewayCredentialWithoutLocalKey: true, + }; + const setupNim = vi.fn(async () => recoveredSelection); + const { deps, calls } = createDeps({ + setupNim, + hydrateCredentialEnv: vi.fn(() => null), + isInferenceRouteReady: vi.fn(() => true), + }); + + const result = await handleProviderInferenceState({ + ...baseOptions(deps, session), + resume: true, + authoritativeResumeConfig: true, + sandboxName: "my-assistant", + selectedMessagingChannels: ["telegram"], + }); + + expect(setupNim).toHaveBeenCalledWith( + { type: "nvidia" }, + "my-assistant", + null, + true, + "nemoclaw", + expect.any(Function), + expect.any(Function), + session.sessionId, + ); + expect(calls.setupInference).toHaveBeenCalledWith( + "my-assistant", + "mock/channels-rebuild", + "compatible-endpoint", + "https://compatible.example.test/v1", + "COMPATIBLE_API_KEY", + null, + [], + expect.objectContaining({ + skipHostInferenceSmoke: true, + reuseGatewayCredentialWithoutLocalKey: true, + reservationSessionId: session.sessionId, + }), + ); + expect(result).toMatchObject({ + provider: "compatible-endpoint", + model: "mock/channels-rebuild", + endpointUrl: "https://compatible.example.test/v1", + }); + }); +}); diff --git a/src/lib/onboard/machine/handlers/provider-inference-recovery-gating.test.ts b/src/lib/onboard/machine/handlers/provider-inference-recovery-gating.test.ts index b56bf39d51..2de32e5c4c 100644 --- a/src/lib/onboard/machine/handlers/provider-inference-recovery-gating.test.ts +++ b/src/lib/onboard/machine/handlers/provider-inference-recovery-gating.test.ts @@ -59,6 +59,101 @@ describe("provider inference recovery gating", () => { ); }); + it("allows a preflighted authoritative rebuild to recover from its incomplete session", async () => { + const session = createSession({ + provider: "compatible-endpoint", + model: "mock/channels-rebuild", + endpointUrl: "https://compatible.example.test/v1", + credentialEnv: "COMPATIBLE_API_KEY", + preferredInferenceApi: "openai-completions", + }); + session.sandboxName = "dc-after"; + const { deps, calls } = createDeps(); + + await handleProviderInferenceState({ + ...baseOptions(deps, session), + resume: true, + forceProviderSelection: true, + authoritativeResumeConfig: true, + sandboxName: "dc-after", + }); + + expect(calls.setupNim).toHaveBeenCalledWith( + { type: "nvidia" }, + "dc-after", + null, + true, + "nemoclaw", + expect.any(Function), + expect.any(Function), + session.sessionId, + ); + }); + + it("rejects an authoritative incomplete session for a different sandbox", async () => { + const session = createSession({ + sandboxName: "dc-before", + provider: "compatible-endpoint", + model: "mock/channels-rebuild", + endpointUrl: "https://compatible.example.test/v1", + credentialEnv: "COMPATIBLE_API_KEY", + preferredInferenceApi: "openai-completions", + }); + const { deps, calls } = createDeps(); + + await handleProviderInferenceState({ + ...baseOptions(deps, session), + resume: true, + forceProviderSelection: true, + authoritativeResumeConfig: true, + sandboxName: "dc-after", + }); + + expect(calls.setupNim).toHaveBeenCalledWith( + { type: "nvidia" }, + "dc-after", + null, + false, + "nemoclaw", + expect.any(Function), + expect.any(Function), + session.sessionId, + ); + }); + + it("rejects an authoritative incomplete session with a foreign reservation", async () => { + const session = createSession(); + session.sandboxName = "dc-after"; + const entry: SandboxEntry = { + name: "dc-after", + pendingRouteReservation: true, + reservationSessionId: "session-other", + }; + const getAuthority = vi.fn((_name: string, sessionId: string | null | undefined) => + classifySandboxRecoveryAuthority(entry, sessionId), + ); + const { deps, calls } = createDeps({ getSandboxRecoveryAuthority: getAuthority }); + + await handleProviderInferenceState({ + ...baseOptions(deps, session), + resume: true, + forceProviderSelection: true, + authoritativeResumeConfig: true, + sandboxName: "dc-after", + }); + + expect(calls.setupNim).toHaveBeenCalledWith( + { type: "nvidia" }, + "dc-after", + null, + false, + "nemoclaw", + expect.any(Function), + expect.any(Function), + session.sessionId, + ); + }); + it("allows recovery for a registered sandbox", async () => { const getAuthority = vi.fn((name: string) => name === "dc-after" ? ("authorized" as const) : ("missing" as const), @@ -154,10 +249,9 @@ describe("provider inference recovery gating", () => { expect(getAuthority).toHaveBeenCalledWith("dc-after", session.sessionId); }); - it("forces route setup and rechecks ownership after recorded selection (#6630)", async () => { + it("rechecks an authoritative incomplete session after recorded selection (#6630)", async () => { const session = createSession(); session.sandboxName = "dc-after"; - session.steps.sandbox.status = "complete"; const persistedAfterSelection = createSession({ sessionId: "session-after-selection" }); let authority: "authorized" | "unauthorized" = "authorized"; const getAuthority = vi.fn((_name: string, sessionId: string | null | undefined) => @@ -197,6 +291,7 @@ describe("provider inference recovery gating", () => { await handleProviderInferenceState({ ...baseOptions(deps, session), resume: true, + authoritativeResumeConfig: true, sandboxName: "dc-after", }); diff --git a/src/lib/onboard/machine/handlers/provider-inference-recovery.ts b/src/lib/onboard/machine/handlers/provider-inference-recovery.ts index d12bd1acae..c5cba918bd 100644 --- a/src/lib/onboard/machine/handlers/provider-inference-recovery.ts +++ b/src/lib/onboard/machine/handlers/provider-inference-recovery.ts @@ -21,11 +21,23 @@ interface ProviderRecoverySetupOptions { isRecordedProviderRecoveryAuthorized?: () => boolean; } +interface ProviderRecoveryOptions { + /** + * Rebuild recreate replaces the durable session after deleting the old sandbox, so its sandbox + * step must remain incomplete until creation succeeds. The locked rebuild pipeline validates the + * target before deletion, then writes that exact identity into the pending session before onboard. + * Remove this exception once #6666 replaces the handoff with a dedicated provider-recovery + * authorization receipt. + */ + authoritativeResumeConfig?: boolean; +} + export function createRecovery( fresh: boolean, sandboxName: string | null, session: Session | null, deps: ProviderRecoveryDeps, + options: ProviderRecoveryOptions = {}, ): { sessionId: string | null; shouldRecover(): boolean; @@ -46,7 +58,9 @@ export function createRecovery( ? deps.getSandboxRecoveryAuthority(sandboxName, sessionId) : "missing", sessionSandboxName: - session?.steps?.sandbox?.status === "complete" ? (session.sandboxName ?? null) : null, + session?.steps?.sandbox?.status === "complete" || options.authoritativeResumeConfig + ? (session?.sandboxName ?? null) + : null, }), setupOptions(recoveredRecordedProvider, selectedSandboxName, currentSessionId) { if (!recoveredRecordedProvider) return { reservationSessionId: currentSessionId }; diff --git a/src/lib/onboard/machine/handlers/provider-inference.ts b/src/lib/onboard/machine/handlers/provider-inference.ts index 254304e9e5..db794d2d0c 100644 --- a/src/lib/onboard/machine/handlers/provider-inference.ts +++ b/src/lib/onboard/machine/handlers/provider-inference.ts @@ -344,7 +344,9 @@ export async function handleProviderInferenceState({ clearAutoDetectedCompatibleContextWindow(process.env); let forceInferenceSetup = initialForceInferenceSetup; let recoveredRecordedProvider = false; - const providerRecovery = createRecovery(fresh, sandboxName, session, deps); + const providerRecovery = createRecovery(fresh, sandboxName, session, deps, { + authoritativeResumeConfig, + }); const resumeProviderSelection = !forceProviderSelection && effectiveResume && diff --git a/test/onboard-fsm-live-slices.test.ts b/test/onboard-fsm-live-slices.test.ts index 5b8a2f91bf..9fc4f54915 100644 --- a/test/onboard-fsm-live-slices.test.ts +++ b/test/onboard-fsm-live-slices.test.ts @@ -355,6 +355,8 @@ const { onboard } = require(${onboardPath}); ...(scenario.mode === "authoritative-core-gateway" ? { authoritativeResumeConfig: true, + recreateSandbox: true, + onboardLockAlreadyHeld: true, targetGatewayName: "nemoclaw-9090", targetGatewayPort: 9090, }