Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 20 additions & 1 deletion src/lib/onboard/authoritative-rebuild-target.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,8 @@ describe("prepared provider reconfiguration handoff", () => {
resume: true,
recreateSandbox: true,
onboardLockAlreadyHeld: true,
targetGatewayName: "nemoclaw-8081",
targetGatewayPort: 8081,
rebuildProviderReconfigure: providerTarget,
};

Expand All @@ -115,13 +117,30 @@ describe("prepared provider reconfiguration handoff", () => {
});
});

it("authorizes incomplete-session recovery only for the locked rebuild context", () => {
const recoveryOptions = { ...authorizedOptions, rebuildProviderReconfigure: undefined };
expect(rebuildProviderFlowOptions(recoveryOptions, providerTarget)).toEqual({
authoritativeResumeConfig: true,
forceInferenceSetup: false,
});
for (const options of [
{ ...recoveryOptions, resume: false },
{ ...recoveryOptions, recreateSandbox: false },
{ ...recoveryOptions, onboardLockAlreadyHeld: false },
]) {
expect(() => rebuildProviderFlowOptions(options, providerTarget)).toThrow(
"requires a preflighted locked rebuild resume",
);
}
});

it("rejects an unauthorized or mismatched handoff (#6114)", () => {
expect(() =>
rebuildProviderFlowOptions(
{ ...authorizedOptions, onboardLockAlreadyHeld: false },
providerTarget,
),
).toThrow("requires an authoritative locked rebuild resume");
).toThrow("requires a preflighted locked rebuild resume");
expect(() =>
rebuildProviderFlowOptions(authorizedOptions, {
...providerTarget,
Expand Down
19 changes: 18 additions & 1 deletion src/lib/onboard/authoritative-rebuild-target.ts
Original file line number Diff line number Diff line change
Expand Up @@ -112,8 +112,25 @@ export function rebuildProviderFlowOptions(
opts: OnboardOptions,
target: Parameters<typeof validateRebuildHandoff>[1],
): { authoritativeResumeConfig: boolean; forceInferenceSetup: boolean } {
const authoritativeResumeConfig = opts.authoritativeResumeConfig === true;
if (authoritativeResumeConfig) {
const gateway = resolveAuthoritativeOnboardGatewayBinding(opts);
if (
opts.resume !== true ||
opts.recreateSandbox !== true ||
opts.onboardLockAlreadyHeld !== true ||
!gateway ||
!target.sandboxName ||
!target.provider ||
!target.model
) {
throw new Error(
"Authoritative provider recovery requires a preflighted locked rebuild resume.",
);
}
}
return {
authoritativeResumeConfig: opts.authoritativeResumeConfig === true,
authoritativeResumeConfig,
forceInferenceSetup: validateRebuildHandoff(opts, target),
};
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import { describe, expect, it, vi } from "vitest";

import { createSession } from "../../../state/onboard-session";
import { handleProviderInferenceState } from "./provider-inference";
import { baseOptions, baseSelection, createDeps } from "./provider-inference.test-support";

describe("authoritative provider inference recovery", () => {
it("stays enabled across messaging revalidation", async () => {
const session = createSession({
sandboxName: "my-assistant",
provider: "compatible-endpoint",
model: "mock/channels-rebuild",
endpointUrl: "https://compatible.example.test/v1",
credentialEnv: "COMPATIBLE_API_KEY",
preferredInferenceApi: "openai-completions",
});
const recoveredSelection = {
...baseSelection,
model: "mock/channels-rebuild",
provider: "compatible-endpoint",
endpointUrl: "https://compatible.example.test/v1",
credentialEnv: "COMPATIBLE_API_KEY",
preferredInferenceApi: "openai-completions",
recoveredFromSandbox: true,
skipHostInferenceSmoke: true,
reuseGatewayCredentialWithoutLocalKey: true,
};
const setupNim = vi.fn(async () => recoveredSelection);
const { deps, calls } = createDeps({
setupNim,
hydrateCredentialEnv: vi.fn(() => null),
isInferenceRouteReady: vi.fn(() => true),
});

const result = await handleProviderInferenceState({
...baseOptions(deps, session),
resume: true,
authoritativeResumeConfig: true,
sandboxName: "my-assistant",
selectedMessagingChannels: ["telegram"],
});

expect(setupNim).toHaveBeenCalledWith(
{ type: "nvidia" },
"my-assistant",
null,
true,
"nemoclaw",
expect.any(Function),
expect.any(Function),
session.sessionId,
);
expect(calls.setupInference).toHaveBeenCalledWith(
"my-assistant",
"mock/channels-rebuild",
"compatible-endpoint",
"https://compatible.example.test/v1",
"COMPATIBLE_API_KEY",
null,
[],
expect.objectContaining({
skipHostInferenceSmoke: true,
reuseGatewayCredentialWithoutLocalKey: true,
reservationSessionId: session.sessionId,
}),
);
expect(result).toMatchObject({
provider: "compatible-endpoint",
model: "mock/channels-rebuild",
endpointUrl: "https://compatible.example.test/v1",
});
});
});
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,101 @@ describe("provider inference recovery gating", () => {
);
});

it("allows a preflighted authoritative rebuild to recover from its incomplete session", async () => {
const session = createSession({
provider: "compatible-endpoint",
model: "mock/channels-rebuild",
endpointUrl: "https://compatible.example.test/v1",
credentialEnv: "COMPATIBLE_API_KEY",
preferredInferenceApi: "openai-completions",
});
session.sandboxName = "dc-after";
const { deps, calls } = createDeps();

await handleProviderInferenceState({
...baseOptions(deps, session),
resume: true,
forceProviderSelection: true,
authoritativeResumeConfig: true,
sandboxName: "dc-after",
});

expect(calls.setupNim).toHaveBeenCalledWith(
{ type: "nvidia" },
"dc-after",
null,
true,
"nemoclaw",
expect.any(Function),
expect.any(Function),
session.sessionId,
);
});

it("rejects an authoritative incomplete session for a different sandbox", async () => {
const session = createSession({
sandboxName: "dc-before",
provider: "compatible-endpoint",
model: "mock/channels-rebuild",
endpointUrl: "https://compatible.example.test/v1",
credentialEnv: "COMPATIBLE_API_KEY",
preferredInferenceApi: "openai-completions",
});
const { deps, calls } = createDeps();

await handleProviderInferenceState({
...baseOptions(deps, session),
resume: true,
forceProviderSelection: true,
authoritativeResumeConfig: true,
sandboxName: "dc-after",
});

expect(calls.setupNim).toHaveBeenCalledWith(
{ type: "nvidia" },
"dc-after",
null,
false,
"nemoclaw",
expect.any(Function),
expect.any(Function),
session.sessionId,
);
});

it("rejects an authoritative incomplete session with a foreign reservation", async () => {
const session = createSession();
session.sandboxName = "dc-after";
const entry: SandboxEntry = {
name: "dc-after",
pendingRouteReservation: true,
reservationSessionId: "session-other",
};
const getAuthority = vi.fn((_name: string, sessionId: string | null | undefined) =>
classifySandboxRecoveryAuthority(entry, sessionId),
);
const { deps, calls } = createDeps({ getSandboxRecoveryAuthority: getAuthority });

await handleProviderInferenceState({
...baseOptions(deps, session),
resume: true,
forceProviderSelection: true,
authoritativeResumeConfig: true,
sandboxName: "dc-after",
});

expect(calls.setupNim).toHaveBeenCalledWith(
{ type: "nvidia" },
"dc-after",
null,
false,
"nemoclaw",
expect.any(Function),
expect.any(Function),
session.sessionId,
);
});

it("allows recovery for a registered sandbox", async () => {
const getAuthority = vi.fn((name: string) =>
name === "dc-after" ? ("authorized" as const) : ("missing" as const),
Expand Down Expand Up @@ -154,10 +249,9 @@ describe("provider inference recovery gating", () => {
expect(getAuthority).toHaveBeenCalledWith("dc-after", session.sessionId);
});

it("forces route setup and rechecks ownership after recorded selection (#6630)", async () => {
it("rechecks an authoritative incomplete session after recorded selection (#6630)", async () => {
const session = createSession();
session.sandboxName = "dc-after";
session.steps.sandbox.status = "complete";
const persistedAfterSelection = createSession({ sessionId: "session-after-selection" });
let authority: "authorized" | "unauthorized" = "authorized";
const getAuthority = vi.fn((_name: string, sessionId: string | null | undefined) =>
Expand Down Expand Up @@ -197,6 +291,7 @@ describe("provider inference recovery gating", () => {
await handleProviderInferenceState({
...baseOptions(deps, session),
resume: true,
authoritativeResumeConfig: true,
sandboxName: "dc-after",
});

Expand Down
16 changes: 15 additions & 1 deletion src/lib/onboard/machine/handlers/provider-inference-recovery.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,23 @@ interface ProviderRecoverySetupOptions {
isRecordedProviderRecoveryAuthorized?: () => boolean;
}

interface ProviderRecoveryOptions {
/**
* Rebuild recreate replaces the durable session after deleting the old sandbox, so its sandbox
* step must remain incomplete until creation succeeds. The locked rebuild pipeline validates the
* target before deletion, then writes that exact identity into the pending session before onboard.
* Remove this exception once #6666 replaces the handoff with a dedicated provider-recovery
* authorization receipt.
*/
authoritativeResumeConfig?: boolean;
}

export function createRecovery(
fresh: boolean,
sandboxName: string | null,
session: Session | null,
deps: ProviderRecoveryDeps,
options: ProviderRecoveryOptions = {},
): {
sessionId: string | null;
shouldRecover(): boolean;
Expand All @@ -46,7 +58,9 @@ export function createRecovery(
? deps.getSandboxRecoveryAuthority(sandboxName, sessionId)
: "missing",
sessionSandboxName:
session?.steps?.sandbox?.status === "complete" ? (session.sandboxName ?? null) : null,
session?.steps?.sandbox?.status === "complete" || options.authoritativeResumeConfig
? (session?.sandboxName ?? null)
: null,
}),
setupOptions(recoveredRecordedProvider, selectedSandboxName, currentSessionId) {
if (!recoveredRecordedProvider) return { reservationSessionId: currentSessionId };
Expand Down
4 changes: 3 additions & 1 deletion src/lib/onboard/machine/handlers/provider-inference.ts
Original file line number Diff line number Diff line change
Expand Up @@ -344,7 +344,9 @@ export async function handleProviderInferenceState<Gpu, Agent, Host>({
clearAutoDetectedCompatibleContextWindow(process.env);
let forceInferenceSetup = initialForceInferenceSetup;
let recoveredRecordedProvider = false;
const providerRecovery = createRecovery(fresh, sandboxName, session, deps);
const providerRecovery = createRecovery(fresh, sandboxName, session, deps, {
authoritativeResumeConfig,
});
const resumeProviderSelection =
!forceProviderSelection &&
effectiveResume &&
Expand Down
2 changes: 2 additions & 0 deletions test/onboard-fsm-live-slices.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -355,6 +355,8 @@ const { onboard } = require(${onboardPath});
...(scenario.mode === "authoritative-core-gateway"
? {
authoritativeResumeConfig: true,
recreateSandbox: true,
onboardLockAlreadyHeld: true,
targetGatewayName: "nemoclaw-9090",
targetGatewayPort: 9090,
}
Expand Down