diff --git a/docs/inference/use-local-inference.mdx b/docs/inference/use-local-inference.mdx index 3ae4718fd0..bb40d10208 100644 --- a/docs/inference/use-local-inference.mdx +++ b/docs/inference/use-local-inference.mdx @@ -92,6 +92,8 @@ If the selected model declares that it does not support tool calling, onboarding The validation also requires structured chat-completions tool calls. If the model leaks tool-call JSON as plain message text, onboarding stops so you can choose a model that returns tool calls in the expected response field. If a host-side validation probe times out, NemoClaw retries the Ollama tool-call validation with a larger timeout before failing the setup. +Each Ollama-backed OpenClaw agent passthrough checks the registered route to see whether the selected model is still loaded. +If the daemon is reachable but the model is unloaded, NemoClaw sends a bounded warm-up request before dispatching to OpenClaw. On WSL, if you choose the Windows-host Ollama path, NemoClaw uses `host.docker.internal:11434` and pulls missing models through the Ollama HTTP API instead of requiring the `ollama` CLI inside WSL. diff --git a/docs/reference/troubleshooting.mdx b/docs/reference/troubleshooting.mdx index 08d8ede937..005cba7bcf 100644 --- a/docs/reference/troubleshooting.mdx +++ b/docs/reference/troubleshooting.mdx @@ -1359,6 +1359,8 @@ If that line shows `unhealthy`, `unreachable`, or `not probed`, inspect the labe For local Ollama and local vLLM, `Inference (ollama backend)` or the corresponding local-backend line reports the host-side service separately. For Local Ollama, current releases can also print `Inference (auth proxy)` when a proxy token is available. If a local backend or auth-proxy diagnostic fails, start the backend or re-run onboarding so NemoClaw can recreate the proxy token, restart the proxy, and refresh the route. +For Ollama-backed OpenClaw sandboxes, agent passthrough uses the registered host route to warm an unloaded model after an Ollama daemon restart. +If that bounded warm-up fails or times out, NemoClaw reports the result and continues so OpenClaw can emit its canonical backend error. If the endpoint is correct but requests still fail, check for network policy rules that may block the connection. Then verify the credential and base URL for the provider you selected during onboarding. diff --git a/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts b/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts new file mode 100644 index 0000000000..55712f5f5e --- /dev/null +++ b/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts @@ -0,0 +1,261 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; +import { OLLAMA_PORT, OLLAMA_PROXY_PORT } from "../../../core/ports"; +import { + maybeWarmOllamaAfterDaemonRestart, + type OllamaRestartRecoveryDeps, +} from "./ollama-restart-recovery"; + +const unloadedStatus = { + probed: true, + loaded: false, + cpuOnly: false, +}; + +function successfulWarmResult() { + return { + stdout: JSON.stringify({ response: "Hello!", done: true }), + exitCode: 0, + timedOut: false, + }; +} + +function getCommandUrl(command: readonly string[]): string { + return command.find((arg) => arg.startsWith("http://")) ?? ""; +} + +function getCommandBody(command: readonly string[]): Record { + const dataIndex = command.indexOf("-d"); + return JSON.parse(command[dataIndex + 1] ?? "null") as Record; +} + +describe("maybeWarmOllamaAfterDaemonRestart", () => { + it("skips routes that are not local Ollama", () => { + expect( + maybeWarmOllamaAfterDaemonRestart({ provider: "vllm-local", model: "meta/llama" }), + ).toEqual({ kind: "skipped", reason: "not-ollama" }); + }); + + it("skips a local Ollama route without a registered model", () => { + expect(maybeWarmOllamaAfterDaemonRestart({ provider: "ollama-local" })).toEqual({ + kind: "skipped", + reason: "missing-model", + }); + }); + + it("uses the persisted direct bridge route for both the default probe and warm-up", () => { + const runCaptureImpl = vi.fn((_command: readonly string[]) => JSON.stringify({ models: [] })); + const runCaptureExImpl = vi.fn((_command: string[]) => successfulWarmResult()); + + expect( + maybeWarmOllamaAfterDaemonRestart( + { + provider: "ollama-local", + model: "qwen3.6:35b", + endpointUrl: `http://host.openshell.internal:${OLLAMA_PORT}/v1`, + }, + { runCaptureImpl, runCaptureExImpl }, + ), + ).toEqual({ kind: "warmed", ok: true, timedOut: false }); + + expect(getCommandUrl(runCaptureImpl.mock.calls[0][0])).toBe( + `http://host.docker.internal:${OLLAMA_PORT}/api/ps`, + ); + expect(getCommandUrl(runCaptureExImpl.mock.calls[0][0])).toBe( + `http://host.docker.internal:${OLLAMA_PORT}/api/generate`, + ); + expect(getCommandBody(runCaptureExImpl.mock.calls[0][0])).toMatchObject({ + model: "qwen3.6:35b", + stream: false, + think: false, + }); + }); + + it("maps an auth-proxy route back to host loopback", () => { + const runCaptureImpl = vi.fn((_command: readonly string[]) => JSON.stringify({ models: [] })); + const runCaptureExImpl = vi.fn((_command: string[]) => successfulWarmResult()); + + maybeWarmOllamaAfterDaemonRestart( + { + provider: "ollama-local", + model: "qwen3.6:35b", + endpointUrl: `http://host.openshell.internal:${OLLAMA_PROXY_PORT}/v1`, + }, + { runCaptureImpl, runCaptureExImpl }, + ); + + expect(getCommandUrl(runCaptureImpl.mock.calls[0][0])).toBe( + `http://127.0.0.1:${OLLAMA_PORT}/api/ps`, + ); + expect(getCommandUrl(runCaptureExImpl.mock.calls[0][0])).toBe( + `http://127.0.0.1:${OLLAMA_PORT}/api/generate`, + ); + }); + + it("falls back to an allowlisted host instead of probing an arbitrary registry URL", () => { + const runCaptureImpl = vi.fn((_command: readonly string[]) => JSON.stringify({ models: [] })); + const runCaptureExImpl = vi.fn((_command: string[]) => successfulWarmResult()); + + maybeWarmOllamaAfterDaemonRestart( + { + provider: "ollama-local", + model: "qwen3.6:35b", + endpointUrl: `http://example.com:${OLLAMA_PORT}/v1`, + }, + { + getOllamaHost: () => "also.example.com", + runCaptureImpl, + runCaptureExImpl, + }, + ); + + expect(getCommandUrl(runCaptureImpl.mock.calls[0][0])).toContain("http://127.0.0.1:"); + expect(getCommandUrl(runCaptureExImpl.mock.calls[0][0])).toContain("http://127.0.0.1:"); + }); + + it("does not map an unrecognized proxy-port host to host loopback (#6039)", () => { + const runCaptureImpl = vi.fn((_command: readonly string[]) => JSON.stringify({ models: [] })); + const runCaptureExImpl = vi.fn((_command: string[]) => successfulWarmResult()); + + maybeWarmOllamaAfterDaemonRestart( + { + provider: "ollama-local", + model: "qwen3.6:35b", + endpointUrl: `http://example.com:${OLLAMA_PROXY_PORT}/v1`, + }, + { + getOllamaHost: () => "host.docker.internal", + runCaptureImpl, + runCaptureExImpl, + }, + ); + + expect(getCommandUrl(runCaptureImpl.mock.calls[0][0])).toBe( + `http://host.docker.internal:${OLLAMA_PORT}/api/ps`, + ); + expect(getCommandUrl(runCaptureExImpl.mock.calls[0][0])).toBe( + `http://host.docker.internal:${OLLAMA_PORT}/api/generate`, + ); + }); + + it("skips the warm-up when the selected model is already loaded", () => { + const probeRuntimeModelStatus = vi.fn(() => ({ + probed: true, + loaded: true, + cpuOnly: false, + })); + const runCaptureExImpl = vi.fn(() => successfulWarmResult()); + + expect( + maybeWarmOllamaAfterDaemonRestart( + { provider: "ollama-local", model: "qwen3.6:35b" }, + { probeRuntimeModelStatus, runCaptureExImpl }, + ), + ).toEqual({ kind: "skipped", reason: "already-loaded" }); + expect(runCaptureExImpl).not.toHaveBeenCalled(); + }); + + it("skips the warm-up when the daemon probe is unreachable", () => { + const runCaptureExImpl = vi.fn(() => successfulWarmResult()); + + expect( + maybeWarmOllamaAfterDaemonRestart( + { provider: "ollama-local", model: "qwen3.6:35b" }, + { runCaptureImpl: () => "", runCaptureExImpl }, + ), + ).toEqual({ kind: "skipped", reason: "unreachable" }); + expect(runCaptureExImpl).not.toHaveBeenCalled(); + }); + + it("reports a bounded warm-up timeout", () => { + expect( + maybeWarmOllamaAfterDaemonRestart( + { provider: "ollama-local", model: "qwen3.6:35b" }, + { + probeRuntimeModelStatus: () => unloadedStatus, + runCaptureExImpl: () => ({ + stdout: "", + exitCode: 28, + timedOut: true, + }), + }, + ), + ).toEqual({ kind: "warmed", ok: false, timedOut: true, reason: "timeout" }); + }); + + it("does not treat an exit-zero Ollama error body as a successful warm-up", () => { + expect( + maybeWarmOllamaAfterDaemonRestart( + { provider: "ollama-local", model: "missing:latest" }, + { + probeRuntimeModelStatus: () => unloadedStatus, + runCaptureExImpl: () => ({ + stdout: JSON.stringify({ error: "model not found" }), + exitCode: 0, + timedOut: false, + }), + }, + ), + ).toEqual({ kind: "warmed", ok: false, timedOut: false, reason: "ollama-error" }); + }); + + it("accepts a completed thinking-only response from a thinking model", () => { + expect( + maybeWarmOllamaAfterDaemonRestart( + { provider: "ollama-local", model: "qwen3.6:35b" }, + { + probeRuntimeModelStatus: () => unloadedStatus, + runCaptureExImpl: () => ({ + stdout: JSON.stringify({ response: "", thinking: "The model is ready.", done: true }), + exitCode: 0, + timedOut: false, + }), + }, + ), + ).toEqual({ kind: "warmed", ok: true, timedOut: false }); + }); + + it.each([ + ["empty body", ""], + ["malformed JSON", "not-json"], + ["missing done marker", JSON.stringify({ response: "Hello!" })], + ["empty response", JSON.stringify({ response: "", done: true })], + ])("rejects an invalid warm response: %s", (_name, stdout) => { + expect( + maybeWarmOllamaAfterDaemonRestart( + { provider: "ollama-local", model: "qwen3.6:35b" }, + { + probeRuntimeModelStatus: () => unloadedStatus, + runCaptureExImpl: () => ({ stdout, exitCode: 0, timedOut: false }), + }, + ), + ).toEqual({ kind: "warmed", ok: false, timedOut: false, reason: "invalid-response" }); + }); + + it("reports a non-zero warm command exit", () => { + expect( + maybeWarmOllamaAfterDaemonRestart( + { provider: "ollama-local", model: "qwen3.6:35b" }, + { + probeRuntimeModelStatus: () => unloadedStatus, + runCaptureExImpl: () => ({ stdout: "", exitCode: 7, timedOut: false }), + }, + ), + ).toEqual({ kind: "warmed", ok: false, timedOut: false, reason: "command-failed" }); + }); + + it("reports a warm process spawn failure without throwing", () => { + const deps: OllamaRestartRecoveryDeps = { + probeRuntimeModelStatus: () => unloadedStatus, + runCaptureExImpl: () => { + throw new Error("spawn failed"); + }, + }; + + expect( + maybeWarmOllamaAfterDaemonRestart({ provider: "ollama-local", model: "qwen3.6:35b" }, deps), + ).toEqual({ kind: "warmed", ok: false, timedOut: false, reason: "spawn-failed" }); + }); +}); diff --git a/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts b/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts new file mode 100644 index 0000000000..e4cd06927b --- /dev/null +++ b/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts @@ -0,0 +1,234 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Source-of-truth boundary for Ollama restart recovery. +// +// Invalid state: restarting the external Ollama daemon drops its loaded model, +// so the first OpenClaw turn can exhaust its request budget cold-loading it. +// Ollama owns daemon/model lifecycle; NemoClaw owns the persisted inference +// route and the host-side passthrough that can perform a bounded warm-up before +// dispatch. This cannot be fixed at the producer in this PR because Ollama does +// not persist loaded runners across daemon restarts. Focused tests cover direct +// and proxied route translation, unreachable/already-loaded states, timeouts, +// process failures, and semantic response validation. Remove this recovery when +// supported Ollama versions persist runners across restart, or when NemoClaw +// manages daemon lifecycle and can warm the model at restart time instead. + +import { buildValidatedCurlCommandArgs } from "../../../adapters/http/curl-args"; +import { OLLAMA_PORT, OLLAMA_PROXY_PORT } from "../../../core/ports"; +import { + getResolvedOllamaHost, + OLLAMA_HOST_DOCKER_INTERNAL, + OLLAMA_LOCALHOST, + type RunCaptureExFn, +} from "../../../inference/local"; +import { + type OllamaRuntimeModelStatus, + type OllamaRuntimeRunCaptureFn, + probeOllamaRuntimeModelStatus, +} from "../../../inference/ollama-runtime-context"; +import { runCaptureEx } from "../../../runner"; + +export interface OllamaRestartRecoveryRoute { + provider?: string | null; + model?: string | null; + endpointUrl?: string | null; +} + +export interface OllamaRestartRecoveryDeps { + probeRuntimeModelStatus?: ( + model: string, + getOllamaHost: () => string, + runCaptureImpl?: OllamaRuntimeRunCaptureFn, + ) => OllamaRuntimeModelStatus; + runCaptureExImpl?: RunCaptureExFn; + getOllamaHost?: () => string; + runCaptureImpl?: OllamaRuntimeRunCaptureFn; +} + +export type OllamaRestartRecoveryFailureReason = + | "timeout" + | "command-failed" + | "ollama-error" + | "invalid-response" + | "spawn-failed"; + +export type OllamaRestartRecoveryResult = + | { kind: "skipped"; reason: "not-ollama" | "missing-model" | "already-loaded" | "unreachable" } + | { kind: "warmed"; ok: true; timedOut: false } + | { + kind: "warmed"; + ok: false; + timedOut: boolean; + reason: OllamaRestartRecoveryFailureReason; + }; + +export const OLLAMA_LOCAL_PROVIDER = "ollama-local"; +const OLLAMA_RESTART_RECOVERY_TIMEOUT_SECONDS = 300; +const OPENSHELL_HOST_BRIDGE = "host.openshell.internal"; +const ALLOWED_RAW_OLLAMA_HOSTS = new Set([ + OLLAMA_LOCALHOST, + "localhost", + OLLAMA_HOST_DOCKER_INTERNAL, +]); + +function normalizeRouteValue(value: string | null | undefined): string { + return String(value ?? "").trim(); +} + +function normalizeHostname(value: string): string { + return (value.startsWith("[") && value.endsWith("]") ? value.slice(1, -1) : value) + .replace(/\.$/, "") + .toLowerCase(); +} + +function getAllowedFallbackHost(getOllamaHost: () => string): string { + try { + const host = normalizeHostname(getOllamaHost()); + return ALLOWED_RAW_OLLAMA_HOSTS.has(host) ? host : OLLAMA_LOCALHOST; + } catch { + return OLLAMA_LOCALHOST; + } +} + +/** + * Translate the persisted sandbox-facing route back to the host-side daemon. + * Only fixed local bridge names are accepted so edited registry data cannot + * turn this recovery probe into an arbitrary host request. + */ +function resolveRawOllamaHost( + endpointUrl: string | null | undefined, + getOllamaHost: () => string, +): string { + try { + const endpoint = new URL(normalizeRouteValue(endpointUrl)); + const hostname = normalizeHostname(endpoint.hostname); + const port = Number(endpoint.port || (endpoint.protocol === "https:" ? 443 : 80)); + + if ( + endpoint.protocol === "http:" && + hostname === OPENSHELL_HOST_BRIDGE && + port === OLLAMA_PORT + ) { + return OLLAMA_HOST_DOCKER_INTERNAL; + } + if ( + endpoint.protocol === "http:" && + hostname === OPENSHELL_HOST_BRIDGE && + port === OLLAMA_PROXY_PORT + ) { + return OLLAMA_LOCALHOST; + } + if ( + endpoint.protocol === "http:" && + port === OLLAMA_PORT && + ALLOWED_RAW_OLLAMA_HOSTS.has(hostname) + ) { + return hostname; + } + } catch { + // Missing and legacy registry endpoints use the process-local resolved host. + } + + return getAllowedFallbackHost(getOllamaHost); +} + +function buildWarmCommand(model: string, hostname: string): string[] { + const body = JSON.stringify({ + model, + prompt: "Hello, reply in less than 5 words", + stream: false, + think: false, + keep_alive: "15m", + options: { num_predict: 16 }, + }); + return [ + "curl", + ...buildValidatedCurlCommandArgs([ + "-sS", + "--connect-timeout", + "3", + "--max-time", + String(OLLAMA_RESTART_RECOVERY_TIMEOUT_SECONDS), + "-H", + "Content-Type: application/json", + "-d", + body, + `http://${hostname}:${OLLAMA_PORT}/api/generate`, + ]), + ]; +} + +function validateWarmResponse(stdout: string): "ok" | "ollama-error" | "invalid-response" { + try { + const parsed = JSON.parse(stdout) as { + done?: unknown; + error?: unknown; + response?: unknown; + thinking?: unknown; + }; + if (typeof parsed.error === "string" && parsed.error.trim() !== "") { + return "ollama-error"; + } + const response = typeof parsed.response === "string" ? parsed.response.trim() : ""; + const thinking = typeof parsed.thinking === "string" ? parsed.thinking.trim() : ""; + if (parsed.done !== true || (!response && !thinking)) { + return "invalid-response"; + } + return "ok"; + } catch { + return "invalid-response"; + } +} + +/** + * Warm a registered local Ollama model only when `/api/ps` proves that the + * daemon is reachable and the selected model is no longer loaded. + */ +export function maybeWarmOllamaAfterDaemonRestart( + route: OllamaRestartRecoveryRoute, + deps: OllamaRestartRecoveryDeps = {}, +): OllamaRestartRecoveryResult { + if (normalizeRouteValue(route.provider) !== OLLAMA_LOCAL_PROVIDER) { + return { kind: "skipped", reason: "not-ollama" }; + } + + const model = normalizeRouteValue(route.model); + if (!model) { + return { kind: "skipped", reason: "missing-model" }; + } + + const getOllamaHost = deps.getOllamaHost ?? getResolvedOllamaHost; + const rawHost = resolveRawOllamaHost(route.endpointUrl, getOllamaHost); + const probe = deps.probeRuntimeModelStatus ?? probeOllamaRuntimeModelStatus; + let status: OllamaRuntimeModelStatus; + try { + status = probe(model, () => rawHost, deps.runCaptureImpl); + } catch { + return { kind: "skipped", reason: "unreachable" }; + } + if (!status.probed) { + return { kind: "skipped", reason: "unreachable" }; + } + if (status.loaded) { + return { kind: "skipped", reason: "already-loaded" }; + } + + const captureEx = deps.runCaptureExImpl ?? runCaptureEx; + try { + const result = captureEx(buildWarmCommand(model, rawHost)); + if (result.timedOut) { + return { kind: "warmed", ok: false, timedOut: true, reason: "timeout" }; + } + if (result.exitCode !== 0) { + return { kind: "warmed", ok: false, timedOut: false, reason: "command-failed" }; + } + const response = validateWarmResponse(result.stdout); + if (response !== "ok") { + return { kind: "warmed", ok: false, timedOut: false, reason: response }; + } + return { kind: "warmed", ok: true, timedOut: false }; + } catch { + return { kind: "warmed", ok: false, timedOut: false, reason: "spawn-failed" }; + } +} diff --git a/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.test.ts b/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.test.ts new file mode 100644 index 0000000000..c4f65b9756 --- /dev/null +++ b/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.test.ts @@ -0,0 +1,211 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; +import { type AgentPassthroughDeps, runAgentPassthrough } from "./passthrough"; +import { runOllamaRestartRecovery } from "./passthrough-ollama-recovery"; + +function makeProcMock() { + const writes: string[] = []; + return { + writes, + proc: { stderr: { write: (value: string) => writes.push(value) } }, + }; +} + +describe("runOllamaRestartRecovery", () => { + it.each([ + ["auth proxy", "http://host.openshell.internal:11435/v1"], + ["WSL direct bridge", "http://host.openshell.internal:11434/v1"], + ])("forwards the persisted %s route to recovery", (_name, endpointUrl) => { + const recoverOllama = vi.fn(() => ({ + kind: "skipped" as const, + reason: "already-loaded" as const, + })); + const { writes, proc } = makeProcMock(); + const route = { + provider: "ollama-local", + model: "qwen3.6:35b", + endpointUrl, + }; + + runOllamaRestartRecovery(route, proc, recoverOllama); + + expect(recoverOllama).toHaveBeenCalledWith(route); + expect(writes.join("")).toContain("Ollama model 'qwen3.6:35b' is already loaded"); + }); + + it("reports a successful warm-up", () => { + const { writes, proc } = makeProcMock(); + + runOllamaRestartRecovery({ provider: "ollama-local", model: "qwen3.6:35b" }, proc, () => ({ + kind: "warmed", + ok: true, + timedOut: false, + })); + + expect(writes.join("")).toContain("Ollama model 'qwen3.6:35b' is loaded and ready"); + }); + + it("reports a timeout before continuing to OpenClaw", () => { + const { writes, proc } = makeProcMock(); + + runOllamaRestartRecovery({ provider: "ollama-local", model: "qwen3.6:35b" }, proc, () => ({ + kind: "warmed", + ok: false, + timedOut: true, + reason: "timeout", + })); + + const stderr = writes.join(""); + expect(stderr).toContain("Checking Ollama model readiness after daemon restart"); + expect(stderr).toContain("Ollama warm-up for 'qwen3.6:35b' timed out"); + expect(stderr).toContain("continuing to OpenClaw dispatch"); + }); + + it.each([ + ["command-failed", "curl exited unsuccessfully"], + ["ollama-error", "Ollama returned an error"], + ["invalid-response", "Ollama returned an invalid response"], + ["spawn-failed", "the warm-up process could not start"], + ] as const)("reports a %s warm-up failure", (reason, message) => { + const { writes, proc } = makeProcMock(); + + runOllamaRestartRecovery({ provider: "ollama-local", model: "qwen3.6:35b" }, proc, () => ({ + kind: "warmed", + ok: false, + timedOut: false, + reason, + })); + + expect(writes.join("")).toContain(message); + }); + + it.each([ + ["already-loaded", "Ollama model 'qwen3.6:35b' is already loaded"], + ["unreachable", "Ollama was unreachable during the restart check"], + ["missing-model", "No Ollama model is recorded for this sandbox"], + ["not-ollama", "Checking Ollama model readiness after daemon restart"], + ] as const)("handles the %s skip reason", (reason, message) => { + const { writes, proc } = makeProcMock(); + + runOllamaRestartRecovery({ provider: "ollama-local", model: "qwen3.6:35b" }, proc, () => ({ + kind: "skipped", + reason, + })); + + expect(writes.join("")).toContain(message); + }); + + it("contains an unexpected recovery exception", () => { + const { writes, proc } = makeProcMock(); + + expect(() => + runOllamaRestartRecovery({ provider: "ollama-local", model: "qwen3.6:35b" }, proc, () => { + throw new Error("unexpected"); + }), + ).not.toThrow(); + expect(writes.join("")).toContain( + "Ollama restart recovery failed unexpectedly; continuing to OpenClaw dispatch", + ); + }); +}); + +function makePassthroughDeps( + route: { provider: string; model: string; endpointUrl: string }, + events: string[], +): AgentPassthroughDeps { + return { + getSandbox: ((name) => ({ name, agent: "openclaw", ...route })) as NonNullable< + AgentPassthroughDeps["getSandbox"] + >, + ensureLive: (async () => ({ state: "present", output: "Phase: Ready" })) as NonNullable< + AgentPassthroughDeps["ensureLive"] + >, + exec: (async () => { + events.push("dispatch"); + }) as NonNullable, + getRecentShieldsAutoRestore: () => ({ kind: "none" }), + process: { + exit: ((code: number) => { + throw new Error(`__exit:${code}`); + }) as (code: number) => never, + stdout: { write: () => true }, + stderr: { write: () => true }, + }, + }; +} + +describe("agent passthrough Ollama recovery ordering", () => { + it("checks an auth-proxy route before JSON dispatch", async () => { + const events: string[] = []; + const route = { + provider: "ollama-local", + model: "qwen3.6:35b", + endpointUrl: "http://host.openshell.internal:11435/v1", + }; + const deps = makePassthroughDeps(route, events); + const runRecovery = vi.fn(() => { + events.push("recovery"); + }); + const execJson = vi.fn(((): never => { + events.push("dispatch"); + throw new Error("__exit:0"); + }) as NonNullable); + + await expect( + runAgentPassthrough( + "alpha", + { extraArgs: ["--agent", "main", "-m", "ping", "--json"] }, + { ...deps, execJson, runOllamaRestartRecovery: runRecovery }, + ), + ).rejects.toThrow("__exit:0"); + + expect(runRecovery).toHaveBeenCalledWith(expect.objectContaining(route), deps.process); + expect(events).toEqual(["recovery", "dispatch"]); + }); + + it("checks a WSL direct route before non-JSON dispatch", async () => { + const events: string[] = []; + const route = { + provider: "ollama-local", + model: "qwen3.6:35b", + endpointUrl: "http://host.openshell.internal:11434/v1", + }; + const deps = makePassthroughDeps(route, events); + const runRecovery = vi.fn(() => { + events.push("recovery"); + }); + + await runAgentPassthrough( + "alpha", + { extraArgs: ["--agent", "main", "-m", "ping"] }, + { ...deps, runOllamaRestartRecovery: runRecovery }, + ); + + expect(runRecovery).toHaveBeenCalledWith(expect.objectContaining(route), deps.process); + expect(events).toEqual(["recovery", "dispatch"]); + }); + + it("does not run Ollama recovery for a non-Ollama route", async () => { + const events: string[] = []; + const deps = makePassthroughDeps( + { + provider: "vllm-local", + model: "meta/llama", + endpointUrl: "http://host.openshell.internal:8000/v1", + }, + events, + ); + const runRecovery = vi.fn(); + + await runAgentPassthrough( + "alpha", + { extraArgs: ["--agent", "main", "-m", "ping"] }, + { ...deps, runOllamaRestartRecovery: runRecovery }, + ); + + expect(runRecovery).not.toHaveBeenCalled(); + expect(events).toEqual(["dispatch"]); + }); +}); diff --git a/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.ts b/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.ts new file mode 100644 index 0000000000..a4be760b23 --- /dev/null +++ b/src/lib/actions/sandbox/agent/passthrough-ollama-recovery.ts @@ -0,0 +1,92 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + maybeWarmOllamaAfterDaemonRestart, + OLLAMA_LOCAL_PROVIDER, + type OllamaRestartRecoveryFailureReason, + type OllamaRestartRecoveryResult, + type OllamaRestartRecoveryRoute, +} from "./ollama-restart-recovery"; + +export { OLLAMA_LOCAL_PROVIDER }; + +export type OllamaRestartRecoveryFn = ( + route: OllamaRestartRecoveryRoute, +) => OllamaRestartRecoveryResult; + +export interface OllamaRestartRecoveryProcess { + stderr: { write(s: string): unknown }; +} + +function describeWarmFailure(reason: OllamaRestartRecoveryFailureReason): string { + switch (reason) { + case "timeout": + return "timed out"; + case "command-failed": + return "curl exited unsuccessfully"; + case "ollama-error": + return "Ollama returned an error"; + case "invalid-response": + return "Ollama returned an invalid response"; + case "spawn-failed": + return "the warm-up process could not start"; + } +} + +function reportRecovery( + route: OllamaRestartRecoveryRoute, + result: OllamaRestartRecoveryResult, + proc: OllamaRestartRecoveryProcess, +): void { + const model = String(route.model ?? "").trim() || "the registered model"; + if (result.kind === "warmed") { + if (result.ok) { + proc.stderr.write(` Ollama model '${model}' is loaded and ready.\n`); + return; + } + proc.stderr.write( + ` Ollama warm-up for '${model}' ${describeWarmFailure(result.reason)}; continuing to OpenClaw dispatch.\n`, + ); + return; + } + + const reason = result.reason; + switch (reason) { + case "already-loaded": + proc.stderr.write(` Ollama model '${model}' is already loaded.\n`); + break; + case "unreachable": + proc.stderr.write( + " Ollama was unreachable during the restart check; continuing to OpenClaw dispatch.\n", + ); + break; + case "missing-model": + proc.stderr.write( + " No Ollama model is recorded for this sandbox; continuing to OpenClaw dispatch.\n", + ); + break; + case "not-ollama": + break; + default: { + const _exhaustive: never = reason; + return _exhaustive; + } + } +} + +/** Run best-effort Ollama recovery without blocking the canonical agent error path. */ +export function runOllamaRestartRecovery( + route: OllamaRestartRecoveryRoute, + proc: OllamaRestartRecoveryProcess, + recoverOllama: OllamaRestartRecoveryFn = maybeWarmOllamaAfterDaemonRestart, +): void { + proc.stderr.write(" Checking Ollama model readiness after daemon restart...\n"); + try { + reportRecovery(route, recoverOllama(route), proc); + } catch { + proc.stderr.write( + " Ollama restart recovery failed unexpectedly; continuing to OpenClaw dispatch.\n", + ); + } +} diff --git a/src/lib/actions/sandbox/agent/passthrough.test.ts b/src/lib/actions/sandbox/agent/passthrough.test.ts index cad840209a..0806724c24 100644 --- a/src/lib/actions/sandbox/agent/passthrough.test.ts +++ b/src/lib/actions/sandbox/agent/passthrough.test.ts @@ -7,7 +7,17 @@ const execMock = vi.hoisted(() => vi.fn(async () => {})); const ensureLiveMock = vi.hoisted(() => vi.fn(async () => ({ state: "present", output: "Phase: Ready" }) as { output?: string }), ); -const getSandboxMock = vi.hoisted(() => vi.fn(() => null as { agent?: string | null } | null)); +const getSandboxMock = vi.hoisted(() => + vi.fn( + () => + null as { + agent?: string | null; + provider?: string | null; + model?: string | null; + endpointUrl?: string | null; + } | null, + ), +); const listAgentsMock = vi.hoisted(() => vi.fn(() => ["custom-terminal", "hermes", "langchain-deepagents-code", "openclaw"]), ); diff --git a/src/lib/actions/sandbox/agent/passthrough.ts b/src/lib/actions/sandbox/agent/passthrough.ts index a9d77d9f96..096ae9096e 100644 --- a/src/lib/actions/sandbox/agent/passthrough.ts +++ b/src/lib/actions/sandbox/agent/passthrough.ts @@ -3,8 +3,8 @@ // Source-of-truth boundary for the `nemoclaw agent` passthrough. // -// The wrapper enforces three host-side mirrors of upstream contracts and one -// advisory diagnostic: +// The wrapper enforces three host-side mirrors of upstream contracts, one +// advisory diagnostic, and one best-effort pre-dispatch recovery: // // 1. Agent-kind guard (registry mirror). // @@ -72,6 +72,12 @@ // source-boundary analysis lives with the focused implementation in // `passthrough-shields-warning.ts`. // +// 5. Ollama restart recovery (best-effort lifecycle bridge). Ollama owns model +// runner lifetime, while NemoClaw owns the registered route and dispatch +// ordering. The focused source-boundary analysis, reporting, and regression +// coverage live in `ollama-restart-recovery.ts` and +// `passthrough-ollama-recovery.ts`. +// // Regression tests: `passthrough.test.ts` covers the Hermes redirect, the // forwarded argv, the registry-miss fallback to OpenClaw, registry and // manifest-resolution fail-closed paths, quoted manifest command rejection, @@ -79,7 +85,8 @@ // unparseable phase fail-closed path, the OpenClaw no-selector rejection, and // the `--flag=value` selector-acceptance branch, plus the OpenClaw JSON // captured transport path used to append failure provenance without polluting -// machine-readable stdout. The focused shields diagnostic owns its tests. +// machine-readable stdout. The focused shields and Ollama modules own their +// diagnostic and recovery tests. // // Removal conditions: // @@ -91,6 +98,8 @@ // missing selector with a clean exit 2 and an actionable message. // - Drop the simple-token parser when terminal runtime manifests expose // argv arrays natively. +// - Drop Ollama pre-dispatch recovery when supported daemon restarts preserve +// loaded runners or NemoClaw manages and warms the daemon lifecycle. import { type AgentDefinition, isTerminalAgent, listAgents, loadAgent } from "../../../agent/defs"; import { CLI_NAME } from "../../../cli/branding"; @@ -101,6 +110,7 @@ import { execSandbox } from "../exec"; import { ensureLiveSandboxOrExit } from "../gateway-state"; import { hasAgentPassthroughHelpToken, printAgentPassthroughHelp } from "./passthrough-help"; import { type AgentJsonPassthroughProcess, runAgentJsonPassthrough } from "./passthrough-json"; +import { OLLAMA_LOCAL_PROVIDER, runOllamaRestartRecovery } from "./passthrough-ollama-recovery"; import { maybeEmitShieldsRelockWarning } from "./passthrough-shields-warning"; export { @@ -134,6 +144,7 @@ export interface AgentPassthroughDeps { ensureLive?: typeof ensureLiveSandboxOrExit; exec?: typeof execSandbox; execJson?: typeof runAgentJsonPassthrough; + runOllamaRestartRecovery?: typeof runOllamaRestartRecovery; getRecentShieldsAutoRestore?: (sandboxName: string) => ShieldsAutoRestoreReadResult; process?: { exit(code: number): never; @@ -144,7 +155,13 @@ export interface AgentPassthroughDeps { type RegistryReadResult = | { kind: "missing" } - | { kind: "agent"; agent: string | null } + | { + kind: "agent"; + agent: string | null; + provider: string | null; + model: string | null; + endpointUrl: string | null; + } | { kind: "error"; message: string }; type ResolvedRegistryReadResult = Exclude; type TerminalCommandResult = @@ -158,7 +175,13 @@ function readSandboxAgentFromRegistry( try { const sandbox = getSandbox(sandboxName); if (!sandbox) return { kind: "missing" }; - return { kind: "agent", agent: sandbox.agent ?? null }; + return { + kind: "agent", + agent: sandbox.agent ?? null, + provider: sandbox.provider ?? null, + model: sandbox.model ?? null, + endpointUrl: sandbox.endpointUrl ?? null, + }; } catch (error) { return { kind: "error", message: (error as Error).message ?? String(error) }; } @@ -425,6 +448,10 @@ export async function runAgentPassthrough( rejectNoTargetSelector(proc); } if (isOpenClawPassthroughCommand(command)) { + if (lookup.kind === "agent" && lookup.provider === OLLAMA_LOCAL_PROVIDER) { + const recoverOllama = deps.runOllamaRestartRecovery ?? runOllamaRestartRecovery; + recoverOllama(lookup, proc); + } maybeEmitShieldsRelockWarning(proc, sandboxName, deps.getRecentShieldsAutoRestore); } if (isOpenClawPassthroughCommand(command) && requestsOpenClawJsonOutput(extraArgs)) { diff --git a/test/e2e/live/gpu-e2e.test.ts b/test/e2e/live/gpu-e2e.test.ts index dc230edacb..e385691980 100644 --- a/test/e2e/live/gpu-e2e.test.ts +++ b/test/e2e/live/gpu-e2e.test.ts @@ -5,6 +5,7 @@ import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; import { resultText } from "../fixtures/clients/index.ts"; import { trustedSandboxShellScript } from "../fixtures/clients/sandbox.ts"; import { expect, test } from "../fixtures/e2e-test.ts"; +import { parseOpenClawAgentText } from "./common-egress-agent-helpers.ts"; import { assertGpuInstallProofs, assertNvidiaAvailable, @@ -89,6 +90,14 @@ function assertSmallContextCompactionPolicy(configText: string): void { }); } +function loadedOllamaModels(raw: string): string[] { + const parsed = JSON.parse(raw) as { models?: Array<{ name?: unknown; model?: unknown }> }; + return (parsed.models ?? []).flatMap((entry) => { + const name = typeof entry.name === "string" ? entry.name : entry.model; + return typeof name === "string" && name.trim() ? [name.trim()] : []; + }); +} + test("GPU Ollama onboard enables CUDA, auth proxy, and sandbox inference", { timeout: TIMEOUT_MS, }, async ({ artifacts, cleanup, host, sandbox, skip }) => { @@ -222,4 +231,70 @@ test("GPU Ollama onboard enables CUDA, auth proxy, and sandbox inference", { ); expect(chat.exitCode, resultText(chat)).toBe(0); expect(chatContent(chat.stdout)).toMatch(/pong/i); + + const restart = await host.command( + "bash", + [ + "-c", + `set -euo pipefail +if sudo -n systemctl restart ollama 2>/dev/null; then + restart_mode=system +elif systemctl --user restart ollama 2>/dev/null; then + restart_mode=user +else + pkill -f '[o]llama serve' 2>/dev/null || true + OLLAMA_HOST=127.0.0.1:11434 nohup ollama serve >/tmp/nemoclaw-gpu-e2e-ollama.log 2>&1 & + restart_mode=manual +fi +for attempt in $(seq 1 60); do + tags_json="$(curl -fsS --connect-timeout 2 http://127.0.0.1:11434/api/tags 2>/dev/null || true)" + if [ -n "$tags_json" ]; then + ps_json="$(curl -fsS --connect-timeout 2 http://127.0.0.1:11434/api/ps 2>/dev/null || true)" + if [ -n "$ps_json" ]; then + printf 'restart_mode=%s\n%s\n' "$restart_mode" "$ps_json" + exit 0 + fi + fi + sleep 1 +done +echo 'Ollama did not become ready after restart' >&2 +exit 1`, + ], + { artifactName: "ollama-daemon-restart-unloaded", env: env(), timeoutMs: 90_000 }, + ); + expect(restart.exitCode, resultText(restart)).toBe(0); + const restartLines = restart.stdout.trim().split("\n"); + expect(restartLines[0]).toMatch(/^restart_mode=(system|user|manual)$/u); + expect(loadedOllamaModels(restartLines.slice(1).join("\n"))).toEqual([]); + + const recovered = await host.nemoclaw( + [ + SANDBOX_NAME, + "agent", + "--agent", + "main", + "--json", + "--session-id", + `e2e-gpu-ollama-restart-${Date.now()}-${process.pid}`, + "-m", + "Reply with exactly one word: PONG", + ], + { + artifactName: "agent-after-ollama-daemon-restart", + env: env(), + timeoutMs: 12 * 60_000, + }, + ); + expect(recovered.exitCode, resultText(recovered)).toBe(0); + expect(resultText(recovered)).toContain("Checking Ollama model readiness after daemon restart"); + expect(resultText(recovered)).toContain(`Ollama model '${model}' is loaded and ready.`); + expect(parseOpenClawAgentText(recovered.stdout)).toMatch(/pong/i); + + const loaded = await host.command("curl", ["-fsS", "http://127.0.0.1:11434/api/ps"], { + artifactName: "ollama-model-loaded-after-recovery", + env: env(), + timeoutMs: 30_000, + }); + expect(loaded.exitCode, resultText(loaded)).toBe(0); + expect(loadedOllamaModels(loaded.stdout)).toContain(model); });