diff --git a/docs/inference/use-local-inference.mdx b/docs/inference/use-local-inference.mdx index bd80cb4e84..f36c28d5cd 100644 --- a/docs/inference/use-local-inference.mdx +++ b/docs/inference/use-local-inference.mdx @@ -91,6 +91,8 @@ If the selected model declares that it does not support tool calling, onboarding The validation also requires structured chat-completions tool calls. If the model leaks tool-call JSON as plain message text, onboarding stops so you can choose a model that returns tool calls in the expected response field. If a host-side validation probe times out, NemoClaw retries the Ollama tool-call validation with a larger timeout before failing the setup. +After an Ollama daemon restart, the first OpenClaw agent passthrough command checks whether the registered `ollama-local` model is loaded. +If the daemon is reachable but the model is unloaded, NemoClaw sends a bounded warm-up request before dispatching the agent command. On WSL, if you choose the Windows-host Ollama path, NemoClaw uses `host.docker.internal:11434` and pulls missing models through the Ollama HTTP API instead of requiring the `ollama` CLI inside WSL. ### WSL with Windows-Host Ollama diff --git a/docs/reference/troubleshooting.mdx b/docs/reference/troubleshooting.mdx index 861ffc077e..8507d2d366 100644 --- a/docs/reference/troubleshooting.mdx +++ b/docs/reference/troubleshooting.mdx @@ -1098,6 +1098,8 @@ For local Ollama and local vLLM, `$$nemoclaw status` also prints an `Infe If that line shows `unreachable`, start the local backend first and then retry the request. For Local Ollama, current releases also print `Inference (auth proxy)` when a proxy token is available. If the backend is healthy but the auth proxy is `unauthorized` or `unreachable`, re-run onboarding so NemoClaw can recreate the proxy token, restart the proxy, and refresh the route. +For Ollama-backed OpenClaw sandboxes, NemoClaw warms the registered model on the first agent passthrough after an Ollama daemon restart when the daemon is reachable but the model is unloaded. +If the command still times out, confirm that the Ollama daemon is reachable and that the selected model can load with `ollama run `. If the endpoint is correct but requests still fail, check for network policy rules that may block the connection. Then verify the credential and base URL for the provider you selected during onboarding. diff --git a/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts b/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts new file mode 100644 index 0000000000..9fb20fc647 --- /dev/null +++ b/src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts @@ -0,0 +1,182 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; +import { maybeWarmOllamaAfterDaemonRestart } from "./ollama-restart-recovery"; + +describe("maybeWarmOllamaAfterDaemonRestart", () => { + it("skips non-Ollama routes", () => { + const runCaptureExImpl = vi.fn(); + + const result = maybeWarmOllamaAfterDaemonRestart( + { provider: "vllm-local", model: "meta/llama" }, + { runCaptureExImpl }, + ); + + expect(result).toEqual({ kind: "skipped", reason: "not-ollama" }); + expect(runCaptureExImpl).not.toHaveBeenCalled(); + }); + + it("skips Ollama routes without a selected model", () => { + const runCaptureExImpl = vi.fn(); + + const result = maybeWarmOllamaAfterDaemonRestart( + { provider: "ollama-local", model: " " }, + { runCaptureExImpl }, + ); + + expect(result).toEqual({ kind: "skipped", reason: "missing-model" }); + expect(runCaptureExImpl).not.toHaveBeenCalled(); + }); + + it("skips while Ollama is unreachable so backend-unavailable stays fast", () => { + const runCaptureExImpl = vi.fn(); + const probeRuntimeModelStatus = vi.fn(() => ({ + probed: false, + loaded: false, + cpuOnly: false, + })); + + const result = maybeWarmOllamaAfterDaemonRestart( + { provider: "ollama-local", model: "qwen3.6:35b" }, + { probeRuntimeModelStatus, runCaptureExImpl }, + ); + + expect(result).toEqual({ kind: "skipped", reason: "unreachable" }); + expect(probeRuntimeModelStatus).toHaveBeenCalledWith( + "qwen3.6:35b", + expect.any(Function), + undefined, + ); + expect(runCaptureExImpl).not.toHaveBeenCalled(); + }); + + it("skips when the runtime status probe throws", () => { + const runCaptureExImpl = vi.fn(); + const probeRuntimeModelStatus = vi.fn(() => { + throw new Error("curl probe failed"); + }); + + const result = maybeWarmOllamaAfterDaemonRestart( + { provider: "ollama-local", model: "qwen3.6:35b" }, + { probeRuntimeModelStatus, runCaptureExImpl }, + ); + + expect(result).toEqual({ kind: "skipped", reason: "unreachable" }); + expect(runCaptureExImpl).not.toHaveBeenCalled(); + }); + + it("skips when the selected Ollama model is already loaded", () => { + const runCaptureExImpl = vi.fn(); + const probeRuntimeModelStatus = vi.fn(() => ({ + probed: true, + loaded: true, + cpuOnly: false, + })); + + const result = maybeWarmOllamaAfterDaemonRestart( + { provider: "ollama-local", model: "qwen3.6:35b" }, + { probeRuntimeModelStatus, runCaptureExImpl }, + ); + + expect(result).toEqual({ kind: "skipped", reason: "already-loaded" }); + expect(runCaptureExImpl).not.toHaveBeenCalled(); + }); + + it("warms when Ollama is reachable but the selected model is not loaded", () => { + const runCaptureExImpl = vi.fn(() => ({ + stdout: '{"done":true}', + stderr: "", + exitCode: 0, + timedOut: false, + })); + const probeRuntimeModelStatus = vi.fn(() => ({ + probed: true, + loaded: false, + cpuOnly: false, + })); + + const result = maybeWarmOllamaAfterDaemonRestart( + { provider: "ollama-local", model: "qwen3.6:35b" }, + { + getOllamaHost: () => "127.0.0.1", + probeRuntimeModelStatus, + runCaptureExImpl, + }, + ); + + expect(result).toEqual({ kind: "warmed", ok: true, timedOut: false }); + expect(runCaptureExImpl).toHaveBeenCalledTimes(1); + const [command] = runCaptureExImpl.mock.calls[0] as unknown as [string[]]; + expect(command).toContain("--max-time"); + expect(command).toContain("300"); + expect(command).toContain("http://127.0.0.1:11434/api/generate"); + expect(command.join("\n")).toContain('"model":"qwen3.6:35b"'); + }); + + it("uses the host-aware runtime probe before warming", () => { + const runCaptureImpl = vi.fn((cmd: string | string[]) => { + const command = Array.isArray(cmd) ? cmd.join(" ") : cmd; + expect(command).toContain("http://192.0.2.44:11434/api/ps"); + return JSON.stringify({ models: [] }); + }); + const runCaptureExImpl = vi.fn(() => ({ + stdout: '{"done":true}', + stderr: "", + exitCode: 0, + timedOut: false, + })); + + const result = maybeWarmOllamaAfterDaemonRestart( + { provider: "ollama-local", model: "qwen3.6:35b" }, + { + getOllamaHost: () => "192.0.2.44", + runCaptureImpl, + runCaptureExImpl, + }, + ); + + expect(result).toEqual({ kind: "warmed", ok: true, timedOut: false }); + expect(runCaptureImpl).toHaveBeenCalledTimes(1); + expect(runCaptureExImpl).toHaveBeenCalledTimes(1); + }); + + it("returns a failed warm result without throwing when the warmup probe has no output", () => { + const runCaptureExImpl = vi.fn(() => ({ + stdout: "", + stderr: "curl: (28) Operation timed out", + exitCode: 28, + timedOut: true, + })); + const probeRuntimeModelStatus = vi.fn(() => ({ + probed: true, + loaded: false, + cpuOnly: false, + })); + + const result = maybeWarmOllamaAfterDaemonRestart( + { provider: "ollama-local", model: "qwen3.6:35b" }, + { probeRuntimeModelStatus, runCaptureExImpl }, + ); + + expect(result).toEqual({ kind: "warmed", ok: false, timedOut: true }); + }); + + it("returns a failed warm result without throwing when the warmup probe throws", () => { + const runCaptureExImpl = vi.fn(() => { + throw new Error("spawn failed"); + }); + const probeRuntimeModelStatus = vi.fn(() => ({ + probed: true, + loaded: false, + cpuOnly: false, + })); + + const result = maybeWarmOllamaAfterDaemonRestart( + { provider: "ollama-local", model: "qwen3.6:35b" }, + { probeRuntimeModelStatus, runCaptureExImpl }, + ); + + expect(result).toEqual({ kind: "warmed", ok: false, timedOut: false }); + }); +}); diff --git a/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts b/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts new file mode 100644 index 0000000000..4ac02117f7 --- /dev/null +++ b/src/lib/actions/sandbox/agent/ollama-restart-recovery.ts @@ -0,0 +1,88 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { RunCaptureExFn } from "../../../inference/local"; +import { getOllamaProbeCommand, getResolvedOllamaHost } from "../../../inference/local"; +import { + type OllamaRuntimeModelStatus, + type OllamaRuntimeRunCaptureFn, + probeOllamaRuntimeModelStatus, +} from "../../../inference/ollama-runtime-context"; + +const { runCaptureEx } = require("../../../runner") as typeof import("../../../runner"); + +export interface OllamaRestartRecoveryRoute { + provider?: string | null; + model?: string | null; +} + +export interface OllamaRestartRecoveryDeps { + probeRuntimeModelStatus?: ( + model: string, + getOllamaHost: () => string, + runCaptureImpl?: OllamaRuntimeRunCaptureFn, + ) => OllamaRuntimeModelStatus; + runCaptureExImpl?: RunCaptureExFn; + getOllamaHost?: () => string; + runCaptureImpl?: OllamaRuntimeRunCaptureFn; +} + +export type OllamaRestartRecoveryResult = + | { kind: "skipped"; reason: "not-ollama" | "missing-model" | "already-loaded" | "unreachable" } + | { kind: "warmed"; ok: boolean; timedOut: boolean }; + +const OLLAMA_PROVIDER = "ollama-local"; +const OLLAMA_RESTART_RECOVERY_TIMEOUT_SECONDS = 300; + +/** + * Converts optional route metadata into a comparable string value. + */ +function normalizeRouteValue(value: string | null | undefined): string { + return String(value ?? "").trim(); +} + +/** + * Best-effort recovery for the local-Ollama agent path after a daemon restart. + * + * Killing Ollama drops the model runner even after the daemon comes back. The + * first post-restart OpenClaw agent request can then spend its whole request + * budget cold-loading the model and exit non-zero. We only warm when `/api/ps` + * proves the daemon is reachable but the selected model is not loaded, so a + * genuinely down backend still reaches OpenClaw's existing clear + * backend-unavailable error without an added long probe delay. + */ +export function maybeWarmOllamaAfterDaemonRestart( + route: OllamaRestartRecoveryRoute, + deps: OllamaRestartRecoveryDeps = {}, +): OllamaRestartRecoveryResult { + if (normalizeRouteValue(route.provider) !== OLLAMA_PROVIDER) { + return { kind: "skipped", reason: "not-ollama" }; + } + + const model = normalizeRouteValue(route.model); + if (!model) { + return { kind: "skipped", reason: "missing-model" }; + } + + const probe = deps.probeRuntimeModelStatus ?? probeOllamaRuntimeModelStatus; + let status: OllamaRuntimeModelStatus; + try { + status = probe(model, deps.getOllamaHost ?? getResolvedOllamaHost, deps.runCaptureImpl); + } catch { + return { kind: "skipped", reason: "unreachable" }; + } + if (!status.probed) { + return { kind: "skipped", reason: "unreachable" }; + } + if (status.loaded) { + return { kind: "skipped", reason: "already-loaded" }; + } + + const captureEx = deps.runCaptureExImpl ?? runCaptureEx; + try { + const result = captureEx(getOllamaProbeCommand(model, OLLAMA_RESTART_RECOVERY_TIMEOUT_SECONDS)); + return { kind: "warmed", ok: Boolean(result.stdout), timedOut: result.timedOut }; + } catch { + return { kind: "warmed", ok: false, timedOut: false }; + } +} diff --git a/src/lib/actions/sandbox/agent/passthrough.test.ts b/src/lib/actions/sandbox/agent/passthrough.test.ts index de1bf2d267..0ad896c449 100644 --- a/src/lib/actions/sandbox/agent/passthrough.test.ts +++ b/src/lib/actions/sandbox/agent/passthrough.test.ts @@ -7,7 +7,16 @@ const execMock = vi.hoisted(() => vi.fn(async () => {})); const ensureLiveMock = vi.hoisted(() => vi.fn(async () => ({ state: "present", output: "Phase: Ready" }) as { output?: string }), ); -const getSandboxMock = vi.hoisted(() => vi.fn(() => null as { agent?: string | null } | null)); +const getSandboxMock = vi.hoisted(() => + vi.fn( + () => + null as { + agent?: string | null; + provider?: string | null; + model?: string | null; + } | null, + ), +); const listAgentsMock = vi.hoisted(() => vi.fn(() => ["custom-terminal", "hermes", "langchain-deepagents-code", "openclaw"]), ); @@ -112,6 +121,98 @@ describe("runAgentPassthrough", () => { ); }); + it("checks Ollama model readiness before OpenClaw JSON dispatch", async () => { + const execJson = vi.fn(() => { + throw new Error("__exit:0"); + }); + const maybeWarmOllamaAfterDaemonRestart = vi.fn(() => ({ + kind: "skipped" as const, + reason: "already-loaded" as const, + })); + getSandboxMock.mockReturnValueOnce({ + agent: "openclaw", + provider: "ollama-local", + model: "qwen3.6:35b", + }); + const { proc } = makeProcMock(); + + await expect( + runAgentPassthrough( + "alpha", + { + extraArgs: ["--agent", "work", "--session-id", "s-1", "-m", "ping", "--json"], + }, + { execJson, maybeWarmOllamaAfterDaemonRestart, process: proc }, + ), + ).rejects.toThrow("__exit:0"); + + expect(maybeWarmOllamaAfterDaemonRestart).toHaveBeenCalledWith({ + provider: "ollama-local", + model: "qwen3.6:35b", + }); + expect(maybeWarmOllamaAfterDaemonRestart.mock.invocationCallOrder[0]).toBeLessThan( + execJson.mock.invocationCallOrder[0], + ); + }); + + it("reports failed Ollama restart warm-up before OpenClaw dispatch", async () => { + const execJson = vi.fn(() => { + throw new Error("__exit:0"); + }); + const maybeWarmOllamaAfterDaemonRestart = vi.fn(() => ({ + kind: "warmed" as const, + ok: false, + timedOut: true, + })); + getSandboxMock.mockReturnValueOnce({ + agent: "openclaw", + provider: "ollama-local", + model: "qwen3.6:35b", + }); + const { writes, proc } = makeProcMock(); + + await expect( + runAgentPassthrough( + "alpha", + { + extraArgs: ["--agent", "work", "--session-id", "s-1", "-m", "ping", "--json"], + }, + { execJson, maybeWarmOllamaAfterDaemonRestart, process: proc }, + ), + ).rejects.toThrow("__exit:0"); + + const stderr = writes.join(""); + expect(stderr).toContain("Checking Ollama model readiness after daemon restart"); + expect(stderr).toContain( + "Ollama warm-up after restart did not complete cleanly for 'qwen3.6:35b'", + ); + expect(maybeWarmOllamaAfterDaemonRestart.mock.invocationCallOrder[0]).toBeLessThan( + execJson.mock.invocationCallOrder[0], + ); + }); + + it("does not run Ollama restart recovery for non-Ollama registered routes", async () => { + const maybeWarmOllamaAfterDaemonRestart = vi.fn(); + getSandboxMock.mockReturnValueOnce({ + agent: "openclaw", + provider: "vllm-local", + model: "meta/llama", + }); + + await runAgentPassthrough( + "alpha", + { extraArgs: ["--agent", "work", "--session-id", "s-1", "-m", "ping"] }, + { maybeWarmOllamaAfterDaemonRestart }, + ); + + expect(maybeWarmOllamaAfterDaemonRestart).not.toHaveBeenCalled(); + expect(execMock).toHaveBeenCalledWith( + "alpha", + ["openclaw", "agent", "--agent", "work", "--session-id", "s-1", "-m", "ping"], + { tty: false }, + ); + }); + it("keeps --json as a message value on the normal passthrough path", async () => { const execJson = vi.fn(((): never => { throw new Error("__unexpected-json"); diff --git a/src/lib/actions/sandbox/agent/passthrough.ts b/src/lib/actions/sandbox/agent/passthrough.ts index a9d77d9f96..48461f1974 100644 --- a/src/lib/actions/sandbox/agent/passthrough.ts +++ b/src/lib/actions/sandbox/agent/passthrough.ts @@ -99,6 +99,11 @@ import { parseSandboxPhase } from "../../../state/gateway"; import * as registry from "../../../state/registry"; import { execSandbox } from "../exec"; import { ensureLiveSandboxOrExit } from "../gateway-state"; +import { + maybeWarmOllamaAfterDaemonRestart, + type OllamaRestartRecoveryResult, + type OllamaRestartRecoveryRoute, +} from "./ollama-restart-recovery"; import { hasAgentPassthroughHelpToken, printAgentPassthroughHelp } from "./passthrough-help"; import { type AgentJsonPassthroughProcess, runAgentJsonPassthrough } from "./passthrough-json"; import { maybeEmitShieldsRelockWarning } from "./passthrough-shields-warning"; @@ -134,6 +139,9 @@ export interface AgentPassthroughDeps { ensureLive?: typeof ensureLiveSandboxOrExit; exec?: typeof execSandbox; execJson?: typeof runAgentJsonPassthrough; + maybeWarmOllamaAfterDaemonRestart?: ( + route: OllamaRestartRecoveryRoute, + ) => OllamaRestartRecoveryResult | undefined; getRecentShieldsAutoRestore?: (sandboxName: string) => ShieldsAutoRestoreReadResult; process?: { exit(code: number): never; @@ -144,13 +152,16 @@ export interface AgentPassthroughDeps { type RegistryReadResult = | { kind: "missing" } - | { kind: "agent"; agent: string | null } + | { kind: "agent"; agent: string | null; provider: string | null; model: string | null } | { kind: "error"; message: string }; type ResolvedRegistryReadResult = Exclude; type TerminalCommandResult = | { kind: "command"; argv: string[] } | { kind: "unsupported"; message: string }; +/** + * Reads registered agent and inference route metadata for a sandbox. + */ function readSandboxAgentFromRegistry( sandboxName: string, getSandbox: typeof registry.getSandbox = registry.getSandbox, @@ -158,12 +169,31 @@ function readSandboxAgentFromRegistry( try { const sandbox = getSandbox(sandboxName); if (!sandbox) return { kind: "missing" }; - return { kind: "agent", agent: sandbox.agent ?? null }; + return { + kind: "agent", + agent: sandbox.agent ?? null, + provider: sandbox.provider ?? null, + model: sandbox.model ?? null, + }; } catch (error) { return { kind: "error", message: (error as Error).message ?? String(error) }; } } +/** + * Builds an Ollama restart-recovery route from registry metadata when applicable. + */ +function getOllamaRestartRecoveryRoute( + lookup: ResolvedRegistryReadResult, +): OllamaRestartRecoveryRoute | null { + if (lookup.kind !== "agent") return null; + if (lookup.provider !== "ollama-local") return null; + return { provider: lookup.provider, model: lookup.model }; +} + +/** + * Rejects registered agents that cannot use the OpenClaw passthrough path. + */ function rejectNonOpenclawAgent( sandboxName: string, agent: string, @@ -379,6 +409,9 @@ function rejectUnparseablePhase( return proc.exit(2); } +/** + * Rejects passthrough dispatch when sandbox state is not ready for an agent. + */ function rejectNotReadyForAgent( sandboxName: string, phase: string, @@ -400,6 +433,9 @@ function rejectNotReadyForAgent( return proc.exit(1); } +/** + * Dispatches agent passthrough commands after validating sandbox and agent state. + */ export async function runAgentPassthrough( sandboxName: string, { extraArgs = [] }: AgentPassthroughOptions = {}, @@ -425,6 +461,18 @@ export async function runAgentPassthrough( rejectNoTargetSelector(proc); } if (isOpenClawPassthroughCommand(command)) { + const route = getOllamaRestartRecoveryRoute(lookup); + if (route) { + const recoverOllama = + deps.maybeWarmOllamaAfterDaemonRestart ?? maybeWarmOllamaAfterDaemonRestart; + proc.stderr.write("Checking Ollama model readiness after daemon restart...\n"); + const recovery = recoverOllama(route); + if (recovery?.kind === "warmed" && (!recovery.ok || recovery.timedOut)) { + proc.stderr.write( + `Ollama warm-up after restart did not complete cleanly for '${route.model}'; continuing to OpenClaw dispatch.\n`, + ); + } + } maybeEmitShieldsRelockWarning(proc, sandboxName, deps.getRecentShieldsAutoRestore); } if (isOpenClawPassthroughCommand(command) && requestsOpenClawJsonOutput(extraArgs)) {