Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions docs/inference/use-local-inference.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,8 @@ If the selected model declares that it does not support tool calling, onboarding
The validation also requires structured chat-completions tool calls.
If the model leaks tool-call JSON as plain message text, onboarding stops so you can choose a model that returns tool calls in the expected response field.
If a host-side validation probe times out, NemoClaw retries the Ollama tool-call validation with a larger timeout before failing the setup.
After an Ollama daemon restart, the first OpenClaw agent passthrough command checks whether the registered `ollama-local` model is loaded.
If the daemon is reachable but the model is unloaded, NemoClaw sends a bounded warm-up request before dispatching the agent command.
On WSL, if you choose the Windows-host Ollama path, NemoClaw uses `host.docker.internal:11434` and pulls missing models through the Ollama HTTP API instead of requiring the `ollama` CLI inside WSL.

### WSL with Windows-Host Ollama
Expand Down
2 changes: 2 additions & 0 deletions docs/reference/troubleshooting.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -1098,6 +1098,8 @@ For local Ollama and local vLLM, `$$nemoclaw <name> status` also prints an `Infe
If that line shows `unreachable`, start the local backend first and then retry the request.
For Local Ollama, current releases also print `Inference (auth proxy)` when a proxy token is available.
If the backend is healthy but the auth proxy is `unauthorized` or `unreachable`, re-run onboarding so NemoClaw can recreate the proxy token, restart the proxy, and refresh the route.
For Ollama-backed OpenClaw sandboxes, NemoClaw warms the registered model on the first agent passthrough after an Ollama daemon restart when the daemon is reachable but the model is unloaded.
If the command still times out, confirm that the Ollama daemon is reachable and that the selected model can load with `ollama run <model>`.

If the endpoint is correct but requests still fail, check for network policy rules that may block the connection.
Then verify the credential and base URL for the provider you selected during onboarding.
Expand Down
182 changes: 182 additions & 0 deletions src/lib/actions/sandbox/agent/ollama-restart-recovery.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,182 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import { describe, expect, it, vi } from "vitest";
import { maybeWarmOllamaAfterDaemonRestart } from "./ollama-restart-recovery";

describe("maybeWarmOllamaAfterDaemonRestart", () => {
it("skips non-Ollama routes", () => {
const runCaptureExImpl = vi.fn();

const result = maybeWarmOllamaAfterDaemonRestart(
{ provider: "vllm-local", model: "meta/llama" },
{ runCaptureExImpl },
);

expect(result).toEqual({ kind: "skipped", reason: "not-ollama" });
expect(runCaptureExImpl).not.toHaveBeenCalled();
});

it("skips Ollama routes without a selected model", () => {
const runCaptureExImpl = vi.fn();

const result = maybeWarmOllamaAfterDaemonRestart(
{ provider: "ollama-local", model: " " },
{ runCaptureExImpl },
);

expect(result).toEqual({ kind: "skipped", reason: "missing-model" });
expect(runCaptureExImpl).not.toHaveBeenCalled();
});

it("skips while Ollama is unreachable so backend-unavailable stays fast", () => {
const runCaptureExImpl = vi.fn();
const probeRuntimeModelStatus = vi.fn(() => ({
probed: false,
loaded: false,
cpuOnly: false,
}));

const result = maybeWarmOllamaAfterDaemonRestart(
{ provider: "ollama-local", model: "qwen3.6:35b" },
{ probeRuntimeModelStatus, runCaptureExImpl },
);

expect(result).toEqual({ kind: "skipped", reason: "unreachable" });
expect(probeRuntimeModelStatus).toHaveBeenCalledWith(
"qwen3.6:35b",
expect.any(Function),
undefined,
);
expect(runCaptureExImpl).not.toHaveBeenCalled();
});

it("skips when the runtime status probe throws", () => {
const runCaptureExImpl = vi.fn();
const probeRuntimeModelStatus = vi.fn(() => {
throw new Error("curl probe failed");
});

const result = maybeWarmOllamaAfterDaemonRestart(
{ provider: "ollama-local", model: "qwen3.6:35b" },
{ probeRuntimeModelStatus, runCaptureExImpl },
);

expect(result).toEqual({ kind: "skipped", reason: "unreachable" });
expect(runCaptureExImpl).not.toHaveBeenCalled();
});

it("skips when the selected Ollama model is already loaded", () => {
const runCaptureExImpl = vi.fn();
const probeRuntimeModelStatus = vi.fn(() => ({
probed: true,
loaded: true,
cpuOnly: false,
}));

const result = maybeWarmOllamaAfterDaemonRestart(
{ provider: "ollama-local", model: "qwen3.6:35b" },
{ probeRuntimeModelStatus, runCaptureExImpl },
);

expect(result).toEqual({ kind: "skipped", reason: "already-loaded" });
expect(runCaptureExImpl).not.toHaveBeenCalled();
});

it("warms when Ollama is reachable but the selected model is not loaded", () => {
const runCaptureExImpl = vi.fn(() => ({
stdout: '{"done":true}',
stderr: "",
exitCode: 0,
timedOut: false,
}));
const probeRuntimeModelStatus = vi.fn(() => ({
probed: true,
loaded: false,
cpuOnly: false,
}));

const result = maybeWarmOllamaAfterDaemonRestart(
{ provider: "ollama-local", model: "qwen3.6:35b" },
{
getOllamaHost: () => "127.0.0.1",
probeRuntimeModelStatus,
runCaptureExImpl,
},
);

expect(result).toEqual({ kind: "warmed", ok: true, timedOut: false });
expect(runCaptureExImpl).toHaveBeenCalledTimes(1);
const [command] = runCaptureExImpl.mock.calls[0] as unknown as [string[]];
expect(command).toContain("--max-time");
expect(command).toContain("300");
expect(command).toContain("http://127.0.0.1:11434/api/generate");
expect(command.join("\n")).toContain('"model":"qwen3.6:35b"');
});

it("uses the host-aware runtime probe before warming", () => {
const runCaptureImpl = vi.fn((cmd: string | string[]) => {
const command = Array.isArray(cmd) ? cmd.join(" ") : cmd;
expect(command).toContain("http://192.0.2.44:11434/api/ps");
return JSON.stringify({ models: [] });
});
const runCaptureExImpl = vi.fn(() => ({
stdout: '{"done":true}',
stderr: "",
exitCode: 0,
timedOut: false,
}));

const result = maybeWarmOllamaAfterDaemonRestart(
{ provider: "ollama-local", model: "qwen3.6:35b" },
{
getOllamaHost: () => "192.0.2.44",
runCaptureImpl,
runCaptureExImpl,
},
);

expect(result).toEqual({ kind: "warmed", ok: true, timedOut: false });
expect(runCaptureImpl).toHaveBeenCalledTimes(1);
expect(runCaptureExImpl).toHaveBeenCalledTimes(1);
});

it("returns a failed warm result without throwing when the warmup probe has no output", () => {
const runCaptureExImpl = vi.fn(() => ({
stdout: "",
stderr: "curl: (28) Operation timed out",
exitCode: 28,
timedOut: true,
}));
const probeRuntimeModelStatus = vi.fn(() => ({
probed: true,
loaded: false,
cpuOnly: false,
}));

const result = maybeWarmOllamaAfterDaemonRestart(
{ provider: "ollama-local", model: "qwen3.6:35b" },
{ probeRuntimeModelStatus, runCaptureExImpl },
);

expect(result).toEqual({ kind: "warmed", ok: false, timedOut: true });
});

it("returns a failed warm result without throwing when the warmup probe throws", () => {
const runCaptureExImpl = vi.fn(() => {
throw new Error("spawn failed");
});
const probeRuntimeModelStatus = vi.fn(() => ({
probed: true,
loaded: false,
cpuOnly: false,
}));

const result = maybeWarmOllamaAfterDaemonRestart(
{ provider: "ollama-local", model: "qwen3.6:35b" },
{ probeRuntimeModelStatus, runCaptureExImpl },
);

expect(result).toEqual({ kind: "warmed", ok: false, timedOut: false });
});
});
88 changes: 88 additions & 0 deletions src/lib/actions/sandbox/agent/ollama-restart-recovery.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import type { RunCaptureExFn } from "../../../inference/local";
import { getOllamaProbeCommand, getResolvedOllamaHost } from "../../../inference/local";
import {
type OllamaRuntimeModelStatus,
type OllamaRuntimeRunCaptureFn,
probeOllamaRuntimeModelStatus,
} from "../../../inference/ollama-runtime-context";

const { runCaptureEx } = require("../../../runner") as typeof import("../../../runner");

export interface OllamaRestartRecoveryRoute {
provider?: string | null;
model?: string | null;
}

export interface OllamaRestartRecoveryDeps {
probeRuntimeModelStatus?: (
model: string,
getOllamaHost: () => string,
runCaptureImpl?: OllamaRuntimeRunCaptureFn,
) => OllamaRuntimeModelStatus;
runCaptureExImpl?: RunCaptureExFn;
getOllamaHost?: () => string;
runCaptureImpl?: OllamaRuntimeRunCaptureFn;
}

export type OllamaRestartRecoveryResult =
| { kind: "skipped"; reason: "not-ollama" | "missing-model" | "already-loaded" | "unreachable" }
| { kind: "warmed"; ok: boolean; timedOut: boolean };

const OLLAMA_PROVIDER = "ollama-local";
const OLLAMA_RESTART_RECOVERY_TIMEOUT_SECONDS = 300;

/**
* Converts optional route metadata into a comparable string value.
*/
function normalizeRouteValue(value: string | null | undefined): string {
return String(value ?? "").trim();
}

/**
* Best-effort recovery for the local-Ollama agent path after a daemon restart.
*
* Killing Ollama drops the model runner even after the daemon comes back. The
* first post-restart OpenClaw agent request can then spend its whole request
* budget cold-loading the model and exit non-zero. We only warm when `/api/ps`
* proves the daemon is reachable but the selected model is not loaded, so a
* genuinely down backend still reaches OpenClaw's existing clear
* backend-unavailable error without an added long probe delay.
*/
export function maybeWarmOllamaAfterDaemonRestart(
route: OllamaRestartRecoveryRoute,
deps: OllamaRestartRecoveryDeps = {},
): OllamaRestartRecoveryResult {
if (normalizeRouteValue(route.provider) !== OLLAMA_PROVIDER) {
return { kind: "skipped", reason: "not-ollama" };
}

const model = normalizeRouteValue(route.model);
if (!model) {
return { kind: "skipped", reason: "missing-model" };
}

const probe = deps.probeRuntimeModelStatus ?? probeOllamaRuntimeModelStatus;
let status: OllamaRuntimeModelStatus;
try {
status = probe(model, deps.getOllamaHost ?? getResolvedOllamaHost, deps.runCaptureImpl);
} catch {
return { kind: "skipped", reason: "unreachable" };
}
if (!status.probed) {
return { kind: "skipped", reason: "unreachable" };
}
if (status.loaded) {
return { kind: "skipped", reason: "already-loaded" };
}

const captureEx = deps.runCaptureExImpl ?? runCaptureEx;
try {
const result = captureEx(getOllamaProbeCommand(model, OLLAMA_RESTART_RECOVERY_TIMEOUT_SECONDS));
return { kind: "warmed", ok: Boolean(result.stdout), timedOut: result.timedOut };
} catch {
return { kind: "warmed", ok: false, timedOut: false };
}
}
Loading
Loading