diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index f2ef7754..e664dacc 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -67,6 +67,8 @@ jobs: echo "tag already exists: ${RELEASE_VERSION}" >&2 exit 1 fi + - name: Verify recommended model sources + run: python3 deploy/verify_model_sources.py --source-root . - name: Verify release readiness on the exact commit env: GH_TOKEN: ${{ github.token }} diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index d8e0c95f..20966354 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -20,6 +20,16 @@ concurrency: cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: + model-sources: + name: Verify Recommended Model Sources + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v7 + with: + persist-credentials: false + - name: Verify immutable Hugging Face revisions + run: python3 deploy/verify_model_sources.py --source-root . --allow-unavailable reuse: name: REUSE Compliance uses: SchmiedmayerLab/.github/.github/workflows/reuse.yml@v0.2 diff --git a/deploy/carina/bootstrap.sh b/deploy/carina/bootstrap.sh index 33d7e526..c2778142 100755 --- a/deploy/carina/bootstrap.sh +++ b/deploy/carina/bootstrap.sh @@ -75,6 +75,7 @@ printf 'Installing the locked vLLM environment.\n' "${root}/bootstrap/bin/uv" pip sync \ --require-hashes --python "${root}/vllm/bin/python" images/gpu/vllm-requirements.txt install -m 0444 images/gpu/heartwood_vllm.py "${root}/vllm/bin/heartwood_vllm.py" +install -m 0444 images/gpu/sitecustomize.py "${root}/vllm/bin/sitecustomize.py" install -m 0555 images/gpu/heartwood-vllm "${root}/vllm/bin/heartwood-vllm" export PATH="${root}/bootstrap/bin:${PATH}" diff --git a/deploy/verify_model_sources.py b/deploy/verify_model_sources.py new file mode 100644 index 00000000..7d7109ec --- /dev/null +++ b/deploy/verify_model_sources.py @@ -0,0 +1,147 @@ +# This source file is part of the Heartwood open-source project +# +# SPDX-FileCopyrightText: 2026 Stanford University and the project authors (see CONTRIBUTORS.md) +# +# SPDX-License-Identifier: MIT + +"""Verify that recommended Hugging Face model revisions still resolve exactly.""" + +from __future__ import annotations + +import argparse +import json +import re +import time +import tomllib +import urllib.error +import urllib.parse +import urllib.request +from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +_SNAPSHOT_CATALOG = Path("images/generic/local-runtime/snapshots.toml") +_COMMIT_PATTERN = re.compile(r"^[0-9a-f]{40}$") + + +class ModelSourceVerificationError(RuntimeError): + """Raised when a recommended model source cannot be verified.""" + + +class ModelSourceUnavailableError(RuntimeError): + """Raised when the provider cannot be reached after bounded retries.""" + + +@dataclass(frozen=True) +class ModelSource: + """One immutable recommended model source.""" + + model_id: str + repository: str + revision: str + + @property + def api_url(self) -> str: + """Return the Hugging Face revision endpoint.""" + repository = urllib.parse.quote(self.repository, safe="/") + revision = urllib.parse.quote(self.revision, safe="") + return f"https://huggingface.co/api/models/{repository}/revision/{revision}" + + +JsonFetcher = Callable[[str, float], dict[str, Any]] + + +def load_model_sources(source_root: Path) -> tuple[ModelSource, ...]: + """Load and validate every recommended model pin from the catalog.""" + catalog_path = source_root / _SNAPSHOT_CATALOG + with catalog_path.open("rb") as file: + catalog = tomllib.load(file) + snapshots = catalog.get("snapshots") + if not isinstance(snapshots, dict) or not snapshots: + raise ModelSourceVerificationError("model snapshot catalog contains no snapshots") + sources: list[ModelSource] = [] + for model_id, value in sorted(snapshots.items()): + if not isinstance(model_id, str) or not isinstance(value, dict): + raise ModelSourceVerificationError("model snapshot catalog is malformed") + repository = value.get("source_repository") + revision = value.get("source_revision") + if not isinstance(repository, str) or repository.count("/") != 1: + raise ModelSourceVerificationError(f"{model_id}: invalid Hugging Face repository") + if not isinstance(revision, str) or _COMMIT_PATTERN.fullmatch(revision) is None: + raise ModelSourceVerificationError(f"{model_id}: revision must be a commit SHA") + sources.append(ModelSource(model_id, repository, revision)) + return tuple(sources) + + +def verify_model_source(source: ModelSource, *, fetch_json: JsonFetcher) -> None: + """Require the remote repository and immutable commit to match the catalog.""" + payload = fetch_json(source.api_url, 20) + repository = payload.get("id") + if repository is None: + repository = payload.get("modelId") + revision = payload.get("sha") + if repository != source.repository: + raise ModelSourceVerificationError( + f"{source.model_id}: expected repository {source.repository}, got {repository!r}" + ) + if revision != source.revision: + raise ModelSourceVerificationError( + f"{source.model_id}: expected revision {source.revision}, got {revision!r}" + ) + + +def _fetch_json(url: str, timeout: float) -> dict[str, Any]: + request = urllib.request.Request(url, headers={"User-Agent": "heartwood-release-verifier"}) + last_error: Exception | None = None + for attempt in range(3): + try: + with urllib.request.urlopen(request, timeout=timeout) as response: + payload = json.load(response) + if not isinstance(payload, dict): + raise ModelSourceVerificationError("Hugging Face returned a non-object response") + return payload + except urllib.error.HTTPError as error: + if error.code < 500 and error.code not in {408, 425, 429}: + raise ModelSourceVerificationError( + f"Hugging Face rejected the immutable revision request with HTTP {error.code}" + ) from error + last_error = error + if attempt < 2: + time.sleep(attempt + 1) + except (OSError, ValueError, urllib.error.URLError) as error: + last_error = error + if attempt < 2: + time.sleep(attempt + 1) + raise ModelSourceUnavailableError(f"Hugging Face is unavailable: {last_error}") + + +def main() -> int: + """Verify all catalog model sources.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--source-root", type=Path, default=Path.cwd()) + parser.add_argument( + "--allow-unavailable", + action="store_true", + help="warn instead of failing when the provider is temporarily unavailable", + ) + args = parser.parse_args() + try: + sources = load_model_sources(args.source_root.resolve()) + for source in sources: + try: + verify_model_source(source, fetch_json=_fetch_json) + print(f"Verified {source.repository}@{source.revision}") + except ModelSourceUnavailableError as error: + if not args.allow_unavailable: + raise + print(f"WARNING: {source.model_id}: {error}") + except (OSError, tomllib.TOMLDecodeError, ModelSourceVerificationError) as error: + parser.error(str(error)) + except ModelSourceUnavailableError as error: + parser.error(str(error)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/design/03-architecture.md b/design/03-architecture.md index c481fdd1..418dd66a 100644 --- a/design/03-architecture.md +++ b/design/03-architecture.md @@ -116,7 +116,7 @@ The gateway owns one model-catalog service. OpenAI and Anthropic catalogs use th A model profile remains the single non-secret execution configuration consumed by policy and OpenHands: a stable profile id, LiteLLM model identifier, base URL when needed, declared normalized completion endpoint, capability tier, and one credential binding. Selecting a catalog entry materializes or replaces one profile for that connection. Researcher interfaces do not require profile ids, credential-storage details, policy endpoints, or LiteLLM prefixes. -Hosted model limits remain upstream-owned. The catalog projects context metadata reported by the provider and the pinned OpenHands/LiteLLM stack, while OpenHands owns provider-specific request shaping. Heartwood does not maintain a second cloud-model limit table. A Heartwood-managed local profile additionally persists the selected runtime context, passes the total capacity to the inference server, and divides it into an input allowance and bounded output budget through OpenHands' native token options. OpenHands' per-event character guard is derived from that input allowance so code and tool output are not prematurely truncated by its smaller default. The runtime, agent, CLI, browser, and notebook therefore describe one capacity. +Hosted model limits remain upstream-owned. The catalog projects context metadata reported by the provider and the pinned OpenHands/LiteLLM stack, while OpenHands owns provider-specific request shaping. Heartwood does not maintain a second cloud-model limit table. A Heartwood-managed local model records its source-declared capacity, while launch selects an effective power-of-two tier from 16,384 through 1,048,576 tokens bounded by that capacity and conservative observed RAM or GPU-memory headroom. The 1,048,576-token validation ceiling accommodates current long-context metadata while bounding implausibly large values; it is not a default or a runtime guarantee. The launcher passes the effective capacity to the inference server, persists it in the active profile, and divides it into an input allowance and bounded output budget through OpenHands' native token options. OpenHands' per-event character guard is derived from that input allowance so code and tool output are not prematurely truncated by its smaller default. The runtime, agent, CLI, browser, and notebook therefore describe one effective capacity. A credential binding identifies a managed identity, environment-backed platform secret, mounted file, or session-only value. Project configuration stores the binding, never the secret. The CLI and same-origin web interface use the same gateway operation to establish a selection; a submitted token is never returned, logged, audited, written to project state, or accepted as a command-line argument. Session-only values must be entered again after the gateway restarts; durable bindings are owned and protected by the deployment. @@ -130,7 +130,9 @@ Local profiles use the same contract. The local connection lists models reported An **Other model** path accepts a Hugging Face `owner/model` identifier and an optional revision. The gateway uses the maintained Hugging Face client to inspect repository metadata and resolve the request to a full immutable commit. When an NVIDIA vLLM deployment is available, it selects a complete snapshot only when the repository has standard configuration and weights and reports a text-generation architecture supported by the packaged tool-call parser; embedding models and unknown parser families fail before transfer. Otherwise, when the portable llama.cpp runtime is available, it selects one single-file GGUF and prefers a uniquely identifiable balanced quantization such as Q4_K_M. It rejects split GGUF files, ambiguous choices, repositories without complete size or digest metadata, unsupported formats, and runtime mismatches with a not-yet-supported message linked to the repository issue chooser. This is intentionally a best-effort compatibility layer rather than a model-format framework. -Before transfer, the normalized plan contains the chosen runtime, immutable source, selected file when applicable, expected bytes, minimum free space, a conservative CPU/RAM/GPU estimate, context window, source-reported license posture, and `recommended` or `user-selected` catalog provenance. Reviewed recommendations declare a 32,768-token local context. User-selected models use supported repository metadata up to that same cap and fall back to 16,384 tokens when no usable value is available. Heartwood keeps this deterministic instead of silently shrinking context according to the current machine; the launcher compares the persisted choice with observed RAM and GPU memory and warns before startup when the conservative estimate exceeds available resources. +Before transfer, the normalized plan contains the chosen runtime, immutable source, selected file when applicable, expected bytes, minimum free space, a conservative CPU/RAM/GPU estimate, context capacity, source-reported license posture, and `recommended` or `user-selected` catalog provenance. Reviewed recommendations currently declare a 32,768-token local capacity. User-selected models use supported repository metadata up to 1,048,576 tokens and default to 32,768 tokens when no usable value is available. At launch, Heartwood applies the same deterministic tier planner across platforms: it reserves resource headroom, selects the largest supported tier that fits the estimate, reports the reason, and uses the 32,768-token safe tier when model-size or memory evidence is unavailable. This explicit effective value keeps the runtime and all interaction surfaces aligned; runtime-native model validation remains the final compatibility check. A model with less than the normal 16,384-token agent minimum remains eligible only for compatibility fixtures and fails normal capability validation. + +Every OpenHands agent receives the upstream `LLMSummarizingCondenser`. When a profile declares an input budget, token-based condensation begins at 75% of that allowance, leaving room for system instructions, tool schemas, the next response, and summary recovery. The upstream 240-event threshold remains as a soft fallback when token metadata is unavailable or unusually sparse, and OpenHands' context-exceeded request remains a hard recovery trigger. The condenser uses the same policy-authorized model route under a separate usage identifier, preserves the first two and recent events, and lets OpenHands own token counting, overflow recovery, and summary construction. Heartwood does not maintain a parallel transcript summarizer. Condensation extends a session but cannot guarantee verbatim retention of old detail, so durable facts and outputs remain in project files and audit records rather than relying on model history alone. The plan and verified download metadata persist in `.heartwood/config.toml`, so a new gateway process projects the same state. Single-file GGUF content is checked against its source SHA-256; snapshots are staged atomically and receive an exact local manifest. The image may include a portable llama.cpp server binary or an explicit GPU vLLM runtime, but model weights live in project storage or a platform disk. NVIDIA images and the Carina native bundle isolate the same hash-locked CUDA 11.8 vLLM environment. Its launcher retains patched Transformers 5 behavior while applying legacy subclass initialization only to vLLM's bundled configuration classes, preserving the current Terra driver baseline without carrying a vulnerable Transformers 4 release. The launcher backports the upstream fix for [GHSA-8fr4-5q9j-m8gm](https://github.com/vllm-project/vllm/security/advisories/GHSA-8fr4-5q9j-m8gm) by removing the affected configuration from vLLM's private registry before any model configuration is loaded. The lock also overrides vLLM's vulnerable exact xgrammar pin with the first patched compatible release, overrides the legacy HTTP graph with a fully patched idna release, and verifies both dependencies with vLLM's structured-output backend. Image construction and native installation run these security regressions in addition to verifying the CUDA build and normal model-configuration integration. A live launch repeats the integration preflight and initializes CUDA before starting the server so a missing GPU, incompatible driver, vulnerable raw launcher, or dependency mismatch fails before model loading begins. Background downloads report transferred and expected bytes through the gateway; successful verification records the standard loopback model profile. Download completion produces `compute-required`, not runtime readiness or a model-capability claim, until `heartwood launch` starts and verifies the selected server. diff --git a/design/07-testing-eval.md b/design/07-testing-eval.md index 6a22f73b..2bc6475f 100644 --- a/design/07-testing-eval.md +++ b/design/07-testing-eval.md @@ -17,9 +17,10 @@ Verification runs only on synthetic data in public development and continuous in 1. **Pure-code tests.** Detector evidence, endpoint normalization, deployment policy, credentials, action settings, upstream analyzer composition, audit chaining and scrubbing, adapters, model connections, provider catalogs, model settings, recommendation filtering, Hugging Face repository inspection, CPU and GPU plan selection, unsupported and ambiguous format errors, artifact integrity, and Skill metadata are tested without a model call or network access. 2. **Deterministic replay.** Synthetic command and event fixtures verify session reconstruction, approval state, audit export, and the ordered cohort, baseline, and aggregate-export results. Prompt or response content from controlled-data sessions must never become a fixture. 3. **Skill tests.** Every bundled Skill has metadata validation, root-confinement checks, deterministic script tests, and synthetic failure cases. The reference fixture contains 24 participants, both outcome classes, target and non-target condition records, and a count-floor boundary; tests require the cohort, training-only baseline, and aggregate export to agree. The `verified` tier means repository verification; it is not cryptographic, clinical, statistical, security, or institutional certification. -4. **Interface contracts.** CLI, notebook, REST, WebSocket, Server-Sent Events, and web view models consume the same gateway services and session events. Cross-interface tests change project model and action settings through one surface, restart the gateway, and require the other surfaces to observe the persisted state; concurrent configuration tests require independent gateway stores to preserve both updates. Tests require the recommendation catalog, arbitrary Hugging Face plan, immutable provenance, automatic runtime choice, resource guidance, transfer status, verified selection, and unsupported-model issue path to agree across interfaces. Vitest, Testing Library, and Playwright cover first-run readiness, provider-source preparation, local-model preparation, compute gating, focus and settings refresh, grouped action review, and desktop and narrow Jupyter layouts. Packaged-gateway and Jupyter-proxy tests cover deployment transports. A real-browser system test uses the built interface, live gateway, OpenHands SDK, loopback model fixture, terminal tool, session-localized inputs, all three reference Skills, repeated action approval, successful and failed tool outcomes, persisted aggregate artifacts, audit download, and CLI replay of the browser-created session. -5. **Runtime integration.** Native AMD64 and ARM64 jobs build the no-weight runtime. With runtime networking disabled, a deterministic OpenAI-compatible fixture exposes a model-list route, the shared catalog selects that model, and a real OpenHands conversation executes the target-condition cohort Skill before exercising allow, reject, low-risk automatic execution, medium-risk confirmation, audit export, native Skill load, REST, notebook, and proxy paths. Native release validation exercises the packaged generic and emulated Carina installer layouts without downloading a model or vLLM. A process-level Carina emulator drives the real CLI through GPU-partition discovery, a filtered Slurm handoff, snapshot verification and staging, runtime preflight, loopback readiness, noninteractive setup, doctor, session opening, process shutdown, and scratch cleanup; it verifies software contracts but is not evidence of real Slurm, NVIDIA, vLLM, or model behavior. A separate job downloads and verifies a tiny CI-only artifact on the runner, mounts it read-only, and performs a real llama.cpp load and completion with runtime networking disabled. +4. **Interface contracts.** CLI, notebook, REST, WebSocket, Server-Sent Events, and web view models consume the same gateway services and session events. Cross-interface tests change project model and action settings through one surface, restart the gateway, and require the other surfaces to observe the persisted state; concurrent configuration tests require independent gateway stores to preserve both updates. Tests require the recommendation catalog, arbitrary Hugging Face plan, immutable provenance, automatic runtime choice, resource-bounded context tier, persisted effective profile budget, resource guidance, transfer status, verified selection, and unsupported-model issue path to agree across interfaces. Vitest, Testing Library, and Playwright cover first-run readiness, provider-source preparation, local-model preparation, compute gating, focus and settings refresh, grouped action review, and desktop and narrow Jupyter layouts. Packaged-gateway and Jupyter-proxy tests cover deployment transports. A real-browser system test uses the built interface, live gateway, OpenHands SDK, loopback model fixture, terminal tool, session-localized inputs, all three reference Skills, repeated action approval, successful and failed tool outcomes, persisted aggregate artifacts, audit download, and CLI replay of the browser-created session. +5. **Runtime integration.** Native AMD64 and ARM64 jobs build the no-weight runtime. With runtime networking disabled, a deterministic OpenAI-compatible fixture exposes a model-list route, the shared catalog selects that model, and a real OpenHands conversation executes the target-condition cohort Skill before exercising allow, reject, low-risk automatic execution, medium-risk confirmation, audit export, native Skill load, REST, notebook, and proxy paths. Context tests require model metadata up to the guarded 1M ceiling, deterministic resource-bounded tiers, the 32K unknown-resource fallback, and the persisted runtime/input/output budget to remain aligned. Adapter tests require every OpenHands agent to receive the upstream summarizing condenser with a threshold derived from the active input budget and an event-count fallback when provider metadata is absent. Native release validation exercises the packaged generic and emulated Carina installer layouts without downloading a model or vLLM. A process-level Carina emulator drives the real CLI through GPU-partition discovery, a filtered Slurm handoff, snapshot verification and staging, runtime preflight, loopback readiness, noninteractive setup, doctor, session opening, process shutdown, and scratch cleanup; it verifies software contracts but is not evidence of real Slurm, NVIDIA, vLLM, or model behavior. A separate job downloads and verifies a tiny CI-only artifact on the runner, mounts it read-only, and performs a real llama.cpp load and completion with runtime networking disabled. CPU-hosted GPU-image verification imports the pinned runtime, checks its CUDA-enabled build metadata and reviewed security backports, exercises vLLM's tokenizer cache in a child process, and resolves a Qwen architecture through vLLM's isolated model-inspection subprocess. A live launch with an attached GPU separately initializes CUDA before model loading. 6. **Capable-model acceptance.** A resource-qualified local or scheduled release run explicitly downloads a pinned recommended artifact outside the image, mounts it read-only, disables container networking, and requires a native OpenHands tool proposal, successful execution of the target-condition cohort Skill, exact aggregate counts and quality flags, model-route records, action-mode records, no error events, and audit export. This gate validates model-to-agent interoperability but does not replace deterministic approval tests or establish a biomedical, production, or benchmark-backed capability tier. +7. **External-source preflight.** Pull requests and releases resolve every recommended Hugging Face repository at its immutable commit through the provider API. Unit tests validate catalog structure without network access; the pull-request preflight detects deleted repositories, mismatched revisions, and copy errors when the provider responds but reports a warning during a bounded provider outage. Release creation remains strict and fails unless every immutable source resolves. ## Capability Tiers diff --git a/docs/container-images.md b/docs/container-images.md index 466d1b76..5e14e28b 100644 --- a/docs/container-images.md +++ b/docs/container-images.md @@ -141,7 +141,7 @@ docker run --rm -it \ heartwood launch --plain ``` -The portable image runs llama.cpp on CPU. Attaching a GPU does not accelerate that path. To use the explicit AMD64 NVIDIA variant on a 16 GB GPU, download `qwen25-coder-7b-instruct-awq-vllm`, retain the same project mount, and start the container with GPU access such as Docker's `--gpus all`. Larger GPUs can use `qwen25-7b-instruct-vllm`. The image supplies vLLM but still downloads model weights only after that explicit project-level command. Before startup, Heartwood initializes CUDA and reports the selected 32,768-token context together with estimated and observed RAM and GPU memory. +The portable image runs llama.cpp on CPU. Attaching a GPU does not accelerate that path. To use the explicit AMD64 NVIDIA variant on a 16 GB GPU, download `qwen25-7b-instruct-awq-vllm`, retain the same project mount, and start the container with GPU access such as Docker's `--gpus all`. Larger GPUs can use `qwen25-7b-instruct-vllm`. The image supplies vLLM but still downloads model weights only after that explicit project-level command. Before startup, Heartwood initializes CUDA, selects a power-of-two context tier from 16K through 1M bounded by model capacity and conservative observed memory, and persists that effective budget for every interface. The reviewed Qwen recommendations currently have a 32K capacity; capacities above 128K are intended only for compatible long-context models on memory-rich systems. ## Runtime Security Controls diff --git a/docs/getting-started-offline.md b/docs/getting-started-offline.md index 6dab51c7..8574ce5c 100644 --- a/docs/getting-started-offline.md +++ b/docs/getting-started-offline.md @@ -120,7 +120,9 @@ heartwood launch --web During launch, Heartwood reports each stage while it verifies the model, checks available RAM and GPU memory, validates the inference runtime, starts the loopback server, waits for model readiness, validates the shared project setup, and opens the selected interface. Large models can take several minutes to load. Heartwood reports elapsed time every 15 seconds and writes runtime details to `.heartwood/logs/local-model.log`. -Recommended local models use a 32,768-token context window. A user-selected Hugging Face model uses its declared context metadata, capped at 32,768 tokens for predictable local resource use, or a 16,384-token fallback when the repository does not publish that metadata. The same total capacity is persisted in `.heartwood/config.toml`, passed to llama.cpp or vLLM, divided into an input allowance and bounded output budget for OpenHands, and shown by the CLI and browser. Heartwood warns before startup when the selected model and context appear too large for available RAM or GPU memory. It does not silently reduce the window because doing so would make model behavior differ across launches. +The current recommended local models support up to 32,768 tokens. Another compatible Hugging Face model may declare a longer capacity up to 1,048,576 tokens; 32,768 tokens remains the safe default when the repository does not publish usable metadata. During launch, Heartwood checks the model size and available RAM or GPU memory, chooses the largest power-of-two tier from 16K through 1M that fits its conservative estimate, and explains the selection. If resource information is unavailable, it uses at most 32K. The effective value is passed to llama.cpp or vLLM and saved in `.heartwood/config.toml`, so the CLI, browser, notebook, and OpenHands use the same input and output budget. + +Use 32K as the normal local baseline and 64K for larger analysis tasks on capable hardware. A 128K window is useful for unusually large working sets when the model and memory have been validated. Capacities above 128K are advanced options for long-context models on memory-rich systems; they increase KV-cache memory, prompt-processing time, and the amount of irrelevant history the model must search. A larger advertised limit does not by itself improve agent quality. Heartwood therefore expands capacity only when both model metadata and observed resources support it, while OpenHands automatically summarizes older working history before the active input budget is exhausted. Leave the launch command running while using the terminal or browser. Press `Ctrl+C` or exit the interface to stop the server. The model files and project state remain on disk. diff --git a/docs/platform-support.md b/docs/platform-support.md index e628f4c2..ce86dd78 100644 --- a/docs/platform-support.md +++ b/docs/platform-support.md @@ -23,7 +23,7 @@ An implemented feature, automated test, or published artifact is not evidence of | Platform path | Available artifact | Current claim | Required deployment work | |---|---|---|---| | Generic Linux or Jupyter environment | Generic release container for AMD64 and ARM64 | Implemented and CI-validated | Validate the host's identity, storage, network, model, and data controls | -| Terra Jupyter | Terra-derived release image for AMD64 | Implemented, CI-validated, and partially live-validated with synthetic data | Complete one immutable-image model interaction and grouped decision workflow, then obtain any required institutional review | +| Terra Jupyter | Terra-derived release image for AMD64 | Implemented and CI-validated; the `0.2.0-beta.2` platform path is partially live-validated with synthetic data | Publish and validate a corrected immutable GPU image, then obtain any required institutional review | | Stanford Carina | Native release installer and bundle | Implemented and CI-validated | Complete the synthetic workflow from the published release on Carina, then obtain any required institutional review | No listed path is automatically approved for protected health information or another controlled dataset. @@ -42,7 +42,7 @@ The published tags are `0.2.0-beta.2-terra`, `edge-terra`, and immutable `sha- None: +def apply_transformers_compatibility() -> None: + """Keep vLLM's pinned configuration classes compatible with Transformers 5.""" from transformers.configuration_utils import PreTrainedConfig + from transformers.tokenization_utils_base import PreTrainedTokenizerBase + if getattr(PreTrainedConfig, "_heartwood_compatibility_applied", False): + return original = PreTrainedConfig.__init_subclass__.__func__ def compatible_init_subclass( @@ -33,6 +38,18 @@ def compatible_init_subclass( original(cls, *args, **kwargs) PreTrainedConfig.__init_subclass__ = classmethod(compatible_init_subclass) + PreTrainedConfig._heartwood_compatibility_applied = True + + if not hasattr(PreTrainedTokenizerBase, "all_special_tokens_extended"): + + @property + def all_special_tokens_extended( + self: PreTrainedTokenizerBase, + ) -> list[object]: + added_by_content = {str(token): token for token in self.added_tokens_decoder.values()} + return [added_by_content.get(token, token) for token in self.all_special_tokens] + + PreTrainedTokenizerBase.all_special_tokens_extended = all_special_tokens_extended def _apply_vllm_security_backport() -> type[object]: @@ -48,6 +65,8 @@ def _verify_runtime(removed_config: type[object]) -> None: import idna # noqa: F401 import transformers import xgrammar # noqa: F401 + from transformers.tokenization_utils_base import PreTrainedTokenizerBase + from vllm.model_executor.models.registry import ModelRegistry from vllm.transformers_utils import config as config_module from vllm.transformers_utils.tokenizer import get_tokenizer # noqa: F401 from vllm.v1.structured_output import backend_xgrammar # noqa: F401 @@ -58,6 +77,37 @@ def _verify_runtime(removed_config: type[object]) -> None: raise RuntimeError("the reviewed xgrammar security override is unavailable") if version("idna") != "3.18": raise RuntimeError("the reviewed idna security override is unavailable") + if not hasattr(PreTrainedTokenizerBase, "all_special_tokens_extended"): + raise RuntimeError("the reviewed Transformers tokenizer compatibility is unavailable") + + tokenizer_check = subprocess.run( + [ + sys.executable, + "-c", + """ +from tokenizers import Tokenizer +from tokenizers.models import WordLevel +from transformers import PreTrainedTokenizerFast +from vllm.transformers_utils.tokenizer import get_cached_tokenizer + +backend = Tokenizer(WordLevel({"": 0, "test": 1}, unk_token="")) +tokenizer = PreTrainedTokenizerFast( + tokenizer_object=backend, + unk_token="", + eos_token="", +) +cached = get_cached_tokenizer(tokenizer) +assert [str(token) for token in cached.all_special_tokens_extended] == cached.all_special_tokens +assert cached.encode("test", add_special_tokens=False) == [1] +""", + ], + check=False, + capture_output=True, + text=True, + ) + if tokenizer_check.returncode != 0: + detail = tokenizer_check.stderr.strip() or tokenizer_check.stdout.strip() + raise RuntimeError(f"vLLM tokenizer subprocess compatibility failed: {detail}") vulnerable_module = importlib.import_module(removed_config.__module__) dynamic_loader_called = False @@ -106,6 +156,13 @@ def reject_dynamic_loader(*_args: object, **_kwargs: object) -> type[object]: if dynamic_loader_called: raise RuntimeError("the vulnerable vLLM dynamic configuration loader was called") + registered_model = ModelRegistry.models.get("Qwen2ForCausalLM") + if registered_model is None: + raise RuntimeError("the Qwen2 architecture is unavailable in the reviewed vLLM runtime") + model_info = registered_model.inspect_model_cls() + if model_info.architecture != "Qwen2ForCausalLM": + raise RuntimeError(f"unexpected inspected model architecture: {model_info.architecture}") + print( f"Transformers {transformers.__version__} integration and " "vLLM GHSA-8fr4-5q9j-m8gm, xgrammar GHSA-7rgv-gqhr-fxg3, and " @@ -115,7 +172,7 @@ def reject_dynamic_loader(*_args: object, **_kwargs: object) -> type[object]: def main() -> None: """Validate or start the vLLM command-line interface.""" - _apply_transformers_compatibility() + apply_transformers_compatibility() removed_config = _apply_vllm_security_backport() if sys.argv[1:] == ["__heartwood_verify_runtime__"]: _verify_runtime(removed_config) diff --git a/images/gpu/sitecustomize.py b/images/gpu/sitecustomize.py new file mode 100644 index 00000000..a8966f05 --- /dev/null +++ b/images/gpu/sitecustomize.py @@ -0,0 +1,13 @@ +# This source file is part of the Heartwood open-source project +# +# SPDX-FileCopyrightText: 2026 Stanford University and the project authors (see CONTRIBUTORS.md) +# +# SPDX-License-Identifier: MIT + +"""Apply the reviewed Transformers compatibility hook in every vLLM process.""" + +from __future__ import annotations + +from heartwood_vllm import apply_transformers_compatibility + +apply_transformers_compatibility() diff --git a/images/platform/Dockerfile b/images/platform/Dockerfile index e948e7dd..c81ecd7a 100644 --- a/images/platform/Dockerfile +++ b/images/platform/Dockerfile @@ -115,6 +115,7 @@ COPY images ./images RUN if [ "${HEARTWOOD_GPU_RUNTIME}" = "vllm" ]; then \ install -m 0444 images/gpu/heartwood_vllm.py /opt/heartwood-vllm/bin/heartwood_vllm.py; \ + install -m 0444 images/gpu/sitecustomize.py /opt/heartwood-vllm/bin/sitecustomize.py; \ install -m 0555 images/gpu/heartwood-vllm /opt/heartwood-vllm/bin/heartwood-vllm; \ fi @@ -134,7 +135,8 @@ RUN if [ -d "${HEARTWOOD_JUPYTER_PREFIX}" ]; then \ /opt/heartwood/.venv/bin/python -m ipykernel install \ --prefix "${HEARTWOOD_JUPYTER_PREFIX}" \ --name heartwood \ - --display-name "Python 3 (Heartwood)"; \ + --display-name "Python 3 (Heartwood)" \ + --env IPYTHONDIR "/tmp/heartwood-ipython"; \ fi WORKDIR ${HEARTWOOD_PLATFORM_HOME} diff --git a/images/platform/scripts/terra_jupyter_contract_smoke.sh b/images/platform/scripts/terra_jupyter_contract_smoke.sh index b9587b80..62e873a4 100755 --- a/images/platform/scripts/terra_jupyter_contract_smoke.sh +++ b/images/platform/scripts/terra_jupyter_contract_smoke.sh @@ -75,6 +75,9 @@ spec = heartwood.get("spec", {}) argv = spec.get("argv", []) if not argv or "/opt/heartwood/.venv/bin/python" not in argv[0]: raise SystemExit(f"Heartwood kernel does not use the Heartwood Python: {argv}") +environment = spec.get("env", {}) +if environment.get("IPYTHONDIR") != "/tmp/heartwood-ipython": + raise SystemExit(f"Heartwood kernel does not isolate incompatible IPython startup files: {environment}") ' "${heartwood_python}" - <<'PY' diff --git a/packages/cli/src/heartwood/cli/__init__.py b/packages/cli/src/heartwood/cli/__init__.py index 00a18edc..47aa3b5c 100644 --- a/packages/cli/src/heartwood/cli/__init__.py +++ b/packages/cli/src/heartwood/cli/__init__.py @@ -1055,7 +1055,7 @@ def _format_model_artifacts(catalog: dict[str, object]) -> str: lines.append(f" {item.get('label')}: {item.get('purpose')}") context_window = item.get("context_window") if isinstance(context_window, int): - lines.append(f" Context: {context_window:,} tokens") + lines.append(f" Context capacity: up to {context_window:,} tokens") lines.append(f" {item.get('availability_reason')}") resources = item.get("recommended_resource_envelope") if isinstance(resources, str): @@ -1078,7 +1078,9 @@ def _format_model_repository(inspection: dict[str, object]) -> str: size = model.get("size_bytes") size_gib = float(size) / (1024**3) if isinstance(size, int | float) else 0 context_window = model.get("context_window") - context_label = f"{context_window:,} tokens" if isinstance(context_window, int) else "Unknown" + context_label = ( + f"up to {context_window:,} tokens" if isinstance(context_window, int) else "Unknown" + ) runtime = "CPU" if model.get("runtime") == "llama-cpp" else "NVIDIA GPU" lines = [ "Heartwood model plan", @@ -1088,7 +1090,7 @@ def _format_model_repository(inspection: dict[str, object]) -> str: f"Revision: {model.get('source_revision')}", f"Runtime: {runtime}", f"Download: {size_gib:.2f} GiB", - f"Context: {context_label}", + f"Context capacity: {context_label}", f"Selection: {inspection.get('selection_reason')}", f"License: {model.get('license_posture')}", "", diff --git a/packages/cli/src/heartwood/cli/_launch.py b/packages/cli/src/heartwood/cli/_launch.py index 14cf464a..aa7b31c4 100644 --- a/packages/cli/src/heartwood/cli/_launch.py +++ b/packages/cli/src/heartwood/cli/_launch.py @@ -18,19 +18,23 @@ import time import urllib.request from collections.abc import Callable, Mapping, Sequence -from dataclasses import dataclass +from dataclasses import dataclass, replace from pathlib import Path +from typing import Literal from packaging.version import InvalidVersion, Version from heartwood.adapters.platform import select_platform_adapter from heartwood.cli._model_snapshot import verify_snapshot from heartwood.gateway import ( + LocalContextPlan, ProjectConfig, ProjectConfigStore, ProjectContext, + estimate_local_runtime_memory, has_authenticated_jupyter_proxy, jupyter_proxy_url, + plan_local_context_window, verify_model_artifact, ) @@ -105,9 +109,9 @@ def format(self) -> str: f"Model: {self.model_root if self.model_root is not None else 'not selected'}", f"Runtime: {self.runtime if self.runtime is not None else 'not selected'}", ( - f"Context: {self.context_window:,} tokens" + f"Context capacity: up to {self.context_window:,} tokens" if self.context_window - else "Context: not selected" + else "Context capacity: not selected" ), f"State: {self.state_root}", f"Project: {self.project_root}", @@ -124,7 +128,7 @@ class LocalRuntimeSelection: """Persisted local-model selection normalized for runtime launch.""" model_root: Path - runtime: str + runtime: Literal["llama-cpp", "vllm"] model_id: str size_bytes: int | None artifact_sha256: str | None @@ -249,7 +253,10 @@ def _run_runtime(options: LaunchOptions, env: Mapping[str, str]) -> int: print(f"{_runtime_label(runtime_kind)} executable is unavailable: {runtime_executable}") return 69 runtime_env = _runtime_environment(env, project=options.project) - _print_resource_assessment(selection, runtime_env) + context_plan = _context_plan(selection, runtime_env) + selection = replace(selection, context_window=context_plan.effective_window) + runtime_env["HEARTWOOD_LOCAL_MODEL_CONTEXT"] = str(selection.context_window) + _print_resource_assessment(selection, runtime_env, context_plan=context_plan) _stage(2, 6, "Validate the local inference runtime") if runtime_kind == "vllm": preflight_error = _preflight_vllm(runtime_executable, runtime_env) @@ -321,11 +328,15 @@ def _run_runtime(options: LaunchOptions, env: Mapping[str, str]) -> int: model_id=model_id, timeout=options.startup_timeout, ): - print( - f"{_runtime_label(runtime_kind)} did not become ready after " - f"{options.startup_timeout} seconds." - ) - _print_runtime_failure(log_path, runtime.poll()) + exit_code = runtime.poll() + if exit_code is None: + print( + f"{_runtime_label(runtime_kind)} did not become ready after " + f"{options.startup_timeout} seconds." + ) + else: + print(f"{_runtime_label(runtime_kind)} exited before becoming ready.") + _print_runtime_failure(log_path, exit_code) return 70 print( f"{_runtime_label(runtime_kind)} is ready after " @@ -333,7 +344,11 @@ def _run_runtime(options: LaunchOptions, env: Mapping[str, str]) -> int: ) runtime_env["HEARTWOOD_LOCAL_RUNTIME_ACTIVE"] = "1" _stage(5, 6, "Validate the shared Heartwood setup") - setup_code = _ensure_setup(options, runtime_env) + setup_code = _ensure_setup( + options, + runtime_env, + context_window=selection.context_window, + ) if setup_code != 0: return setup_code interaction, label = _interaction_command(options, env=env) @@ -474,9 +489,13 @@ def _local_model_selection( if selection is None: return None model = selection.resolved_path(project) - runtime = selection.runtime - if runtime == "auto": + runtime: Literal["llama-cpp", "vllm"] + if selection.runtime == "auto": runtime = "llama-cpp" if model.suffix.casefold() == ".gguf" else "vllm" + elif selection.runtime == "llama-cpp": + runtime = "llama-cpp" + else: + runtime = "vllm" return LocalRuntimeSelection( model_root=model, runtime=runtime, @@ -490,31 +509,64 @@ def _local_model_selection( def _print_resource_assessment( selection: LocalRuntimeSelection, env: Mapping[str, str], + *, + context_plan: LocalContextPlan | None = None, ) -> None: """Report the selected context and a conservative memory preflight.""" + plan = _context_plan(selection, env) if context_plan is None else context_plan model_bytes = selection.size_bytes if model_bytes is None and selection.model_root.exists(): model_bytes = _model_size(selection.model_root) + print( + f"Context window: {plan.effective_window:,} tokens selected automatically " + f"(model capacity: {plan.model_limit:,})." + ) + print(f"Context selection: {plan.reason}") if model_bytes is None: - print(f"Context window: {selection.context_window:,} tokens.") print("Resource check: model size is unavailable; memory could not be estimated.") return - context_bytes = selection.context_window * 128 * 1024 system_required = max( 8 * 1024**3, - int(model_bytes * 1.25) + context_bytes + 4 * 1024**3, + estimate_local_runtime_memory( + context_window=plan.effective_window, + model_size_bytes=model_bytes, + runtime="llama-cpp", + ), ) system_available = _available_system_memory_bytes() - print(f"Context window: {selection.context_window:,} tokens.") _print_memory_result("RAM", system_required, system_available) if selection.runtime == "vllm": - gpu_required = int(model_bytes * 1.15) + context_bytes + 2 * 1024**3 + gpu_required = estimate_local_runtime_memory( + context_window=plan.effective_window, + model_size_bytes=model_bytes, + runtime="vllm", + ) gpu_available = _available_gpu_memory_bytes(env) _print_memory_result("GPU memory", gpu_required, gpu_available) +def _context_plan( + selection: LocalRuntimeSelection, + env: Mapping[str, str], +) -> LocalContextPlan: + model_bytes = selection.size_bytes + if model_bytes is None and selection.model_root.exists(): + model_bytes = _model_size(selection.model_root) + available = ( + _available_gpu_memory_bytes(env) + if selection.runtime == "vllm" + else _available_system_memory_bytes() + ) + return plan_local_context_window( + model_limit=selection.context_window, + model_size_bytes=model_bytes, + runtime=selection.runtime, + available_memory_bytes=available, + ) + + def _print_memory_result(label: str, required: int, available: int | None) -> None: estimate = _format_bytes(required) if available is None: @@ -673,6 +725,7 @@ def _runtime_environment( "HEARTWOOD_IMAGE_FLAVOR", "HEARTWOOD_PLATFORM", "HEARTWOOD_PLATFORM_HOME", + "HEARTWOOD_LOCAL_MODEL_CONTEXT", "HEARTWOOD_LOCAL_RUNTIME_ACTIVE", "LOCAL_SCRATCH_JOB", "SLURM_JOB_ID", @@ -740,7 +793,12 @@ def _catalog_contains_model(payload: object, model_id: str) -> bool: ) -def _ensure_setup(options: LaunchOptions, env: Mapping[str, str] | None = None) -> int: +def _ensure_setup( + options: LaunchOptions, + env: Mapping[str, str] | None = None, + *, + context_window: int | None = None, +) -> int: runtime_env = _runtime_environment( os.environ if env is None else env, project=options.project, @@ -788,6 +846,8 @@ def _ensure_setup(options: LaunchOptions, env: Mapping[str, str] | None = None) ).returncode if setup_code != 0: return setup_code + if context_window is not None: + _persist_effective_context(store, context_window) doctor_command = ( sys.executable, "-m", @@ -802,6 +862,24 @@ def _ensure_setup(options: LaunchOptions, env: Mapping[str, str] | None = None) ).returncode +def _persist_effective_context(store: ProjectConfigStore, context_window: int) -> None: + """Keep the active local OpenHands profile aligned with the launched runtime.""" + output_budget = min(4096, max(512, context_window // 8)) + + def update(config: ProjectConfig) -> ProjectConfig: + profile = config.model_settings.profile() + if not profile.is_local: + raise LaunchConfigurationError("the active model profile is not local") + updated = replace( + profile, + max_input_tokens=context_window - output_budget, + max_output_tokens=output_budget, + ) + return config.with_model_settings(config.model_settings.with_profile(updated)) + + store.update(update) + + def _resolve_slurm_partition(requested: str | None, env: Mapping[str, str]) -> str: partitions = _discover_slurm_gpu_partitions(env) if requested: diff --git a/packages/cli/tests/test_launch.py b/packages/cli/tests/test_launch.py index dcf06f5d..4093408b 100644 --- a/packages/cli/tests/test_launch.py +++ b/packages/cli/tests/test_launch.py @@ -30,6 +30,7 @@ _gguf_file, _interaction_command, _model_size, + _persist_effective_context, _preflight_vllm, _print_resource_assessment, _print_runtime_failure, @@ -45,7 +46,13 @@ run_launch, ) from heartwood.cli._model_snapshot import verify_snapshot -from heartwood.gateway import ProjectConfig, ProjectConfigStore, ProjectContext +from heartwood.gateway import ( + ModelSettings, + ProjectConfig, + ProjectConfigStore, + ProjectContext, + model_profile_from_preset, +) def _options( @@ -53,6 +60,7 @@ def _options( *, selected: bool = True, runtime: str = "vllm", + context_window: int = 32_768, **overrides: object, ) -> LaunchOptions: tmp_path.mkdir(parents=True, exist_ok=True) @@ -72,6 +80,7 @@ def _options( path=project.models_dir / ("model.gguf" if runtime == "llama-cpp" else "model"), runtime=runtime, model_id="test-model", + context_window=context_window, ) values: dict[str, object] = { "project": project, @@ -121,8 +130,8 @@ def test_carina_plan_preserves_project_and_exports_no_credentials( assert plan.platform_id == "carina" assert plan.allocation_required - assert plan.context_window == 16_384 - assert "Context: 16,384 tokens" in plan.format() + assert plan.context_window == 32_768 + assert "Context capacity: up to 32,768 tokens" in plan.format() assert plan.allocation_command[:3] == ("srun", "--pty", "--partition=gpu") assert f"--chdir={tmp_path}" in plan.allocation_command export = next(item for item in plan.allocation_command if item.startswith("--export=")) @@ -258,7 +267,7 @@ def test_launch_scrubs_resource_and_runtime_preflight_environments( ) monkeypatch.setattr( "heartwood.cli._launch._print_resource_assessment", - lambda _selection, env: observed.append(dict(env)), + lambda _selection, env, **_kwargs: observed.append(dict(env)), ) def stop_after_preflight(_executable: Path, env: dict[str, str]) -> str: @@ -418,7 +427,7 @@ def test_launch_supervises_selected_vllm_and_opens_project_chat( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: - options = _options(tmp_path) + options = _options(tmp_path, context_window=131_072) model = tmp_path / ".heartwood" / "models" / "model" _snapshot(model) executable = tmp_path / "vllm" @@ -455,14 +464,30 @@ def completed( "heartwood.cli._launch._resolve_runtime_executable", lambda _kind: executable ) monkeypatch.setattr("heartwood.cli._launch._preflight_vllm", lambda *_args: None) + monkeypatch.setattr( + "heartwood.cli._launch._available_gpu_memory_bytes", + lambda _env: 48 * 1024**3, + ) monkeypatch.setattr("heartwood.cli._launch.subprocess.Popen", FakeProcess) monkeypatch.setattr("heartwood.cli._launch._wait_for_runtime", lambda *_args, **_kwargs: True) - monkeypatch.setattr("heartwood.cli._launch._ensure_setup", lambda *_args, **_kwargs: 0) + setup_options: list[dict[str, object]] = [] + + def ensure_setup(*_args: object, **kwargs: object) -> int: + setup_options.append(kwargs) + return 0 + + monkeypatch.setattr( + "heartwood.cli._launch._ensure_setup", + ensure_setup, + ) monkeypatch.setattr("heartwood.cli._launch.subprocess.run", completed) assert run_launch(options, env={"HEARTWOOD_PLATFORM": "generic"}) == 0 assert observed_processes[0][0] == str(executable) assert "--enable-auto-tool-choice" in observed_processes[0] + context_index = observed_processes[0].index("--max-model-len") + assert observed_processes[0][context_index + 1] == "131072" + assert setup_options == [{"context_window": 131_072}] assert observed_runs[-1][1] == tmp_path assert "--workspace" not in observed_runs[-1][0] assert (tmp_path / ".heartwood" / "logs" / "local-model.log").is_file() @@ -514,6 +539,48 @@ def kill(self) -> None: assert "Runtime log:" in output +def test_launch_reports_early_runtime_exit_without_claiming_timeout( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + options = _options(tmp_path, startup_timeout=600) + _snapshot(options.project.models_dir / "model") + executable = tmp_path / "vllm" + executable.write_text("#!/bin/sh\n", encoding="utf-8") + executable.chmod(0o755) + + class FailedProcess: + def __init__(self, _command: object, **_kwargs: object) -> None: + pass + + def poll(self) -> int: + return 42 + + def terminate(self) -> None: + pass + + def wait(self, timeout: float | None = None) -> int: + del timeout + return 42 + + def kill(self) -> None: + pass + + monkeypatch.setattr( + "heartwood.cli._launch._resolve_runtime_executable", lambda _kind: executable + ) + monkeypatch.setattr("heartwood.cli._launch._preflight_vllm", lambda *_args: None) + monkeypatch.setattr("heartwood.cli._launch.subprocess.Popen", FailedProcess) + monkeypatch.setattr("heartwood.cli._launch._wait_for_runtime", lambda *_args, **_kwargs: False) + + assert run_launch(options, env={"HEARTWOOD_PLATFORM": "generic"}) == 70 + output = capsys.readouterr().out + assert "vLLM exited before becoming ready" in output + assert "after 600 seconds" not in output + assert "vLLM exited with status 42" in output + + def test_llama_cpp_command_uses_the_selected_gguf(tmp_path: Path) -> None: model = tmp_path / "model.gguf" model.write_bytes(b"synthetic") @@ -559,7 +626,8 @@ def test_resource_assessment_reports_context_and_memory_warnings( _print_resource_assessment(selection, {"PATH": "/usr/bin"}) output = capsys.readouterr().out - assert "Context window: 32,768 tokens" in output + assert "Context window: 16,384 tokens selected automatically" in output + assert "model capacity: 32,768" in output assert "Warning: RAM may be insufficient" in output assert "Warning: GPU memory may be insufficient" in output @@ -873,6 +941,27 @@ def failed(command: Sequence[str], **_kwargs: object) -> subprocess.CompletedPro assert "setup" in observed[0] +def test_effective_context_is_persisted_in_the_shared_local_profile(tmp_path: Path) -> None: + options = _options(tmp_path) + adapter = select_platform_adapter({"HEARTWOOD_PLATFORM": "generic"}) + store = ProjectConfigStore( + options.project, + ProjectConfig( + platform_id=adapter.adapter_id, + policy=adapter.default_policy_profile(), + ), + ) + profile = model_profile_from_preset("local-openai-compatible", "heartwood-local-model") + settings = ModelSettings().with_profile(profile).selecting(profile.profile_id) + store.select_model_source("local", settings) + + _persist_effective_context(store, 65_536) + + persisted = store.load().model_settings.profile() + assert persisted.max_input_tokens == 61_440 + assert persisted.max_output_tokens == 4_096 + + def test_partition_discovery_and_selection( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, diff --git a/packages/compliance/tests/test_container_assets.py b/packages/compliance/tests/test_container_assets.py index a535a721..2a4358dd 100644 --- a/packages/compliance/tests/test_container_assets.py +++ b/packages/compliance/tests/test_container_assets.py @@ -114,6 +114,7 @@ def test_platform_image_adds_heartwood_without_replacing_terra_runtime() -> None assert " jq \\" in platform assert "/opt/heartwood/.venv/bin:${PATH}" not in platform assert "ipykernel install" in platform + assert '--env IPYTHONDIR "/tmp/heartwood-ipython"' in platform assert "heartwood-workspace" not in platform assert "heartwood-project" not in platform assert "USER ${HEARTWOOD_PLATFORM_USER}" in platform @@ -220,6 +221,8 @@ def test_gpu_runtime_is_isolated_pinned_and_no_weight() -> None: launcher = _read("images/gpu/start_vllm.sh") verifier = _read("images/gpu/verify_runtime.sh") compatibility = _read("images/gpu/heartwood_vllm.py") + sitecustomize = _read("images/gpu/sitecustomize.py") + executable = _read("images/gpu/heartwood-vllm") lock = _read("images/gpu/vllm-requirements.txt") overrides = _read("images/gpu/vllm-overrides.txt") @@ -233,6 +236,7 @@ def test_gpu_runtime_is_isolated_pinned_and_no_weight() -> None: assert ( "images/gpu/heartwood_vllm.py /opt/heartwood-vllm/bin/heartwood_vllm.py" in dockerfile ) + assert "images/gpu/sitecustomize.py /opt/heartwood-vllm/bin/sitecustomize.py" in dockerfile assert "images/gpu/heartwood-vllm /opt/heartwood-vllm/bin/heartwood-vllm" in dockerfile assert generic.index("uv venv /opt/heartwood-vllm") < generic.index("COPY packages ./packages") assert platform.index("uv venv /opt/heartwood-vllm") < platform.index( @@ -282,6 +286,16 @@ def test_gpu_runtime_is_isolated_pinned_and_no_weight() -> None: assert "GPU runtime image contains a model artifact" in verifier assert 'cls.__module__.startswith("vllm.")' in compatibility assert "PreTrainedConfig.__init_subclass__ = classmethod" in compatibility + assert "_heartwood_compatibility_applied" in compatibility + assert 'hasattr(PreTrainedTokenizerBase, "all_special_tokens_extended")' in compatibility + assert "self.added_tokens_decoder.values()" in compatibility + assert 'ModelRegistry.models.get("Qwen2ForCausalLM")' in compatibility + assert "registered_model.inspect_model_cls()" in compatibility + assert "get_cached_tokenizer" in compatibility + assert "tokenizer_check = subprocess.run" in compatibility + assert "apply_transformers_compatibility" in sitecustomize + assert 'export PYTHONPATH="${runtime_bin}"' in executable + assert "${PYTHONPATH" not in executable assert '_VULNERABLE_CONFIG_TYPE = "Llama_Nemotron_Nano_VL"' in compatibility assert "_CONFIG_REGISTRY.pop(_VULNERABLE_CONFIG_TYPE, None)" in compatibility assert ( @@ -394,6 +408,7 @@ def test_carina_native_launch_requires_verified_synthetic_allocation() -> None: assert "micromamba create" in bootstrap assert 'images/gpu/heartwood_vllm.py "${root}/vllm/bin/heartwood_vllm.py"' in bootstrap + assert 'images/gpu/sitecustomize.py "${root}/vllm/bin/sitecustomize.py"' in bootstrap assert 'images/gpu/heartwood-vllm "${root}/vllm/bin/heartwood-vllm"' in bootstrap assert '"${root}/vllm/bin/heartwood-vllm" __heartwood_verify_runtime__' in bootstrap assert "micromamba install" in bootstrap diff --git a/packages/compliance/tests/test_documentation_assets.py b/packages/compliance/tests/test_documentation_assets.py index df3918fb..13dc4ab3 100644 --- a/packages/compliance/tests/test_documentation_assets.py +++ b/packages/compliance/tests/test_documentation_assets.py @@ -441,6 +441,9 @@ def test_terra_runbook_tracks_platform_and_model_setup() -> None: assert "real Terra workspace validation" in runbook assert "heartwood serve" in runbook assert "heartwood launch --web" in runbook + assert "## Step 7: Stop Compute Safely" in runbook + assert "Do not open the terminal route merely to confirm the pause" in runbook + assert runbook.count("heartwood models download qwen25-7b-instruct-q4_k_m") == 1 assert "edge-terra-coder" not in runbook assert "edge-terra-smoke" not in runbook diff --git a/packages/compliance/tests/test_model_source_verification.py b/packages/compliance/tests/test_model_source_verification.py new file mode 100644 index 00000000..ea2bc5fe --- /dev/null +++ b/packages/compliance/tests/test_model_source_verification.py @@ -0,0 +1,98 @@ +# This source file is part of the Heartwood open-source project +# +# SPDX-FileCopyrightText: 2026 Stanford University and the project authors (see CONTRIBUTORS.md) +# +# SPDX-License-Identifier: MIT + +from __future__ import annotations + +import importlib.util +import sys +import urllib.error +from email.message import Message +from pathlib import Path +from types import ModuleType + +import pytest + + +def _verifier() -> ModuleType: + path = Path("deploy/verify_model_sources.py") + spec = importlib.util.spec_from_file_location("verify_model_sources", path) + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def test_repository_model_sources_are_immutable_and_use_revision_routes() -> None: + verifier = _verifier() + sources = verifier.load_model_sources(Path.cwd()) + + assert {source.model_id for source in sources} == { + "qwen25-7b-instruct-awq-vllm", + "qwen25-7b-instruct-vllm", + } + assert all(len(source.revision) == 40 for source in sources) + assert all(f"/revision/{source.revision}" in source.api_url for source in sources) + + +def test_model_source_verification_requires_exact_repository_and_revision() -> None: + verifier = _verifier() + source = verifier.ModelSource( + "synthetic-model", + "example/model", + "a" * 40, + ) + + verifier.verify_model_source( + source, + fetch_json=lambda _url, _timeout: {"id": "example/model", "sha": "a" * 40}, + ) + verifier.verify_model_source( + source, + fetch_json=lambda _url, _timeout: {"modelId": "example/model", "sha": "a" * 40}, + ) + with pytest.raises(verifier.ModelSourceVerificationError, match="expected revision"): + verifier.verify_model_source( + source, + fetch_json=lambda _url, _timeout: {"id": "example/model", "sha": "b" * 40}, + ) + with pytest.raises(verifier.ModelSourceVerificationError, match="expected repository"): + verifier.verify_model_source( + source, + fetch_json=lambda _url, _timeout: {"id": "other/model", "sha": "a" * 40}, + ) + + +def test_model_source_fetch_distinguishes_missing_revisions_from_provider_outages( + monkeypatch: pytest.MonkeyPatch, +) -> None: + verifier = _verifier() + monkeypatch.setattr(verifier.time, "sleep", lambda _delay: None) + + def unavailable(_request: object, *, timeout: float) -> object: + del timeout + raise urllib.error.URLError("synthetic outage") + + monkeypatch.setattr(verifier.urllib.request, "urlopen", unavailable) + with pytest.raises(verifier.ModelSourceUnavailableError, match="synthetic outage"): + verifier._fetch_json("https://huggingface.co/api/models/example/model", 1) + + def missing(request: object, *, timeout: float) -> object: + del timeout + raise urllib.error.HTTPError(str(request), 404, "Not Found", Message(), None) + + monkeypatch.setattr(verifier.urllib.request, "urlopen", missing) + with pytest.raises(verifier.ModelSourceVerificationError, match="HTTP 404"): + verifier._fetch_json("https://huggingface.co/api/models/example/model", 1) + + def timeout(request: object, *, timeout: float) -> object: + del timeout + raise urllib.error.HTTPError(str(request), 408, "Request Timeout", Message(), None) + + monkeypatch.setattr(verifier.urllib.request, "urlopen", timeout) + with pytest.raises(verifier.ModelSourceUnavailableError, match="HTTP Error 408"): + verifier._fetch_json("https://huggingface.co/api/models/example/model", 1) diff --git a/packages/compliance/tests/test_release_governance.py b/packages/compliance/tests/test_release_governance.py index 925caa61..c1ddec15 100644 --- a/packages/compliance/tests/test_release_governance.py +++ b/packages/compliance/tests/test_release_governance.py @@ -234,7 +234,12 @@ def test_main_validation_owns_release_readiness_dependencies() -> None: def test_release_gate_is_fail_fast_and_uses_readiness_check() -> None: workflow = Path(".github/workflows/create-release.yml").read_text(encoding="utf-8") + validation = Path(".github/workflows/validate.yml").read_text(encoding="utf-8") assert "--required-check 'Release Candidate Ready'" in workflow + assert "python3 deploy/verify_model_sources.py --source-root ." in workflow + assert ( + "python3 deploy/verify_model_sources.py --source-root . --allow-unavailable" in validation + ) assert "for attempt in" not in workflow assert "sleep 30" not in workflow assert "release-required-checks.txt" not in workflow diff --git a/packages/gateway/src/heartwood/gateway/__init__.py b/packages/gateway/src/heartwood/gateway/__init__.py index 3c587eff..f77418cb 100644 --- a/packages/gateway/src/heartwood/gateway/__init__.py +++ b/packages/gateway/src/heartwood/gateway/__init__.py @@ -18,6 +18,11 @@ from heartwood.gateway._asgi import GatewayAsgiApp from heartwood.gateway._gateway import SessionGateway from heartwood.gateway._jupyter import has_authenticated_jupyter_proxy, jupyter_proxy_url +from heartwood.gateway._local_model_contract import ( + LocalContextPlan, + estimate_local_runtime_memory, + plan_local_context_window, +) from heartwood.gateway._local_models import ( HuggingFaceModelRepository, LocalModelChoice, @@ -109,6 +114,7 @@ "GatewayAsgiApp", "GatewayEventStream", "HuggingFaceModelRepository", + "LocalContextPlan", "LocalModelChoice", "LocalModelDownloadManager", "LocalModelDownloadPlan", @@ -157,6 +163,7 @@ "custom_model_connection", "download_model_artifact", "download_model_snapshot", + "estimate_local_runtime_memory", "has_authenticated_jupyter_proxy", "inspect_deployment", "jupyter_proxy_url", @@ -168,6 +175,7 @@ "model_profile_from_preset", "model_settings_from_mapping", "persist_deployment_profile", + "plan_local_context_window", "recommended_model_choices", "verify_model_artifact", "verify_model_snapshot", diff --git a/packages/gateway/src/heartwood/gateway/_local_model_contract.py b/packages/gateway/src/heartwood/gateway/_local_model_contract.py index a869eaaf..09400e0d 100644 --- a/packages/gateway/src/heartwood/gateway/_local_model_contract.py +++ b/packages/gateway/src/heartwood/gateway/_local_model_contract.py @@ -4,8 +4,134 @@ # # SPDX-License-Identifier: MIT -"""Shared local-model context limits.""" +"""Shared local-model context limits and resource-aware planning.""" -DEFAULT_LOCAL_CONTEXT_WINDOW = 16_384 +from __future__ import annotations + +from dataclasses import dataclass +from typing import Literal + +DEFAULT_LOCAL_CONTEXT_WINDOW = 32_768 MINIMUM_LOCAL_CONTEXT_WINDOW = 2_048 -MAXIMUM_LOCAL_CONTEXT_WINDOW = 32_768 +MINIMUM_AGENT_CONTEXT_WINDOW = 16_384 +MAXIMUM_LOCAL_CONTEXT_WINDOW = 1_048_576 +LOCAL_CONTEXT_WINDOW_TIERS = ( + 16_384, + 32_768, + 65_536, + 131_072, + 262_144, + 524_288, + 1_048_576, +) + +_CONTEXT_BYTES_PER_TOKEN = 128 * 1024 +_MEMORY_UTILIZATION = 0.8 +_GIB = 1024**3 + +type LocalRuntimeKind = Literal["llama-cpp", "vllm"] + + +@dataclass(frozen=True, slots=True) +class LocalContextPlan: + """One deterministic effective context choice for a local runtime launch.""" + + model_limit: int + effective_window: int + resource: Literal["RAM", "GPU memory"] + available_bytes: int | None + estimated_required_bytes: int | None + reason: str + + +def plan_local_context_window( + *, + model_limit: int, + model_size_bytes: int | None, + runtime: LocalRuntimeKind, + available_memory_bytes: int | None, +) -> LocalContextPlan: + """Choose a stable context tier bounded by model capacity and available memory.""" + if not MINIMUM_LOCAL_CONTEXT_WINDOW <= model_limit <= MAXIMUM_LOCAL_CONTEXT_WINDOW: + raise ValueError( + f"model context limit must be between {MINIMUM_LOCAL_CONTEXT_WINDOW} and " + f"{MAXIMUM_LOCAL_CONTEXT_WINDOW} tokens" + ) + resource: Literal["RAM", "GPU memory"] = "GPU memory" if runtime == "vllm" else "RAM" + fallback = _tier_at_or_below(min(model_limit, DEFAULT_LOCAL_CONTEXT_WINDOW), model_limit) + if model_size_bytes is None or model_size_bytes <= 0 or available_memory_bytes is None: + return LocalContextPlan( + model_limit=model_limit, + effective_window=fallback, + resource=resource, + available_bytes=available_memory_bytes, + estimated_required_bytes=( + estimate_local_runtime_memory( + context_window=fallback, + model_size_bytes=model_size_bytes, + runtime=runtime, + ) + if model_size_bytes is not None and model_size_bytes > 0 + else None + ), + reason=( + f"Selected the {fallback:,}-token safe default because {resource.lower()} " + "or model-size information is unavailable." + ), + ) + + usable_memory = int(available_memory_bytes * _MEMORY_UTILIZATION) + fixed_bytes, model_multiplier = _runtime_memory_parameters(runtime) + context_budget = usable_memory - int(model_size_bytes * model_multiplier) - fixed_bytes + resource_limit = max(0, context_budget // _CONTEXT_BYTES_PER_TOKEN) + bounded_limit = min(model_limit, resource_limit) + effective = _tier_at_or_below(bounded_limit, min(model_limit, MINIMUM_AGENT_CONTEXT_WINDOW)) + required = estimate_local_runtime_memory( + context_window=effective, + model_size_bytes=model_size_bytes, + runtime=runtime, + ) + if effective == model_limit: + reason = f"Selected the model's full {effective:,}-token context." + elif bounded_limit >= MINIMUM_AGENT_CONTEXT_WINDOW: + reason = ( + f"Selected {effective:,} of {model_limit:,} supported tokens to retain " + f"conservative {resource.lower()} headroom." + ) + else: + reason = ( + f"Selected the minimum viable {effective:,}-token agent context; observed " + f"{resource.lower()} is below the conservative estimate." + ) + return LocalContextPlan( + model_limit=model_limit, + effective_window=effective, + resource=resource, + available_bytes=available_memory_bytes, + estimated_required_bytes=required, + reason=reason, + ) + + +def _tier_at_or_below(limit: int, fallback: int) -> int: + eligible = [tier for tier in LOCAL_CONTEXT_WINDOW_TIERS if tier <= limit] + return max(eligible) if eligible else fallback + + +def _runtime_memory_parameters(runtime: LocalRuntimeKind) -> tuple[int, float]: + return (2 * _GIB, 1.15) if runtime == "vllm" else (4 * _GIB, 1.25) + + +def estimate_local_runtime_memory( + *, + context_window: int, + model_size_bytes: int, + runtime: LocalRuntimeKind, +) -> int: + """Estimate model, runtime, and context memory with conservative headroom.""" + fixed_bytes, model_multiplier = _runtime_memory_parameters(runtime) + return ( + int(model_size_bytes * model_multiplier) + + fixed_bytes + + (context_window * _CONTEXT_BYTES_PER_TOKEN) + ) diff --git a/packages/gateway/src/heartwood/gateway/_local_models.py b/packages/gateway/src/heartwood/gateway/_local_models.py index 9eef39d7..10c36c55 100644 --- a/packages/gateway/src/heartwood/gateway/_local_models.py +++ b/packages/gateway/src/heartwood/gateway/_local_models.py @@ -109,6 +109,10 @@ def validate(self) -> None: raise ModelRepositoryError("local model license posture must not be empty") if self.context_window < 2048: raise ModelRepositoryError("local model context window must be at least 2048 tokens") + if self.context_window > MAXIMUM_LOCAL_CONTEXT_WINDOW: + raise ModelRepositoryError( + f"local model context window must be at most {MAXIMUM_LOCAL_CONTEXT_WINDOW} tokens" + ) if self.runtime == "llama-cpp": if self.source_path is None or not self.source_path.casefold().endswith(".gguf"): raise ModelRepositoryError("CPU models require one GGUF file") diff --git a/packages/gateway/src/heartwood/gateway/_model_artifacts.py b/packages/gateway/src/heartwood/gateway/_model_artifacts.py index 2c81ac02..9290be9b 100644 --- a/packages/gateway/src/heartwood/gateway/_model_artifacts.py +++ b/packages/gateway/src/heartwood/gateway/_model_artifacts.py @@ -143,7 +143,9 @@ def validate(self) -> None: msg = "artifact storage metadata is invalid" raise ModelArtifactError(msg) if not MINIMUM_LOCAL_CONTEXT_WINDOW <= self.context_window <= MAXIMUM_LOCAL_CONTEXT_WINDOW: - raise ModelArtifactError("context_window must be between 2048 and 32768 tokens") + raise ModelArtifactError( + f"context_window must be between 2048 and {MAXIMUM_LOCAL_CONTEXT_WINDOW} tokens" + ) if len(self.artifact_sha256) != 64 or any( character not in "0123456789abcdef" for character in self.artifact_sha256 ): diff --git a/packages/gateway/src/heartwood/gateway/_model_snapshots.py b/packages/gateway/src/heartwood/gateway/_model_snapshots.py index d76f2208..1df08972 100644 --- a/packages/gateway/src/heartwood/gateway/_model_snapshots.py +++ b/packages/gateway/src/heartwood/gateway/_model_snapshots.py @@ -94,7 +94,9 @@ def validate(self) -> None: if self.expected_size_bytes <= 0 or self.minimum_free_bytes < self.expected_size_bytes: raise ModelSnapshotError("snapshot storage metadata is invalid") if not MINIMUM_LOCAL_CONTEXT_WINDOW <= self.context_window <= MAXIMUM_LOCAL_CONTEXT_WINDOW: - raise ModelSnapshotError("context_window must be between 2048 and 32768 tokens") + raise ModelSnapshotError( + f"context_window must be between 2048 and {MAXIMUM_LOCAL_CONTEXT_WINDOW} tokens" + ) def safe_dict(self) -> dict[str, object]: """Return non-secret catalog metadata.""" diff --git a/packages/gateway/src/heartwood/gateway/_openhands_sdk.py b/packages/gateway/src/heartwood/gateway/_openhands_sdk.py index f4eb39b1..943a5546 100644 --- a/packages/gateway/src/heartwood/gateway/_openhands_sdk.py +++ b/packages/gateway/src/heartwood/gateway/_openhands_sdk.py @@ -62,8 +62,22 @@ def __call__(self, **options: object) -> object: """Build one OpenHands agent context.""" +class _Llm(Protocol): + def model_copy(self, *, update: dict[str, object]) -> _Llm: + """Copy the model route with condenser-specific options.""" + + def reset_metrics(self) -> None: + """Separate condenser usage from the agent model metrics.""" + + +class _CondenserFactory(Protocol): + def __call__(self, **options: object) -> object: + """Build one OpenHands history condenser.""" + + class _SdkModule(Protocol): AgentContext: _AgentContextFactory + LLMSummarizingCondenser: _CondenserFactory ConversationFactory = Callable[[Callable[[object], None]], _Conversation] @@ -77,6 +91,9 @@ class _SdkModule(Protocol): _AGENT_LLM_TIMEOUT_SECONDS = 180 _AGENT_LLM_DEFAULT_MAX_MESSAGE_CHARS = 30_000 _AGENT_LLM_ESTIMATED_CHARS_PER_TOKEN = 4 +_AGENT_CONDENSER_INPUT_FRACTION = 0.75 +_AGENT_CONDENSER_MAX_EVENTS = 240 +_AGENT_CONDENSER_KEEP_FIRST = 2 class OpenHandsSdkBackend: @@ -293,6 +310,7 @@ def _default_conversation_factory( # pragma: no cover - container integration context = _agent_context(sdk, skills) agent = sdk.Agent( llm=llm, + condenser=_context_condenser(sdk, llm, self.profile), tools=[ sdk.Tool( name=tools_module.TerminalTool.name, @@ -462,6 +480,23 @@ def _llm_max_message_chars(profile: ModelProfile) -> int: ) +def _context_condenser(sdk: _SdkModule, llm: _Llm, profile: ModelProfile) -> object: + """Build OpenHands' native rolling summary within the active input budget.""" + max_tokens = ( + max(1, int(profile.max_input_tokens * _AGENT_CONDENSER_INPUT_FRACTION)) + if profile.max_input_tokens is not None + else None + ) + condenser_llm = llm.model_copy(update={"usage_id": "heartwood-condenser", "stream": False}) + condenser_llm.reset_metrics() + return sdk.LLMSummarizingCondenser( + llm=condenser_llm, + max_tokens=max_tokens, + max_size=_AGENT_CONDENSER_MAX_EVENTS, + keep_first=_AGENT_CONDENSER_KEEP_FIRST, + ) + + def _backend_error(error: Exception) -> BackendEvent: return BackendEvent( kind=BackendEventKind.ERROR, @@ -469,7 +504,10 @@ def _backend_error(error: Exception) -> BackendEvent: ) -def _agent_context(sdk: _SdkModule, skills: list[object]) -> object: +def _agent_context( + sdk: _SdkModule, + skills: list[object], +) -> object: """Build the context from explicitly verified Skills only.""" return sdk.AgentContext( skills=skills, @@ -478,8 +516,11 @@ def _agent_context(sdk: _SdkModule, skills: list[object]) -> object: load_project_skills=False, system_message_suffix=( "Operate only inside the configured project directory. Do not inspect or modify " - "the reserved .heartwood directory. Follow Heartwood data-use, egress, and " - "aggregate-export controls." + "reserved .heartwood state. An explicitly loaded Skill may read or execute only " + "the files under the Skill location returned by invoke_skill; never modify that " + "location or inspect neighboring .heartwood content. Resolve a Skill-relative file " + "such as scripts/run.py from the returned Skill location, never from the project " + "directory. Follow Heartwood data-use, egress, and aggregate-export controls." ), ) diff --git a/packages/gateway/src/heartwood/gateway/_project_config.py b/packages/gateway/src/heartwood/gateway/_project_config.py index dd270c69..c2dfe9e0 100644 --- a/packages/gateway/src/heartwood/gateway/_project_config.py +++ b/packages/gateway/src/heartwood/gateway/_project_config.py @@ -125,7 +125,8 @@ def validate(self, project: ProjectContext) -> None: raise ProjectConfigError("local model minimum_free_bytes must cover its size") if not MINIMUM_LOCAL_CONTEXT_WINDOW <= self.context_window <= MAXIMUM_LOCAL_CONTEXT_WINDOW: raise ProjectConfigError( - "local model context_window must be between 2048 and 32768 tokens" + f"local model context_window must be between 2048 and " + f"{MAXIMUM_LOCAL_CONTEXT_WINDOW} tokens" ) for field_name, value in ( ("license_posture", self.license_posture), diff --git a/packages/gateway/tests/test_gateway_contract.py b/packages/gateway/tests/test_gateway_contract.py index e9876fe2..a374e227 100644 --- a/packages/gateway/tests/test_gateway_contract.py +++ b/packages/gateway/tests/test_gateway_contract.py @@ -585,7 +585,7 @@ def without_packaged_runtimes(path: Path) -> bool: gateway.env["CUDA_VISIBLE_DEVICES"] = "0" fully_available = cast(list[dict[str, JsonValue]], gateway.model_artifacts()["models"]) assert all(model["available"] for model in fully_available) - assert fully_available[0]["model_id"] == "qwen25-coder-7b-instruct-awq-vllm" + assert fully_available[0]["model_id"] == "qwen25-7b-instruct-awq-vllm" assert fully_available[0]["availability_reason"] == "Recommended for this deployment" diff --git a/packages/gateway/tests/test_local_models.py b/packages/gateway/tests/test_local_models.py index 80ceb8df..6b5a9a8c 100644 --- a/packages/gateway/tests/test_local_models.py +++ b/packages/gateway/tests/test_local_models.py @@ -22,6 +22,7 @@ ModelSnapshot, load_model_artifact_catalog, load_model_snapshot_catalog, + plan_local_context_window, recommended_model_choices, ) @@ -43,11 +44,59 @@ def test_repository_plan_selects_balanced_gguf_for_cpu() -> None: assert plan.model.source_path == "model-q4_k_m.gguf" assert plan.model.source_revision == "1" * 40 assert plan.model.catalog_source == "user-selected" - assert plan.model.context_window == 16_384 + assert plan.model.context_window == 32_768 assert "CPU cores" in str(plan.model.minimum_resource_envelope) assert "balanced single-file GGUF" in plan.selection_reason +def test_context_planner_uses_stable_model_and_memory_bounded_tiers() -> None: + t4 = plan_local_context_window( + model_limit=131_072, + model_size_bytes=5 * 1024**3, + runtime="vllm", + available_memory_bytes=16 * 1024**3, + ) + larger_gpu = plan_local_context_window( + model_limit=131_072, + model_size_bytes=5 * 1024**3, + runtime="vllm", + available_memory_bytes=48 * 1024**3, + ) + long_context_gpu = plan_local_context_window( + model_limit=1_048_576, + model_size_bytes=5 * 1024**3, + runtime="vllm", + available_memory_bytes=80 * 1024**3, + ) + unknown = plan_local_context_window( + model_limit=131_072, + model_size_bytes=None, + runtime="llama-cpp", + available_memory_bytes=None, + ) + + assert t4.effective_window == 32_768 + assert t4.resource == "GPU memory" + assert "headroom" in t4.reason + assert larger_gpu.effective_window == 131_072 + assert "full" in larger_gpu.reason + assert long_context_gpu.effective_window == 262_144 + assert "headroom" in long_context_gpu.reason + assert unknown.effective_window == 32_768 + assert "safe default" in unknown.reason + + +def test_context_planner_never_exceeds_a_non_tier_model_limit() -> None: + plan = plan_local_context_window( + model_limit=48_000, + model_size_bytes=4 * 1024**3, + runtime="llama-cpp", + available_memory_bytes=64 * 1024**3, + ) + + assert plan.effective_window == 32_768 + + def test_repository_plan_prefers_standard_snapshot_when_gpu_runtime_is_available() -> None: repository = _repository( _file("config.json", 100), @@ -72,7 +121,7 @@ def test_repository_plan_bounds_context_from_model_metadata() -> None: repository = _repository( _file("config.json", 100), _file("model.safetensors", 1024, digest="a" * 64), - context_window=131_072, + context_window=2_097_152, ) plan = repository.plan( @@ -81,7 +130,7 @@ def test_repository_plan_bounds_context_from_model_metadata() -> None: gpu_available=True, ) - assert plan.model.context_window == 32_768 + assert plan.model.context_window == 1_048_576 def test_repository_plan_reports_unsupported_formats_and_runtime_mismatch() -> None: @@ -271,6 +320,7 @@ def to_dict(self) -> dict[str, str]: (lambda choice: replace(choice, minimum_free_bytes=1), "storage metadata"), (lambda choice: replace(choice, license_posture=" "), "license posture"), (lambda choice: replace(choice, context_window=1024), "at least 2048"), + (lambda choice: replace(choice, context_window=1_048_577), "at most 1048576"), (lambda choice: replace(choice, source_path="model.bin"), "one GGUF file"), ( lambda choice: replace(choice, source_path="../model.gguf"), @@ -327,7 +377,7 @@ def test_central_catalog_exposes_only_recommended_models() -> None: assert {choice.model_id for choice in choices} == { "qwen25-7b-instruct-q4_k_m", "qwen25-coder-7b-instruct-q4_k_m", - "qwen25-coder-7b-instruct-awq-vllm", + "qwen25-7b-instruct-awq-vllm", "qwen25-7b-instruct-vllm", } assert all(choice.recommended_resource_envelope for choice in choices) diff --git a/packages/gateway/tests/test_model_artifacts.py b/packages/gateway/tests/test_model_artifacts.py index 2ae8918d..49e75ceb 100644 --- a/packages/gateway/tests/test_model_artifacts.py +++ b/packages/gateway/tests/test_model_artifacts.py @@ -359,7 +359,7 @@ def download( ({"source_revision": "latest"}, "immutable revision"), ({"artifact_size_bytes": 0}, "storage metadata"), ({"minimum_free_bytes": 1}, "storage metadata"), - ({"context_window": 32_769}, "between 2048 and 32768"), + ({"context_window": 1_048_577}, "between 2048 and 1048576"), ({"artifact_sha256": "ABC"}, "lowercase SHA-256"), ], ) diff --git a/packages/gateway/tests/test_model_snapshots.py b/packages/gateway/tests/test_model_snapshots.py index d1338271..acad69cb 100644 --- a/packages/gateway/tests/test_model_snapshots.py +++ b/packages/gateway/tests/test_model_snapshots.py @@ -37,9 +37,9 @@ def test_repository_snapshot_catalog_pins_the_carina_demo_model() -> None: assert snapshot.minimum_free_bytes >= snapshot.expected_size_bytes assert snapshot.context_window == 32_768 - terra_snapshot = catalog.snapshot("qwen25-coder-7b-instruct-awq-vllm") + terra_snapshot = catalog.snapshot("qwen25-7b-instruct-awq-vllm") assert terra_snapshot.runtime_profile == "vllm-cuda" - assert terra_snapshot.source_repository == "Qwen/Qwen2.5-Coder-7B-Instruct-AWQ" + assert terra_snapshot.source_repository == "Qwen/Qwen2.5-7B-Instruct-AWQ" assert terra_snapshot.source_revision == "b25037543e9394b818fdfca67ab2a00ecc7dd641" assert terra_snapshot.minimum_free_bytes >= terra_snapshot.expected_size_bytes assert terra_snapshot.context_window == 32_768 @@ -178,7 +178,7 @@ def test_snapshot_metadata_rejects_floating_revisions() -> None: ({"purpose": ""}, "purpose must be"), ({"expected_size_bytes": 0}, "storage metadata"), ({"minimum_free_bytes": 1}, "storage metadata"), - ({"context_window": 32_769}, "between 2048 and 32768"), + ({"context_window": 1_048_577}, "between 2048 and 1048576"), ], ) def test_snapshot_metadata_rejects_unsafe_values(changes: dict[str, object], message: str) -> None: diff --git a/packages/gateway/tests/test_openhands_sdk.py b/packages/gateway/tests/test_openhands_sdk.py index 04e55397..03c90ee0 100644 --- a/packages/gateway/tests/test_openhands_sdk.py +++ b/packages/gateway/tests/test_openhands_sdk.py @@ -23,6 +23,7 @@ _agent_context, _analyzed_risk, _configure_upstream_defaults, + _context_condenser, _llm_max_message_chars, _llm_resilience_options, _security_configuration, @@ -49,13 +50,22 @@ def agent_context(**options: object) -> object: captured.update(options) return object() - context = _agent_context(SimpleNamespace(AgentContext=agent_context), ["verified-skill"]) + context = _agent_context( + SimpleNamespace(AgentContext=agent_context), + ["verified-skill"], + ) assert context is not None assert captured["skills"] == ["verified-skill"] assert captured["load_user_skills"] is False assert captured["load_public_skills"] is False assert captured["load_project_skills"] is False + suffix = str(captured["system_message_suffix"]) + assert "location returned by invoke_skill" in suffix + assert "scripts/run.py" in suffix + assert "never from the project directory" in suffix + assert "/opt/heartwood" not in suffix + assert str(Path.cwd()) not in suffix def test_terminal_tool_masks_all_configured_provider_environment_keys() -> None: @@ -150,6 +160,70 @@ def test_openhands_aligns_local_event_capacity_with_input_budget() -> None: assert _llm_max_message_chars(hosted) == 30_000 +def test_openhands_context_condenser_uses_the_active_model_budget() -> None: + copied: dict[str, object] = {} + options: dict[str, object] = {} + + class FakeLlm: + reset = False + + def model_copy(self, *, update: dict[str, object]) -> FakeLlm: + copied.update(update) + return FakeLlm() + + def reset_metrics(self) -> None: + self.reset = True + + def condenser(**values: object) -> object: + options.update(values) + return object() + + profile = ModelProfile( + profile_id="local", + model="openai/local", + base_url="http://127.0.0.1:8765/v1", + policy_endpoint="http://127.0.0.1:8765/v1/chat/completions", + credential_kind="none", + max_input_tokens=28_672, + max_output_tokens=4_096, + ) + + _context_condenser( + SimpleNamespace(LLMSummarizingCondenser=condenser), + FakeLlm(), + profile, + ) + + assert copied == {"usage_id": "heartwood-condenser", "stream": False} + assert options["max_tokens"] == 21_504 + assert options["max_size"] == 240 + assert options["keep_first"] == 2 + assert cast(FakeLlm, options["llm"]).reset is True + + +def test_openhands_context_condenser_keeps_an_event_limit_without_token_metadata() -> None: + llm = SimpleNamespace(model_copy=lambda **_kwargs: SimpleNamespace(reset_metrics=lambda: None)) + options: dict[str, object] = {} + + def condenser(**values: object) -> object: + options.update(values) + return object() + + sdk = SimpleNamespace(LLMSummarizingCondenser=condenser) + profile = ModelProfile( + profile_id="hosted", + model="openai/model", + policy_endpoint="https://api.openai.com/v1/chat/completions", + credential_kind="environment", + api_key_env="OPENAI_API_KEY", + ) + + _context_condenser(sdk, llm, profile) + + assert options["max_tokens"] is None + assert options["max_size"] == 240 + + def test_openhands_security_configuration_uses_upstream_defense_in_depth() -> None: security = import_module("openhands.sdk.security") analyzer, always = _security_configuration("always-confirm") diff --git a/packages/gateway/tests/test_project_config.py b/packages/gateway/tests/test_project_config.py index 810fd4d9..96a18f87 100644 --- a/packages/gateway/tests/test_project_config.py +++ b/packages/gateway/tests/test_project_config.py @@ -282,9 +282,9 @@ def test_project_config_rejects_symlink(tmp_path: Path) -> None: LocalModelSelection( artifact_id="model", path=".heartwood/models/model", - context_window=32_769, + context_window=1_048_577, ), - "between 2048 and 32768", + "between 2048 and 1048576", ), ], ) diff --git a/packages/webui/src/App.test.tsx b/packages/webui/src/App.test.tsx index fc2c7f61..0df84dd2 100644 --- a/packages/webui/src/App.test.tsx +++ b/packages/webui/src/App.test.tsx @@ -990,6 +990,7 @@ describe("App", () => { expect( within(modelPlan as HTMLElement).getByText(/Recommended: 8 CPU cores/u), ).toBeInTheDocument(); + expect(modelPlan).toHaveTextContent("Up to 32,768 tokens"); expect( within(modelPlan as HTMLElement).getByText(`Revision: ${"1".repeat(40)}`), ).toBeInTheDocument(); diff --git a/packages/webui/src/components/UtilitySheet.tsx b/packages/webui/src/components/UtilitySheet.tsx index 90b913f7..38f7f016 100644 --- a/packages/webui/src/components/UtilitySheet.tsx +++ b/packages/webui/src/components/UtilitySheet.tsx @@ -435,8 +435,8 @@ const SettingsContent = (props: UtilitySheetProps) => { {localComputeLabel(model.runtime)} ·{" "} - {formatBytes(model.size_bytes)} ·{" "} - {model.context_window.toLocaleString()} token context + {formatBytes(model.size_bytes)} · Up to{" "} + {model.context_window.toLocaleString()} tokens {model.purpose} {model.availability_reason} @@ -720,8 +720,8 @@ const CustomLocalModelSetup = ({ {plan.model.label} {localComputeLabel(plan.model.runtime)} ·{" "} - {formatBytes(plan.model.size_bytes)} ·{" "} - {plan.model.context_window.toLocaleString()} token context + {formatBytes(plan.model.size_bytes)} · Up to{" "} + {plan.model.context_window.toLocaleString()} tokens {plan.selection_reason} {plan.model.purpose}