From 533f29af4a0076dc52c87a504111e934a9e68731 Mon Sep 17 00:00:00 2001 From: Paul Schmiedmayer Date: Sun, 12 Jul 2026 16:25:52 -0700 Subject: [PATCH] Add Platform-Aware Installation And Launch --- .github/workflows/native-release.yml | 81 +++++ README.md | 10 + deploy/carina/launch-interactive.sh | 78 +--- deploy/carina/verify_model_snapshot.py | 61 +--- deploy/install.sh | 158 ++++++++ deploy/package-native.sh | 28 ++ deploy/tests/native_installer_smoke.sh | 66 ++++ design/03-architecture.md | 8 + design/08-development.md | 2 + design/09-implementation-plan.md | 5 +- docs/carina-cli.md | 47 ++- docs/container-images.md | 2 + docs/platform-support.md | 2 +- .../heartwood/adapters/platform/__init__.py | 13 +- .../src/heartwood/adapters/platform/carina.py | 7 +- .../src/heartwood/adapters/platform/terra.py | 58 +++ .../adapters/tests/test_generic_adapters.py | 20 +- packages/cli/src/heartwood/cli/__init__.py | 53 ++- packages/cli/src/heartwood/cli/_launch.py | 337 ++++++++++++++++++ .../cli/src/heartwood/cli/_model_snapshot.py | 69 ++++ packages/cli/tests/test_launch.py | 301 ++++++++++++++++ .../compliance/tests/test_container_assets.py | 42 ++- .../src/heartwood/gateway/_readiness.py | 7 +- packages/gateway/tests/test_readiness.py | 17 +- 24 files changed, 1306 insertions(+), 166 deletions(-) create mode 100644 .github/workflows/native-release.yml create mode 100755 deploy/install.sh create mode 100755 deploy/package-native.sh create mode 100755 deploy/tests/native_installer_smoke.sh create mode 100644 packages/adapters/src/heartwood/adapters/platform/terra.py create mode 100644 packages/cli/src/heartwood/cli/_launch.py create mode 100644 packages/cli/src/heartwood/cli/_model_snapshot.py create mode 100644 packages/cli/tests/test_launch.py diff --git a/.github/workflows/native-release.yml b/.github/workflows/native-release.yml new file mode 100644 index 00000000..ca4416a0 --- /dev/null +++ b/.github/workflows/native-release.yml @@ -0,0 +1,81 @@ +# This source file is part of the Heartwood open-source project +# +# SPDX-FileCopyrightText: 2026 Stanford University and the project authors (see CONTRIBUTORS.md) +# +# SPDX-License-Identifier: MIT + +name: Native Release Assets + +on: + pull_request: + paths: + - ".github/workflows/native-release.yml" + - "deploy/**" + - "packages/**" + - "pyproject.toml" + - "uv.lock" + release: + types: [published] + workflow_dispatch: + +permissions: + contents: read + +jobs: + package: + name: Build And Verify Native Assets + runs-on: ubuntu-24.04 + steps: + - name: Checkout repository + uses: actions/checkout@v7 + with: + fetch-depth: 0 + persist-credentials: false + - name: Build native assets + env: + RELEASE_VERSION: ${{ github.event.release.tag_name || github.sha }} + run: deploy/package-native.sh dist "${RELEASE_VERSION}" + - name: Verify local installer path + run: >- + dist/heartwood-installer + --bundle dist/heartwood-native.tar.gz + --checksums dist/SHA256SUMS + --root "${RUNNER_TEMP}/heartwood" + --platform generic + --dry-run + - name: Test native installation layout + run: deploy/tests/native_installer_smoke.sh dist + - name: Upload workflow artifact + uses: actions/upload-artifact@v7 + with: + name: heartwood-native-assets + path: | + dist/heartwood-installer + dist/heartwood-native.tar.gz + dist/SHA256SUMS + if-no-files-found: error + + publish: + name: Publish Native Release Assets + if: github.event_name == 'release' + needs: package + runs-on: ubuntu-24.04 + permissions: + attestations: write + contents: write + id-token: write + steps: + - name: Download native assets + uses: actions/download-artifact@v8 + with: + name: heartwood-native-assets + path: dist + - name: Attest native assets + uses: actions/attest@v4 + with: + subject-path: dist/* + - name: Publish release assets + env: + GH_TOKEN: ${{ github.token }} + RELEASE_TAG: ${{ github.event.release.tag_name }} + run: gh release upload "${RELEASE_TAG}" dist/* --clobber --repo "${GITHUB_REPOSITORY}" diff --git a/README.md b/README.md index bb12cac9..86a594aa 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,16 @@ The default runtime uses the generic platform policy and synthetic OMOP data-sou See [Platform Support](docs/platform-support.md) for platform-specific evidence and limitations. All of Us, AnVIL, Seven Bridges, Velsera, DNAnexus, and UK Biobank Research Analysis Platform are design targets rather than supported platforms. +Tagged releases publish a verified native bundle for environments where a platform image is not appropriate. Download the release installer, verify it before execution, and use the installed environment-aware launcher: + +```bash +./heartwood-installer --root /persistent/project/heartwood +export PATH="/persistent/project/heartwood/bin:${PATH}" +heartwood launch --model-root /persistent/project/models/ +``` + +`heartwood launch --dry-run` reports the detected platform, storage, model, and compute plan without changing state. On Carina it asks before invoking Slurm; Terra and generic containers use their already-provisioned compute. See [Carina CLI Pilot](docs/carina-cli.md) for the synthetic native GPU workflow. + ## Researcher Experience ![Heartwood synthetic reference analysis](docs/assets/web-reference-analysis.png) diff --git a/deploy/carina/launch-interactive.sh b/deploy/carina/launch-interactive.sh index 8ba06215..030c65b0 100755 --- a/deploy/carina/launch-interactive.sh +++ b/deploy/carina/launch-interactive.sh @@ -7,9 +7,6 @@ set -euo pipefail -script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -repo_root="$(cd "${script_dir}/../.." && pwd)" - environment_root="" model_root="" state_root="" @@ -29,70 +26,19 @@ done : "${SLURM_JOB_ID:?launch Heartwood inside a Slurm compute allocation}" : "${LOCAL_SCRATCH_JOB:?Carina job-local scratch is unavailable}" -if [[ ! -x "${environment_root}/heartwood/bin/heartwood" ]]; then - echo "Heartwood environment is unavailable; run deploy/carina/bootstrap.sh first" >&2 - exit 69 -fi -if [[ ! -x "${environment_root}/vllm/bin/vllm" ]]; then - echo "vLLM environment is unavailable; run deploy/carina/bootstrap.sh first" >&2 +heartwood="${environment_root}/heartwood/bin/heartwood" +if [[ ! -x "${heartwood}" ]]; then + echo "Heartwood environment is unavailable; run the native installer first" >&2 exit 69 fi -"${environment_root}/heartwood/bin/python" "${script_dir}/verify_model_snapshot.py" "${model_root}" -mkdir -p "${state_root}" -staged_model="$(mktemp -d "${LOCAL_SCRATCH_JOB%/}/heartwood-model.XXXXXX")" -runtime_pid="" -cleanup() { - if [[ -n "${runtime_pid}" ]]; then - kill "${runtime_pid}" >/dev/null 2>&1 || true - for _ in {1..10}; do - if ! kill -0 "${runtime_pid}" >/dev/null 2>&1; then - break - fi - sleep 1 - done - if kill -0 "${runtime_pid}" >/dev/null 2>&1; then - kill -KILL "${runtime_pid}" >/dev/null 2>&1 || true - fi - wait "${runtime_pid}" >/dev/null 2>&1 || true - fi - rm -rf "${staged_model}" -} -trap cleanup EXIT INT TERM -cp -a "${model_root}/." "${staged_model}/" - -unset GH_TOKEN GITHUB_TOKEN HF_TOKEN HUGGING_FACE_HUB_TOKEN -unset OPENAI_API_KEY ANTHROPIC_API_KEY AZURE_API_KEY export HEARTWOOD_PLATFORM=carina -export HEARTWOOD_AGENT_BACKEND=openhands-sdk -export HEARTWOOD_LOCAL_MODEL_PATH="${staged_model}" -export HEARTWOOD_LOCAL_MODEL_ALIAS="${model_id}" -export HEARTWOOD_VLLM_EXECUTABLE="${environment_root}/vllm/bin/vllm" -export PATH="${environment_root}/heartwood/bin:${PATH}" - -bash "${repo_root}/images/gpu/start_vllm.sh" >"${state_root}/vllm-${SLURM_JOB_ID}.log" 2>&1 & -runtime_pid="$!" - -python - <<'PY' -import json -import time -import urllib.request - -deadline = time.time() + 300 -while time.time() < deadline: - try: - with urllib.request.urlopen("http://127.0.0.1:8765/v1/models", timeout=2) as response: - if response.status == 200 and json.load(response).get("data"): - break - except OSError: - time.sleep(1) -else: - raise SystemExit("vLLM did not become ready within 300 seconds") -PY - -workspace="${state_root}/sessions" -if [[ ! -f "${state_root}/setup.json" ]]; then - heartwood --workspace "${workspace}" setup \ - --model-source local --model-id "${model_id}" --non-interactive --yes -fi -heartwood --workspace "${workspace}" --session-id carina-demo chat +exec "${heartwood}" \ + --workspace "${state_root}/sessions" \ + --session-id carina-demo \ + launch \ + --inside-allocation \ + --environment-root "${environment_root}" \ + --model-root "${model_root}" \ + --state-root "${state_root}" \ + --model-id "${model_id}" diff --git a/deploy/carina/verify_model_snapshot.py b/deploy/carina/verify_model_snapshot.py index be419ef8..b3acd178 100644 --- a/deploy/carina/verify_model_snapshot.py +++ b/deploy/carina/verify_model_snapshot.py @@ -8,67 +8,10 @@ from __future__ import annotations -import hashlib -import os -import re import sys -from pathlib import Path, PurePosixPath +from pathlib import Path -_ENTRY = re.compile(r"^([0-9a-fA-F]{64}) [ *](.+)$") - - -def verify_snapshot(root: Path) -> None: - """Reject unlisted, missing, linked, duplicated, or modified snapshot files.""" - manifest = root / "SHA256SUMS" - if not root.is_dir() or not manifest.is_file() or manifest.is_symlink(): - raise ValueError("model root must contain a regular SHA256SUMS manifest") - expected: dict[str, str] = {} - for line_number, line in enumerate(manifest.read_text(encoding="utf-8").splitlines(), 1): - match = _ENTRY.fullmatch(line) - if match is None: - raise ValueError(f"invalid SHA256SUMS entry on line {line_number}") - digest, name = match.groups() - manifest_relative = PurePosixPath(name) - if ( - manifest_relative.is_absolute() - or ".." in manifest_relative.parts - or name in {"", "SHA256SUMS"} - ): - raise ValueError(f"unsafe SHA256SUMS path on line {line_number}") - normalized = manifest_relative.as_posix() - if normalized in expected: - raise ValueError(f"duplicate SHA256SUMS path: {normalized}") - expected[normalized] = digest.lower() - - actual: set[str] = set() - for path in root.rglob("*"): - snapshot_relative = path.relative_to(root).as_posix() - if path.is_symlink(): - raise ValueError(f"model snapshot contains a symbolic link: {snapshot_relative}") - if path.is_file() and snapshot_relative != "SHA256SUMS": - actual.add(snapshot_relative) - if actual != set(expected): - missing = sorted(set(expected) - actual) - unlisted = sorted(actual - set(expected)) - detail = "; ".join( - item - for item in ( - f"missing: {', '.join(missing)}" if missing else "", - f"unlisted: {', '.join(unlisted)}" if unlisted else "", - ) - if item - ) - raise ValueError(f"model snapshot does not match SHA256SUMS coverage ({detail})") - - for relative_name, expected_digest in expected.items(): - hasher = hashlib.sha256() - descriptor = os.open(root / relative_name, os.O_RDONLY | os.O_NOFOLLOW) - with os.fdopen(descriptor, "rb") as file: - while chunk := file.read(1024 * 1024): - hasher.update(chunk) - digest = hasher.hexdigest() - if digest != expected_digest: - raise ValueError(f"SHA-256 mismatch: {relative_name}") +from heartwood.cli._model_snapshot import verify_snapshot def main() -> int: diff --git a/deploy/install.sh b/deploy/install.sh new file mode 100755 index 00000000..dabaee60 --- /dev/null +++ b/deploy/install.sh @@ -0,0 +1,158 @@ +#!/usr/bin/env bash +# This source file is part of the Heartwood open-source project +# +# SPDX-FileCopyrightText: 2026 Stanford University and the project authors (see CONTRIBUTORS.md) +# +# SPDX-License-Identifier: MIT + +set -euo pipefail + +repository="SchmiedmayerLab/heartwood" +root="${HOME}/.local/share/heartwood" +version="latest" +platform="auto" +bundle="" +checksums="" +dry_run="false" + +usage() { + cat <<'EOF' +Usage: heartwood-installer [options] + + --root PATH Installation and runtime root + --version VERSION GitHub release tag, or latest (default) + --platform NAME auto, carina, or generic + --bundle PATH Use a local heartwood-native.tar.gz + --checksums PATH SHA256SUMS for a local bundle + --dry-run Verify and display the installation without changing it +EOF +} + +while (($#)); do + case "$1" in + --root) root="${2:?missing installation root}"; shift 2 ;; + --version) version="${2:?missing version}"; shift 2 ;; + --platform) platform="${2:?missing platform}"; shift 2 ;; + --bundle) bundle="${2:?missing bundle}"; shift 2 ;; + --checksums) checksums="${2:?missing checksum manifest}"; shift 2 ;; + --dry-run) dry_run="true"; shift ;; + --help|-h) usage; exit 0 ;; + *) echo "unknown argument: $1" >&2; usage >&2; exit 64 ;; + esac +done + +if [[ "${platform}" == "auto" ]]; then + if [[ -n "${SLURM_CLUSTER_NAME:-}" || -n "${HEARTWOOD_CARINA:-}" || "${HEARTWOOD_PLATFORM:-}" == "carina" ]]; then + platform="carina" + else + platform="generic" + fi +fi +if [[ "${platform}" != "carina" && "${platform}" != "generic" ]]; then + echo "unsupported native platform: ${platform}" >&2 + exit 64 +fi + +workspace="$(mktemp -d)" +installation_staging="" +cleanup() { + rm -rf "${workspace}" + if [[ -n "${installation_staging}" && -d "${installation_staging}" ]]; then + rm -rf "${installation_staging}" + fi +} +trap cleanup EXIT + +if [[ -z "${bundle}" ]]; then + if ! command -v curl >/dev/null 2>&1; then + echo "curl is required to retrieve a GitHub Release" >&2 + exit 69 + fi + if [[ "${version}" == "latest" ]]; then + release_root="https://github.com/${repository}/releases/latest/download" + else + release_root="https://github.com/${repository}/releases/download/${version}" + fi + bundle="${workspace}/heartwood-native.tar.gz" + checksums="${workspace}/SHA256SUMS" + curl --fail --location --silent --show-error "${release_root}/heartwood-native.tar.gz" --output "${bundle}" + curl --fail --location --silent --show-error "${release_root}/SHA256SUMS" --output "${checksums}" +elif [[ -z "${checksums}" ]]; then + echo "--checksums is required with --bundle" >&2 + exit 64 +fi + +if [[ ! -f "${bundle}" || ! -f "${checksums}" ]]; then + echo "bundle and checksum manifest must be regular files" >&2 + exit 66 +fi +checksum_line_count="$(wc -l <"${checksums}" | tr -d ' ')" +checksum_line="$(cat "${checksums}")" +if [[ "${checksum_line_count}" != "1" || ! "${checksum_line}" =~ ^[0-9a-f]{64}[[:space:]][[:space:]]heartwood-native\.tar\.gz$ ]]; then + echo "checksum manifest must contain exactly heartwood-native.tar.gz" >&2 + exit 66 +fi +if [[ "${bundle}" != "${workspace}/heartwood-native.tar.gz" ]]; then + cp "${bundle}" "${workspace}/heartwood-native.tar.gz" +fi +if [[ "${checksums}" != "${workspace}/SHA256SUMS" ]]; then + cp "${checksums}" "${workspace}/SHA256SUMS" +fi +( + cd "${workspace}" + sha256sum --check --strict SHA256SUMS +) + +release_version="$(tar -xOf "${workspace}/heartwood-native.tar.gz" heartwood/HEARTWOOD_VERSION)" +if [[ ! "${release_version}" =~ ^[A-Za-z0-9][A-Za-z0-9._+-]{0,127}$ ]]; then + echo "bundle contains an unsafe release version" >&2 + exit 66 +fi + +printf 'Heartwood native installation\n\n' +printf 'Platform: %s\n' "${platform}" +printf 'Version: %s\n' "${release_version}" +printf 'Root: %s\n' "${root}" +if [[ "${dry_run}" == "true" ]]; then + printf 'Result: verified dry run; no files changed\n' + exit 0 +fi + +versions_root="${root}/versions" +source_root="${versions_root}/${release_version}" +runtime_root="${root}/runtimes/${release_version}" +mkdir -p "${versions_root}" "${root}/runtimes" "${root}/bin" +if [[ ! -d "${source_root}" ]]; then + installation_staging="$(mktemp -d "${versions_root}/.heartwood-${release_version}.XXXXXX")" + tar -xzf "${workspace}/heartwood-native.tar.gz" -C "${installation_staging}" --strip-components=1 + mv "${installation_staging}" "${source_root}" + installation_staging="" +fi + +if [[ "${platform}" == "carina" ]]; then + ( + cd "${source_root}" + deploy/carina/bootstrap.sh --environment-root "${runtime_root}" + ) +else + if ! command -v uv >/dev/null 2>&1; then + echo "uv is required for a generic native installation" >&2 + exit 69 + fi + ( + cd "${source_root}" + UV_PROJECT_ENVIRONMENT="${runtime_root}/heartwood" uv sync --locked --no-dev --all-extras + ) +fi + +ln -sfn "${source_root}" "${root}/current" +command_path="${runtime_root}/heartwood/bin/heartwood" +if [[ ! -x "${command_path}" ]]; then + echo "installed heartwood command is unavailable" >&2 + exit 70 +fi +printf '#!/usr/bin/env bash\nexport HEARTWOOD_INSTALL_ROOT=%q\nexport HEARTWOOD_NATIVE_ROOT=%q\nexport HEARTWOOD_NATIVE_VERSION=%q\nexport HEARTWOOD_VERSION=%q\nexport HEARTWOOD_HOME=%q\nexec %q "$@"\n' \ + "${source_root}" "${root}" "${release_version}" "${release_version}" "${root}/state" "${command_path}" \ + >"${root}/bin/heartwood" +chmod +x "${root}/bin/heartwood" +printf '\nInstalled %s\nAdd %s to PATH.\n' "${release_version}" "${root}/bin" diff --git a/deploy/package-native.sh b/deploy/package-native.sh new file mode 100755 index 00000000..626a4804 --- /dev/null +++ b/deploy/package-native.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# This source file is part of the Heartwood open-source project +# +# SPDX-FileCopyrightText: 2026 Stanford University and the project authors (see CONTRIBUTORS.md) +# +# SPDX-License-Identifier: MIT + +set -euo pipefail + +output_dir="${1:-dist}" +version="${2:-$(git describe --tags --always --dirty)}" +archive="${output_dir}/heartwood-native.tar.gz" +workspace="$(mktemp -d)" +cleanup() { + rm -rf "${workspace}" +} +trap cleanup EXIT + +mkdir -p "${output_dir}" "${workspace}/heartwood" +git archive --format=tar HEAD | tar -xf - -C "${workspace}/heartwood" +printf '%s\n' "${version}" >"${workspace}/heartwood/HEARTWOOD_VERSION" +tar -czf "${archive}" -C "${workspace}" heartwood +( + cd "${output_dir}" + sha256sum "$(basename "${archive}")" >SHA256SUMS +) +cp deploy/install.sh "${output_dir}/heartwood-installer" +chmod +x "${output_dir}/heartwood-installer" diff --git a/deploy/tests/native_installer_smoke.sh b/deploy/tests/native_installer_smoke.sh new file mode 100755 index 00000000..13de5d8b --- /dev/null +++ b/deploy/tests/native_installer_smoke.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# This source file is part of the Heartwood open-source project +# +# SPDX-FileCopyrightText: 2026 Stanford University and the project authors (see CONTRIBUTORS.md) +# +# SPDX-License-Identifier: MIT + +set -euo pipefail + +assets="${1:?asset directory is required}" +workspace="$(mktemp -d)" +cleanup() { + rm -rf "${workspace}" +} +trap cleanup EXIT + +mkdir -p "${workspace}/bin" +cat >"${workspace}/bin/uv" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +: "${UV_PROJECT_ENVIRONMENT:?UV_PROJECT_ENVIRONMENT is required}" +mkdir -p "${UV_PROJECT_ENVIRONMENT}/bin" +cat >"${UV_PROJECT_ENVIRONMENT}/bin/heartwood" <<'COMMAND' +#!/usr/bin/env bash +printf '%s|%s|%s|%s\n' "${HEARTWOOD_INSTALL_ROOT}" "${HEARTWOOD_NATIVE_VERSION}" "${HEARTWOOD_VERSION}" "${HEARTWOOD_HOME}" +COMMAND +chmod +x "${UV_PROJECT_ENVIRONMENT}/bin/heartwood" +EOF +chmod +x "${workspace}/bin/uv" + +PATH="${workspace}/bin:${PATH}" "${assets}/heartwood-installer" \ + --bundle "${assets}/heartwood-native.tar.gz" \ + --checksums "${assets}/SHA256SUMS" \ + --root "${workspace}/installation" \ + --platform generic + +test -x "${workspace}/installation/bin/heartwood" +test -L "${workspace}/installation/current" +output="$("${workspace}/installation/bin/heartwood")" +case "${output}" in + "${workspace}/installation/versions/"*"|"*"|"*"|${workspace}/installation/state") ;; + *) echo "installed command did not receive native installation metadata" >&2; exit 1 ;; +esac + +printf '%064d heartwood-native.tar.gz\n' 0 >"${workspace}/invalid-SHA256SUMS" +if "${assets}/heartwood-installer" \ + --bundle "${assets}/heartwood-native.tar.gz" \ + --checksums "${workspace}/invalid-SHA256SUMS" \ + --root "${workspace}/invalid" \ + --platform generic; then + echo "installer accepted a corrupted checksum" >&2 + exit 1 +fi +test ! -e "${workspace}/invalid" + +digest="$(cut -d ' ' -f 1 "${assets}/SHA256SUMS")" +printf '%s ../heartwood-native.tar.gz\n' "${digest}" >"${workspace}/unsafe-SHA256SUMS" +if "${assets}/heartwood-installer" \ + --bundle "${assets}/heartwood-native.tar.gz" \ + --checksums "${workspace}/unsafe-SHA256SUMS" \ + --root "${workspace}/unsafe" \ + --platform generic; then + echo "installer accepted an unsafe checksum manifest" >&2 + exit 1 +fi +test ! -e "${workspace}/unsafe" diff --git a/design/03-architecture.md b/design/03-architecture.md index da78e343..a99a2d9e 100644 --- a/design/03-architecture.md +++ b/design/03-architecture.md @@ -151,6 +151,14 @@ Notebook support reuses the gateway and web UI through the platform's authentica Provider-specific execution adapters are not part of the default architecture. A platform adapter may supply identity, endpoint, visibility, and static-catalog metadata. Narrow catalog adapters call maintained SDK listing operations only; OpenHands and LiteLLM remain the provider execution and compatibility layer. +### Installation And Launch Lifecycle + +Native deployments use a small standalone installer only before the `heartwood` command exists. The installer verifies a versioned GitHub Release bundle, places its immutable source payload under a versioned installation root, creates locked runtime environments, and publishes a stable command link. It never downloads model weights, stores credentials, submits compute jobs, or selects a data route. + +After installation, `heartwood launch` owns the common lifecycle contract. A platform launch adapter detects whether compute is already provisioned, returns a typed resource proposal, and optionally constructs a scheduler request. Scheduler submission requires a separate explicit user decision and is never implied by action-confirmation settings. Once compute is available, the command verifies the selected model snapshot, stages it into platform scratch when required, starts the configured OpenAI-compatible runtime, waits for readiness, opens the normal Heartwood conversation, and cleans up only the processes and temporary files it created. + +Carina implements scheduler acquisition through Slurm. Terra and generic container environments report already-provisioned compute and use the same post-allocation lifecycle without invoking Slurm. Shell and batch files remain thin compatibility and debugging wrappers over this contract; they do not own model policy, session state, or agent execution. + ## Audit And Durability OpenHands persists the operational conversation state needed for resume. Heartwood records researcher and agent messages in its in-boundary session event stream so every client can reconstruct the same transcript, and derives a separate hash-chained audit record from the session events. The audit record stores route ids, endpoint decisions, tool names, risk levels, result status, activated Skill ids, and human decisions. Prompt text, model response text, action summaries, filesystem paths, row values, and secrets are excluded from exported records; message content and action summaries remain available only in the in-boundary conversation and session event stores. diff --git a/design/08-development.md b/design/08-development.md index f71dc56c..7421d2d6 100644 --- a/design/08-development.md +++ b/design/08-development.md @@ -45,3 +45,5 @@ Pure-code safety paths have enforced coverage, while CLI, notebook, REST, transp ## Supply Chain Application dependencies are hash-locked, image base versions and downloaded runtime archives are pinned, and optional model artifacts require immutable revisions, byte sizes, SHA-256 digests, and license metadata. Published images contain no credentials or model weights. BuildKit attestations are implemented for the generic image. Failed staging digests remain untagged and cannot move a public channel; graph-aware GHCR retention, cryptographic image signing, real Skill signature verification, generated notices, formal release channels, and signature-policy enforcement remain future release controls. + +Native installation assets are built from the tagged source tree by GitHub Actions. Each release publishes a fixed-name source bundle, a standalone installer, and a SHA-256 manifest; provenance covers the downloadable artifacts. Pull requests build the same bundle and exercise verification and dry-run installation without publishing. The installer rejects a missing or mismatched manifest and supports an explicitly supplied local bundle for restricted-network transfer workflows. diff --git a/design/09-implementation-plan.md b/design/09-implementation-plan.md index cd7e389d..dd00b730 100644 --- a/design/09-implementation-plan.md +++ b/design/09-implementation-plan.md @@ -35,6 +35,8 @@ Unchecked items are planned work and are not current capability or support claim - The CLI, web UI, and notebook bridge use one Heartwood command and event contract for tasks, messages, actions, allow or reject decisions, pause and resume, replay, settings, and audit export. The gateway also owns persisted session creation, listing, title metadata, status derivation, and selection used by the web session rail. - The CLI opens a full-screen Textual conversation in a capable terminal, replays the gateway-owned session, executes blocking gateway turns in a background worker, renders action decisions and lifecycle events, and retains a line-oriented fallback plus deterministic one-shot commands. A framework-neutral interaction controller keeps terminal presentation separate from session commands; it does not create another OpenHands conversation or state store. - Bare `heartwood` evaluates typed deployment readiness and opens setup, recovery guidance, or the conversation. `heartwood doctor` is read-only and requires the detected platform, setup source, selected model, connection, policy, credential reference, and action mode to agree; `heartwood setup` restores the prior configuration if route validation fails and persists the validated result through the same gateway stores used by every interface. Explicit non-interactive setup supports reproducible deployment automation. +- `heartwood launch` detects Carina, Terra, or generic execution, renders a complete compute and runtime plan, supports dry-run and no-allocation modes, requires separate consent before a Carina Slurm request, and reuses one packaged model-verification, vLLM-supervision, setup, conversation, and cleanup path after compute is available. +- Deterministic platform selection provides conservative Carina, Terra, and generic adapters. The Terra adapter reports the already-provisioned interactive environment and local-runtime policy without adding a Terra API client or claiming controlled-data integration. - The web UI is conversation-first and implements persisted session navigation, title editing, typed platform and dataset context, chronological model and tool activity, inline action decisions, a stable composer, responsive session and utility sheets, repository-verification labels for Skills, readable audit activity, local, platform, cloud, and custom model connections, advanced model profiles, and byte-level local-model download progress. Boundary evidence and workflow progress are omitted until typed gateway events provide them rather than being inferred or presented as placeholders. - **Ask Every Time** maps to OpenHands `AlwaysConfirm`. **Auto-Approve Low Risk** maps to OpenHands `ConfirmRisky` with a `MEDIUM` threshold and unknown actions confirmed. Deployment policy controls which modes may be selected. - The deterministic backend is limited to unit tests, replay, and no-model integration checks. @@ -53,6 +55,7 @@ Unchecked items are planned work and are not current capability or support claim - Generic images build natively for `linux/amd64` and `linux/arm64` and publish one multi-platform manifest. Terra publishes a separate AMD64 Docker schema-2 manifest compatible with Leonardo image detection. - Explicit generic and Terra NVIDIA targets install the artifact-hash-locked vLLM runtime in an isolated environment while retaining the same Heartwood payload and no-weight policy. Pull requests build both targets; main creates digest-checked immutable tags and promotes both moving channels only after both candidates pass. The portable images remain the defaults. A native Carina deployment uses the same lock through separate Micromamba-managed Heartwood and vLLM environments, requires exact manifest coverage, stages into fresh job scratch, and binds inference to loopback. +- GitHub Actions builds a fixed-name native source bundle, standalone installer, and checksum manifest. Pull requests verify and install the bundle with a hermetic dependency substitute; tagged releases attest and publish the same assets. Installation uses versioned source and runtime roots, supports approved local-bundle transfer, and leaves models, credentials, and compute acquisition outside installation. - CI verifies no-weight image contents, OpenHands loopback orchestration, the synthetic reference cohort, both confirmation modes, native Skill loading, fresh named-volume ownership and cross-container recovery, separately mounted llama.cpp inference in the generic and Terra images, a three-Skill browser session replayed through the CLI, reproducible synthetic desktop and notebook-viewport screenshots, Jupyter startup, proxy routing, Terra image contracts, and registry media types. An opt-in workflow-dispatch job requires the pinned 7B agent artifact to execute the cohort Skill and produce the expected aggregate artifact through a network-disabled OpenHands terminal action without making it a pull-request dependency. - Main publication stages untagged images by digest, validates the exact candidates, creates and verifies immutable commit tags, and moves the generic and Terra channel tags only after every required candidate check passes. - Python, TypeScript, documentation, licensing, secret scanning, dependency review, CodeQL, container checks, and synthetic replay are repository gates. @@ -72,6 +75,7 @@ Unchecked items are planned work and are not current capability or support claim 11. **First-run and persistence setup is not unified.** The generic runtime uses a state volume and a model-cache volume, path overrides remain distributed, and no shared setup flow verifies durable storage or migrates prior state. [Issue #22](https://github.com/SchmiedmayerLab/heartwood/issues/22) owns this work. 12. **Agent events are returned as completed batches.** The full-screen CLI remains responsive while a turn runs, but the gateway cannot yet project durable token or tool lifecycle updates incrementally or interrupt an active OpenHands turn. [Issue #24](https://github.com/SchmiedmayerLab/heartwood/issues/24) owns the typed OpenHands event and lifecycle contract. 13. **Carina and NVIDIA execution are not live-validated.** Platform detection, setup, policy, native environment locks, Slurm launchers, GPU image targets, publication contracts, and the Stanford AI API Gateway connection are implemented and CI-testable, but a real Carina L40S allocation, a native container GPU run, a live Stanford gateway request, and the complete synthetic acceptance have not been recorded. +14. **Native release and launch are not live-validated.** Bundle construction, checksum enforcement, versioned installation, command publication, platform detection, scheduler consent, local-runtime supervision, and cleanup are automated, but no tagged native release has been installed and exercised in a real Carina allocation or approved Terra environment. ## Priority 1 — Release-Candidate Runtime Contract @@ -80,7 +84,6 @@ Unchecked items are planned work and are not current capability or support claim ### Deliverables - [ ] Replace the implicit synthetic data-source default with an explicit unconfigured data source. Enable the synthetic OMOP adapter only through a named fixture or demonstration configuration. -- [ ] Add a minimal Terra platform adapter selected from detector evidence. It must expose platform identity, persistent paths, proxy assumptions, and a conservative default policy without implementing a parallel Terra client. - [ ] Make the gateway the sole writer for an active session. Route CLI operations through the running gateway or enforce an interprocess session lock, and add concurrent-command, duplicate-writer, interrupted-append, and recovery tests. - [ ] Implement the canonical versioned state root, one-volume default, optional split model cache, migration, restart checks, and shared first-run setup defined in [Issue #22](https://github.com/SchmiedmayerLab/heartwood/issues/22). - [ ] Implement `heartwood doctor` as a read-only environment, storage, accelerator, model-route, credential-reference, and policy diagnostic. Implement `heartwood setup` as the shared resumable first-run flow over the same gateway settings used by the CLI, web UI, and notebook bridge. Platform adapters may contribute detected defaults and validation steps, but setup must show its evidence, require confirmation before mutation or download, and support explicit non-interactive configuration for automation. Make bare `heartwood` state-aware in an interactive terminal: open setup when required configuration is absent, present concise recovery choices when diagnostics fail, and open the conversation when setup is valid; retain help-only behavior for non-interactive invocation and never mutate state merely because no subcommand was supplied. diff --git a/docs/carina-cli.md b/docs/carina-cli.md index f48cb7f9..05810873 100644 --- a/docs/carina-cli.md +++ b/docs/carina-cli.md @@ -14,7 +14,7 @@ This guide defines the synthetic-only Heartwood pilot on Stanford Carina. The re ## Safety Boundary -Use a new project directory containing only the public Heartwood repository, a reviewed public model, and synthetic fixtures. Do not point Heartwood at an existing research directory, enumerate unrelated project storage, print the full environment, or copy controlled data into job-local scratch. The first pilot uses **Ask Every Time** and one CLI process; terminal and file tools still run with the researcher's Unix permissions. +Use a new project directory containing only the verified Heartwood installation, a reviewed public model, and synthetic fixtures. Do not point Heartwood at an existing research directory, enumerate unrelated project storage, print the full environment, or copy controlled data into job-local scratch. The first pilot uses **Ask Every Time** and one CLI process; terminal and file tools still run with the researcher's Unix permissions. ## Prepare Persistent Storage @@ -22,49 +22,58 @@ On a login node, select protected project paths owned by the pilot: ```bash export HEARTWOOD_ROOT=/projects///heartwood-pilot -mkdir -p "${HEARTWOOD_ROOT}/environments" "${HEARTWOOD_ROOT}/models" "${HEARTWOOD_ROOT}/state" +mkdir -p "${HEARTWOOD_ROOT}/models" ``` -Clone the public repository over HTTPS. Carina does not support GitHub SSH access. Do not leave a GitHub token in the job environment. +Do not leave a GitHub token in the job environment. -## Install The Native Environments +## Install A Tagged Release -Load Carina's supported Micromamba module, enter the repository, and run: +Load Carina's supported Micromamba module. Download the standalone installer from the selected GitHub Release, review its checksum or attestation, and run: ```bash -deploy/carina/bootstrap.sh --environment-root "${HEARTWOOD_ROOT}/environments" +chmod +x heartwood-installer +./heartwood-installer --root "${HEARTWOOD_ROOT}" --platform carina --version +export PATH="${HEARTWOOD_ROOT}/bin:${PATH}" ``` -The bootstrap creates separate locked Heartwood and vLLM environments. This prevents the NVIDIA inference dependency set from changing the OpenHands application environment. +The installer retrieves and verifies the matching native bundle, keeps the immutable source payload and runtime under versioned directories, creates separate locked Heartwood and vLLM environments, and publishes `${HEARTWOOD_ROOT}/bin/heartwood`. For restricted-network transfer, place `heartwood-native.tar.gz`, `SHA256SUMS`, and `heartwood-installer` through the approved path and use `--bundle` with `--checksums`. The installer never downloads model weights or stores credentials. ## Stage A Reviewed Model Place the approved Hugging Face snapshot under `${HEARTWOOD_ROOT}/models/` using an approved transfer path. Add a `SHA256SUMS` file containing every regular model file. The launcher rejects missing, unlisted, linked, or modified files and stages the verified snapshot into a fresh job-scratch directory. Model acquisition is deliberately separate from agent startup; the launcher never downloads weights. -## Start An Interactive Allocation +## Review And Launch + +Preview the complete launch plan without requesting compute: ```bash -srun --pty --partition=gpu --gres=gpu:1 --cpus-per-task=8 --mem=64G --time=02:00:00 bash +export HEARTWOOD_PLATFORM=carina +heartwood --workspace "${HEARTWOOD_ROOT}/state/sessions" doctor +heartwood launch \ + --model-root "${HEARTWOOD_ROOT}/models/" \ + --dry-run ``` -Inside the allocation, confirm the detected state without changing it: +Launch the session: ```bash -export HEARTWOOD_PLATFORM=carina -export PATH="${HEARTWOOD_ROOT}/environments/heartwood/bin:${PATH}" -heartwood --workspace "${HEARTWOOD_ROOT}/state/sessions" doctor +heartwood launch --model-root "${HEARTWOOD_ROOT}/models/" ``` -Start verified local inference and the CLI: +On a login node, Heartwood displays the `gpu` partition, GPU, CPU, memory, and time request and asks before invoking `srun`. Scheduler consent is independent of agent action confirmation. Inside the allocation, the same command verifies the model manifest, stages it into `$LOCAL_SCRATCH_JOB`, removes unrelated provider and source-control credentials, starts vLLM on `127.0.0.1:8765`, waits for `/v1/models`, configures the Local connection, and opens `heartwood chat`. vLLM stops when the CLI exits. Use `--no-allocate` to prohibit scheduler submission or `--yes-request-allocation` only in reviewed automation. + +The compatibility scripts under `deploy/carina` remain available from the installed source payload for troubleshooting; they delegate to the packaged launch contract and are not the normal researcher workflow. + +## Action Confirmation + +Setup defaults to **Ask Every Time**. After validating the synthetic workflow, a researcher may explicitly select: ```bash -deploy/carina/launch-interactive.sh \ - --environment-root "${HEARTWOOD_ROOT}/environments" \ - --model-root "${HEARTWOOD_ROOT}/models/" \ - --state-root "${HEARTWOOD_ROOT}/state" +heartwood actions set auto-approve-low-risk ``` -The launcher requires an active allocation and job-local scratch, verifies the model manifest, stages it into `$LOCAL_SCRATCH_JOB`, removes unrelated provider and source-control credentials, starts vLLM on `127.0.0.1:8765`, waits for `/v1/models`, configures the Local connection, and opens `heartwood chat`. vLLM stops when the CLI exits. +This uses OpenHands risk analysis: low-risk actions continue automatically, while medium-, high-, and unknown-risk actions still require **Allow once** or **Reject**. Heartwood does not expose unconditional auto-approval. This software option does not establish approval for controlled data. ## Stanford AI API Gateway diff --git a/docs/container-images.md b/docs/container-images.md index b731c60c..bc1ee4ac 100644 --- a/docs/container-images.md +++ b/docs/container-images.md @@ -79,6 +79,8 @@ The service starts without a secret or model. Configure a profile from the web s The state volume contains sessions, non-secret model and action settings, installed Skills, OpenHands state, workspaces, and audit data. The separate model volume allows large weights to use a different quota and retention policy and also owns Hugging Face transfer metadata through `HF_HOME`. Override `HEARTWOOD_MODEL_CACHE` and `HF_HOME` together when mounting a different model path. [Issue #22](https://github.com/SchmiedmayerLab/heartwood/issues/22) tracks a canonical versioned root and one-volume default while preserving the split cache as an advanced option; the current two-volume layout remains the supported contract until that migration is implemented and restart-tested. +Native environments use the same application and dependency locks through GitHub Release assets rather than a platform image. The release publishes `heartwood-installer`, `heartwood-native.tar.gz`, and `SHA256SUMS`; pull requests build and dry-run the same assets without publishing. Native installation contains no model weights or credentials and does not request compute. The installed `heartwood launch` command owns platform-aware compute planning and runtime startup. + Run an explicitly mounted local model in the same container: ```bash diff --git a/docs/platform-support.md b/docs/platform-support.md index 355c8f1d..4de5eb8e 100644 --- a/docs/platform-support.md +++ b/docs/platform-support.md @@ -18,7 +18,7 @@ This matrix records current repository implementation and validation status. Hea |---|---|---|---|---| | Generic Linux or Jupyter environment | Implemented | `edge` and `sha-` for `linux/amd64` and `linux/arm64` | Native architecture builds; exact-digest no-weight, real OpenHands loopback, action-confirmation, mounted llama.cpp, and fresh named-volume checks before tag promotion; live-browser cohort, baseline, aggregate-export, audit, and CLI replay; responsive Chromium and notebook-proxy contracts | Self-hosted deployments must validate their own identity, storage, network, and data controls. | | Terra Jupyter | Implemented platform-derived image | `edge-terra` and `sha--terra` for `linux/amd64` as Docker schema 2 | Real pinned Terra base on main; exact-digest Jupyter environment, Heartwood kernel, entrypoint, `/notebooks/...` route, Leonardo-compatible manifest, OpenHands reference cohort, mounted llama.cpp, CLI, web, notebook proxy, and audit checks before tag promotion | Real Terra workspace validation remains required before a supported or institution-approved deployment claim. | -| Stanford Carina CLI | Implemented platform and launcher contracts | Native Micromamba environments; no Carina-specific image is published; `edge-gpu-nvidia` is only the equivalent generic container target | Deterministic platform detection, conservative local-only policy, coherent setup and doctor state, hash-locked isolated vLLM environment, exact model-snapshot verification, Slurm launcher contracts, Stanford gateway manifest, and synthetic contract tests | A real GPU allocation, reviewed model, and content-free synthetic acceptance remain required before live-validated status. No controlled-data or institutional-approval claim. | +| Stanford Carina CLI | Implemented native installation and launch contracts | Verified native release bundle; no Carina-specific image is published; `edge-gpu-nvidia` is only the equivalent generic container target | Deterministic platform detection, release-bundle verification, locked isolated environments, exact model-snapshot verification, explicit Slurm consent, shared runtime supervision, Stanford gateway manifest, and both OpenHands confirmation modes | A real GPU allocation, reviewed model, and content-free synthetic acceptance remain required before live-validated status. No controlled-data or institutional-approval claim. | | All of Us or AnVIL through Terra | Design target only | No separately validated image | No platform-specific live evidence | Not currently supported as a distinct deployment. Dataset policy, identity, image base, and live control-plane behavior require separate validation. | | Seven Bridges or Velsera | Design target only | None | None | Not currently supported. | | DNAnexus or UK Biobank Research Analysis Platform | Design target only | None | None | Not currently supported. | diff --git a/packages/adapters/src/heartwood/adapters/platform/__init__.py b/packages/adapters/src/heartwood/adapters/platform/__init__.py index 45c6eb3b..0791069a 100644 --- a/packages/adapters/src/heartwood/adapters/platform/__init__.py +++ b/packages/adapters/src/heartwood/adapters/platform/__init__.py @@ -13,13 +13,22 @@ from heartwood.adapters import PlatformAdapter from heartwood.adapters.platform.carina import CarinaPlatformAdapter from heartwood.adapters.platform.generic import GenericPlatformAdapter +from heartwood.adapters.platform.terra import TerraPlatformAdapter from heartwood.detector import Platform, detect_platform -__all__ = ["CarinaPlatformAdapter", "GenericPlatformAdapter", "select_platform_adapter"] +__all__ = [ + "CarinaPlatformAdapter", + "GenericPlatformAdapter", + "TerraPlatformAdapter", + "select_platform_adapter", +] def select_platform_adapter(env: Mapping[str, str]) -> PlatformAdapter: """Select the implemented adapter from deterministic environment evidence.""" - if detect_platform(env).platform is Platform.CARINA: + platform = detect_platform(env).platform + if platform is Platform.CARINA: return CarinaPlatformAdapter() + if platform is Platform.TERRA: + return TerraPlatformAdapter() return GenericPlatformAdapter() diff --git a/packages/adapters/src/heartwood/adapters/platform/carina.py b/packages/adapters/src/heartwood/adapters/platform/carina.py index a900a82c..57579759 100644 --- a/packages/adapters/src/heartwood/adapters/platform/carina.py +++ b/packages/adapters/src/heartwood/adapters/platform/carina.py @@ -56,7 +56,10 @@ def default_policy_profile(self) -> PolicyProfile: allowed_model_endpoints=("http://127.0.0.1:8765/v1/chat/completions",), allowed_model_catalog_endpoints=("http://127.0.0.1:8765/v1/models",), allowed_capability_tiers=("supervised", "experimental"), - allowed_action_confirmation_modes=("always-confirm",), + allowed_action_confirmation_modes=("always-confirm", "confirm-risky"), credential_allowlist=(), - notes="Synthetic-only Carina policy with loopback inference and explicit actions.", + notes=( + "Synthetic-only Carina policy with loopback inference; Ask Every Time is the " + "default and researchers may explicitly select low-risk auto-approval." + ), ) diff --git a/packages/adapters/src/heartwood/adapters/platform/terra.py b/packages/adapters/src/heartwood/adapters/platform/terra.py new file mode 100644 index 00000000..c434fbba --- /dev/null +++ b/packages/adapters/src/heartwood/adapters/platform/terra.py @@ -0,0 +1,58 @@ +# This source file is part of the Heartwood open-source project +# +# SPDX-FileCopyrightText: 2026 Stanford University and the project authors (see CONTRIBUTORS.md) +# +# SPDX-License-Identifier: MIT + +"""Minimal Terra interactive-runtime platform adapter.""" + +from __future__ import annotations + +from collections.abc import Mapping +from pathlib import Path + +from heartwood.adapters import AdapterDetection +from heartwood.detector import Platform, detect_platform +from heartwood.schemas import PolicyProfile + + +class TerraPlatformAdapter: + """Platform contract for an already-provisioned Terra Jupyter runtime.""" + + @property + def adapter_id(self) -> str: + """Return the stable platform adapter id.""" + return "terra" + + def detect(self, env: Mapping[str, str]) -> AdapterDetection: + """Detect Terra from deterministic workspace markers.""" + detection = detect_platform(env) + if detection.platform is Platform.TERRA: + return AdapterDetection(self.adapter_id, detection.confidence, detection.evidence) + return AdapterDetection( + self.adapter_id, + 0.0, + ("Terra platform evidence not found", *detection.evidence), + ) + + def data_mounts(self) -> tuple[Path, ...]: + """Return no implicit controlled-data mount.""" + return () + + def credential_allowlist(self) -> tuple[str, ...]: + """Return no implicit provider credential.""" + return () + + def default_policy_profile(self) -> PolicyProfile: + """Return the conservative local-runtime Terra policy.""" + return PolicyProfile( + policy_id="terra-local-default", + platform_id=self.adapter_id, + deny_egress_by_default=True, + allowed_model_endpoints=("http://127.0.0.1:8765/v1/chat/completions",), + allowed_model_catalog_endpoints=("http://127.0.0.1:8765/v1/models",), + allowed_capability_tiers=("supervised", "experimental"), + allowed_action_confirmation_modes=("always-confirm", "confirm-risky"), + credential_allowlist=(), + notes="Terra local-runtime policy; deployment controls remain authoritative.", + ) diff --git a/packages/adapters/tests/test_generic_adapters.py b/packages/adapters/tests/test_generic_adapters.py index c9fad970..2b7e240e 100644 --- a/packages/adapters/tests/test_generic_adapters.py +++ b/packages/adapters/tests/test_generic_adapters.py @@ -18,7 +18,12 @@ assert_registry_adapter_conforms, ) from heartwood.adapters.data import DataSourceBoundaryError, LocalFilesystemDataSourceAdapter -from heartwood.adapters.platform import CarinaPlatformAdapter, GenericPlatformAdapter +from heartwood.adapters.platform import ( + CarinaPlatformAdapter, + GenericPlatformAdapter, + TerraPlatformAdapter, + select_platform_adapter, +) from heartwood.adapters.registry import LocalRegistryAdapter, RegistryBoundaryError @@ -35,10 +40,21 @@ def test_carina_platform_adapter_conforms_and_defaults_to_local_only() -> None: assert detection.confidence > 0.0 assert adapter.data_mounts() == () assert policy.platform_id == "carina" - assert policy.allowed_action_confirmation_modes == ("always-confirm",) + assert policy.allowed_action_confirmation_modes == ("always-confirm", "confirm-risky") assert policy.credential_allowlist == () +def test_terra_platform_adapter_conforms_and_uses_provisioned_compute() -> None: + adapter = TerraPlatformAdapter() + assert_platform_adapter_conforms(adapter) + detection = adapter.detect({"GOOGLE_PROJECT": "synthetic-project"}) + policy = adapter.default_policy_profile() + assert detection.adapter_id == "terra" + assert detection.confidence > 0.0 + assert policy.platform_id == "terra" + assert select_platform_adapter({"GOOGLE_PROJECT": "synthetic-project"}).adapter_id == "terra" + + def test_local_filesystem_data_adapter_conforms() -> None: assert_data_source_adapter_conforms(LocalFilesystemDataSourceAdapter.synthetic_omop()) diff --git a/packages/cli/src/heartwood/cli/__init__.py b/packages/cli/src/heartwood/cli/__init__.py index e0e5fc24..8552e049 100644 --- a/packages/cli/src/heartwood/cli/__init__.py +++ b/packages/cli/src/heartwood/cli/__init__.py @@ -22,6 +22,7 @@ import uvicorn from heartwood.cli._interactive import InteractiveSession, command_help +from heartwood.cli._launch import LaunchOptions, run_launch from heartwood.compliance import ReviewerPacketGenerator from heartwood.gateway import ( ActionSettingsError, @@ -49,7 +50,7 @@ __all__ = ["__version__", "main"] -__version__ = "0.0.0" +__version__ = os.environ.get("HEARTWOOD_VERSION", "0.0.0") _PROG = "heartwood" _DEFAULT_WORKSPACE = Path(".heartwood") / "sessions" @@ -126,6 +127,29 @@ def _build_parser() -> argparse.ArgumentParser: ) setup.add_argument("--yes", action="store_true", help="Confirm the displayed configuration.") + launch = subparsers.add_parser( + "launch", help="Prepare platform compute and open an interactive Heartwood session." + ) + launch.add_argument("--model-root", type=Path, help="Verified local-model snapshot directory.") + launch.add_argument("--state-root", type=Path, help="Persistent Heartwood state root.") + launch.add_argument("--environment-root", type=Path, help="Native runtime environment root.") + launch.add_argument("--vllm-executable", type=Path, help="Explicit vLLM executable.") + launch.add_argument("--model-id", default="heartwood-local-model") + launch.add_argument("--partition", default="gpu") + launch.add_argument("--gpus", type=int, default=1) + launch.add_argument("--cpus", type=int, default=8) + launch.add_argument("--memory", default="64G") + launch.add_argument("--time", dest="time_limit", default="02:00:00") + launch.add_argument("--dry-run", action="store_true") + launch.add_argument("--no-allocate", action="store_true") + launch.add_argument( + "--yes-request-allocation", + action="store_true", + help="Confirm the displayed scheduler request without an interactive prompt.", + ) + launch.add_argument("--inside-allocation", action="store_true", help=argparse.SUPPRESS) + launch.add_argument("--plain", action="store_true", help="Open the line-oriented chat.") + allow = subparsers.add_parser( "allow", aliases=["approve"], @@ -281,6 +305,33 @@ def main(argv: Sequence[str] | None = None) -> int: return _handle_doctor(workspace=args.workspace, as_json=args.json) if args.command == "setup": return _handle_setup(parser, args) + if args.command == "launch": + if args.gpus < 1 or args.cpus < 1: + parser.error("--gpus and --cpus must be positive") + state_root = args.state_root or args.workspace.parent + if args.workspace.parent != state_root: + parser.error("--state-root must be the parent of --workspace") + return run_launch( + LaunchOptions( + workspace=args.workspace, + session_id=args.session_id, + model_root=args.model_root, + state_root=state_root, + environment_root=args.environment_root, + vllm_executable=args.vllm_executable, + model_id=args.model_id, + partition=args.partition, + gpus=args.gpus, + cpus=args.cpus, + memory=args.memory, + time_limit=args.time_limit, + dry_run=args.dry_run, + no_allocate=args.no_allocate, + yes_request_allocation=args.yes_request_allocation, + inside_allocation=args.inside_allocation, + plain=args.plain, + ) + ) if args.command is None and sys.stdin.isatty(): readiness = inspect_deployment(args.workspace) if readiness.state == "setup-required": diff --git a/packages/cli/src/heartwood/cli/_launch.py b/packages/cli/src/heartwood/cli/_launch.py new file mode 100644 index 00000000..8df4dbb8 --- /dev/null +++ b/packages/cli/src/heartwood/cli/_launch.py @@ -0,0 +1,337 @@ +# This source file is part of the Heartwood open-source project +# +# SPDX-FileCopyrightText: 2026 Stanford University and the project authors (see CONTRIBUTORS.md) +# +# SPDX-License-Identifier: MIT + +"""Environment-aware native runtime launch orchestration.""" + +from __future__ import annotations + +import json +import os +import shlex +import shutil +import subprocess +import sys +import tempfile +import time +import urllib.request +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path + +from heartwood.adapters.platform import select_platform_adapter +from heartwood.cli._model_snapshot import verify_snapshot + +InputFunction = Callable[[str], str] +RunFunction = Callable[[Sequence[str]], int] + + +@dataclass(frozen=True, slots=True) +class LaunchOptions: + """Validated launch inputs shared by platform implementations.""" + + workspace: Path + session_id: str + model_root: Path | None + state_root: Path + environment_root: Path | None + vllm_executable: Path | None + model_id: str + partition: str + gpus: int + cpus: int + memory: str + time_limit: str + dry_run: bool + no_allocate: bool + yes_request_allocation: bool + inside_allocation: bool + plain: bool + + +@dataclass(frozen=True, slots=True) +class LaunchPlan: + """Human-reviewable compute and runtime launch proposal.""" + + platform_id: str + allocation_required: bool + allocation_command: tuple[str, ...] + model_root: Path | None + state_root: Path + + def format(self) -> str: + """Render the launch proposal without secrets.""" + compute = "Slurm allocation required" if self.allocation_required else "already provisioned" + lines = [ + "Heartwood launch plan", + "", + f"Platform: {self.platform_id}", + f"Compute: {compute}", + f"Model: {self.model_root if self.model_root is not None else 'not selected'}", + f"State: {self.state_root}", + ] + if self.allocation_command: + lines.append(f"Request: {shlex.join(self.allocation_command)}") + return "\n".join(lines) + + +def build_launch_plan(options: LaunchOptions, env: Mapping[str, str]) -> LaunchPlan: + """Build a platform-specific launch plan without changing external state.""" + platform_id = select_platform_adapter(env).adapter_id + allocation_required = platform_id == "carina" and not env.get("SLURM_JOB_ID") + command: tuple[str, ...] = () + if allocation_required: + command = ( + "srun", + "--pty", + f"--partition={options.partition}", + f"--gres=gpu:{options.gpus}", + f"--cpus-per-task={options.cpus}", + f"--mem={options.memory}", + f"--time={options.time_limit}", + *_reentry_command(options), + ) + return LaunchPlan( + platform_id, allocation_required, command, options.model_root, options.state_root + ) + + +def run_launch( + options: LaunchOptions, + *, + env: Mapping[str, str] | None = None, + input_fn: InputFunction = input, + run_fn: RunFunction | None = None, +) -> int: + """Execute a reviewed allocation request or the in-allocation runtime.""" + active_env = dict(os.environ if env is None else env) + plan = build_launch_plan(options, active_env) + print(plan.format()) + if options.dry_run: + return 0 + if plan.allocation_required: + if options.no_allocate: + print("\nA GPU allocation is required; rerun without --no-allocate.") + return 1 + if not options.yes_request_allocation: + try: + approved = input_fn("\nRequest this GPU allocation? [y/N]: ").strip().lower() == "y" + except EOFError: + approved = False + if not approved: + print("Allocation cancelled.") + return 1 + runner = run_fn or _run_command + return runner(plan.allocation_command) + return _run_runtime(options, active_env) + + +def _reentry_command(options: LaunchOptions) -> tuple[str, ...]: + command = [ + sys.executable, + "-m", + "heartwood.cli", + "--workspace", + str(options.workspace), + "--session-id", + options.session_id, + "launch", + "--inside-allocation", + "--state-root", + str(options.state_root), + "--model-id", + options.model_id, + ] + if options.model_root is not None: + command.extend(("--model-root", str(options.model_root))) + if options.environment_root is not None: + command.extend(("--environment-root", str(options.environment_root))) + if options.vllm_executable is not None: + command.extend(("--vllm-executable", str(options.vllm_executable))) + if options.plain: + command.append("--plain") + return tuple(command) + + +def _run_command(command: Sequence[str]) -> int: + return subprocess.run(command, check=False).returncode + + +def _run_runtime(options: LaunchOptions, env: Mapping[str, str]) -> int: + if options.model_root is None: + print("A verified model snapshot is required; pass --model-root.") + return 64 + try: + verify_snapshot(options.model_root) + except (OSError, UnicodeError, ValueError) as error: + print(f"Model verification failed: {error}") + return 66 + vllm = _resolve_vllm(options, env) + if not vllm.is_file() or not os.access(vllm, os.X_OK): + print(f"vLLM executable is unavailable: {vllm}") + return 69 + + scratch_parent = Path(env.get("LOCAL_SCRATCH_JOB", str(options.state_root / "scratch"))) + scratch_parent.mkdir(parents=True, exist_ok=True) + staged_model = Path(tempfile.mkdtemp(prefix="heartwood-model.", dir=scratch_parent)) + runtime: subprocess.Popen[str] | None = None + try: + shutil.copytree(options.model_root, staged_model, dirs_exist_ok=True, symlinks=False) + try: + verify_snapshot(staged_model) + except (OSError, UnicodeError, ValueError) as error: + print(f"Staged model verification failed: {error}") + return 66 + runtime_log = options.state_root / "runtime" + runtime_log.mkdir(parents=True, exist_ok=True) + log_file = (runtime_log / "vllm.log").open("a", encoding="utf-8") + try: + runtime = subprocess.Popen( + _vllm_command(vllm, staged_model, options.model_id), + stdout=log_file, + stderr=subprocess.STDOUT, + text=True, + env=_runtime_environment(env), + ) + finally: + log_file.close() + if not _wait_for_runtime(runtime): + print(f"vLLM did not become ready; inspect {runtime_log / 'vllm.log'}") + return 70 + setup_code = _ensure_setup(options, env) + if setup_code != 0: + return setup_code + chat = [ + sys.executable, + "-m", + "heartwood.cli", + "--workspace", + str(options.workspace), + "--session-id", + options.session_id, + "chat", + ] + if options.plain: + chat.append("--plain") + return subprocess.run(chat, check=False, env=_runtime_environment(env)).returncode + finally: + if runtime is not None and runtime.poll() is None: + runtime.terminate() + try: + runtime.wait(timeout=10) + except subprocess.TimeoutExpired: + runtime.kill() + runtime.wait() + shutil.rmtree(staged_model, ignore_errors=True) + + +def _resolve_vllm(options: LaunchOptions, env: Mapping[str, str]) -> Path: + if options.vllm_executable is not None: + return options.vllm_executable + configured = env.get("HEARTWOOD_VLLM_EXECUTABLE") + if configured: + return Path(configured) + if options.environment_root is not None: + return options.environment_root / "vllm" / "bin" / "vllm" + native_root = env.get("HEARTWOOD_NATIVE_ROOT") + native_version = env.get("HEARTWOOD_NATIVE_VERSION") + if native_root and native_version: + return Path(native_root) / "runtimes" / native_version / "vllm" / "bin" / "vllm" + return Path("/opt/heartwood-vllm/bin/vllm") + + +def _vllm_command(executable: Path, model: Path, model_id: str) -> tuple[str, ...]: + return ( + str(executable), + "serve", + str(model), + "--host", + "127.0.0.1", + "--port", + "8765", + "--served-model-name", + model_id, + "--max-model-len", + "8192", + "--enable-auto-tool-choice", + "--tool-call-parser", + "hermes", + ) + + +def _runtime_environment(env: Mapping[str, str]) -> dict[str, str]: + allowed_names = ( + "PATH", + "HOME", + "USER", + "LOGNAME", + "SHELL", + "TMPDIR", + "TMP", + "TEMP", + "LANG", + "LC_ALL", + "LC_CTYPE", + "TERM", + "VIRTUAL_ENV", + "PYTHONPATH", + "LD_LIBRARY_PATH", + "DYLD_LIBRARY_PATH", + "CUDA_VISIBLE_DEVICES", + "NVIDIA_VISIBLE_DEVICES", + "NVIDIA_DRIVER_CAPABILITIES", + "XDG_CACHE_HOME", + "HF_HOME", + "TRANSFORMERS_CACHE", + "TORCH_HOME", + ) + result = {name: env[name] for name in allowed_names if name in env} + result.update( + { + "HEARTWOOD_AGENT_BACKEND": "openhands-sdk", + "HEARTWOOD_LOCAL_RUNTIME_HOST": "127.0.0.1", + "HEARTWOOD_LOCAL_RUNTIME_PORT": "8765", + } + ) + return result + + +def _wait_for_runtime(runtime: subprocess.Popen[str], timeout: float = 300) -> bool: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if runtime.poll() is not None: + return False + try: + with urllib.request.urlopen("http://127.0.0.1:8765/v1/models", timeout=2) as response: + payload = json.load(response) + if response.status == 200 and payload.get("data"): + return True + except (OSError, ValueError): + time.sleep(1) + return False + + +def _ensure_setup(options: LaunchOptions, env: Mapping[str, str] | None = None) -> int: + if (options.state_root / "setup.json").is_file(): + return 0 + command = ( + sys.executable, + "-m", + "heartwood.cli", + "--workspace", + str(options.workspace), + "setup", + "--model-source", + "local", + "--model-id", + options.model_id, + "--non-interactive", + "--yes", + ) + return subprocess.run( + command, + check=False, + env=_runtime_environment(os.environ if env is None else env), + ).returncode diff --git a/packages/cli/src/heartwood/cli/_model_snapshot.py b/packages/cli/src/heartwood/cli/_model_snapshot.py new file mode 100644 index 00000000..57e979f8 --- /dev/null +++ b/packages/cli/src/heartwood/cli/_model_snapshot.py @@ -0,0 +1,69 @@ +# This source file is part of the Heartwood open-source project +# +# SPDX-FileCopyrightText: 2026 Stanford University and the project authors (see CONTRIBUTORS.md) +# +# SPDX-License-Identifier: MIT + +"""Exact manifest verification for staged local-model snapshots.""" + +from __future__ import annotations + +import hashlib +import os +import re +from pathlib import Path, PurePosixPath + +_ENTRY = re.compile(r"^([0-9a-fA-F]{64}) [ *](.+)$") + + +def verify_snapshot(root: Path) -> None: + """Reject unlisted, missing, linked, duplicated, or modified snapshot files.""" + manifest = root / "SHA256SUMS" + if not root.is_dir() or not manifest.is_file() or manifest.is_symlink(): + raise ValueError("model root must contain a regular SHA256SUMS manifest") + expected: dict[str, str] = {} + for line_number, line in enumerate(manifest.read_text(encoding="utf-8").splitlines(), 1): + match = _ENTRY.fullmatch(line) + if match is None: + raise ValueError(f"invalid SHA256SUMS entry on line {line_number}") + digest, name = match.groups() + manifest_relative = PurePosixPath(name) + if ( + manifest_relative.is_absolute() + or ".." in manifest_relative.parts + or name in {"", "SHA256SUMS"} + ): + raise ValueError(f"unsafe SHA256SUMS path on line {line_number}") + normalized = manifest_relative.as_posix() + if normalized in expected: + raise ValueError(f"duplicate SHA256SUMS path: {normalized}") + expected[normalized] = digest.lower() + + actual: set[str] = set() + for path in root.rglob("*"): + snapshot_relative = path.relative_to(root).as_posix() + if path.is_symlink(): + raise ValueError(f"model snapshot contains a symbolic link: {snapshot_relative}") + if path.is_file() and snapshot_relative != "SHA256SUMS": + actual.add(snapshot_relative) + if actual != set(expected): + missing = sorted(set(expected) - actual) + unlisted = sorted(actual - set(expected)) + detail = "; ".join( + item + for item in ( + f"missing: {', '.join(missing)}" if missing else "", + f"unlisted: {', '.join(unlisted)}" if unlisted else "", + ) + if item + ) + raise ValueError(f"model snapshot does not match SHA256SUMS coverage ({detail})") + + for relative_name, expected_digest in expected.items(): + hasher = hashlib.sha256() + descriptor = os.open(root / relative_name, os.O_RDONLY | os.O_NOFOLLOW) + with os.fdopen(descriptor, "rb") as file: + while chunk := file.read(1024 * 1024): + hasher.update(chunk) + if hasher.hexdigest() != expected_digest: + raise ValueError(f"SHA-256 mismatch: {relative_name}") diff --git a/packages/cli/tests/test_launch.py b/packages/cli/tests/test_launch.py new file mode 100644 index 00000000..7ce3a0b6 --- /dev/null +++ b/packages/cli/tests/test_launch.py @@ -0,0 +1,301 @@ +# This source file is part of the Heartwood open-source project +# +# SPDX-FileCopyrightText: 2026 Stanford University and the project authors (see CONTRIBUTORS.md) +# +# SPDX-License-Identifier: MIT + +from __future__ import annotations + +import hashlib +import subprocess +from collections.abc import Sequence +from pathlib import Path + +import pytest + +from heartwood.cli._launch import ( + LaunchOptions, + _ensure_setup, + _resolve_vllm, + _runtime_environment, + _wait_for_runtime, + build_launch_plan, + run_launch, +) +from heartwood.cli._model_snapshot import verify_snapshot + + +def _options(tmp_path: Path, **overrides: object) -> LaunchOptions: + values: dict[str, object] = { + "workspace": tmp_path / "state" / "sessions", + "session_id": "launch-test", + "model_root": tmp_path / "model", + "state_root": tmp_path / "state", + "environment_root": tmp_path / "environment", + "vllm_executable": tmp_path / "vllm", + "model_id": "test-model", + "partition": "gpu", + "gpus": 1, + "cpus": 8, + "memory": "64G", + "time_limit": "02:00:00", + "dry_run": False, + "no_allocate": False, + "yes_request_allocation": False, + "inside_allocation": False, + "plain": True, + } + values.update(overrides) + return LaunchOptions(**values) # type: ignore[arg-type] + + +def _model(root: Path) -> None: + root.mkdir() + weights = root / "weights.safetensors" + weights.write_bytes(b"synthetic") + digest = hashlib.sha256(weights.read_bytes()).hexdigest() + (root / "SHA256SUMS").write_text(f"{digest} weights.safetensors\n", encoding="utf-8") + + +def test_carina_launch_plan_requests_reviewable_gpu_allocation(tmp_path: Path) -> None: + options = _options(tmp_path) + plan = build_launch_plan(options, {"HEARTWOOD_PLATFORM": "carina"}) + + assert plan.platform_id == "carina" + assert plan.allocation_required + assert plan.allocation_command[:3] == ("srun", "--pty", "--partition=gpu") + assert "--gres=gpu:1" in plan.allocation_command + assert "--inside-allocation" in plan.allocation_command + assert "Slurm allocation required" in plan.format() + + +def test_carina_launch_requires_explicit_allocation_consent( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + called = False + + def runner(_command: object) -> int: + nonlocal called + called = True + return 0 + + code = run_launch( + _options(tmp_path), + env={"HEARTWOOD_PLATFORM": "carina"}, + input_fn=lambda _prompt: "n", + run_fn=runner, + ) + + assert code == 1 + assert not called + assert "Allocation cancelled" in capsys.readouterr().out + + +def test_carina_launch_submits_exact_plan_after_consent(tmp_path: Path) -> None: + observed: list[str] = [] + + def runner(command: object) -> int: + observed.extend(command) # type: ignore[arg-type] + return 23 + + code = run_launch( + _options(tmp_path, yes_request_allocation=True), + env={"HEARTWOOD_PLATFORM": "carina"}, + run_fn=runner, + ) + + assert code == 23 + assert observed[0] == "srun" + assert "--cpus-per-task=8" in observed + + +def test_launch_dry_run_and_no_allocate_never_submit(tmp_path: Path) -> None: + def unexpected(_command: object) -> int: + pytest.fail("scheduler must not be called") + + assert ( + run_launch( + _options(tmp_path, dry_run=True), + env={"HEARTWOOD_PLATFORM": "carina"}, + run_fn=unexpected, + ) + == 0 + ) + assert ( + run_launch( + _options(tmp_path, no_allocate=True), + env={"HEARTWOOD_PLATFORM": "carina"}, + run_fn=unexpected, + ) + == 1 + ) + + terra = build_launch_plan(_options(tmp_path), {"GOOGLE_PROJECT": "synthetic-project"}) + assert terra.platform_id == "terra" + assert not terra.allocation_required + + +def test_direct_launch_reports_model_and_runtime_failures( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + env = {"HEARTWOOD_PLATFORM": "generic"} + assert run_launch(_options(tmp_path, model_root=None), env=env) == 64 + assert run_launch(_options(tmp_path), env=env) == 66 + _model(tmp_path / "model") + assert run_launch(_options(tmp_path), env=env) == 69 + assert "vLLM executable is unavailable" in capsys.readouterr().out + + +def test_direct_launch_supervises_runtime_and_opens_existing_chat( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _model(tmp_path / "model") + executable = tmp_path / "vllm" + executable.write_text("#!/bin/sh\n", encoding="utf-8") + executable.chmod(0o755) + (tmp_path / "state").mkdir() + (tmp_path / "state" / "setup.json").write_text("{}", encoding="utf-8") + observed: list[tuple[str, ...]] = [] + + class FakeProcess: + def __init__(self, command: object, **_kwargs: object) -> None: + observed.append(tuple(command)) # type: ignore[arg-type] + self.running = True + + def poll(self) -> int | None: + return None if self.running else 0 + + def terminate(self) -> None: + self.running = False + + def wait(self, timeout: float | None = None) -> int: + del timeout + return 0 + + def kill(self) -> None: + self.running = False + + monkeypatch.setattr("heartwood.cli._launch.subprocess.Popen", FakeProcess) + monkeypatch.setattr("heartwood.cli._launch._wait_for_runtime", lambda _runtime: True) + monkeypatch.setattr( + "heartwood.cli._launch.subprocess.run", + lambda command, **_kwargs: subprocess.CompletedProcess(command, 0), + ) + + assert run_launch(_options(tmp_path), env={"HEARTWOOD_PLATFORM": "generic"}) == 0 + assert observed[0][0] == str(executable) + assert "--enable-auto-tool-choice" in observed[0] + assert not any((tmp_path / "state" / "scratch").iterdir()) + + +def test_model_snapshot_rejects_malformed_unsafe_and_modified_content(tmp_path: Path) -> None: + model = tmp_path / "model" + model.mkdir() + (model / "SHA256SUMS").write_text("not-a-manifest\n", encoding="utf-8") + with pytest.raises(ValueError, match="invalid SHA256SUMS"): + verify_snapshot(model) + + digest = hashlib.sha256(b"expected").hexdigest() + (model / "SHA256SUMS").write_text(f"{digest} ../weights\n", encoding="utf-8") + with pytest.raises(ValueError, match="unsafe SHA256SUMS"): + verify_snapshot(model) + + (model / "weights").write_bytes(b"modified") + (model / "SHA256SUMS").write_text(f"{digest} weights\n{digest} weights\n", encoding="utf-8") + with pytest.raises(ValueError, match="duplicate SHA256SUMS"): + verify_snapshot(model) + + (model / "SHA256SUMS").write_text(f"{digest} weights\n", encoding="utf-8") + with pytest.raises(ValueError, match="SHA-256 mismatch"): + verify_snapshot(model) + + +def test_model_snapshot_rejects_unlisted_and_linked_files(tmp_path: Path) -> None: + model = tmp_path / "model" + _model(model) + (model / "unlisted").write_text("value", encoding="utf-8") + with pytest.raises(ValueError, match="unlisted"): + verify_snapshot(model) + (model / "unlisted").unlink() + (model / "link").symlink_to(model / "weights.safetensors") + with pytest.raises(ValueError, match="symbolic link"): + verify_snapshot(model) + + +def test_launch_runtime_helpers_scrub_credentials_and_resolve_native_vllm(tmp_path: Path) -> None: + env = { + "OPENAI_API_KEY": "secret", + "GH_TOKEN": "secret", + "STANFORD_AI_API_KEY": "secret", + "AWS_SECRET_ACCESS_KEY": "secret", + "CUSTOM_PROVIDER_TOKEN": "secret", + "PATH": "/usr/bin", + "CUDA_VISIBLE_DEVICES": "0", + "HEARTWOOD_NATIVE_ROOT": str(tmp_path), + "HEARTWOOD_NATIVE_VERSION": "v1", + } + runtime_env = _runtime_environment(env) + assert "OPENAI_API_KEY" not in runtime_env + assert "GH_TOKEN" not in runtime_env + assert "STANFORD_AI_API_KEY" not in runtime_env + assert "AWS_SECRET_ACCESS_KEY" not in runtime_env + assert "CUSTOM_PROVIDER_TOKEN" not in runtime_env + assert runtime_env["PATH"] == "/usr/bin" + assert runtime_env["CUDA_VISIBLE_DEVICES"] == "0" + assert runtime_env["HEARTWOOD_AGENT_BACKEND"] == "openhands-sdk" + assert _resolve_vllm(_options(tmp_path, vllm_executable=None, environment_root=None), env) == ( + tmp_path / "runtimes" / "v1" / "vllm" / "bin" / "vllm" + ) + + +def test_runtime_readiness_stops_when_process_exits() -> None: + class ExitedProcess: + def poll(self) -> int: + return 1 + + assert not _wait_for_runtime(ExitedProcess(), timeout=0.1) # type: ignore[arg-type] + + +def test_setup_helper_invokes_non_interactive_local_setup( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + observed: list[str] = [] + + def completed( + command: Sequence[str], **_kwargs: object + ) -> subprocess.CompletedProcess[Sequence[str]]: + observed.extend(command) + return subprocess.CompletedProcess(command, 7) + + monkeypatch.setattr("heartwood.cli._launch.subprocess.run", completed) + assert _ensure_setup(_options(tmp_path), {"OPENAI_API_KEY": "secret"}) == 7 + assert "--non-interactive" in observed + assert "--model-source" in observed + + (tmp_path / "state").mkdir() + (tmp_path / "state" / "setup.json").write_text("{}", encoding="utf-8") + observed.clear() + assert _ensure_setup(_options(tmp_path)) == 0 + assert not observed + + +def test_direct_launch_reports_staged_verification_failure( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + _model(tmp_path / "model") + executable = tmp_path / "vllm" + executable.write_text("#!/bin/sh\n", encoding="utf-8") + executable.chmod(0o755) + calls = 0 + + def verifier(_root: Path) -> None: + nonlocal calls + calls += 1 + if calls == 2: + raise ValueError("copy changed") + + monkeypatch.setattr("heartwood.cli._launch.verify_snapshot", verifier) + + assert run_launch(_options(tmp_path), env={"HEARTWOOD_PLATFORM": "generic"}) == 66 + assert "Staged model verification failed: copy changed" in capsys.readouterr().out + assert not any((tmp_path / "state" / "scratch").iterdir()) diff --git a/packages/compliance/tests/test_container_assets.py b/packages/compliance/tests/test_container_assets.py index 0e1a20d5..7381e13b 100644 --- a/packages/compliance/tests/test_container_assets.py +++ b/packages/compliance/tests/test_container_assets.py @@ -240,6 +240,7 @@ def test_vllm_launcher_enforces_loopback_and_tool_calling(tmp_path: Path) -> Non def test_carina_launcher_requires_verified_synthetic_allocation() -> None: bootstrap = _read("deploy/carina/bootstrap.sh") launcher = _read("deploy/carina/launch-interactive.sh") + launch_runtime = _read("packages/cli/src/heartwood/cli/_launch.py") batch = _read("deploy/carina/interactive.sbatch") environment = _toml("images/generic/image-flavors.toml") @@ -250,15 +251,15 @@ def test_carina_launcher_requires_verified_synthetic_allocation() -> None: assert '"${root}/vllm/bin/python"' in bootstrap assert "SLURM_JOB_ID" in launcher assert "LOCAL_SCRATCH_JOB" in launcher - assert "verify_model_snapshot.py" in launcher - assert 'repo_root="$(cd "${script_dir}/../.." && pwd)"' in launcher - assert 'kill -KILL "${runtime_pid}"' in launcher - assert 'mktemp -d "${LOCAL_SCRATCH_JOB%/}/heartwood-model.XXXXXX"' in launcher - assert 'rm -rf "${staged_model}"' in launcher - assert "unset GH_TOKEN GITHUB_TOKEN HF_TOKEN" in launcher - assert "unset OPENAI_API_KEY ANTHROPIC_API_KEY AZURE_API_KEY" in launcher - assert "--model-source local" in launcher - assert "127.0.0.1:8765/v1/models" in launcher + assert "--inside-allocation" in launcher + assert "HEARTWOOD_PLATFORM=carina" in launcher + assert "verify_snapshot(options.model_root)" in launch_runtime + assert 'env.get("LOCAL_SCRATCH_JOB"' in launch_runtime + assert "allowed_names" in launch_runtime + assert "result = {name: env[name] for name in allowed_names if name in env}" in launch_runtime + assert '"OPENAI_API_KEY"' not in launch_runtime + assert '"--model-source"' in launch_runtime + assert "127.0.0.1:8765/v1/models" in launch_runtime assert "#SBATCH --partition=gpu" in batch assert "#SBATCH --gres=gpu:1" in batch assert "${script_dir}/launch-interactive.sh" in batch @@ -273,7 +274,7 @@ def test_carina_model_verifier_requires_exact_manifest_coverage(tmp_path: Path) digest = hashlib.sha256(weights.read_bytes()).hexdigest() (model / "SHA256SUMS").write_text(f"{digest} weights.safetensors\n", encoding="utf-8") verifier = _repo_root() / "deploy/carina/verify_model_snapshot.py" - verifier_source = verifier.read_text(encoding="utf-8") + verifier_source = _read("packages/cli/src/heartwood/cli/_model_snapshot.py") assert "file.read(1024 * 1024)" in verifier_source assert "os.O_NOFOLLOW" in verifier_source assert ".read_bytes()" not in verifier_source @@ -292,6 +293,27 @@ def test_carina_model_verifier_requires_exact_manifest_coverage(tmp_path: Path) assert linked.returncode == 66 +def test_native_release_assets_are_verified_before_installation() -> None: + installer = _read("deploy/install.sh") + packager = _read("deploy/package-native.sh") + workflow = _read(".github/workflows/native-release.yml") + smoke = _read("deploy/tests/native_installer_smoke.sh") + + assert "sha256sum --check --strict" in installer + assert "--bundle" in installer + assert "--dry-run" in installer + assert "HEARTWOOD_INSTALL_ROOT" in installer + assert "checksum manifest must contain exactly heartwood-native.tar.gz" in installer + assert "[A-Za-z0-9._+-]{0,127}" in installer + assert "git archive --format=tar HEAD" in packager + assert "actions/attest@v4" in workflow + assert "gh release upload" in workflow + assert "Build And Verify Native Assets" in workflow + assert "native_installer_smoke.sh" in workflow + assert "installer accepted a corrupted checksum" in smoke + assert "installer accepted an unsafe checksum manifest" in smoke + + def test_gpu_publication_builds_only_explicit_main_variants() -> None: workflow = _read(".github/workflows/gpu-container-image.yml") dependency_review = _read(".github/workflows/dependency-review.yml") diff --git a/packages/gateway/src/heartwood/gateway/_readiness.py b/packages/gateway/src/heartwood/gateway/_readiness.py index c4aaacb9..5b4e38cf 100644 --- a/packages/gateway/src/heartwood/gateway/_readiness.py +++ b/packages/gateway/src/heartwood/gateway/_readiness.py @@ -216,6 +216,7 @@ def persist_deployment_profile( policy_path = state_root / "policy.json" connections_path = state_root / "model-connections.json" setup_path = state_root / "setup.json" + platform_policy = adapter.default_policy_profile() if model_source == "stanford-ai-api-gateway": connections = _stanford_connection_manifest() policy: object = { @@ -226,14 +227,16 @@ def persist_deployment_profile( "allowed_model_endpoints": [f"{_STANFORD_ROOT}/chat/completions"], "allowed_model_catalog_endpoints": [f"{_STANFORD_ROOT}/models"], "allowed_capability_tiers": ["supervised", "experimental"], - "allowed_action_confirmation_modes": ["always-confirm"], + "allowed_action_confirmation_modes": list( + platform_policy.allowed_action_confirmation_modes + ), "credential_allowlist": ["STANFORD_AI_API_KEY"], "aggregate_count_floor": 20, "notes": "Stanford gateway route; data eligibility requires deployment approval.", } else: connections = {"schema_version": "heartwood.model-connections.v1", "connections": []} - policy = adapter.default_policy_profile().model_dump(mode="json") + policy = platform_policy.model_dump(mode="json") _atomic_json(connections_path, connections) _atomic_json(policy_path, policy) _atomic_json( diff --git a/packages/gateway/tests/test_readiness.py b/packages/gateway/tests/test_readiness.py index d40c8b46..34964aba 100644 --- a/packages/gateway/tests/test_readiness.py +++ b/packages/gateway/tests/test_readiness.py @@ -248,9 +248,24 @@ def test_local_setup_persists_conservative_restart_configuration(tmp_path: Path) assert connections.stat().st_mode & 0o777 == 0o600 payload = json.loads(policy.read_text(encoding="utf-8")) assert payload["platform_id"] == "carina" - assert payload["allowed_action_confirmation_modes"] == ["always-confirm"] + assert payload["allowed_action_confirmation_modes"] == ["always-confirm", "confirm-risky"] assert payload["credential_allowlist"] == [] + gateway = SessionGateway(workspace=workspace, env={"HEARTWOOD_PLATFORM": "carina"}) + selected = gateway.select_action_confirmation_mode("confirm-risky") + assert selected["confirmation_mode"] == "confirm-risky" + + +def test_stanford_setup_inherits_carina_confirmation_modes(tmp_path: Path) -> None: + workspace = tmp_path / "state" / "sessions" + _, policy, _ = persist_deployment_profile( + workspace, + model_source="stanford-ai-api-gateway", + env={"HEARTWOOD_PLATFORM": "carina"}, + ) + payload = json.loads(policy.read_text(encoding="utf-8")) + assert payload["allowed_action_confirmation_modes"] == ["always-confirm", "confirm-risky"] + def test_stanford_setup_is_discovered_after_gateway_restart(tmp_path: Path) -> None: workspace = tmp_path / "state" / "sessions"