From 7299becb74df36d29e285461c6b2266f12223edd Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 15 Jul 2026 20:50:45 -0700 Subject: [PATCH 01/74] feat(installer): prepare DGX Station host prerequisites Signed-off-by: Aaron Erickson --- docs/get-started/prerequisites.mdx | 7 + docs/get-started/quickstart.mdx | 5 + scripts/checks/vitest-project-overlap.ts | 1 + scripts/install.sh | 157 ++++- scripts/prepare-dgx-station-host.sh | 656 ++++++++++++++++++ test/install-express-prompt.test.ts | 3 + test/install-station-host-preparation.test.ts | 324 +++++++++ vitest.config.ts | 2 + 8 files changed, 1128 insertions(+), 27 deletions(-) create mode 100755 scripts/prepare-dgx-station-host.sh create mode 100644 test/install-station-host-preparation.test.ts diff --git a/docs/get-started/prerequisites.mdx b/docs/get-started/prerequisites.mdx index b8b28670bd..aee76ff15b 100644 --- a/docs/get-started/prerequisites.mdx +++ b/docs/get-started/prerequisites.mdx @@ -41,6 +41,13 @@ If the group change is not active in the current shell, the installer exits with If you choose the native Linux Ollama install path, the onboard wizard also requires `zstd` for Ollama archive extraction. The installer also requires `strings` from `binutils` to verify the OpenShell binary before it continues with OpenShell install work. +On an Ubuntu 24.04 ARM64 DGX Station GB300, accepting express install prepares the host with NVIDIA open driver `610.43.02`, Docker CE `29.6.1` with Buildx, and NVIDIA Container Toolkit `1.19.1`. +The preparation probes package and runtime state first, reuses exact matches, and installs only missing pinned packages. +It requires Secure Boot to be disabled, matching headers for the running kernel, at least 20 GiB free on the root filesystem, and no active agent, inference, or Docker workloads. +It does not install a host CUDA toolkit or Docker Compose. +If an existing prerequisite has a different version, preparation stops instead of automatically upgrading or downgrading it. +After installing missing pinned packages, the installer exits with status `10`; reboot, sign in, and rerun the same installer command to resume express setup. + NemoClaw needs Docker access. On personal Linux development machines, adding your user to the `docker` group is the standard way to run Docker without sudo. diff --git a/docs/get-started/quickstart.mdx b/docs/get-started/quickstart.mdx index 1cfdb81341..d1d61ba87e 100644 --- a/docs/get-started/quickstart.mdx +++ b/docs/get-started/quickstart.mdx @@ -121,6 +121,8 @@ Use these details when your first-run path needs more control. DGX Spark, DGX Station, and Windows WSL offer an interactive express-install path that chooses a managed local inference option for the platform. On DGX Station, accepting the express prompt selects the pinned `nemotron-3-ultra-550b-a55b` managed-vLLM recipe and completes onboarding without more provider, model, policy, or sandbox-name choices. + The Station path probes the validated driver, Docker, Buildx, and NVIDIA Container Toolkit versions, reuses exact matches, and installs missing pinned packages without additional choices. + After it installs missing pinned packages, the installer exits with status `10` at the validated reboot boundary; reboot, sign in, and rerun the same installer command to resume the accepted recipe without another prompt. Pass `--station-deepseek` to use DeepSeek V4 Flash for a Station demo instead. Refer to [Platform Support](../reference/platform-support) and [Choose an Inference Provider](../inference/learn-and-choose/choose-inference-provider) for the current platform behavior. @@ -190,6 +192,9 @@ Use these details when your first-run path needs more control. Express install switches onboarding to non-interactive mode, allows `sudo` password prompts for required host changes, and selects the managed local inference path for that platform. DGX Spark uses managed vLLM with `qwen3.6-35b-a3b-nvfp4` by default. DGX Station express install explicitly selects `nemotron-3-ultra-550b-a55b` instead of the Station managed-vLLM profile default, `deepseek-v4-flash`, and discloses the approximately `352 GB` model download before confirmation. + Before onboarding, the Station path checks for NVIDIA open driver `610.43.02`, Docker CE `29.6.1` with Buildx, and NVIDIA Container Toolkit `1.19.1`. + It reuses exact versions, installs missing pinned packages, and refuses to automatically upgrade or downgrade an existing mismatched version. + Installing missing pinned packages exits with status `10` for a reboot; after you sign in and rerun the same installer command, the accepted express recipe resumes without another prompt. To select DeepSeek V4 Flash while retaining the one-confirmation Station express flow, run `curl -fsSL https://www.nvidia.com/nemoclaw.sh | bash -s -- --station-deepseek`. Unless `NEMOCLAW_POLICY_TIER` is set, express install applies policy in `suggested` mode with the `balanced` tier, including the base sandbox policy and supported package, model, web-search, and local-inference presets. Express install uses `my-assistant` as the sandbox name across all platforms unless `NEMOCLAW_SANDBOX_NAME` is set. diff --git a/scripts/checks/vitest-project-overlap.ts b/scripts/checks/vitest-project-overlap.ts index 83dfdf033e..9e5415c7ee 100644 --- a/scripts/checks/vitest-project-overlap.ts +++ b/scripts/checks/vitest-project-overlap.ts @@ -46,6 +46,7 @@ const INSTALLER_INTEGRATION_TESTS = new Set([ "test/install-openshell-version-check.test.ts", "test/install-preflight-docker-bootstrap.test.ts", "test/install-preflight.test.ts", + "test/install-station-host-preparation.test.ts", ]); function normalizeRepoPath(file: string): string { diff --git a/scripts/install.sh b/scripts/install.sh index 591cc4cad6..8b21fe6069 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -2868,6 +2868,7 @@ STATION_ULTRA_VLLM_MODEL="nemotron-3-ultra-550b-a55b" STATION_ULTRA_SERVED_MODEL="nvidia/nemotron-3-ultra-550b-a55b" STATION_DEEPSEEK_VLLM_MODEL="deepseek-v4-flash" STATION_DEEPSEEK_SERVED_MODEL="deepseek-ai/DeepSeek-V4-Flash" +_SELECTED_EXPRESS_PLATFORM="" normalize_station_vllm_model() { printf "%s" "${1:-}" | tr '[:upper:]' '[:lower:]' | sed 's/^[[:space:]]*//; s/[[:space:]]*$//' @@ -2939,6 +2940,122 @@ configure_station_express_model() { fi } +station_express_resume_file() { + local state_dir + state_dir="$(nemoclaw_state_dir)" || return 1 + printf '%s/station-express-resume' "$state_dir" +} + +validate_station_express_resume_model() { + local model="${1:-}" + [[ ${#model} -le 255 && "$model" =~ ^[A-Za-z0-9][A-Za-z0-9._/-]*$ ]] +} + +load_station_express_resume() { + local state_file model line_count + state_file="$(station_express_resume_file)" || return 1 + assert_nemoclaw_state_path_safe "$state_file" + [[ -f "$state_file" ]] || return 1 + line_count="$(wc -l <"$state_file" | tr -d '[:space:]')" + IFS= read -r model <"$state_file" || model="" + if [[ "$line_count" != "1" ]] || ! validate_station_express_resume_model "$model"; then + error "DGX Station express resume state is invalid. Remove ${state_file} and rerun the installer." + fi + NEMOCLAW_VLLM_MODEL="$model" + export NEMOCLAW_VLLM_MODEL +} + +save_station_express_resume() { + local state_file state_dir temp_file model="${NEMOCLAW_VLLM_MODEL:-}" + validate_station_express_resume_model "$model" || error "Cannot save an invalid DGX Station express model selector." + state_file="$(station_express_resume_file)" || error "Could not resolve NemoClaw state for DGX Station express resume." + state_dir="$(ensure_nemoclaw_state_dir)" || error "Could not prepare NemoClaw state for DGX Station express resume." + assert_nemoclaw_state_path_safe "$state_file" + temp_file="$(mktemp "${state_file}.tmp.XXXXXX")" || error "Could not create DGX Station express resume state under ${state_dir}." + chmod 600 "$temp_file" || { + rm -f "$temp_file" + error "Could not secure DGX Station express resume state under ${state_dir}." + } + if ! printf '%s\n' "$model" >"$temp_file"; then + rm -f "$temp_file" + error "Could not write DGX Station express resume state under ${state_dir}." + fi + if ! mv -f "$temp_file" "$state_file"; then + rm -f "$temp_file" + error "Could not publish DGX Station express resume state under ${state_dir}." + fi +} + +clear_station_express_resume() { + local state_file + state_file="$(station_express_resume_file)" || return 0 + assert_nemoclaw_state_path_safe "$state_file" + rm -f "$state_file" +} + +activate_express_install() { + local platform="$1" + _SELECTED_EXPRESS_PLATFORM="$platform" + NON_INTERACTIVE=1 + export NEMOCLAW_NON_INTERACTIVE=1 + export NEMOCLAW_NON_INTERACTIVE_SUDO_MODE=prompt + export NEMOCLAW_YES=1 + export NEMOCLAW_POLICY_MODE=suggested + case "$platform" in + "DGX Spark") + export NEMOCLAW_SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-my-assistant}" + export NEMOCLAW_PROVIDER=install-vllm + if [ -n "${NEMOCLAW_VLLM_MODEL:-}" ]; then + export NEMOCLAW_VLLM_MODEL + fi + ;; + "DGX Station") + export NEMOCLAW_SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-my-assistant}" + export NEMOCLAW_PROVIDER=install-vllm + configure_station_express_model + ;; + "Windows WSL") + export NEMOCLAW_PROVIDER=install-windows-ollama + ;; + esac +} + +run_station_host_preparation() { + local helper="${SCRIPT_DIR}/prepare-dgx-station-host.sh" + [[ -f "$helper" ]] || error "DGX Station host preparation helper is missing: ${helper}" + bash "$helper" --apply +} + +ensure_station_express_host() { + [[ "${_SELECTED_EXPRESS_PLATFORM:-}" == "DGX Station" ]] || return 0 + + info "Checking pinned DGX Station host prerequisites. Exact matches are reused." + local status=0 + run_station_host_preparation || status=$? + case "$status" in + 0) + ok "DGX Station host prerequisites are ready" + ;; + 10) + save_station_express_resume + warn "DGX Station host prerequisites were installed and require a reboot." + info "Run: sudo reboot" + info "After signing in again, rerun the same NemoClaw installer command; express setup resumes without another choice." + exit 10 + ;; + *) + error "DGX Station host preparation failed. Review the station-bootstrap log above, correct the reported host state, and rerun the installer." + ;; + esac +} + +prepare_installer_host() { + maybe_offer_express_install + ensure_station_express_host + ensure_docker + ensure_openshell_build_deps +} + # Prompt the user to opt into express install on supported platforms. Sets the # non-interactive + provider/model env vars when accepted. Skipped when # the user already passed --non-interactive, set NEMOCLAW_PROVIDER, or has @@ -2972,6 +3089,7 @@ describe_express_install() { inference_summary="managed local vLLM with NVIDIA Nemotron 3 Ultra 550B" inference_disclosure="Managed vLLM pulls the pinned Station image and approximately 352 GB model, then runs a local inference container." fi + printf " Station host setup reuses exact prerequisite versions, installs missing pinned driver, Docker, and NVIDIA Container Toolkit packages, and may require one reboot.\n" sandbox_summary="${NEMOCLAW_SANDBOX_NAME:-my-assistant}" ;; "Windows WSL") @@ -3025,6 +3143,11 @@ maybe_offer_express_install() { info "Detected ${platform}. Skipping express prompt (NEMOCLAW_NO_EXPRESS=1)." return 0 fi + if [ "$platform" = "DGX Station" ] && load_station_express_resume; then + info "Detected DGX Station. Resuming the accepted express install after host preparation." + activate_express_install "$platform" + return 0 + fi if [ "${NON_INTERACTIVE:-}" = "1" ]; then info "Detected ${platform}. Skipping express prompt (--non-interactive set)." return 0 @@ -3060,28 +3183,7 @@ maybe_offer_express_install() { case "$reply" in "" | y | yes) info "Using express install for ${platform}." - NON_INTERACTIVE=1 - export NEMOCLAW_NON_INTERACTIVE=1 - export NEMOCLAW_NON_INTERACTIVE_SUDO_MODE=prompt - export NEMOCLAW_YES=1 - export NEMOCLAW_POLICY_MODE=suggested - case "$platform" in - "DGX Spark") - export NEMOCLAW_SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-my-assistant}" - export NEMOCLAW_PROVIDER=install-vllm - if [ -n "${NEMOCLAW_VLLM_MODEL:-}" ]; then - export NEMOCLAW_VLLM_MODEL - fi - ;; - "DGX Station") - export NEMOCLAW_SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-my-assistant}" - export NEMOCLAW_PROVIDER=install-vllm - configure_station_express_model - ;; - "Windows WSL") - export NEMOCLAW_PROVIDER=install-windows-ollama - ;; - esac + activate_express_install "$platform" ;; *) info "Skipping express install. Continuing with interactive flow." @@ -3160,15 +3262,13 @@ main() { # still collect acceptance before Node.js or the CLI are installed. preflight_usage_notice_prompt - ensure_docker - ensure_openshell_build_deps - # Offer express install on supported platforms (DGX Spark / Station / WSL). # Runs AFTER the third-party notice so the user has explicitly accepted the # license before opting into the unattended path. Express only sets the # provider/model/policy + non-interactive vars; license acceptance is - # already recorded by preflight above. - maybe_offer_express_install + # already recorded by preflight above. Station selection runs its pinned + # host prerequisite preparation before the generic Docker bootstrap. + prepare_installer_host _INSTALL_START=$SECONDS bash "${SCRIPT_DIR}/setup-jetson.sh" @@ -3237,6 +3337,9 @@ main() { fi finalize_install + if [[ "${_SELECTED_EXPRESS_PLATFORM:-}" == "DGX Station" ]]; then + clear_station_express_resume + fi } # Print the completion summary, then propagate a fatal/non-zero result when the diff --git a/scripts/prepare-dgx-station-host.sh b/scripts/prepare-dgx-station-host.sh new file mode 100755 index 0000000000..42b9d3e104 --- /dev/null +++ b/scripts/prepare-dgx-station-host.sh @@ -0,0 +1,656 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -Eeuo pipefail +umask 077 + +readonly SCRIPT_VERSION="2026-07-15.1" +readonly REBOOT_REQUIRED_EXIT=10 +readonly MIN_FREE_KIB=$((20 * 1024 * 1024)) + +readonly CUDA_KEYRING_URL="https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/sbsa/cuda-keyring_1.1-1_all.deb" +readonly CUDA_KEYRING_SHA256="6ea7d2737648936820e85677177957a0f6521b840d98eb0bbae0a4f003fa7249" +readonly CUDA_KEY_FINGERPRINT="EB693B3035CD5710E231E123A4B469963BF863CC" # gitleaks:allow -- public NVIDIA signing-key fingerprint +readonly DOCKER_KEY_URL="https://download.docker.com/linux/ubuntu/gpg" +readonly DOCKER_KEY_SHA256="1500c1f56fa9e26b9b8f42452a553675796ade0807cdce11975eb98170b3a570" # gitleaks:allow -- public Docker GPG-key integrity pin +readonly DOCKER_KEY_FINGERPRINT="9DC858229FC7DD38854AE2D88D81803C0EBFCD88" + +readonly DRIVER_VERSION="610.43.02" +readonly DOCKER_VERSION="29.6.1" +readonly TOOLKIT_VERSION="1.19.1" +readonly ACCEPTANCE_IMAGE="ubuntu@sha256:7f622ca8766bccb22f04242ecb6f19f770b2f08827dc4b8c707de5e78a6da7ab" +readonly STATE_DIR="${HOME}/.local/state/station-bootstrap" +readonly INSTALL_BOOT_MARKER="${STATE_DIR}/install-boot-id" + +readonly -a PACKAGE_SPECS=( + "dkms=1:3.4.0-1ubuntu1" + "nvidia-driver-pinning-610=610-2ubuntu1" + "nvidia-driver-open=610.43.02-1ubuntu1" + "containerd.io=2.2.6-1~ubuntu.24.04~noble" + "docker-buildx-plugin=0.35.0-1~ubuntu.24.04~noble" + "docker-ce=5:29.6.1-1~ubuntu.24.04~noble" + "docker-ce-cli=5:29.6.1-1~ubuntu.24.04~noble" + "libnvidia-container-tools=1.19.1-1" + "libnvidia-container1=1.19.1-1" + "nvidia-container-toolkit=1.19.1-1" + "nvidia-container-toolkit-base=1.19.1-1" +) + +MODE="" +LOG_FILE="" + +info() { + printf '[station-prepare] %s %s\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$*" +} + +warn() { + printf '[station-prepare] %s WARNING: %s\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$*" >&2 +} + +fatal() { + printf '[station-prepare] %s ERROR: %s\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$*" >&2 + exit 1 +} + +on_error() { + local rc=$? + local line=${1:-unknown} + printf '[station-prepare] %s ERROR: command failed at line %s (exit %s)\n' \ + "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$line" "$rc" >&2 + exit "$rc" +} + +usage() { + cat <<'EOF' +Usage: prepare-dgx-station-host.sh --check|--apply|--verify + + --check Read-only eligibility and current-state report. + --apply Install exact prerequisites or finish post-reboot runtime setup. + --verify Read-only host verification plus ephemeral GPU container tests. + +Exit 10 from --apply means an operator-controlled reboot is required. After +the reboot, run --apply once more, followed by --verify. +EOF +} + +is_valid_mode() { + case "${1:-}" in + --check | --apply | --verify) return 0 ;; + *) return 1 ;; + esac +} + +is_station_product() { + local product=${1:-} + [[ "$product" == *"Station GB300"* || "$product" == *"DGX Station"* ]] +} + +is_expected_failed_unit() { + case "${1:-}" in + cloud-init.service | fwupd-refresh.service | NetworkManager-wait-online.service | systemd-networkd-wait-online.service | \ + sssd-autofs.socket | sssd-nss.socket | sssd-pam-priv.socket | sssd-pam.socket) + return 0 + ;; + *) return 1 ;; + esac +} + +is_driver_transitional_unit() { + [[ "${1:-}" == "nvidia-persistenced.service" ]] +} + +package_name() { + printf '%s\n' "${1%%=*}" +} + +package_expected_version() { + printf '%s\n' "${1#*=}" +} + +acquire_sudo() { + if sudo -n true >/dev/null 2>&1; then + info "sudo=noninteractive" + return + fi + + info "sudo=interactive_authentication_required" + sudo -v +} + +installed_version() { + dpkg-query -W -f='${Version}' "$1" 2>/dev/null || true +} + +package_is_exact() { + local spec=$1 + local name expected actual + name="$(package_name "$spec")" + expected="$(package_expected_version "$spec")" + actual="$(installed_version "$name")" + [[ "$actual" == "$expected" ]] +} + +package_state() { + local spec=$1 + local name expected actual + name="$(package_name "$spec")" + expected="$(package_expected_version "$spec")" + actual="$(installed_version "$name")" + if [[ -z "$actual" ]]; then + printf 'missing\n' + elif [[ "$actual" == "$expected" ]]; then + printf 'exact\n' + else + printf 'mismatch\n' + fi +} + +assert_no_package_mismatches() { + local spec state name expected actual mismatch=0 + for spec in "${PACKAGE_SPECS[@]}"; do + state="$(package_state "$spec")" + [[ "$state" == "mismatch" ]] || continue + name="$(package_name "$spec")" + expected="$(package_expected_version "$spec")" + actual="$(installed_version "$name")" + warn "package=${name} status=mismatch actual=${actual} expected=${expected}" + mismatch=1 + done + ((mismatch == 0)) || fatal "Existing Station prerequisite versions differ from the validated pins; refusing to upgrade or downgrade them automatically" +} + +all_packages_exact() { + local spec + for spec in "${PACKAGE_SPECS[@]}"; do + package_is_exact "$spec" || return 1 + done + return 0 +} + +setup_log() { + local log_dir="${HOME}/station-bootstrap-logs" + mkdir -p "$log_dir" + chmod 0700 "$log_dir" + LOG_FILE="${log_dir}/station-prepare-${MODE#--}-$(date -u '+%Y%m%dT%H%M%SZ').log" + exec > >(tee -a "$LOG_FILE") 2>&1 + info "version=${SCRIPT_VERSION} mode=${MODE} log=${LOG_FILE}" +} + +require_command() { + command -v "$1" >/dev/null 2>&1 || fatal "Required command is missing: $1" +} + +check_platform() { + local arch product + arch="$(uname -m)" + [[ "$arch" == "aarch64" || "$arch" == "arm64" ]] || fatal "Expected ARM64, found ${arch}" + + [[ -r /etc/os-release ]] || fatal "/etc/os-release is unavailable" + # shellcheck disable=SC1091 + source /etc/os-release + [[ "${ID:-}" == "ubuntu" && "${VERSION_ID:-}" == "24.04" ]] \ + || fatal "Expected Ubuntu 24.04, found ${PRETTY_NAME:-unknown}" + + product="$(= MIN_FREE_KIB)) || fatal "At least 20 GiB free is required; found $((available / 1024 / 1024)) GiB" + info "root_free_gib=$((available / 1024 / 1024))" +} + +check_network() { + local host + for host in developer.download.nvidia.com download.docker.com registry-1.docker.io; do + getent ahosts "$host" >/dev/null 2>&1 || fatal "DNS resolution failed for ${host}" + done + info "network=required_vendor_hosts_resolve" +} + +check_package_managers_idle() { + local active + active="$(ps -eo pid=,comm= | awk '$2 ~ /^(apt|apt-get|dpkg|unattended-upgrade)$/ {print}')" + [[ -z "$active" ]] || fatal "A package-manager process is active: ${active}" + info "package_manager=idle" +} + +check_failed_units() { + local unit unexpected=0 + local -a units=() + mapfile -t units < <(systemctl --failed --no-legend --plain 2>/dev/null | awk 'NF {print $1}') + if ((${#units[@]} == 0)); then + info "failed_units=none" + return 0 + fi + for unit in "${units[@]}"; do + if is_expected_failed_unit "$unit"; then + warn "pre-existing OEM failed unit allowed for this pilot: ${unit}" + elif is_driver_transitional_unit "$unit" && all_packages_exact; then + warn "driver unit failure allowed only until post-reboot verification: ${unit}" + else + warn "unexpected failed unit: ${unit}" + unexpected=1 + fi + done + ((unexpected == 0)) || fatal "Unexpected failed units block Station preparation" +} + +check_no_workloads() { + local processes matches listeners containers="" + processes="$(ps -eo pid=,ppid=,comm=,args=)" + matches="$(awk -v self="$$" -v parent="$PPID" ' + { + pid=$1 + ppid=$2 + comm=tolower($3) + $1=$2=$3="" + args=tolower($0) + if (pid == self || pid == parent) next + if (comm ~ /^(vllm|nemoclaw|openshell)$/ || + args ~ /(^|[[:space:]\/])(vllm|nemoclaw|openshell)([[:space:]:]|\.js([[:space:]]|$)|$)/) print + } + ' <<<"$processes")" + [[ -z "$matches" ]] || fatal "Agent or inference workload is active: ${matches}" + + listeners="$(ss -H -ltn 2>/dev/null | awk '$4 ~ /:8000$/ {print}')" + [[ -z "$listeners" ]] || fatal "Port 8000 is already listening: ${listeners}" + + if command -v docker >/dev/null 2>&1; then + if containers="$(docker ps -aq 2>/dev/null)"; then + : + elif [[ "$MODE" == "--apply" ]] && containers="$(sudo -n docker ps -aq 2>/dev/null)"; then + info "docker_access=sudo_until_group_membership_is_active" + elif systemctl is-active --quiet docker.service; then + fatal "Docker is active but inaccessible to this login; start a new login session with docker-group membership" + else + info "docker_daemon=inactive" + containers="" + fi + fi + [[ -z "$containers" ]] || fatal "Existing Docker containers block host preparation: ${containers}" + info "workloads=none port_8000=free" +} + +driver_loaded_exact() { + local loaded + command -v nvidia-smi >/dev/null 2>&1 || return 1 + loaded="$(nvidia-smi --query-gpu=driver_version --format=csv,noheader 2>/dev/null | head -n1 | tr -d '[:space:]')" + [[ "$loaded" == "$DRIVER_VERSION" ]] +} + +write_install_boot_marker() { + mkdir -p "$STATE_DIR" + chmod 0700 "$STATE_DIR" + cp /proc/sys/kernel/random/boot_id "$INSTALL_BOOT_MARKER" + chmod 0600 "$INSTALL_BOOT_MARKER" +} + +install_boot_marker_matches_current_boot() { + [[ -r "$INSTALL_BOOT_MARKER" ]] || return 1 + [[ "$(tr -d '[:space:]' <"$INSTALL_BOOT_MARKER")" == "$(tr -d '[:space:]' /dev/null \ + | awk -F: '$1 == "fpr" {print $10}' \ + | grep -Fxq "$expected" || fatal "Expected signing-key fingerprint ${expected} was not found in ${path}" +} + +configure_repositories() { + local tmp cuda_deb docker_asc docker_gpg docker_list + tmp="$(mktemp -d)" + cuda_deb="${tmp}/cuda-keyring.deb" + docker_asc="${tmp}/docker.asc" + docker_gpg="${tmp}/docker.gpg" + docker_list="${tmp}/docker.list" + + info "Downloading and verifying official repository keys" + curl --fail --silent --show-error --location "$CUDA_KEYRING_URL" --output "$cuda_deb" + verify_file_sha256 "$cuda_deb" "$CUDA_KEYRING_SHA256" + sudo dpkg -i "$cuda_deb" + verify_key_fingerprint /usr/share/keyrings/cuda-archive-keyring.gpg "$CUDA_KEY_FINGERPRINT" + + curl --fail --silent --show-error --location "$DOCKER_KEY_URL" --output "$docker_asc" + verify_file_sha256 "$docker_asc" "$DOCKER_KEY_SHA256" + verify_key_fingerprint "$docker_asc" "$DOCKER_KEY_FINGERPRINT" + gpg --batch --yes --dearmor --output "$docker_gpg" "$docker_asc" + sudo install -d -m 0755 /etc/apt/keyrings + sudo install -m 0644 "$docker_gpg" /etc/apt/keyrings/docker.gpg + printf '%s\n' \ + 'deb [arch=arm64 signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu noble stable' \ + >"$docker_list" + sudo install -m 0644 "$docker_list" /etc/apt/sources.list.d/docker.list + + rm -rf "$tmp" + info "repository_keys=verified" +} + +validate_package_availability() { + local spec + for spec in "${PACKAGE_SPECS[@]}"; do + apt-cache show "$spec" >/dev/null 2>&1 || fatal "Exact package version is unavailable: ${spec}" + done + info "exact_package_versions=available" +} + +simulate_install() { + local simulation + simulation="$(apt-get -s install --no-install-recommends "${PACKAGE_SPECS[@]}")" \ + || fatal "APT simulation failed" + printf '%s\n' "$simulation" + if grep -Eq '^(Remv |Purg )' <<<"$simulation"; then + fatal "APT simulation proposed a package removal" + fi + info "apt_simulation=no_removals" +} + +install_packages() { + configure_repositories + info "Refreshing package metadata" + sudo apt-get update + validate_package_availability + simulate_install + info "Installing pinned Station prerequisites" + sudo env DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + "${PACKAGE_SPECS[@]}" + + local spec + for spec in "${PACKAGE_SPECS[@]}"; do + package_is_exact "$spec" || fatal "Installed package does not match ${spec}" + done + info "pinned_packages=installed" +} + +ensure_docker_group() { + local user_name=${SUDO_USER:-$USER} + getent group docker >/dev/null 2>&1 || fatal "Docker group is missing after package installation" + if ! id -nG "$user_name" | tr ' ' '\n' | grep -Fxq docker; then + sudo usermod -aG docker "$user_name" + info "docker_group=added user=${user_name}; a new login is required" + else + info "docker_group=present user=${user_name}" + fi +} + +refresh_cdi() { + sudo systemctl enable --now nvidia-cdi-refresh.path + if ! sudo systemctl restart nvidia-cdi-refresh.service; then + warn "Packaged CDI refresh failed; generating the exact fallback spec" + sudo install -d -m 0755 /etc/cdi + sudo nvidia-ctk cdi generate --output=/etc/cdi/nvidia.yaml + fi + nvidia-ctk cdi list | grep -Fxq 'nvidia.com/gpu=all' || fatal "CDI device nvidia.com/gpu=all is unavailable" + info "cdi=nvidia.com/gpu=all" +} + +ensure_acceptance_image() { + if ! sudo docker image inspect "$ACCEPTANCE_IMAGE" >/dev/null 2>&1; then + info "Pulling digest-pinned ARM64 acceptance image" + sudo docker pull --platform linux/arm64 "$ACCEPTANCE_IMAGE" + fi +} + +run_cdi_test_sudo() { + sudo docker run --rm --device nvidia.com/gpu=all "$ACCEPTANCE_IMAGE" nvidia-smi +} + +run_gpus_test_sudo() { + sudo docker run --rm --gpus all "$ACCEPTANCE_IMAGE" nvidia-smi +} + +ensure_cdi_runtime() { + if run_cdi_test_sudo; then + info "cdi_contract=pass_without_configuration_change" + return 0 + fi + + warn "CDI GPU launch failed; refreshing the NVIDIA CDI device spec" + refresh_cdi + run_cdi_test_sudo || fatal "CDI Docker GPU test failed after CDI refresh" + info "cdi_contract=pass_after_refresh" +} + +configure_docker_runtime_if_needed() { + local backup_dir timestamp + if run_gpus_test_sudo; then + info "docker_gpus_contract=pass_without_configuration_change" + return 0 + fi + + warn "Docker --gpus all failed; applying the reviewed NVIDIA runtime registration" + [[ -z "$(sudo docker ps -aq)" ]] || fatal "Docker became non-idle before runtime configuration" + timestamp="$(date -u '+%Y%m%dT%H%M%SZ')" + backup_dir="/var/backups/station-bootstrap/${timestamp}" + sudo install -d -m 0700 "$backup_dir" + if sudo test -e /etc/docker/daemon.json; then + sudo cp --preserve=all /etc/docker/daemon.json "${backup_dir}/daemon.json" + else + sudo touch "${backup_dir}/daemon.json.absent" + sudo chmod 0600 "${backup_dir}/daemon.json.absent" + fi + sudo nvidia-ctk runtime configure --runtime=docker + sudo systemctl restart docker.service + run_gpus_test_sudo || fatal "Docker --gpus all still fails after NVIDIA runtime registration" + run_cdi_test_sudo || fatal "CDI launch regressed after NVIDIA runtime registration" + info "docker_gpus_contract=pass backup=${backup_dir}" +} + +finish_runtime() { + sudo systemctl enable --now containerd.service docker.service + ensure_docker_group + ensure_acceptance_image + ensure_cdi_runtime + configure_docker_runtime_if_needed + [[ -z "$(sudo docker ps -aq)" ]] || fatal "Acceptance tests left a Docker container behind" + info "runtime_setup=complete" +} + +verify_apply_state() { + local spec + for spec in "${PACKAGE_SPECS[@]}"; do + package_is_exact "$spec" || fatal "Package verification failed: ${spec}" + done + verify_gpu + systemctl is-active --quiet nvidia-persistenced.service || fatal "nvidia-persistenced.service is not active" + systemctl is-active --quiet containerd.service || fatal "containerd.service is not active" + systemctl is-active --quiet docker.service || fatal "docker.service is not active" + nvidia-ctk cdi list | grep -Fxq 'nvidia.com/gpu=all' || fatal "CDI verification failed" + sudo docker image inspect "$ACCEPTANCE_IMAGE" >/dev/null 2>&1 || fatal "Digest-pinned acceptance image is missing" + [[ -z "$(sudo docker ps -aq)" ]] || fatal "Verification found a leftover Docker container" + info "STATION_HOST_READY" +} + +verify_gpu() { + local row name driver corrected uncorrected + row="$(nvidia-smi \ + --query-gpu=name,driver_version,ecc.errors.corrected.volatile.total,ecc.errors.uncorrected.volatile.total \ + --format=csv,noheader,nounits)" || fatal "nvidia-smi failed" + IFS=',' read -r name driver corrected uncorrected <<<"$row" + name="${name#"${name%%[![:space:]]*}"}" + driver="${driver//[[:space:]]/}" + corrected="${corrected//[[:space:]]/}" + uncorrected="${uncorrected//[[:space:]]/}" + [[ "$name" == *"GB300"* ]] || fatal "Expected NVIDIA GB300, found ${name}" + [[ "$driver" == "$DRIVER_VERSION" ]] || fatal "Expected driver ${DRIVER_VERSION}, found ${driver}" + [[ "$corrected" == "0" && "$uncorrected" == "0" ]] \ + || fatal "ECC must be 0/0, found corrected=${corrected} uncorrected=${uncorrected}" + info "gpu=${name} driver=${driver} ecc_corrected=${corrected} ecc_uncorrected=${uncorrected}" +} + +verify_host() { + local spec user_name=${SUDO_USER:-$USER} + for spec in "${PACKAGE_SPECS[@]}"; do + package_is_exact "$spec" || fatal "Package verification failed: ${spec}" + done + verify_gpu + systemctl is-active --quiet nvidia-persistenced.service || fatal "nvidia-persistenced.service is not active" + systemctl is-active --quiet containerd.service || fatal "containerd.service is not active" + systemctl is-active --quiet docker.service || fatal "docker.service is not active" + id -nG "$user_name" | tr ' ' '\n' | grep -Fxq docker || fatal "${user_name} is not in the docker group" + docker info >/dev/null 2>&1 || fatal "${user_name} cannot access Docker; start a new login session" + nvidia-ctk cdi list | grep -Fxq 'nvidia.com/gpu=all' || fatal "CDI verification failed" + docker image inspect "$ACCEPTANCE_IMAGE" >/dev/null 2>&1 || fatal "Digest-pinned acceptance image is missing; run --apply" + docker run --rm --device nvidia.com/gpu=all "$ACCEPTANCE_IMAGE" nvidia-smi >/dev/null + docker run --rm --gpus all "$ACCEPTANCE_IMAGE" nvidia-smi >/dev/null + [[ -z "$(docker ps -aq)" ]] || fatal "Verification left a Docker container behind" + info "docker=$(docker version --format '{{.Server.Version}}') expected_docker=${DOCKER_VERSION} toolkit=$(nvidia-ctk --version | head -n1) expected_toolkit=${TOOLKIT_VERSION}" + info "STATION_HOST_READY" +} + +run_check() { + common_preflight + print_package_status + if all_packages_exact; then + if install_boot_marker_matches_current_boot; then + warn "Package installation completed in the current boot; reboot is required" + info "CHECK_RESULT=REBOOT_REQUIRED" + elif driver_loaded_exact; then + info "CHECK_RESULT=PACKAGES_AND_DRIVER_PRESENT" + else + warn "Exact packages are installed but driver ${DRIVER_VERSION} is not loaded; reboot is required" + info "CHECK_RESULT=REBOOT_REQUIRED" + fi + else + info "CHECK_RESULT=READY_TO_APPLY" + fi +} + +run_apply() { + require_command apt-cache + require_command apt-get + require_command curl + require_command gpg + require_command grep + require_command sha256sum + require_command sudo + acquire_sudo + common_preflight + + if [[ -e /var/run/reboot-required ]]; then + if all_packages_exact && ! driver_loaded_exact; then + warn "A reboot is required before runtime setup can continue" + exit "$REBOOT_REQUIRED_EXIT" + fi + fatal "An unrelated reboot is already pending" + fi + + if ! all_packages_exact; then + assert_no_package_mismatches + install_packages + ensure_docker_group + sudo systemctl enable containerd.service docker.service nvidia-cdi-refresh.path + write_install_boot_marker + info "APPLY_RESULT=REBOOT_REQUIRED" + info "Run: sudo reboot" + exit "$REBOOT_REQUIRED_EXIT" + fi + + if install_boot_marker_matches_current_boot; then + warn "Package installation completed in the current boot" + info "APPLY_RESULT=REBOOT_REQUIRED" + info "Run: sudo reboot" + exit "$REBOOT_REQUIRED_EXIT" + fi + + driver_loaded_exact || { + warn "Pinned packages are installed but driver ${DRIVER_VERSION} is not loaded" + info "APPLY_RESULT=REBOOT_REQUIRED" + info "Run: sudo reboot" + exit "$REBOOT_REQUIRED_EXIT" + } + + finish_runtime + verify_apply_state + rm -f "$INSTALL_BOOT_MARKER" + info "APPLY_RESULT=COMPLETE" +} + +run_verify() { + common_preflight + require_command docker + require_command nvidia-ctk + require_command nvidia-smi + all_packages_exact || fatal "Pinned prerequisite packages are incomplete; run --apply" + driver_loaded_exact || fatal "Pinned driver is not loaded; reboot, then run --apply" + verify_host +} + +main() { + if (($# != 1)) || ! is_valid_mode "${1:-}"; then + usage >&2 + exit 2 + fi + MODE=$1 + setup_log + trap 'on_error "$LINENO"' ERR + case "$MODE" in + --check) run_check ;; + --apply) run_apply ;; + --verify) run_verify ;; + esac +} + +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + main "$@" +fi diff --git a/test/install-express-prompt.test.ts b/test/install-express-prompt.test.ts index 65ba31f39f..03a7142357 100644 --- a/test/install-express-prompt.test.ts +++ b/test/install-express-prompt.test.ts @@ -218,6 +218,9 @@ detect_express_platform /Express install will configure managed local vLLM with NVIDIA Nemotron 3 Ultra 550B/, ); expect(output).toMatch(/approximately 352 GB model/); + expect(output).toMatch( + /installs missing pinned driver, Docker, and NVIDIA Container Toolkit packages/, + ); expect(output).toMatch(/Using express install for DGX Station/); expect(output).toMatch( /RESULT NON_INTERACTIVE=1 SUDO_MODE=prompt PROVIDER=install-vllm MODEL=nvidia\/nemotron-3-ultra-550b-a55b VLLM_MODEL=nemotron-3-ultra-550b-a55b POLICY=suggested YES=1 SANDBOX=my-assistant/, diff --git a/test/install-station-host-preparation.test.ts b/test/install-station-host-preparation.test.ts new file mode 100644 index 0000000000..f2e193c1ea --- /dev/null +++ b/test/install-station-host-preparation.test.ts @@ -0,0 +1,324 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { INSTALLER_PAYLOAD, TEST_SYSTEM_PATH } from "./helpers/installer-sourced-env"; + +const REPO_ROOT = path.resolve(import.meta.dirname, ".."); +const STATION_PREPARE = path.join(REPO_ROOT, "scripts", "prepare-dgx-station-host.sh"); + +function runSourced(script: string, body: string, extraEnv: Record = {}) { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-station-host-")); + const result = spawnSync( + "bash", + ["--noprofile", "--norc", "-c", `source "$SCRIPT_UNDER_TEST" >/dev/null\n${body}`], + { + cwd: REPO_ROOT, + encoding: "utf-8", + env: { + HOME: home, + PATH: TEST_SYSTEM_PATH, + SCRIPT_UNDER_TEST: script, + ...extraEnv, + }, + }, + ); + return { home, result, output: `${result.stdout}${result.stderr}` }; +} + +describe("DGX Station host preparation", () => { + it.each([ + ["", "missing"], + ["5:29.6.1-1~ubuntu.24.04~noble", "exact"], + ["5:30.0.0-1~ubuntu.24.04~noble", "mismatch"], + ])("classifies an installed package version as %s -> %s", (actual, expected) => { + const { result, output } = runSourced( + STATION_PREPARE, + ` +installed_version() { + if [[ "$1" == "docker-ce" ]]; then printf '%s' "$PACKAGE_ACTUAL"; fi +} +package_state 'docker-ce=5:29.6.1-1~ubuntu.24.04~noble' +`, + { PACKAGE_ACTUAL: actual }, + ); + + expect(result.status, output).toBe(0); + expect(result.stdout.trim()).toBe(expected); + }); + + it("refuses to change an existing mismatched prerequisite", () => { + const { result, output } = runSourced( + STATION_PREPARE, + ` +installed_version() { + if [[ "$1" == "docker-ce" ]]; then printf '5:30.0.0-1~ubuntu.24.04~noble'; fi +} +assert_no_package_mismatches +`, + ); + + expect(result.status, output).not.toBe(0); + expect(output).toMatch(/docker-ce status=mismatch/); + expect(output).toMatch(/refusing to upgrade or downgrade them automatically/); + }); + + it("reuses exact packages and proceeds directly to runtime probes", () => { + const { result, output } = runSourced( + STATION_PREPARE, + ` +common_preflight() { :; } +require_command() { :; } +acquire_sudo() { :; } +all_packages_exact() { return 0; } +install_boot_marker_matches_current_boot() { return 1; } +driver_loaded_exact() { return 0; } +install_packages() { printf 'INSTALL_PACKAGES\n'; } +finish_runtime() { printf 'FINISH_RUNTIME\n'; } +verify_apply_state() { printf 'VERIFY_APPLY_STATE\n'; } +run_apply +`, + ); + + expect(result.status, output).toBe(0); + expect(output).toContain("FINISH_RUNTIME"); + expect(output).toContain("VERIFY_APPLY_STATE"); + expect(output).not.toContain("INSTALL_PACKAGES"); + expect(output).toContain("APPLY_RESULT=COMPLETE"); + }); + + it("installs only missing packages and returns the reboot-required contract", () => { + const { result, output } = runSourced( + STATION_PREPARE, + ` +common_preflight() { :; } +require_command() { :; } +acquire_sudo() { :; } +all_packages_exact() { return 1; } +assert_no_package_mismatches() { printf 'NO_MISMATCHES\n'; } +install_packages() { printf 'INSTALL_PACKAGES\n'; } +ensure_docker_group() { printf 'ENSURE_DOCKER_GROUP\n'; } +write_install_boot_marker() { printf 'WRITE_BOOT_MARKER\n'; } +sudo() { printf 'SUDO %s\n' "$*"; } +run_apply +`, + ); + + expect(result.status, output).toBe(10); + expect(output).toContain("NO_MISMATCHES"); + expect(output).toContain("INSTALL_PACKAGES"); + expect(output).toContain("ENSURE_DOCKER_GROUP"); + expect(output).toContain("WRITE_BOOT_MARKER"); + expect(output).toContain("APPLY_RESULT=REBOOT_REQUIRED"); + }); + + it("does not refresh CDI when the GPU launch probe already passes", () => { + const { result, output } = runSourced( + STATION_PREPARE, + ` +run_cdi_test_sudo() { printf 'CDI_TEST\n'; return 0; } +refresh_cdi() { printf 'REFRESH_CDI\n'; } +ensure_cdi_runtime +`, + ); + + expect(result.status, output).toBe(0); + expect(output).toContain("cdi_contract=pass_without_configuration_change"); + expect(output).not.toContain("REFRESH_CDI"); + }); + + it("refreshes CDI once when the initial GPU launch probe fails", () => { + const { result, output } = runSourced( + STATION_PREPARE, + ` +calls=0 +run_cdi_test_sudo() { + calls=$((calls + 1)) + printf 'CDI_TEST_%s\n' "$calls" + [[ "$calls" -gt 1 ]] +} +refresh_cdi() { printf 'REFRESH_CDI\n'; } +ensure_cdi_runtime +`, + ); + + expect(result.status, output).toBe(0); + expect(output).toContain("CDI_TEST_1"); + expect(output).toContain("REFRESH_CDI"); + expect(output).toContain("CDI_TEST_2"); + expect(output).toContain("cdi_contract=pass_after_refresh"); + }); + + it("ignores the installer process while still blocking a real vLLM workload", () => { + const selfOnly = runSourced( + STATION_PREPARE, + ` +ps() { + printf '%s %s bash bash /tmp/NemoClaw/scripts/prepare-dgx-station-host.sh --apply\n' "$$" "$PPID" + printf '%s 1 bash bash /tmp/NemoClaw/scripts/install.sh\n' "$PPID" +} +ss() { :; } +check_no_workloads +`, + ); + expect(selfOnly.result.status, selfOnly.output).toBe(0); + + const active = runSourced( + STATION_PREPARE, + ` +ps() { printf '999 1 python python -m vllm serve model\n'; } +ss() { :; } +check_no_workloads +`, + ); + expect(active.result.status, active.output).not.toBe(0); + expect(active.output).toMatch(/Agent or inference workload is active/); + }); + + it("uses sudo to inspect containers during apply until Docker group access is active", () => { + const { result, output } = runSourced( + STATION_PREPARE, + ` +MODE='--apply' +ps() { printf '%s %s bash bash prepare-dgx-station-host.sh --apply\n' "$$" "$PPID"; } +ss() { :; } +docker() { return 1; } +sudo() { + if [[ "$1" == "-n" ]]; then shift; fi + [[ "$*" == "docker ps -aq" ]] || return 1 +} +systemctl() { return 0; } +check_no_workloads +`, + { PATH: `${path.dirname(process.execPath)}:${TEST_SYSTEM_PATH}` }, + ); + + expect(result.status, output).toBe(0); + expect(output).toContain("docker_access=sudo_until_group_membership_is_active"); + expect(output).toContain("workloads=none"); + }); +}); + +describe("DGX Station express host integration", () => { + it("runs Station preparation before the generic Docker bootstrap", () => { + const { result, output } = runSourced( + INSTALLER_PAYLOAD, + ` +maybe_offer_express_install() { printf 'SELECT_EXPRESS\n'; _SELECTED_EXPRESS_PLATFORM='DGX Station'; } +ensure_station_express_host() { printf 'PREPARE_STATION\n'; } +ensure_docker() { printf 'ENSURE_DOCKER\n'; } +ensure_openshell_build_deps() { printf 'ENSURE_BUILD_DEPS\n'; } +prepare_installer_host +`, + ); + + expect(result.status, output).toBe(0); + expect(result.stdout.trim().split("\n")).toEqual([ + "SELECT_EXPRESS", + "PREPARE_STATION", + "ENSURE_DOCKER", + "ENSURE_BUILD_DEPS", + ]); + }); + + it("persists the selected model when host preparation requires a reboot", () => { + const { home, result, output } = runSourced( + INSTALLER_PAYLOAD, + ` +_SELECTED_EXPRESS_PLATFORM='DGX Station' +NEMOCLAW_VLLM_MODEL='nemotron-3-ultra-550b-a55b' +run_station_host_preparation() { return 10; } +ensure_station_express_host +`, + ); + const stateFile = path.join(home, ".nemoclaw", "station-express-resume"); + + expect(result.status, output).toBe(10); + expect(fs.readFileSync(stateFile, "utf-8")).toBe("nemotron-3-ultra-550b-a55b\n"); + expect(fs.statSync(stateFile).mode & 0o777).toBe(0o600); + expect(output).toMatch(/rerun the same NemoClaw installer command/); + }); + + it("resumes the accepted Station recipe without another prompt", () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-station-resume-")); + const stateDir = path.join(home, ".nemoclaw"); + fs.mkdirSync(stateDir, { mode: 0o700 }); + fs.writeFileSync( + path.join(stateDir, "station-express-resume"), + "nemotron-3-ultra-550b-a55b\n", + { mode: 0o600 }, + ); + const result = spawnSync( + "bash", + [ + "--noprofile", + "--norc", + "-c", + ` +source "$INSTALLER_UNDER_TEST" >/dev/null +detect_express_platform() { printf 'DGX Station'; } +NON_INTERACTIVE='' +NEMOCLAW_PROVIDER='' +NEMOCLAW_NO_EXPRESS='' +maybe_offer_express_install +printf 'RESULT PLATFORM=%s PROVIDER=%s MODEL=%s VLLM_MODEL=%s\n' \ + "$_SELECTED_EXPRESS_PLATFORM" "$NEMOCLAW_PROVIDER" "\${NEMOCLAW_MODEL:-}" "$NEMOCLAW_VLLM_MODEL" +`, + ], + { + cwd: REPO_ROOT, + encoding: "utf-8", + env: { + HOME: home, + PATH: TEST_SYSTEM_PATH, + INSTALLER_UNDER_TEST: INSTALLER_PAYLOAD, + }, + }, + ); + const output = `${result.stdout}${result.stderr}`; + + expect(result.status, output).toBe(0); + expect(output).toMatch(/Resuming the accepted express install/); + expect(output).not.toMatch(/Run express install with these settings/); + expect(output).toMatch( + /RESULT PLATFORM=DGX Station PROVIDER=install-vllm MODEL=nvidia\/nemotron-3-ultra-550b-a55b VLLM_MODEL=nemotron-3-ultra-550b-a55b/, + ); + }); + + it("rejects a multi-line Station resume state", () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-station-resume-invalid-")); + const stateDir = path.join(home, ".nemoclaw"); + fs.mkdirSync(stateDir, { mode: 0o700 }); + fs.writeFileSync( + path.join(stateDir, "station-express-resume"), + "nemotron-3-ultra-550b-a55b\nunexpected\n", + { mode: 0o600 }, + ); + const result = spawnSync( + "bash", + [ + "--noprofile", + "--norc", + "-c", + `source "$INSTALLER_UNDER_TEST" >/dev/null; load_station_express_resume`, + ], + { + cwd: REPO_ROOT, + encoding: "utf-8", + env: { + HOME: home, + PATH: TEST_SYSTEM_PATH, + INSTALLER_UNDER_TEST: INSTALLER_PAYLOAD, + }, + }, + ); + const output = `${result.stdout}${result.stderr}`; + + expect(result.status, output).not.toBe(0); + expect(output).toMatch(/resume state is invalid/); + }); +}); diff --git a/vitest.config.ts b/vitest.config.ts index bd368fa6b0..6d5509c6b1 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -124,6 +124,7 @@ export default defineConfig({ "test/install-clone-ref.test.ts", "test/install-preflight.test.ts", "test/install-preflight-docker-bootstrap.test.ts", + "test/install-station-host-preparation.test.ts", "test/install-openshell-version-check.test.ts", ], }, @@ -142,6 +143,7 @@ export default defineConfig({ "test/install-clone-ref.test.ts", "test/install-preflight.test.ts", "test/install-preflight-docker-bootstrap.test.ts", + "test/install-station-host-preparation.test.ts", "test/install-openshell-version-check.test.ts", ], // Slow tests that spawn real bash install.sh processes. Explicit From 35c26d3d55f4055091398df85741a2717a5b2fd8 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 15 Jul 2026 21:18:48 -0700 Subject: [PATCH 02/74] fix(installer): harden station host preparation Signed-off-by: Aaron Erickson --- docs/get-started/prerequisites.mdx | 5 + docs/get-started/quickstart.mdx | 2 + scripts/install.sh | 9 +- scripts/prepare-dgx-station-host.sh | 72 ++++++-- test/install-express-prompt.test.ts | 1 + test/install-station-host-preparation.test.ts | 173 ++++++++++++++++++ 6 files changed, 246 insertions(+), 16 deletions(-) diff --git a/docs/get-started/prerequisites.mdx b/docs/get-started/prerequisites.mdx index aee76ff15b..99b85dc069 100644 --- a/docs/get-started/prerequisites.mdx +++ b/docs/get-started/prerequisites.mdx @@ -48,6 +48,11 @@ It does not install a host CUDA toolkit or Docker Compose. If an existing prerequisite has a different version, preparation stops instead of automatically upgrading or downgrading it. After installing missing pinned packages, the installer exits with status `10`; reboot, sign in, and rerun the same installer command to resume express setup. + +DGX Station remains Deferred. +Full NemoClaw onboarding with this recipe has not completed end-to-end validation on physical DGX Station hardware. + + NemoClaw needs Docker access. On personal Linux development machines, adding your user to the `docker` group is the standard way to run Docker without sudo. diff --git a/docs/get-started/quickstart.mdx b/docs/get-started/quickstart.mdx index d1d61ba87e..d0bd7bb8b3 100644 --- a/docs/get-started/quickstart.mdx +++ b/docs/get-started/quickstart.mdx @@ -123,6 +123,7 @@ Use these details when your first-run path needs more control. On DGX Station, accepting the express prompt selects the pinned `nemotron-3-ultra-550b-a55b` managed-vLLM recipe and completes onboarding without more provider, model, policy, or sandbox-name choices. The Station path probes the validated driver, Docker, Buildx, and NVIDIA Container Toolkit versions, reuses exact matches, and installs missing pinned packages without additional choices. After it installs missing pinned packages, the installer exits with status `10` at the validated reboot boundary; reboot, sign in, and rerun the same installer command to resume the accepted recipe without another prompt. + This automation does not change Station's Deferred support status; physical end-to-end validation remains open. Pass `--station-deepseek` to use DeepSeek V4 Flash for a Station demo instead. Refer to [Platform Support](../reference/platform-support) and [Choose an Inference Provider](../inference/learn-and-choose/choose-inference-provider) for the current platform behavior. @@ -195,6 +196,7 @@ Use these details when your first-run path needs more control. Before onboarding, the Station path checks for NVIDIA open driver `610.43.02`, Docker CE `29.6.1` with Buildx, and NVIDIA Container Toolkit `1.19.1`. It reuses exact versions, installs missing pinned packages, and refuses to automatically upgrade or downgrade an existing mismatched version. Installing missing pinned packages exits with status `10` for a reboot; after you sign in and rerun the same installer command, the accepted express recipe resumes without another prompt. + This automation does not change Station's Deferred support status; physical end-to-end validation remains open. To select DeepSeek V4 Flash while retaining the one-confirmation Station express flow, run `curl -fsSL https://www.nvidia.com/nemoclaw.sh | bash -s -- --station-deepseek`. Unless `NEMOCLAW_POLICY_TIER` is set, express install applies policy in `suggested` mode with the `balanced` tier, including the base sandbox policy and supported package, model, web-search, and local-inference presets. Express install uses `my-assistant` as the sandbox name across all platforms unless `NEMOCLAW_SANDBOX_NAME` is set. diff --git a/scripts/install.sh b/scripts/install.sh index 8b21fe6069..32469a36f6 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -3090,6 +3090,7 @@ describe_express_install() { inference_disclosure="Managed vLLM pulls the pinned Station image and approximately 352 GB model, then runs a local inference container." fi printf " Station host setup reuses exact prerequisite versions, installs missing pinned driver, Docker, and NVIDIA Container Toolkit packages, and may require one reboot.\n" + printf " DGX Station remains Deferred; this recipe has not completed end-to-end validation on physical hardware.\n" sandbox_summary="${NEMOCLAW_SANDBOX_NAME:-my-assistant}" ;; "Windows WSL") @@ -3143,6 +3144,10 @@ maybe_offer_express_install() { info "Detected ${platform}. Skipping express prompt (NEMOCLAW_NO_EXPRESS=1)." return 0 fi + if [ -n "${NEMOCLAW_PROVIDER:-}" ]; then + info "Detected ${platform}. Skipping express prompt (NEMOCLAW_PROVIDER=${NEMOCLAW_PROVIDER} already set)." + return 0 + fi if [ "$platform" = "DGX Station" ] && load_station_express_resume; then info "Detected DGX Station. Resuming the accepted express install after host preparation." activate_express_install "$platform" @@ -3152,10 +3157,6 @@ maybe_offer_express_install() { info "Detected ${platform}. Skipping express prompt (--non-interactive set)." return 0 fi - if [ -n "${NEMOCLAW_PROVIDER:-}" ]; then - info "Detected ${platform}. Skipping express prompt (NEMOCLAW_PROVIDER=${NEMOCLAW_PROVIDER} already set)." - return 0 - fi local reply="" if [ -t 0 ]; then info "Detected ${platform}." diff --git a/scripts/prepare-dgx-station-host.sh b/scripts/prepare-dgx-station-host.sh index 42b9d3e104..ad9faa05d3 100755 --- a/scripts/prepare-dgx-station-host.sh +++ b/scripts/prepare-dgx-station-host.sh @@ -11,6 +11,7 @@ readonly MIN_FREE_KIB=$((20 * 1024 * 1024)) readonly CUDA_KEYRING_URL="https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/sbsa/cuda-keyring_1.1-1_all.deb" readonly CUDA_KEYRING_SHA256="6ea7d2737648936820e85677177957a0f6521b840d98eb0bbae0a4f003fa7249" +readonly CUDA_KEYRING_PACKAGE_VERSION="1.1-1" readonly CUDA_KEY_FINGERPRINT="EB693B3035CD5710E231E123A4B469963BF863CC" # gitleaks:allow -- public NVIDIA signing-key fingerprint readonly DOCKER_KEY_URL="https://download.docker.com/linux/ubuntu/gpg" readonly DOCKER_KEY_SHA256="1500c1f56fa9e26b9b8f42452a553675796ade0807cdce11975eb98170b3a570" # gitleaks:allow -- public Docker GPG-key integrity pin @@ -39,6 +40,7 @@ readonly -a PACKAGE_SPECS=( MODE="" LOG_FILE="" +DOCKER_GROUP_ADDED=0 info() { printf '[station-prepare] %s %s\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$*" @@ -283,8 +285,7 @@ check_no_workloads() { elif systemctl is-active --quiet docker.service; then fatal "Docker is active but inaccessible to this login; start a new login session with docker-group membership" else - info "docker_daemon=inactive" - containers="" + fatal "Docker is installed but inactive, so existing container state cannot be verified safely; start Docker and rerun preparation" fi fi [[ -z "$containers" ]] || fatal "Existing Docker containers block host preparation: ${containers}" @@ -357,6 +358,42 @@ verify_key_fingerprint() { | grep -Fxq "$expected" || fatal "Expected signing-key fingerprint ${expected} was not found in ${path}" } +ensure_cuda_keyring() { + local cuda_deb=$1 actual verification + actual="$(installed_version cuda-keyring)" + if [[ -z "$actual" ]]; then + curl --fail --silent --show-error --location "$CUDA_KEYRING_URL" --output "$cuda_deb" + verify_file_sha256 "$cuda_deb" "$CUDA_KEYRING_SHA256" + sudo dpkg -i "$cuda_deb" + package_is_exact "cuda-keyring=${CUDA_KEYRING_PACKAGE_VERSION}" \ + || fatal "Installed cuda-keyring does not match ${CUDA_KEYRING_PACKAGE_VERSION}" + elif [[ "$actual" == "$CUDA_KEYRING_PACKAGE_VERSION" ]]; then + verification="$(dpkg -V cuda-keyring 2>&1)" \ + || fatal "Unable to verify the installed cuda-keyring package" + [[ -z "$verification" ]] || fatal "Installed cuda-keyring files differ from the package manifest: ${verification}" + info "cuda_keyring=exact version=${actual}" + else + fatal "Existing cuda-keyring version ${actual} differs from validated pin ${CUDA_KEYRING_PACKAGE_VERSION}; refusing to upgrade or downgrade it automatically" + fi + + sudo test ! -L /usr/share/keyrings/cuda-archive-keyring.gpg \ + || fatal "CUDA repository keyring must not be a symbolic link" + verify_key_fingerprint /usr/share/keyrings/cuda-archive-keyring.gpg "$CUDA_KEY_FINGERPRINT" +} + +install_exact_file_or_reuse() { + local source=$1 target=$2 mode=$3 label=$4 + sudo test ! -L "$target" || fatal "${label} must not be a symbolic link: ${target}" + if sudo test -e "$target"; then + sudo cmp -s "$source" "$target" \ + || fatal "Existing ${label} differs from the validated content; refusing to overwrite ${target}" + info "${label}=exact path=${target}" + return 0 + fi + sudo install -m "$mode" "$source" "$target" + info "${label}=installed path=${target}" +} + configure_repositories() { local tmp cuda_deb docker_asc docker_gpg docker_list tmp="$(mktemp -d)" @@ -366,21 +403,18 @@ configure_repositories() { docker_list="${tmp}/docker.list" info "Downloading and verifying official repository keys" - curl --fail --silent --show-error --location "$CUDA_KEYRING_URL" --output "$cuda_deb" - verify_file_sha256 "$cuda_deb" "$CUDA_KEYRING_SHA256" - sudo dpkg -i "$cuda_deb" - verify_key_fingerprint /usr/share/keyrings/cuda-archive-keyring.gpg "$CUDA_KEY_FINGERPRINT" + ensure_cuda_keyring "$cuda_deb" curl --fail --silent --show-error --location "$DOCKER_KEY_URL" --output "$docker_asc" verify_file_sha256 "$docker_asc" "$DOCKER_KEY_SHA256" verify_key_fingerprint "$docker_asc" "$DOCKER_KEY_FINGERPRINT" gpg --batch --yes --dearmor --output "$docker_gpg" "$docker_asc" sudo install -d -m 0755 /etc/apt/keyrings - sudo install -m 0644 "$docker_gpg" /etc/apt/keyrings/docker.gpg + install_exact_file_or_reuse "$docker_gpg" /etc/apt/keyrings/docker.gpg 0644 docker_repository_key printf '%s\n' \ 'deb [arch=arm64 signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu noble stable' \ >"$docker_list" - sudo install -m 0644 "$docker_list" /etc/apt/sources.list.d/docker.list + install_exact_file_or_reuse "$docker_list" /etc/apt/sources.list.d/docker.list 0644 docker_repository_source rm -rf "$tmp" info "repository_keys=verified" @@ -427,6 +461,7 @@ ensure_docker_group() { getent group docker >/dev/null 2>&1 || fatal "Docker group is missing after package installation" if ! id -nG "$user_name" | tr ' ' '\n' | grep -Fxq docker; then sudo usermod -aG docker "$user_name" + DOCKER_GROUP_ADDED=1 info "docker_group=added user=${user_name}; a new login is required" else info "docker_group=present user=${user_name}" @@ -436,9 +471,10 @@ ensure_docker_group() { refresh_cdi() { sudo systemctl enable --now nvidia-cdi-refresh.path if ! sudo systemctl restart nvidia-cdi-refresh.service; then - warn "Packaged CDI refresh failed; generating the exact fallback spec" - sudo install -d -m 0755 /etc/cdi - sudo nvidia-ctk cdi generate --output=/etc/cdi/nvidia.yaml + warn "Packaged CDI refresh failed; refusing to create a persistent manual CDI specification" + sudo systemctl status nvidia-cdi-refresh.service --no-pager || true + sudo journalctl -u nvidia-cdi-refresh.service --no-pager -n 50 || true + fatal "nvidia-cdi-refresh.service failed; fix the packaged service and rerun preparation" fi nvidia-ctk cdi list | grep -Fxq 'nvidia.com/gpu=all' || fatal "CDI device nvidia.com/gpu=all is unavailable" info "cdi=nvidia.com/gpu=all" @@ -579,7 +615,9 @@ run_check() { run_apply() { require_command apt-cache require_command apt-get + require_command cmp require_command curl + require_command dpkg require_command gpg require_command grep require_command sha256sum @@ -622,6 +660,12 @@ run_apply() { finish_runtime verify_apply_state + if ((DOCKER_GROUP_ADDED == 1)); then + warn "Docker group membership was added and requires a new login before onboarding" + info "APPLY_RESULT=REBOOT_REQUIRED" + info "Run: sudo reboot" + exit "$REBOOT_REQUIRED_EXIT" + fi rm -f "$INSTALL_BOOT_MARKER" info "APPLY_RESULT=COMPLETE" } @@ -642,7 +686,11 @@ main() { exit 2 fi MODE=$1 - setup_log + if [[ "$MODE" == "--apply" ]]; then + setup_log + else + info "version=${SCRIPT_VERSION} mode=${MODE} log=disabled_read_only" + fi trap 'on_error "$LINENO"' ERR case "$MODE" in --check) run_check ;; diff --git a/test/install-express-prompt.test.ts b/test/install-express-prompt.test.ts index 03a7142357..497a116d6f 100644 --- a/test/install-express-prompt.test.ts +++ b/test/install-express-prompt.test.ts @@ -221,6 +221,7 @@ detect_express_platform expect(output).toMatch( /installs missing pinned driver, Docker, and NVIDIA Container Toolkit packages/, ); + expect(output).toMatch(/DGX Station remains Deferred/); expect(output).toMatch(/Using express install for DGX Station/); expect(output).toMatch( /RESULT NON_INTERACTIVE=1 SUDO_MODE=prompt PROVIDER=install-vllm MODEL=nvidia\/nemotron-3-ultra-550b-a55b VLLM_MODEL=nemotron-3-ultra-550b-a55b POLICY=suggested YES=1 SANDBOX=my-assistant/, diff --git a/test/install-station-host-preparation.test.ts b/test/install-station-host-preparation.test.ts index f2e193c1ea..4555f2e07d 100644 --- a/test/install-station-host-preparation.test.ts +++ b/test/install-station-host-preparation.test.ts @@ -25,6 +25,8 @@ function runSourced(script: string, body: string, extraEnv: Record { + const { result, output } = runSourced( + STATION_PREPARE, + ` +MODE='--apply' +ps() { printf '%s %s bash bash prepare-dgx-station-host.sh --apply\n' "$$" "$PPID"; } +ss() { :; } +docker() { return 1; } +sudo() { return 1; } +systemctl() { return 1; } +check_no_workloads +`, + { PATH: `${path.dirname(process.execPath)}:${TEST_SYSTEM_PATH}` }, + ); + + expect(result.status, output).not.toBe(0); + expect(output).toMatch(/container state cannot be verified safely/); + }); + + it("refuses an installed CUDA keyring version that differs from the pin", () => { + const { result, output } = runSourced( + STATION_PREPARE, + ` +installed_version() { printf '2.0-1'; } +ensure_cuda_keyring "$HOME/cuda-keyring.deb" +`, + ); + + expect(result.status, output).not.toBe(0); + expect(output).toMatch(/refusing to upgrade or downgrade it automatically/); + }); + + it("reuses an exact verified CUDA keyring without downloading it again", () => { + const { result, output } = runSourced( + STATION_PREPARE, + ` +installed_version() { printf '1.1-1'; } +dpkg() { :; } +curl() { printf 'DOWNLOAD\n'; } +sudo() { "$@"; } +verify_key_fingerprint() { printf 'VERIFIED_FINGERPRINT\n'; } +ensure_cuda_keyring "$HOME/cuda-keyring.deb" +`, + ); + + expect(result.status, output).toBe(0); + expect(output).toContain("cuda_keyring=exact version=1.1-1"); + expect(output).toContain("VERIFIED_FINGERPRINT"); + expect(output).not.toContain("DOWNLOAD"); + }); + + it("reuses exact repository files and refuses to overwrite mismatched content", () => { + const { result, output } = runSourced( + STATION_PREPARE, + ` +printf 'validated\n' >"$HOME/source" +cp "$HOME/source" "$HOME/target" +sudo() { "$@"; } +install_exact_file_or_reuse "$HOME/source" "$HOME/target" 0644 test_repository_file +printf 'modified\n' >"$HOME/target" +install_exact_file_or_reuse "$HOME/source" "$HOME/target" 0644 test_repository_file +`, + ); + + expect(result.status, output).not.toBe(0); + expect(output).toMatch(/test_repository_file=exact/); + expect(output).toMatch(/refusing to overwrite/); + }); + + it("requires a new login after adding Docker group membership", () => { + const { result, output } = runSourced( + STATION_PREPARE, + ` +common_preflight() { :; } +require_command() { :; } +acquire_sudo() { :; } +all_packages_exact() { return 0; } +install_boot_marker_matches_current_boot() { return 1; } +driver_loaded_exact() { return 0; } +finish_runtime() { DOCKER_GROUP_ADDED=1; printf 'FINISH_RUNTIME\n'; } +verify_apply_state() { printf 'VERIFY_APPLY_STATE\n'; } +run_apply +`, + ); + + expect(result.status, output).toBe(10); + expect(output).toContain("VERIFY_APPLY_STATE"); + expect(output).toContain("APPLY_RESULT=REBOOT_REQUIRED"); + expect(output).toMatch(/new login before onboarding/); + }); + + it("diagnoses packaged CDI refresh failure without writing a manual specification", () => { + const { result, output } = runSourced( + STATION_PREPARE, + ` +sudo() { + printf 'SUDO %s\n' "$*" + [[ "$*" != "systemctl restart nvidia-cdi-refresh.service" ]] +} +refresh_cdi +`, + ); + + expect(result.status, output).not.toBe(0); + expect(output).toContain("systemctl status nvidia-cdi-refresh.service --no-pager"); + expect(output).toContain("journalctl -u nvidia-cdi-refresh.service --no-pager -n 50"); + expect(output).toMatch(/refusing to create a persistent manual CDI specification/); + expect(output).not.toContain("cdi generate"); + }); + + it("accepts a successful packaged CDI refresh", () => { + const { result, output } = runSourced( + STATION_PREPARE, + ` +sudo() { printf 'SUDO %s\n' "$*"; } +nvidia-ctk() { + [[ "$*" == "cdi list" ]] && printf 'nvidia.com/gpu=all\n' +} +refresh_cdi +`, + ); + + expect(result.status, output).toBe(0); + expect(output).toContain("systemctl restart nvidia-cdi-refresh.service"); + expect(output).toContain("cdi=nvidia.com/gpu=all"); + expect(output).not.toContain("systemctl status"); + }); + + it.each(["--check", "--verify"])("keeps %s read-only under HOME", (mode) => { + const { home, result, output } = runSourced( + STATION_PREPARE, + ` +run_check() { :; } +run_verify() { :; } +main "$READ_MODE" +`, + { READ_MODE: mode }, + ); + + expect(result.status, output).toBe(0); + expect(output).toContain("log=disabled_read_only"); + expect(fs.existsSync(path.join(home, "station-bootstrap-logs"))).toBe(false); + }); }); describe("DGX Station express host integration", () => { @@ -277,6 +423,8 @@ printf 'RESULT PLATFORM=%s PROVIDER=%s MODEL=%s VLLM_MODEL=%s\n' \ PATH: TEST_SYSTEM_PATH, INSTALLER_UNDER_TEST: INSTALLER_PAYLOAD, }, + timeout: 15_000, + killSignal: "SIGKILL", }, ); const output = `${result.stdout}${result.stderr}`; @@ -289,6 +437,29 @@ printf 'RESULT PLATFORM=%s PROVIDER=%s MODEL=%s VLLM_MODEL=%s\n' \ ); }); + it("preserves an explicit provider even when Station resume state exists", () => { + const { result, output } = runSourced( + INSTALLER_PAYLOAD, + ` +mkdir -p "$HOME/.nemoclaw" +chmod 0700 "$HOME/.nemoclaw" +printf 'nemotron-3-ultra-550b-a55b\n' >"$HOME/.nemoclaw/station-express-resume" +chmod 0600 "$HOME/.nemoclaw/station-express-resume" +detect_express_platform() { printf 'DGX Station'; } +NON_INTERACTIVE='' +NEMOCLAW_PROVIDER='openai' +NEMOCLAW_NO_EXPRESS='' +maybe_offer_express_install +printf 'RESULT PROVIDER=%s\n' "$NEMOCLAW_PROVIDER" +`, + ); + + expect(result.status, output).toBe(0); + expect(output).toContain("NEMOCLAW_PROVIDER=openai already set"); + expect(output).toContain("RESULT PROVIDER=openai"); + expect(output).not.toContain("Resuming the accepted express install"); + }); + it("rejects a multi-line Station resume state", () => { const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-station-resume-invalid-")); const stateDir = path.join(home, ".nemoclaw"); @@ -314,6 +485,8 @@ printf 'RESULT PLATFORM=%s PROVIDER=%s MODEL=%s VLLM_MODEL=%s\n' \ PATH: TEST_SYSTEM_PATH, INSTALLER_UNDER_TEST: INSTALLER_PAYLOAD, }, + timeout: 15_000, + killSignal: "SIGKILL", }, ); const output = `${result.stdout}${result.stderr}`; From 9763bffd230401d48d8a33668a4dd03b934ddf3b Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 15 Jul 2026 21:21:55 -0700 Subject: [PATCH 03/74] test(installer): cover station resume symlinks Signed-off-by: Aaron Erickson --- test/install-station-host-preparation.test.ts | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/test/install-station-host-preparation.test.ts b/test/install-station-host-preparation.test.ts index 4555f2e07d..fe060fef46 100644 --- a/test/install-station-host-preparation.test.ts +++ b/test/install-station-host-preparation.test.ts @@ -389,6 +389,49 @@ ensure_station_express_host expect(output).toMatch(/rerun the same NemoClaw installer command/); }); + it("rejects a resume-state symlink without loading its target", () => { + const { home, result, output } = runSourced( + INSTALLER_PAYLOAD, + ` +mkdir -p "$HOME/.nemoclaw" +chmod 0700 "$HOME/.nemoclaw" +printf 'deepseek-v4-flash\n' >"$HOME/resume-target" +ln -s "$HOME/resume-target" "$HOME/.nemoclaw/station-express-resume" +load_station_express_resume +`, + ); + const target = path.join(home, "resume-target"); + const stateFile = path.join(home, ".nemoclaw", "station-express-resume"); + + expect(result.status, output).toBe(1); + expect(output).toMatch(/Refusing symbolic link in NemoClaw state path/); + expect(fs.readFileSync(target, "utf-8")).toBe("deepseek-v4-flash\n"); + expect(fs.lstatSync(stateFile).isSymbolicLink()).toBe(true); + }); + + it("rejects a resume-state symlink without modifying its target", () => { + const { home, result, output } = runSourced( + INSTALLER_PAYLOAD, + ` +mkdir -p "$HOME/.nemoclaw" +chmod 0700 "$HOME/.nemoclaw" +printf 'preserve-this-target\n' >"$HOME/resume-target" +ln -s "$HOME/resume-target" "$HOME/.nemoclaw/station-express-resume" +_SELECTED_EXPRESS_PLATFORM='DGX Station' +NEMOCLAW_VLLM_MODEL='nemotron-3-ultra-550b-a55b' +run_station_host_preparation() { return 10; } +ensure_station_express_host +`, + ); + const target = path.join(home, "resume-target"); + const stateFile = path.join(home, ".nemoclaw", "station-express-resume"); + + expect(result.status, output).toBe(1); + expect(output).toMatch(/Refusing symbolic link in NemoClaw state path/); + expect(fs.readFileSync(target, "utf-8")).toBe("preserve-this-target\n"); + expect(fs.lstatSync(stateFile).isSymbolicLink()).toBe(true); + }); + it("resumes the accepted Station recipe without another prompt", () => { const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-station-resume-")); const stateDir = path.join(home, ".nemoclaw"); From 8e5fb909b9d29115e58ff4d9d5d8f1c1b21a1dbe Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 15 Jul 2026 21:31:17 -0700 Subject: [PATCH 04/74] test(installer): cover station resume boundaries Signed-off-by: Aaron Erickson --- test/install-station-host-preparation.test.ts | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/test/install-station-host-preparation.test.ts b/test/install-station-host-preparation.test.ts index fe060fef46..5814d99b35 100644 --- a/test/install-station-host-preparation.test.ts +++ b/test/install-station-host-preparation.test.ts @@ -347,6 +347,22 @@ main "$READ_MODE" expect(output).toContain("log=disabled_read_only"); expect(fs.existsSync(path.join(home, "station-bootstrap-logs"))).toBe(false); }); + + it("fails verification when exact packages are present but the driver is not loaded", () => { + const { result, output } = runSourced( + STATION_PREPARE, + ` +common_preflight() { :; } +require_command() { :; } +all_packages_exact() { return 0; } +driver_loaded_exact() { return 1; } +run_verify +`, + ); + + expect(result.status, output).not.toBe(0); + expect(output).toMatch(/Pinned driver is not loaded/); + }); }); describe("DGX Station express host integration", () => { @@ -503,6 +519,31 @@ printf 'RESULT PROVIDER=%s\n' "$NEMOCLAW_PROVIDER" expect(output).not.toContain("Resuming the accepted express install"); }); + it("does not load Station resume state on DGX Spark", () => { + const { result, output } = runSourced( + INSTALLER_PAYLOAD, + ` +mkdir -p "$HOME/.nemoclaw" +chmod 0700 "$HOME/.nemoclaw" +printf 'nemotron-3-ultra-550b-a55b\n' >"$HOME/.nemoclaw/station-express-resume" +chmod 0600 "$HOME/.nemoclaw/station-express-resume" +detect_express_platform() { printf 'DGX Spark'; } +NON_INTERACTIVE='1' +NEMOCLAW_PROVIDER='' +NEMOCLAW_NO_EXPRESS='' +NEMOCLAW_VLLM_MODEL='' +maybe_offer_express_install +printf 'RESULT MODEL=%s\n' "$NEMOCLAW_VLLM_MODEL" +`, + ); + + expect(result.status, output).toBe(0); + expect(output).toContain("Detected DGX Spark. Skipping express prompt (--non-interactive set)"); + expect(output).toContain("RESULT MODEL="); + expect(output).not.toContain("Resuming the accepted express install"); + expect(output).not.toContain("nemotron-3-ultra-550b-a55b"); + }); + it("rejects a multi-line Station resume state", () => { const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-station-resume-invalid-")); const stateDir = path.join(home, ".nemoclaw"); From 34ac3fbd3f1bcc2a15ec036dabe233bd04010928 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 15 Jul 2026 21:39:10 -0700 Subject: [PATCH 05/74] fix(installer): secure station bootstrap state Signed-off-by: Aaron Erickson --- scripts/install.sh | 2 + scripts/prepare-dgx-station-host.sh | 10 ++++ test/install-station-host-preparation.test.ts | 52 +++++++++++++++++++ 3 files changed, 64 insertions(+) diff --git a/scripts/install.sh b/scripts/install.sh index 32469a36f6..0f100e1ab9 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -3051,6 +3051,8 @@ ensure_station_express_host() { prepare_installer_host() { maybe_offer_express_install + # Intentional ordering: Station preparation owns the reboot boundary before + # generic Docker bootstrap; ensure_station_express_host is a no-op elsewhere. ensure_station_express_host ensure_docker ensure_openshell_build_deps diff --git a/scripts/prepare-dgx-station-host.sh b/scripts/prepare-dgx-station-host.sh index ad9faa05d3..93573ed9bf 100755 --- a/scripts/prepare-dgx-station-host.sh +++ b/scripts/prepare-dgx-station-host.sh @@ -299,14 +299,24 @@ driver_loaded_exact() { [[ "$loaded" == "$DRIVER_VERSION" ]] } +assert_station_state_dir_safe() { + local path + for path in "${HOME}/.local" "${HOME}/.local/state" "$STATE_DIR"; do + [[ ! -L "$path" ]] || fatal "Refusing symbolic link in Station bootstrap state path: ${path}" + done +} + write_install_boot_marker() { + assert_station_state_dir_safe mkdir -p "$STATE_DIR" + assert_station_state_dir_safe chmod 0700 "$STATE_DIR" cp /proc/sys/kernel/random/boot_id "$INSTALL_BOOT_MARKER" chmod 0600 "$INSTALL_BOOT_MARKER" } install_boot_marker_matches_current_boot() { + assert_station_state_dir_safe [[ -r "$INSTALL_BOOT_MARKER" ]] || return 1 [[ "$(tr -d '[:space:]' <"$INSTALL_BOOT_MARKER")" == "$(tr -d '[:space:]' = {}) { const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-station-host-")); @@ -33,6 +37,22 @@ function runSourced(script: string, body: string, extraEnv: Record { + it("keeps documented Station pins and Deferred status aligned", () => { + const helper = fs.readFileSync(STATION_PREPARE, "utf-8"); + const docs = STATION_DOCS.map((doc) => fs.readFileSync(doc, "utf-8")); + const pinnedValues = ["DRIVER_VERSION", "DOCKER_VERSION", "TOOLKIT_VERSION"].map((name) => { + const value = helper.match(new RegExp(`readonly ${name}="([^"]+)"`))?.[1]; + expect(value, `${name} must remain declared in the Station helper`).toBeTruthy(); + return value as string; + }); + + for (const doc of docs) { + for (const version of pinnedValues) expect(doc).toContain(version); + expect(doc).toMatch(/(?:DGX )?Station(?: remains|'s) Deferred/); + expect(doc).toMatch(/physical (?:DGX Station )?hardware|physical end-to-end validation/); + } + }); + it.each([ ["", "missing"], ["5:29.6.1-1~ubuntu.24.04~noble", "exact"], @@ -363,6 +383,21 @@ run_verify expect(result.status, output).not.toBe(0); expect(output).toMatch(/Pinned driver is not loaded/); }); + + it("rejects a symlinked Station bootstrap state directory", () => { + const { home, result, output } = runSourced( + STATION_PREPARE, + ` +mkdir -p "$HOME/.local/state" "$HOME/redirect-target" +ln -s "$HOME/redirect-target" "$HOME/.local/state/station-bootstrap" +write_install_boot_marker +`, + ); + + expect(result.status, output).not.toBe(0); + expect(output).toMatch(/Refusing symbolic link in Station bootstrap state path/); + expect(fs.existsSync(path.join(home, "redirect-target", "install-boot-id"))).toBe(false); + }); }); describe("DGX Station express host integration", () => { @@ -387,6 +422,23 @@ prepare_installer_host ]); }); + it("skips Station preparation before Docker bootstrap on non-Station platforms", () => { + const { result, output } = runSourced( + INSTALLER_PAYLOAD, + ` +maybe_offer_express_install() { _SELECTED_EXPRESS_PLATFORM='DGX Spark'; } +run_station_host_preparation() { printf 'PREPARE_STATION\n'; } +ensure_docker() { printf 'ENSURE_DOCKER\n'; } +ensure_openshell_build_deps() { printf 'ENSURE_BUILD_DEPS\n'; } +prepare_installer_host +`, + ); + + expect(result.status, output).toBe(0); + expect(output).not.toContain("PREPARE_STATION"); + expect(result.stdout.trim().split("\n")).toEqual(["ENSURE_DOCKER", "ENSURE_BUILD_DEPS"]); + }); + it("persists the selected model when host preparation requires a reboot", () => { const { home, result, output } = runSourced( INSTALLER_PAYLOAD, From c56f6c52e243df8d2b52a0dc102f418b59d4303f Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 15 Jul 2026 21:46:48 -0700 Subject: [PATCH 06/74] fix(installer): recheck workloads before runtime changes Signed-off-by: Aaron Erickson --- scripts/prepare-dgx-station-host.sh | 1 + test/install-station-host-preparation.test.ts | 26 ++++++++++++++++++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/scripts/prepare-dgx-station-host.sh b/scripts/prepare-dgx-station-host.sh index 93573ed9bf..6ee446a436 100755 --- a/scripts/prepare-dgx-station-host.sh +++ b/scripts/prepare-dgx-station-host.sh @@ -535,6 +535,7 @@ configure_docker_runtime_if_needed() { sudo touch "${backup_dir}/daemon.json.absent" sudo chmod 0600 "${backup_dir}/daemon.json.absent" fi + check_no_workloads sudo nvidia-ctk runtime configure --runtime=docker sudo systemctl restart docker.service run_gpus_test_sudo || fatal "Docker --gpus all still fails after NVIDIA runtime registration" diff --git a/test/install-station-host-preparation.test.ts b/test/install-station-host-preparation.test.ts index b195eb8a21..73654a6d06 100644 --- a/test/install-station-host-preparation.test.ts +++ b/test/install-station-host-preparation.test.ts @@ -334,6 +334,27 @@ refresh_cdi expect(output).not.toContain("cdi generate"); }); + it("rechecks every workload gate immediately before Docker runtime mutation", () => { + const { result, output } = runSourced( + STATION_PREPARE, + ` +run_gpus_test_sudo() { return 1; } +sudo() { + [[ "$*" == "docker ps -aq" ]] && return 0 + [[ "$*" == "test -e /etc/docker/daemon.json" ]] && return 1 + printf 'SUDO %s\n' "$*" +} +check_no_workloads() { printf 'RECHECK_ALL_WORKLOADS\n'; return 1; } +configure_docker_runtime_if_needed +`, + ); + + expect(result.status, output).not.toBe(0); + expect(output).toContain("RECHECK_ALL_WORKLOADS"); + expect(output).not.toContain("nvidia-ctk runtime configure"); + expect(output).not.toContain("systemctl restart docker.service"); + }); + it("accepts a successful packaged CDI refresh", () => { const { result, output } = runSourced( STATION_PREPARE, @@ -427,7 +448,10 @@ prepare_installer_host INSTALLER_PAYLOAD, ` maybe_offer_express_install() { _SELECTED_EXPRESS_PLATFORM='DGX Spark'; } -run_station_host_preparation() { printf 'PREPARE_STATION\n'; } +ensure_station_express_host() { + [[ "$_SELECTED_EXPRESS_PLATFORM" == 'DGX Station' ]] && printf 'PREPARE_STATION\n' + return 0 +} ensure_docker() { printf 'ENSURE_DOCKER\n'; } ensure_openshell_build_deps() { printf 'ENSURE_BUILD_DEPS\n'; } prepare_installer_host From 88d1156ab4aaa0a667cbd4e82827dd3ac271878b Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 15 Jul 2026 21:56:09 -0700 Subject: [PATCH 07/74] test(installer): prove Station helper ships in bootstrap Signed-off-by: Aaron Erickson --- install.sh | 4 ++ scripts/install.sh | 3 + test/install-station-host-preparation.test.ts | 64 +++++++++++++++++++ 3 files changed, 71 insertions(+) diff --git a/install.sh b/install.sh index ab867b4c89..2f705acd33 100755 --- a/install.sh +++ b/install.sh @@ -91,6 +91,10 @@ exec_installer_from_ref() { legacy_script="${source_root}/install.sh" if has_payload_marker "$payload_script"; then + # The public curl|bash boundary deliberately executes from the complete + # selected-ref checkout, not from a standalone payload file. Installer + # helpers beside scripts/install.sh (including DGX Station preparation) + # are therefore staged from the same ref before payload execution. verify_downloaded_script "$payload_script" "versioned installer" NEMOCLAW_INSTALL_REF="$ref" NEMOCLAW_INSTALL_TAG="$ref" NEMOCLAW_BOOTSTRAP_PAYLOAD=1 \ bash "$payload_script" "$@" diff --git a/scripts/install.sh b/scripts/install.sh index 0f100e1ab9..7bed8d7260 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -3021,6 +3021,9 @@ activate_express_install() { } run_station_host_preparation() { + # Public curl|bash starts in the root bootstrap, which clones the complete + # selected ref before executing this payload. Keep the sibling lookup and + # fail-closed check so Station preparation cannot drift from that ref. local helper="${SCRIPT_DIR}/prepare-dgx-station-host.sh" [[ -f "$helper" ]] || error "DGX Station host preparation helper is missing: ${helper}" bash "$helper" --apply diff --git a/test/install-station-host-preparation.test.ts b/test/install-station-host-preparation.test.ts index 73654a6d06..9e0e4ece2d 100644 --- a/test/install-station-host-preparation.test.ts +++ b/test/install-station-host-preparation.test.ts @@ -9,6 +9,7 @@ import { describe, expect, it } from "vitest"; import { INSTALLER_PAYLOAD, TEST_SYSTEM_PATH } from "./helpers/installer-sourced-env"; const REPO_ROOT = path.resolve(import.meta.dirname, ".."); +const PUBLIC_BOOTSTRAP = path.join(REPO_ROOT, "install.sh"); const STATION_PREPARE = path.join(REPO_ROOT, "scripts", "prepare-dgx-station-host.sh"); const STATION_DOCS = [ path.join(REPO_ROOT, "docs", "get-started", "prerequisites.mdx"), @@ -422,6 +423,69 @@ write_install_boot_marker }); describe("DGX Station express host integration", () => { + it("ships and invokes Station preparation through the public curl bootstrap", () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-station-public-bootstrap-")); + const fakeBin = path.join(tmp, "bin"); + fs.mkdirSync(fakeBin); + fs.writeFileSync( + path.join(fakeBin, "git"), + `#!/usr/bin/env bash +set -euo pipefail +if [ "\${1:-}" = "init" ]; then + target="\${@: -1}" + mkdir -p "$target/scripts" + cat > "$target/scripts/install.sh" <<'PAYLOAD' +#!/usr/bin/env bash +# NEMOCLAW_VERSIONED_INSTALLER_PAYLOAD=1 +set -euo pipefail +source "\${INSTALLER_UNDER_TEST:?}" >/dev/null +SCRIPT_DIR="$(cd "$(dirname "\${BASH_SOURCE[0]}")" && pwd)" +maybe_offer_express_install() { _SELECTED_EXPRESS_PLATFORM='DGX Station'; } +ensure_docker() { printf 'ENSURE_DOCKER\\n'; } +ensure_openshell_build_deps() { printf 'ENSURE_BUILD_DEPS\\n'; } +prepare_installer_host +PAYLOAD + cat > "$target/scripts/prepare-dgx-station-host.sh" <<'HELPER' +#!/usr/bin/env bash +set -euo pipefail +[ "\${1:-}" = "--apply" ] +printf 'PREPARE_STATION\\n' +HELPER + chmod +x "$target/scripts/install.sh" "$target/scripts/prepare-dgx-station-host.sh" + exit 0 +fi +if [ "\${1:-}" = "-C" ]; then shift 2; fi +case "\${1:-}" in + remote|fetch|checkout) exit 0 ;; +esac +exit 0 +`, + { mode: 0o755 }, + ); + + const result = spawnSync("bash", [], { + cwd: tmp, + input: fs.readFileSync(PUBLIC_BOOTSTRAP, "utf-8"), + encoding: "utf-8", + env: { + ...process.env, + HOME: tmp, + PATH: `${fakeBin}:${TEST_SYSTEM_PATH}`, + INSTALLER_UNDER_TEST: INSTALLER_PAYLOAD, + NEMOCLAW_INSTALL_REF: "refs/tags/station-fixture", + }, + timeout: 15_000, + killSignal: "SIGKILL", + }); + const output = `${result.stdout}${result.stderr}`; + + expect(result.status, output).toBe(0); + expect(output).toContain("DGX Station host prerequisites are ready"); + expect(output.indexOf("PREPARE_STATION")).toBeGreaterThanOrEqual(0); + expect(output.indexOf("PREPARE_STATION")).toBeLessThan(output.indexOf("ENSURE_DOCKER")); + expect(output.indexOf("ENSURE_DOCKER")).toBeLessThan(output.indexOf("ENSURE_BUILD_DEPS")); + }); + it("runs Station preparation before the generic Docker bootstrap", () => { const { result, output } = runSourced( INSTALLER_PAYLOAD, From 7c2ee15fd201d1c374c57c8d987455c9fd1dd31d Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 15 Jul 2026 23:11:49 -0700 Subject: [PATCH 08/74] ci: retry credentialed e2e after runner loss Signed-off-by: Aaron Erickson From 80fdb0a876dce033d1cc194d0bd76376150196d6 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 15 Jul 2026 23:34:47 -0700 Subject: [PATCH 09/74] fix(installer): constrain Docker runtime repair Signed-off-by: Aaron Erickson --- docs/get-started/prerequisites.mdx | 2 + scripts/prepare-dgx-station-host.sh | 17 ++++++- test/install-station-host-preparation.test.ts | 47 +++++++++++++++++++ 3 files changed, 65 insertions(+), 1 deletion(-) diff --git a/docs/get-started/prerequisites.mdx b/docs/get-started/prerequisites.mdx index 99b85dc069..bb5a2657a6 100644 --- a/docs/get-started/prerequisites.mdx +++ b/docs/get-started/prerequisites.mdx @@ -43,6 +43,8 @@ The installer also requires `strings` from `binutils` to verify the OpenShell bi On an Ubuntu 24.04 ARM64 DGX Station GB300, accepting express install prepares the host with NVIDIA open driver `610.43.02`, Docker CE `29.6.1` with Buildx, and NVIDIA Container Toolkit `1.19.1`. The preparation probes package and runtime state first, reuses exact matches, and installs only missing pinned packages. +If the `docker --gpus all` acceptance probe fails, preparation registers the NVIDIA Docker runtime only when `docker info` diagnoses that runtime as absent; any other launch failure stops without changing daemon configuration. +That registration remains required until the same acceptance probe succeeds through a replacement Docker and NVIDIA runtime integration, so preparation does not remove it automatically. It requires Secure Boot to be disabled, matching headers for the running kernel, at least 20 GiB free on the root filesystem, and no active agent, inference, or Docker workloads. It does not install a host CUDA toolkit or Docker Compose. If an existing prerequisite has a different version, preparation stops instead of automatically upgrading or downgrading it. diff --git a/scripts/prepare-dgx-station-host.sh b/scripts/prepare-dgx-station-host.sh index 6ee446a436..c94f441aa4 100755 --- a/scripts/prepare-dgx-station-host.sh +++ b/scripts/prepare-dgx-station-host.sh @@ -505,6 +505,14 @@ run_gpus_test_sudo() { sudo docker run --rm --gpus all "$ACCEPTANCE_IMAGE" nvidia-smi } +docker_has_nvidia_runtime_sudo() { + local runtimes + runtimes="$( + sudo docker info --format '{{range $name, $_ := .Runtimes}}{{println $name}}{{end}}' + )" || fatal "Could not inspect Docker runtimes after the --gpus all probe failed" + grep -Fxq 'nvidia' <<<"$runtimes" +} + ensure_cdi_runtime() { if run_cdi_test_sudo; then info "cdi_contract=pass_without_configuration_change" @@ -524,7 +532,14 @@ configure_docker_runtime_if_needed() { return 0 fi - warn "Docker --gpus all failed; applying the reviewed NVIDIA runtime registration" + if docker_has_nvidia_runtime_sudo; then + fatal "Docker --gpus all failed even though the NVIDIA runtime is registered; daemon configuration was left unchanged. Inspect the failed container launch and rerun preparation." + fi + + # Persistent registration is the supported repair only for the diagnosed + # missing-runtime state. It remains required until this acceptance probe + # succeeds through a replacement Docker/NVIDIA runtime integration. + warn "Docker --gpus all failed and Docker reports no NVIDIA runtime; applying the reviewed NVIDIA runtime registration" [[ -z "$(sudo docker ps -aq)" ]] || fatal "Docker became non-idle before runtime configuration" timestamp="$(date -u '+%Y%m%dT%H%M%SZ')" backup_dir="/var/backups/station-bootstrap/${timestamp}" diff --git a/test/install-station-host-preparation.test.ts b/test/install-station-host-preparation.test.ts index 9e0e4ece2d..1b65d2707d 100644 --- a/test/install-station-host-preparation.test.ts +++ b/test/install-station-host-preparation.test.ts @@ -340,6 +340,7 @@ refresh_cdi STATION_PREPARE, ` run_gpus_test_sudo() { return 1; } +docker_has_nvidia_runtime_sudo() { return 1; } sudo() { [[ "$*" == "docker ps -aq" ]] && return 0 [[ "$*" == "test -e /etc/docker/daemon.json" ]] && return 1 @@ -356,6 +357,52 @@ configure_docker_runtime_if_needed expect(output).not.toContain("systemctl restart docker.service"); }); + it("leaves Docker unchanged when the NVIDIA runtime is already registered", () => { + const { result, output } = runSourced( + STATION_PREPARE, + ` +run_gpus_test_sudo() { return 1; } +docker_has_nvidia_runtime_sudo() { return 0; } +sudo() { printf 'SUDO %s\n' "$*"; } +configure_docker_runtime_if_needed +`, + ); + + expect(result.status, output).not.toBe(0); + expect(output).toMatch(/NVIDIA runtime is registered/); + expect(output).toMatch(/daemon configuration was left unchanged/); + expect(output).not.toContain("nvidia-ctk runtime configure"); + expect(output).not.toContain("systemctl restart docker.service"); + }); + + it("registers the NVIDIA runtime only when Docker reports it missing", () => { + const { result, output } = runSourced( + STATION_PREPARE, + ` +calls=0 +run_gpus_test_sudo() { + calls=$((calls + 1)) + [[ "$calls" -gt 1 ]] +} +run_cdi_test_sudo() { return 0; } +docker_has_nvidia_runtime_sudo() { return 1; } +check_no_workloads() { printf 'RECHECK_ALL_WORKLOADS\n'; } +sudo() { + [[ "$*" == "docker ps -aq" ]] && return 0 + [[ "$*" == "test -e /etc/docker/daemon.json" ]] && return 1 + printf 'SUDO %s\n' "$*" +} +configure_docker_runtime_if_needed +`, + ); + + expect(result.status, output).toBe(0); + expect(output).toContain("Docker reports no NVIDIA runtime"); + expect(output).toContain("nvidia-ctk runtime configure --runtime=docker"); + expect(output).toContain("systemctl restart docker.service"); + expect(output).toContain("docker_gpus_contract=pass"); + }); + it("accepts a successful packaged CDI refresh", () => { const { result, output } = runSourced( STATION_PREPARE, From fb2be367c5580158353a7a21ca6893345358119b Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 15 Jul 2026 23:44:49 -0700 Subject: [PATCH 10/74] test(installer): guard Station project assignment Signed-off-by: Aaron Erickson --- test/test-boundary-guards.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/test/test-boundary-guards.test.ts b/test/test-boundary-guards.test.ts index 3237fbf5b9..6f0a5115ae 100644 --- a/test/test-boundary-guards.test.ts +++ b/test/test-boundary-guards.test.ts @@ -696,6 +696,7 @@ describe("Vitest project membership boundary", () => { ["test/install-openshell-version-check.test.ts", "installer-integration"], ["test/install-preflight-docker-bootstrap.test.ts", "installer-integration"], ["test/install-preflight.test.ts", "installer-integration"], + ["test/install-station-host-preparation.test.ts", "installer-integration"], ["test/package-contract/example.test.js", "package-contract"], ["test/e2e/support/example.test.js", "e2e-support"], ["test/e2e/live/example.spec.ts", "e2e-live"], From fb6adefa516a01f859383dd55f3328dda111e2cd Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 16 Jul 2026 08:27:10 -0700 Subject: [PATCH 11/74] fix(installer): complete Station GPU prerequisites Signed-off-by: Aaron Erickson --- docs/get-started/prerequisites.mdx | 5 +- docs/get-started/quickstart.mdx | 5 +- scripts/prepare-dgx-station-host.sh | 35 +++++-- test/install-station-host-preparation.test.ts | 91 +++++++++++++++++-- 4 files changed, 118 insertions(+), 18 deletions(-) diff --git a/docs/get-started/prerequisites.mdx b/docs/get-started/prerequisites.mdx index bb5a2657a6..c5abbe2ab3 100644 --- a/docs/get-started/prerequisites.mdx +++ b/docs/get-started/prerequisites.mdx @@ -42,7 +42,10 @@ If you choose the native Linux Ollama install path, the onboard wizard also requ The installer also requires `strings` from `binutils` to verify the OpenShell binary before it continues with OpenShell install work. On an Ubuntu 24.04 ARM64 DGX Station GB300, accepting express install prepares the host with NVIDIA open driver `610.43.02`, Docker CE `29.6.1` with Buildx, and NVIDIA Container Toolkit `1.19.1`. -The preparation probes package and runtime state first, reuses exact matches, and installs only missing pinned packages. +The preparation probes package and runtime state first, reuses exact matches, and installs only missing pinned packages, including the NVIDIA Container Toolkit libraries and `nvidia-ctk` CLI. +Package downloads from the Ubuntu, NVIDIA, and Docker repository hosts can proceed concurrently within one serialized APT transaction. +After reboot, preparation enables NVIDIA's packaged CDI refresh service, requires the `nvidia.com/gpu=all` device, and verifies it with a real container launch. +If the packaged refresh does not produce that device, it uses NVIDIA's documented transient generation fallback at `/var/run/cdi/nvidia.yaml`; it does not create a persistent manual CDI configuration under `/etc`. If the `docker --gpus all` acceptance probe fails, preparation registers the NVIDIA Docker runtime only when `docker info` diagnoses that runtime as absent; any other launch failure stops without changing daemon configuration. That registration remains required until the same acceptance probe succeeds through a replacement Docker and NVIDIA runtime integration, so preparation does not remove it automatically. It requires Secure Boot to be disabled, matching headers for the running kernel, at least 20 GiB free on the root filesystem, and no active agent, inference, or Docker workloads. diff --git a/docs/get-started/quickstart.mdx b/docs/get-started/quickstart.mdx index d0bd7bb8b3..aeba937e08 100644 --- a/docs/get-started/quickstart.mdx +++ b/docs/get-started/quickstart.mdx @@ -122,6 +122,8 @@ Use these details when your first-run path needs more control. DGX Spark, DGX Station, and Windows WSL offer an interactive express-install path that chooses a managed local inference option for the platform. On DGX Station, accepting the express prompt selects the pinned `nemotron-3-ultra-550b-a55b` managed-vLLM recipe and completes onboarding without more provider, model, policy, or sandbox-name choices. The Station path probes the validated driver, Docker, Buildx, and NVIDIA Container Toolkit versions, reuses exact matches, and installs missing pinned packages without additional choices. + It establishes NVIDIA CDI through the packaged refresh service, with NVIDIA's transient `/var/run/cdi/nvidia.yaml` generation command as a fallback, then proves both CDI and `--gpus all` with real container launches. + APT fetches from separate Ubuntu, NVIDIA, and Docker repository hosts concurrently while package installation and host mutations remain serialized. After it installs missing pinned packages, the installer exits with status `10` at the validated reboot boundary; reboot, sign in, and rerun the same installer command to resume the accepted recipe without another prompt. This automation does not change Station's Deferred support status; physical end-to-end validation remains open. Pass `--station-deepseek` to use DeepSeek V4 Flash for a Station demo instead. @@ -193,8 +195,9 @@ Use these details when your first-run path needs more control. Express install switches onboarding to non-interactive mode, allows `sudo` password prompts for required host changes, and selects the managed local inference path for that platform. DGX Spark uses managed vLLM with `qwen3.6-35b-a3b-nvfp4` by default. DGX Station express install explicitly selects `nemotron-3-ultra-550b-a55b` instead of the Station managed-vLLM profile default, `deepseek-v4-flash`, and discloses the approximately `352 GB` model download before confirmation. - Before onboarding, the Station path checks for NVIDIA open driver `610.43.02`, Docker CE `29.6.1` with Buildx, and NVIDIA Container Toolkit `1.19.1`. + Before onboarding, the Station path checks for NVIDIA open driver `610.43.02`, Docker CE `29.6.1` with Buildx, NVIDIA Container Toolkit `1.19.1`, and the NVIDIA CDI device `nvidia.com/gpu=all`. It reuses exact versions, installs missing pinned packages, and refuses to automatically upgrade or downgrade an existing mismatched version. + APT downloads from separate repository hosts can overlap, but package installation, driver activation, service changes, and runtime acceptance checks remain serialized. Installing missing pinned packages exits with status `10` for a reboot; after you sign in and rerun the same installer command, the accepted express recipe resumes without another prompt. This automation does not change Station's Deferred support status; physical end-to-end validation remains open. To select DeepSeek V4 Flash while retaining the one-confirmation Station express flow, run `curl -fsSL https://www.nvidia.com/nemoclaw.sh | bash -s -- --station-deepseek`. diff --git a/scripts/prepare-dgx-station-host.sh b/scripts/prepare-dgx-station-host.sh index c94f441aa4..d13bd76b4e 100755 --- a/scripts/prepare-dgx-station-host.sh +++ b/scripts/prepare-dgx-station-host.sh @@ -5,7 +5,7 @@ set -Eeuo pipefail umask 077 -readonly SCRIPT_VERSION="2026-07-15.1" +readonly SCRIPT_VERSION="2026-07-16.1" readonly REBOOT_REQUIRED_EXIT=10 readonly MIN_FREE_KIB=$((20 * 1024 * 1024)) @@ -23,6 +23,7 @@ readonly TOOLKIT_VERSION="1.19.1" readonly ACCEPTANCE_IMAGE="ubuntu@sha256:7f622ca8766bccb22f04242ecb6f19f770b2f08827dc4b8c707de5e78a6da7ab" readonly STATE_DIR="${HOME}/.local/state/station-bootstrap" readonly INSTALL_BOOT_MARKER="${STATE_DIR}/install-boot-id" +readonly -a APT_DOWNLOAD_OPTIONS=("-o" "Acquire::Queue-Mode=host") readonly -a PACKAGE_SPECS=( "dkms=1:3.4.0-1ubuntu1" @@ -440,7 +441,7 @@ validate_package_availability() { simulate_install() { local simulation - simulation="$(apt-get -s install --no-install-recommends "${PACKAGE_SPECS[@]}")" \ + simulation="$(apt-get "${APT_DOWNLOAD_OPTIONS[@]}" -s install --no-install-recommends "${PACKAGE_SPECS[@]}")" \ || fatal "APT simulation failed" printf '%s\n' "$simulation" if grep -Eq '^(Remv |Purg )' <<<"$simulation"; then @@ -451,12 +452,12 @@ simulate_install() { install_packages() { configure_repositories - info "Refreshing package metadata" - sudo apt-get update + info "Refreshing package metadata with one download connection per repository host" + sudo apt-get "${APT_DOWNLOAD_OPTIONS[@]}" update validate_package_availability simulate_install - info "Installing pinned Station prerequisites" - sudo env DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + info "Installing pinned Station prerequisites with parallel downloads across repository hosts" + sudo env DEBIAN_FRONTEND=noninteractive apt-get "${APT_DOWNLOAD_OPTIONS[@]}" install -y --no-install-recommends \ "${PACKAGE_SPECS[@]}" local spec @@ -481,13 +482,27 @@ ensure_docker_group() { refresh_cdi() { sudo systemctl enable --now nvidia-cdi-refresh.path if ! sudo systemctl restart nvidia-cdi-refresh.service; then - warn "Packaged CDI refresh failed; refusing to create a persistent manual CDI specification" + warn "Packaged CDI refresh failed; collecting diagnostics before the NVIDIA transient fallback" + sudo systemctl status nvidia-cdi-refresh.service --no-pager || true + sudo journalctl -u nvidia-cdi-refresh.service --no-pager -n 50 || true + elif nvidia-ctk cdi list | grep -Fxq 'nvidia.com/gpu=all'; then + info "cdi=nvidia.com/gpu=all source=packaged_refresh_service" + return 0 + else + warn "Packaged CDI refresh completed without advertising nvidia.com/gpu=all" sudo systemctl status nvidia-cdi-refresh.service --no-pager || true sudo journalctl -u nvidia-cdi-refresh.service --no-pager -n 50 || true - fatal "nvidia-cdi-refresh.service failed; fix the packaged service and rerun preparation" fi - nvidia-ctk cdi list | grep -Fxq 'nvidia.com/gpu=all' || fatal "CDI device nvidia.com/gpu=all is unavailable" - info "cdi=nvidia.com/gpu=all" + + # This is NVIDIA's documented fallback and writes the same transient /var/run + # specification as nvidia-cdi-refresh.service. It does not create a persistent + # manual CDI configuration under /etc. + warn "Generating NVIDIA's transient CDI specification at /var/run/cdi/nvidia.yaml" + sudo nvidia-ctk cdi generate --output=/var/run/cdi/nvidia.yaml \ + || fatal "Direct transient CDI generation failed; inspect the packaged service diagnostics" + nvidia-ctk cdi list | grep -Fxq 'nvidia.com/gpu=all' \ + || fatal "CDI device nvidia.com/gpu=all is unavailable after direct transient generation" + info "cdi=nvidia.com/gpu=all source=direct_transient_fallback" } ensure_acceptance_image() { diff --git a/test/install-station-host-preparation.test.ts b/test/install-station-host-preparation.test.ts index 1b65d2707d..4cbe56aefd 100644 --- a/test/install-station-host-preparation.test.ts +++ b/test/install-station-host-preparation.test.ts @@ -139,6 +139,35 @@ run_apply expect(output).toContain("APPLY_RESULT=REBOOT_REQUIRED"); }); + it("installs the exact NVIDIA Container Toolkit package contract with safe parallel fetches", () => { + const { result, output } = runSourced( + STATION_PREPARE, + ` +configure_repositories() { printf 'CONFIGURE_REPOSITORIES\n'; } +validate_package_availability() { printf 'VALIDATE_PACKAGES\n'; } +simulate_install() { printf 'SIMULATE_INSTALL\n'; } +package_is_exact() { return 0; } +sudo() { printf 'SUDO %s\n' "$*"; } +install_packages +`, + ); + + expect(result.status, output).toBe(0); + expect(output).toContain("apt-get -o Acquire::Queue-Mode=host update"); + expect(output).toContain( + "apt-get -o Acquire::Queue-Mode=host install -y --no-install-recommends", + ); + for (const spec of [ + "libnvidia-container-tools=1.19.1-1", + "libnvidia-container1=1.19.1-1", + "nvidia-container-toolkit=1.19.1-1", + "nvidia-container-toolkit-base=1.19.1-1", + ]) { + expect(output).toContain(spec); + } + expect(output).toContain("pinned_packages=installed"); + }); + it("does not refresh CDI when the GPU launch probe already passes", () => { const { result, output } = runSourced( STATION_PREPARE, @@ -316,23 +345,72 @@ run_apply expect(output).toMatch(/new login before onboarding/); }); - it("diagnoses packaged CDI refresh failure without writing a manual specification", () => { + it("falls back to NVIDIA's transient CDI generation when the packaged refresh fails", () => { const { result, output } = runSourced( STATION_PREPARE, ` +generated=0 sudo() { printf 'SUDO %s\n' "$*" - [[ "$*" != "systemctl restart nvidia-cdi-refresh.service" ]] + if [[ "$*" == "systemctl restart nvidia-cdi-refresh.service" ]]; then return 1; fi + if [[ "$*" == "nvidia-ctk cdi generate --output=/var/run/cdi/nvidia.yaml" ]]; then generated=1; fi + return 0 +} +nvidia-ctk() { + if [[ "$*" == "cdi list" && "$generated" == "1" ]]; then printf 'nvidia.com/gpu=all\n'; fi } refresh_cdi `, ); - expect(result.status, output).not.toBe(0); + expect(result.status, output).toBe(0); expect(output).toContain("systemctl status nvidia-cdi-refresh.service --no-pager"); expect(output).toContain("journalctl -u nvidia-cdi-refresh.service --no-pager -n 50"); - expect(output).toMatch(/refusing to create a persistent manual CDI specification/); - expect(output).not.toContain("cdi generate"); + expect(output).toContain("nvidia-ctk cdi generate --output=/var/run/cdi/nvidia.yaml"); + expect(output).toContain("cdi=nvidia.com/gpu=all source=direct_transient_fallback"); + expect(output).not.toContain("/etc/cdi"); + }); + + it("uses the transient CDI fallback when the packaged refresh produces no GPU device", () => { + const { result, output } = runSourced( + STATION_PREPARE, + ` +generated=0 +sudo() { + printf 'SUDO %s\n' "$*" + if [[ "$*" == "nvidia-ctk cdi generate --output=/var/run/cdi/nvidia.yaml" ]]; then generated=1; fi + return 0 +} +nvidia-ctk() { + if [[ "$*" == "cdi list" && "$generated" == "1" ]]; then printf 'nvidia.com/gpu=all\n'; fi +} +refresh_cdi +`, + ); + + expect(result.status, output).toBe(0); + expect(output).toMatch(/completed without advertising nvidia\.com\/gpu=all/); + expect(output).toContain("nvidia-ctk cdi generate --output=/var/run/cdi/nvidia.yaml"); + expect(output).toContain("cdi=nvidia.com/gpu=all source=direct_transient_fallback"); + }); + + it("fails closed when direct transient CDI generation also fails", () => { + const { result, output } = runSourced( + STATION_PREPARE, + ` +sudo() { + printf 'SUDO %s\n' "$*" + if [[ "$*" == "systemctl restart nvidia-cdi-refresh.service" ]]; then return 1; fi + if [[ "$*" == "nvidia-ctk cdi generate --output=/var/run/cdi/nvidia.yaml" ]]; then return 1; fi + return 0 +} +refresh_cdi +`, + ); + + expect(result.status, output).not.toBe(0); + expect(output).toContain("nvidia-ctk cdi generate --output=/var/run/cdi/nvidia.yaml"); + expect(output).toMatch(/Direct transient CDI generation failed/); }); it("rechecks every workload gate immediately before Docker runtime mutation", () => { @@ -417,8 +495,9 @@ refresh_cdi expect(result.status, output).toBe(0); expect(output).toContain("systemctl restart nvidia-cdi-refresh.service"); - expect(output).toContain("cdi=nvidia.com/gpu=all"); + expect(output).toContain("cdi=nvidia.com/gpu=all source=packaged_refresh_service"); expect(output).not.toContain("systemctl status"); + expect(output).not.toContain("cdi generate"); }); it.each(["--check", "--verify"])("keeps %s read-only under HOME", (mode) => { From c1edd7b8e3f041c86f551e338817702bf1d1a5b9 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 16 Jul 2026 08:31:31 -0700 Subject: [PATCH 12/74] fix(installer): preserve Station host policies Signed-off-by: Aaron Erickson --- docs/get-started/prerequisites.mdx | 4 +- docs/get-started/quickstart.mdx | 2 - scripts/prepare-dgx-station-host.sh | 34 +++++++++--- test/install-station-host-preparation.test.ts | 55 +++++++++++++++++-- 4 files changed, 78 insertions(+), 17 deletions(-) diff --git a/docs/get-started/prerequisites.mdx b/docs/get-started/prerequisites.mdx index c5abbe2ab3..aa62e79ab1 100644 --- a/docs/get-started/prerequisites.mdx +++ b/docs/get-started/prerequisites.mdx @@ -43,9 +43,9 @@ The installer also requires `strings` from `binutils` to verify the OpenShell bi On an Ubuntu 24.04 ARM64 DGX Station GB300, accepting express install prepares the host with NVIDIA open driver `610.43.02`, Docker CE `29.6.1` with Buildx, and NVIDIA Container Toolkit `1.19.1`. The preparation probes package and runtime state first, reuses exact matches, and installs only missing pinned packages, including the NVIDIA Container Toolkit libraries and `nvidia-ctk` CLI. -Package downloads from the Ubuntu, NVIDIA, and Docker repository hosts can proceed concurrently within one serialized APT transaction. -After reboot, preparation enables NVIDIA's packaged CDI refresh service, requires the `nvidia.com/gpu=all` device, and verifies it with a real container launch. +After reboot, preparation enables NVIDIA's packaged CDI refresh path and service, requires the `nvidia.com/gpu=all` device, and verifies it with a real container launch. If the packaged refresh does not produce that device, it uses NVIDIA's documented transient generation fallback at `/var/run/cdi/nvidia.yaml`; it does not create a persistent manual CDI configuration under `/etc`. +If an administrator has customized the packaged CDI refresh environment, preparation stops instead of bypassing that policy with default direct generation. If the `docker --gpus all` acceptance probe fails, preparation registers the NVIDIA Docker runtime only when `docker info` diagnoses that runtime as absent; any other launch failure stops without changing daemon configuration. That registration remains required until the same acceptance probe succeeds through a replacement Docker and NVIDIA runtime integration, so preparation does not remove it automatically. It requires Secure Boot to be disabled, matching headers for the running kernel, at least 20 GiB free on the root filesystem, and no active agent, inference, or Docker workloads. diff --git a/docs/get-started/quickstart.mdx b/docs/get-started/quickstart.mdx index aeba937e08..be6d40d3c6 100644 --- a/docs/get-started/quickstart.mdx +++ b/docs/get-started/quickstart.mdx @@ -123,7 +123,6 @@ Use these details when your first-run path needs more control. On DGX Station, accepting the express prompt selects the pinned `nemotron-3-ultra-550b-a55b` managed-vLLM recipe and completes onboarding without more provider, model, policy, or sandbox-name choices. The Station path probes the validated driver, Docker, Buildx, and NVIDIA Container Toolkit versions, reuses exact matches, and installs missing pinned packages without additional choices. It establishes NVIDIA CDI through the packaged refresh service, with NVIDIA's transient `/var/run/cdi/nvidia.yaml` generation command as a fallback, then proves both CDI and `--gpus all` with real container launches. - APT fetches from separate Ubuntu, NVIDIA, and Docker repository hosts concurrently while package installation and host mutations remain serialized. After it installs missing pinned packages, the installer exits with status `10` at the validated reboot boundary; reboot, sign in, and rerun the same installer command to resume the accepted recipe without another prompt. This automation does not change Station's Deferred support status; physical end-to-end validation remains open. Pass `--station-deepseek` to use DeepSeek V4 Flash for a Station demo instead. @@ -197,7 +196,6 @@ Use these details when your first-run path needs more control. DGX Station express install explicitly selects `nemotron-3-ultra-550b-a55b` instead of the Station managed-vLLM profile default, `deepseek-v4-flash`, and discloses the approximately `352 GB` model download before confirmation. Before onboarding, the Station path checks for NVIDIA open driver `610.43.02`, Docker CE `29.6.1` with Buildx, NVIDIA Container Toolkit `1.19.1`, and the NVIDIA CDI device `nvidia.com/gpu=all`. It reuses exact versions, installs missing pinned packages, and refuses to automatically upgrade or downgrade an existing mismatched version. - APT downloads from separate repository hosts can overlap, but package installation, driver activation, service changes, and runtime acceptance checks remain serialized. Installing missing pinned packages exits with status `10` for a reboot; after you sign in and rerun the same installer command, the accepted express recipe resumes without another prompt. This automation does not change Station's Deferred support status; physical end-to-end validation remains open. To select DeepSeek V4 Flash while retaining the one-confirmation Station express flow, run `curl -fsSL https://www.nvidia.com/nemoclaw.sh | bash -s -- --station-deepseek`. diff --git a/scripts/prepare-dgx-station-host.sh b/scripts/prepare-dgx-station-host.sh index d13bd76b4e..35ecd3334f 100755 --- a/scripts/prepare-dgx-station-host.sh +++ b/scripts/prepare-dgx-station-host.sh @@ -23,7 +23,7 @@ readonly TOOLKIT_VERSION="1.19.1" readonly ACCEPTANCE_IMAGE="ubuntu@sha256:7f622ca8766bccb22f04242ecb6f19f770b2f08827dc4b8c707de5e78a6da7ab" readonly STATE_DIR="${HOME}/.local/state/station-bootstrap" readonly INSTALL_BOOT_MARKER="${STATE_DIR}/install-boot-id" -readonly -a APT_DOWNLOAD_OPTIONS=("-o" "Acquire::Queue-Mode=host") +readonly CDI_REFRESH_ENV_FILE="/etc/nvidia-container-toolkit/nvidia-cdi-refresh.env" readonly -a PACKAGE_SPECS=( "dkms=1:3.4.0-1ubuntu1" @@ -441,7 +441,7 @@ validate_package_availability() { simulate_install() { local simulation - simulation="$(apt-get "${APT_DOWNLOAD_OPTIONS[@]}" -s install --no-install-recommends "${PACKAGE_SPECS[@]}")" \ + simulation="$(apt-get -s install --no-install-recommends "${PACKAGE_SPECS[@]}")" \ || fatal "APT simulation failed" printf '%s\n' "$simulation" if grep -Eq '^(Remv |Purg )' <<<"$simulation"; then @@ -452,12 +452,12 @@ simulate_install() { install_packages() { configure_repositories - info "Refreshing package metadata with one download connection per repository host" - sudo apt-get "${APT_DOWNLOAD_OPTIONS[@]}" update + info "Refreshing package metadata" + sudo apt-get update validate_package_availability simulate_install - info "Installing pinned Station prerequisites with parallel downloads across repository hosts" - sudo env DEBIAN_FRONTEND=noninteractive apt-get "${APT_DOWNLOAD_OPTIONS[@]}" install -y --no-install-recommends \ + info "Installing pinned Station prerequisites" + sudo env DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ "${PACKAGE_SPECS[@]}" local spec @@ -479,8 +479,23 @@ ensure_docker_group() { fi } +cdi_refresh_has_custom_environment() { + local grep_status + sudo test -e "$CDI_REFRESH_ENV_FILE" || return 1 + if sudo grep -Eq '^[[:space:]]*[^#[:space:]]' "$CDI_REFRESH_ENV_FILE"; then + return 0 + else + grep_status=$? + fi + ((grep_status == 1)) && return 1 + fatal "Could not inspect ${CDI_REFRESH_ENV_FILE} before CDI fallback" +} + refresh_cdi() { - sudo systemctl enable --now nvidia-cdi-refresh.path + sudo systemctl enable nvidia-cdi-refresh.path nvidia-cdi-refresh.service \ + || fatal "Could not enable the packaged NVIDIA CDI refresh lifecycle" + sudo systemctl start nvidia-cdi-refresh.path \ + || fatal "Could not activate the packaged NVIDIA CDI refresh path" if ! sudo systemctl restart nvidia-cdi-refresh.service; then warn "Packaged CDI refresh failed; collecting diagnostics before the NVIDIA transient fallback" sudo systemctl status nvidia-cdi-refresh.service --no-pager || true @@ -497,6 +512,9 @@ refresh_cdi() { # This is NVIDIA's documented fallback and writes the same transient /var/run # specification as nvidia-cdi-refresh.service. It does not create a persistent # manual CDI configuration under /etc. + if cdi_refresh_has_custom_environment; then + fatal "${CDI_REFRESH_ENV_FILE} contains administrator overrides; refusing to bypass them with default direct CDI generation" + fi warn "Generating NVIDIA's transient CDI specification at /var/run/cdi/nvidia.yaml" sudo nvidia-ctk cdi generate --output=/var/run/cdi/nvidia.yaml \ || fatal "Direct transient CDI generation failed; inspect the packaged service diagnostics" @@ -678,7 +696,7 @@ run_apply() { assert_no_package_mismatches install_packages ensure_docker_group - sudo systemctl enable containerd.service docker.service nvidia-cdi-refresh.path + sudo systemctl enable containerd.service docker.service nvidia-cdi-refresh.path nvidia-cdi-refresh.service write_install_boot_marker info "APPLY_RESULT=REBOOT_REQUIRED" info "Run: sudo reboot" diff --git a/test/install-station-host-preparation.test.ts b/test/install-station-host-preparation.test.ts index 4cbe56aefd..622561e7fb 100644 --- a/test/install-station-host-preparation.test.ts +++ b/test/install-station-host-preparation.test.ts @@ -136,10 +136,13 @@ run_apply expect(output).toContain("INSTALL_PACKAGES"); expect(output).toContain("ENSURE_DOCKER_GROUP"); expect(output).toContain("WRITE_BOOT_MARKER"); + expect(output).toContain( + "systemctl enable containerd.service docker.service nvidia-cdi-refresh.path nvidia-cdi-refresh.service", + ); expect(output).toContain("APPLY_RESULT=REBOOT_REQUIRED"); }); - it("installs the exact NVIDIA Container Toolkit package contract with safe parallel fetches", () => { + it("installs the exact NVIDIA Container Toolkit package contract", () => { const { result, output } = runSourced( STATION_PREPARE, ` @@ -153,10 +156,8 @@ install_packages ); expect(result.status, output).toBe(0); - expect(output).toContain("apt-get -o Acquire::Queue-Mode=host update"); - expect(output).toContain( - "apt-get -o Acquire::Queue-Mode=host install -y --no-install-recommends", - ); + expect(output).toContain("apt-get update"); + expect(output).toContain("apt-get install -y --no-install-recommends"); for (const spec of [ "libnvidia-container-tools=1.19.1-1", "libnvidia-container1=1.19.1-1", @@ -353,6 +354,7 @@ generated=0 sudo() { printf 'SUDO %s\n' "$*" if [[ "$*" == "systemctl restart nvidia-cdi-refresh.service" ]]; then return 1; fi + if [[ "$*" == "test -e $CDI_REFRESH_ENV_FILE" ]]; then return 1; fi if [[ "$*" == "nvidia-ctk cdi generate --output=/var/run/cdi/nvidia.yaml" ]]; then generated=1; fi return 0 } @@ -378,6 +380,7 @@ refresh_cdi generated=0 sudo() { printf 'SUDO %s\n' "$*" + if [[ "$*" == "test -e $CDI_REFRESH_ENV_FILE" ]]; then return 1; fi if [[ "$*" == "nvidia-ctk cdi generate --output=/var/run/cdi/nvidia.yaml" ]]; then generated=1; fi return 0 } @@ -401,6 +404,7 @@ refresh_cdi sudo() { printf 'SUDO %s\n' "$*" if [[ "$*" == "systemctl restart nvidia-cdi-refresh.service" ]]; then return 1; fi + if [[ "$*" == "test -e $CDI_REFRESH_ENV_FILE" ]]; then return 1; fi if [[ "$*" == "nvidia-ctk cdi generate --output=/var/run/cdi/nvidia.yaml" ]]; then return 1; fi return 0 } @@ -413,6 +417,45 @@ refresh_cdi expect(output).toMatch(/Direct transient CDI generation failed/); }); + it("fails closed when direct CDI generation produces no GPU device", () => { + const { result, output } = runSourced( + STATION_PREPARE, + ` +sudo() { + printf 'SUDO %s\n' "$*" + if [[ "$*" == "systemctl restart nvidia-cdi-refresh.service" ]]; then return 1; fi + if [[ "$*" == "test -e $CDI_REFRESH_ENV_FILE" ]]; then return 1; fi + return 0 +} +nvidia-ctk() { :; } +refresh_cdi +`, + ); + + expect(result.status, output).not.toBe(0); + expect(output).toContain("nvidia-ctk cdi generate --output=/var/run/cdi/nvidia.yaml"); + expect(output).toMatch(/unavailable after direct transient generation/); + }); + + it("does not bypass an administrator-customized CDI refresh environment", () => { + const { result, output } = runSourced( + STATION_PREPARE, + ` +sudo() { + printf 'SUDO %s\n' "$*" + if [[ "$*" == "systemctl restart nvidia-cdi-refresh.service" ]]; then return 1; fi + return 0 +} +refresh_cdi +`, + ); + + expect(result.status, output).not.toBe(0); + expect(output).toMatch(/contains administrator overrides/); + expect(output).toMatch(/refusing to bypass/); + expect(output).not.toContain("nvidia-ctk cdi generate --output=/var/run/cdi/nvidia.yaml"); + }); + it("rechecks every workload gate immediately before Docker runtime mutation", () => { const { result, output } = runSourced( STATION_PREPARE, @@ -494,6 +537,8 @@ refresh_cdi ); expect(result.status, output).toBe(0); + expect(output).toContain("systemctl enable nvidia-cdi-refresh.path nvidia-cdi-refresh.service"); + expect(output).toContain("systemctl start nvidia-cdi-refresh.path"); expect(output).toContain("systemctl restart nvidia-cdi-refresh.service"); expect(output).toContain("cdi=nvidia.com/gpu=all source=packaged_refresh_service"); expect(output).not.toContain("systemctl status"); From b2230f94273ee50293bb030c79420939717251d6 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 16 Jul 2026 08:34:39 -0700 Subject: [PATCH 13/74] fix(installer): persist Station CDI lifecycle Signed-off-by: Aaron Erickson --- scripts/prepare-dgx-station-host.sh | 23 ++++++++++++++++++- test/install-station-host-preparation.test.ts | 21 +++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/scripts/prepare-dgx-station-host.sh b/scripts/prepare-dgx-station-host.sh index 35ecd3334f..7baa54ca3e 100755 --- a/scripts/prepare-dgx-station-host.sh +++ b/scripts/prepare-dgx-station-host.sh @@ -42,6 +42,7 @@ readonly -a PACKAGE_SPECS=( MODE="" LOG_FILE="" DOCKER_GROUP_ADDED=0 +CDI_LIFECYCLE_READY=0 info() { printf '[station-prepare] %s %s\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$*" @@ -491,11 +492,28 @@ cdi_refresh_has_custom_environment() { fatal "Could not inspect ${CDI_REFRESH_ENV_FILE} before CDI fallback" } -refresh_cdi() { +ensure_cdi_refresh_lifecycle() { + ((CDI_LIFECYCLE_READY == 0)) || return 0 sudo systemctl enable nvidia-cdi-refresh.path nvidia-cdi-refresh.service \ || fatal "Could not enable the packaged NVIDIA CDI refresh lifecycle" sudo systemctl start nvidia-cdi-refresh.path \ || fatal "Could not activate the packaged NVIDIA CDI refresh path" + CDI_LIFECYCLE_READY=1 + info "cdi_refresh_lifecycle=enabled" +} + +verify_cdi_refresh_lifecycle() { + systemctl is-enabled --quiet nvidia-cdi-refresh.path \ + || fatal "nvidia-cdi-refresh.path is not enabled" + systemctl is-enabled --quiet nvidia-cdi-refresh.service \ + || fatal "nvidia-cdi-refresh.service is not enabled" + systemctl is-active --quiet nvidia-cdi-refresh.path \ + || fatal "nvidia-cdi-refresh.path is not active" + info "cdi_refresh_lifecycle=verified" +} + +refresh_cdi() { + ensure_cdi_refresh_lifecycle if ! sudo systemctl restart nvidia-cdi-refresh.service; then warn "Packaged CDI refresh failed; collecting diagnostics before the NVIDIA transient fallback" sudo systemctl status nvidia-cdi-refresh.service --no-pager || true @@ -547,6 +565,7 @@ docker_has_nvidia_runtime_sudo() { } ensure_cdi_runtime() { + ensure_cdi_refresh_lifecycle if run_cdi_test_sudo; then info "cdi_contract=pass_without_configuration_change" return 0 @@ -610,6 +629,7 @@ verify_apply_state() { systemctl is-active --quiet nvidia-persistenced.service || fatal "nvidia-persistenced.service is not active" systemctl is-active --quiet containerd.service || fatal "containerd.service is not active" systemctl is-active --quiet docker.service || fatal "docker.service is not active" + verify_cdi_refresh_lifecycle nvidia-ctk cdi list | grep -Fxq 'nvidia.com/gpu=all' || fatal "CDI verification failed" sudo docker image inspect "$ACCEPTANCE_IMAGE" >/dev/null 2>&1 || fatal "Digest-pinned acceptance image is missing" [[ -z "$(sudo docker ps -aq)" ]] || fatal "Verification found a leftover Docker container" @@ -642,6 +662,7 @@ verify_host() { systemctl is-active --quiet nvidia-persistenced.service || fatal "nvidia-persistenced.service is not active" systemctl is-active --quiet containerd.service || fatal "containerd.service is not active" systemctl is-active --quiet docker.service || fatal "docker.service is not active" + verify_cdi_refresh_lifecycle id -nG "$user_name" | tr ' ' '\n' | grep -Fxq docker || fatal "${user_name} is not in the docker group" docker info >/dev/null 2>&1 || fatal "${user_name} cannot access Docker; start a new login session" nvidia-ctk cdi list | grep -Fxq 'nvidia.com/gpu=all' || fatal "CDI verification failed" diff --git a/test/install-station-host-preparation.test.ts b/test/install-station-host-preparation.test.ts index 622561e7fb..40aa8245a9 100644 --- a/test/install-station-host-preparation.test.ts +++ b/test/install-station-host-preparation.test.ts @@ -173,6 +173,7 @@ install_packages const { result, output } = runSourced( STATION_PREPARE, ` +sudo() { printf 'SUDO %s\n' "$*"; } run_cdi_test_sudo() { printf 'CDI_TEST\n'; return 0; } refresh_cdi() { printf 'REFRESH_CDI\n'; } ensure_cdi_runtime @@ -180,6 +181,8 @@ ensure_cdi_runtime ); expect(result.status, output).toBe(0); + expect(output).toContain("systemctl enable nvidia-cdi-refresh.path nvidia-cdi-refresh.service"); + expect(output).toContain("systemctl start nvidia-cdi-refresh.path"); expect(output).toContain("cdi_contract=pass_without_configuration_change"); expect(output).not.toContain("REFRESH_CDI"); }); @@ -189,6 +192,7 @@ ensure_cdi_runtime STATION_PREPARE, ` calls=0 +ensure_cdi_refresh_lifecycle() { printf 'ENSURE_CDI_LIFECYCLE\n'; } run_cdi_test_sudo() { calls=$((calls + 1)) printf 'CDI_TEST_%s\n' "$calls" @@ -201,6 +205,7 @@ ensure_cdi_runtime expect(result.status, output).toBe(0); expect(output).toContain("CDI_TEST_1"); + expect(output).toContain("ENSURE_CDI_LIFECYCLE"); expect(output).toContain("REFRESH_CDI"); expect(output).toContain("CDI_TEST_2"); expect(output).toContain("cdi_contract=pass_after_refresh"); @@ -545,6 +550,22 @@ refresh_cdi expect(output).not.toContain("cdi generate"); }); + it("verifies the durable packaged CDI refresh lifecycle", () => { + const { result, output } = runSourced( + STATION_PREPARE, + ` +systemctl() { printf 'SYSTEMCTL %s\n' "$*"; } +verify_cdi_refresh_lifecycle +`, + ); + + expect(result.status, output).toBe(0); + expect(output).toContain("SYSTEMCTL is-enabled --quiet nvidia-cdi-refresh.path"); + expect(output).toContain("SYSTEMCTL is-enabled --quiet nvidia-cdi-refresh.service"); + expect(output).toContain("SYSTEMCTL is-active --quiet nvidia-cdi-refresh.path"); + expect(output).toContain("cdi_refresh_lifecycle=verified"); + }); + it.each(["--check", "--verify"])("keeps %s read-only under HOME", (mode) => { const { home, result, output } = runSourced( STATION_PREPARE, From 26eef0914174d752852311d2d0e5f4911d0c3b90 Mon Sep 17 00:00:00 2001 From: Senthil Kumar Ravichandran Date: Thu, 16 Jul 2026 09:03:17 -0700 Subject: [PATCH 14/74] fix(installer): enforce DGX Station preparation boundaries Signed-off-by: Senthil Kumar Ravichandran --- docs/get-started/prerequisites.mdx | 12 +- docs/get-started/quickstart.mdx | 22 +- scripts/install.sh | 96 +++++- scripts/prepare-dgx-station-host.sh | 246 ++++++++++++-- test/install-express-prompt.test.ts | 75 +++++ test/install-station-host-preparation.test.ts | 308 +++++++++++++++++- 6 files changed, 689 insertions(+), 70 deletions(-) diff --git a/docs/get-started/prerequisites.mdx b/docs/get-started/prerequisites.mdx index aa62e79ab1..65cfa80c0f 100644 --- a/docs/get-started/prerequisites.mdx +++ b/docs/get-started/prerequisites.mdx @@ -41,17 +41,21 @@ If the group change is not active in the current shell, the installer exits with If you choose the native Linux Ollama install path, the onboard wizard also requires `zstd` for Ollama archive extraction. The installer also requires `strings` from `binutils` to verify the OpenShell binary before it continues with OpenShell install work. -On an Ubuntu 24.04 ARM64 DGX Station GB300, accepting express install prepares the host with NVIDIA open driver `610.43.02`, Docker CE `29.6.1` with Buildx, and NVIDIA Container Toolkit `1.19.1`. +On a DGX Station GB300 running the generic Ubuntu 24.04 ARM64 image, accepting express install prepares the host with NVIDIA open driver `610.43.02`, Docker CE `29.6.1` with Buildx, and NVIDIA Container Toolkit `1.19.1`. +DGX OS, NVIDIA BaseOS images, and other Station generations are outside this automatic preparation boundary and stop before host preparation. +On those systems, set `NEMOCLAW_PROVIDER` or `NEMOCLAW_NO_EXPRESS=1` explicitly to continue without Station host automation. The preparation probes package and runtime state first, reuses exact matches, and installs only missing pinned packages, including the NVIDIA Container Toolkit libraries and `nvidia-ctk` CLI. +It permits only the reviewed factory transition from `dkms` `3.0.11-1ubuntu13` to `1:3.4.0-1ubuntu1`. After reboot, preparation enables NVIDIA's packaged CDI refresh path and service, requires the `nvidia.com/gpu=all` device, and verifies it with a real container launch. If the packaged refresh does not produce that device, it uses NVIDIA's documented transient generation fallback at `/var/run/cdi/nvidia.yaml`; it does not create a persistent manual CDI configuration under `/etc`. If an administrator has customized the packaged CDI refresh environment, preparation stops instead of bypassing that policy with default direct generation. If the `docker --gpus all` acceptance probe fails, preparation registers the NVIDIA Docker runtime only when `docker info` diagnoses that runtime as absent; any other launch failure stops without changing daemon configuration. -That registration remains required until the same acceptance probe succeeds through a replacement Docker and NVIDIA runtime integration, so preparation does not remove it automatically. +If registration or a post-change acceptance probe fails, preparation restores the prior Docker daemon configuration; if restoration fails, it reports the backup path and stops. +After successful registration, the runtime remains configured until the same acceptance probe succeeds through a replacement Docker and NVIDIA runtime integration. It requires Secure Boot to be disabled, matching headers for the running kernel, at least 20 GiB free on the root filesystem, and no active agent, inference, or Docker workloads. It does not install a host CUDA toolkit or Docker Compose. -If an existing prerequisite has a different version, preparation stops instead of automatically upgrading or downgrading it. -After installing missing pinned packages, the installer exits with status `10`; reboot, sign in, and rerun the same installer command to resume express setup. +If any other existing prerequisite version differs, preparation stops instead of changing it automatically. +After changing pinned packages, the installer exits with status `10`; reboot, sign in, and run the printed command, which pins the exact accepted NemoClaw commit before resuming express setup. DGX Station remains Deferred. diff --git a/docs/get-started/quickstart.mdx b/docs/get-started/quickstart.mdx index be6d40d3c6..9123c6d01f 100644 --- a/docs/get-started/quickstart.mdx +++ b/docs/get-started/quickstart.mdx @@ -119,11 +119,13 @@ Use these details when your first-run path needs more control. If the installer adds you to the `docker` group, run the printed `newgrp docker` command before you rerun it. On macOS, start Docker Desktop or Colima first. - DGX Spark, DGX Station, and Windows WSL offer an interactive express-install path that chooses a managed local inference option for the platform. - On DGX Station, accepting the express prompt selects the pinned `nemotron-3-ultra-550b-a55b` managed-vLLM recipe and completes onboarding without more provider, model, policy, or sandbox-name choices. - The Station path probes the validated driver, Docker, Buildx, and NVIDIA Container Toolkit versions, reuses exact matches, and installs missing pinned packages without additional choices. + DGX Spark, Station GB300 hosts running the generic Ubuntu 24.04 ARM64 image, and Windows WSL offer an interactive express-install path that chooses a managed local inference option for the platform. + On that Station configuration, accepting the express prompt selects the pinned `nemotron-3-ultra-550b-a55b` managed-vLLM recipe and completes onboarding without more provider, model, policy, or sandbox-name choices. + DGX OS, NVIDIA BaseOS images, and other Station generations stop before host preparation. + Set `NEMOCLAW_PROVIDER` or `NEMOCLAW_NO_EXPRESS=1` explicitly to continue on an unqualified Station without host automation. + It probes the pinned driver, Docker, Buildx, and NVIDIA Container Toolkit versions, reuses exact matches, installs missing pins, and permits only the reviewed factory `dkms` transition. It establishes NVIDIA CDI through the packaged refresh service, with NVIDIA's transient `/var/run/cdi/nvidia.yaml` generation command as a fallback, then proves both CDI and `--gpus all` with real container launches. - After it installs missing pinned packages, the installer exits with status `10` at the validated reboot boundary; reboot, sign in, and rerun the same installer command to resume the accepted recipe without another prompt. + After it changes pinned packages, the installer exits with status `10` at the required reboot boundary; reboot, sign in, and run the printed exact-commit command to resume the accepted recipe without another prompt. This automation does not change Station's Deferred support status; physical end-to-end validation remains open. Pass `--station-deepseek` to use DeepSeek V4 Flash for a Station demo instead. Refer to [Platform Support](../reference/platform-support) and [Choose an Inference Provider](../inference/learn-and-choose/choose-inference-provider) for the current platform behavior. @@ -190,13 +192,17 @@ Use these details when your first-run path needs more control. curl -fsSL https://www.nvidia.com/nemoclaw.sh | bash ``` - On DGX Spark, DGX Station, and Windows WSL, interactive installation offers express install after you accept the third-party software notice. + On DGX Spark, Station GB300 hosts running the generic Ubuntu 24.04 ARM64 image, and Windows WSL, interactive installation offers express install after you accept the third-party software notice. Express install switches onboarding to non-interactive mode, allows `sudo` password prompts for required host changes, and selects the managed local inference path for that platform. DGX Spark uses managed vLLM with `qwen3.6-35b-a3b-nvfp4` by default. DGX Station express install explicitly selects `nemotron-3-ultra-550b-a55b` instead of the Station managed-vLLM profile default, `deepseek-v4-flash`, and discloses the approximately `352 GB` model download before confirmation. - Before onboarding, the Station path checks for NVIDIA open driver `610.43.02`, Docker CE `29.6.1` with Buildx, NVIDIA Container Toolkit `1.19.1`, and the NVIDIA CDI device `nvidia.com/gpu=all`. - It reuses exact versions, installs missing pinned packages, and refuses to automatically upgrade or downgrade an existing mismatched version. - Installing missing pinned packages exits with status `10` for a reboot; after you sign in and rerun the same installer command, the accepted express recipe resumes without another prompt. + Before onboarding, the Station path requires Station GB300 with the generic Ubuntu 24.04 ARM64 image and checks for NVIDIA open driver `610.43.02`, Docker CE `29.6.1` with Buildx, and NVIDIA Container Toolkit `1.19.1`. + DGX OS, NVIDIA BaseOS images, and other Station generations are outside this automatic preparation boundary. + Set `NEMOCLAW_PROVIDER` or `NEMOCLAW_NO_EXPRESS=1` to continue without Station host automation. + Preparation reuses exact versions, installs missing pinned packages, permits only the reviewed `dkms` transition from `3.0.11-1ubuntu13` to `1:3.4.0-1ubuntu1`, and refuses every other mismatched version. + It requires the NVIDIA CDI device `nvidia.com/gpu=all`, using the packaged refresh service or NVIDIA's transient fallback, and verifies CDI and `--gpus all` with real container launches. + If NVIDIA Docker runtime registration or a post-change acceptance probe fails, preparation restores the prior Docker daemon configuration. + Changing pinned packages exits with status `10` for a reboot; after you sign in and run the printed exact-commit command, the accepted express recipe resumes without another prompt. This automation does not change Station's Deferred support status; physical end-to-end validation remains open. To select DeepSeek V4 Flash while retaining the one-confirmation Station express flow, run `curl -fsSL https://www.nvidia.com/nemoclaw.sh | bash -s -- --station-deepseek`. Unless `NEMOCLAW_POLICY_TIER` is set, express install applies policy in `suggested` mode with the `balanced` tier, including the base sandbox policy and supported package, model, web-search, and local-inference presets. diff --git a/scripts/install.sh b/scripts/install.sh index 7bed8d7260..66e49b5178 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -2859,16 +2859,37 @@ detect_express_platform() { fi case "$model" in *DGX*Spark*) printf "DGX Spark" ;; - *DGX*Station* | *Station*GB300*) printf "DGX Station" ;; + *Station*GB300*) + if [ -e /etc/dgx-release ] || [ -L /etc/dgx-release ]; then + printf "Unsupported DGX Station OS" + else + printf "DGX Station" + fi + ;; + *DGX*Station*) printf "Unsupported DGX Station generation" ;; *) ;; esac } +validate_express_platform_boundary() { + case "${1:-}" in + "Unsupported DGX Station OS") + if [ "${NEMOCLAW_NO_EXPRESS:-}" = "1" ] || [ -n "${NEMOCLAW_PROVIDER:-}" ]; then return 0; fi + error "DGX OS/BaseOS is outside the validated Station express boundary. Use the generic Ubuntu 24.04 ARM64 image." + ;; + "Unsupported DGX Station generation") + if [ "${NEMOCLAW_NO_EXPRESS:-}" = "1" ] || [ -n "${NEMOCLAW_PROVIDER:-}" ]; then return 0; fi + error "This DGX Station generation is outside the validated Station GB300 express boundary." + ;; + esac +} + STATION_ULTRA_VLLM_MODEL="nemotron-3-ultra-550b-a55b" STATION_ULTRA_SERVED_MODEL="nvidia/nemotron-3-ultra-550b-a55b" STATION_DEEPSEEK_VLLM_MODEL="deepseek-v4-flash" STATION_DEEPSEEK_SERVED_MODEL="deepseek-ai/DeepSeek-V4-Flash" _SELECTED_EXPRESS_PLATFORM="" +_STATION_EXPRESS_RESUME_REVISION="" normalize_station_vllm_model() { printf "%s" "${1:-}" | tr '[:upper:]' '[:lower:]' | sed 's/^[[:space:]]*//; s/[[:space:]]*$//' @@ -2905,6 +2926,7 @@ validate_station_deepseek_override() { preflight_explicit_express_flags() { local platform platform="$(detect_express_platform)" + validate_express_platform_boundary "$platform" validate_station_deepseek_override "$platform" } @@ -2951,23 +2973,66 @@ validate_station_express_resume_model() { [[ ${#model} -le 255 && "$model" =~ ^[A-Za-z0-9][A-Za-z0-9._/-]*$ ]] } +validate_station_express_resume_revision() { + [[ "${1:-}" =~ ^[0-9a-f]{40}$ ]] +} + +station_installer_revision() { + local revision + revision="$(git -C "${SCRIPT_DIR}/.." rev-parse --verify 'HEAD^{commit}' 2>/dev/null)" \ + || error "Could not resolve the exact NemoClaw revision for DGX Station reboot resume." + validate_station_express_resume_revision "$revision" \ + || error "Resolved NemoClaw revision is invalid: ${revision}" + printf '%s' "$revision" +} + +portable_file_mode() { + stat -c '%a' "$1" 2>/dev/null || stat -f '%Lp' "$1" +} + +assert_station_express_resume_file_safe() { + local state_file=$1 state_dir mode + state_dir="$(dirname "$state_file")" + [[ -d "$state_dir" && -O "$state_dir" ]] \ + || error "DGX Station express resume directory is not owned by the current user: ${state_dir}" + mode="$(portable_file_mode "$state_dir")" \ + || error "Could not inspect DGX Station express resume directory permissions: ${state_dir}" + (((8#$mode & 0077) == 0)) \ + || error "DGX Station express resume directory must not be accessible by group or other users: ${state_dir}" + [[ -f "$state_file" && -O "$state_file" ]] \ + || error "DGX Station express resume state must be a regular file owned by the current user: ${state_file}" + mode="$(portable_file_mode "$state_file")" \ + || error "Could not inspect DGX Station express resume state permissions: ${state_file}" + [[ "$mode" == "600" ]] || error "DGX Station express resume state must have mode 0600: ${state_file}" +} + load_station_express_resume() { - local state_file model line_count + local state_file revision_line model_line line_count saved_revision current_revision state_file="$(station_express_resume_file)" || return 1 assert_nemoclaw_state_path_safe "$state_file" - [[ -f "$state_file" ]] || return 1 + [[ -e "$state_file" || -L "$state_file" ]] || return 1 + assert_station_express_resume_file_safe "$state_file" line_count="$(wc -l <"$state_file" | tr -d '[:space:]')" - IFS= read -r model <"$state_file" || model="" - if [[ "$line_count" != "1" ]] || ! validate_station_express_resume_model "$model"; then + revision_line="$(sed -n '1p' "$state_file")" + model_line="$(sed -n '2p' "$state_file")" + saved_revision="${revision_line#revision=}" + NEMOCLAW_VLLM_MODEL="${model_line#model=}" + if [[ "$line_count" != "2" || "$revision_line" != "revision=${saved_revision}" || "$model_line" != "model=${NEMOCLAW_VLLM_MODEL}" ]] \ + || ! validate_station_express_resume_revision "$saved_revision" \ + || ! validate_station_express_resume_model "$NEMOCLAW_VLLM_MODEL"; then error "DGX Station express resume state is invalid. Remove ${state_file} and rerun the installer." fi - NEMOCLAW_VLLM_MODEL="$model" + current_revision="$(station_installer_revision)" + if [[ "$current_revision" != "$saved_revision" ]]; then + error "DGX Station express resume requires NemoClaw revision ${saved_revision}, but this installer is ${current_revision}. Rerun with: curl -fsSL https://www.nvidia.com/nemoclaw.sh | NEMOCLAW_INSTALL_TAG=${saved_revision} bash" + fi export NEMOCLAW_VLLM_MODEL } save_station_express_resume() { - local state_file state_dir temp_file model="${NEMOCLAW_VLLM_MODEL:-}" + local state_file state_dir temp_file revision model="${NEMOCLAW_VLLM_MODEL:-}" validate_station_express_resume_model "$model" || error "Cannot save an invalid DGX Station express model selector." + revision="$(station_installer_revision)" state_file="$(station_express_resume_file)" || error "Could not resolve NemoClaw state for DGX Station express resume." state_dir="$(ensure_nemoclaw_state_dir)" || error "Could not prepare NemoClaw state for DGX Station express resume." assert_nemoclaw_state_path_safe "$state_file" @@ -2976,7 +3041,7 @@ save_station_express_resume() { rm -f "$temp_file" error "Could not secure DGX Station express resume state under ${state_dir}." } - if ! printf '%s\n' "$model" >"$temp_file"; then + if ! printf 'revision=%s\nmodel=%s\n' "$revision" "$model" >"$temp_file"; then rm -f "$temp_file" error "Could not write DGX Station express resume state under ${state_dir}." fi @@ -2984,12 +3049,17 @@ save_station_express_resume() { rm -f "$temp_file" error "Could not publish DGX Station express resume state under ${state_dir}." fi + assert_station_express_resume_file_safe "$state_file" + _STATION_EXPRESS_RESUME_REVISION="$revision" } clear_station_express_resume() { local state_file state_file="$(station_express_resume_file)" || return 0 assert_nemoclaw_state_path_safe "$state_file" + [[ -e "$state_file" || -L "$state_file" ]] || return 0 + [[ -f "$state_file" && -O "$state_file" ]] \ + || error "Refusing to remove invalid DGX Station express resume state: ${state_file}" rm -f "$state_file" } @@ -3041,9 +3111,12 @@ ensure_station_express_host() { ;; 10) save_station_express_resume + local revision + revision="${_STATION_EXPRESS_RESUME_REVISION}" warn "DGX Station host prerequisites were installed and require a reboot." info "Run: sudo reboot" - info "After signing in again, rerun the same NemoClaw installer command; express setup resumes without another choice." + info "After signing in again, rerun the accepted revision:" + info "curl -fsSL https://www.nvidia.com/nemoclaw.sh | NEMOCLAW_INSTALL_TAG=${revision} bash" exit 10 ;; *) @@ -3094,7 +3167,7 @@ describe_express_install() { inference_summary="managed local vLLM with NVIDIA Nemotron 3 Ultra 550B" inference_disclosure="Managed vLLM pulls the pinned Station image and approximately 352 GB model, then runs a local inference container." fi - printf " Station host setup reuses exact prerequisite versions, installs missing pinned driver, Docker, and NVIDIA Container Toolkit packages, and may require one reboot.\n" + printf " Station host setup reuses exact prerequisite versions, applies the reviewed factory DKMS transition when present, installs missing pinned driver, Docker, and NVIDIA Container Toolkit packages, and may require one reboot.\n" printf " DGX Station remains Deferred; this recipe has not completed end-to-end validation on physical hardware.\n" sandbox_summary="${NEMOCLAW_SANDBOX_NAME:-my-assistant}" ;; @@ -3138,6 +3211,7 @@ describe_express_install() { maybe_offer_express_install() { local platform platform="$(detect_express_platform)" + validate_express_platform_boundary "$platform" validate_station_deepseek_override "$platform" # Not on a platform we have an express recipe for — say nothing. if [ -z "$platform" ]; then @@ -3146,10 +3220,12 @@ maybe_offer_express_install() { # On a supported platform but a skip condition applies — explain why so # the user understands they could have gotten express otherwise. if [ "${NEMOCLAW_NO_EXPRESS:-}" = "1" ]; then + if [ "$platform" = "DGX Station" ]; then clear_station_express_resume; fi info "Detected ${platform}. Skipping express prompt (NEMOCLAW_NO_EXPRESS=1)." return 0 fi if [ -n "${NEMOCLAW_PROVIDER:-}" ]; then + if [ "$platform" = "DGX Station" ]; then clear_station_express_resume; fi info "Detected ${platform}. Skipping express prompt (NEMOCLAW_PROVIDER=${NEMOCLAW_PROVIDER} already set)." return 0 fi diff --git a/scripts/prepare-dgx-station-host.sh b/scripts/prepare-dgx-station-host.sh index 7baa54ca3e..ec54744f2a 100755 --- a/scripts/prepare-dgx-station-host.sh +++ b/scripts/prepare-dgx-station-host.sh @@ -20,13 +20,17 @@ readonly DOCKER_KEY_FINGERPRINT="9DC858229FC7DD38854AE2D88D81803C0EBFCD88" readonly DRIVER_VERSION="610.43.02" readonly DOCKER_VERSION="29.6.1" readonly TOOLKIT_VERSION="1.19.1" +readonly FACTORY_DKMS_VERSION="3.0.11-1ubuntu13" +readonly TARGET_DKMS_VERSION="1:3.4.0-1ubuntu1" readonly ACCEPTANCE_IMAGE="ubuntu@sha256:7f622ca8766bccb22f04242ecb6f19f770b2f08827dc4b8c707de5e78a6da7ab" readonly STATE_DIR="${HOME}/.local/state/station-bootstrap" readonly INSTALL_BOOT_MARKER="${STATE_DIR}/install-boot-id" readonly CDI_REFRESH_ENV_FILE="/etc/nvidia-container-toolkit/nvidia-cdi-refresh.env" +readonly TRANSIENT_CDI_SPEC_PATH="/var/run/cdi/nvidia.yaml" +readonly TRANSIENT_CDI_SPEC_CANONICAL_PATH="/run/cdi/nvidia.yaml" readonly -a PACKAGE_SPECS=( - "dkms=1:3.4.0-1ubuntu1" + "dkms=${TARGET_DKMS_VERSION}" "nvidia-driver-pinning-610=610-2ubuntu1" "nvidia-driver-open=610.43.02-1ubuntu1" "containerd.io=2.2.6-1~ubuntu.24.04~noble" @@ -87,13 +91,12 @@ is_valid_mode() { is_station_product() { local product=${1:-} - [[ "$product" == *"Station GB300"* || "$product" == *"DGX Station"* ]] + [[ "$product" == *"Station"* && "$product" == *"GB300"* ]] } -is_expected_failed_unit() { +is_preparation_critical_unit() { case "${1:-}" in - cloud-init.service | fwupd-refresh.service | NetworkManager-wait-online.service | systemd-networkd-wait-online.service | \ - sssd-autofs.socket | sssd-nss.socket | sssd-pam-priv.socket | sssd-pam.socket) + containerd.service | docker.service | nvidia-cdi-refresh.service | nvidia-persistenced.service) return 0 ;; *) return 1 ;; @@ -145,6 +148,8 @@ package_state() { printf 'missing\n' elif [[ "$actual" == "$expected" ]]; then printf 'exact\n' + elif [[ "$name" == "dkms" && "$actual" == "$FACTORY_DKMS_VERSION" && "$expected" == "$TARGET_DKMS_VERSION" ]]; then + printf 'approved-transition\n' else printf 'mismatch\n' fi @@ -154,6 +159,13 @@ assert_no_package_mismatches() { local spec state name expected actual mismatch=0 for spec in "${PACKAGE_SPECS[@]}"; do state="$(package_state "$spec")" + if [[ "$state" == "approved-transition" ]]; then + name="$(package_name "$spec")" + expected="$(package_expected_version "$spec")" + actual="$(installed_version "$name")" + info "package=${name} status=approved_transition actual=${actual} expected=${expected}" + continue + fi [[ "$state" == "mismatch" ]] || continue name="$(package_name "$spec")" expected="$(package_expected_version "$spec")" @@ -161,7 +173,7 @@ assert_no_package_mismatches() { warn "package=${name} status=mismatch actual=${actual} expected=${expected}" mismatch=1 done - ((mismatch == 0)) || fatal "Existing Station prerequisite versions differ from the validated pins; refusing to upgrade or downgrade them automatically" + ((mismatch == 0)) || fatal "Existing Station prerequisite versions differ from the validated pins or approved factory transition; refusing to change them automatically" } all_packages_exact() { @@ -185,6 +197,10 @@ require_command() { command -v "$1" >/dev/null 2>&1 || fatal "Required command is missing: $1" } +file_mode() { + stat -c '%a' "$1" 2>/dev/null || stat -f '%Lp' "$1" +} + check_platform() { local arch product arch="$(uname -m)" @@ -195,6 +211,8 @@ check_platform() { source /etc/os-release [[ "${ID:-}" == "ubuntu" && "${VERSION_ID:-}" == "24.04" ]] \ || fatal "Expected Ubuntu 24.04, found ${PRETTY_NAME:-unknown}" + [[ ! -e /etc/dgx-release && ! -L /etc/dgx-release ]] \ + || fatal "DGX OS/BaseOS is outside this recipe's validated boundary; use the generic Ubuntu 24.04 ARM64 image" product="$(/dev/null | awk 'NF {print $1}') + failed_output="$(systemctl --failed --no-legend --plain 2>/dev/null)" \ + || fatal "Unable to inspect failed system services" + while IFS= read -r unit; do + [[ -n "$unit" ]] && units+=("$unit") + done < <(awk 'NF {print $1}' <<<"$failed_output") if ((${#units[@]} == 0)); then info "failed_units=none" return 0 fi for unit in "${units[@]}"; do - if is_expected_failed_unit "$unit"; then - warn "pre-existing OEM failed unit allowed for this pilot: ${unit}" - elif is_driver_transitional_unit "$unit" && all_packages_exact; then + if is_driver_transitional_unit "$unit" && all_packages_exact && ! driver_loaded_exact; then warn "driver unit failure allowed only until post-reboot verification: ${unit}" + elif is_preparation_critical_unit "$unit"; then + warn "failed preparation-critical unit: ${unit}" + blocking=1 else - warn "unexpected failed unit: ${unit}" - unexpected=1 + warn "pre-existing failed unit outside the Station preparation transaction: ${unit}" fi done - ((unexpected == 0)) || fatal "Unexpected failed units block Station preparation" + ((blocking == 0)) || fatal "Failed driver or container runtime units block Station preparation" } check_no_workloads() { @@ -302,25 +324,55 @@ driver_loaded_exact() { } assert_station_state_dir_safe() { - local path + local path mode for path in "${HOME}/.local" "${HOME}/.local/state" "$STATE_DIR"; do [[ ! -L "$path" ]] || fatal "Refusing symbolic link in Station bootstrap state path: ${path}" + [[ ! -e "$path" || -d "$path" ]] || fatal "Station bootstrap state path is not a directory: ${path}" + if [[ -e "$path" ]]; then + [[ -O "$path" ]] || fatal "Station bootstrap state path is not owned by the current user: ${path}" + mode="$(file_mode "$path")" + (((8#$mode & 0022) == 0)) || fatal "Station bootstrap state path is group- or other-writable: ${path}" + fi done } +assert_install_boot_marker_safe() { + local mode + [[ ! -L "$INSTALL_BOOT_MARKER" ]] || fatal "Refusing symbolic link for Station bootstrap boot marker: ${INSTALL_BOOT_MARKER}" + [[ ! -e "$INSTALL_BOOT_MARKER" || -f "$INSTALL_BOOT_MARKER" ]] \ + || fatal "Station bootstrap boot marker is not a regular file: ${INSTALL_BOOT_MARKER}" + if [[ -e "$INSTALL_BOOT_MARKER" ]]; then + [[ -O "$INSTALL_BOOT_MARKER" ]] || fatal "Station bootstrap boot marker is not owned by the current user" + mode="$(file_mode "$INSTALL_BOOT_MARKER")" + [[ "$mode" == "600" ]] || fatal "Station bootstrap boot marker must have mode 0600" + fi +} + write_install_boot_marker() { + local temp_file assert_station_state_dir_safe mkdir -p "$STATE_DIR" assert_station_state_dir_safe chmod 0700 "$STATE_DIR" - cp /proc/sys/kernel/random/boot_id "$INSTALL_BOOT_MARKER" - chmod 0600 "$INSTALL_BOOT_MARKER" + assert_install_boot_marker_safe + temp_file="$(mktemp "${INSTALL_BOOT_MARKER}.tmp.XXXXXX")" + chmod 0600 "$temp_file" + tr -d '[:space:]' "$temp_file" + printf '\n' >>"$temp_file" + mv -f "$temp_file" "$INSTALL_BOOT_MARKER" + assert_install_boot_marker_safe } install_boot_marker_matches_current_boot() { + local installed_boot current_boot assert_station_state_dir_safe - [[ -r "$INSTALL_BOOT_MARKER" ]] || return 1 - [[ "$(tr -d '[:space:]' <"$INSTALL_BOOT_MARKER")" == "$(tr -d '[:space:]' { + const result = spawnSync( + "bash", + [ + "--noprofile", + "--norc", + "-c", + ` +source "$INSTALLER_UNDER_TEST" >/dev/null +validate_express_platform_boundary "$EXPRESS_PLATFORM" +printf 'NON_EXPRESS_ALLOWED\n' +`, + ], + { + cwd: path.join(import.meta.dirname, ".."), + encoding: "utf-8", + env: { + HOME: fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-express-platform-override-")), + PATH: TEST_SYSTEM_PATH, + INSTALLER_UNDER_TEST: INSTALLER_PAYLOAD, + EXPRESS_PLATFORM: platform, + ...overrides, + }, + }, + ); + const output = `${result.stdout}${result.stderr}`; + + expect(result.status, output).toBe(0); + expect(output).toContain("NON_EXPRESS_ALLOWED"); + }); + it.each([ ["NEMOCLAW_NO_EXPRESS", "1", /cannot be combined with NEMOCLAW_NO_EXPRESS=1/], ["NON_INTERACTIVE", "1", /cannot be combined with --non-interactive/], @@ -444,6 +478,47 @@ detect_express_platform } }); + it("classifies older DGX Station generations as unsupported", () => { + const result = detectExpressPlatformForProductName("NVIDIA DGX Station A100"); + + expect(result.status, `${result.stdout}${result.stderr}`).toBe(0); + expect(result.stdout).toBe("Unsupported DGX Station generation"); + }); + + it.each([ + "Unsupported DGX Station OS", + "Unsupported DGX Station generation", + ])("rejects %s before the express prompt", (platform) => { + const result = spawnSync( + "bash", + [ + "--noprofile", + "--norc", + "-c", + ` +source "$INSTALLER_UNDER_TEST" >/dev/null +validate_express_platform_boundary "$EXPRESS_PLATFORM" +printf 'PROMPT_REACHED\n' +`, + ], + { + cwd: path.join(import.meta.dirname, ".."), + encoding: "utf-8", + env: { + HOME: fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-express-platform-reject-")), + PATH: TEST_SYSTEM_PATH, + INSTALLER_UNDER_TEST: INSTALLER_PAYLOAD, + EXPRESS_PLATFORM: platform, + }, + }, + ); + const output = `${result.stdout}${result.stderr}`; + + expect(result.status, output).not.toBe(0); + expect(output).toMatch(/outside the validated Station/); + expect(output).not.toContain("PROMPT_REACHED"); + }); + it("maps Windows WSL express install to Windows-host Ollama", () => { const result = runExpressPromptWithTty("\n", "pipe", "Windows WSL"); const output = `${result.stdout}${result.stderr}`; diff --git a/test/install-station-host-preparation.test.ts b/test/install-station-host-preparation.test.ts index 40aa8245a9..6dcecc400d 100644 --- a/test/install-station-host-preparation.test.ts +++ b/test/install-station-host-preparation.test.ts @@ -11,6 +11,7 @@ import { INSTALLER_PAYLOAD, TEST_SYSTEM_PATH } from "./helpers/installer-sourced const REPO_ROOT = path.resolve(import.meta.dirname, ".."); const PUBLIC_BOOTSTRAP = path.join(REPO_ROOT, "install.sh"); const STATION_PREPARE = path.join(REPO_ROOT, "scripts", "prepare-dgx-station-host.sh"); +const STATION_REVISION = "a".repeat(40); const STATION_DOCS = [ path.join(REPO_ROOT, "docs", "get-started", "prerequisites.mdx"), path.join(REPO_ROOT, "docs", "get-started", "quickstart.mdx"), @@ -41,7 +42,13 @@ describe("DGX Station host preparation", () => { it("keeps documented Station pins and Deferred status aligned", () => { const helper = fs.readFileSync(STATION_PREPARE, "utf-8"); const docs = STATION_DOCS.map((doc) => fs.readFileSync(doc, "utf-8")); - const pinnedValues = ["DRIVER_VERSION", "DOCKER_VERSION", "TOOLKIT_VERSION"].map((name) => { + const pinnedValues = [ + "DRIVER_VERSION", + "DOCKER_VERSION", + "TOOLKIT_VERSION", + "FACTORY_DKMS_VERSION", + "TARGET_DKMS_VERSION", + ].map((name) => { const value = helper.match(new RegExp(`readonly ${name}="([^"]+)"`))?.[1]; expect(value, `${name} must remain declared in the Station helper`).toBeTruthy(); return value as string; @@ -74,6 +81,50 @@ package_state 'docker-ce=5:29.6.1-1~ubuntu.24.04~noble' expect(result.stdout.trim()).toBe(expected); }); + it.each([ + ["Dell Pro Max with Station GB300", true], + ["NVIDIA DGX Station GB300", true], + ["NVIDIA DGX Station A100", false], + ["Dell Pro Max with Station GB200", false], + ["Dell Pro Max with GB300", false], + ])("accepts only Station GB300 DMI: %s", (product, accepted) => { + const { result } = runSourced(STATION_PREPARE, `is_station_product "$PRODUCT"`, { + PRODUCT: product, + }); + + expect(result.status === 0).toBe(accepted); + }); + + it("allows only the reviewed factory DKMS transition", () => { + const approved = runSourced( + STATION_PREPARE, + ` +installed_version() { + if [[ "$1" == "dkms" ]]; then printf '%s' "$DKMS_ACTUAL"; fi +} +package_state 'dkms=1:3.4.0-1ubuntu1' +assert_no_package_mismatches +`, + { DKMS_ACTUAL: "3.0.11-1ubuntu13" }, + ); + expect(approved.result.status, approved.output).toBe(0); + expect(approved.output).toContain("approved-transition"); + expect(approved.output).toContain("status=approved_transition"); + + const arbitrary = runSourced( + STATION_PREPARE, + ` +installed_version() { + if [[ "$1" == "dkms" ]]; then printf '%s' "$DKMS_ACTUAL"; fi +} +assert_no_package_mismatches +`, + { DKMS_ACTUAL: "3.2.0-1" }, + ); + expect(arbitrary.result.status, arbitrary.output).not.toBe(0); + expect(arbitrary.output).toMatch(/dkms status=mismatch/); + }); + it("refuses to change an existing mismatched prerequisite", () => { const { result, output } = runSourced( STATION_PREPARE, @@ -87,7 +138,42 @@ assert_no_package_mismatches expect(result.status, output).not.toBe(0); expect(output).toMatch(/docker-ce status=mismatch/); - expect(output).toMatch(/refusing to upgrade or downgrade them automatically/); + expect(output).toMatch(/refusing to change them automatically/); + }); + + it("warns about unrelated failed units but blocks failed runtime units", () => { + const unrelated = runSourced( + STATION_PREPARE, + ` +systemctl() { printf 'cloud-init.service loaded failed failed Cloud init\n'; } +check_failed_units +`, + ); + expect(unrelated.result.status, unrelated.output).toBe(0); + expect(unrelated.output).toMatch(/outside the Station preparation transaction/); + + const critical = runSourced( + STATION_PREPARE, + ` +systemctl() { printf 'docker.service loaded failed failed Docker\n'; } +check_failed_units +`, + ); + expect(critical.result.status, critical.output).not.toBe(0); + expect(critical.output).toMatch(/failed preparation-critical unit: docker.service/); + }); + + it("fails closed when failed-service inspection is unavailable", () => { + const { result, output } = runSourced( + STATION_PREPARE, + ` +systemctl() { return 1; } +check_failed_units +`, + ); + + expect(result.status, output).not.toBe(0); + expect(output).toMatch(/Unable to inspect failed system services/); }); it("reuses exact packages and proceeds directly to runtime probes", () => { @@ -114,7 +200,7 @@ run_apply expect(output).toContain("APPLY_RESULT=COMPLETE"); }); - it("installs only missing packages and returns the reboot-required contract", () => { + it("applies the reviewed factory DKMS transition and returns the reboot-required contract", () => { const { result, output } = runSourced( STATION_PREPARE, ` @@ -122,9 +208,12 @@ common_preflight() { :; } require_command() { :; } acquire_sudo() { :; } all_packages_exact() { return 1; } -assert_no_package_mismatches() { printf 'NO_MISMATCHES\n'; } +installed_version() { + if [[ "$1" == "dkms" ]]; then printf '3.0.11-1ubuntu13'; fi +} install_packages() { printf 'INSTALL_PACKAGES\n'; } ensure_docker_group() { printf 'ENSURE_DOCKER_GROUP\n'; } +check_no_workloads() { printf 'RECHECK_ALL_WORKLOADS\n'; } write_install_boot_marker() { printf 'WRITE_BOOT_MARKER\n'; } sudo() { printf 'SUDO %s\n' "$*"; } run_apply @@ -132,9 +221,10 @@ run_apply ); expect(result.status, output).toBe(10); - expect(output).toContain("NO_MISMATCHES"); + expect(output).toContain("package=dkms status=approved_transition"); expect(output).toContain("INSTALL_PACKAGES"); expect(output).toContain("ENSURE_DOCKER_GROUP"); + expect(output).toContain("RECHECK_ALL_WORKLOADS"); expect(output).toContain("WRITE_BOOT_MARKER"); expect(output).toContain( "systemctl enable containerd.service docker.service nvidia-cdi-refresh.path nvidia-cdi-refresh.service", @@ -149,6 +239,7 @@ run_apply configure_repositories() { printf 'CONFIGURE_REPOSITORIES\n'; } validate_package_availability() { printf 'VALIDATE_PACKAGES\n'; } simulate_install() { printf 'SIMULATE_INSTALL\n'; } +check_no_workloads() { printf 'RECHECK_ALL_WORKLOADS\n'; } package_is_exact() { return 0; } sudo() { printf 'SUDO %s\n' "$*"; } install_packages @@ -158,6 +249,7 @@ install_packages expect(result.status, output).toBe(0); expect(output).toContain("apt-get update"); expect(output).toContain("apt-get install -y --no-install-recommends"); + expect(output).toContain("RECHECK_ALL_WORKLOADS"); for (const spec of [ "libnvidia-container-tools=1.19.1-1", "libnvidia-container1=1.19.1-1", @@ -173,6 +265,7 @@ install_packages const { result, output } = runSourced( STATION_PREPARE, ` +check_no_workloads() { printf 'RECHECK_ALL_WORKLOADS\n'; } sudo() { printf 'SUDO %s\n' "$*"; } run_cdi_test_sudo() { printf 'CDI_TEST\n'; return 0; } refresh_cdi() { printf 'REFRESH_CDI\n'; } @@ -181,6 +274,7 @@ ensure_cdi_runtime ); expect(result.status, output).toBe(0); + expect(output).toContain("RECHECK_ALL_WORKLOADS"); expect(output).toContain("systemctl enable nvidia-cdi-refresh.path nvidia-cdi-refresh.service"); expect(output).toContain("systemctl start nvidia-cdi-refresh.path"); expect(output).toContain("cdi_contract=pass_without_configuration_change"); @@ -283,6 +377,7 @@ check_no_workloads const { result, output } = runSourced( STATION_PREPARE, ` +assert_root_directory_safe() { :; } installed_version() { printf '2.0-1'; } ensure_cuda_keyring "$HOME/cuda-keyring.deb" `, @@ -296,6 +391,8 @@ ensure_cuda_keyring "$HOME/cuda-keyring.deb" const { result, output } = runSourced( STATION_PREPARE, ` +assert_root_directory_safe() { :; } +assert_root_regular_file_safe() { :; } installed_version() { printf '1.1-1'; } dpkg() { :; } curl() { printf 'DOWNLOAD\n'; } @@ -317,6 +414,8 @@ ensure_cuda_keyring "$HOME/cuda-keyring.deb" ` printf 'validated\n' >"$HOME/source" cp "$HOME/source" "$HOME/target" +assert_root_directory_safe() { :; } +assert_root_regular_file_safe() { :; } sudo() { "$@"; } install_exact_file_or_reuse "$HOME/source" "$HOME/target" 0644 test_repository_file printf 'modified\n' >"$HOME/target" @@ -329,6 +428,53 @@ install_exact_file_or_reuse "$HOME/source" "$HOME/target" 0644 test_repository_f expect(output).toMatch(/refusing to overwrite/); }); + it("rejects privileged files with unsafe ownership, mode, type, or parent metadata", () => { + for (const metadata of ["1000 0 644", "0 0 666"]) { + const { result, output } = runSourced( + STATION_PREPARE, + ` +sudo() { + if [[ "$1" == "test" ]]; then return 0; fi + if [[ "$1" == "stat" ]]; then printf '%s\n' "$ROOT_METADATA"; return 0; fi + return 1 +} +assert_root_regular_file_safe /etc/example 0644 test_file +`, + { ROOT_METADATA: metadata }, + ); + expect(result.status, output).not.toBe(0); + expect(output).toMatch(/root-owned regular file/); + } + + const unsafeType = runSourced( + STATION_PREPARE, + ` +sudo() { + [[ "$*" == "test ! -L /etc/example" ]] && return 0 + [[ "$*" == "test -f /etc/example" ]] && return 1 + return 1 +} +assert_root_regular_file_safe /etc/example 0644 test_file +`, + ); + expect(unsafeType.result.status, unsafeType.output).not.toBe(0); + expect(unsafeType.output).toMatch(/root-owned regular file/); + + const unsafeParent = runSourced( + STATION_PREPARE, + ` +sudo() { + if [[ "$1" == "test" ]]; then return 0; fi + if [[ "$1" == "stat" ]]; then printf '0 0 777\n'; return 0; fi + return 1 +} +assert_root_directory_safe /etc/apt/keyrings test_directory +`, + ); + expect(unsafeParent.result.status, unsafeParent.output).not.toBe(0); + expect(unsafeParent.output).toMatch(/not group- or other-writable/); + }); + it("requires a new login after adding Docker group membership", () => { const { result, output } = runSourced( STATION_PREPARE, @@ -356,6 +502,8 @@ run_apply STATION_PREPARE, ` generated=0 +check_no_workloads() { printf 'RECHECK_ALL_WORKLOADS\n'; } +assert_transient_cdi_output_safe() { printf 'CDI_OUTPUT_SAFE %s\n' "$1"; } sudo() { printf 'SUDO %s\n' "$*" if [[ "$*" == "systemctl restart nvidia-cdi-refresh.service" ]]; then return 1; fi @@ -373,6 +521,7 @@ refresh_cdi expect(result.status, output).toBe(0); expect(output).toContain("systemctl status nvidia-cdi-refresh.service --no-pager"); expect(output).toContain("journalctl -u nvidia-cdi-refresh.service --no-pager -n 50"); + expect(output).toContain("RECHECK_ALL_WORKLOADS"); expect(output).toContain("nvidia-ctk cdi generate --output=/var/run/cdi/nvidia.yaml"); expect(output).toContain("cdi=nvidia.com/gpu=all source=direct_transient_fallback"); expect(output).not.toContain("/etc/cdi"); @@ -383,6 +532,8 @@ refresh_cdi STATION_PREPARE, ` generated=0 +check_no_workloads() { printf 'RECHECK_ALL_WORKLOADS\n'; } +assert_transient_cdi_output_safe() { printf 'CDI_OUTPUT_SAFE %s\n' "$1"; } sudo() { printf 'SUDO %s\n' "$*" if [[ "$*" == "test -e $CDI_REFRESH_ENV_FILE" ]]; then return 1; fi @@ -397,6 +548,7 @@ refresh_cdi ); expect(result.status, output).toBe(0); + expect(output).toContain("RECHECK_ALL_WORKLOADS"); expect(output).toMatch(/completed without advertising nvidia\.com\/gpu=all/); expect(output).toContain("nvidia-ctk cdi generate --output=/var/run/cdi/nvidia.yaml"); expect(output).toContain("cdi=nvidia.com/gpu=all source=direct_transient_fallback"); @@ -406,6 +558,8 @@ refresh_cdi const { result, output } = runSourced( STATION_PREPARE, ` +check_no_workloads() { printf 'RECHECK_ALL_WORKLOADS\n'; } +assert_transient_cdi_output_safe() { printf 'CDI_OUTPUT_SAFE %s\n' "$1"; } sudo() { printf 'SUDO %s\n' "$*" if [[ "$*" == "systemctl restart nvidia-cdi-refresh.service" ]]; then return 1; fi @@ -418,6 +572,9 @@ refresh_cdi ); expect(result.status, output).not.toBe(0); + expect(output).toContain("systemctl status nvidia-cdi-refresh.service --no-pager"); + expect(output).toContain("journalctl -u nvidia-cdi-refresh.service --no-pager -n 50"); + expect(output).toContain("RECHECK_ALL_WORKLOADS"); expect(output).toContain("nvidia-ctk cdi generate --output=/var/run/cdi/nvidia.yaml"); expect(output).toMatch(/Direct transient CDI generation failed/); }); @@ -426,6 +583,8 @@ refresh_cdi const { result, output } = runSourced( STATION_PREPARE, ` +check_no_workloads() { printf 'RECHECK_ALL_WORKLOADS\n'; } +assert_transient_cdi_output_safe() { printf 'CDI_OUTPUT_SAFE %s\n' "$1"; } sudo() { printf 'SUDO %s\n' "$*" if [[ "$*" == "systemctl restart nvidia-cdi-refresh.service" ]]; then return 1; fi @@ -446,6 +605,7 @@ refresh_cdi const { result, output } = runSourced( STATION_PREPARE, ` +check_no_workloads() { printf 'RECHECK_ALL_WORKLOADS\n'; } sudo() { printf 'SUDO %s\n' "$*" if [[ "$*" == "systemctl restart nvidia-cdi-refresh.service" ]]; then return 1; fi @@ -461,6 +621,26 @@ refresh_cdi expect(output).not.toContain("nvidia-ctk cdi generate --output=/var/run/cdi/nvidia.yaml"); }); + it("rejects a symbolic link at the privileged transient CDI output", () => { + const { result, output } = runSourced( + STATION_PREPARE, + ` +ensure_root_directory_safe() { printf 'ENSURE_ROOT_DIRECTORY_SAFE\n'; } +readlink() { printf '/run/cdi\n'; } +sudo() { + printf 'SUDO %s\n' "$*" + if [[ "$*" == "test ! -L /run/cdi/nvidia.yaml" ]]; then return 1; fi + return 0 +} +assert_transient_cdi_output_safe 0 +`, + ); + + expect(result.status, output).not.toBe(0); + expect(output).toContain("ENSURE_ROOT_DIRECTORY_SAFE"); + expect(output).toMatch(/must not be a symbolic link/); + }); + it("rechecks every workload gate immediately before Docker runtime mutation", () => { const { result, output } = runSourced( STATION_PREPARE, @@ -513,9 +693,16 @@ run_gpus_test_sudo() { run_cdi_test_sudo() { return 0; } docker_has_nvidia_runtime_sudo() { return 1; } check_no_workloads() { printf 'RECHECK_ALL_WORKLOADS\n'; } +ensure_root_directory_safe() { :; } +assert_root_directory_safe() { :; } +assert_root_regular_file_safe() { :; } +root_regular_file_is_safe() { return 0; } sudo() { - [[ "$*" == "docker ps -aq" ]] && return 0 - [[ "$*" == "test -e /etc/docker/daemon.json" ]] && return 1 + if [[ "$*" == "mktemp -d /var/backups/station-bootstrap/docker-runtime.XXXXXXXXXX" ]]; then + printf '/var/backups/station-bootstrap/docker-runtime.TEST' + return 0 + fi + [[ "$*" == "test -e /etc/docker/daemon.json" || "$*" == "test -L /etc/docker/daemon.json" ]] && return 1 printf 'SUDO %s\n' "$*" } configure_docker_runtime_if_needed @@ -529,10 +716,41 @@ configure_docker_runtime_if_needed expect(output).toContain("docker_gpus_contract=pass"); }); + it("restores the prior Docker configuration when a post-mutation launch probe fails", () => { + const { result, output } = runSourced( + STATION_PREPARE, + ` +run_gpus_test_sudo() { printf 'GPU_PROBE\n'; return 1; } +run_cdi_test_sudo() { return 0; } +docker_has_nvidia_runtime_sudo() { return 1; } +check_no_workloads() { printf 'RECHECK_ALL_WORKLOADS\n'; } +ensure_root_directory_safe() { :; } +assert_root_directory_safe() { :; } +assert_root_regular_file_safe() { :; } +root_regular_file_is_safe() { return 0; } +sudo() { + if [[ "$*" == "mktemp -d /var/backups/station-bootstrap/docker-runtime.XXXXXXXXXX" ]]; then + printf '/var/backups/station-bootstrap/docker-runtime.TEST' + return 0 + fi + [[ "$*" == "test -e /etc/docker/daemon.json" || "$*" == "test -L /etc/docker/daemon.json" ]] && return 1 + printf 'SUDO %s\n' "$*" +} +configure_docker_runtime_if_needed +`, + ); + + expect(result.status, output).not.toBe(0); + expect(output).toContain("Restoring the Docker daemon configuration"); + expect(output).toContain("rm -f -- /etc/docker/daemon.json"); + expect(output).toMatch(/prior Docker daemon configuration was restored/); + }); + it("accepts a successful packaged CDI refresh", () => { const { result, output } = runSourced( STATION_PREPARE, ` +check_no_workloads() { printf 'RECHECK_ALL_WORKLOADS\n'; } sudo() { printf 'SUDO %s\n' "$*"; } nvidia-ctk() { [[ "$*" == "cdi list" ]] && printf 'nvidia.com/gpu=all\n' @@ -612,6 +830,25 @@ write_install_boot_marker expect(output).toMatch(/Refusing symbolic link in Station bootstrap state path/); expect(fs.existsSync(path.join(home, "redirect-target", "install-boot-id"))).toBe(false); }); + + it("rejects a direct boot-marker symlink without modifying its target", () => { + const { home, result, output } = runSourced( + STATION_PREPARE, + ` +mkdir -p "$HOME/.local/state/station-bootstrap" +chmod 0700 "$HOME/.local/state/station-bootstrap" +printf 'preserve-this-target\n' >"$HOME/marker-target" +ln -s "$HOME/marker-target" "$HOME/.local/state/station-bootstrap/install-boot-id" +write_install_boot_marker +`, + ); + + expect(result.status, output).not.toBe(0); + expect(output).toMatch(/Refusing symbolic link for Station bootstrap boot marker/); + expect(fs.readFileSync(path.join(home, "marker-target"), "utf-8")).toBe( + "preserve-this-target\n", + ); + }); }); describe("DGX Station express host integration", () => { @@ -725,6 +962,7 @@ prepare_installer_host ` _SELECTED_EXPRESS_PLATFORM='DGX Station' NEMOCLAW_VLLM_MODEL='nemotron-3-ultra-550b-a55b' +station_installer_revision() { printf '${STATION_REVISION}'; } run_station_host_preparation() { return 10; } ensure_station_express_host `, @@ -732,9 +970,11 @@ ensure_station_express_host const stateFile = path.join(home, ".nemoclaw", "station-express-resume"); expect(result.status, output).toBe(10); - expect(fs.readFileSync(stateFile, "utf-8")).toBe("nemotron-3-ultra-550b-a55b\n"); + expect(fs.readFileSync(stateFile, "utf-8")).toBe( + `revision=${STATION_REVISION}\nmodel=nemotron-3-ultra-550b-a55b\n`, + ); expect(fs.statSync(stateFile).mode & 0o777).toBe(0o600); - expect(output).toMatch(/rerun the same NemoClaw installer command/); + expect(output).toContain(`NEMOCLAW_INSTALL_TAG=${STATION_REVISION}`); }); it("rejects a resume-state symlink without loading its target", () => { @@ -767,6 +1007,7 @@ printf 'preserve-this-target\n' >"$HOME/resume-target" ln -s "$HOME/resume-target" "$HOME/.nemoclaw/station-express-resume" _SELECTED_EXPRESS_PLATFORM='DGX Station' NEMOCLAW_VLLM_MODEL='nemotron-3-ultra-550b-a55b' +station_installer_revision() { printf '${STATION_REVISION}'; } run_station_host_preparation() { return 10; } ensure_station_express_host `, @@ -786,7 +1027,7 @@ ensure_station_express_host fs.mkdirSync(stateDir, { mode: 0o700 }); fs.writeFileSync( path.join(stateDir, "station-express-resume"), - "nemotron-3-ultra-550b-a55b\n", + `revision=${STATION_REVISION}\nmodel=nemotron-3-ultra-550b-a55b\n`, { mode: 0o600 }, ); const result = spawnSync( @@ -798,6 +1039,7 @@ ensure_station_express_host ` source "$INSTALLER_UNDER_TEST" >/dev/null detect_express_platform() { printf 'DGX Station'; } +station_installer_revision() { printf '${STATION_REVISION}'; } NON_INTERACTIVE='' NEMOCLAW_PROVIDER='' NEMOCLAW_NO_EXPRESS='' @@ -829,7 +1071,7 @@ printf 'RESULT PLATFORM=%s PROVIDER=%s MODEL=%s VLLM_MODEL=%s\n' \ }); it("preserves an explicit provider even when Station resume state exists", () => { - const { result, output } = runSourced( + const { home, result, output } = runSourced( INSTALLER_PAYLOAD, ` mkdir -p "$HOME/.nemoclaw" @@ -849,6 +1091,28 @@ printf 'RESULT PROVIDER=%s\n' "$NEMOCLAW_PROVIDER" expect(output).toContain("NEMOCLAW_PROVIDER=openai already set"); expect(output).toContain("RESULT PROVIDER=openai"); expect(output).not.toContain("Resuming the accepted express install"); + expect(fs.existsSync(path.join(home, ".nemoclaw", "station-express-resume"))).toBe(false); + }); + + it("clears pending Station resume state when express install is explicitly disabled", () => { + const { home, result, output } = runSourced( + INSTALLER_PAYLOAD, + ` +mkdir -p "$HOME/.nemoclaw" +chmod 0700 "$HOME/.nemoclaw" +printf 'revision=${STATION_REVISION}\nmodel=nemotron-3-ultra-550b-a55b\n' >"$HOME/.nemoclaw/station-express-resume" +chmod 0600 "$HOME/.nemoclaw/station-express-resume" +detect_express_platform() { printf 'DGX Station'; } +NON_INTERACTIVE='' +NEMOCLAW_PROVIDER='' +NEMOCLAW_NO_EXPRESS='1' +maybe_offer_express_install +`, + ); + + expect(result.status, output).toBe(0); + expect(output).toContain("NEMOCLAW_NO_EXPRESS=1"); + expect(fs.existsSync(path.join(home, ".nemoclaw", "station-express-resume"))).toBe(false); }); it("does not load Station resume state on DGX Spark", () => { @@ -882,7 +1146,7 @@ printf 'RESULT MODEL=%s\n' "$NEMOCLAW_VLLM_MODEL" fs.mkdirSync(stateDir, { mode: 0o700 }); fs.writeFileSync( path.join(stateDir, "station-express-resume"), - "nemotron-3-ultra-550b-a55b\nunexpected\n", + `revision=${STATION_REVISION}\nmodel=nemotron-3-ultra-550b-a55b\nunexpected\n`, { mode: 0o600 }, ); const result = spawnSync( @@ -910,4 +1174,24 @@ printf 'RESULT MODEL=%s\n' "$NEMOCLAW_VLLM_MODEL" expect(result.status, output).not.toBe(0); expect(output).toMatch(/resume state is invalid/); }); + + it("rejects resume under a different installer revision with exact rerun guidance", () => { + const savedRevision = "b".repeat(40); + const currentRevision = "c".repeat(40); + const { result, output } = runSourced( + INSTALLER_PAYLOAD, + ` +mkdir -p "$HOME/.nemoclaw" +chmod 0700 "$HOME/.nemoclaw" +printf 'revision=${savedRevision}\nmodel=nemotron-3-ultra-550b-a55b\n' >"$HOME/.nemoclaw/station-express-resume" +chmod 0600 "$HOME/.nemoclaw/station-express-resume" +station_installer_revision() { printf '${currentRevision}'; } +load_station_express_resume +`, + ); + + expect(result.status, output).not.toBe(0); + expect(output).toContain(`requires NemoClaw revision ${savedRevision}`); + expect(output).toContain(`NEMOCLAW_INSTALL_TAG=${savedRevision}`); + }); }); From a560a5df674b02fa761ac13204ab0b4c4254a475 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 16 Jul 2026 10:14:57 -0700 Subject: [PATCH 15/74] feat(vllm): add probe-gated dual DGX Station serving Signed-off-by: Aaron Erickson --- .../rebuild-local-provider-recreate.test.ts | 1 + src/lib/inference/context-window.ts | 24 +- src/lib/inference/local-vllm-auth.test.ts | 219 +++ src/lib/inference/local.ts | 210 ++- src/lib/inference/vllm-api-key.test.ts | 73 + src/lib/inference/vllm-api-key.ts | 131 ++ src/lib/inference/vllm-docker-env.test.ts | 65 +- src/lib/inference/vllm-docker-env.ts | 111 ++ src/lib/inference/vllm-dual-station.test.ts | 602 ++++++ src/lib/inference/vllm-models.test.ts | 87 + src/lib/inference/vllm-models.ts | 118 +- src/lib/inference/vllm-ownership.test.ts | 105 ++ .../vllm-station-cluster-lifecycle.test.ts | 971 ++++++++++ .../vllm-station-cluster-lifecycle.ts | 1360 ++++++++++++++ .../inference/vllm-station-cluster.test.ts | 842 +++++++++ src/lib/inference/vllm-station-cluster.ts | 1631 +++++++++++++++++ .../inference/vllm-station-lifecycle-lock.ts | 27 + .../vllm-station-model-staging.test.ts | 258 +++ .../inference/vllm-station-model-staging.ts | 674 +++++++ src/lib/inference/vllm.test.ts | 91 +- src/lib/inference/vllm.ts | 488 ++++- src/lib/onboard.ts | 2 +- src/lib/onboard/inference-providers/types.ts | 1 + .../inference-providers/vllm-local.test.ts | 104 ++ .../onboard/inference-providers/vllm-local.ts | 18 +- .../inference-selection-validation.test.ts | 39 + .../onboard/inference-selection-validation.ts | 28 +- src/lib/onboard/provider-host-state.ts | 20 +- src/lib/onboard/setup-inference.ts | 4 + src/lib/onboard/setup-nim-flow.test.ts | 2 +- src/lib/onboard/setup-nim-flow.ts | 2 +- src/lib/onboard/setup-nim-vllm.test.ts | 120 ++ src/lib/onboard/setup-nim-vllm.ts | 107 +- 33 files changed, 8372 insertions(+), 163 deletions(-) create mode 100644 src/lib/inference/local-vllm-auth.test.ts create mode 100644 src/lib/inference/vllm-api-key.test.ts create mode 100644 src/lib/inference/vllm-api-key.ts create mode 100644 src/lib/inference/vllm-dual-station.test.ts create mode 100644 src/lib/inference/vllm-ownership.test.ts create mode 100644 src/lib/inference/vllm-station-cluster-lifecycle.test.ts create mode 100644 src/lib/inference/vllm-station-cluster-lifecycle.ts create mode 100644 src/lib/inference/vllm-station-cluster.test.ts create mode 100644 src/lib/inference/vllm-station-cluster.ts create mode 100644 src/lib/inference/vllm-station-lifecycle-lock.ts create mode 100644 src/lib/inference/vllm-station-model-staging.test.ts create mode 100644 src/lib/inference/vllm-station-model-staging.ts create mode 100644 src/lib/onboard/inference-providers/vllm-local.test.ts diff --git a/src/lib/actions/sandbox/rebuild-local-provider-recreate.test.ts b/src/lib/actions/sandbox/rebuild-local-provider-recreate.test.ts index e7439fe3e3..3a13a926d8 100644 --- a/src/lib/actions/sandbox/rebuild-local-provider-recreate.test.ts +++ b/src/lib/actions/sandbox/rebuild-local-provider-recreate.test.ts @@ -111,6 +111,7 @@ const localProviderScenarios = [ applyLocalInferenceRoute, run: () => ({ status: 0 }), VLLM_LOCAL_CREDENTIAL_ENV: "NEMOCLAW_VLLM_LOCAL_TOKEN", + getManagedVllmProviderBinding: () => null, ...unusedCommonInferenceDeps, }, ), diff --git a/src/lib/inference/context-window.ts b/src/lib/inference/context-window.ts index 4f0aa6158e..433a091814 100644 --- a/src/lib/inference/context-window.ts +++ b/src/lib/inference/context-window.ts @@ -10,10 +10,12 @@ * window kept for a cloud model → silent under-utilization). */ -import { VLLM_PORT } from "../core/ports"; import { DEFAULT_CONTEXT_WINDOW } from "./config"; import { + getLocalProviderHealthEndpoint, + getManagedDualStationVllmProviderBinding, getOllamaWarmupCommand, + probeVllmModels, type RunCaptureFn, resolveOllamaRuntimeContextWindow, } from "./local"; @@ -44,10 +46,22 @@ const defaultContextWindowDeps: ContextWindowDeps = { probeVllmContextWindow: (model: string): number | null => { // Same source onboard uses: GET /v1/models on the host vLLM server and read // max_model_len (handles both NemoClaw-launched and bring-your-own vLLM). - const { runCapture } = require("../runner") as { runCapture: RunCaptureFn }; - const raw = runCapture(["curl", "-sf", `http://127.0.0.1:${VLLM_PORT}/v1/models`], { - ignoreError: true, - }); + let managedBinding: ReturnType; + try { + managedBinding = getManagedDualStationVllmProviderBinding(); + } catch { + return null; + } + let raw: string; + if (managedBinding) { + const result = probeVllmModels(managedBinding.baseUrl, managedBinding.apiKey); + raw = result.ok ? result.body : ""; + } else { + const { runCapture } = require("../runner") as { runCapture: RunCaptureFn }; + const endpoint = getLocalProviderHealthEndpoint("vllm-local"); + if (!endpoint) return null; + raw = runCapture(["curl", "-sf", endpoint], { ignoreError: true }); + } if (!raw) return null; let parsed: unknown; try { diff --git a/src/lib/inference/local-vllm-auth.test.ts b/src/lib/inference/local-vllm-auth.test.ts new file mode 100644 index 0000000000..e8a24ac705 --- /dev/null +++ b/src/lib/inference/local-vllm-auth.test.ts @@ -0,0 +1,219 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const lifecycle = vi.hoisted(() => ({ + baseUrl: vi.fn<() => string | null>(), +})); + +vi.mock("./vllm-station-cluster-lifecycle", () => ({ + getDualStationManagedVllmBaseUrl: lifecycle.baseUrl, +})); + +import { + CONTAINER_REACHABILITY_IMAGE, + getLocalProviderBaseUrl, + getLocalProviderContainerReachabilityCheck, + getLocalProviderHealthCheck, + getLocalProviderHealthEndpoint, + getManagedDualStationVllmProviderBinding, + LOCAL_INFERENCE_SANDBOX_HOST_URL_ENV, + probeLocalProviderHealth, + probeVllmModels, + validateLocalProvider, +} from "./local"; + +const BASE_URL = "http://10.40.0.1:8000"; +const API_KEY = "e".repeat(64); + +beforeEach(() => { + vi.stubEnv(LOCAL_INFERENCE_SANDBOX_HOST_URL_ENV, undefined); + lifecycle.baseUrl.mockReset(); + lifecycle.baseUrl.mockReturnValue(BASE_URL); +}); + +afterEach(() => vi.unstubAllEnvs()); + +describe("managed dual-Station vLLM authentication", () => { + it("keeps an explicit sandbox host override ahead of managed endpoint recovery", () => { + const loadApiKeyImpl = vi.fn(() => API_KEY); + expect(getLocalProviderBaseUrl("vllm-local", { hostUrl: "http://explicit-host" })).toBe( + "http://explicit-host:8000/v1", + ); + expect( + getManagedDualStationVllmProviderBinding({ + hostUrl: "http://explicit-host", + loadApiKeyImpl, + }), + ).toBeNull(); + expect(loadApiKeyImpl).not.toHaveBeenCalled(); + }); + + it("does not load a stale or unsafe key without a recovered managed endpoint", () => { + const loadApiKeyImpl = vi.fn(() => { + throw new Error(`unsafe ${API_KEY}`); + }); + lifecycle.baseUrl.mockReturnValue(null); + + expect(getManagedDualStationVllmProviderBinding({ loadApiKeyImpl })).toBeNull(); + expect(loadApiKeyImpl).not.toHaveBeenCalled(); + }); + + it("keeps legacy health unauthenticated even when stale key loading would fail", () => { + const loadVllmApiKeyImpl = vi.fn(() => { + throw new Error(`unsafe ${API_KEY}`); + }); + const runCurlProbeImpl = vi.fn(() => ({ + ok: true, + httpStatus: 200, + curlStatus: 0, + body: '{"data":[{"id":"served/model"}]}', + stderr: "", + message: "HTTP 200", + })); + lifecycle.baseUrl.mockReturnValue(null); + + const result = probeLocalProviderHealth("vllm-local", { + model: "served/model", + loadVllmApiKeyImpl, + runCurlProbeImpl, + }); + + expect(result?.ok).toBe(true); + expect(loadVllmApiKeyImpl).not.toHaveBeenCalled(); + expect(runCurlProbeImpl).toHaveBeenCalledWith([ + "-sS", + "--connect-timeout", + "3", + "--max-time", + "5", + "http://127.0.0.1:8000/v1/models", + ]); + }); + + it("returns one atomic provider endpoint and credential binding", () => { + expect(getManagedDualStationVllmProviderBinding({ loadApiKeyImpl: () => API_KEY })).toEqual({ + baseUrl: `${BASE_URL}/v1`, + apiKey: API_KEY, + }); + }); + + it("uses /health only for unauthenticated availability checks", () => { + expect(getLocalProviderHealthEndpoint("vllm-local")).toBe(`${BASE_URL}/v1/models`); + expect(getLocalProviderHealthCheck("vllm-local")).toEqual([ + "curl", + "-sf", + "--connect-timeout", + "3", + "--max-time", + "5", + "--noproxy", + "*", + "--write-out", + "%{http_code}", + `${BASE_URL}/health`, + ]); + expect(getLocalProviderContainerReachabilityCheck("vllm-local")).toEqual([ + "docker", + "--context", + "default", + "run", + "--rm", + "--add-host", + "host.openshell.internal:host-gateway", + CONTAINER_REACHABILITY_IMAGE, + "--connect-timeout", + "5", + "--max-time", + "10", + "--noproxy", + "*", + "-sf", + "-w", + "%{http_code}", + `${BASE_URL}/health`, + ]); + }); + + it("pins reachability and diagnostics to the local daemon despite a persisted remote context", () => { + const capture = vi.fn((argv: readonly string[]) => (argv[0] === "curl" ? "200" : "")); + + const result = validateLocalProvider("vllm-local", capture, () => undefined); + const dockerCommands = capture.mock.calls + .map(([argv]) => argv) + .filter((argv) => argv[0] === "docker"); + + expect(result.ok).toBe(false); + expect(dockerCommands).toHaveLength(5); + expect( + dockerCommands.every((argv) => argv.slice(0, 3).join(" ") === "docker --context default"), + ).toBe(true); + }); + + it("passes bearer auth through a private curl config and cleans it up", () => { + let configPath = ""; + const result = probeVllmModels(`${BASE_URL}/v1`, API_KEY, { + runCurlProbeImpl: (argv, options) => { + expect(argv).not.toContain(API_KEY); + expect(argv.at(-1)).toBe(`${BASE_URL}/v1/models`); + const configIndex = argv.indexOf("--config"); + configPath = argv[configIndex + 1] ?? ""; + expect(options?.trustedConfigFiles).toEqual([configPath]); + expect(options?.pinnedAddresses).toEqual([]); + expect(fs.readFileSync(configPath, "utf8")).toContain(`Authorization: Bearer ${API_KEY}`); + return { + ok: true, + httpStatus: 200, + curlStatus: 0, + body: '{"data":[{"id":"served/model"}]}', + stderr: "", + message: "HTTP 200", + }; + }, + }); + + expect(result.ok).toBe(true); + expect(configPath).not.toBe(""); + expect(fs.existsSync(configPath)).toBe(false); + }); + + it("keeps authenticated model inventory authoritative for configured-model health", () => { + const runCurlProbeImpl = vi.fn((argv: string[]) => ({ + ok: true, + httpStatus: 200, + curlStatus: 0, + body: '{"data":[{"id":"different/model"}]}', + stderr: "", + message: "HTTP 200", + })); + const result = probeLocalProviderHealth("vllm-local", { + model: "required/model", + loadVllmApiKeyImpl: () => API_KEY, + runCurlProbeImpl, + }); + + expect(runCurlProbeImpl).toHaveBeenCalledOnce(); + const probeArgv = runCurlProbeImpl.mock.calls[0]?.[0] ?? []; + expect(probeArgv).not.toContain(API_KEY); + expect(result?.ok).toBe(false); + expect(result?.failureLabel).toBe("unhealthy"); + expect(result?.detail).toContain("required/model"); + expect(result?.detail).toContain("different/model"); + }); + + it("fails closed before model inventory when the managed key is absent", () => { + const runCurlProbeImpl = vi.fn(); + const result = probeLocalProviderHealth("vllm-local", { + model: "required/model", + loadVllmApiKeyImpl: () => null, + runCurlProbeImpl, + }); + + expect(runCurlProbeImpl).not.toHaveBeenCalled(); + expect(result?.ok).toBe(false); + expect(result?.failureLabel).toBe("unauthorized"); + expect(result?.detail).not.toContain(API_KEY); + }); +}); diff --git a/src/lib/inference/local.ts b/src/lib/inference/local.ts index e2bcb0eab8..b611acc9d3 100644 --- a/src/lib/inference/local.ts +++ b/src/lib/inference/local.ts @@ -45,7 +45,9 @@ import { resetOllamaRuntimeContextWindowAutoState, resolveOllamaRuntimeContextWindow as resolveOllamaRuntimeContextWindowWithHost, } from "./ollama-runtime-context"; +import { loadDualStationVllmApiKey } from "./vllm-api-key"; import { applyVllmRuntimeContextWindow as applyVllmRuntimeContextWindowFromModels } from "./vllm-runtime-context"; +import { getDualStationManagedVllmBaseUrl } from "./vllm-station-cluster-lifecycle"; export type { OllamaRuntimeModelStatus } from "./ollama-runtime-context"; @@ -250,6 +252,8 @@ export interface LocalProviderHealthProbeOptions { * state root (written by inference/ollama/proxy.ts during onboard). */ loadOllamaProxyTokenImpl?: () => string | null; + /** Reads the managed dual-Station vLLM key. Injectable so tests stay deterministic. */ + loadVllmApiKeyImpl?: () => string | null; } function defaultLoadOllamaProxyToken(): string | null { @@ -272,6 +276,52 @@ function runLocalCurlProbe(argv: string[], opts: CurlProbeOptions = {}): CurlPro return runCurlProbe(argv, { ...opts, env: buildSubprocessEnv(), replaceEnv: true }); } +export interface VllmModelsProbeOptions { + runCurlProbeImpl?: (argv: string[], opts?: CurlProbeOptions) => CurlProbeResult; +} + +/** Query vLLM's authoritative model inventory without exposing its bearer in process argv. */ +export function probeVllmModels( + baseUrl: string, + apiKey: string, + options: VllmModelsProbeOptions = {}, +): CurlProbeResult { + const runCurlProbeImpl = options.runCurlProbeImpl ?? runLocalCurlProbe; + let authConfig: ReturnType | undefined; + try { + authConfig = createBearerAuthConfig(apiKey, { prefix: "nemoclaw-vllm-auth" }); + return runCurlProbeImpl( + [ + "-sS", + "--connect-timeout", + "3", + "--max-time", + "5", + ...authConfig.args, + `${baseUrl.replace(/\/+$/, "")}/models`, + ], + { + trustedConfigFiles: authConfig.trustedConfigFiles, + // Managed dual-Station endpoints are recovered from owned container + // labels and use a direct-attached RFC1918 rail. Never delegate that + // request through an ambient HTTP proxy. + pinnedAddresses: [], + }, + ); + } catch { + return { + ok: false, + httpStatus: 0, + curlStatus: 1, + body: "", + stderr: "", + message: "Could not prepare the authenticated vLLM model probe.", + }; + } finally { + authConfig?.cleanup(); + } +} + // A 200 response on `/api/tags` alone is not enough to call Ollama healthy — // a captive HTTP_PROXY, a stale listener, or a stub on the loopback port can // all answer with arbitrary 2xx bodies that look healthy at the curl-status @@ -349,22 +399,58 @@ function normalizeLocalInferenceHostUrl(raw: string | null | undefined): string return null; } -function getLocalInferenceSandboxHostUrl(): string { +function configuredLocalInferenceHostUrl(hostUrl?: string | null): string | null { return ( - normalizeLocalInferenceHostUrl(process.env[LOCAL_INFERENCE_SANDBOX_HOST_URL_ENV]) || - HOST_GATEWAY_URL + normalizeLocalInferenceHostUrl(hostUrl) || + normalizeLocalInferenceHostUrl(process.env[LOCAL_INFERENCE_SANDBOX_HOST_URL_ENV]) ); } +function recoveredManagedDualStationVllmBaseUrl(): string | null { + return configuredLocalInferenceHostUrl() ? null : getDualStationManagedVllmBaseUrl(); +} + +export interface ManagedDualStationVllmProviderBinding { + baseUrl: string; + apiKey: string; +} + +export interface ManagedDualStationVllmProviderBindingOptions { + hostUrl?: string | null; + getManagedBaseUrlImpl?: () => string | null; + loadApiKeyImpl?: () => string | null; +} + +/** Recover one endpoint+credential pair only from an owned managed-dual container. */ +export function getManagedDualStationVllmProviderBinding( + options: ManagedDualStationVllmProviderBindingOptions = {}, +): ManagedDualStationVllmProviderBinding | null { + const configuredHostUrl = configuredLocalInferenceHostUrl(options.hostUrl); + if (configuredHostUrl) return null; + + const managedBaseUrl = (options.getManagedBaseUrlImpl ?? getDualStationManagedVllmBaseUrl)(); + if (!managedBaseUrl) return null; + const apiKey = (options.loadApiKeyImpl ?? loadDualStationVllmApiKey)(); + if (!apiKey) { + throw new Error("Managed dual-Station vLLM authentication is missing."); + } + return { baseUrl: `${managedBaseUrl}/v1`, apiKey }; +} + export function getLocalProviderBaseUrl( provider: string, options: { hostUrl?: string | null } = {}, ): string | null { - const hostUrl = - normalizeLocalInferenceHostUrl(options.hostUrl) || getLocalInferenceSandboxHostUrl(); + const configuredHostUrl = configuredLocalInferenceHostUrl(options.hostUrl); + const hostUrl = configuredHostUrl || HOST_GATEWAY_URL; switch (provider) { - case "vllm-local": + case "vllm-local": { + if (!configuredHostUrl) { + const dualStationBaseUrl = recoveredManagedDualStationVllmBaseUrl(); + if (dualStationBaseUrl) return `${dualStationBaseUrl}/v1`; + } return `${hostUrl}:${VLLM_PORT}/v1`; + } case "ollama-local": // Containers reach Ollama through the auth proxy, not directly. return `${hostUrl}:${getOllamaContainerPort()}/v1`; @@ -375,8 +461,10 @@ export function getLocalProviderBaseUrl( export function getLocalProviderValidationBaseUrl(provider: string): string | null { switch (provider) { - case "vllm-local": - return `http://127.0.0.1:${VLLM_PORT}/v1`; + case "vllm-local": { + const dualStationBaseUrl = recoveredManagedDualStationVllmBaseUrl(); + return dualStationBaseUrl ? `${dualStationBaseUrl}/v1` : `http://127.0.0.1:${VLLM_PORT}/v1`; + } case "ollama-local": return `http://${getResolvedOllamaHost()}:${OLLAMA_PORT}/v1`; default: @@ -386,8 +474,12 @@ export function getLocalProviderValidationBaseUrl(provider: string): string | nu export function getLocalProviderHealthEndpoint(provider: string): string | null { switch (provider) { - case "vllm-local": - return `http://127.0.0.1:${VLLM_PORT}/v1/models`; + case "vllm-local": { + const dualStationBaseUrl = recoveredManagedDualStationVllmBaseUrl(); + return dualStationBaseUrl + ? `${dualStationBaseUrl}/v1/models` + : `http://127.0.0.1:${VLLM_PORT}/v1/models`; + } case "ollama-local": return `http://${getResolvedOllamaHost()}:${OLLAMA_PORT}/api/tags`; default: @@ -395,8 +487,32 @@ export function getLocalProviderHealthEndpoint(provider: string): string | null } } +/** Lightweight endpoint used only to prove that the local service is reachable. */ +export function getLocalProviderAvailabilityEndpoint(provider: string): string | null { + if (provider === "vllm-local") { + const dualStationBaseUrl = recoveredManagedDualStationVllmBaseUrl(); + if (dualStationBaseUrl) return `${dualStationBaseUrl}/health`; + } + return getLocalProviderHealthEndpoint(provider); +} + export function getLocalProviderHealthCheck(provider: string): string[] | null { - const endpoint = getLocalProviderHealthEndpoint(provider); + const endpoint = getLocalProviderAvailabilityEndpoint(provider); + if (provider === "vllm-local" && endpoint?.endsWith("/health")) { + return [ + "curl", + "-sf", + "--connect-timeout", + "3", + "--max-time", + "5", + "--noproxy", + "*", + "--write-out", + "%{http_code}", + endpoint, + ]; + } return endpoint ? ["curl", ...buildValidatedCurlCommandArgs(["-sf", endpoint])] : null; } @@ -554,14 +670,46 @@ export function probeLocalProviderHealth( provider: string, options: LocalProviderHealthProbeOptions = {}, ): LocalProviderHealthStatus | null { - const endpoint = getLocalProviderHealthEndpoint(provider); const providerLabel = getLocalProviderLabel(provider); - if (!endpoint || !providerLabel) { - return null; + if (!providerLabel) return null; + + let managedBinding: ManagedDualStationVllmProviderBinding | null = null; + if (provider === "vllm-local") { + try { + managedBinding = getManagedDualStationVllmProviderBinding({ + loadApiKeyImpl: options.loadVllmApiKeyImpl, + }); + } catch (error) { + const missingAuth = + error instanceof Error && + error.message === "Managed dual-Station vLLM authentication is missing."; + return { + ok: false, + providerLabel, + endpoint: + getLocalProviderHealthEndpoint(provider) ?? `http://127.0.0.1:${VLLM_PORT}/v1/models`, + failureLabel: missingAuth ? "unauthorized" : "unhealthy", + probeLabel: "vllm backend", + detail: missingAuth + ? "Local vLLM requires its managed bearer credential, but no private key is available. Re-run `nemoclaw onboard` to repair the dual-Station provider." + : "Local vLLM authentication state is unsafe or unreadable. Re-run `nemoclaw onboard` to repair the managed dual-Station provider.", + }; + } } + const endpoint = managedBinding + ? `${managedBinding.baseUrl}/models` + : provider === "vllm-local" + ? `http://127.0.0.1:${VLLM_PORT}/v1/models` + : getLocalProviderHealthEndpoint(provider); + if (!endpoint) return null; const runCurlProbeImpl = options.runCurlProbeImpl ?? runLocalCurlProbe; - const result = runCurlProbeImpl(["-sS", "--connect-timeout", "3", "--max-time", "5", endpoint]); + let result: CurlProbeResult; + if (managedBinding) { + result = probeVllmModels(managedBinding.baseUrl, managedBinding.apiKey, { runCurlProbeImpl }); + } else { + result = runCurlProbeImpl(["-sS", "--connect-timeout", "3", "--max-time", "5", endpoint]); + } // Per #3265 the status line is renamed `Inference ():` for local // providers so the upcoming `Inference (auth proxy):` subprobe lines render @@ -661,9 +809,10 @@ export function probeLocalProviderHealth( export function getLocalProviderContainerReachabilityCheck(provider: string): string[] | null { switch (provider) { - case "vllm-local": + case "vllm-local": { + const dualStationBaseUrl = recoveredManagedDualStationVllmBaseUrl(); return [ - "docker", + ...(dualStationBaseUrl ? ["docker", "--context", "default"] : ["docker"]), "run", "--rm", "--add-host", @@ -673,9 +822,14 @@ export function getLocalProviderContainerReachabilityCheck(provider: string): st "5", "--max-time", "10", + ...(dualStationBaseUrl ? ["--noproxy", "*"] : []), "-sf", - `http://host.openshell.internal:${VLLM_PORT}/v1/models`, + ...(dualStationBaseUrl ? ["-w", "%{http_code}"] : []), + dualStationBaseUrl + ? `${dualStationBaseUrl}/health` + : `http://host.openshell.internal:${VLLM_PORT}/v1/models`, ]; + } case "ollama-local": // Check the auth proxy port, not Ollama directly. The proxy listens // on 0.0.0.0 and is reachable from containers; Ollama is on 127.0.0.1. @@ -735,7 +889,7 @@ export function validateLocalProvider( case "vllm-local": return { ok: false, - message: `Local vLLM was selected, but nothing is responding on http://127.0.0.1:${VLLM_PORT}.`, + message: `Local vLLM was selected, but nothing is responding on ${getLocalProviderHealthEndpoint(provider) ?? "the configured endpoint"}.`, }; case "ollama-local": return { @@ -770,7 +924,7 @@ export function validateLocalProvider( case "vllm-local": return { ok: false, - message: `Local vLLM is responding on 127.0.0.1, but the Docker container reachability check failed for http://host.openshell.internal:${VLLM_PORT}. This may be a Docker networking issue — the sandbox uses a different network path and may still work.`, + message: `Local vLLM is responding on the host, but the Docker container reachability check failed for ${getContainerCheckUrl(provider)}. This may be a Docker networking issue — the sandbox uses a different network path and may still work.`, diagnostic, }; case "ollama-local": @@ -790,8 +944,12 @@ export function validateLocalProvider( function getContainerCheckUrl(provider: string): string { switch (provider) { - case "vllm-local": - return `http://host.openshell.internal:${VLLM_PORT}/v1/models`; + case "vllm-local": { + const dualStationBaseUrl = recoveredManagedDualStationVllmBaseUrl(); + return dualStationBaseUrl + ? `${dualStationBaseUrl}/health` + : `http://host.openshell.internal:${VLLM_PORT}/v1/models`; + } case "ollama-local": return `http://host.openshell.internal:${getOllamaContainerPort()}/api/tags`; default: @@ -801,11 +959,15 @@ function getContainerCheckUrl(provider: string): string { function collectContainerDiagnostic(provider: string, capture: RunCaptureFn): string { const url = getContainerCheckUrl(provider); + const dockerCommand = + provider === "vllm-local" && recoveredManagedDualStationVllmBaseUrl() + ? ["docker", "--context", "default"] + : ["docker"]; try { // Get HTTP status code const httpStatus = capture( [ - "docker", + ...dockerCommand, "run", "--rm", "--add-host", @@ -828,7 +990,7 @@ function collectContainerDiagnostic(provider: string, capture: RunCaptureFn): st // Get /etc/hosts to see host-gateway resolution const hostsOutput = capture( [ - "docker", + ...dockerCommand, "run", "--rm", "--add-host", diff --git a/src/lib/inference/vllm-api-key.test.ts b/src/lib/inference/vllm-api-key.test.ts new file mode 100644 index 0000000000..6b550c63d3 --- /dev/null +++ b/src/lib/inference/vllm-api-key.test.ts @@ -0,0 +1,73 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { afterEach, describe, expect, it } from "vitest"; + +import { + dualStationVllmApiKeyPath, + ensureDualStationVllmApiKey, + loadDualStationVllmApiKey, +} from "./vllm-api-key"; + +const temporaryDirectories: string[] = []; + +function temporaryDirectory(): string { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-vllm-key-")); + temporaryDirectories.push(directory); + return directory; +} + +afterEach(() => { + for (const directory of temporaryDirectories.splice(0)) { + fs.rmSync(directory, { force: true, recursive: true }); + } +}); + +describe("dual-Station vLLM API key persistence", () => { + it("creates one private 256-bit key and reuses it", () => { + const stateDir = path.join(temporaryDirectory(), "state"); + const generated = ensureDualStationVllmApiKey({ + stateDir, + randomBytes: () => Buffer.alloc(32, 0xab), + }); + + expect(generated).toBe("ab".repeat(32)); + expect(loadDualStationVllmApiKey({ stateDir })).toBe(generated); + expect(ensureDualStationVllmApiKey({ stateDir, randomBytes: () => Buffer.alloc(32, 1) })).toBe( + generated, + ); + expect(fs.statSync(stateDir).mode & 0o777).toBe(0o700); + expect(fs.statSync(dualStationVllmApiKeyPath(stateDir)).mode & 0o777).toBe(0o600); + }); + + it("returns null when no key has been provisioned", () => { + expect(loadDualStationVllmApiKey({ stateDir: temporaryDirectory() })).toBeNull(); + }); + + it("rejects malformed or overly permissive key files", () => { + const stateDir = temporaryDirectory(); + const filePath = dualStationVllmApiKeyPath(stateDir); + fs.writeFileSync(filePath, `${"ab".repeat(32)}\n`, { mode: 0o644 }); + expect(() => loadDualStationVllmApiKey({ stateDir })).toThrow("group or others"); + + fs.chmodSync(filePath, 0o600); + fs.writeFileSync(filePath, "not-a-key\n"); + expect(() => loadDualStationVllmApiKey({ stateDir })).toThrow("malformed"); + }); + + it("refuses to follow a symbolic-link key path", () => { + const root = temporaryDirectory(); + const stateDir = path.join(root, "state"); + fs.mkdirSync(stateDir, { mode: 0o700 }); + const target = path.join(root, "target"); + fs.writeFileSync(target, `${"cd".repeat(32)}\n`, { mode: 0o600 }); + fs.symlinkSync(target, dualStationVllmApiKeyPath(stateDir)); + + expect(() => loadDualStationVllmApiKey({ stateDir })).toThrow("symbolic link"); + expect(() => ensureDualStationVllmApiKey({ stateDir })).toThrow("symbolic link"); + }); +}); diff --git a/src/lib/inference/vllm-api-key.ts b/src/lib/inference/vllm-api-key.ts new file mode 100644 index 0000000000..a2e5e98c26 --- /dev/null +++ b/src/lib/inference/vllm-api-key.ts @@ -0,0 +1,131 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import crypto from "node:crypto"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { DEFAULT_GATEWAY_PORT } from "../core/ports"; +import { nemoclawStateRoot } from "../state/state-root"; +import { ensureLocalAdapterStateDir } from "./local-adapter-lifecycle"; + +export const DUAL_STATION_VLLM_API_KEY_FILE = "dual-station-vllm-api-key"; +export const DUAL_STATION_VLLM_API_KEY_PATTERN = /^[a-f0-9]{64}$/; + +export interface DualStationVllmApiKeyOptions { + stateDir?: string; + randomBytes?: (size: number) => Buffer; +} + +function defaultStateDir(): string { + // The managed vLLM service is host-global rather than gateway-scoped. Every + // gateway therefore reads the same key even when NEMOCLAW_GATEWAY_PORT is + // changed for a second sandbox. + return nemoclawStateRoot(os.homedir(), DEFAULT_GATEWAY_PORT); +} + +export function dualStationVllmApiKeyPath(stateDir = defaultStateDir()): string { + return path.join(stateDir, DUAL_STATION_VLLM_API_KEY_FILE); +} + +function assertPrivateRegularFile(stat: fs.Stats, filePath: string): void { + if (!stat.isFile()) { + throw new Error(`Refusing to read dual-Station vLLM API key from non-file path: ${filePath}`); + } + if ((stat.mode & 0o077) !== 0) { + throw new Error( + `Dual-Station vLLM API key file must not be accessible by group or others: ${filePath}`, + ); + } + if (typeof process.getuid === "function" && stat.uid !== process.getuid()) { + throw new Error(`Dual-Station vLLM API key file is not owned by the current user: ${filePath}`); + } +} + +/** Load the host-global managed endpoint key, failing closed on unsafe state. */ +export function loadDualStationVllmApiKey( + options: Pick = {}, +): string | null { + const filePath = dualStationVllmApiKeyPath(options.stateDir ?? defaultStateDir()); + let before: fs.Stats; + try { + before = fs.lstatSync(filePath); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; + throw error; + } + if (before.isSymbolicLink()) { + throw new Error( + `Refusing to read dual-Station vLLM API key through a symbolic link: ${filePath}`, + ); + } + assertPrivateRegularFile(before, filePath); + if (before.size < 64 || before.size > 65) { + throw new Error(`Dual-Station vLLM API key file is malformed: ${filePath}`); + } + + const noFollow = fs.constants.O_NOFOLLOW ?? 0; + const nonBlock = fs.constants.O_NONBLOCK ?? 0; + let fd: number | undefined; + try { + fd = fs.openSync(filePath, fs.constants.O_RDONLY | noFollow | nonBlock); + const opened = fs.fstatSync(fd); + assertPrivateRegularFile(opened, filePath); + if (opened.dev !== before.dev || opened.ino !== before.ino) { + throw new Error( + `Dual-Station vLLM API key file changed while it was being opened: ${filePath}`, + ); + } + const value = fs.readFileSync(fd, "utf8").trim(); + if (!DUAL_STATION_VLLM_API_KEY_PATTERN.test(value)) { + throw new Error(`Dual-Station vLLM API key file is malformed: ${filePath}`); + } + return value; + } finally { + if (fd !== undefined) fs.closeSync(fd); + } +} + +/** Create the managed endpoint key once, or reuse the existing private key. */ +export function ensureDualStationVllmApiKey(options: DualStationVllmApiKeyOptions = {}): string { + const stateDir = options.stateDir ?? defaultStateDir(); + ensureLocalAdapterStateDir(stateDir); + const existing = loadDualStationVllmApiKey({ stateDir }); + if (existing) return existing; + + const randomBytes = options.randomBytes ?? crypto.randomBytes; + const value = randomBytes(32).toString("hex"); + if (!DUAL_STATION_VLLM_API_KEY_PATTERN.test(value)) { + throw new Error("Could not generate a valid dual-Station vLLM API key"); + } + + const filePath = dualStationVllmApiKeyPath(stateDir); + const noFollow = fs.constants.O_NOFOLLOW ?? 0; + let fd: number | undefined; + try { + fd = fs.openSync( + filePath, + fs.constants.O_WRONLY | fs.constants.O_CREAT | fs.constants.O_EXCL | noFollow, + 0o600, + ); + const opened = fs.fstatSync(fd); + assertPrivateRegularFile(opened, filePath); + fs.writeFileSync(fd, `${value}\n`, "utf8"); + fs.fsyncSync(fd); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "EEXIST") { + const raced = loadDualStationVllmApiKey({ stateDir }); + if (raced) return raced; + } + throw error; + } finally { + if (fd !== undefined) fs.closeSync(fd); + } + + const persisted = loadDualStationVllmApiKey({ stateDir }); + if (persisted !== value) { + throw new Error("Could not verify the persisted dual-Station vLLM API key"); + } + return value; +} diff --git a/src/lib/inference/vllm-docker-env.test.ts b/src/lib/inference/vllm-docker-env.test.ts index af06687ff3..cf0395559c 100644 --- a/src/lib/inference/vllm-docker-env.test.ts +++ b/src/lib/inference/vllm-docker-env.test.ts @@ -2,7 +2,11 @@ // SPDX-License-Identifier: Apache-2.0 import { afterEach, describe, expect, it, vi } from "vitest"; -import { buildVllmDockerEnv } from "./vllm-docker-env"; +import { + buildLocalDualStationDockerEnv, + buildRemoteVllmDockerEnv, + buildVllmDockerEnv, +} from "./vllm-docker-env"; afterEach(() => { vi.unstubAllEnvs(); @@ -38,4 +42,63 @@ describe("managed vLLM Docker client environment", () => { expect(env.DOCKER_CONTEXT).toBe("requested-context"); expect(env.DOCKER_HOST).toBeUndefined(); }); + + it("pins a canonical SSH daemon and strips incompatible ambient Docker selectors", () => { + vi.stubEnv("DOCKER_API_VERSION", "1.48"); + vi.stubEnv("DOCKER_CERT_PATH", "/tmp/ambient-docker-certs"); + vi.stubEnv("DOCKER_CONFIG", "/tmp/nemoclaw-docker-config"); + vi.stubEnv("DOCKER_CONTEXT", "ambient-context"); + vi.stubEnv("DOCKER_HOST", "tcp://ambient.example.test:2376"); + vi.stubEnv("DOCKER_TLS", "1"); + vi.stubEnv("DOCKER_TLS_VERIFY", "1"); + vi.stubEnv("SSH_AUTH_SOCK", "/tmp/ssh-agent.sock"); + vi.stubEnv("OPENSHELL_GATEWAY_AUTH_TOKEN", "must-not-cross-ssh"); + vi.stubEnv("UNRELATED_SECRET", "do-not-forward"); + + const env = buildRemoteVllmDockerEnv("ssh://station@dgx-peer.example.test:22"); + + expect(env).toEqual( + expect.objectContaining({ + DOCKER_HOST: "ssh://station@dgx-peer.example.test:22", + SSH_AUTH_SOCK: "/tmp/ssh-agent.sock", + }), + ); + expect(env.DOCKER_API_VERSION).toBeUndefined(); + expect(env.DOCKER_CERT_PATH).toBeUndefined(); + expect(env.DOCKER_CONFIG).toBeUndefined(); + expect(env.DOCKER_CONTEXT).toBeUndefined(); + expect(env.DOCKER_TLS).toBeUndefined(); + expect(env.DOCKER_TLS_VERIFY).toBeUndefined(); + expect(env.UNRELATED_SECRET).toBeUndefined(); + expect(env.OPENSHELL_GATEWAY_AUTH_TOKEN).toBeUndefined(); + }); + + it("pins the dual-Station head to the physical host default Docker daemon", () => { + const env = buildLocalDualStationDockerEnv( + { SAFE_MARKER: "kept" }, + { + DOCKER_HOST: "ssh://wrong-daemon", + DOCKER_CONTEXT: "wrong-context", + DOCKER_CONFIG: "/tmp/wrong-config", + }, + ); + + expect(env.SAFE_MARKER).toBe("kept"); + expect(env.DOCKER_HOST).toBeUndefined(); + expect(env.DOCKER_CONTEXT).toBe("default"); + expect(env.DOCKER_CONFIG).toBeUndefined(); + }); + + it.each([ + "tcp://dgx-peer.example.test:2376", + "ssh://station:secret@dgx-peer.example.test", + "ssh://dgx-peer.example.test/", + "ssh://dgx-peer.example.test?context=other", + "ssh://Dgx-Peer.example.test", + " ssh://dgx-peer.example.test", + ])("rejects a non-canonical or unsafe remote Docker target: %s", (sshUri) => { + expect(() => buildRemoteVllmDockerEnv(sshUri)).toThrow( + "Remote Docker host must be a canonical ssh://[user@]host[:port] URI", + ); + }); }); diff --git a/src/lib/inference/vllm-docker-env.ts b/src/lib/inference/vllm-docker-env.ts index 986d2286e9..3ffe0f2f69 100644 --- a/src/lib/inference/vllm-docker-env.ts +++ b/src/lib/inference/vllm-docker-env.ts @@ -1,6 +1,8 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { isIP } from "node:net"; + import { buildSubprocessEnv } from "../subprocess-env"; const DOCKER_CLIENT_ENV_NAMES = [ @@ -13,6 +15,75 @@ const DOCKER_CLIENT_ENV_NAMES = [ "DOCKER_TLS_VERIFY", ] as const; +const REMOTE_DOCKER_INCOMPATIBLE_ENV_NAMES = [ + "DOCKER_API_VERSION", + "DOCKER_CERT_PATH", + "DOCKER_CONFIG", + "DOCKER_CONTEXT", + "DOCKER_TLS", + "DOCKER_TLS_VERIFY", +] as const; +const SSH_TRANSPORT_ENV_NAMES = [ + "HOME", + "PATH", + "USER", + "LOGNAME", + "SHELL", + "SSH_AUTH_SOCK", + "LANG", + "LC_ALL", + "LC_CTYPE", + "TMPDIR", +] as const; +const CANONICAL_SSH_HOST_PATTERN = + /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?(?:\.[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?)*$/; +const CANONICAL_SSH_USERNAME_PATTERN = /^[A-Za-z_][A-Za-z0-9._-]*$/; + +function validateRemoteDockerSshUri(value: string): string { + const invalid = () => + new Error( + "Remote Docker host must be a canonical ssh://[user@]host[:port] URI without a password, path, query, or fragment", + ); + if (typeof value !== "string" || !value || value !== value.trim() || value.includes("\0")) { + throw invalid(); + } + + let parsed: URL; + try { + parsed = new URL(value); + } catch { + throw invalid(); + } + + if ( + parsed.protocol !== "ssh:" || + !parsed.hostname || + parsed.password || + parsed.pathname || + parsed.search || + parsed.hash || + (parsed.username && !CANONICAL_SSH_USERNAME_PATTERN.test(parsed.username)) + ) { + throw invalid(); + } + + const bracketedIpv6 = parsed.hostname.startsWith("[") && parsed.hostname.endsWith("]"); + const bareHostname = bracketedIpv6 ? parsed.hostname.slice(1, -1) : parsed.hostname; + const validHostname = bracketedIpv6 + ? bareHostname === bareHostname.toLowerCase() && isIP(bareHostname) === 6 + : isIP(bareHostname) === 4 || CANONICAL_SSH_HOST_PATTERN.test(bareHostname); + const port = parsed.port ? Number(parsed.port) : null; + if (!validHostname || (port !== null && (!Number.isInteger(port) || port < 1 || port > 65535))) { + throw invalid(); + } + + const canonical = `ssh://${parsed.username ? `${parsed.username}@` : ""}${parsed.hostname}${ + parsed.port ? `:${parsed.port}` : "" + }`; + if (value !== canonical) throw invalid(); + return canonical; +} + /** * Use one Docker client selection for every managed-vLLM subprocess while * retaining the repository's child-process environment sanitization. @@ -32,3 +103,43 @@ export function buildVllmDockerEnv( } return env; } + +/** Select the physical host's default Docker daemon, ignoring ambient client routing. */ +export function buildLocalDualStationDockerEnv( + extra: Record = {}, + source: NodeJS.ProcessEnv = process.env, +): Record { + const env = buildVllmDockerEnv(extra, source); + for (const name of DOCKER_CLIENT_ENV_NAMES) delete env[name]; + // Docker otherwise falls back to config.json's persisted currentContext, + // which may point at a remote daemon even with every selector env unset. + env.DOCKER_CONTEXT = "default"; + return env; +} + +/** Minimal environment shared by strict SSH probes and Docker's SSH helper. */ +export function buildVllmSshTransportEnv( + extra: Record = {}, + source: NodeJS.ProcessEnv = process.env, +): Record { + const env: Record = {}; + for (const name of SSH_TRANSPORT_ENV_NAMES) { + const value = source[name]; + if (value !== undefined) env[name] = value; + } + return { ...env, ...extra }; +} + +/** + * Select one explicitly configured Docker-over-SSH daemon without allowing an + * ambient context, client config, API pin, or TCP/TLS settings to influence it. + */ +export function buildRemoteVllmDockerEnv( + sshUri: string, + source: NodeJS.ProcessEnv = process.env, +): Record { + const remoteHost = validateRemoteDockerSshUri(sshUri); + const env = buildVllmSshTransportEnv({ DOCKER_HOST: remoteHost }, source); + for (const name of REMOTE_DOCKER_INCOMPATIBLE_ENV_NAMES) delete env[name]; + return env; +} diff --git a/src/lib/inference/vllm-dual-station.test.ts b/src/lib/inference/vllm-dual-station.test.ts new file mode 100644 index 0000000000..e967ac1fc4 --- /dev/null +++ b/src/lib/inference/vllm-dual-station.test.ts @@ -0,0 +1,602 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { EventEmitter } from "node:events"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + areContainersRunning: vi.fn(), + cleanup: vi.fn(), + dockerCapture: vi.fn(), + dockerForceRm: vi.fn(), + dockerImageInspectFormat: vi.fn(), + dockerPullWithProgressWatchdog: vi.fn(), + dockerRunDetached: vi.fn(), + dockerSpawn: vi.fn(), + dockerStop: vi.fn(), + ensureApiKey: vi.fn(), + findUnwritableTreePath: vi.fn(), + getManagedBaseUrl: vi.fn(), + getGpuIndicesByName: vi.fn(), + loadApiKey: vi.fn(), + measureDirectorySizeBytes: vi.fn(), + preflightGpuRuntime: vi.fn(), + preflightOwnership: vi.fn(), + probeCapability: vi.fn(), + probeDockerStorage: vi.fn(), + probeHostStorage: vi.fn(), + runCapture: vi.fn(), + runCurlProbe: vi.fn(), + startManaged: vi.fn(), + stageModelSnapshot: vi.fn(), + withLifecycle: vi.fn(), +})); + +vi.mock("../runner", async (importOriginal) => ({ + ...(await importOriginal()), + runCapture: mocks.runCapture, +})); + +vi.mock("../adapters/http/probe", () => ({ + runCurlProbe: mocks.runCurlProbe, +})); + +vi.mock("../adapters/docker", () => ({ + dockerCapture: mocks.dockerCapture, + dockerForceRm: mocks.dockerForceRm, + dockerImageInspectFormat: mocks.dockerImageInspectFormat, + dockerPullWithProgressWatchdog: mocks.dockerPullWithProgressWatchdog, + dockerRunDetached: mocks.dockerRunDetached, + dockerSpawn: mocks.dockerSpawn, + dockerStop: mocks.dockerStop, +})); + +vi.mock("./nim", () => ({ + getGpuIndicesByName: mocks.getGpuIndicesByName, +})); + +vi.mock("./vllm-storage", async (importOriginal) => ({ + ...(await importOriginal()), + findUnwritableTreePath: mocks.findUnwritableTreePath, + measureDirectorySizeBytes: mocks.measureDirectorySizeBytes, + probeDockerStorage: mocks.probeDockerStorage, + probeHostStorage: mocks.probeHostStorage, +})); + +vi.mock("./vllm-station-cluster", async (importOriginal) => ({ + ...(await importOriginal()), + probeDualStationVllmCapability: mocks.probeCapability, +})); + +vi.mock("./vllm-station-model-staging", () => ({ + stageDualStationModelSnapshot: mocks.stageModelSnapshot, +})); + +vi.mock("./vllm-station-cluster-lifecycle", () => ({ + areDualStationManagedVllmContainersRunning: mocks.areContainersRunning, + cleanupDualStationManagedVllm: mocks.cleanup, + getDualStationManagedVllmBaseUrl: mocks.getManagedBaseUrl, + preflightDualStationGpuRuntime: mocks.preflightGpuRuntime, + preflightDualStationManagedVllm: mocks.preflightOwnership, + startDualStationManagedVllm: mocks.startManaged, + withDualStationManagedVllmLifecycle: mocks.withLifecycle, +})); + +vi.mock("./vllm-api-key", () => ({ + ensureDualStationVllmApiKey: mocks.ensureApiKey, + loadDualStationVllmApiKey: mocks.loadApiKey, +})); + +import { detectVllmProfile, installVllm } from "./vllm"; +import { DUAL_STATION_VLLM_RUNTIME, type DualStationVllmPlan } from "./vllm-station-cluster"; + +const API_KEY = "ab".repeat(32); +const HEAD_ID = "a".repeat(64); +const WORKER_ID = "b".repeat(64); +const HEAD_BASE_URL = "http://192.168.100.1:8000"; + +function plan(): DualStationVllmPlan { + return { + peerSshTarget: "nvidia@station-b", + peerDockerHost: "ssh://nvidia@station-b", + runtime: DUAL_STATION_VLLM_RUNTIME, + local: { + hostname: "station-a", + home: "/home/nvidia", + uid: 1000, + gpu: { index: 0, name: "NVIDIA GB300", uuid: "GPU-aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa" }, + }, + peer: { + hostname: "station-b", + home: "/home/nvidia", + uid: 1000, + gpu: { index: 0, name: "NVIDIA GB300", uuid: "GPU-bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb" }, + }, + rails: [ + { + index: 0, + subnet: "192.168.100.0/30", + local: { + rdmaDevice: "mlx5_0", + uverbsDevice: "/dev/infiniband/uverbs0", + netdev: "enp1s0f0np0", + macAddress: "02:00:00:00:00:01", + pciAddress: "0000:01:00.0", + address: "192.168.100.1", + }, + peer: { + rdmaDevice: "mlx5_0", + uverbsDevice: "/dev/infiniband/uverbs0", + netdev: "enp1s0f0np0", + macAddress: "02:00:00:00:00:02", + pciAddress: "0000:01:00.0", + address: "192.168.100.2", + }, + }, + { + index: 1, + subnet: "192.168.200.0/30", + local: { + rdmaDevice: "mlx5_1", + uverbsDevice: "/dev/infiniband/uverbs1", + netdev: "enp1s0f1np1", + macAddress: "02:00:00:00:01:01", + pciAddress: "0000:01:00.1", + address: "192.168.200.1", + }, + peer: { + rdmaDevice: "mlx5_1", + uverbsDevice: "/dev/infiniband/uverbs1", + netdev: "enp1s0f1np1", + macAddress: "02:00:00:00:01:02", + pciAddress: "0000:01:00.1", + address: "192.168.200.2", + }, + }, + ], + masterAddress: "192.168.100.1", + roceGidIndex: 3, + }; +} + +function successfulSpawn(): EventEmitter & { stdout: EventEmitter; stderr: EventEmitter } { + const child = new EventEmitter() as EventEmitter & { + stdout: EventEmitter; + stderr: EventEmitter; + }; + child.stdout = new EventEmitter(); + child.stderr = new EventEmitter(); + process.nextTick(() => child.emit("exit", 0)); + return child; +} + +const originalEnv = { ...process.env }; +let logSpy: ReturnType; +let errorSpy: ReturnType; +let mkdirSpy: ReturnType; +let stdoutSpy: ReturnType; + +beforeEach(() => { + vi.clearAllMocks(); + process.env.NEMOCLAW_VLLM_MODEL = "nemotron-3-ultra-550b-a55b"; + process.env.NEMOCLAW_DGX_STATION_PEER = "nvidia@station-b"; + process.env.HF_TOKEN = "hf_test"; + delete process.env.NEMOCLAW_VLLM_EXTRA_ARGS_JSON; + delete process.env.VLLM_API_KEY; + + logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); + errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + mkdirSpy = vi.spyOn(fs, "mkdirSync").mockImplementation(() => undefined); + stdoutSpy = vi.spyOn(process.stdout, "write").mockImplementation(() => true); + + mocks.probeCapability.mockReturnValue({ + kind: "ready", + plan: plan(), + peerModelSnapshot: "ready", + }); + mocks.stageModelSnapshot.mockResolvedValue({ ok: true, transferred: true }); + mocks.preflightOwnership.mockReturnValue({ ok: true }); + mocks.preflightGpuRuntime.mockReturnValue({ ok: true }); + mocks.getManagedBaseUrl.mockReturnValue(null); + mocks.ensureApiKey.mockReturnValue(API_KEY); + mocks.loadApiKey.mockReturnValue(API_KEY); + mocks.startManaged.mockReturnValue({ + ok: true, + baseUrl: HEAD_BASE_URL, + headContainerId: HEAD_ID, + workerContainerId: WORKER_ID, + reusedExisting: false, + }); + mocks.withLifecycle.mockImplementation(async (operation) => await operation()); + mocks.areContainersRunning.mockReturnValue(true); + mocks.cleanup.mockReturnValue({ ok: true, removedContainerIds: [] }); + mocks.findUnwritableTreePath.mockReturnValue(null); + mocks.measureDirectorySizeBytes.mockReturnValue(0n); + mocks.probeDockerStorage.mockReturnValue({ + ok: true, + capacity: { availableBytes: 1_000_000_000_000n, path: "/docker", source: "Docker" }, + }); + mocks.probeHostStorage.mockReturnValue({ + ok: true, + capacity: { + availableBytes: 1_000_000_000_000n, + path: path.join(os.homedir(), ".cache", "huggingface"), + source: "Hugging Face cache", + }, + }); + mocks.dockerImageInspectFormat.mockReturnValue(`sha256:${"c".repeat(64)}`); + mocks.dockerPullWithProgressWatchdog.mockResolvedValue({ + status: 0, + signal: null, + output: "", + timedOut: false, + timeoutKind: null, + }); + mocks.dockerSpawn.mockReturnValue(successfulSpawn()); + mocks.runCurlProbe.mockImplementation((args: string[]) => { + const url = args.at(-1) ?? ""; + const authenticated = args.includes("--config"); + return url.endsWith("/health") + ? { ok: true, httpStatus: 200, message: "ok", body: "" } + : authenticated + ? { + ok: true, + httpStatus: 200, + message: "ok", + body: JSON.stringify({ data: [{ id: "nvidia/nemotron-3-ultra-550b-a55b" }] }), + } + : { ok: false, httpStatus: 401, message: "unauthorized", body: "" }; + }); + mocks.runCapture.mockImplementation((args: readonly string[]) => { + switch (args[0]) { + case "sh": + return "/usr/bin/tool\n"; + case "curl": + return "200"; + default: + return ""; + } + }); +}); + +afterEach(() => { + logSpy.mockRestore(); + errorSpy.mockRestore(); + mkdirSpy.mockRestore(); + stdoutSpy.mockRestore(); + process.env = { ...originalEnv }; +}); + +describe("dual DGX Station vLLM install orchestration", () => { + it("pulls both immutable images, preflights GPUs, authenticates, and starts worker plus head", async () => { + const clusterPlan = plan(); + let lifecycleActive = false; + let capabilityCalls = 0; + const probeImplementation = mocks.runCurlProbe.getMockImplementation(); + mocks.withLifecycle.mockImplementation(async (operation) => { + lifecycleActive = true; + try { + return await operation(); + } finally { + lifecycleActive = false; + } + }); + mocks.runCurlProbe.mockImplementation((args: string[], options?: unknown) => { + expect(lifecycleActive).toBe(true); + return probeImplementation?.(args, options); + }); + mocks.areContainersRunning.mockImplementation(() => { + expect(lifecycleActive).toBe(true); + return true; + }); + mocks.stageModelSnapshot.mockImplementation(async () => { + expect(lifecycleActive).toBe(true); + return { ok: true, transferred: false }; + }); + mocks.probeCapability.mockImplementation(() => { + capabilityCalls += 1; + expect(lifecycleActive).toBe(capabilityCalls === 2); + return { + kind: "ready", + plan: clusterPlan, + peerModelSnapshot: "ready", + }; + }); + const profile = detectVllmProfile({ platform: "station", type: "nvidia" }); + + await expect( + installVllm(profile!, { hasImage: true, nonInteractive: true, promptFn: vi.fn() }), + ).resolves.toEqual({ ok: true }); + + expect(mocks.preflightOwnership).toHaveBeenCalledWith(clusterPlan); + expect(mocks.dockerPullWithProgressWatchdog).toHaveBeenCalledTimes(2); + const localPullOptions = mocks.dockerPullWithProgressWatchdog.mock.calls[0][1]; + const peerPullOptions = mocks.dockerPullWithProgressWatchdog.mock.calls[1][1]; + expect(peerPullOptions.env.DOCKER_HOST).toBe("ssh://nvidia@station-b"); + expect(localPullOptions.env.DOCKER_HOST).not.toBe(peerPullOptions.env.DOCKER_HOST); + expect(localPullOptions.env.DOCKER_CONTEXT).toBe("default"); + expect(peerPullOptions.env.DOCKER_CONTEXT).toBeUndefined(); + expect(localPullOptions.env.VLLM_API_KEY).toBeUndefined(); + expect(peerPullOptions.env.VLLM_API_KEY).toBeUndefined(); + expect(mocks.preflightGpuRuntime).toHaveBeenCalledWith(clusterPlan); + expect(mocks.preflightGpuRuntime.mock.invocationCallOrder[0]).toBeGreaterThan( + mocks.dockerPullWithProgressWatchdog.mock.invocationCallOrder[1], + ); + expect(mocks.ensureApiKey.mock.invocationCallOrder[0]).toBeGreaterThan( + mocks.preflightGpuRuntime.mock.invocationCallOrder[0], + ); + expect(mocks.ensureApiKey.mock.invocationCallOrder[0]).toBeGreaterThan( + mocks.dockerSpawn.mock.invocationCallOrder[0], + ); + expect(mocks.startManaged).toHaveBeenCalledWith(clusterPlan, { apiKey: API_KEY }); + expect(mocks.startManaged.mock.invocationCallOrder[0]).toBeGreaterThan( + mocks.dockerSpawn.mock.invocationCallOrder[0], + ); + expect(mocks.areContainersRunning).toHaveBeenCalledWith(clusterPlan); + expect(mocks.withLifecycle).toHaveBeenCalledTimes(2); + expect(lifecycleActive).toBe(false); + expect(mocks.probeCapability).toHaveBeenCalledTimes(2); + expect(mocks.stageModelSnapshot).toHaveBeenCalledWith(clusterPlan); + + const readinessArgs = mocks.runCurlProbe.mock.calls[0][0] as string[]; + expect(readinessArgs).toContain(`${HEAD_BASE_URL}/health`); + expect(readinessArgs.join(" ")).not.toContain("/v1/models"); + expect(mocks.runCurlProbe.mock.calls[0][1]).toMatchObject({ pinnedAddresses: [] }); + const unauthenticatedArgs = mocks.runCurlProbe.mock.calls[1][0] as string[]; + const authenticatedArgs = mocks.runCurlProbe.mock.calls[2][0] as string[]; + expect(unauthenticatedArgs).toContain(`${HEAD_BASE_URL}/v1/models`); + expect(unauthenticatedArgs).not.toContain("--config"); + expect(authenticatedArgs).toContain("--config"); + expect(authenticatedArgs).not.toContain(API_KEY); + expect(mocks.runCurlProbe.mock.calls[1][1]).toMatchObject({ pinnedAddresses: [] }); + expect(mocks.runCurlProbe.mock.calls[2][1]).toMatchObject({ pinnedAddresses: [] }); + expect(mocks.dockerSpawn.mock.calls[0][1].env.VLLM_API_KEY).toBeUndefined(); + expect(mocks.dockerSpawn.mock.calls[0][1].env.DOCKER_CONTEXT).toBe("default"); + expect(mocks.dockerImageInspectFormat.mock.calls[0][2].env.DOCKER_CONTEXT).toBe("default"); + }); + + it("selects pinned Nemotron Ultra without prompting when an explicit peer qualifies", async () => { + delete process.env.NEMOCLAW_VLLM_MODEL; + const promptFn = vi.fn(); + const profile = detectVllmProfile({ platform: "station", type: "nvidia" }); + + await expect( + installVllm(profile!, { hasImage: true, nonInteractive: true, promptFn }), + ).resolves.toEqual({ ok: true }); + + expect(promptFn).not.toHaveBeenCalled(); + const [downloadArgs] = mocks.dockerSpawn.mock.calls[0] as [string[]]; + expect(downloadArgs).toEqual( + expect.arrayContaining([ + "nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4", + "--revision", + DUAL_STATION_VLLM_RUNTIME.modelRevision, + ]), + ); + expect(mocks.probeCapability).toHaveBeenCalledTimes(2); + }); + + it("fails through the normal gated-model resolver before side effects without an HF token", async () => { + delete process.env.NEMOCLAW_VLLM_MODEL; + delete process.env.HF_TOKEN; + delete process.env.HUGGING_FACE_HUB_TOKEN; + const beforeInstall = vi.fn(); + const profile = detectVllmProfile({ platform: "station", type: "nvidia" }); + + await expect( + installVllm(profile!, { + hasImage: true, + nonInteractive: true, + promptFn: vi.fn(), + beforeInstall, + }), + ).resolves.toEqual({ ok: false }); + + expect(mocks.probeCapability).toHaveBeenCalledTimes(1); + expect(beforeInstall).not.toHaveBeenCalled(); + expect(mocks.preflightOwnership).not.toHaveBeenCalled(); + expect(mocks.dockerPullWithProgressWatchdog).not.toHaveBeenCalled(); + expect(mocks.dockerSpawn).not.toHaveBeenCalled(); + expect(mocks.stageModelSnapshot).not.toHaveBeenCalled(); + expect(errorSpy).toHaveBeenCalledWith(expect.stringContaining("gated on Hugging Face")); + }); + + it("skips only the model picker for an interactive qualified-peer install", async () => { + delete process.env.NEMOCLAW_VLLM_MODEL; + const promptFn = vi.fn().mockResolvedValue("y"); + const profile = detectVllmProfile({ platform: "station", type: "nvidia" }); + + await expect( + installVllm(profile!, { hasImage: true, nonInteractive: false, promptFn }), + ).resolves.toEqual({ ok: true }); + + expect(promptFn).toHaveBeenCalledTimes(1); + expect(promptFn).toHaveBeenCalledWith(" Continue? [y/N]: "); + expect(promptFn).not.toHaveBeenCalledWith(expect.stringContaining("Choose model")); + }); + + it("keeps an explicit model override ahead of peer-driven automatic selection", async () => { + process.env.NEMOCLAW_VLLM_MODEL = "nemotron-3-nano-4b"; + mocks.runCapture.mockReturnValue(""); + const profile = detectVllmProfile({ platform: "station", type: "nvidia" }); + + await expect( + installVllm(profile!, { hasImage: true, nonInteractive: true, promptFn: vi.fn() }), + ).resolves.toEqual({ ok: false }); + + expect(mocks.probeCapability).not.toHaveBeenCalled(); + expect(mocks.stageModelSnapshot).not.toHaveBeenCalled(); + expect(errorSpy).toHaveBeenCalledWith(" vLLM install failed: docker not found on PATH"); + }); + + it("stages a missing peer snapshot after download and requires a ready re-probe", async () => { + const clusterPlan = plan(); + mocks.probeCapability + .mockReturnValueOnce({ + kind: "ready", + plan: clusterPlan, + peerModelSnapshot: "staging-required", + }) + .mockReturnValueOnce({ + kind: "ready", + plan: clusterPlan, + peerModelSnapshot: "ready", + }); + const profile = detectVllmProfile({ platform: "station", type: "nvidia" }); + + await expect( + installVllm(profile!, { hasImage: true, nonInteractive: true, promptFn: vi.fn() }), + ).resolves.toEqual({ ok: true }); + + expect(mocks.stageModelSnapshot).toHaveBeenCalledWith(clusterPlan); + expect(mocks.stageModelSnapshot.mock.invocationCallOrder[0]).toBeGreaterThan( + mocks.dockerSpawn.mock.invocationCallOrder[0], + ); + expect(mocks.probeCapability.mock.invocationCallOrder[1]).toBeGreaterThan( + mocks.stageModelSnapshot.mock.invocationCallOrder[0], + ); + expect(mocks.startManaged.mock.invocationCallOrder[0]).toBeGreaterThan( + mocks.probeCapability.mock.invocationCallOrder[1], + ); + }); + + it("fails closed before re-probe or launch when peer snapshot staging fails", async () => { + mocks.probeCapability.mockReturnValue({ + kind: "ready", + plan: plan(), + peerModelSnapshot: "staging-required", + }); + mocks.stageModelSnapshot.mockResolvedValue({ ok: false, reason: "peer transfer timed out" }); + const profile = detectVllmProfile({ platform: "station", type: "nvidia" }); + + await expect( + installVllm(profile!, { hasImage: true, nonInteractive: true, promptFn: vi.fn() }), + ).resolves.toEqual({ ok: false }); + + expect(mocks.probeCapability).toHaveBeenCalledTimes(1); + expect(mocks.startManaged).not.toHaveBeenCalled(); + expect(errorSpy).toHaveBeenCalledWith(" vLLM install failed: peer transfer timed out"); + }); + + it("fails before callbacks, prompts, or Docker work when a configured peer is incapable", async () => { + const beforeInstall = vi.fn(); + mocks.probeCapability.mockReturnValue({ + kind: "unavailable", + code: "peer-fabric-unavailable", + reason: "peer fabric is incomplete", + }); + const profile = detectVllmProfile({ platform: "station", type: "nvidia" }); + + await expect( + installVllm(profile!, { + hasImage: true, + nonInteractive: true, + promptFn: vi.fn(), + beforeInstall, + }), + ).resolves.toEqual({ ok: false }); + + expect(beforeInstall).not.toHaveBeenCalled(); + expect(mocks.runCapture).not.toHaveBeenCalled(); + expect(mocks.preflightOwnership).not.toHaveBeenCalled(); + expect(mocks.dockerPullWithProgressWatchdog).not.toHaveBeenCalled(); + expect(mocks.ensureApiKey).not.toHaveBeenCalled(); + expect(errorSpy).toHaveBeenCalledWith( + " Dual DGX Station setup unavailable: peer fabric is incomplete", + ); + }); + + it("stops before key creation and model download when either GPU runtime smoke fails", async () => { + mocks.preflightGpuRuntime.mockReturnValue({ + ok: false, + reason: "worker GPU smoke did not expose exactly the discovered GPU", + }); + const profile = detectVllmProfile({ platform: "station", type: "nvidia" }); + + await expect( + installVllm(profile!, { hasImage: true, nonInteractive: true, promptFn: vi.fn() }), + ).resolves.toEqual({ ok: false }); + + expect(mocks.dockerPullWithProgressWatchdog).toHaveBeenCalledTimes(2); + expect(mocks.ensureApiKey).not.toHaveBeenCalled(); + expect(mocks.dockerSpawn).not.toHaveBeenCalled(); + expect(mocks.startManaged).not.toHaveBeenCalled(); + expect(errorSpy).toHaveBeenCalledWith( + " vLLM install failed: worker GPU smoke did not expose exactly the discovered GPU", + ); + }); + + it("refuses teardown when the qualified topology changes during the model download", async () => { + const originalPlan = plan(); + const changedPlan = plan(); + changedPlan.peer.hostname = "station-b-replaced"; + mocks.probeCapability + .mockReturnValueOnce({ + kind: "ready", + plan: originalPlan, + peerModelSnapshot: "ready", + }) + .mockReturnValueOnce({ + kind: "ready", + plan: changedPlan, + peerModelSnapshot: "ready", + }); + const profile = detectVllmProfile({ platform: "station", type: "nvidia" }); + + await expect( + installVllm(profile!, { hasImage: true, nonInteractive: true, promptFn: vi.fn() }), + ).resolves.toEqual({ ok: false }); + + expect(mocks.dockerSpawn).toHaveBeenCalledTimes(1); + expect(mocks.ensureApiKey).not.toHaveBeenCalled(); + expect(mocks.startManaged).not.toHaveBeenCalled(); + expect(errorSpy).toHaveBeenCalledWith( + " vLLM install failed: dual-Station topology changed during download; rerun setup against a stable pair.", + ); + }); + + it("rolls back a new pair when unauthenticated model inventory is exposed", async () => { + mocks.runCurlProbe.mockImplementation((args: string[]) => ({ + ok: true, + httpStatus: 200, + message: "ok", + body: args.at(-1)?.endsWith("/v1/models") + ? JSON.stringify({ data: [{ id: "nvidia/nemotron-3-ultra-550b-a55b" }] }) + : "", + })); + const profile = detectVllmProfile({ platform: "station", type: "nvidia" }); + + await expect( + installVllm(profile!, { hasImage: true, nonInteractive: true, promptFn: vi.fn() }), + ).resolves.toEqual({ ok: false }); + + expect(mocks.cleanup).toHaveBeenCalledWith( + expect.objectContaining({ masterAddress: "192.168.100.1" }), + ); + expect(errorSpy).toHaveBeenCalledWith( + " vLLM install failed: unauthenticated model inventory returned HTTP 200; expected vLLM to reject it with HTTP 401", + ); + }); + + it("stops before storage or image work when container ownership is not exact", async () => { + mocks.preflightOwnership.mockReturnValue({ + ok: false, + reason: "worker container ownership is foreign; refusing mutation", + }); + const profile = detectVllmProfile({ platform: "station", type: "nvidia" }); + + await expect( + installVllm(profile!, { hasImage: true, nonInteractive: true, promptFn: vi.fn() }), + ).resolves.toEqual({ ok: false }); + + expect(mocks.probeHostStorage).not.toHaveBeenCalled(); + expect(mocks.dockerImageInspectFormat).not.toHaveBeenCalled(); + expect(mocks.dockerPullWithProgressWatchdog).not.toHaveBeenCalled(); + expect(mocks.dockerSpawn).not.toHaveBeenCalled(); + expect(mocks.startManaged).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/inference/vllm-models.test.ts b/src/lib/inference/vllm-models.test.ts index 6db54eb9f0..557f4d48a7 100644 --- a/src/lib/inference/vllm-models.test.ts +++ b/src/lib/inference/vllm-models.test.ts @@ -5,6 +5,7 @@ import { describe, expect, it } from "vitest"; import { assertGatedModelAccess, + buildNemotronUltraDistributedServeCommand, buildVllmServeCommand, DEFAULT_VLLM_MODEL, modelsForPlatform, @@ -101,6 +102,84 @@ describe("vllm model registry", () => { ); }); + it("builds the pinned two-Station Nemotron Ultra vLLM v0.22 head command", () => { + const cmd = buildNemotronUltraDistributedServeCommand({ + nodeRank: 0, + masterAddr: "192.168.240.1", + masterPort: 29501, + }); + + expect(cmd).toBe( + [ + "export VLLM_WEIGHT_OFFLOADING_DISABLE_PIN_MEMORY=1", + "&& export VLLM_NVFP4_GEMM_BACKEND=flashinfer-trtllm", + "&& export VLLM_FLOAT32_MATMUL_PRECISION=high", + "&& vllm serve nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4", + "--tensor-parallel-size 2", + "--pipeline-parallel-size 1", + "--data-parallel-size 1", + "--port 8000", + "--trust-remote-code", + "--nnodes 2", + "--node-rank 0", + "--master-addr 192.168.240.1", + "--master-port 29501", + "--max-model-len 32768", + "--revision 183968f87ae4cedce3039313cac1fd43d112c578", + "--served-model-name nvidia/nemotron-3-ultra-550b-a55b", + "--host 192.168.240.1", + "--cpu-offload-gb 150", + "--cpu-offload-params experts", + "--max-num-seqs 16", + "--gpu-memory-utilization 0.85", + "--reasoning-parser nemotron_v3", + "--enable-auto-tool-choice", + "--tool-call-parser qwen3_coder", + `--default-chat-template-kwargs '{"enable_thinking":true,"force_nonempty_content":true}'`, + ].join(" "), + ); + expect(cmd).not.toContain("--headless"); + expect(cmd).not.toContain("--kernel_config"); + expect(cmd).not.toContain("--speculative-config"); + }); + + it("adds headless mode only to the rank-1 Nemotron Ultra command", () => { + const head = buildNemotronUltraDistributedServeCommand({ + nodeRank: 0, + masterAddr: "192.168.240.1", + masterPort: 29501, + }); + const worker = buildNemotronUltraDistributedServeCommand({ + nodeRank: 1, + masterAddr: "192.168.240.1", + masterPort: 29501, + }); + + expect(worker).toBe( + head + .replace("--node-rank 0", "--node-rank 1") + .replace("--master-port 29501", "--master-port 29501 --headless") + .replace("--host 192.168.240.1", "--host 127.0.0.1"), + ); + }); + + it("rejects invalid two-Station Nemotron Ultra rendezvous options", () => { + expect(() => + buildNemotronUltraDistributedServeCommand({ + nodeRank: 0, + masterAddr: "station a", + masterPort: 29501, + }), + ).toThrow(/masterAddr/); + expect(() => + buildNemotronUltraDistributedServeCommand({ + nodeRank: 1, + masterAddr: "192.168.240.1", + masterPort: 70000, + }), + ).toThrow(/masterPort/); + }); + it("rejects an unknown NEMOCLAW_VLLM_MODEL with a helpful message", () => { expect(() => selectVllmModelFromEnv({ NEMOCLAW_VLLM_MODEL: "made-up-model" } as NodeJS.ProcessEnv), @@ -132,6 +211,14 @@ describe("vllm model registry", () => { ); }); + it("keeps the pinned Nemotron Ultra recipe behind Hugging Face access", () => { + const ultra = VLLM_MODELS.find((m) => m.envValue === "nemotron-3-ultra-550b-a55b"); + expect(ultra?.gated).toBe(true); + expect(() => assertGatedModelAccess(ultra!, {} as NodeJS.ProcessEnv)).toThrow( + /gated on Hugging Face/, + ); + }); + it("never rejects a non-gated model regardless of token state", () => { const qwen = VLLM_MODELS.find((m) => m.envValue === "qwen3.6-27b"); expect(() => assertGatedModelAccess(qwen!, {} as NodeJS.ProcessEnv)).not.toThrow(); diff --git a/src/lib/inference/vllm-models.ts b/src/lib/inference/vllm-models.ts index ae46263125..a735e45054 100644 --- a/src/lib/inference/vllm-models.ts +++ b/src/lib/inference/vllm-models.ts @@ -27,6 +27,8 @@ * envelope, and tool-call behaviour validated. */ +import net from "node:net"; + export type VllmPlatform = "spark" | "station" | "linux"; export interface VllmRuntimeOverride { @@ -245,7 +247,7 @@ export const VLLM_MODELS: readonly VllmModelDef[] = [ "--default-chat-template-kwargs", `'{"enable_thinking":true,"force_nonempty_content":true}'`, ], - gated: false, + gated: true, platforms: ["station"], serveEnv: { VLLM_WEIGHT_OFFLOADING_DISABLE_PIN_MEMORY: "1", @@ -472,6 +474,120 @@ function shellQuote(value: string): string { return `'${value.replace(/'/g, `'\"'\"'`)}'`; } +function rewriteVllmArgs( + args: readonly string[], + overrides: Readonly>, + omittedFlags: ReadonlySet = new Set(), +): string[] { + const result: string[] = []; + const remainingOverrides = new Set(Object.keys(overrides)); + for (let index = 0; index < args.length; index += 1) { + const arg = args[index]; + if (omittedFlags.has(arg)) { + if (index === args.length - 1) throw new Error(`Missing value for vLLM argument '${arg}'.`); + index += 1; + continue; + } + if (Object.prototype.hasOwnProperty.call(overrides, arg)) { + if (index === args.length - 1) throw new Error(`Missing value for vLLM argument '${arg}'.`); + result.push(arg, overrides[arg]); + remainingOverrides.delete(arg); + index += 1; + continue; + } + result.push(arg); + } + if (remainingOverrides.size > 0) { + throw new Error(`Cannot override missing vLLM argument '${[...remainingOverrides][0]}'.`); + } + return result; +} + +export interface NemotronUltraDistributedServeOptions { + /** vLLM multiprocessing rank: 0 serves the API and 1 runs headless. */ + nodeRank: 0 | 1; + /** Routable rank-0 address used by both nodes for the distributed rendezvous. */ + masterAddr: string; + /** Routable rank-0 port used by both nodes for the distributed rendezvous. */ + masterPort: number; +} + +/** + * Build one side of the validated two-Station Nemotron Ultra vLLM v0.22 + * direct-tensor-parallel launch. Existing callers keep the single-node + * registry command unless they opt into this rank/address/port API. + */ +export function buildNemotronUltraDistributedServeCommand( + options: NemotronUltraDistributedServeOptions, +): string { + if (options.nodeRank !== 0 && options.nodeRank !== 1) { + throw new Error("Nemotron Ultra distributed nodeRank must be 0 or 1."); + } + const masterAddr = options.masterAddr.trim(); + if (net.isIP(masterAddr) !== 4) { + throw new Error("Nemotron Ultra distributed masterAddr must be a canonical IPv4 address."); + } + if ( + !Number.isInteger(options.masterPort) || + options.masterPort < 1 || + options.masterPort > 65535 + ) { + throw new Error("Nemotron Ultra distributed masterPort must be an integer from 1 to 65535."); + } + + const model = VLLM_MODELS.find( + (candidate) => candidate.envValue === "nemotron-3-ultra-550b-a55b", + ); + if (!model?.revision || !model.servedModelId) { + throw new Error( + "Nemotron Ultra distributed serving requires a pinned revision and served model id.", + ); + } + + const sharedArgs = rewriteVllmArgs(SHARED_VLLM_ARGS, { "--tensor-parallel-size": "2" }); + const modelArgs = rewriteVllmArgs( + model.modelArgs, + { + // Rank 0 binds only to the selected direct-attach RoCE address. This + // keeps the API off the management network while still giving the + // OpenShell route a host-reachable endpoint. The lifecycle also enables + // vLLM bearer authentication. The headless worker exposes no API. + "--host": options.nodeRank === 0 ? masterAddr : "127.0.0.1", + "--max-num-seqs": "16", + "--gpu-memory-utilization": "0.85", + }, + new Set(["--kernel_config", "--speculative-config"]), + ); + const serveEnv = { + ...model.serveEnv, + VLLM_FLOAT32_MATMUL_PRECISION: "high", + }; + const envPrefix = `${Object.entries(serveEnv) + .map(([key, value]) => `export ${key}=${value}`) + .join(" && ")} && `; + const args = [ + ...sharedArgs, + "--nnodes", + "2", + "--node-rank", + String(options.nodeRank), + "--master-addr", + shellQuote(masterAddr), + "--master-port", + String(options.masterPort), + ...(options.nodeRank === 1 ? ["--headless"] : []), + "--max-model-len", + "32768", + "--revision", + model.revision, + "--served-model-name", + model.servedModelId, + ...modelArgs, + ]; + + return `${envPrefix}vllm serve ${model.id} ${args.join(" ")}`; +} + /** * Build the `vllm serve` command line for the supplied model: the shared * serving flags merged with the model-specific args from the registry. diff --git a/src/lib/inference/vllm-ownership.test.ts b/src/lib/inference/vllm-ownership.test.ts new file mode 100644 index 0000000000..744f0d295d --- /dev/null +++ b/src/lib/inference/vllm-ownership.test.ts @@ -0,0 +1,105 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + dockerCapture: vi.fn(), +})); + +vi.mock("../adapters/docker", async (importOriginal) => ({ + ...(await importOriginal()), + dockerCapture: mocks.dockerCapture, +})); + +import { + isNemoClawManagedVllmRunning, + NEMOCLAW_VLLM_CONTAINER_NAME, + NEMOCLAW_VLLM_MANAGED_LABEL, +} from "./vllm"; +import { + DUAL_STATION_VLLM_CLUSTER_LABEL, + DUAL_STATION_VLLM_ENDPOINT_LABEL, + DUAL_STATION_VLLM_ROLE_LABEL, +} from "./vllm-station-cluster-lifecycle"; + +const MANAGED_CONTAINER_ID = "a".repeat(64); + +function vllmContainerRow( + containerName: string, + { + id = MANAGED_CONTAINER_ID, + label = "true", + state = "exited", + dualRole = "", + dualEndpoint = "", + dualCluster = "", + } = {}, +): string { + return [id, containerName, state, label, dualRole, dualEndpoint, dualCluster].join("|"); +} + +beforeEach(() => vi.clearAllMocks()); + +describe("managed vLLM ownership", () => { + it("recognizes only the exact running container with the managed label", () => { + mocks.dockerCapture.mockReturnValue( + vllmContainerRow(NEMOCLAW_VLLM_CONTAINER_NAME, { state: "running" }), + ); + + expect(isNemoClawManagedVllmRunning()).toBe(true); + expect(mocks.dockerCapture).toHaveBeenCalledWith( + [ + "container", + "ls", + "--all", + "--no-trunc", + "--filter", + `name=^/${NEMOCLAW_VLLM_CONTAINER_NAME}$`, + "--format", + [ + "{{.ID}}", + "{{.Names}}", + "{{.State}}", + `{{.Label "${NEMOCLAW_VLLM_MANAGED_LABEL}"}}`, + `{{.Label "${DUAL_STATION_VLLM_ROLE_LABEL}"}}`, + `{{.Label "${DUAL_STATION_VLLM_ENDPOINT_LABEL}"}}`, + `{{.Label "${DUAL_STATION_VLLM_CLUSTER_LABEL}"}}`, + ].join("|"), + ], + expect.objectContaining({ timeout: 10_000 }), + ); + }); + + it("recognizes a running dual-Station head without treating it as legacy ownership", () => { + mocks.dockerCapture.mockReturnValue( + vllmContainerRow(NEMOCLAW_VLLM_CONTAINER_NAME, { + state: "running", + dualRole: "head", + dualEndpoint: "http://192.168.100.1:8000", + dualCluster: "f".repeat(64), + }), + ); + + expect(isNemoClawManagedVllmRunning()).toBe(true); + }); + + it.each([ + vllmContainerRow(NEMOCLAW_VLLM_CONTAINER_NAME), + vllmContainerRow(NEMOCLAW_VLLM_CONTAINER_NAME, { label: "" }), + vllmContainerRow(NEMOCLAW_VLLM_CONTAINER_NAME, { label: "false", state: "running" }), + "", + "malformed", + `${vllmContainerRow(NEMOCLAW_VLLM_CONTAINER_NAME)}\n${vllmContainerRow(NEMOCLAW_VLLM_CONTAINER_NAME)}`, + ])("fails closed for inspect output %j", (output) => { + mocks.dockerCapture.mockReturnValue(output); + expect(isNemoClawManagedVllmRunning()).toBe(false); + }); + + it("fails closed when Docker inspection throws", () => { + mocks.dockerCapture.mockImplementation(() => { + throw new Error("docker unavailable"); + }); + expect(isNemoClawManagedVllmRunning()).toBe(false); + }); +}); diff --git a/src/lib/inference/vllm-station-cluster-lifecycle.test.ts b/src/lib/inference/vllm-station-cluster-lifecycle.test.ts new file mode 100644 index 0000000000..8da461a585 --- /dev/null +++ b/src/lib/inference/vllm-station-cluster-lifecycle.test.ts @@ -0,0 +1,971 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { AsyncLocalStorage } from "node:async_hooks"; +import { describe, expect, it, vi } from "vitest"; +import { DUAL_STATION_VLLM_RUNTIME, type DualStationVllmPlan } from "./vllm-station-cluster"; +import { + areDualStationManagedVllmContainersRunning, + buildDualStationGpuSmokeRunArgs, + buildDualStationVllmRunArgs, + cleanupDualStationManagedVllm, + DUAL_STATION_VLLM_API_KEY_FINGERPRINT_LABEL, + DUAL_STATION_VLLM_CLUSTER_LABEL, + DUAL_STATION_VLLM_ENDPOINT_LABEL, + DUAL_STATION_VLLM_GPU_LABEL, + DUAL_STATION_VLLM_GPU_SMOKE_LABEL, + DUAL_STATION_VLLM_HEAD_CONTAINER_NAME, + DUAL_STATION_VLLM_LAUNCH_CONTRACT_LABEL, + DUAL_STATION_VLLM_LAUNCH_SCHEMA_LABEL, + DUAL_STATION_VLLM_MANAGED_LABEL, + DUAL_STATION_VLLM_ROLE_LABEL, + DUAL_STATION_VLLM_TRANSACTION_LABEL, + DUAL_STATION_VLLM_WORKER_CONTAINER_NAME, + type DualStationDockerOptions, + type DualStationVllmLifecycleDeps, + dualStationVllmApiKeyFingerprint, + dualStationVllmClusterId, + dualStationVllmLaunchContract, + getDualStationManagedVllmBaseUrl, + preflightDualStationGpuRuntime, + preflightDualStationManagedVllm, + startDualStationManagedVllm, + withDualStationManagedVllmLifecycle, +} from "./vllm-station-cluster-lifecycle"; + +const WORKER_ID = "a".repeat(64); +const HEAD_ID = "b".repeat(64); +const WORKER_SMOKE_ID = "c".repeat(64); +const HEAD_SMOKE_ID = "d".repeat(64); +const API_KEY = "e".repeat(64); +const START_CONFIG = { apiKey: API_KEY }; +const API_KEY_FINGERPRINT = dualStationVllmApiKeyFingerprint(API_KEY); +const TRANSACTION_ID = "1".repeat(32); + +function fixturePlan(): DualStationVllmPlan { + return { + peerSshTarget: "nvidia@station-b", + peerDockerHost: "ssh://nvidia@station-b", + runtime: DUAL_STATION_VLLM_RUNTIME, + local: { + hostname: "station-a", + home: "/home/local", + uid: 1000, + gpu: { + index: 0, + name: "NVIDIA GB300", + uuid: "GPU-aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", + }, + }, + peer: { + hostname: "station-b", + home: "/home/nvidia", + uid: 1001, + gpu: { + index: 1, + name: "NVIDIA GB300 Grace Blackwell Superchip", + uuid: "GPU-99999999-8888-7777-6666-555555555555", + }, + }, + rails: [ + { + index: 0, + subnet: "192.168.240.0/30", + local: { + rdmaDevice: "mlx5_0", + netdev: "cx8a0", + macAddress: "02:00:00:00:00:01", + uverbsDevice: "/dev/infiniband/uverbs0", + pciAddress: "0001:03:00.0", + address: "192.168.240.1", + }, + peer: { + rdmaDevice: "mlx5_0", + netdev: "cx8b0", + macAddress: "02:00:00:00:00:02", + uverbsDevice: "/dev/infiniband/uverbs0", + pciAddress: "0002:03:00.0", + address: "192.168.240.2", + }, + }, + { + index: 1, + subnet: "192.168.240.4/30", + local: { + rdmaDevice: "mlx5_1", + netdev: "cx8a1", + macAddress: "02:00:00:00:00:05", + uverbsDevice: "/dev/infiniband/uverbs1", + pciAddress: "0001:03:00.1", + address: "192.168.240.5", + }, + peer: { + rdmaDevice: "mlx5_1", + netdev: "cx8b1", + macAddress: "02:00:00:00:00:06", + uverbsDevice: "/dev/infiniband/uverbs1", + pciAddress: "0002:03:00.1", + address: "192.168.240.6", + }, + }, + ], + masterAddress: "192.168.240.1", + roceGidIndex: 3, + }; +} + +type FakeContainer = { + id: string; + name: string; + state: string; + image: string; + labels: Record; +}; + +function dockerValues(args: readonly string[], flag: string): string[] { + return args.flatMap((arg, index) => + arg === flag && index < args.length - 1 ? [args[index + 1]] : [], + ); +} + +function raise(message: string): never { + throw new Error(message); +} + +function row(container: FakeContainer): string { + return [ + container.id, + container.name, + container.state, + container.image, + container.labels[DUAL_STATION_VLLM_MANAGED_LABEL] ?? "", + container.labels[DUAL_STATION_VLLM_ROLE_LABEL] ?? "", + container.labels[DUAL_STATION_VLLM_ENDPOINT_LABEL] ?? "", + container.labels[DUAL_STATION_VLLM_CLUSTER_LABEL] ?? "", + container.labels[DUAL_STATION_VLLM_GPU_LABEL] ?? "", + container.labels[DUAL_STATION_VLLM_LAUNCH_SCHEMA_LABEL] ?? "", + container.labels[DUAL_STATION_VLLM_LAUNCH_CONTRACT_LABEL] ?? "", + container.labels[DUAL_STATION_VLLM_API_KEY_FINGERPRINT_LABEL] ?? "", + container.labels[DUAL_STATION_VLLM_TRANSACTION_LABEL] ?? "", + ].join("\t"); +} + +function fakeContainer( + role: "head" | "worker", + overrides: Partial = {}, +): FakeContainer { + return { + id: role === "head" ? HEAD_ID : WORKER_ID, + name: + role === "head" + ? DUAL_STATION_VLLM_HEAD_CONTAINER_NAME + : DUAL_STATION_VLLM_WORKER_CONTAINER_NAME, + state: "running", + image: DUAL_STATION_VLLM_RUNTIME.image, + labels: { + [DUAL_STATION_VLLM_MANAGED_LABEL]: "true", + [DUAL_STATION_VLLM_ROLE_LABEL]: role, + [DUAL_STATION_VLLM_ENDPOINT_LABEL]: + role === "head" ? "http://192.168.240.1:8000" : "headless", + [DUAL_STATION_VLLM_CLUSTER_LABEL]: dualStationVllmClusterId(fixturePlan()), + [DUAL_STATION_VLLM_GPU_LABEL]: + role === "head" ? fixturePlan().local.gpu.uuid : fixturePlan().peer.gpu.uuid, + [DUAL_STATION_VLLM_LAUNCH_SCHEMA_LABEL]: "1", + [DUAL_STATION_VLLM_LAUNCH_CONTRACT_LABEL]: dualStationVllmLaunchContract(fixturePlan(), role), + [DUAL_STATION_VLLM_API_KEY_FINGERPRINT_LABEL]: API_KEY_FINGERPRINT, + [DUAL_STATION_VLLM_TRANSACTION_LABEL]: TRANSACTION_ID, + }, + ...overrides, + }; +} + +function harness( + options: { + failRole?: "head" | "worker"; + invalidIdRole?: "head" | "worker"; + failSmokeTarget?: "local" | "peer"; + failSmokeCleanupTarget?: "local" | "peer"; + missingImageTarget?: "local" | "peer"; + smokeGpuOutput?: Partial>; + lateCreateRole?: "head" | "worker"; + failedRoleForeignTransaction?: "head" | "worker"; + } = {}, +) { + const containers = new Map(); + const operations: Array<{ kind: "capture" | "rm" | "run"; target: string; value: string }> = []; + const captureOptions: Array = []; + const rmOptions: Array = []; + const runCalls: Array<{ + args: readonly string[]; + options: DualStationDockerOptions | undefined; + }> = []; + const buildRemoteDockerEnv = vi.fn((sshUri: string) => ({ + TARGET: "peer", + DOCKER_HOST: sshUri, + VLLM_API_KEY: "ambient-must-be-stripped", + })); + let nonceCounter = 0; + let transactionCounter = 0; + let lifecycleLockActive = 0; + let maxLifecycleLockActive = 0; + let lifecycleLockTail = Promise.resolve(); + const lifecycleLockContext = new AsyncLocalStorage(); + let lateContainer: { targetName: string; container: FakeContainer } | null = null; + + async function acquireLifecycleLock(operation: () => Promise | T): Promise { + const previous = lifecycleLockTail; + let release: () => void = () => undefined; + lifecycleLockTail = new Promise((resolve) => { + release = resolve; + }); + await previous; + lifecycleLockActive += 1; + maxLifecycleLockActive = Math.max(maxLifecycleLockActive, lifecycleLockActive); + try { + return await lifecycleLockContext.run(true, operation); + } finally { + lifecycleLockActive -= 1; + release(); + } + } + + function target(optionsArg?: DualStationDockerOptions): string { + return String(optionsArg?.env?.TARGET ?? "unknown"); + } + + function key(targetName: string, name: string): string { + return `${targetName}:${name}`; + } + + const deps: DualStationVllmLifecycleDeps = { + buildLocalDockerEnv: () => ({ + TARGET: "local", + VLLM_API_KEY: "ambient-must-be-stripped", + }), + buildRemoteDockerEnv, + createProbeNonce: () => { + nonceCounter += 1; + return nonceCounter.toString(16).padStart(32, "0"); + }, + createTransactionId: () => { + transactionCounter += 1; + return transactionCounter.toString(16).padStart(32, "0"); + }, + loadApiKey: () => API_KEY, + localInterfaceAddresses: () => [fixturePlan().masterAddress], + waitBeforeReconcile: async () => { + const pending = lateContainer; + lateContainer = null; + return pending + ? void containers.set(key(pending.targetName, pending.container.name), [pending.container]) + : undefined; + }, + withLifecycleLock: async (operation: () => Promise | T) => + lifecycleLockContext.getStore() ? await operation() : await acquireLifecycleLock(operation), + dockerCapture: (args, optionsArg) => { + captureOptions.push(optionsArg); + const targetName = target(optionsArg); + switch (args[0]) { + case "image": { + operations.push({ kind: "capture", target: targetName, value: `image:${args.at(-1)}` }); + return options.missingImageTarget === targetName + ? raise("missing image") + : `sha256:${"f".repeat(64)}\n`; + } + case "wait": + operations.push({ kind: "capture", target: targetName, value: `wait:${args[1]}` }); + return "0\n"; + case "logs": { + operations.push({ kind: "capture", target: targetName, value: `logs:${args[1]}` }); + const defaultUuid = + targetName === "local" ? fixturePlan().local.gpu.uuid : fixturePlan().peer.gpu.uuid; + return `${options.smokeGpuOutput?.[targetName as "local" | "peer"] ?? defaultUuid}\n`; + } + default: + break; + } + const filter = dockerValues(args, "--filter")[0] ?? ""; + const name = filter.replace(/^name=\^\//, "").replace(/\$$/, ""); + operations.push({ kind: "capture", target: targetName, value: name }); + const isSmokeInspection = + dockerValues(args, "--format")[0]?.includes(DUAL_STATION_VLLM_GPU_SMOKE_LABEL) ?? false; + return (containers.get(key(targetName, name)) ?? []) + .map((container) => + isSmokeInspection + ? [ + container.id, + container.name, + container.image, + container.labels[DUAL_STATION_VLLM_GPU_SMOKE_LABEL] ?? "", + container.labels[DUAL_STATION_VLLM_ROLE_LABEL] ?? "", + ].join("\t") + : row(container), + ) + .join("\n"); + }, + dockerRunDetached: (args, optionsArg) => { + const targetName = target(optionsArg); + const name = dockerValues(args, "--name")[0]; + runCalls.push({ args: [...args], options: optionsArg }); + operations.push({ kind: "run", target: targetName, value: name }); + const labels = Object.fromEntries( + dockerValues(args, "--label").map((label) => { + const separator = label.indexOf("="); + return [label.slice(0, separator), label.slice(separator + 1)]; + }), + ); + switch (name.startsWith("nemoclaw-vllm-gpu-smoke-")) { + case true: + return options.failSmokeTarget === targetName + ? { status: 1, stdout: "", stderr: "smoke failed" } + : (() => { + const smokeContainer: FakeContainer = { + id: targetName === "local" ? HEAD_SMOKE_ID : WORKER_SMOKE_ID, + name, + state: "exited", + image: DUAL_STATION_VLLM_RUNTIME.image, + labels, + }; + containers.set(key(targetName, name), [smokeContainer]); + return { status: 0, stdout: `${smokeContainer.id}\n` }; + })(); + default: + break; + } + const role = name === DUAL_STATION_VLLM_HEAD_CONTAINER_NAME ? "head" : "worker"; + const imageIndex = args.indexOf("/bin/bash") + 1; + const container = fakeContainer(role, { + image: args[imageIndex], + labels, + }); + return options.failedRoleForeignTransaction === role + ? (() => { + container.labels[DUAL_STATION_VLLM_TRANSACTION_LABEL] = "f".repeat(32); + containers.set(key(targetName, name), [container]); + return { status: 1, stdout: "", stderr: "ambiguous failed create" }; + })() + : options.lateCreateRole === role + ? (() => { + lateContainer = { targetName, container }; + return { status: null, stdout: "", stderr: "timed out" }; + })() + : options.failRole === role + ? { status: 1, stdout: "", stderr: "failed" } + : (() => { + containers.set(key(targetName, name), [container]); + return { + status: 0, + stdout: + options.invalidIdRole === role ? "not-a-container-id\n" : `${container.id}\n`, + }; + })(); + }, + dockerForceRm: (containerId, optionsArg) => { + rmOptions.push(optionsArg); + const targetName = target(optionsArg); + operations.push({ kind: "rm", target: targetName, value: containerId }); + const shouldFail = + options.failSmokeCleanupTarget === targetName && + (containerId === WORKER_SMOKE_ID || containerId === HEAD_SMOKE_ID); + const match = [...containers.entries()].find( + ([containerKey, entries]) => + containerKey.startsWith(`${targetName}:`) && + entries.some((entry) => entry.id === containerId), + ); + return shouldFail || !match + ? { status: 1 } + : (() => { + const [containerKey, entries] = match; + const remaining = entries.filter((entry) => entry.id !== containerId); + containers.set(containerKey, remaining); + return { status: 0 }; + })(); + }, + }; + + function seed(targetName: "local" | "peer", container: FakeContainer): void { + const containerKey = key(targetName, container.name); + containers.set(containerKey, [...(containers.get(containerKey) ?? []), container]); + } + + return { + buildRemoteDockerEnv, + captureOptions, + containers, + deps, + getMaxLifecycleLockActive: () => maxLifecycleLockActive, + operations, + rmOptions, + runCalls, + seed, + }; +} + +describe("dual-Station managed vLLM run argv", () => { + it.each(["head", "worker"] as const)("builds the exact %s launch contract", (role) => { + const plan = fixturePlan(); + const args = buildDualStationVllmRunArgs(plan, role, TRANSACTION_ID, API_KEY_FINGERPRINT); + const env = dockerValues(args, "--env"); + const expectedNode = role === "head" ? plan.local : plan.peer; + const expectedNetdev = role === "head" ? "cx8a0" : "cx8b0"; + + expect(args).toEqual( + expect.arrayContaining(["--network", "host", "--shm-size", "16g", "--read-only"]), + ); + expect(dockerValues(args, "--tmpfs")).toEqual([ + "/tmp:rw,nosuid,nodev,size=17179869184", + "/root/.cache:rw,nosuid,nodev,size=68719476736", + ]); + expect(args).toEqual( + expect.arrayContaining([ + "--cap-drop", + "ALL", + "--cap-add", + "IPC_LOCK", + "--cap-add", + "DAC_READ_SEARCH", + ]), + ); + expect(dockerValues(args, "--ulimit")).toEqual([ + "memlock=-1", + "stack=67108864", + "nofile=1048576:1048576", + ]); + expect(dockerValues(args, "--gpus")).toEqual([`device=${expectedNode.gpu.uuid}`]); + expect(args.filter((arg) => arg.startsWith("--device="))).toEqual([ + "--device=/dev/infiniband/uverbs0", + "--device=/dev/infiniband/uverbs1", + ]); + expect(dockerValues(args, "--volume")).toEqual([ + `${expectedNode.home}/.cache/huggingface/hub:/root/.cache/huggingface/hub:ro`, + ]); + expect(dockerValues(args, "--volume").join("\n")).not.toContain("/huggingface/token"); + expect(env).toEqual( + expect.arrayContaining([ + "NCCL_IB_HCA=mlx5_0,mlx5_1", + "NCCL_IB_DISABLE=0", + "NCCL_IB_GID_INDEX=3", + `NCCL_SOCKET_IFNAME=${expectedNetdev}`, + `GLOO_SOCKET_IFNAME=${expectedNetdev}`, + `TP_SOCKET_IFNAME=${expectedNetdev}`, + `OMPI_MCA_btl_tcp_if_include=${expectedNetdev}`, + `MN_IF_NAME=${expectedNetdev}`, + "NCCL_IGNORE_CPU_AFFINITY=1", + "UCX_NET_DEVICES=mlx5_0:1,mlx5_1:1", + "UCX_IB_GID_INDEX=3", + "HF_HUB_OFFLINE=1", + "TRANSFORMERS_OFFLINE=1", + ]), + ); + expect(args).toContain(plan.runtime.image); + expect(dockerValues(args, "--label")).toEqual( + expect.arrayContaining([ + `${DUAL_STATION_VLLM_LAUNCH_SCHEMA_LABEL}=1`, + `${DUAL_STATION_VLLM_LAUNCH_CONTRACT_LABEL}=${dualStationVllmLaunchContract(plan, role)}`, + `${DUAL_STATION_VLLM_API_KEY_FINGERPRINT_LABEL}=${API_KEY_FINGERPRINT}`, + `${DUAL_STATION_VLLM_TRANSACTION_LABEL}=${TRANSACTION_ID}`, + ]), + ); + expect(args).not.toContain("--privileged"); + expect(args.join("\n")).not.toContain("--device=/dev/infiniband:"); + expect(dockerValues(args, "--env").filter((name) => name === "VLLM_API_KEY")).toEqual( + role === "head" ? ["VLLM_API_KEY"] : [], + ); + expect(args).not.toContain(API_KEY); + expect(args.join("\n")).not.toMatch(/HF_TOKEN|HUGGING_FACE_HUB_TOKEN|docker run/u); + const command = args.at(-1) ?? ""; + const imageIndex = args.indexOf(plan.runtime.image); + expect(args.slice(imageIndex - 2, imageIndex + 2)).toEqual([ + "--entrypoint", + "/bin/bash", + plan.runtime.image, + "-lc", + ]); + expect(args.filter((arg) => arg === "-lc")).toHaveLength(1); + expect(imageIndex).toBe(args.length - 3); + expect(command).toContain(`--node-rank ${role === "head" ? "0" : "1"}`); + expect(command).toContain(role === "head" ? "--host 192.168.240.1" : "--host 127.0.0.1"); + expect(command.includes("--headless")).toBe(role === "worker"); + }); + + it.each(["head", "worker"] as const)("builds a bounded, no-pull %s GPU smoke command", (role) => { + const nonce = "1".repeat(32); + const { args, containerName } = buildDualStationGpuSmokeRunArgs(fixturePlan(), role, nonce); + + expect(containerName).toBe(`nemoclaw-vllm-gpu-smoke-${role}-${nonce}`); + expect(dockerValues(args, "--gpus")).toEqual([ + `device=${role === "head" ? fixturePlan().local.gpu.uuid : fixturePlan().peer.gpu.uuid}`, + ]); + expect(dockerValues(args, "--network")).toEqual(["none"]); + expect(dockerValues(args, "--cap-drop")).toEqual(["ALL"]); + expect(dockerValues(args, "--label")).toEqual( + expect.arrayContaining([ + `${DUAL_STATION_VLLM_GPU_SMOKE_LABEL}=${nonce}`, + `${DUAL_STATION_VLLM_ROLE_LABEL}=${role}`, + ]), + ); + expect(args).toContain("--pull=never"); + expect(args).toContain(DUAL_STATION_VLLM_RUNTIME.image); + expect(args).toEqual(expect.arrayContaining(["--entrypoint", "nvidia-smi"])); + expect(args).not.toContain("--rm"); + expect(args.join("\n")).not.toContain("VLLM_API_KEY"); + }); + + it.each([ + "/dev/infiniband/rdma_cm", + "/dev/infiniband/uverbs0", + ])("rejects an unsafe or duplicate verbs character device: %s", (uverbsDevice) => { + const plan = fixturePlan(); + plan.rails[1].local.uverbsDevice = uverbsDevice; + + expect(() => + buildDualStationVllmRunArgs(plan, "head", TRANSACTION_ID, API_KEY_FINGERPRINT), + ).toThrow(/rails must use two distinct devices|rail endpoint is invalid/u); + }); +}); + +describe("dual-Station managed vLLM lifecycle", () => { + it("provides a read-only ownership preflight before download work", () => { + const fake = harness(); + + expect(preflightDualStationManagedVllm(fixturePlan(), fake.deps)).toEqual({ ok: true }); + expect(fake.operations.filter((operation) => operation.kind === "capture")).toHaveLength(2); + expect(fake.operations.some((operation) => operation.kind === "run")).toBe(false); + expect(fake.operations.some((operation) => operation.kind === "rm")).toBe(false); + }); + + it("proves exact-image GPU execution on both daemons and removes both exact probe IDs", async () => { + const fake = harness(); + + expect(await preflightDualStationGpuRuntime(fixturePlan(), fake.deps)).toEqual({ ok: true }); + expect(fake.operations.filter((operation) => operation.kind === "run")).toHaveLength(2); + expect(fake.operations.filter((operation) => operation.kind === "rm")).toEqual([ + { kind: "rm", target: "peer", value: WORKER_SMOKE_ID }, + { kind: "rm", target: "local", value: HEAD_SMOKE_ID }, + ]); + expect( + fake.operations.filter( + (operation) => operation.kind === "capture" && operation.value.startsWith("image:"), + ), + ).toHaveLength(2); + for (const call of fake.runCalls) { + expect(call.args).toContain("--pull=never"); + expect(call.args).toContain(DUAL_STATION_VLLM_RUNTIME.image); + expect(call.options?.env?.VLLM_API_KEY).toBeUndefined(); + } + for (const options of [...fake.captureOptions, ...fake.rmOptions]) { + expect(options?.env?.VLLM_API_KEY).toBeUndefined(); + } + }); + + it("does not mutate either daemon unless both exact pinned images are present", async () => { + const fake = harness({ missingImageTarget: "local" }); + + expect(await preflightDualStationGpuRuntime(fixturePlan(), fake.deps)).toEqual({ + ok: false, + reason: "head pinned vLLM image is not present or could not be inspected", + }); + expect(fake.operations.some((operation) => operation.kind === "run")).toBe(false); + expect(fake.operations.some((operation) => operation.kind === "rm")).toBe(false); + }); + + it("removes an exact failed GPU probe and never starts the managed containers", async () => { + const fake = harness({ smokeGpuOutput: { peer: "GPU-not-the-discovered-device" } }); + + expect(await startDualStationManagedVllm(fixturePlan(), START_CONFIG, fake.deps)).toEqual({ + ok: false, + reason: "worker GPU smoke did not expose exactly the discovered GPU", + rollbackErrors: [], + }); + expect(fake.operations.filter((operation) => operation.kind === "run")).toHaveLength(1); + expect(fake.operations.filter((operation) => operation.kind === "rm")).toEqual([ + { kind: "rm", target: "peer", value: WORKER_SMOKE_ID }, + ]); + }); + + it.each([ + "short", + "A".repeat(64), + ])("rejects an unsafe API key before probing: %s", async (apiKey) => { + const fake = harness(); + + expect(await startDualStationManagedVllm(fixturePlan(), { apiKey }, fake.deps)).toMatchObject({ + ok: false, + reason: expect.stringContaining("64 lowercase hexadecimal"), + }); + expect(fake.operations).toEqual([]); + }); + + it("starts the worker before the head and returns validated exact IDs", async () => { + const fake = harness(); + + expect(await startDualStationManagedVllm(fixturePlan(), START_CONFIG, fake.deps)).toEqual({ + ok: true, + baseUrl: "http://192.168.240.1:8000", + headContainerId: HEAD_ID, + workerContainerId: WORKER_ID, + reusedExisting: false, + }); + expect(fake.operations.filter((operation) => operation.kind === "run")).toEqual([ + { + kind: "run", + target: "peer", + value: `nemoclaw-vllm-gpu-smoke-worker-${"1".padStart(32, "0")}`, + }, + { + kind: "run", + target: "local", + value: `nemoclaw-vllm-gpu-smoke-head-${"2".padStart(32, "0")}`, + }, + { kind: "run", target: "peer", value: DUAL_STATION_VLLM_WORKER_CONTAINER_NAME }, + { kind: "run", target: "local", value: DUAL_STATION_VLLM_HEAD_CONTAINER_NAME }, + ]); + const headRun = fake.runCalls.find( + ({ args }) => dockerValues(args, "--name")[0] === DUAL_STATION_VLLM_HEAD_CONTAINER_NAME, + ); + expect(headRun?.options?.env?.VLLM_API_KEY).toBe(API_KEY); + expect(headRun?.args).toContain("VLLM_API_KEY"); + expect(headRun?.args).not.toContain(API_KEY); + for (const call of fake.runCalls.filter((call) => call !== headRun)) { + expect(call.options?.env?.VLLM_API_KEY).toBeUndefined(); + expect(call.args).not.toContain(API_KEY); + } + for (const options of [...fake.captureOptions, ...fake.rmOptions]) { + expect(options?.env?.VLLM_API_KEY).toBeUndefined(); + } + expect(fake.buildRemoteDockerEnv).toHaveBeenCalledWith("ssh://nvidia@station-b"); + }); + + it("reuses an already-running exact pair without tearing down the working service", async () => { + const fake = harness(); + fake.seed("local", fakeContainer("head")); + fake.seed("peer", fakeContainer("worker")); + + expect(await startDualStationManagedVllm(fixturePlan(), START_CONFIG, fake.deps)).toEqual({ + ok: true, + baseUrl: "http://192.168.240.1:8000", + headContainerId: HEAD_ID, + workerContainerId: WORKER_ID, + reusedExisting: true, + }); + expect(fake.operations.filter((operation) => operation.kind === "rm")).toEqual([ + { kind: "rm", target: "peer", value: WORKER_SMOKE_ID }, + { kind: "rm", target: "local", value: HEAD_SMOKE_ID }, + ]); + }); + + it.each([ + [DUAL_STATION_VLLM_LAUNCH_CONTRACT_LABEL, "f".repeat(64)], + [DUAL_STATION_VLLM_API_KEY_FINGERPRINT_LABEL, "d".repeat(64)], + ])("recreates an owned pair whose %s no longer matches", async (label, value) => { + const fake = harness(); + const head = fakeContainer("head"); + const worker = fakeContainer("worker"); + head.labels[label] = value; + worker.labels[label] = value; + fake.seed("local", head); + fake.seed("peer", worker); + + expect(await startDualStationManagedVllm(fixturePlan(), START_CONFIG, fake.deps)).toMatchObject( + { + ok: true, + reusedExisting: false, + }, + ); + expect(fake.operations.filter((operation) => operation.kind === "rm")).toEqual( + expect.arrayContaining([ + { kind: "rm", target: "local", value: HEAD_ID }, + { kind: "rm", target: "peer", value: WORKER_ID }, + ]), + ); + }); + + it("recreates a mixed pair from different lifecycle transactions", async () => { + const fake = harness(); + const worker = fakeContainer("worker"); + worker.labels[DUAL_STATION_VLLM_TRANSACTION_LABEL] = "f".repeat(32); + fake.seed("local", fakeContainer("head")); + fake.seed("peer", worker); + + expect(await startDualStationManagedVllm(fixturePlan(), START_CONFIG, fake.deps)).toMatchObject( + { + ok: true, + reusedExisting: false, + }, + ); + expect(fake.operations.filter((operation) => operation.kind === "rm")).toEqual( + expect.arrayContaining([ + { kind: "rm", target: "local", value: HEAD_ID }, + { kind: "rm", target: "peer", value: WORKER_ID }, + ]), + ); + }); + + it("serializes concurrent same-plan starts so only one worker is created", async () => { + const fake = harness(); + + const results = await Promise.all([ + startDualStationManagedVllm(fixturePlan(), START_CONFIG, fake.deps), + startDualStationManagedVllm(fixturePlan(), START_CONFIG, fake.deps), + ]); + + expect(results).toEqual([ + expect.objectContaining({ ok: true, reusedExisting: false }), + expect.objectContaining({ ok: true, reusedExisting: true }), + ]); + expect( + fake.operations.filter( + (operation) => + operation.kind === "run" && operation.value === DUAL_STATION_VLLM_WORKER_CONTAINER_NAME, + ), + ).toHaveLength(1); + expect(fake.getMaxLifecycleLockActive()).toBe(1); + }); + + it("holds the lifecycle lease after start until validation commits", async () => { + const fake = harness(); + let releaseValidation: () => void = () => undefined; + let reportStarted: () => void = () => undefined; + const validationGate = new Promise((resolve) => { + releaseValidation = resolve; + }); + const started = new Promise((resolve) => { + reportStarted = resolve; + }); + const first = withDualStationManagedVllmLifecycle(async () => { + const result = await startDualStationManagedVllm(fixturePlan(), START_CONFIG, fake.deps); + reportStarted(); + await validationGate; + return result; + }, fake.deps); + await started; + + const second = startDualStationManagedVllm(fixturePlan(), START_CONFIG, fake.deps); + await Promise.resolve(); + expect( + fake.operations.filter( + (operation) => + operation.kind === "run" && operation.value === DUAL_STATION_VLLM_WORKER_CONTAINER_NAME, + ), + ).toHaveLength(1); + + releaseValidation(); + expect(await Promise.all([first, second])).toEqual([ + expect.objectContaining({ ok: true, reusedExisting: false }), + expect.objectContaining({ ok: true, reusedExisting: true }), + ]); + expect(fake.getMaxLifecycleLockActive()).toBe(1); + }); + + it("reconciles a late worker create and rolls back only its transaction", async () => { + const fake = harness({ lateCreateRole: "worker" }); + + expect(await startDualStationManagedVllm(fixturePlan(), START_CONFIG, fake.deps)).toEqual({ + ok: false, + reason: "worker container failed to start", + rollbackErrors: [], + }); + expect(fake.operations.filter((operation) => operation.kind === "rm")).toContainEqual({ + kind: "rm", + target: "peer", + value: WORKER_ID, + }); + }); + + it("never rolls back an ambiguous worker labeled by another transaction", async () => { + const fake = harness({ failedRoleForeignTransaction: "worker" }); + + expect(await startDualStationManagedVllm(fixturePlan(), START_CONFIG, fake.deps)).toEqual({ + ok: false, + reason: "worker container failed to start", + rollbackErrors: [], + }); + expect(fake.operations.filter((operation) => operation.kind === "rm")).not.toContainEqual({ + kind: "rm", + target: "peer", + value: WORKER_ID, + }); + expect(fake.containers.get(`peer:${DUAL_STATION_VLLM_WORKER_CONTAINER_NAME}`)).toHaveLength(1); + }); + + it("migrates only the exact legacy single-Station head when the peer name is absent", async () => { + const fake = harness(); + fake.seed( + "local", + fakeContainer("head", { + labels: { [DUAL_STATION_VLLM_MANAGED_LABEL]: "true" }, + }), + ); + + expect(await startDualStationManagedVllm(fixturePlan(), START_CONFIG, fake.deps)).toMatchObject( + { + ok: true, + reusedExisting: false, + }, + ); + expect(fake.operations.filter((operation) => operation.kind === "rm")).toContainEqual({ + kind: "rm", + target: "local", + value: HEAD_ID, + }); + }); + + it("refuses a same-image worker that belongs to another physical cluster plan", () => { + const fake = harness(); + const unrelatedWorker = fakeContainer("worker"); + unrelatedWorker.labels[DUAL_STATION_VLLM_CLUSTER_LABEL] = "f".repeat(64); + fake.seed("peer", unrelatedWorker); + + expect(preflightDualStationManagedVllm(fixturePlan(), fake.deps)).toMatchObject({ + ok: false, + reason: expect.stringContaining("foreign"), + }); + expect(fake.operations.some((operation) => operation.kind === "rm")).toBe(false); + }); + + it("fails before mutation when an exact name has foreign ownership", async () => { + const fake = harness(); + fake.seed( + "local", + fakeContainer("head", { + labels: { + [DUAL_STATION_VLLM_MANAGED_LABEL]: "false", + [DUAL_STATION_VLLM_ROLE_LABEL]: "head", + [DUAL_STATION_VLLM_ENDPOINT_LABEL]: "http://192.168.240.1:8000", + }, + }), + ); + + expect(await startDualStationManagedVllm(fixturePlan(), START_CONFIG, fake.deps)).toMatchObject( + { + ok: false, + reason: expect.stringContaining("foreign"), + }, + ); + expect(fake.operations.some((operation) => operation.kind === "run")).toBe(false); + expect(fake.operations.some((operation) => operation.kind === "rm")).toBe(false); + }); + + it("rolls back the exact worker ID when the head fails", async () => { + const fake = harness({ failRole: "head" }); + + expect(await startDualStationManagedVllm(fixturePlan(), START_CONFIG, fake.deps)).toEqual({ + ok: false, + reason: "head container failed to start", + rollbackErrors: [], + }); + expect(fake.operations.filter((operation) => operation.kind === "rm")).toEqual([ + { kind: "rm", target: "peer", value: WORKER_SMOKE_ID }, + { kind: "rm", target: "local", value: HEAD_SMOKE_ID }, + { kind: "rm", target: "peer", value: WORKER_ID }, + ]); + }); + + it("rejects an invalid docker-run ID and recovers only its exact owned ID", async () => { + const fake = harness({ invalidIdRole: "worker" }); + + expect(await startDualStationManagedVllm(fixturePlan(), START_CONFIG, fake.deps)).toEqual({ + ok: false, + reason: "worker container failed to start", + rollbackErrors: [], + }); + expect(fake.operations.filter((operation) => operation.kind === "rm")).toEqual([ + { kind: "rm", target: "peer", value: WORKER_SMOKE_ID }, + { kind: "rm", target: "local", value: HEAD_SMOKE_ID }, + { kind: "rm", target: "peer", value: WORKER_ID }, + ]); + }); + + it("cleans up exact owned IDs head-first and reports both-running state", async () => { + const fake = harness(); + fake.seed("local", fakeContainer("head")); + fake.seed("peer", fakeContainer("worker")); + + expect(areDualStationManagedVllmContainersRunning(fixturePlan(), fake.deps)).toBe(true); + expect(await cleanupDualStationManagedVllm(fixturePlan(), fake.deps)).toEqual({ + ok: true, + removedContainerIds: [HEAD_ID, WORKER_ID], + }); + expect(fake.operations.filter((operation) => operation.kind === "rm")).toEqual([ + { kind: "rm", target: "local", value: HEAD_ID }, + { kind: "rm", target: "peer", value: WORKER_ID }, + ]); + expect(areDualStationManagedVllmContainersRunning(fixturePlan(), fake.deps)).toBe(false); + }); + + it("refuses all cleanup when either exact name is ambiguous", async () => { + const fake = harness(); + fake.seed("local", fakeContainer("head")); + fake.seed("local", fakeContainer("head", { id: "c".repeat(64) })); + fake.seed("peer", fakeContainer("worker")); + + expect(await cleanupDualStationManagedVllm(fixturePlan(), fake.deps)).toMatchObject({ + ok: false, + reason: expect.stringContaining("ambiguous"), + }); + expect(fake.operations.some((operation) => operation.kind === "rm")).toBe(false); + }); +}); + +describe("managed dual-Station base URL recovery", () => { + it("returns only a running, owned RFC1918 head endpoint with a bounded inspect", () => { + const fake = harness(); + fake.seed("local", fakeContainer("head")); + + expect(getDualStationManagedVllmBaseUrl(fake.deps)).toBe("http://192.168.240.1:8000"); + expect( + fake.operations.some( + (operation) => + operation.kind === "capture" && + operation.target === "local" && + operation.value === DUAL_STATION_VLLM_HEAD_CONTAINER_NAME, + ), + ).toBe(true); + expect(fake.captureOptions.at(-1)).toMatchObject({ timeout: 10_000 }); + expect(fake.captureOptions.at(-1)?.env?.VLLM_API_KEY).toBeUndefined(); + }); + + it.each([ + ["missing persisted key", { loadApiKey: () => null }], + ["mismatched persisted key", { loadApiKey: () => "a".repeat(64) }], + ["endpoint absent from local interfaces", { localInterfaceAddresses: () => [] }], + ])("rejects day-2 recovery with %s", (_case, overrides) => { + const fake = harness(); + fake.seed("local", fakeContainer("head")); + + expect(getDualStationManagedVllmBaseUrl({ ...fake.deps, ...overrides })).toBeNull(); + }); + + it.each([ + "http://8.8.8.8:8000", + "http://0.0.0.0:8000", + "http://192.168.240.1:8000/", + ])("rejects unsafe or non-canonical endpoint label %s", (endpoint) => { + const fake = harness(); + fake.seed( + "local", + fakeContainer("head", { + labels: { + [DUAL_STATION_VLLM_MANAGED_LABEL]: "true", + [DUAL_STATION_VLLM_ROLE_LABEL]: "head", + [DUAL_STATION_VLLM_ENDPOINT_LABEL]: endpoint, + }, + }), + ); + + expect(getDualStationManagedVllmBaseUrl(fake.deps)).toBeNull(); + }); + + it("rejects an owned-looking head that does not use the pinned runtime image", () => { + const fake = harness(); + fake.seed( + "local", + fakeContainer("head", { + image: + "nvcr.io/nvidia/vllm:forged@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + }), + ); + + expect(getDualStationManagedVllmBaseUrl(fake.deps)).toBeNull(); + }); +}); diff --git a/src/lib/inference/vllm-station-cluster-lifecycle.ts b/src/lib/inference/vllm-station-cluster-lifecycle.ts new file mode 100644 index 0000000000..9eb4a1b528 --- /dev/null +++ b/src/lib/inference/vllm-station-cluster-lifecycle.ts @@ -0,0 +1,1360 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash, randomBytes } from "node:crypto"; +import net from "node:net"; +import os from "node:os"; +import path from "node:path"; + +import { dockerCapture, dockerForceRm, dockerRunDetached } from "../adapters/docker"; +import { DUAL_STATION_VLLM_API_KEY_PATTERN, loadDualStationVllmApiKey } from "./vllm-api-key"; +import { buildLocalDualStationDockerEnv, buildRemoteVllmDockerEnv } from "./vllm-docker-env"; +import { buildNemotronUltraDistributedServeCommand } from "./vllm-models"; +import { DUAL_STATION_VLLM_RUNTIME, type DualStationVllmPlan } from "./vllm-station-cluster"; +import { withDualStationVllmLifecycleLock } from "./vllm-station-lifecycle-lock"; + +export const DUAL_STATION_VLLM_HEAD_CONTAINER_NAME = "nemoclaw-vllm"; +export const DUAL_STATION_VLLM_WORKER_CONTAINER_NAME = "nemoclaw-vllm-worker"; +export const DUAL_STATION_VLLM_MANAGED_LABEL = "com.nvidia.nemoclaw.managed-vllm"; +export const DUAL_STATION_VLLM_ROLE_LABEL = "com.nvidia.nemoclaw.vllm-role"; +export const DUAL_STATION_VLLM_ENDPOINT_LABEL = "com.nvidia.nemoclaw.vllm-endpoint"; +export const DUAL_STATION_VLLM_CLUSTER_LABEL = "com.nvidia.nemoclaw.vllm-cluster"; +export const DUAL_STATION_VLLM_GPU_LABEL = "com.nvidia.nemoclaw.vllm-gpu"; +export const DUAL_STATION_VLLM_LAUNCH_SCHEMA_LABEL = "com.nvidia.nemoclaw.vllm-launch-schema"; +export const DUAL_STATION_VLLM_LAUNCH_CONTRACT_LABEL = "com.nvidia.nemoclaw.vllm-launch-contract"; +export const DUAL_STATION_VLLM_API_KEY_FINGERPRINT_LABEL = + "com.nvidia.nemoclaw.vllm-api-key-fingerprint"; +export const DUAL_STATION_VLLM_TRANSACTION_LABEL = "com.nvidia.nemoclaw.vllm-transaction"; +export const DUAL_STATION_VLLM_GPU_SMOKE_LABEL = "com.nvidia.nemoclaw.gpu-smoke"; +export const DUAL_STATION_VLLM_MASTER_PORT = 29501; + +const HEAD_API_PORT = 8000; +const HF_CACHE_CONTAINER_DIR = "/root/.cache/huggingface"; +const HF_HUB_CACHE_CONTAINER_DIR = `${HF_CACHE_CONTAINER_DIR}/hub`; +const DOCKER_INSPECT_TIMEOUT_MS = 10_000; +const DOCKER_MUTATION_TIMEOUT_MS = 60_000; +const DOCKER_GPU_SMOKE_TIMEOUT_MS = 30_000; +const DOCKER_LATE_CREATE_RECONCILE_ATTEMPTS = 5; +const DOCKER_LATE_CREATE_RECONCILE_INTERVAL_MS = 250; +const DOCKER_CONTAINER_ID_PATTERN = /^[a-f0-9]{64}$/; +const CLUSTER_ID_PATTERN = /^[a-f0-9]{64}$/; +const SHA256_HEX_PATTERN = /^[a-f0-9]{64}$/; +const IMMUTABLE_IMAGE_PATTERN = /^[^\s@]+(?:\/[^\s@]+)+@sha256:[a-f0-9]{64}$/; +const IMAGE_ID_PATTERN = /^sha256:[a-f0-9]{64}$/; +const SAFE_DEVICE_NAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_.:-]{0,63}$/; +const SAFE_UVERBS_DEVICE_PATTERN = /^\/dev\/infiniband\/uverbs[0-9]+$/; +const SAFE_GPU_UUID_PATTERN = /^GPU-[A-Za-z0-9-]{8,123}$/; +const GPU_SMOKE_NONCE_PATTERN = /^[a-f0-9]{32}$/; +const TRANSACTION_ID_PATTERN = /^[a-f0-9]{32}$/; +const GPU_SMOKE_CONTAINER_PREFIX = "nemoclaw-vllm-gpu-smoke"; +const DUAL_STATION_VLLM_LAUNCH_SCHEMA = "1"; +const API_KEY_FINGERPRINT_DOMAIN = "nemoclaw-dual-station-vllm-api-key\0"; + +export type DualStationVllmRole = "head" | "worker"; + +export interface DualStationDockerOptions { + env?: NodeJS.ProcessEnv; + ignoreError?: boolean; + suppressOutput?: boolean; + timeout?: number; +} + +export interface DualStationDockerResult { + status: number | null; + stdout?: string | Buffer | null; + stderr?: string | Buffer | null; + error?: Error; + signal?: NodeJS.Signals | null; +} + +export interface DualStationVllmLifecycleDeps { + dockerCapture(args: readonly string[], options?: DualStationDockerOptions): string; + dockerForceRm(containerId: string, options?: DualStationDockerOptions): DualStationDockerResult; + dockerRunDetached( + args: readonly string[], + options?: DualStationDockerOptions, + ): DualStationDockerResult; + buildLocalDockerEnv(): Record; + buildRemoteDockerEnv(sshUri: string): Record; + createProbeNonce(): string; + createTransactionId(): string; + waitBeforeReconcile(ms: number): Promise; + withLifecycleLock(operation: () => Promise | T): Promise; + loadApiKey(): string | null; + localInterfaceAddresses(): readonly string[]; +} + +export interface DualStationVllmStartConfig { + /** A caller-owned 256-bit API key. It is never persisted by the lifecycle. */ + apiKey: string; +} + +export type StartDualStationVllmResult = + | { + ok: true; + baseUrl: string; + headContainerId: string; + workerContainerId: string; + /** True when an already-running exact owned pair was left untouched. */ + reusedExisting: boolean; + } + | { ok: false; reason: string; rollbackErrors: string[] }; + +export type CleanupDualStationVllmResult = + | { ok: true; removedContainerIds: string[] } + | { ok: false; reason: string }; + +export type PreflightDualStationVllmResult = { ok: true } | { ok: false; reason: string }; + +type ManagedContainerSpec = { + role: DualStationVllmRole; + name: string; + endpoint: string; + clusterId: string; + gpuUuid: string; + image: string; + launchContract: string; + apiKeyFingerprint: string | null; + env: Record; +}; + +type ManagedContainerInspection = + | { kind: "absent" } + | { + kind: "managed"; + containerId: string; + running: boolean; + transactionId: string; + reusable: boolean; + } + | { kind: "legacy-managed"; containerId: string; running: boolean } + | { kind: "foreign" | "ambiguous" | "unknown" }; + +type GpuSmokeSpec = { + role: DualStationVllmRole; + containerName: string; + nonce: string; + image: string; + expectedGpuUuid: string; + env: Record; +}; + +type GpuSmokeInspection = + | { kind: "absent" } + | { kind: "owned"; containerId: string } + | { kind: "foreign" | "ambiguous" | "unknown" }; + +const DEFAULT_DEPS: DualStationVllmLifecycleDeps = { + dockerCapture, + dockerForceRm, + dockerRunDetached, + buildLocalDockerEnv: buildLocalDualStationDockerEnv, + buildRemoteDockerEnv: buildRemoteVllmDockerEnv, + createProbeNonce: () => randomBytes(16).toString("hex"), + createTransactionId: () => randomBytes(16).toString("hex"), + waitBeforeReconcile: (ms) => new Promise((resolve) => setTimeout(resolve, ms)), + withLifecycleLock: withDualStationVllmLifecycleLock, + loadApiKey: loadDualStationVllmApiKey, + localInterfaceAddresses: () => + Object.values(os.networkInterfaces()).flatMap((addresses) => + (addresses ?? []).flatMap((address) => + address.family === "IPv4" && !address.internal ? [address.address] : [], + ), + ), +}; + +function depsWith( + overrides: Partial = {}, +): DualStationVllmLifecycleDeps { + return { ...DEFAULT_DEPS, ...overrides }; +} + +function isRfc1918Ipv4(address: string): boolean { + if (net.isIP(address) !== 4) return false; + const octets = address.split(".").map(Number); + return ( + octets[0] === 10 || + (octets[0] === 172 && octets[1] >= 16 && octets[1] <= 31) || + (octets[0] === 192 && octets[1] === 168) + ); +} + +function assertSafePlan(plan: DualStationVllmPlan): void { + if ( + !IMMUTABLE_IMAGE_PATTERN.test(plan.runtime.image) || + plan.runtime.image !== DUAL_STATION_VLLM_RUNTIME.image || + plan.runtime.modelId !== DUAL_STATION_VLLM_RUNTIME.modelId || + plan.runtime.modelRevision !== DUAL_STATION_VLLM_RUNTIME.modelRevision || + plan.runtime.servedModelId !== DUAL_STATION_VLLM_RUNTIME.servedModelId || + plan.runtime.tensorParallelSize !== DUAL_STATION_VLLM_RUNTIME.tensorParallelSize || + plan.runtime.nodeCount !== DUAL_STATION_VLLM_RUNTIME.nodeCount + ) { + throw new Error("Dual-Station vLLM requires the exact pinned runtime contract."); + } + if (plan.rails.length !== 2 || plan.masterAddress !== plan.rails[0]?.local.address) { + throw new Error("Dual-Station vLLM requires exactly two ordered rails and a rail-0 master."); + } + if (!Number.isInteger(plan.roceGidIndex) || plan.roceGidIndex < 0 || plan.roceGidIndex > 4095) { + throw new Error("Dual-Station vLLM requires a valid shared RoCE GID index."); + } + for (const [role, node] of [ + ["local", plan.local], + ["peer", plan.peer], + ] as const) { + if ( + !path.posix.isAbsolute(node.home) || + path.posix.normalize(node.home) !== node.home || + node.home.includes(":") + ) { + throw new Error(`Dual-Station ${role} home must be a normalized absolute POSIX path.`); + } + if (!SAFE_GPU_UUID_PATTERN.test(node.gpu.uuid)) { + throw new Error(`Dual-Station ${role} GB300 UUID is invalid.`); + } + } + for (const side of ["local", "peer"] as const) { + const endpoints = plan.rails.map((rail) => rail[side]); + if ( + new Set(endpoints.map((endpoint) => endpoint.rdmaDevice)).size !== 2 || + new Set(endpoints.map((endpoint) => endpoint.netdev)).size !== 2 || + new Set(endpoints.map((endpoint) => endpoint.uverbsDevice)).size !== 2 + ) { + throw new Error(`Dual-Station ${side} rails must use two distinct devices.`); + } + for (const endpoint of endpoints) { + if ( + !SAFE_DEVICE_NAME_PATTERN.test(endpoint.rdmaDevice) || + !SAFE_DEVICE_NAME_PATTERN.test(endpoint.netdev) || + !SAFE_UVERBS_DEVICE_PATTERN.test(endpoint.uverbsDevice) || + !isRfc1918Ipv4(endpoint.address) + ) { + throw new Error(`Dual-Station ${side} rail endpoint is invalid.`); + } + } + } +} + +function clusterIdForPlan(plan: DualStationVllmPlan): string { + const identity = { + runtime: plan.runtime, + local: { + hostname: plan.local.hostname, + gpuUuid: plan.local.gpu.uuid, + }, + peer: { + hostname: plan.peer.hostname, + gpuUuid: plan.peer.gpu.uuid, + }, + rails: plan.rails.map((rail) => ({ + index: rail.index, + subnet: rail.subnet, + local: rail.local, + peer: rail.peer, + })), + masterAddress: plan.masterAddress, + roceGidIndex: plan.roceGidIndex, + }; + return createHash("sha256").update(JSON.stringify(identity)).digest("hex"); +} + +/** Stable identity binding both managed containers to one exact physical plan. */ +export function dualStationVllmClusterId(plan: DualStationVllmPlan): string { + assertSafePlan(plan); + return clusterIdForPlan(plan); +} + +function assertSafeStartConfig(config: DualStationVllmStartConfig): void { + if (!DUAL_STATION_VLLM_API_KEY_PATTERN.test(config.apiKey)) { + throw new Error( + "Dual-Station vLLM API key must be exactly 64 lowercase hexadecimal characters.", + ); + } +} + +/** Domain-separated, non-secret binding for the host-persisted service key. */ +export function dualStationVllmApiKeyFingerprint(apiKey: string): string { + if (!DUAL_STATION_VLLM_API_KEY_PATTERN.test(apiKey)) { + throw new Error( + "Dual-Station vLLM API key must be exactly 64 lowercase hexadecimal characters.", + ); + } + return createHash("sha256").update(API_KEY_FINGERPRINT_DOMAIN).update(apiKey).digest("hex"); +} + +function withoutVllmApiKey(env: Record): Record { + const sanitized = { ...env }; + delete sanitized.VLLM_API_KEY; + return sanitized; +} + +function endpointFor(plan: DualStationVllmPlan, role: DualStationVllmRole): string { + return role === "head" ? `http://${plan.masterAddress}:${String(HEAD_API_PORT)}` : "headless"; +} + +function nameFor(role: DualStationVllmRole): string { + return role === "head" + ? DUAL_STATION_VLLM_HEAD_CONTAINER_NAME + : DUAL_STATION_VLLM_WORKER_CONTAINER_NAME; +} + +function appendEnv(args: string[], name: string, value: string): void { + args.push("--env", `${name}=${value}`); +} + +/** Build the deterministic shell-free launch argv before per-operation labels. */ +function buildDualStationVllmBaseRunArgs( + plan: DualStationVllmPlan, + role: DualStationVllmRole, +): string[] { + assertSafePlan(plan); + const node = role === "head" ? plan.local : plan.peer; + const endpoints = plan.rails.map((rail) => (role === "head" ? rail.local : rail.peer)); + const endpoint = endpointFor(plan, role); + const clusterId = clusterIdForPlan(plan); + const args = [ + "--pull=never", + "--restart", + "unless-stopped", + "--network", + "host", + "--shm-size", + "16g", + "--read-only", + "--tmpfs", + "/tmp:rw,nosuid,nodev,size=17179869184", + "--tmpfs", + "/root/.cache:rw,nosuid,nodev,size=68719476736", + "--cap-drop", + "ALL", + "--cap-add", + "IPC_LOCK", + "--cap-add", + "DAC_READ_SEARCH", + "--ulimit", + "memlock=-1", + "--ulimit", + "stack=67108864", + "--ulimit", + "nofile=1048576:1048576", + "--gpus", + `device=${node.gpu.uuid}`, + ...endpoints.map((endpoint) => `--device=${endpoint.uverbsDevice}`), + "--volume", + `${node.home}/.cache/huggingface/hub:${HF_HUB_CACHE_CONTAINER_DIR}:ro`, + "--label", + `${DUAL_STATION_VLLM_MANAGED_LABEL}=true`, + "--label", + `${DUAL_STATION_VLLM_ROLE_LABEL}=${role}`, + "--label", + `${DUAL_STATION_VLLM_ENDPOINT_LABEL}=${endpoint}`, + "--label", + `${DUAL_STATION_VLLM_CLUSTER_LABEL}=${clusterId}`, + "--label", + `${DUAL_STATION_VLLM_GPU_LABEL}=${node.gpu.uuid}`, + "--name", + nameFor(role), + ]; + + appendEnv(args, "HF_HOME", HF_CACHE_CONTAINER_DIR); + appendEnv(args, "HF_HUB_OFFLINE", "1"); + appendEnv(args, "TRANSFORMERS_OFFLINE", "1"); + appendEnv(args, "VLLM_HOST_IP", endpoints[0].address); + appendEnv(args, "NCCL_IB_HCA", endpoints.map((item) => item.rdmaDevice).join(",")); + appendEnv(args, "NCCL_IB_DISABLE", "0"); + appendEnv(args, "NCCL_IB_GID_INDEX", String(plan.roceGidIndex)); + appendEnv(args, "NCCL_IB_TC", "106"); + appendEnv(args, "NCCL_IB_QPS_PER_CONNECTION", "4"); + appendEnv(args, "NCCL_NET_GDR_LEVEL", "PHB"); + appendEnv(args, "NCCL_IB_PCI_RELAXED_ORDERING", "1"); + appendEnv(args, "NCCL_SOCKET_IFNAME", endpoints[0].netdev); + appendEnv(args, "GLOO_SOCKET_IFNAME", endpoints[0].netdev); + appendEnv(args, "TP_SOCKET_IFNAME", endpoints[0].netdev); + appendEnv(args, "OMPI_MCA_btl_tcp_if_include", endpoints[0].netdev); + appendEnv(args, "MN_IF_NAME", endpoints[0].netdev); + appendEnv(args, "NCCL_IGNORE_CPU_AFFINITY", "1"); + appendEnv(args, "UCX_NET_DEVICES", endpoints.map((item) => `${item.rdmaDevice}:1`).join(",")); + appendEnv(args, "UCX_TLS", "rc_x,cuda_copy,cuda_ipc,gdr_copy"); + appendEnv(args, "UCX_IB_GID_INDEX", String(plan.roceGidIndex)); + appendEnv(args, "UCX_RNDV_THRESH", "8192"); + + if (role === "head") { + // Docker resolves this bare name from the head docker-run subprocess. The + // secret value must never enter argv, labels, or the worker environment. + args.push("--env", "VLLM_API_KEY"); + } + + args.push( + "--entrypoint", + "/bin/bash", + plan.runtime.image, + "-lc", + buildNemotronUltraDistributedServeCommand({ + nodeRank: role === "head" ? 0 : 1, + masterAddr: plan.masterAddress, + masterPort: DUAL_STATION_VLLM_MASTER_PORT, + }), + ); + return args; +} + +/** Stable digest of the exact role-local launch argv and its schema. */ +export function dualStationVllmLaunchContract( + plan: DualStationVllmPlan, + role: DualStationVllmRole, +): string { + const contract = { + schema: DUAL_STATION_VLLM_LAUNCH_SCHEMA, + role, + args: buildDualStationVllmBaseRunArgs(plan, role), + }; + return createHash("sha256").update(JSON.stringify(contract)).digest("hex"); +} + +/** Build the complete launch argv with non-secret transaction/config bindings. */ +export function buildDualStationVllmRunArgs( + plan: DualStationVllmPlan, + role: DualStationVllmRole, + transactionId: string, + apiKeyFingerprint: string, +): string[] { + if (!TRANSACTION_ID_PATTERN.test(transactionId)) { + throw new Error("Dual-Station vLLM transaction ID is invalid."); + } + if (!SHA256_HEX_PATTERN.test(apiKeyFingerprint)) { + throw new Error("Dual-Station vLLM API key fingerprint is invalid."); + } + const args = buildDualStationVllmBaseRunArgs(plan, role); + const nameIndex = args.indexOf("--name"); + args.splice( + nameIndex, + 0, + "--label", + `${DUAL_STATION_VLLM_LAUNCH_SCHEMA_LABEL}=${DUAL_STATION_VLLM_LAUNCH_SCHEMA}`, + "--label", + `${DUAL_STATION_VLLM_LAUNCH_CONTRACT_LABEL}=${dualStationVllmLaunchContract(plan, role)}`, + "--label", + `${DUAL_STATION_VLLM_API_KEY_FINGERPRINT_LABEL}=${apiKeyFingerprint}`, + "--label", + `${DUAL_STATION_VLLM_TRANSACTION_LABEL}=${transactionId}`, + ); + return args; +} + +/** Build a no-network, no-pull GPU runtime probe that only executes nvidia-smi. */ +export function buildDualStationGpuSmokeRunArgs( + plan: DualStationVllmPlan, + role: DualStationVllmRole, + nonce: string, +): { containerName: string; args: string[] } { + assertSafePlan(plan); + if (!GPU_SMOKE_NONCE_PATTERN.test(nonce)) { + throw new Error("Dual-Station GPU smoke nonce is invalid."); + } + const node = role === "head" ? plan.local : plan.peer; + const containerName = `${GPU_SMOKE_CONTAINER_PREFIX}-${role}-${nonce}`; + return { + containerName, + args: [ + "--pull=never", + "--network", + "none", + "--cap-drop", + "ALL", + "--gpus", + `device=${node.gpu.uuid}`, + "--label", + `${DUAL_STATION_VLLM_GPU_SMOKE_LABEL}=${nonce}`, + "--label", + `${DUAL_STATION_VLLM_ROLE_LABEL}=${role}`, + "--name", + containerName, + "--entrypoint", + "nvidia-smi", + plan.runtime.image, + "--query-gpu=uuid", + "--format=csv,noheader", + ], + }; +} + +function specsForPlan( + plan: DualStationVllmPlan, + deps: DualStationVllmLifecycleDeps, + config?: DualStationVllmStartConfig, +): { head: ManagedContainerSpec; worker: ManagedContainerSpec } { + assertSafePlan(plan); + const clusterId = clusterIdForPlan(plan); + const apiKeyFingerprint = config ? dualStationVllmApiKeyFingerprint(config.apiKey) : null; + return { + head: { + role: "head", + name: DUAL_STATION_VLLM_HEAD_CONTAINER_NAME, + endpoint: endpointFor(plan, "head"), + clusterId, + gpuUuid: plan.local.gpu.uuid, + image: plan.runtime.image, + launchContract: dualStationVllmLaunchContract(plan, "head"), + apiKeyFingerprint, + env: withoutVllmApiKey(deps.buildLocalDockerEnv()), + }, + worker: { + role: "worker", + name: DUAL_STATION_VLLM_WORKER_CONTAINER_NAME, + endpoint: endpointFor(plan, "worker"), + clusterId, + gpuUuid: plan.peer.gpu.uuid, + image: plan.runtime.image, + launchContract: dualStationVllmLaunchContract(plan, "worker"), + apiKeyFingerprint, + env: withoutVllmApiKey(deps.buildRemoteDockerEnv(plan.peerDockerHost)), + }, + }; +} + +const INSPECTION_FORMAT = [ + "{{.ID}}", + "{{.Names}}", + "{{.State}}", + "{{.Image}}", + `{{.Label \"${DUAL_STATION_VLLM_MANAGED_LABEL}\"}}`, + `{{.Label \"${DUAL_STATION_VLLM_ROLE_LABEL}\"}}`, + `{{.Label \"${DUAL_STATION_VLLM_ENDPOINT_LABEL}\"}}`, + `{{.Label \"${DUAL_STATION_VLLM_CLUSTER_LABEL}\"}}`, + `{{.Label \"${DUAL_STATION_VLLM_GPU_LABEL}\"}}`, + `{{.Label \"${DUAL_STATION_VLLM_LAUNCH_SCHEMA_LABEL}\"}}`, + `{{.Label \"${DUAL_STATION_VLLM_LAUNCH_CONTRACT_LABEL}\"}}`, + `{{.Label \"${DUAL_STATION_VLLM_API_KEY_FINGERPRINT_LABEL}\"}}`, + `{{.Label \"${DUAL_STATION_VLLM_TRANSACTION_LABEL}\"}}`, +].join("\t"); + +function inspectRows( + containerName: string, + env: Record, + deps: DualStationVllmLifecycleDeps, +): string[][] | null { + try { + const output = deps.dockerCapture( + [ + "container", + "ls", + "--all", + "--no-trunc", + "--filter", + `name=^/${containerName}$`, + "--format", + INSPECTION_FORMAT, + ], + { env, timeout: DOCKER_INSPECT_TIMEOUT_MS }, + ); + if (!output.trim()) return []; + return output + .split(/\r?\n/) + .filter(Boolean) + .map((line) => line.split("\t")); + } catch { + return null; + } +} + +function inspectManagedContainer( + spec: ManagedContainerSpec, + deps: DualStationVllmLifecycleDeps, +): ManagedContainerInspection { + const rows = inspectRows(spec.name, spec.env, deps); + if (rows === null) return { kind: "unknown" }; + if (rows.length === 0) return { kind: "absent" }; + if (rows.length !== 1) return { kind: "ambiguous" }; + const [ + containerId, + name, + state, + image, + managed, + role, + endpoint, + clusterId, + gpuUuid, + launchSchema, + launchContract, + apiKeyFingerprint, + transactionId, + ] = rows[0]; + if (!containerId || rows[0].length !== 13 || !DOCKER_CONTAINER_ID_PATTERN.test(containerId)) { + return { kind: "unknown" }; + } + if ( + spec.role === "head" && + name === spec.name && + image === spec.image && + managed === "true" && + !role && + !endpoint && + !clusterId && + !gpuUuid && + !launchSchema && + !launchContract && + !apiKeyFingerprint && + !transactionId + ) { + return { kind: "legacy-managed", containerId, running: state === "running" }; + } + if ( + name !== spec.name || + image !== spec.image || + managed !== "true" || + role !== spec.role || + endpoint !== spec.endpoint || + clusterId !== spec.clusterId || + gpuUuid !== spec.gpuUuid || + launchSchema !== DUAL_STATION_VLLM_LAUNCH_SCHEMA || + !SHA256_HEX_PATTERN.test(launchContract) || + !SHA256_HEX_PATTERN.test(apiKeyFingerprint) || + !TRANSACTION_ID_PATTERN.test(transactionId) + ) { + return { kind: "foreign" }; + } + const reusable = + launchContract === spec.launchContract && + (spec.apiKeyFingerprint === null || apiKeyFingerprint === spec.apiKeyFingerprint); + return { kind: "managed", containerId, running: state === "running", transactionId, reusable }; +} + +const GPU_SMOKE_INSPECTION_FORMAT = [ + "{{.ID}}", + "{{.Names}}", + "{{.Image}}", + `{{.Label \"${DUAL_STATION_VLLM_GPU_SMOKE_LABEL}\"}}`, + `{{.Label \"${DUAL_STATION_VLLM_ROLE_LABEL}\"}}`, +].join("\t"); + +function inspectGpuSmokeContainer( + spec: GpuSmokeSpec, + deps: DualStationVllmLifecycleDeps, +): GpuSmokeInspection { + let output: string; + try { + output = deps.dockerCapture( + [ + "container", + "ls", + "--all", + "--no-trunc", + "--filter", + `name=^/${spec.containerName}$`, + "--format", + GPU_SMOKE_INSPECTION_FORMAT, + ], + { env: spec.env, timeout: DOCKER_INSPECT_TIMEOUT_MS }, + ); + } catch { + return { kind: "unknown" }; + } + if (!output.trim()) return { kind: "absent" }; + const rows = output + .trim() + .split(/\r?\n/) + .filter(Boolean) + .map((line) => line.split("\t")); + if (rows.length !== 1) return { kind: "ambiguous" }; + const [containerId, name, image, nonce, role] = rows[0]; + if (!containerId || rows[0].length !== 5 || !DOCKER_CONTAINER_ID_PATTERN.test(containerId)) { + return { kind: "unknown" }; + } + if ( + name !== spec.containerName || + image !== spec.image || + nonce !== spec.nonce || + role !== spec.role + ) { + return { kind: "foreign" }; + } + return { kind: "owned", containerId }; +} + +function pinnedImageIsPresent( + spec: ManagedContainerSpec, + deps: DualStationVllmLifecycleDeps, +): boolean { + try { + const output = deps.dockerCapture(["image", "inspect", "--format", "{{.Id}}", spec.image], { + env: spec.env, + timeout: DOCKER_INSPECT_TIMEOUT_MS, + }); + const ids = output.trim().split(/\r?\n/).filter(Boolean); + return ids.length === 1 && IMAGE_ID_PATTERN.test(ids[0]); + } catch { + return false; + } +} + +function mutationSucceeded(result: DualStationDockerResult): boolean { + return result.status === 0 && !result.error && !result.signal; +} + +function resultContainerId(result: DualStationDockerResult): string | null { + const value = String(result.stdout ?? "").trim(); + return DOCKER_CONTAINER_ID_PATTERN.test(value) ? value : null; +} + +function removeGpuSmokeExact( + spec: GpuSmokeSpec, + containerId: string, + deps: DualStationVllmLifecycleDeps, +): boolean { + try { + return mutationSucceeded( + deps.dockerForceRm(containerId, { + env: spec.env, + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_MUTATION_TIMEOUT_MS, + }), + ); + } catch { + return false; + } +} + +function runGpuSmoke( + plan: DualStationVllmPlan, + managedSpec: ManagedContainerSpec, + expectedGpuUuid: string, + deps: DualStationVllmLifecycleDeps, +): PreflightDualStationVllmResult { + let nonce: string; + try { + nonce = deps.createProbeNonce(); + } catch { + return { ok: false, reason: `${managedSpec.role} GPU smoke nonce generation failed` }; + } + let built: ReturnType; + try { + built = buildDualStationGpuSmokeRunArgs(plan, managedSpec.role, nonce); + } catch (error) { + return { ok: false, reason: (error as Error).message }; + } + const spec: GpuSmokeSpec = { + role: managedSpec.role, + containerName: built.containerName, + nonce, + image: managedSpec.image, + expectedGpuUuid, + env: managedSpec.env, + }; + const before = inspectGpuSmokeContainer(spec, deps); + if (before.kind !== "absent") { + return { + ok: false, + reason: `${spec.role} GPU smoke name ownership is ${before.kind}; refusing mutation`, + }; + } + + let failureReason: string | null = null; + let runResult: DualStationDockerResult | null = null; + try { + runResult = deps.dockerRunDetached(built.args, { + env: spec.env, + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_GPU_SMOKE_TIMEOUT_MS, + }); + } catch { + failureReason = `${spec.role} GPU smoke container failed to launch`; + } + + const capturedId = runResult ? resultContainerId(runResult) : null; + let cleanupId = capturedId; + const observed = inspectGpuSmokeContainer(spec, deps); + if (!cleanupId && observed.kind === "owned") cleanupId = observed.containerId; + + if (!failureReason && (!runResult || !mutationSucceeded(runResult))) { + failureReason = `${spec.role} GPU smoke container failed to launch`; + } + if (!failureReason && !capturedId) { + failureReason = `${spec.role} GPU smoke returned an invalid container ID`; + } + if (!failureReason && observed.kind !== "owned") { + failureReason = `${spec.role} GPU smoke ownership is ${observed.kind}`; + } + if ( + !failureReason && + observed.kind === "owned" && + capturedId && + observed.containerId !== capturedId + ) { + failureReason = `${spec.role} GPU smoke container ID did not match exact-name inspection`; + } + + if (!failureReason && capturedId) { + try { + const exitCode = deps.dockerCapture(["wait", capturedId], { + env: spec.env, + timeout: DOCKER_GPU_SMOKE_TIMEOUT_MS, + }); + if (exitCode.trim() !== "0") { + failureReason = `${spec.role} GPU smoke exited unsuccessfully`; + } + } catch { + failureReason = `${spec.role} GPU smoke did not finish within the bounded wait`; + } + } + + if (!failureReason && capturedId) { + try { + const visibleGpuUuids = deps + .dockerCapture(["logs", capturedId], { + env: spec.env, + timeout: DOCKER_INSPECT_TIMEOUT_MS, + }) + .trim() + .split(/\r?\n/) + .map((line) => line.trim()) + .filter(Boolean); + if (visibleGpuUuids.length !== 1 || visibleGpuUuids[0] !== spec.expectedGpuUuid) { + failureReason = `${spec.role} GPU smoke did not expose exactly the discovered GPU`; + } + } catch { + failureReason = `${spec.role} GPU smoke output could not be validated`; + } + } + + if (cleanupId && !removeGpuSmokeExact(spec, cleanupId, deps)) { + const cleanupReason = `failed to remove exact ${spec.role} GPU smoke container ${cleanupId}`; + failureReason = failureReason ? `${failureReason}; ${cleanupReason}` : cleanupReason; + } + if (!cleanupId && observed.kind !== "absent") { + const cleanupReason = `${spec.role} GPU smoke could not identify an exact owned ID for cleanup`; + failureReason = failureReason ? `${failureReason}; ${cleanupReason}` : cleanupReason; + } + return failureReason ? { ok: false, reason: failureReason } : { ok: true }; +} + +function removeExact( + spec: ManagedContainerSpec, + containerId: string, + deps: DualStationVllmLifecycleDeps, +): boolean { + try { + return mutationSucceeded( + deps.dockerForceRm(containerId, { + env: spec.env, + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_MUTATION_TIMEOUT_MS, + }), + ); + } catch { + return false; + } +} + +function removeTransactionExact( + spec: ManagedContainerSpec, + containerId: string, + transactionId: string, + deps: DualStationVllmLifecycleDeps, +): boolean { + const inspection = inspectManagedContainer(spec, deps); + if (inspection.kind === "absent") return true; + if ( + inspection.kind !== "managed" || + !inspection.reusable || + inspection.containerId !== containerId || + inspection.transactionId !== transactionId + ) { + return false; + } + return removeExact(spec, containerId, deps); +} + +function rollbackExact( + entries: readonly { spec: ManagedContainerSpec; containerId: string }[], + transactionId: string, + deps: DualStationVllmLifecycleDeps, +): string[] { + const errors: string[] = []; + const seen = new Set(); + for (const { spec, containerId } of entries) { + const key = `${spec.role}:${containerId}`; + if (seen.has(key)) continue; + seen.add(key); + if (!removeTransactionExact(spec, containerId, transactionId, deps)) { + errors.push(`failed to remove ${spec.role} container ${containerId}`); + } + } + return errors; +} + +async function recoverManagedId( + spec: ManagedContainerSpec, + transactionId: string, + deps: DualStationVllmLifecycleDeps, +): Promise { + for (let attempt = 0; attempt < DOCKER_LATE_CREATE_RECONCILE_ATTEMPTS; attempt += 1) { + const inspection = inspectManagedContainer(spec, deps); + if ( + inspection.kind === "managed" && + inspection.reusable && + inspection.transactionId === transactionId + ) { + return inspection; + } + if (inspection.kind !== "absent" && inspection.kind !== "unknown") return null; + if (attempt < DOCKER_LATE_CREATE_RECONCILE_ATTEMPTS - 1) { + await deps.waitBeforeReconcile(DOCKER_LATE_CREATE_RECONCILE_INTERVAL_MS); + } + } + return null; +} + +async function startOne( + plan: DualStationVllmPlan, + spec: ManagedContainerSpec, + config: DualStationVllmStartConfig, + transactionId: string, + deps: DualStationVllmLifecycleDeps, +): Promise<{ ok: true; containerId: string } | { ok: false; containerId: string | null }> { + let result: DualStationDockerResult | null = null; + try { + const env = spec.role === "head" ? { ...spec.env, VLLM_API_KEY: config.apiKey } : spec.env; + result = deps.dockerRunDetached( + buildDualStationVllmRunArgs(plan, spec.role, transactionId, spec.apiKeyFingerprint ?? ""), + { + env, + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_MUTATION_TIMEOUT_MS, + }, + ); + } catch { + // A timed-out Docker client may still have committed the create. Reconcile + // only this unguessable transaction before deciding what may be rolled back. + } + const capturedId = result ? resultContainerId(result) : null; + const observed = await recoverManagedId(spec, transactionId, deps); + if ( + result && + mutationSucceeded(result) && + capturedId && + observed?.kind === "managed" && + observed.containerId === capturedId && + observed.running + ) { + return { ok: true, containerId: capturedId }; + } + return { + ok: false, + containerId: observed?.kind === "managed" ? observed.containerId : null, + }; +} + +function unsafeInspectionReason( + role: DualStationVllmRole, + inspection: ManagedContainerInspection, +): string | null { + if ( + inspection.kind === "managed" || + inspection.kind === "legacy-managed" || + inspection.kind === "absent" + ) { + return null; + } + return `${role} container ownership is ${inspection.kind}; refusing mutation`; +} + +function ownershipTopologyReason( + head: ManagedContainerInspection, + worker: ManagedContainerInspection, +): string | null { + const headReason = unsafeInspectionReason("head", head); + if (headReason) return headReason; + const workerReason = unsafeInspectionReason("worker", worker); + if (workerReason) return workerReason; + if (head.kind === "legacy-managed" && worker.kind !== "absent") { + return "legacy single-Station head can only migrate when the peer worker name is absent"; + } + return null; +} + +function ownershipPreflightForSpecs( + specs: ReturnType, + deps: DualStationVllmLifecycleDeps, +): PreflightDualStationVllmResult { + const head = inspectManagedContainer(specs.head, deps); + const worker = inspectManagedContainer(specs.worker, deps); + const reason = ownershipTopologyReason(head, worker); + return reason ? { ok: false, reason } : { ok: true }; +} + +function gpuRuntimePreflightForSpecs( + plan: DualStationVllmPlan, + specs: ReturnType, + deps: DualStationVllmLifecycleDeps, +): PreflightDualStationVllmResult { + const ownershipBefore = ownershipPreflightForSpecs(specs, deps); + if (!ownershipBefore.ok) return ownershipBefore; + + // Validate both exact digest references before either daemon is mutated. + for (const spec of [specs.worker, specs.head]) { + if (!pinnedImageIsPresent(spec, deps)) { + return { + ok: false, + reason: `${spec.role} pinned vLLM image is not present or could not be inspected`, + }; + } + } + + for (const [spec, gpuUuid] of [ + [specs.worker, plan.peer.gpu.uuid], + [specs.head, plan.local.gpu.uuid], + ] as const) { + const smoke = runGpuSmoke(plan, spec, gpuUuid, deps); + if (!smoke.ok) return smoke; + } + + // Exact names may have changed while the bounded probes ran. + return ownershipPreflightForSpecs(specs, deps); +} + +/** Read-only ownership preflight used before downloads and repeated by start. */ +export function preflightDualStationManagedVllm( + plan: DualStationVllmPlan, + overrides: Partial = {}, +): PreflightDualStationVllmResult { + const deps = depsWith(overrides); + let specs: ReturnType; + try { + specs = specsForPlan(plan, deps); + } catch (error) { + return { ok: false, reason: (error as Error).message }; + } + return ownershipPreflightForSpecs(specs, deps); +} + +/** + * After both pinned images are installed, prove GPU-container execution on + * each daemon without pulling an image, starting vLLM, or receiving an API key. + */ +export async function preflightDualStationGpuRuntime( + plan: DualStationVllmPlan, + overrides: Partial = {}, +): Promise { + const deps = depsWith(overrides); + try { + const specs = specsForPlan(plan, deps); + return await deps.withLifecycleLock(() => gpuRuntimePreflightForSpecs(plan, specs, deps)); + } catch (error) { + return { ok: false, reason: `dual-Station lifecycle lock failed: ${(error as Error).message}` }; + } +} + +/** + * Hold the host-global lease across start, readiness/auth validation, and any + * rollback. The callback's successful return is the lifecycle commit point. + */ +export function withDualStationManagedVllmLifecycle( + operation: () => Promise | T, + overrides: Partial = {}, +): Promise { + return depsWith(overrides).withLifecycleLock(operation); +} + +/** Start rank 1 first, then rank 0, rolling back only exact newly created IDs. */ +export async function startDualStationManagedVllm( + plan: DualStationVllmPlan, + config: DualStationVllmStartConfig, + overrides: Partial = {}, +): Promise { + const deps = depsWith(overrides); + let specs: ReturnType; + try { + assertSafeStartConfig(config); + specs = specsForPlan(plan, deps, config); + } catch (error) { + return { ok: false, reason: (error as Error).message, rollbackErrors: [] }; + } + + try { + return await deps.withLifecycleLock(async () => { + // Deliberately repeat the post-pull smoke here: model download can be long, + // and this closes that TOCTOU window immediately before owned replacement. + const gpuRuntimePreflight = gpuRuntimePreflightForSpecs(plan, specs, deps); + if (!gpuRuntimePreflight.ok) { + return { ...gpuRuntimePreflight, rollbackErrors: [] }; + } + + const existingHead = inspectManagedContainer(specs.head, deps); + const existingWorker = inspectManagedContainer(specs.worker, deps); + const topologyReason = ownershipTopologyReason(existingHead, existingWorker); + if (topologyReason) return { ok: false, reason: topologyReason, rollbackErrors: [] }; + + if ( + existingHead.kind === "managed" && + existingHead.running && + existingHead.reusable && + existingWorker.kind === "managed" && + existingWorker.running && + existingWorker.reusable && + existingHead.transactionId === existingWorker.transactionId + ) { + return { + ok: true, + baseUrl: specs.head.endpoint, + headContainerId: existingHead.containerId, + workerContainerId: existingWorker.containerId, + reusedExisting: true, + }; + } + + let transactionId: string; + try { + transactionId = deps.createTransactionId(); + } catch { + return { + ok: false, + reason: "dual-Station lifecycle transaction generation failed", + rollbackErrors: [], + }; + } + if (!TRANSACTION_ID_PATTERN.test(transactionId)) { + return { + ok: false, + reason: "dual-Station lifecycle transaction ID is invalid", + rollbackErrors: [], + }; + } + + for (const [spec, inspection] of [ + [specs.head, existingHead], + [specs.worker, existingWorker], + ] as const) { + if ( + (inspection.kind === "managed" || inspection.kind === "legacy-managed") && + !removeExact(spec, inspection.containerId, deps) + ) { + return { + ok: false, + reason: `failed to remove existing owned ${spec.role} container`, + rollbackErrors: [], + }; + } + } + + const worker = await startOne(plan, specs.worker, config, transactionId, deps); + if (!worker.ok) { + const rollbackErrors = worker.containerId + ? rollbackExact( + [{ spec: specs.worker, containerId: worker.containerId }], + transactionId, + deps, + ) + : []; + return { ok: false, reason: "worker container failed to start", rollbackErrors }; + } + + const head = await startOne(plan, specs.head, config, transactionId, deps); + if (!head.ok) { + const rollback = [ + ...(head.containerId ? [{ spec: specs.head, containerId: head.containerId }] : []), + { spec: specs.worker, containerId: worker.containerId }, + ]; + return { + ok: false, + reason: "head container failed to start", + rollbackErrors: rollbackExact(rollback, transactionId, deps), + }; + } + + const finalHead = inspectManagedContainer(specs.head, deps); + const finalWorker = inspectManagedContainer(specs.worker, deps); + if ( + finalHead.kind !== "managed" || + !finalHead.running || + !finalHead.reusable || + finalHead.transactionId !== transactionId || + finalHead.containerId !== head.containerId || + finalWorker.kind !== "managed" || + !finalWorker.running || + !finalWorker.reusable || + finalWorker.transactionId !== transactionId || + finalWorker.containerId !== worker.containerId + ) { + return { + ok: false, + reason: "dual-Station containers did not remain running", + rollbackErrors: rollbackExact( + [ + { spec: specs.head, containerId: head.containerId }, + { spec: specs.worker, containerId: worker.containerId }, + ], + transactionId, + deps, + ), + }; + } + + return { + ok: true, + baseUrl: specs.head.endpoint, + headContainerId: head.containerId, + workerContainerId: worker.containerId, + reusedExisting: false, + }; + }); + } catch (error) { + return { + ok: false, + reason: `dual-Station lifecycle lock failed: ${(error as Error).message}`, + rollbackErrors: [], + }; + } +} + +/** Remove only containers whose complete dual-Station ownership tuple matches. */ +function cleanupDualStationManagedVllmUnlocked( + plan: DualStationVllmPlan, + deps: DualStationVllmLifecycleDeps, +): CleanupDualStationVllmResult { + let specs: ReturnType; + try { + specs = specsForPlan(plan, deps); + } catch (error) { + return { ok: false, reason: (error as Error).message }; + } + const head = inspectManagedContainer(specs.head, deps); + const worker = inspectManagedContainer(specs.worker, deps); + if (head.kind === "legacy-managed" || worker.kind === "legacy-managed") { + return { ok: false, reason: "refusing dual-Station cleanup of a legacy single-host container" }; + } + for (const [role, inspection] of [ + ["head", head], + ["worker", worker], + ] as const) { + const reason = unsafeInspectionReason(role, inspection); + if (reason) return { ok: false, reason }; + } + + const removedContainerIds: string[] = []; + for (const [spec, inspection] of [ + [specs.head, head], + [specs.worker, worker], + ] as const) { + if (inspection.kind !== "managed") continue; + if (!removeExact(spec, inspection.containerId, deps)) { + return { ok: false, reason: `failed to remove owned ${spec.role} container` }; + } + removedContainerIds.push(inspection.containerId); + } + return { ok: true, removedContainerIds }; +} + +/** Serialize cleanup with start so fixed names cannot transfer between owners. */ +export async function cleanupDualStationManagedVllm( + plan: DualStationVllmPlan, + overrides: Partial = {}, +): Promise { + const deps = depsWith(overrides); + try { + return await deps.withLifecycleLock(() => cleanupDualStationManagedVllmUnlocked(plan, deps)); + } catch (error) { + return { ok: false, reason: `dual-Station lifecycle lock failed: ${(error as Error).message}` }; + } +} + +/** Read-only exact-ownership/running check across both Docker daemons. */ +export function areDualStationManagedVllmContainersRunning( + plan: DualStationVllmPlan, + overrides: Partial = {}, +): boolean { + const deps = depsWith(overrides); + try { + const specs = specsForPlan(plan, deps); + const head = inspectManagedContainer(specs.head, deps); + const worker = inspectManagedContainer(specs.worker, deps); + return ( + head.kind === "managed" && + head.running && + head.reusable && + worker.kind === "managed" && + worker.running && + worker.reusable && + head.transactionId === worker.transactionId + ); + } catch { + return false; + } +} + +function validatedManagedBaseUrl(value: string): string | null { + let parsed: URL; + try { + parsed = new URL(value); + } catch { + return null; + } + if ( + parsed.protocol !== "http:" || + parsed.username || + parsed.password || + parsed.pathname !== "/" || + parsed.search || + parsed.hash || + parsed.port !== String(HEAD_API_PORT) || + !isRfc1918Ipv4(parsed.hostname) + ) { + return null; + } + const canonical = `http://${parsed.hostname}:${String(HEAD_API_PORT)}`; + return value === canonical ? canonical : null; +} + +/** Recover the local managed head endpoint without trusting persisted user input. */ +export function getDualStationManagedVllmBaseUrl( + overrides: Partial = {}, +): string | null { + const deps = depsWith(overrides); + const env = withoutVllmApiKey(deps.buildLocalDockerEnv()); + const rows = inspectRows(DUAL_STATION_VLLM_HEAD_CONTAINER_NAME, env, deps); + if (!rows || rows.length !== 1 || rows[0].length !== 13) return null; + const [ + containerId, + name, + state, + image, + managed, + role, + endpoint, + clusterId, + gpuUuid, + launchSchema, + launchContract, + apiKeyFingerprint, + transactionId, + ] = rows[0]; + if ( + !DOCKER_CONTAINER_ID_PATTERN.test(containerId) || + name !== DUAL_STATION_VLLM_HEAD_CONTAINER_NAME || + state !== "running" || + image !== DUAL_STATION_VLLM_RUNTIME.image || + managed !== "true" || + role !== "head" || + !CLUSTER_ID_PATTERN.test(clusterId) || + !SAFE_GPU_UUID_PATTERN.test(gpuUuid) || + launchSchema !== DUAL_STATION_VLLM_LAUNCH_SCHEMA || + !SHA256_HEX_PATTERN.test(launchContract) || + !SHA256_HEX_PATTERN.test(apiKeyFingerprint) || + !TRANSACTION_ID_PATTERN.test(transactionId) + ) { + return null; + } + let apiKey: string | null; + try { + apiKey = deps.loadApiKey(); + } catch { + return null; + } + if (!apiKey || dualStationVllmApiKeyFingerprint(apiKey) !== apiKeyFingerprint) return null; + const baseUrl = validatedManagedBaseUrl(endpoint); + if (!baseUrl || !deps.localInterfaceAddresses().includes(new URL(baseUrl).hostname)) return null; + return baseUrl; +} diff --git a/src/lib/inference/vllm-station-cluster.test.ts b/src/lib/inference/vllm-station-cluster.test.ts new file mode 100644 index 0000000000..463809d7c3 --- /dev/null +++ b/src/lib/inference/vllm-station-cluster.test.ts @@ -0,0 +1,842 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { SpawnSyncOptionsWithStringEncoding } from "node:child_process"; + +import { describe, expect, it, vi } from "vitest"; + +import { buildRemoteVllmDockerEnv } from "./vllm-docker-env"; +import { + createStationClusterProbeDeps, + DUAL_STATION_VLLM_RUNTIME, + NEMOCLAW_DGX_STATION_PEER_ENV, + parseStationHostProbe, + probeDualStationVllmCapability, + type StationClusterProbeDeps, + type StationHostProbe, + type StationProbeCommandResult, + type StationRailConnectivityRequest, +} from "./vllm-station-cluster"; + +const LOCAL_HOME = "/home/local"; +const PEER_HOME = "/home/nvidia"; +const STRICT_DOCKER_SSH_CONFIG = [ + "batchmode yes", + "stricthostkeychecking true", + "permitlocalcommand no", + "forwardagent no", + "forwardx11 no", + "forwardx11trusted no", + "tunnel false", + "updatehostkeys no", + "controlmaster false", + "controlpersist no", + "sendenv LANG", + "sendenv LC_*", +].join("\n"); + +function snapshotPath(home: string): string { + return [ + home, + ".cache/huggingface/hub", + `models--${DUAL_STATION_VLLM_RUNTIME.modelId.replace("/", "--")}`, + "snapshots", + DUAL_STATION_VLLM_RUNTIME.modelRevision, + ].join("/"); +} + +function rail( + rdmaDevice: string, + netdev: string, + pciAddress: string, + address: string, + gidIndexes: number[] = [3, 5], +) { + const hostOctet = netdev.includes("a") ? "aa" : "bb"; + const railOctet = netdev.endsWith("0") ? "00" : "01"; + return { + rdmaDevice, + port: 1, + netdev, + macAddress: `02:00:00:${hostOctet}:00:${railOctet}`, + uverbsDevice: `/dev/infiniband/uverbs${rdmaDevice.endsWith("0") ? "0" : "1"}`, + pciAddress, + pciName: `${pciAddress} Ethernet controller: NVIDIA ConnectX-8 SuperNIC`, + state: "4: ACTIVE", + linkLayer: "Ethernet", + speedMbps: 400_000, + mtu: 9000, + ipv4Addresses: [{ address, prefixLength: 30 }], + roceV2Ipv4Gids: gidIndexes.map((index) => ({ index, address })), + }; +} + +function setRailAddress( + item: ReturnType, + address: string, + prefixLength: number, +): void { + item.ipv4Addresses = [{ address, prefixLength }]; + item.roceV2Ipv4Gids = item.roceV2Ipv4Gids.map((gid) => ({ ...gid, address })); +} + +function hostFixture(side: "local" | "peer"): StationHostProbe { + const isLocal = side === "local"; + const home = isLocal ? LOCAL_HOME : PEER_HOME; + return { + schemaVersion: 1, + hostname: isLocal ? "station-a" : "station-b", + productName: "NVIDIA DGX Station GB300", + architecture: "aarch64", + home, + uid: isLocal ? 1000 : 1001, + gpus: isLocal + ? [{ index: 0, name: "NVIDIA GB300", uuid: "GPU-aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee" }] + : [ + { + index: 0, + name: "NVIDIA RTX PRO 6000", + uuid: "GPU-11111111-2222-3333-4444-555555555555", + }, + { + index: 1, + name: "NVIDIA GB300 Grace Blackwell Superchip", + uuid: "GPU-99999999-8888-7777-6666-555555555555", + }, + ], + docker: { reachable: true, nvidiaRuntime: true }, + rsyncAvailable: true, + nvidiaPeermemLoaded: true, + rails: isLocal + ? [ + rail("mlx5_0", "cx8a0", "0001:03:00.0", "192.168.240.1"), + rail("mlx5_1", "cx8a1", "0001:03:00.1", "192.168.240.5"), + ] + : [ + // Deliberately reverse inventory order; matching is by subnet. + rail("mlx5_1", "cx8b1", "0002:03:00.1", "192.168.240.6"), + rail("mlx5_0", "cx8b0", "0002:03:00.0", "192.168.240.2"), + ], + modelSnapshot: { + modelId: DUAL_STATION_VLLM_RUNTIME.modelId, + revision: DUAL_STATION_VLLM_RUNTIME.modelRevision, + path: snapshotPath(home), + directoryExists: !isLocal, + complete: !isLocal, + shardCount: isLocal ? 0 : 113, + reason: isLocal ? "not staged yet" : "", + }, + }; +} + +function command(stdout: unknown, status = 0): StationProbeCommandResult { + return { status, stdout: typeof stdout === "string" ? stdout : JSON.stringify(stdout) }; +} + +function connectivityResponse( + requests: readonly StationRailConnectivityRequest[], + alter: (check: Record, index: number) => void = () => undefined, +): StationProbeCommandResult { + const checks = requests.map((request, index) => { + const check: Record = { + ...request, + routeDevice: request.netdev, + routeSource: request.sourceAddress, + routeGateway: null, + routeScope: "link", + peerMac: request.expectedPeerMac, + peerNeighborState: "REACHABLE", + jumboPing: true, + }; + alter(check, index); + return check; + }); + return command({ schemaVersion: 1, checks }); +} + +type FixtureDeps = StationClusterProbeDeps & { + calls: { + sshConfig: ReturnType; + localHost: ReturnType; + peerHost: ReturnType; + localConnectivity: ReturnType; + peerConnectivity: ReturnType; + }; +}; + +function fixtureDeps( + local = hostFixture("local"), + peer = hostFixture("peer"), + options: { + localConnectivityAlter?: (check: Record, index: number) => void; + peerConnectivityAlter?: (check: Record, index: number) => void; + } = {}, +): FixtureDeps { + const localHost = vi.fn(() => command(local)); + const peerHost = vi.fn(() => command(peer)); + const sshConfig = vi.fn(() => command(STRICT_DOCKER_SSH_CONFIG)); + const localConnectivity = vi.fn((requests: readonly StationRailConnectivityRequest[]) => + connectivityResponse(requests, options.localConnectivityAlter), + ); + const peerConnectivity = vi.fn( + (_target: string, requests: readonly StationRailConnectivityRequest[]) => + connectivityResponse(requests, options.peerConnectivityAlter), + ); + return { + probePeerSshConfig: sshConfig, + probeLocalHost: localHost, + probePeerHost: peerHost, + probeLocalConnectivity: localConnectivity, + probePeerConnectivity: peerConnectivity, + calls: { sshConfig, localHost, peerHost, localConnectivity, peerConnectivity }, + }; +} + +function runWith(deps: StationClusterProbeDeps, target = "nvidia@station-b") { + return probeDualStationVllmCapability({ + env: { [NEMOCLAW_DGX_STATION_PEER_ENV]: target }, + deps, + }); +} + +describe("probeDualStationVllmCapability", () => { + it.each([ + undefined, + "", + " ", + ])("does no work when the explicit peer is absent or blank (%s)", (value) => { + const deps = fixtureDeps(); + const env = value === undefined ? {} : { [NEMOCLAW_DGX_STATION_PEER_ENV]: value }; + + expect(probeDualStationVllmCapability({ env, deps })).toEqual({ kind: "not-configured" }); + expect(deps.calls.sshConfig).not.toHaveBeenCalled(); + expect(deps.calls.localHost).not.toHaveBeenCalled(); + expect(deps.calls.peerHost).not.toHaveBeenCalled(); + expect(deps.calls.localConnectivity).not.toHaveBeenCalled(); + expect(deps.calls.peerConnectivity).not.toHaveBeenCalled(); + }); + + it.each([ + "ssh://station-b", + "-oProxyCommand=bad", + "station-a,station-b", + "station-b:2222", + "user name@station-b", + "station-b;id", + "station-b$(id)", + "user@station-b@other", + " station-b", + "station-b\nother", + "Station-B", + "station_b", + "station..b", + "station-b.", + "1user@station-b", + ])("rejects a non-single-host peer value without executing: %s", (target) => { + const deps = fixtureDeps(); + + expect(runWith(deps, target)).toMatchObject({ kind: "unavailable", code: "invalid-peer" }); + expect(deps.calls.sshConfig).not.toHaveBeenCalled(); + expect(deps.calls.localHost).not.toHaveBeenCalled(); + expect(deps.calls.peerHost).not.toHaveBeenCalled(); + }); + + it("rejects Docker-over-SSH when the effective operator config weakens peer trust", () => { + const deps = fixtureDeps(); + deps.probePeerSshConfig = () => + command( + STRICT_DOCKER_SSH_CONFIG.replace( + "stricthostkeychecking true", + "stricthostkeychecking false", + ), + ); + + expect(runWith(deps)).toMatchObject({ + kind: "unavailable", + code: "peer-ssh-config-unsafe", + }); + expect(deps.calls.localHost).not.toHaveBeenCalled(); + expect(deps.calls.peerHost).not.toHaveBeenCalled(); + }); + + it("rejects an SSH config that can SendEnv arbitrary NemoClaw secrets", () => { + const deps = fixtureDeps(); + deps.probePeerSshConfig = () => command(`${STRICT_DOCKER_SSH_CONFIG}\nsendenv *`); + + expect(runWith(deps)).toMatchObject({ + kind: "unavailable", + code: "peer-ssh-config-unsafe", + }); + expect(deps.calls.localHost).not.toHaveBeenCalled(); + }); + + it("rejects an ambient Docker client override before mixing it with local hardware", () => { + const deps = fixtureDeps(); + + expect( + probeDualStationVllmCapability({ + env: { + [NEMOCLAW_DGX_STATION_PEER_ENV]: "nvidia@station-b", + DOCKER_CONTEXT: "remote-builder", + }, + deps, + }), + ).toMatchObject({ kind: "unavailable", code: "local-docker-unavailable" }); + expect(deps.calls.sshConfig).not.toHaveBeenCalled(); + expect(deps.calls.localHost).not.toHaveBeenCalled(); + }); + + it.each([ + "station-b", + "nvidia@station-b", + "_svc@192.168.50.20", + ])("returns a downstream-compatible canonical Docker-over-SSH URI for %s", (target) => { + expect(runWith(fixtureDeps(), target)).toMatchObject({ + kind: "ready", + peerModelSnapshot: "ready", + plan: { peerSshTarget: target, peerDockerHost: `ssh://${target}` }, + }); + expect(buildRemoteVllmDockerEnv(`ssh://${target}`, {}).DOCKER_HOST).toBe(`ssh://${target}`); + }); + + it("returns a deterministic two-rail TP2 plan and permits one auxiliary non-GB300 GPU", () => { + const deps = fixtureDeps(); + + const result = runWith(deps); + + expect(result).toMatchObject({ + kind: "ready", + plan: { + peerSshTarget: "nvidia@station-b", + peerDockerHost: "ssh://nvidia@station-b", + runtime: { + image: + "vllm/vllm-openai@sha256:0fec7ec5f3e6bc168e54899935fb0557da908a4832a1dbc88e2debcf2f889416", + modelRevision: "183968f87ae4cedce3039313cac1fd43d112c578", + tensorParallelSize: 2, + nodeCount: 2, + }, + local: { home: LOCAL_HOME, uid: 1000, gpu: { index: 0, name: "NVIDIA GB300" } }, + peer: { + home: PEER_HOME, + uid: 1001, + gpu: { index: 1, name: "NVIDIA GB300 Grace Blackwell Superchip" }, + }, + masterAddress: "192.168.240.1", + roceGidIndex: 3, + rails: [ + { + subnet: "192.168.240.0/30", + local: { + rdmaDevice: "mlx5_0", + netdev: "cx8a0", + uverbsDevice: "/dev/infiniband/uverbs0", + address: "192.168.240.1", + }, + peer: { + rdmaDevice: "mlx5_0", + netdev: "cx8b0", + uverbsDevice: "/dev/infiniband/uverbs0", + address: "192.168.240.2", + }, + }, + { + subnet: "192.168.240.4/30", + local: { + rdmaDevice: "mlx5_1", + netdev: "cx8a1", + uverbsDevice: "/dev/infiniband/uverbs1", + address: "192.168.240.5", + }, + peer: { + rdmaDevice: "mlx5_1", + netdev: "cx8b1", + uverbsDevice: "/dev/infiniband/uverbs1", + address: "192.168.240.6", + }, + }, + ], + }, + }); + expect(deps.calls.peerHost).toHaveBeenCalledWith("nvidia@station-b"); + expect(deps.calls.localConnectivity).toHaveBeenCalledWith([ + { + netdev: "cx8a0", + sourceAddress: "192.168.240.1", + peerAddress: "192.168.240.2", + expectedPeerMac: "02:00:00:bb:00:00", + }, + { + netdev: "cx8a1", + sourceAddress: "192.168.240.5", + peerAddress: "192.168.240.6", + expectedPeerMac: "02:00:00:bb:00:01", + }, + ]); + expect(deps.calls.peerConnectivity).toHaveBeenCalledWith("nvidia@station-b", [ + { + netdev: "cx8b0", + sourceAddress: "192.168.240.2", + peerAddress: "192.168.240.1", + expectedPeerMac: "02:00:00:aa:00:00", + }, + { + netdev: "cx8b1", + sourceAddress: "192.168.240.6", + peerAddress: "192.168.240.5", + expectedPeerMac: "02:00:00:aa:00:01", + }, + ]); + }); + + it("uses the lowest common RoCEv2 IPv4 GID when preferred index 3 is unavailable", () => { + const local = hostFixture("local"); + const peer = hostFixture("peer"); + for (const item of [...local.rails, ...peer.rails]) { + item.roceV2Ipv4Gids = item.roceV2Ipv4Gids.map((gid) => ({ ...gid, index: 5 })); + } + + expect(runWith(fixtureDeps(local, peer))).toMatchObject({ + kind: "ready", + plan: { roceGidIndex: 5 }, + }); + }); + + it.each([ + "DGX-Station", + "P3830", + "NVIDIA Station GB300", + ])("accepts an existing Station firmware product identifier: %s", (productName) => { + const peer = hostFixture("peer"); + peer.productName = productName; + + expect(runWith(fixtureDeps(hostFixture("local"), peer))).toMatchObject({ + kind: "ready", + peerModelSnapshot: "ready", + }); + }); + + it.each([ + { + name: "non-Station peer", + code: "peer-not-station", + mutate: (host: StationHostProbe) => { + host.productName = "Generic Linux Workstation"; + }, + }, + { + name: "more than one peer GB300", + code: "peer-gpu-unavailable", + mutate: (host: StationHostProbe) => { + host.gpus.push({ index: 2, name: "NVIDIA GB300", uuid: "GPU-aaaa-bbbb-cccc-dddd" }); + }, + }, + { + name: "missing peer NVIDIA runtime", + code: "peer-docker-unavailable", + mutate: (host: StationHostProbe) => { + host.docker.nvidiaRuntime = false; + }, + }, + { + name: "missing peer nvidia_peermem", + code: "peer-fabric-unavailable", + mutate: (host: StationHostProbe) => { + host.nvidiaPeermemLoaded = false; + }, + }, + { + name: "slow peer rail", + code: "peer-fabric-unavailable", + mutate: (host: StationHostProbe) => { + host.rails[0].speedMbps = 200_000; + }, + }, + { + name: "non-jumbo peer rail", + code: "peer-fabric-unavailable", + mutate: (host: StationHostProbe) => { + host.rails[1].mtu = 1500; + }, + }, + { + name: "unsupported peer RDMA port", + code: "peer-fabric-unavailable", + mutate: (host: StationHostProbe) => { + host.rails[0].port = 2; + }, + }, + { + name: "missing peer uverbs character device", + code: "peer-fabric-unavailable", + mutate: (host: StationHostProbe) => { + host.rails[0].uverbsDevice = ""; + }, + }, + { + name: "duplicate peer uverbs mapping", + code: "peer-fabric-unavailable", + mutate: (host: StationHostProbe) => { + host.rails[1].uverbsDevice = host.rails[0].uverbsDevice; + }, + }, + { + name: "incomplete peer snapshot", + code: "peer-model-cache-unavailable", + mutate: (host: StationHostProbe) => { + host.modelSnapshot.complete = false; + }, + }, + { + name: "wrong peer snapshot revision", + code: "peer-model-cache-unavailable", + mutate: (host: StationHostProbe) => { + host.modelSnapshot.revision = "f".repeat(40); + }, + }, + { + name: "truncated peer snapshot manifest", + code: "peer-model-cache-unavailable", + mutate: (host: StationHostProbe) => { + host.modelSnapshot.shardCount = 1; + }, + }, + ])("fails closed for $name", ({ code, mutate }) => { + const peer = hostFixture("peer"); + mutate(peer); + + expect(runWith(fixtureDeps(hostFixture("local"), peer))).toMatchObject({ + kind: "unavailable", + code, + }); + }); + + it("qualifies a missing peer snapshot when exact staging prerequisites exist", () => { + const peer = hostFixture("peer"); + peer.modelSnapshot.directoryExists = false; + peer.modelSnapshot.complete = false; + peer.modelSnapshot.shardCount = 0; + peer.modelSnapshot.reason = "snapshot directory is missing"; + + expect(runWith(fixtureDeps(hostFixture("local"), peer))).toMatchObject({ + kind: "ready", + peerModelSnapshot: "staging-required", + }); + }); + + it.each([ + { side: "local", code: "local-model-staging-unavailable" }, + { side: "peer", code: "peer-model-staging-unavailable" }, + ])("requires rsync on the $side host before qualifying a missing snapshot", ({ side, code }) => { + const local = hostFixture("local"); + const peer = hostFixture("peer"); + peer.modelSnapshot.directoryExists = false; + peer.modelSnapshot.complete = false; + peer.modelSnapshot.shardCount = 0; + peer.modelSnapshot.reason = "snapshot directory is missing"; + (side === "local" ? local : peer).rsyncAvailable = false; + + expect(runWith(fixtureDeps(local, peer))).toMatchObject({ kind: "unavailable", code }); + }); + + it("rejects rails that do not form two distinct shared direct subnets", () => { + const peer = hostFixture("peer"); + peer.rails[0].ipv4Addresses = [{ address: "10.20.30.2", prefixLength: 30 }]; + peer.rails[0].roceV2Ipv4Gids = [{ index: 3, address: "10.20.30.2" }]; + + expect(runWith(fixtureDeps(hostFixture("local"), peer))).toMatchObject({ + kind: "unavailable", + code: "fabric-mismatch", + }); + }); + + it("rejects two otherwise matching switched /24 rail networks", () => { + const local = hostFixture("local"); + const peer = hostFixture("peer"); + setRailAddress(local.rails[0], "192.168.100.1", 24); + setRailAddress(local.rails[1], "192.168.101.1", 24); + setRailAddress(peer.rails.find((item) => item.rdmaDevice === "mlx5_0")!, "192.168.100.2", 24); + setRailAddress(peer.rails.find((item) => item.rdmaDevice === "mlx5_1")!, "192.168.101.2", 24); + + expect(runWith(fixtureDeps(local, peer))).toMatchObject({ + kind: "unavailable", + code: "fabric-mismatch", + }); + }); + + it("rejects public addresses even when they form matching /30 rail networks", () => { + const local = hostFixture("local"); + const peer = hostFixture("peer"); + setRailAddress(local.rails[0], "203.0.113.1", 30); + setRailAddress(local.rails[1], "198.51.100.5", 30); + setRailAddress(peer.rails.find((item) => item.rdmaDevice === "mlx5_0")!, "203.0.113.2", 30); + setRailAddress(peer.rails.find((item) => item.rdmaDevice === "mlx5_1")!, "198.51.100.6", 30); + + expect(runWith(fixtureDeps(local, peer))).toMatchObject({ + kind: "unavailable", + code: "fabric-mismatch", + }); + }); + + it("rejects asymmetric RoCEv2 IPv4 GID indexes", () => { + const peer = hostFixture("peer"); + peer.rails[0].roceV2Ipv4Gids = peer.rails[0].roceV2Ipv4Gids.map((gid) => ({ + ...gid, + index: 7, + })); + + expect(runWith(fixtureDeps(hostFixture("local"), peer))).toMatchObject({ + kind: "unavailable", + code: "gid-mismatch", + }); + }); + + it("rejects an SSH target that resolves back to the local Station identity", () => { + const local = hostFixture("local"); + const peer = hostFixture("peer"); + peer.gpus[1].uuid = local.gpus[0].uuid; + + expect(runWith(fixtureDeps(local, peer))).toMatchObject({ + kind: "unavailable", + code: "fabric-mismatch", + }); + }); + + it("allows distinct factory-imaged Stations that report the same hostname", () => { + const local = hostFixture("local"); + const peer = hostFixture("peer"); + peer.hostname = local.hostname; + + expect(runWith(fixtureDeps(local, peer))).toMatchObject({ kind: "ready" }); + }); + + it("rejects a local route that traverses a gateway", () => { + const deps = fixtureDeps(hostFixture("local"), hostFixture("peer"), { + localConnectivityAlter: (check, index) => { + check.routeGateway = index === 0 ? "192.168.240.254" : check.routeGateway; + }, + }); + + expect(runWith(deps)).toMatchObject({ + kind: "unavailable", + code: "local-connectivity-failed", + }); + expect(deps.calls.peerConnectivity).not.toHaveBeenCalled(); + }); + + it("rejects a peer rail that cannot pass an MTU-9000 ping", () => { + const deps = fixtureDeps(hostFixture("local"), hostFixture("peer"), { + peerConnectivityAlter: (check, index) => { + check.jumboPing = index === 1 ? false : check.jumboPing; + }, + }); + + expect(runWith(deps)).toMatchObject({ + kind: "unavailable", + code: "peer-connectivity-failed", + }); + }); + + it("rejects a route without a scope-link connected-prefix proof", () => { + const deps = fixtureDeps(hostFixture("local"), hostFixture("peer"), { + localConnectivityAlter: (check, index) => { + check.routeScope = index === 0 ? "global" : check.routeScope; + }, + }); + + expect(runWith(deps)).toMatchObject({ + kind: "unavailable", + code: "local-connectivity-failed", + }); + }); + + it("rejects a neighbor MAC that does not identify the matched peer rail", () => { + const deps = fixtureDeps(hostFixture("local"), hostFixture("peer"), { + peerConnectivityAlter: (check, index) => { + check.peerMac = index === 0 ? "02:00:00:cc:00:00" : check.peerMac; + }, + }); + + expect(runWith(deps)).toMatchObject({ + kind: "unavailable", + code: "peer-connectivity-failed", + }); + }); + + it("fails closed on command failure or malformed host JSON", () => { + const localFailure = fixtureDeps(); + localFailure.probeLocalHost = () => command("", 1); + expect(runWith(localFailure)).toMatchObject({ + kind: "unavailable", + code: "local-probe-failed", + }); + + const peerMalformed = fixtureDeps(); + peerMalformed.probePeerHost = () => command("not-json"); + expect(runWith(peerMalformed)).toMatchObject({ + kind: "unavailable", + code: "peer-probe-failed", + }); + }); +}); + +describe("probe command boundary", () => { + it("audits the exact effective SSH config used later by Docker transport", () => { + const spawn = vi.fn( + ( + _file: string, + _args: readonly string[], + _options: SpawnSyncOptionsWithStringEncoding, + ): StationProbeCommandResult => command(STRICT_DOCKER_SSH_CONFIG), + ); + const deps = createStationClusterProbeDeps(spawn); + + deps.probePeerSshConfig("nvidia@station-b"); + + const [file, args, options] = spawn.mock.calls[0]; + expect(file).toBe("ssh"); + expect(args).toEqual(["-G", "-T", "-o", "ConnectTimeout=30", "--", "nvidia@station-b"]); + expect(options.input).toBe(""); + expect(options.timeout).toBe(20_000); + }); + + it("uses a fixed stdin script and strict pretrusted SSH without discovery or prompting", () => { + const spawn = vi.fn( + ( + _file: string, + _args: readonly string[], + _options: SpawnSyncOptionsWithStringEncoding, + ): StationProbeCommandResult => command({}), + ); + const deps = createStationClusterProbeDeps(spawn); + + deps.probePeerHost("nvidia@station-b"); + + const [file, args, options] = spawn.mock.calls[0]; + expect(file).toBe("ssh"); + expect(args).toEqual( + expect.arrayContaining([ + "BatchMode=yes", + "StrictHostKeyChecking=yes", + "NumberOfPasswordPrompts=0", + "ConnectTimeout=5", + "ClearAllForwardings=yes", + "--", + "nvidia@station-b", + "python3 -", + ]), + ); + expect(args.join(" ")).not.toMatch(/keyscan|accept-new|StrictHostKeyChecking=no/); + expect(options.input).toEqual(expect.stringContaining('docker", "info')); + expect(options.input).toEqual(expect.stringContaining("/sys/firmware/devicetree/base/model")); + expect(options.input).toEqual(expect.stringContaining("shard_path.is_absolute()")); + expect(options.input).toEqual(expect.stringContaining('shutil.which("rsync")')); + expect(options.input).toEqual(expect.stringContaining('"directoryExists"')); + expect(options.input).toEqual(expect.stringContaining("len(shards) != 113")); + expect(options.input).toEqual( + expect.stringContaining("observed_tensor_size != expected_total_size"), + ); + expect(options.input).toEqual(expect.stringContaining('"infiniband_verbs"')); + expect(options.input).toEqual(expect.stringContaining('re.fullmatch(r"uverbs[0-9]+"')); + expect(options.input).toEqual(expect.stringContaining("len(names) != 1")); + expect(options.input).toEqual(expect.stringContaining("stat.S_ISCHR")); + expect(options.timeout).toBe(20_000); + expect(options.maxBuffer).toBe(1024 * 1024); + }); + + it("removes OPENSHELL secrets from the direct SSH probe environment", () => { + vi.stubEnv("OPENSHELL_GATEWAY_AUTH_TOKEN", "must-not-cross-ssh"); + const spawn = vi.fn( + ( + _file: string, + _args: readonly string[], + _options: SpawnSyncOptionsWithStringEncoding, + ): StationProbeCommandResult => command({}), + ); + const deps = createStationClusterProbeDeps(spawn); + + deps.probePeerHost("nvidia@station-b"); + + const [, , options] = spawn.mock.calls[0]; + expect(options.env?.OPENSHELL_GATEWAY_AUTH_TOKEN).toBeUndefined(); + expect(options.env?.PATH).toBeTruthy(); + vi.unstubAllEnvs(); + }); + + it("pins the local hardware probe to Docker's physical default context", () => { + const spawn = vi.fn( + ( + _file: string, + _args: readonly string[], + _options: SpawnSyncOptionsWithStringEncoding, + ): StationProbeCommandResult => command({}), + ); + const deps = createStationClusterProbeDeps(spawn); + + deps.probeLocalHost(); + + const [file, , options] = spawn.mock.calls[0]; + expect(file).toBe("python3"); + expect(options.env?.DOCKER_CONTEXT).toBe("default"); + expect(options.env?.DOCKER_HOST).toBeUndefined(); + expect(options.env?.DOCKER_CONFIG).toBeUndefined(); + }); + + it("passes only validated discovered rail values to the fixed peer connectivity script", () => { + const spawn = vi.fn( + ( + _file: string, + _args: readonly string[], + _options: SpawnSyncOptionsWithStringEncoding, + ): StationProbeCommandResult => command({}), + ); + const deps = createStationClusterProbeDeps(spawn); + const requests = [ + { + netdev: "cx8b0", + sourceAddress: "192.168.240.2", + peerAddress: "192.168.240.1", + expectedPeerMac: "02:00:00:aa:00:00", + }, + { + netdev: "cx8b1", + sourceAddress: "192.168.240.6", + peerAddress: "192.168.240.5", + expectedPeerMac: "02:00:00:aa:00:01", + }, + ]; + + deps.probePeerConnectivity("station-b", requests); + + const [, args, options] = spawn.mock.calls[0]; + expect(args.at(-1)).toBe( + "python3 - cx8b0 192.168.240.2 192.168.240.1 cx8b1 192.168.240.6 192.168.240.5", + ); + expect(options.input).toEqual(expect.stringContaining('"ping", "-4", "-M", "do"')); + expect(options.input).toEqual( + expect.stringContaining('"route", "get", peer, "from", source, "oif", netdev'), + ); + expect(options.input).toEqual( + expect.stringContaining('"route", "show", "exact", network, "dev", netdev'), + ); + expect(options.input).toEqual( + expect.stringContaining('"neighbor", "show", "to", peer, "dev", netdev'), + ); + expect(options.input).toEqual(expect.stringContaining('"-I", source, peer')); + }); +}); + +describe("parseStationHostProbe", () => { + it("rejects unsupported schemas and unsafe device names", () => { + const unsupported = { ...hostFixture("local"), schemaVersion: 2 }; + expect(() => parseStationHostProbe(JSON.stringify(unsupported))).toThrow(/schema version/); + + const unsafe = hostFixture("local"); + unsafe.rails[0].netdev = "cx8;touch /tmp/pwned"; + expect(() => parseStationHostProbe(JSON.stringify(unsafe))).toThrow(/unsafe device name/); + + const unsafeUverbs = hostFixture("local"); + unsafeUverbs.rails[0].uverbsDevice = "/dev/infiniband/../mem"; + expect(() => parseStationHostProbe(JSON.stringify(unsafeUverbs))).toThrow(/uverbs/); + }); +}); diff --git a/src/lib/inference/vllm-station-cluster.ts b/src/lib/inference/vllm-station-cluster.ts new file mode 100644 index 0000000000..58749346b1 --- /dev/null +++ b/src/lib/inference/vllm-station-cluster.ts @@ -0,0 +1,1631 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { type SpawnSyncOptionsWithStringEncoding, spawnSync } from "node:child_process"; +import net from "node:net"; +import path from "node:path"; + +import { buildSubprocessEnv } from "../subprocess-env"; +import { buildVllmSshTransportEnv } from "./vllm-docker-env"; +import { NEMOTRON_ULTRA_STATION_IMAGE, VLLM_MODELS } from "./vllm-models"; + +export const NEMOCLAW_DGX_STATION_PEER_ENV = "NEMOCLAW_DGX_STATION_PEER"; + +const HOST_PROBE_SCHEMA_VERSION = 1; +const CONNECTIVITY_PROBE_SCHEMA_VERSION = 1; +const COMMAND_TIMEOUT_MS = 20_000; +const CONNECT_TIMEOUT_SECONDS = 5; +const MAX_PROBE_OUTPUT_BYTES = 1024 * 1024; +const PREFERRED_ROCE_GID_INDEX = 3; +const EXPECTED_ULTRA_WEIGHT_SHARDS = 113; +const DIRECT_RAIL_PREFIX_LENGTH = 30; +const DUAL_STATION_LOCAL_DOCKER_OVERRIDE_ENV_NAMES = [ + "DOCKER_API_VERSION", + "DOCKER_CERT_PATH", + "DOCKER_CONFIG", + "DOCKER_CONTEXT", + "DOCKER_HOST", + "DOCKER_TLS", + "DOCKER_TLS_VERIFY", +] as const; +const CANONICAL_SSH_HOST_PATTERN = + /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?(?:\.[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?)*$/; +const CANONICAL_SSH_USERNAME_PATTERN = /^[A-Za-z_][A-Za-z0-9._-]*$/; + +const ultraModel = VLLM_MODELS.find((model) => model.envValue === "nemotron-3-ultra-550b-a55b"); +if (!ultraModel?.revision) { + throw new Error("Nemotron Ultra must have an immutable Hugging Face revision"); +} + +export const DUAL_STATION_VLLM_RUNTIME = Object.freeze({ + image: NEMOTRON_ULTRA_STATION_IMAGE.arm64.ref, + modelId: ultraModel.id, + modelRevision: ultraModel.revision, + servedModelId: ultraModel.servedModelId ?? ultraModel.id, + tensorParallelSize: 2 as const, + nodeCount: 2 as const, +}); + +export interface StationGpuProbe { + index: number; + name: string; + uuid: string; +} + +export interface StationIpv4AddressProbe { + address: string; + prefixLength: number; +} + +export interface StationRoceGidProbe { + index: number; + address: string; +} + +export interface StationRailProbe { + rdmaDevice: string; + port: number; + netdev: string; + macAddress: string; + uverbsDevice: string; + pciAddress: string; + pciName: string; + state: string; + linkLayer: string; + speedMbps: number; + mtu: number; + ipv4Addresses: StationIpv4AddressProbe[]; + roceV2Ipv4Gids: StationRoceGidProbe[]; +} + +export interface StationModelSnapshotProbe { + modelId: string; + revision: string; + path: string; + directoryExists: boolean; + complete: boolean; + shardCount: number; + reason: string; +} + +export interface StationHostProbe { + schemaVersion: 1; + hostname: string; + productName: string; + architecture: string; + home: string; + uid: number; + gpus: StationGpuProbe[]; + docker: { + reachable: boolean; + nvidiaRuntime: boolean; + }; + rsyncAvailable: boolean; + nvidiaPeermemLoaded: boolean; + rails: StationRailProbe[]; + modelSnapshot: StationModelSnapshotProbe; +} + +export interface StationRailConnectivityRequest { + netdev: string; + sourceAddress: string; + peerAddress: string; + expectedPeerMac: string; +} + +export interface StationRailConnectivityProbe { + netdev: string; + sourceAddress: string; + peerAddress: string; + routeDevice: string; + routeSource: string; + routeGateway: string | null; + routeScope: string; + peerMac: string; + peerNeighborState: string; + jumboPing: boolean; +} + +export interface StationProbeCommandResult { + status: number | null; + stdout: string; + stderr?: string; + error?: string; +} + +export interface StationClusterProbeDeps { + probePeerSshConfig(peerTarget: string): StationProbeCommandResult; + probeLocalHost(): StationProbeCommandResult; + probePeerHost(peerTarget: string): StationProbeCommandResult; + probeLocalConnectivity( + requests: readonly StationRailConnectivityRequest[], + ): StationProbeCommandResult; + probePeerConnectivity( + peerTarget: string, + requests: readonly StationRailConnectivityRequest[], + ): StationProbeCommandResult; +} + +export type StationClusterFailureCode = + | "invalid-peer" + | "local-probe-failed" + | "peer-probe-failed" + | "local-not-station" + | "peer-not-station" + | "local-gpu-unavailable" + | "peer-gpu-unavailable" + | "local-docker-unavailable" + | "peer-docker-unavailable" + | "peer-ssh-config-unsafe" + | "local-model-staging-unavailable" + | "peer-model-staging-unavailable" + | "local-fabric-unavailable" + | "peer-fabric-unavailable" + | "fabric-mismatch" + | "gid-mismatch" + | "peer-model-cache-unavailable" + | "local-connectivity-failed" + | "peer-connectivity-failed" + | "probe-error"; + +export interface DualStationPlanNode { + hostname: string; + home: string; + uid: number; + gpu: StationGpuProbe; +} + +export interface DualStationPlanRailEndpoint { + rdmaDevice: string; + netdev: string; + macAddress: string; + uverbsDevice: string; + pciAddress: string; + address: string; +} + +export interface DualStationPlanRail { + index: number; + subnet: string; + local: DualStationPlanRailEndpoint; + peer: DualStationPlanRailEndpoint; +} + +export interface DualStationVllmPlan { + peerSshTarget: string; + peerDockerHost: string; + runtime: typeof DUAL_STATION_VLLM_RUNTIME; + local: DualStationPlanNode; + peer: DualStationPlanNode; + rails: DualStationPlanRail[]; + masterAddress: string; + roceGidIndex: number; +} + +export type StationClusterCapability = + | { kind: "not-configured" } + | { kind: "unavailable"; code: StationClusterFailureCode; reason: string } + | { + kind: "ready"; + plan: DualStationVllmPlan; + peerModelSnapshot: "ready" | "staging-required"; + }; + +type PlanFailure = Extract; + +type MatchedRail = { + localRail: StationRailProbe; + peerRail: StationRailProbe; + localAddress: StationIpv4AddressProbe; + peerAddress: StationIpv4AddressProbe; + subnet: string; +}; + +type StaticPlan = { + plan: DualStationVllmPlan; + peerModelSnapshot: "ready" | "staging-required"; + localConnectivity: StationRailConnectivityRequest[]; + peerConnectivity: StationRailConnectivityRequest[]; +}; + +type StationProbeSpawn = ( + file: string, + args: readonly string[], + options: SpawnSyncOptionsWithStringEncoding, +) => StationProbeCommandResult; + +const HOST_PROBE_SCRIPT = String.raw` +import csv +import ipaddress +import json +import os +from pathlib import Path +import platform +import re +import shutil +import socket +import stat +import subprocess + +MODEL_ID = ${JSON.stringify(DUAL_STATION_VLLM_RUNTIME.modelId)} +MODEL_REVISION = ${JSON.stringify(DUAL_STATION_VLLM_RUNTIME.modelRevision)} +MODEL_CACHE_NAME = "models--" + MODEL_ID.replace("/", "--") + +def read_text(path): + try: + return Path(path).read_text(encoding="utf-8").rstrip("\x00").strip() + except (OSError, UnicodeError): + return "" + +def run(argv, timeout=5): + try: + result = subprocess.run( + argv, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + timeout=timeout, + check=False, + ) + return result.returncode, result.stdout.strip() + except (FileNotFoundError, OSError, subprocess.TimeoutExpired): + return 127, "" + +def product_name(): + for candidate in ( + "/sys/class/dmi/id/product_name", + "/sys/devices/virtual/dmi/id/product_name", + "/sys/firmware/devicetree/base/model", + ): + value = read_text(candidate) + if value: + return value + return "" + +def gpu_inventory(): + rc, output = run([ + "nvidia-smi", + "--query-gpu=index,name,uuid", + "--format=csv,noheader,nounits", + ]) + if rc != 0: + return [] + result = [] + for row in csv.reader(output.splitlines()): + if len(row) != 3: + continue + try: + index = int(row[0].strip()) + except ValueError: + continue + result.append({ + "index": index, + "name": row[1].strip(), + "uuid": row[2].strip(), + }) + return result + +def docker_state(): + rc, output = run(["docker", "info", "--format", "{{json .Runtimes}}"]) + if rc != 0: + return {"reachable": False, "nvidiaRuntime": False} + try: + runtimes = json.loads(output) + except (TypeError, json.JSONDecodeError): + runtimes = {} + return { + "reachable": True, + "nvidiaRuntime": isinstance(runtimes, dict) and "nvidia" in runtimes, + } + +def ipv4_addresses(netdev): + rc, output = run(["ip", "-j", "-4", "address", "show", "dev", netdev]) + if rc != 0: + return [] + try: + links = json.loads(output) + except json.JSONDecodeError: + return [] + result = [] + for link in links if isinstance(links, list) else []: + for address in link.get("addr_info", []): + if address.get("family") != "inet" or address.get("scope") == "host": + continue + local = address.get("local") + prefix = address.get("prefixlen") + if isinstance(local, str) and isinstance(prefix, int): + result.append({"address": local, "prefixLength": prefix}) + return result + +def roce_v2_ipv4_gids(rdma_device, port, netdev): + base = Path("/sys/class/infiniband") / rdma_device / "ports" / str(port) + types_dir = base / "gid_attrs" / "types" + result = [] + try: + indexes = sorted(types_dir.iterdir(), key=lambda item: int(item.name)) + except (OSError, ValueError): + return result + for entry in indexes: + try: + index = int(entry.name) + except ValueError: + continue + if read_text(entry).lower() != "roce v2": + continue + observed_netdev = read_text(base / "gid_attrs" / "ndevs" / str(index)) + if observed_netdev and observed_netdev != netdev: + continue + raw_gid = read_text(base / "gids" / str(index)) + try: + mapped = ipaddress.IPv6Address(raw_gid).ipv4_mapped + except ipaddress.AddressValueError: + mapped = None + if mapped is not None: + result.append({"index": index, "address": str(mapped)}) + return result + +def uverbs_device(rdma_device): + verbs_dir = Path("/sys/class/infiniband") / rdma_device / "device" / "infiniband_verbs" + try: + names = sorted({ + entry.name + for entry in verbs_dir.iterdir() + if re.fullmatch(r"uverbs[0-9]+", entry.name) + }) + except OSError: + return "" + if len(names) != 1: + return "" + device = Path("/dev/infiniband") / names[0] + try: + if not stat.S_ISCHR(device.stat().st_mode): + return "" + except OSError: + return "" + return str(device) + +def rail_inventory(): + rc, output = run(["ibdev2netdev"]) + if rc != 0: + return [] + rails = [] + pattern = re.compile(r"^(\S+)\s+port\s+(\d+)\s+==>\s+(\S+)\s+\(([^)]*)\)") + for line in output.splitlines(): + match = pattern.match(line.strip()) + if not match: + continue + rdma_device, raw_port, netdev, _reported_state = match.groups() + port = int(raw_port) + device_path = Path("/sys/class/net") / netdev / "device" + try: + pci_address = device_path.resolve(strict=True).name + except OSError: + pci_address = "" + rc_lspci, pci_name = run(["lspci", "-D", "-s", pci_address]) if pci_address else (127, "") + if rc_lspci != 0: + pci_name = "" + try: + speed_mbps = int(read_text(Path("/sys/class/net") / netdev / "speed")) + except ValueError: + speed_mbps = -1 + try: + mtu = int(read_text(Path("/sys/class/net") / netdev / "mtu")) + except ValueError: + mtu = -1 + ib_port = Path("/sys/class/infiniband") / rdma_device / "ports" / str(port) + rails.append({ + "rdmaDevice": rdma_device, + "port": port, + "netdev": netdev, + "macAddress": read_text(Path("/sys/class/net") / netdev / "address").lower(), + "uverbsDevice": uverbs_device(rdma_device), + "pciAddress": pci_address, + "pciName": pci_name, + "state": read_text(ib_port / "state"), + "linkLayer": read_text(ib_port / "link_layer"), + "speedMbps": speed_mbps, + "mtu": mtu, + "ipv4Addresses": ipv4_addresses(netdev), + "roceV2Ipv4Gids": roce_v2_ipv4_gids(rdma_device, port, netdev), + }) + return rails + +def snapshot_state(): + home = Path.home() + snapshot = home / ".cache" / "huggingface" / "hub" / MODEL_CACHE_NAME / "snapshots" / MODEL_REVISION + reasons = [] + shard_count = 0 + index_path = snapshot / "model.safetensors.index.json" + if not snapshot.is_dir(): + reasons.append("snapshot directory is missing") + if not (snapshot / "config.json").is_file(): + reasons.append("config.json is missing") + tokenizer_present = any( + (snapshot / name).is_file() + for name in ("tokenizer.json", "tokenizer.model", "vocab.json") + ) + if not tokenizer_present: + reasons.append("tokenizer assets are missing") + try: + index = json.loads(index_path.read_text(encoding="utf-8")) + weight_map = index.get("weight_map", {}) + metadata = index.get("metadata", {}) + shards = sorted(set(weight_map.values())) if isinstance(weight_map, dict) else [] + if not shards or not all(isinstance(item, str) for item in shards): + reasons.append("weight index is empty or malformed") + shards = [] + if len(shards) != ${String(113)}: + reasons.append("weight index does not list the pinned shard count") + expected_total_size = metadata.get("total_size") if isinstance(metadata, dict) else None + if not isinstance(expected_total_size, int) or expected_total_size <= 0: + reasons.append("weight index total_size is missing or malformed") + shard_count = len(shards) + observed_tensor_size = 0 + for shard in shards: + shard_path = Path(shard) + if ( + shard_path.is_absolute() + or shard_path.name != shard + or shard in (".", "..") + or shard_path.suffix != ".safetensors" + ): + reasons.append("weight index contains an unsafe shard path") + break + candidate = snapshot / shard_path + try: + if not candidate.is_file() or candidate.stat().st_size <= 0: + reasons.append("one or more weight shards are missing") + break + with candidate.open("rb") as handle: + raw_header_size = handle.read(8) + if len(raw_header_size) != 8: + raise ValueError("short safetensors header") + header_size = int.from_bytes(raw_header_size, "little") + if header_size <= 0 or header_size > 128 * 1024 * 1024: + raise ValueError("invalid safetensors header size") + header = json.loads(handle.read(header_size)) + tensor_ranges = [] + for tensor_name, tensor in header.items(): + if tensor_name == "__metadata__": + continue + offsets = tensor.get("data_offsets") if isinstance(tensor, dict) else None + if ( + not isinstance(offsets, list) + or len(offsets) != 2 + or not all(isinstance(offset, int) for offset in offsets) + or offsets[0] < 0 + or offsets[1] < offsets[0] + ): + raise ValueError("invalid safetensors data offsets") + tensor_ranges.append(offsets) + if not tensor_ranges: + raise ValueError("empty safetensors shard") + payload_size = max(offsets[1] for offsets in tensor_ranges) + if candidate.stat().st_size != 8 + header_size + payload_size: + raise ValueError("truncated safetensors shard") + observed_tensor_size += sum(offsets[1] - offsets[0] for offsets in tensor_ranges) + except (OSError, UnicodeError, ValueError, json.JSONDecodeError): + reasons.append("one or more weight shards are unreadable or malformed") + break + if isinstance(expected_total_size, int) and observed_tensor_size != expected_total_size: + reasons.append("weight shard sizes do not match the pinned index") + except (OSError, UnicodeError, json.JSONDecodeError): + reasons.append("model.safetensors.index.json is missing or malformed") + return { + "modelId": MODEL_ID, + "revision": MODEL_REVISION, + "path": str(snapshot), + "directoryExists": snapshot.is_dir(), + "complete": not reasons, + "shardCount": shard_count, + "reason": "; ".join(dict.fromkeys(reasons)), + } + +modules = read_text("/proc/modules").splitlines() +payload = { + "schemaVersion": ${String(HOST_PROBE_SCHEMA_VERSION)}, + "hostname": socket.gethostname(), + "productName": product_name(), + "architecture": platform.machine(), + "home": str(Path.home()), + "uid": os.getuid(), + "gpus": gpu_inventory(), + "docker": docker_state(), + "rsyncAvailable": shutil.which("rsync") is not None, + "nvidiaPeermemLoaded": any(line.split(maxsplit=1)[0] == "nvidia_peermem" for line in modules if line), + "rails": rail_inventory(), + "modelSnapshot": snapshot_state(), +} +print(json.dumps(payload, separators=(",", ":"))) +`; + +const CONNECTIVITY_PROBE_SCRIPT = String.raw` +import ipaddress +import json +import subprocess +import sys + +def run(argv, timeout=5): + try: + result = subprocess.run( + argv, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + timeout=timeout, + check=False, + ) + return result.returncode, result.stdout.strip() + except (FileNotFoundError, OSError, subprocess.TimeoutExpired): + return 127, "" + +if len(sys.argv[1:]) != 6: + raise SystemExit("expected two netdev/source/peer triples") + +checks = [] +for offset in (0, 3): + netdev, source, peer = sys.argv[1 + offset:4 + offset] + route_device = "" + route_source = "" + route_gateway = None + route_scope = "" + peer_mac = "" + peer_neighbor_state = "" + rc, output = run(["ip", "-j", "route", "get", peer, "from", source, "oif", netdev]) + if rc == 0: + try: + routes = json.loads(output) + route = routes[0] if isinstance(routes, list) and routes else {} + route_device = route.get("dev", "") if isinstance(route, dict) else "" + route_source = route.get("prefsrc", route.get("src", "")) if isinstance(route, dict) else "" + route_gateway = route.get("gateway") if isinstance(route, dict) else None + except json.JSONDecodeError: + pass + network = str(ipaddress.ip_network(source + "/${String(DIRECT_RAIL_PREFIX_LENGTH)}", strict=False)) + link_rc, link_output = run(["ip", "-j", "route", "show", "exact", network, "dev", netdev]) + if link_rc == 0: + try: + link_routes = json.loads(link_output) + link_route = link_routes[0] if isinstance(link_routes, list) and link_routes else {} + if ( + isinstance(link_route, dict) + and link_route.get("dst") == network + and link_route.get("dev") == netdev + and link_route.get("gateway") is None + ): + route_scope = link_route.get("scope", "") + except json.JSONDecodeError: + pass + ping_rc, _ = run([ + "ping", "-4", "-M", "do", "-s", "8972", "-c", "1", "-W", "2", + "-I", source, peer, + ]) + neighbor_rc, neighbor_output = run(["ip", "-j", "neighbor", "show", "to", peer, "dev", netdev]) + if neighbor_rc == 0: + try: + neighbors = json.loads(neighbor_output) + neighbor = neighbors[0] if isinstance(neighbors, list) and neighbors else {} + if isinstance(neighbor, dict) and neighbor.get("dst") == peer and neighbor.get("dev") == netdev: + peer_mac = str(neighbor.get("lladdr", "")).lower() + raw_state = neighbor.get("state", "") + peer_neighbor_state = ",".join(raw_state) if isinstance(raw_state, list) else str(raw_state) + except json.JSONDecodeError: + pass + checks.append({ + "netdev": netdev, + "sourceAddress": source, + "peerAddress": peer, + "routeDevice": route_device, + "routeSource": route_source, + "routeGateway": route_gateway, + "routeScope": route_scope, + "peerMac": peer_mac, + "peerNeighborState": peer_neighbor_state, + "jumboPing": ping_rc == 0, + }) + +print(json.dumps({ + "schemaVersion": ${String(CONNECTIVITY_PROBE_SCHEMA_VERSION)}, + "checks": checks, +}, separators=(",", ":"))) +`; + +function defaultSpawn( + file: string, + args: readonly string[], + options: SpawnSyncOptionsWithStringEncoding, +): StationProbeCommandResult { + const result = spawnSync(file, [...args], options); + return { + status: result.status, + stdout: result.stdout ?? "", + stderr: result.stderr ?? "", + error: result.error?.message, + }; +} + +function commandOptions(input: string): SpawnSyncOptionsWithStringEncoding { + return { + encoding: "utf8", + input, + timeout: COMMAND_TIMEOUT_MS, + maxBuffer: MAX_PROBE_OUTPUT_BYTES, + killSignal: "SIGKILL", + windowsHide: true, + env: buildSubprocessEnv({ LC_ALL: "C" }), + }; +} + +function localProbeCommandOptions(input: string): SpawnSyncOptionsWithStringEncoding { + const options = commandOptions(input); + for (const name of DUAL_STATION_LOCAL_DOCKER_OVERRIDE_ENV_NAMES) delete options.env?.[name]; + if (options.env) options.env.DOCKER_CONTEXT = "default"; + return options; +} + +function sshCommandOptions(input: string): SpawnSyncOptionsWithStringEncoding { + return { + ...commandOptions(input), + env: buildVllmSshTransportEnv({ LC_ALL: "C" }), + }; +} + +export function strictStationSshTransportArgs(): string[] { + return [ + "-T", + "-o", + "BatchMode=yes", + "-o", + "StrictHostKeyChecking=yes", + "-o", + "NumberOfPasswordPrompts=0", + "-o", + `ConnectTimeout=${String(CONNECT_TIMEOUT_SECONDS)}`, + "-o", + "ConnectionAttempts=1", + "-o", + "ServerAliveInterval=5", + "-o", + "ServerAliveCountMax=1", + "-o", + "ClearAllForwardings=yes", + "-o", + "ForwardAgent=no", + "-o", + "ForwardX11=no", + "-o", + "ForwardX11Trusted=no", + "-o", + "Tunnel=no", + "-o", + "UpdateHostKeys=no", + "-o", + "ControlMaster=no", + "-o", + "ControlPath=none", + "-o", + "PermitLocalCommand=no", + "-o", + "RemoteCommand=none", + "-o", + "LogLevel=ERROR", + ]; +} + +function strictSshArgs(peerTarget: string, remoteCommand: string): string[] { + return [...strictStationSshTransportArgs(), "--", peerTarget, remoteCommand]; +} + +function connectivityArgv(requests: readonly StationRailConnectivityRequest[]): string[] { + if (requests.length !== 2) { + throw new Error("dual-Station connectivity requires exactly two rail requests"); + } + const args: string[] = []; + for (const request of requests) { + if (!isSafeDeviceName(request.netdev)) throw new Error("unsafe connectivity netdev"); + if (!isIpv4(request.sourceAddress) || !isIpv4(request.peerAddress)) { + throw new Error("invalid connectivity address"); + } + normalizeMacAddress(request.expectedPeerMac, "connectivity peer MAC"); + args.push(request.netdev, request.sourceAddress, request.peerAddress); + } + return args; +} + +/** + * Construct the real read-only probe boundary. The optional spawn injection is + * intentionally lower-level than StationClusterProbeDeps so tests can assert + * the exact SSH trust flags and fixed-stdin behavior without making a network + * connection. + */ +export function createStationClusterProbeDeps( + spawn: StationProbeSpawn = defaultSpawn, +): StationClusterProbeDeps { + return { + probePeerSshConfig: (peerTarget) => { + const validated = validatePeerTarget(peerTarget); + if (!validated.ok) throw new Error(validated.reason); + return spawn( + "ssh", + ["-G", "-T", "-o", "ConnectTimeout=30", "--", validated.target], + sshCommandOptions(""), + ); + }, + probeLocalHost: () => spawn("python3", ["-"], localProbeCommandOptions(HOST_PROBE_SCRIPT)), + probePeerHost: (peerTarget) => { + const validated = validatePeerTarget(peerTarget); + if (!validated.ok) throw new Error(validated.reason); + return spawn( + "ssh", + strictSshArgs(validated.target, "python3 -"), + sshCommandOptions(HOST_PROBE_SCRIPT), + ); + }, + probeLocalConnectivity: (requests) => { + const args = connectivityArgv(requests); + return spawn("python3", ["-", ...args], localProbeCommandOptions(CONNECTIVITY_PROBE_SCRIPT)); + }, + probePeerConnectivity: (peerTarget, requests) => { + const validated = validatePeerTarget(peerTarget); + if (!validated.ok) throw new Error(validated.reason); + const args = connectivityArgv(requests); + const remoteCommand = ["python3", "-", ...args].join(" "); + return spawn( + "ssh", + strictSshArgs(validated.target, remoteCommand), + sshCommandOptions(CONNECTIVITY_PROBE_SCRIPT), + ); + }, + }; +} + +const defaultStationClusterProbeDeps = createStationClusterProbeDeps(); + +type PeerValidation = { ok: true; target: string } | { ok: false; reason: string }; + +export function validatePeerTarget(raw: string): PeerValidation { + if (raw.length === 0 || raw !== raw.trim()) { + return { ok: false, reason: `${NEMOCLAW_DGX_STATION_PEER_ENV} must not contain whitespace` }; + } + if (raw.length > 286) { + return { ok: false, reason: `${NEMOCLAW_DGX_STATION_PEER_ENV} is too long` }; + } + if (/[/,:;`'"\\$(){}[\]<>|&!?*\s\u0000-\u001f\u007f]/.test(raw)) { + return { + ok: false, + reason: `${NEMOCLAW_DGX_STATION_PEER_ENV} must name one SSH host or user@host`, + }; + } + const parts = raw.split("@"); + if (parts.length > 2) { + return { + ok: false, + reason: `${NEMOCLAW_DGX_STATION_PEER_ENV} must name one SSH host or user@host`, + }; + } + const username = parts.length === 2 ? parts[0] : ""; + const hostname = parts.at(-1) ?? ""; + const validHostname = net.isIP(hostname) === 4 || CANONICAL_SSH_HOST_PATTERN.test(hostname); + if ( + !validHostname || + (username.length > 0 && !CANONICAL_SSH_USERNAME_PATTERN.test(username)) || + (parts.length === 2 && username.length === 0) + ) { + return { + ok: false, + reason: `${NEMOCLAW_DGX_STATION_PEER_ENV} must name one canonical SSH host or user@host`, + }; + } + return { ok: true, target: raw }; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function requireRecord(value: unknown, label: string): Record { + if (!isRecord(value)) throw new Error(`${label} must be an object`); + return value; +} + +function requireString(value: unknown, label: string, maxLength = 1024): string { + if ( + typeof value !== "string" || + value.length === 0 || + value.length > maxLength || + /[\u0000-\u001f\u007f]/.test(value) + ) { + throw new Error(`${label} must be a non-empty printable string`); + } + return value; +} + +function requireBoolean(value: unknown, label: string): boolean { + if (typeof value !== "boolean") throw new Error(`${label} must be a boolean`); + return value; +} + +function requireInteger(value: unknown, label: string, min: number, max: number): number { + if (!Number.isInteger(value) || (value as number) < min || (value as number) > max) { + throw new Error(`${label} must be an integer between ${String(min)} and ${String(max)}`); + } + return value as number; +} + +function requireArray(value: unknown, label: string, maxLength: number): unknown[] { + if (!Array.isArray(value) || value.length > maxLength) { + throw new Error(`${label} must be an array with at most ${String(maxLength)} entries`); + } + return value; +} + +function isSafeDeviceName(value: string): boolean { + return /^[A-Za-z0-9][A-Za-z0-9_.:-]{0,63}$/.test(value); +} + +function isIpv4(value: string): boolean { + return net.isIP(value) === 4; +} + +function normalizeMacAddress(value: unknown, label: string, allowEmpty = false): string { + if (allowEmpty && value === "") return ""; + const mac = requireString(value, label, 17).toLowerCase(); + if (!/^(?:[0-9a-f]{2}:){5}[0-9a-f]{2}$/.test(mac)) { + throw new Error(`${label} must be a canonical MAC address`); + } + return mac; +} + +function parseUverbsDevice(value: unknown, label: string): string { + if (value === "") return ""; + const device = requireString(value, label, 64); + if (!/^\/dev\/infiniband\/uverbs[0-9]+$/.test(device)) { + throw new Error(`${label} must be a safe /dev/infiniband/uverbs* character-device path`); + } + return device; +} + +function parseGpu(value: unknown, label: string): StationGpuProbe { + const record = requireRecord(value, label); + const name = requireString(record.name, `${label}.name`, 256); + const uuid = requireString(record.uuid, `${label}.uuid`, 128); + if (!/^GPU-[A-Za-z0-9-]+$/.test(uuid)) throw new Error(`${label}.uuid is invalid`); + return { + index: requireInteger(record.index, `${label}.index`, 0, 1024), + name, + uuid, + }; +} + +function parseIpv4Address(value: unknown, label: string): StationIpv4AddressProbe { + const record = requireRecord(value, label); + const address = requireString(record.address, `${label}.address`, 15); + if (!isIpv4(address)) throw new Error(`${label}.address must be IPv4`); + return { + address, + prefixLength: requireInteger(record.prefixLength, `${label}.prefixLength`, 1, 32), + }; +} + +function parseGid(value: unknown, label: string): StationRoceGidProbe { + const record = requireRecord(value, label); + const address = requireString(record.address, `${label}.address`, 15); + if (!isIpv4(address)) throw new Error(`${label}.address must be IPv4`); + return { + index: requireInteger(record.index, `${label}.index`, 0, 4095), + address, + }; +} + +function parseRail(value: unknown, label: string): StationRailProbe { + const record = requireRecord(value, label); + const rdmaDevice = requireString(record.rdmaDevice, `${label}.rdmaDevice`, 64); + const netdev = requireString(record.netdev, `${label}.netdev`, 64); + if (!isSafeDeviceName(rdmaDevice) || !isSafeDeviceName(netdev)) { + throw new Error(`${label} contains an unsafe device name`); + } + const pciAddress = requireString(record.pciAddress, `${label}.pciAddress`, 32); + if (!/^[0-9A-Fa-f]{4}:[0-9A-Fa-f]{2}:[0-9A-Fa-f]{2}\.[0-7]$/.test(pciAddress)) { + throw new Error(`${label}.pciAddress is invalid`); + } + return { + rdmaDevice, + port: requireInteger(record.port, `${label}.port`, 1, 255), + netdev, + macAddress: normalizeMacAddress(record.macAddress, `${label}.macAddress`), + uverbsDevice: parseUverbsDevice(record.uverbsDevice, `${label}.uverbsDevice`), + pciAddress, + pciName: requireString(record.pciName, `${label}.pciName`, 512), + state: requireString(record.state, `${label}.state`, 128), + linkLayer: requireString(record.linkLayer, `${label}.linkLayer`, 128), + speedMbps: requireInteger(record.speedMbps, `${label}.speedMbps`, -1, 1_000_000), + mtu: requireInteger(record.mtu, `${label}.mtu`, -1, 1_000_000), + ipv4Addresses: requireArray(record.ipv4Addresses, `${label}.ipv4Addresses`, 16).map( + (entry, index) => parseIpv4Address(entry, `${label}.ipv4Addresses[${String(index)}]`), + ), + roceV2Ipv4Gids: requireArray(record.roceV2Ipv4Gids, `${label}.roceV2Ipv4Gids`, 128).map( + (entry, index) => parseGid(entry, `${label}.roceV2Ipv4Gids[${String(index)}]`), + ), + }; +} + +function parseSnapshot(value: unknown, label: string): StationModelSnapshotProbe { + const record = requireRecord(value, label); + const snapshotPath = requireString(record.path, `${label}.path`, 4096); + if ( + !path.posix.isAbsolute(snapshotPath) || + path.posix.normalize(snapshotPath) !== snapshotPath || + snapshotPath.includes(":") + ) { + throw new Error(`${label}.path must be a normalized absolute POSIX path`); + } + return { + modelId: requireString(record.modelId, `${label}.modelId`, 512), + revision: requireString(record.revision, `${label}.revision`, 128), + path: snapshotPath, + directoryExists: requireBoolean(record.directoryExists, `${label}.directoryExists`), + complete: requireBoolean(record.complete, `${label}.complete`), + shardCount: requireInteger(record.shardCount, `${label}.shardCount`, 0, 100_000), + reason: typeof record.reason === "string" ? record.reason.slice(0, 1024) : "", + }; +} + +export function parseStationHostProbe(stdout: string): StationHostProbe { + if (Buffer.byteLength(stdout, "utf8") > MAX_PROBE_OUTPUT_BYTES) { + throw new Error("host probe output is too large"); + } + let parsed: unknown; + try { + parsed = JSON.parse(stdout); + } catch { + throw new Error("host probe did not return valid JSON"); + } + const record = requireRecord(parsed, "host probe"); + if (record.schemaVersion !== HOST_PROBE_SCHEMA_VERSION) { + throw new Error("host probe schema version is unsupported"); + } + const home = requireString(record.home, "host probe.home", 4096); + if (!path.posix.isAbsolute(home) || path.posix.normalize(home) !== home || home.includes(":")) { + throw new Error("host probe.home must be a normalized absolute POSIX path"); + } + const docker = requireRecord(record.docker, "host probe.docker"); + return { + schemaVersion: 1, + hostname: requireString(record.hostname, "host probe.hostname", 256), + productName: requireString(record.productName, "host probe.productName", 512), + architecture: requireString(record.architecture, "host probe.architecture", 64), + home, + uid: requireInteger(record.uid, "host probe.uid", 0, 2_147_483_647), + gpus: requireArray(record.gpus, "host probe.gpus", 64).map((entry, index) => + parseGpu(entry, `host probe.gpus[${String(index)}]`), + ), + docker: { + reachable: requireBoolean(docker.reachable, "host probe.docker.reachable"), + nvidiaRuntime: requireBoolean(docker.nvidiaRuntime, "host probe.docker.nvidiaRuntime"), + }, + rsyncAvailable: requireBoolean(record.rsyncAvailable, "host probe.rsyncAvailable"), + nvidiaPeermemLoaded: requireBoolean( + record.nvidiaPeermemLoaded, + "host probe.nvidiaPeermemLoaded", + ), + rails: requireArray(record.rails, "host probe.rails", 32).map((entry, index) => + parseRail(entry, `host probe.rails[${String(index)}]`), + ), + modelSnapshot: parseSnapshot(record.modelSnapshot, "host probe.modelSnapshot"), + }; +} + +function parseConnectivityProbe(stdout: string): StationRailConnectivityProbe[] { + if (Buffer.byteLength(stdout, "utf8") > MAX_PROBE_OUTPUT_BYTES) { + throw new Error("connectivity probe output is too large"); + } + let parsed: unknown; + try { + parsed = JSON.parse(stdout); + } catch { + throw new Error("connectivity probe did not return valid JSON"); + } + const record = requireRecord(parsed, "connectivity probe"); + if (record.schemaVersion !== CONNECTIVITY_PROBE_SCHEMA_VERSION) { + throw new Error("connectivity probe schema version is unsupported"); + } + return requireArray(record.checks, "connectivity probe.checks", 2).map((value, index) => { + const label = `connectivity probe.checks[${String(index)}]`; + const check = requireRecord(value, label); + const netdev = requireString(check.netdev, `${label}.netdev`, 64); + const sourceAddress = requireString(check.sourceAddress, `${label}.sourceAddress`, 15); + const peerAddress = requireString(check.peerAddress, `${label}.peerAddress`, 15); + const routeDevice = + check.routeDevice === "" ? "" : requireString(check.routeDevice, `${label}.routeDevice`, 64); + const routeSource = + check.routeSource === "" ? "" : requireString(check.routeSource, `${label}.routeSource`, 15); + const routeScope = + check.routeScope === "" ? "" : requireString(check.routeScope, `${label}.routeScope`, 32); + const peerNeighborState = + check.peerNeighborState === "" + ? "" + : requireString(check.peerNeighborState, `${label}.peerNeighborState`, 128); + if ( + !isSafeDeviceName(netdev) || + !isIpv4(sourceAddress) || + !isIpv4(peerAddress) || + (routeDevice !== "" && !isSafeDeviceName(routeDevice)) || + (routeSource !== "" && !isIpv4(routeSource)) + ) { + throw new Error(`${label} contains invalid route data`); + } + let routeGateway: string | null = null; + if (check.routeGateway !== null && check.routeGateway !== undefined) { + routeGateway = requireString(check.routeGateway, `${label}.routeGateway`, 15); + if (!isIpv4(routeGateway)) throw new Error(`${label}.routeGateway must be IPv4`); + } + return { + netdev, + sourceAddress, + peerAddress, + routeDevice, + routeSource, + routeGateway, + routeScope, + peerMac: normalizeMacAddress(check.peerMac, `${label}.peerMac`, true), + peerNeighborState, + jumboPing: requireBoolean(check.jumboPing, `${label}.jumboPing`), + }; + }); +} + +function commandSucceeded(result: StationProbeCommandResult): boolean { + return result.status === 0 && !result.error && result.stdout.trim().length > 0; +} + +function unavailable(code: StationClusterFailureCode, reason: string): PlanFailure { + return { kind: "unavailable", code, reason }; +} + +function dockerSshConfigIsStrict(result: StationProbeCommandResult): boolean { + if (!commandSucceeded(result)) return false; + const values = new Map(); + for (const rawLine of result.stdout.split(/\r?\n/)) { + const line = rawLine.trim(); + if (!line) continue; + const separator = line.search(/\s/); + if (separator <= 0) return false; + const key = line.slice(0, separator).toLowerCase(); + const value = line.slice(separator).trim().toLowerCase(); + values.set(key, [...(values.get(key) ?? []), value]); + } + const exactly = (key: string, allowed: readonly string[]): boolean => { + const observed = values.get(key) ?? []; + return observed.length === 1 && allowed.includes(observed[0]); + }; + const absentOrNone = (key: string): boolean => { + const observed = values.get(key) ?? []; + return observed.length === 0 || (observed.length === 1 && observed[0] === "none"); + }; + const sendEnv = values.get("sendenv") ?? []; + return ( + exactly("batchmode", ["yes"]) && + exactly("stricthostkeychecking", ["yes", "true"]) && + exactly("permitlocalcommand", ["no"]) && + exactly("forwardagent", ["no"]) && + exactly("forwardx11", ["no"]) && + exactly("forwardx11trusted", ["no"]) && + exactly("tunnel", ["false", "no"]) && + exactly("updatehostkeys", ["false", "no"]) && + exactly("controlmaster", ["false", "no"]) && + exactly("controlpersist", ["no", "0"]) && + absentOrNone("controlpath") && + absentOrNone("remotecommand") && + absentOrNone("proxycommand") && + absentOrNone("proxyjump") && + absentOrNone("localcommand") && + !values.has("localforward") && + !values.has("remoteforward") && + !values.has("dynamicforward") && + !values.has("knownhostscommand") && + !values.has("setenv") && + sendEnv.every((value) => value === "lang" || value === "lc_*") + ); +} + +function selectGb300(host: StationHostProbe): StationGpuProbe | null { + const matches = host.gpus.filter((gpu) => /\bGB300\b/i.test(gpu.name)); + return matches.length === 1 ? matches[0] : null; +} + +function isDgxStationProduct(productName: string): boolean { + return ( + /(? /ConnectX[- ]?8|\bCX-?8\b/i.test(rail.pciName)); + if (cx8Rails.length !== 2) return null; + const uniqueRdma = new Set(cx8Rails.map((rail) => rail.rdmaDevice)); + const uniqueNetdev = new Set(cx8Rails.map((rail) => rail.netdev)); + const uniquePci = new Set(cx8Rails.map((rail) => rail.pciAddress)); + const uniqueMac = new Set(cx8Rails.map((rail) => rail.macAddress)); + const uniqueUverbs = new Set(cx8Rails.map((rail) => rail.uverbsDevice)); + if ( + uniqueRdma.size !== 2 || + uniqueNetdev.size !== 2 || + uniquePci.size !== 2 || + uniqueMac.size !== 2 || + uniqueUverbs.size !== 2 || + uniqueUverbs.has("") + ) { + return null; + } + for (const rail of cx8Rails) { + const firstMacOctet = Number.parseInt(rail.macAddress.slice(0, 2), 16); + if ( + rail.macAddress === "00:00:00:00:00:00" || + (firstMacOctet & 1) !== 0 || + rail.port !== 1 || + !/\bACTIVE\b/i.test(rail.state) || + rail.linkLayer.toLowerCase() !== "ethernet" || + rail.speedMbps !== 400_000 || + rail.mtu !== 9000 || + rail.ipv4Addresses.length === 0 || + rail.roceV2Ipv4Gids.length === 0 + ) { + return null; + } + } + return cx8Rails; +} + +function ipv4ToNumber(address: string): number { + return address + .split(".") + .map(Number) + .reduce((value, octet) => value * 256 + octet, 0); +} + +function numberToIpv4(value: number): string { + return [24, 16, 8, 0].map((shift) => Math.floor(value / 2 ** shift) % 256).join("."); +} + +function subnetKey(address: StationIpv4AddressProbe): string { + const hostBits = 32 - address.prefixLength; + const divisor = 2 ** hostBits; + const network = Math.floor(ipv4ToNumber(address.address) / divisor) * divisor; + return `${numberToIpv4(network)}/${String(address.prefixLength)}`; +} + +function isPrivateFabricIpv4(address: string): boolean { + const value = ipv4ToNumber(address); + return ( + (value >= ipv4ToNumber("10.0.0.0") && value <= ipv4ToNumber("10.255.255.255")) || + (value >= ipv4ToNumber("172.16.0.0") && value <= ipv4ToNumber("172.31.255.255")) || + (value >= ipv4ToNumber("192.168.0.0") && value <= ipv4ToNumber("192.168.255.255")) + ); +} + +function sharedAddressPairs( + localRail: StationRailProbe, + peerRail: StationRailProbe, +): Array<{ + localAddress: StationIpv4AddressProbe; + peerAddress: StationIpv4AddressProbe; + subnet: string; +}> { + const matches: Array<{ + localAddress: StationIpv4AddressProbe; + peerAddress: StationIpv4AddressProbe; + subnet: string; + }> = []; + for (const localAddress of localRail.ipv4Addresses) { + for (const peerAddress of peerRail.ipv4Addresses) { + const localSubnet = subnetKey(localAddress); + if ( + localAddress.address !== peerAddress.address && + localAddress.prefixLength === DIRECT_RAIL_PREFIX_LENGTH && + peerAddress.prefixLength === DIRECT_RAIL_PREFIX_LENGTH && + isPrivateFabricIpv4(localAddress.address) && + isPrivateFabricIpv4(peerAddress.address) && + localSubnet === subnetKey(peerAddress) + ) { + matches.push({ localAddress, peerAddress, subnet: localSubnet }); + } + } + } + return matches; +} + +function matchRails( + localRails: readonly StationRailProbe[], + peerRails: readonly StationRailProbe[], +): MatchedRail[] | null { + const permutations = [ + [0, 1], + [1, 0], + ] as const; + const candidates: MatchedRail[][] = []; + for (const permutation of permutations) { + const candidate: MatchedRail[] = []; + let valid = true; + for (let index = 0; index < 2; index += 1) { + const localRail = localRails[index]; + const peerRail = peerRails[permutation[index]]; + const addressMatches = sharedAddressPairs(localRail, peerRail); + if (addressMatches.length !== 1) { + valid = false; + break; + } + candidate.push({ localRail, peerRail, ...addressMatches[0] }); + } + if (valid && new Set(candidate.map((match) => match.subnet)).size === 2) { + candidates.push(candidate); + } + } + if (candidates.length !== 1) return null; + return candidates[0].sort((left, right) => + left.localRail.rdmaDevice.localeCompare(right.localRail.rdmaDevice, undefined, { + numeric: true, + }), + ); +} + +function gidsForMatchedAddress(rail: StationRailProbe, address: string): Set { + return new Set( + rail.roceV2Ipv4Gids.filter((gid) => gid.address === address).map((gid) => gid.index), + ); +} + +function commonGidIndex(matches: readonly MatchedRail[]): number | null { + const sets = matches.flatMap((match) => [ + gidsForMatchedAddress(match.localRail, match.localAddress.address), + gidsForMatchedAddress(match.peerRail, match.peerAddress.address), + ]); + if (sets.some((set) => set.size === 0)) return null; + const common = [...sets[0]].filter((index) => sets.slice(1).every((set) => set.has(index))); + if (common.includes(PREFERRED_ROCE_GID_INDEX)) return PREFERRED_ROCE_GID_INDEX; + return common.sort((left, right) => left - right)[0] ?? null; +} + +function expectedPeerSnapshotPath(peer: StationHostProbe): string { + return path.posix.join( + peer.home, + ".cache", + "huggingface", + "hub", + `models--${DUAL_STATION_VLLM_RUNTIME.modelId.replace("/", "--")}`, + "snapshots", + DUAL_STATION_VLLM_RUNTIME.modelRevision, + ); +} + +function buildStaticPlan( + peerTarget: string, + local: StationHostProbe, + peer: StationHostProbe, +): StaticPlan | PlanFailure { + if (!isDgxStationProduct(local.productName) || !/^(?:aarch64|arm64)$/i.test(local.architecture)) { + return unavailable("local-not-station", "local host is not a verified arm64 DGX Station"); + } + if (!isDgxStationProduct(peer.productName) || !/^(?:aarch64|arm64)$/i.test(peer.architecture)) { + return unavailable("peer-not-station", "configured peer is not a verified arm64 DGX Station"); + } + + const localGpu = selectGb300(local); + if (!localGpu) { + return unavailable("local-gpu-unavailable", "local host must expose exactly one GB300 GPU"); + } + const peerGpu = selectGb300(peer); + if (!peerGpu) { + return unavailable("peer-gpu-unavailable", "configured peer must expose exactly one GB300 GPU"); + } + if (localGpu.uuid === peerGpu.uuid) { + return unavailable( + "fabric-mismatch", + "configured peer resolved to the local Station instead of a distinct host", + ); + } + if (!local.docker.reachable || !local.docker.nvidiaRuntime) { + return unavailable( + "local-docker-unavailable", + "local Docker daemon and NVIDIA runtime could not both be verified", + ); + } + if (!peer.docker.reachable || !peer.docker.nvidiaRuntime) { + return unavailable( + "peer-docker-unavailable", + "peer Docker daemon and NVIDIA runtime could not both be verified", + ); + } + if (!local.nvidiaPeermemLoaded) { + return unavailable("local-fabric-unavailable", "nvidia_peermem is not loaded locally"); + } + if (!peer.nvidiaPeermemLoaded) { + return unavailable("peer-fabric-unavailable", "nvidia_peermem is not loaded on the peer"); + } + + const localRails = qualifiedRails(local); + if (!localRails) { + return unavailable( + "local-fabric-unavailable", + "local host does not have two active Ethernet 400G MTU-9000 CX8 RDMA rails", + ); + } + const peerRails = qualifiedRails(peer); + if (!peerRails) { + return unavailable( + "peer-fabric-unavailable", + "peer does not have two active Ethernet 400G MTU-9000 CX8 RDMA rails", + ); + } + + const snapshot = peer.modelSnapshot; + if ( + snapshot.modelId !== DUAL_STATION_VLLM_RUNTIME.modelId || + snapshot.revision !== DUAL_STATION_VLLM_RUNTIME.modelRevision || + snapshot.path !== expectedPeerSnapshotPath(peer) + ) { + return unavailable( + "peer-model-cache-unavailable", + "peer reported an unexpected Nemotron Ultra snapshot identity or path", + ); + } + const peerSnapshotReady = + snapshot.directoryExists && + snapshot.complete && + snapshot.shardCount === EXPECTED_ULTRA_WEIGHT_SHARDS; + if (!peerSnapshotReady && snapshot.directoryExists) { + return unavailable( + "peer-model-cache-unavailable", + "peer has an incomplete pinned Nemotron Ultra snapshot; refusing to overwrite it", + ); + } + if (!peerSnapshotReady && !local.rsyncAvailable) { + return unavailable( + "local-model-staging-unavailable", + "local rsync is required to stage the pinned Nemotron Ultra snapshot", + ); + } + if (!peerSnapshotReady && !peer.rsyncAvailable) { + return unavailable( + "peer-model-staging-unavailable", + "peer rsync is required to stage the pinned Nemotron Ultra snapshot", + ); + } + + const matches = matchRails(localRails, peerRails); + if (!matches) { + return unavailable( + "fabric-mismatch", + "the two hosts do not expose one unambiguous pair of distinct private /30 CX8 subnets", + ); + } + const gidIndex = commonGidIndex(matches); + if (gidIndex === null) { + return unavailable( + "gid-mismatch", + "the four matched rail endpoints do not share a RoCEv2 IPv4 GID index", + ); + } + + const rails = matches.map( + (match, index): DualStationPlanRail => ({ + index, + subnet: match.subnet, + local: { + rdmaDevice: match.localRail.rdmaDevice, + netdev: match.localRail.netdev, + macAddress: match.localRail.macAddress, + uverbsDevice: match.localRail.uverbsDevice, + pciAddress: match.localRail.pciAddress, + address: match.localAddress.address, + }, + peer: { + rdmaDevice: match.peerRail.rdmaDevice, + netdev: match.peerRail.netdev, + macAddress: match.peerRail.macAddress, + uverbsDevice: match.peerRail.uverbsDevice, + pciAddress: match.peerRail.pciAddress, + address: match.peerAddress.address, + }, + }), + ); + + return { + plan: { + peerSshTarget: peerTarget, + peerDockerHost: `ssh://${peerTarget}`, + runtime: DUAL_STATION_VLLM_RUNTIME, + local: { + hostname: local.hostname, + home: local.home, + uid: local.uid, + gpu: localGpu, + }, + peer: { + hostname: peer.hostname, + home: peer.home, + uid: peer.uid, + gpu: peerGpu, + }, + rails, + masterAddress: rails[0].local.address, + roceGidIndex: gidIndex, + }, + peerModelSnapshot: peerSnapshotReady ? "ready" : "staging-required", + localConnectivity: rails.map((rail) => ({ + netdev: rail.local.netdev, + sourceAddress: rail.local.address, + peerAddress: rail.peer.address, + expectedPeerMac: rail.peer.macAddress, + })), + peerConnectivity: rails.map((rail) => ({ + netdev: rail.peer.netdev, + sourceAddress: rail.peer.address, + peerAddress: rail.local.address, + expectedPeerMac: rail.local.macAddress, + })), + }; +} + +function connectivityMatches( + requests: readonly StationRailConnectivityRequest[], + observed: readonly StationRailConnectivityProbe[], +): boolean { + if (observed.length !== requests.length) return false; + const byKey = new Map( + observed.map((check) => [`${check.netdev}|${check.sourceAddress}|${check.peerAddress}`, check]), + ); + if (byKey.size !== observed.length) return false; + return requests.every((request) => { + const check = byKey.get(`${request.netdev}|${request.sourceAddress}|${request.peerAddress}`); + return Boolean( + check && + check.routeDevice === request.netdev && + check.routeSource === request.sourceAddress && + check.routeGateway === null && + check.routeScope.toLowerCase() === "link" && + check.peerMac === request.expectedPeerMac && + /^(?:REACHABLE|STALE|DELAY|PROBE|PERMANENT|NOARP)(?:,(?:REACHABLE|STALE|DELAY|PROBE|PERMANENT|NOARP))*$/i.test( + check.peerNeighborState, + ) && + check.jumboPing, + ); + }); +} + +function parseHostCommand( + result: StationProbeCommandResult, + code: "local-probe-failed" | "peer-probe-failed", +): StationHostProbe | PlanFailure { + if (!commandSucceeded(result)) { + return unavailable( + code, + code === "local-probe-failed" ? "local host probe failed" : "peer host probe failed", + ); + } + try { + return parseStationHostProbe(result.stdout.trim()); + } catch { + return unavailable( + code, + code === "local-probe-failed" + ? "local host probe returned invalid data" + : "peer host probe returned invalid data", + ); + } +} + +export interface ProbeDualStationVllmOptions { + env?: NodeJS.ProcessEnv; + deps?: StationClusterProbeDeps; +} + +/** + * Read-only, fail-closed capability probe for the explicit two-Station path. + * + * An unset/blank NEMOCLAW_DGX_STATION_PEER returns before touching deps. A + * configured peer is contacted exactly by that pretrusted SSH destination; + * the probe never discovers hosts, changes known_hosts, prompts, or mutates + * either machine. + */ +export function probeDualStationVllmCapability( + options: ProbeDualStationVllmOptions = {}, +): StationClusterCapability { + const env = options.env ?? process.env; + const rawPeer = env[NEMOCLAW_DGX_STATION_PEER_ENV]; + if (rawPeer === undefined || rawPeer === "" || rawPeer.trim() === "") { + return { kind: "not-configured" }; + } + const peerValidation = validatePeerTarget(rawPeer); + if (!peerValidation.ok) return unavailable("invalid-peer", peerValidation.reason); + const localDockerOverride = DUAL_STATION_LOCAL_DOCKER_OVERRIDE_ENV_NAMES.find( + (name) => env[name] !== undefined && String(env[name]).trim() !== "", + ); + if (localDockerOverride) { + return unavailable( + "local-docker-unavailable", + `${localDockerOverride} must be unset so dual-Station setup can bind the physical local Docker daemon`, + ); + } + + const deps = options.deps ?? defaultStationClusterProbeDeps; + try { + const sshConfig = deps.probePeerSshConfig(peerValidation.target); + if (!dockerSshConfigIsStrict(sshConfig)) { + return unavailable( + "peer-ssh-config-unsafe", + "configured peer SSH options are not safe for Docker transport; require BatchMode=yes, StrictHostKeyChecking=yes, no forwarding/proxy/local commands, and no connection sharing", + ); + } + const localResult = parseHostCommand(deps.probeLocalHost(), "local-probe-failed"); + if ("kind" in localResult) return localResult; + const peerResult = parseHostCommand( + deps.probePeerHost(peerValidation.target), + "peer-probe-failed", + ); + if ("kind" in peerResult) return peerResult; + + const staticPlan = buildStaticPlan(peerValidation.target, localResult, peerResult); + if ("kind" in staticPlan) return staticPlan; + + const localConnectivityResult = deps.probeLocalConnectivity(staticPlan.localConnectivity); + if (!commandSucceeded(localConnectivityResult)) { + return unavailable( + "local-connectivity-failed", + "local dual-rail route and jumbo-frame probe failed", + ); + } + let localConnectivity: StationRailConnectivityProbe[]; + try { + localConnectivity = parseConnectivityProbe(localConnectivityResult.stdout.trim()); + } catch { + return unavailable( + "local-connectivity-failed", + "local dual-rail route probe returned invalid data", + ); + } + if (!connectivityMatches(staticPlan.localConnectivity, localConnectivity)) { + return unavailable( + "local-connectivity-failed", + "local routes are not direct on both matched rails or jumbo ping failed", + ); + } + + const peerConnectivityResult = deps.probePeerConnectivity( + peerValidation.target, + staticPlan.peerConnectivity, + ); + if (!commandSucceeded(peerConnectivityResult)) { + return unavailable( + "peer-connectivity-failed", + "peer dual-rail route and jumbo-frame probe failed", + ); + } + let peerConnectivity: StationRailConnectivityProbe[]; + try { + peerConnectivity = parseConnectivityProbe(peerConnectivityResult.stdout.trim()); + } catch { + return unavailable( + "peer-connectivity-failed", + "peer dual-rail route probe returned invalid data", + ); + } + if (!connectivityMatches(staticPlan.peerConnectivity, peerConnectivity)) { + return unavailable( + "peer-connectivity-failed", + "peer routes are not direct on both matched rails or jumbo ping failed", + ); + } + + return { + kind: "ready", + plan: staticPlan.plan, + peerModelSnapshot: staticPlan.peerModelSnapshot, + }; + } catch { + return unavailable("probe-error", "dual-Station capability probe failed closed"); + } +} diff --git a/src/lib/inference/vllm-station-lifecycle-lock.ts b/src/lib/inference/vllm-station-lifecycle-lock.ts new file mode 100644 index 0000000000..c1e7b856c4 --- /dev/null +++ b/src/lib/inference/vllm-station-lifecycle-lock.ts @@ -0,0 +1,27 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import path from "node:path"; + +import { type McpLifecycleLockOptions, withMcpLifecycleLock } from "../state/mcp-lifecycle-lock"; +import { resolveHome, STATE_DIR_NAME } from "../state/state-root"; + +const DUAL_STATION_VLLM_LIFECYCLE_LOCK = "dual-station-vllm:host-global"; + +/** + * Serialize the host-managed dual-Station service across gateway instances. + * + * This deliberately anchors the lease at the default ~/.nemoclaw root instead + * of a gateway-specific state root: every local NemoClaw process controls the + * same two fixed Docker container names on the same pair of daemons. + */ +export function withDualStationVllmLifecycleLock( + operation: () => Promise | T, + options: McpLifecycleLockOptions = {}, +): Promise { + const stateDir = options.stateDir ?? path.join(resolveHome(), STATE_DIR_NAME, "state"); + return withMcpLifecycleLock(DUAL_STATION_VLLM_LIFECYCLE_LOCK, operation, { + ...options, + stateDir, + }); +} diff --git a/src/lib/inference/vllm-station-model-staging.test.ts b/src/lib/inference/vllm-station-model-staging.test.ts new file mode 100644 index 0000000000..e6a482dcde --- /dev/null +++ b/src/lib/inference/vllm-station-model-staging.test.ts @@ -0,0 +1,258 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { DUAL_STATION_VLLM_RUNTIME, type DualStationVllmPlan } from "./vllm-station-cluster"; +import { + type ModelStagingCommandResult, + stageDualStationModelSnapshot, +} from "./vllm-station-model-staging"; + +function plan(): DualStationVllmPlan { + return { + peerSshTarget: "nvidia@station-b", + peerDockerHost: "ssh://nvidia@station-b", + runtime: DUAL_STATION_VLLM_RUNTIME, + local: { + hostname: "station-a", + home: "/home/nvidia", + uid: 1000, + gpu: { index: 0, name: "NVIDIA GB300", uuid: "GPU-a" }, + }, + peer: { + hostname: "station-b", + home: "/home/nvidia", + uid: 1000, + gpu: { index: 0, name: "NVIDIA GB300", uuid: "GPU-b" }, + }, + rails: [ + { + index: 0, + subnet: "192.168.100.0/30", + local: { + rdmaDevice: "mlx5_0", + netdev: "cx8a0", + macAddress: "02:00:00:00:00:01", + uverbsDevice: "/dev/infiniband/uverbs0", + pciAddress: "0000:01:00.0", + address: "192.168.100.1", + }, + peer: { + rdmaDevice: "mlx5_0", + netdev: "cx8b0", + macAddress: "02:00:00:00:00:02", + uverbsDevice: "/dev/infiniband/uverbs0", + pciAddress: "0000:01:00.0", + address: "192.168.100.2", + }, + }, + { + index: 1, + subnet: "192.168.200.0/30", + local: { + rdmaDevice: "mlx5_1", + netdev: "cx8a1", + macAddress: "02:00:00:00:01:01", + uverbsDevice: "/dev/infiniband/uverbs1", + pciAddress: "0000:01:00.1", + address: "192.168.200.1", + }, + peer: { + rdmaDevice: "mlx5_1", + netdev: "cx8b1", + macAddress: "02:00:00:00:01:02", + uverbsDevice: "/dev/infiniband/uverbs1", + pciAddress: "0000:01:00.1", + address: "192.168.200.2", + }, + }, + ], + masterAddress: "192.168.100.1", + roceGidIndex: 3, + }; +} + +function result(stdout = "", status = 0): ModelStagingCommandResult { + return { status, stdout, stderr: "" }; +} + +function manifest(): string { + return JSON.stringify({ + schemaVersion: 1, + files: [{ path: "config.json", size: 2, sha256: "a".repeat(64) }], + directories: [], + totalBytes: 2, + }); +} + +function successfulTransferRunner() { + return vi + .fn() + .mockResolvedValueOnce(result(manifest())) + .mockResolvedValueOnce(result('{"state":"transfer"}')) + .mockResolvedValueOnce(result()) + .mockResolvedValueOnce(result('{"state":"ready"}')); +} + +function stagingSuffix(runCommand: ReturnType): string { + const destination = String(runCommand.mock.calls[2][1].at(-1)); + return destination.match(/\.nemoclaw-staging-[a-f0-9]{32}/)?.[0] ?? ""; +} + +afterEach(() => { + vi.unstubAllEnvs(); +}); + +describe("dual-Station pinned model staging", () => { + it("copies only the audited snapshot through strict SSH and verifies it before install", async () => { + vi.stubEnv("OPENSHELL_GATEWAY_AUTH_TOKEN", "must-not-cross-ssh"); + vi.stubEnv("HF_TOKEN", "must-not-cross-ssh"); + const runCommand = successfulTransferRunner(); + + await expect(stageDualStationModelSnapshot(plan(), { runCommand })).resolves.toEqual({ + ok: true, + transferred: true, + }); + + expect(runCommand).toHaveBeenCalledTimes(4); + expect(runCommand.mock.calls.map((call) => call[0])).toEqual([ + "python3", + "ssh", + "rsync", + "ssh", + ]); + const localArgs = runCommand.mock.calls[0][1] as string[]; + expect(localArgs).toEqual([ + "-", + `/home/nvidia/.cache/huggingface/hub/models--${DUAL_STATION_VLLM_RUNTIME.modelId.replace("/", "--")}/snapshots/${DUAL_STATION_VLLM_RUNTIME.modelRevision}`, + `/home/nvidia/.cache/huggingface/hub/models--${DUAL_STATION_VLLM_RUNTIME.modelId.replace("/", "--")}`, + ]); + expect(runCommand.mock.calls[0][2].input).toContain("snapshot symlink escapes"); + expect(runCommand.mock.calls[0][2].input).toContain("len(shards) != 113"); + + const sshArgs = runCommand.mock.calls[1][1] as string[]; + expect(sshArgs).toEqual( + expect.arrayContaining([ + "BatchMode=yes", + "StrictHostKeyChecking=yes", + "ClearAllForwardings=yes", + "ControlMaster=no", + "--", + "nvidia@station-b", + "python3 -", + ]), + ); + expect(runCommand.mock.calls[1][2].input).toContain("peer pinned snapshot already exists"); + expect(runCommand.mock.calls[1][2].input).toContain("shutil.disk_usage(STAGING.parent).free"); + expect(runCommand.mock.calls[3][2].input).toContain("os.rename(STAGING, FINAL)"); + expect(runCommand.mock.calls[3][2].input).toContain("installed_identity != staged_identity"); + + const rsyncArgs = runCommand.mock.calls[2][1] as string[]; + expect(rsyncArgs).toEqual( + expect.arrayContaining(["--copy-links", "--checksum", "--partial", "--protect-args", "--"]), + ); + expect(rsyncArgs).not.toContain("--delete"); + expect(rsyncArgs.at(-1)).toMatch( + new RegExp( + `^nvidia@station-b:/home/nvidia/\\.cache/huggingface/hub/models--${DUAL_STATION_VLLM_RUNTIME.modelId.replace("/", "--")}/snapshots/\\.nemoclaw-staging-[a-f0-9]{32}/$`, + ), + ); + expect(rsyncArgs.join(" ")).toContain("StrictHostKeyChecking=yes"); + expect(runCommand.mock.calls[2][2]).toMatchObject({ + idleTimeoutMs: 30 * 60 * 1000, + streamOutput: true, + }); + expect(runCommand.mock.calls[2][2].env.OPENSHELL_GATEWAY_AUTH_TOKEN).toBeUndefined(); + expect(runCommand.mock.calls[2][2].env.HF_TOKEN).toBeUndefined(); + }); + + it("does no transfer when the peer already has the exact byte manifest", async () => { + const runCommand = vi + .fn() + .mockResolvedValueOnce(result(manifest())) + .mockResolvedValueOnce(result('{"state":"ready"}')); + + await expect(stageDualStationModelSnapshot(plan(), { runCommand })).resolves.toEqual({ + ok: true, + transferred: false, + }); + expect(runCommand).toHaveBeenCalledTimes(2); + expect(runCommand).not.toHaveBeenCalledWith("rsync", expect.anything(), expect.anything()); + }); + + it("gives concurrent reversed and different local heads disjoint retry-safe staging paths", async () => { + const original = plan(); + const reversed = plan(); + const reversedLocal = reversed.local; + reversed.local = reversed.peer; + reversed.peer = reversedLocal; + reversed.peerSshTarget = "nvidia@station-a"; + reversed.peerDockerHost = "ssh://nvidia@station-a"; + const differentHead = plan(); + differentHead.local.gpu.uuid = "GPU-c"; + const originalRunner = successfulTransferRunner(); + const reversedRunner = successfulTransferRunner(); + const differentHeadRunner = successfulTransferRunner(); + + await Promise.all([ + stageDualStationModelSnapshot(original, { runCommand: originalRunner }), + stageDualStationModelSnapshot(reversed, { runCommand: reversedRunner }), + stageDualStationModelSnapshot(differentHead, { runCommand: differentHeadRunner }), + ]); + + const suffixes = [ + stagingSuffix(originalRunner), + stagingSuffix(reversedRunner), + stagingSuffix(differentHeadRunner), + ]; + expect(suffixes.every((suffix) => /^\.nemoclaw-staging-[a-f0-9]{32}$/.test(suffix))).toBe(true); + expect(new Set(suffixes).size).toBe(3); + }); + + it("reuses the same deterministic partial path for the same ordered pair", async () => { + const firstRunner = successfulTransferRunner(); + const retryRunner = successfulTransferRunner(); + + await stageDualStationModelSnapshot(plan(), { runCommand: firstRunner }); + await stageDualStationModelSnapshot(plan(), { runCommand: retryRunner }); + + expect(stagingSuffix(firstRunner)).toBe(stagingSuffix(retryRunner)); + }); + + it("leaves the private staging tree for a safe retry when rsync fails", async () => { + const runCommand = vi + .fn() + .mockResolvedValueOnce(result(manifest())) + .mockResolvedValueOnce(result('{"state":"transfer"}')) + .mockResolvedValueOnce({ status: 23, stdout: "", stderr: "partial transfer" }); + + await expect(stageDualStationModelSnapshot(plan(), { runCommand })).resolves.toEqual({ + ok: false, + reason: "peer snapshot transfer failed: partial transfer", + }); + expect(runCommand).toHaveBeenCalledTimes(3); + }); + + it("rejects a peer home that cannot be represented without remote shell syntax", async () => { + const unsafe = plan(); + unsafe.peer.home = "/home/nvidia;touch-pwned"; + const runCommand = vi.fn(); + + await expect(stageDualStationModelSnapshot(unsafe, { runCommand })).resolves.toEqual({ + ok: false, + reason: "peer home is unsafe for model staging", + }); + expect(runCommand).not.toHaveBeenCalled(); + }); + + it("fails before SSH when the local manifest is malformed", async () => { + const runCommand = vi.fn().mockResolvedValueOnce(result('{"schemaVersion":1}')); + + await expect(stageDualStationModelSnapshot(plan(), { runCommand })).resolves.toEqual({ + ok: false, + reason: "local pinned snapshot audit returned an invalid manifest", + }); + expect(runCommand).toHaveBeenCalledTimes(1); + }); +}); diff --git a/src/lib/inference/vllm-station-model-staging.ts b/src/lib/inference/vllm-station-model-staging.ts new file mode 100644 index 0000000000..c3e9511cd8 --- /dev/null +++ b/src/lib/inference/vllm-station-model-staging.ts @@ -0,0 +1,674 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { type ChildProcess, spawn } from "node:child_process"; +import { createHash } from "node:crypto"; +import path from "node:path"; + +import { buildVllmSshTransportEnv } from "./vllm-docker-env"; +import { + DUAL_STATION_VLLM_RUNTIME, + type DualStationVllmPlan, + strictStationSshTransportArgs, + validatePeerTarget, +} from "./vllm-station-cluster"; + +const MANIFEST_SCHEMA_VERSION = 1; +const MAX_MANIFEST_BYTES = 2 * 1024 * 1024; +const MAX_COMMAND_OUTPUT_BYTES = 256 * 1024; +const SNAPSHOT_AUDIT_TIMEOUT_MS = 6 * 60 * 60 * 1000; +const RSYNC_TIMEOUT_MS = 12 * 60 * 60 * 1000; +const RSYNC_IDLE_TIMEOUT_MS = 30 * 60 * 1000; +const SAFE_POSIX_PATH_PATTERN = /^\/(?:[A-Za-z0-9._-]+\/)*[A-Za-z0-9._-]+$/; + +export interface ModelStagingCommandOptions { + env: Record; + input?: string; + timeoutMs: number; + idleTimeoutMs?: number; + streamOutput?: boolean; +} + +export interface ModelStagingCommandResult { + status: number | null; + stdout: string; + stderr: string; + error?: string; + timedOut?: boolean; +} + +export interface DualStationModelStagingDeps { + runCommand( + file: string, + args: readonly string[], + options: ModelStagingCommandOptions, + ): Promise; +} + +export type DualStationModelStagingResult = + | { ok: true; transferred: boolean } + | { ok: false; reason: string }; + +interface SnapshotManifest { + schemaVersion: 1; + files: Array<{ path: string; size: number; sha256: string }>; + directories: string[]; + totalBytes: number; +} + +interface StagingPaths { + localModelRoot: string; + localSnapshot: string; + peerSnapshot: string; + peerStaging: string; +} + +function appendBounded(current: string, chunk: string, limit: number): string { + const next = current + chunk; + return next.length <= limit ? next : next.slice(next.length - limit); +} + +function defaultRunCommand( + file: string, + args: readonly string[], + options: ModelStagingCommandOptions, +): Promise { + return new Promise((resolve) => { + let child: ChildProcess; + try { + child = spawn(file, [...args], { + env: options.env, + shell: false, + stdio: [options.input === undefined ? "ignore" : "pipe", "pipe", "pipe"], + windowsHide: true, + }); + } catch (err) { + resolve({ status: null, stdout: "", stderr: "", error: (err as Error).message }); + return; + } + + let stdout = ""; + let stderr = ""; + let settled = false; + let timedOut = false; + let idleTimer: NodeJS.Timeout | undefined; + const absoluteTimer = setTimeout(() => { + timedOut = true; + child.kill("SIGKILL"); + }, options.timeoutMs); + absoluteTimer.unref?.(); + + const resetIdleTimer = (): void => { + if (idleTimer) clearTimeout(idleTimer); + if (options.idleTimeoutMs === undefined) return; + idleTimer = setTimeout(() => { + timedOut = true; + child.kill("SIGKILL"); + }, options.idleTimeoutMs); + idleTimer.unref?.(); + }; + resetIdleTimer(); + + child.stdout?.on("data", (value: Buffer | string) => { + const chunk = String(value); + stdout = appendBounded(stdout, chunk, MAX_MANIFEST_BYTES); + resetIdleTimer(); + if (options.streamOutput) process.stdout.write(chunk); + }); + child.stderr?.on("data", (value: Buffer | string) => { + const chunk = String(value); + stderr = appendBounded(stderr, chunk, MAX_COMMAND_OUTPUT_BYTES); + resetIdleTimer(); + if (options.streamOutput) process.stderr.write(chunk); + }); + child.once("error", (err) => { + if (settled) return; + settled = true; + clearTimeout(absoluteTimer); + if (idleTimer) clearTimeout(idleTimer); + resolve({ status: null, stdout, stderr, error: err.message, timedOut }); + }); + child.once("close", (status) => { + if (settled) return; + settled = true; + clearTimeout(absoluteTimer); + if (idleTimer) clearTimeout(idleTimer); + resolve({ status, stdout, stderr, timedOut }); + }); + child.stdin?.on("error", () => undefined); + child.stdin?.end(options.input); + }); +} + +const defaultDeps: DualStationModelStagingDeps = { runCommand: defaultRunCommand }; + +function snapshotPath(home: string): string { + return path.posix.join( + home, + ".cache", + "huggingface", + "hub", + `models--${DUAL_STATION_VLLM_RUNTIME.modelId.replace("/", "--")}`, + "snapshots", + DUAL_STATION_VLLM_RUNTIME.modelRevision, + ); +} + +function stagingTransactionId(plan: DualStationVllmPlan): string { + const identity = [ + "nemoclaw-dual-station-model-staging-v1", + DUAL_STATION_VLLM_RUNTIME.image, + DUAL_STATION_VLLM_RUNTIME.modelId, + DUAL_STATION_VLLM_RUNTIME.modelRevision, + DUAL_STATION_VLLM_RUNTIME.servedModelId, + String(DUAL_STATION_VLLM_RUNTIME.tensorParallelSize), + String(DUAL_STATION_VLLM_RUNTIME.nodeCount), + plan.local.gpu.uuid, + plan.peer.gpu.uuid, + ].join("\0"); + return createHash("sha256").update(identity, "utf8").digest("hex").slice(0, 32); +} + +function stagingPaths(plan: DualStationVllmPlan): StagingPaths { + for (const [label, home] of [ + ["local", plan.local.home], + ["peer", plan.peer.home], + ] as const) { + if ( + !SAFE_POSIX_PATH_PATTERN.test(home) || + path.posix.normalize(home) !== home || + home.split("/").some((component) => component === "." || component === "..") + ) { + throw new Error(`${label} home is unsafe for model staging`); + } + } + const validatedPeer = validatePeerTarget(plan.peerSshTarget); + if (!validatedPeer.ok || validatedPeer.target !== plan.peerSshTarget) { + throw new Error("peer SSH target is unsafe for model staging"); + } + if ( + plan.runtime.modelId !== DUAL_STATION_VLLM_RUNTIME.modelId || + plan.runtime.modelRevision !== DUAL_STATION_VLLM_RUNTIME.modelRevision || + plan.runtime.image !== DUAL_STATION_VLLM_RUNTIME.image + ) { + throw new Error("dual-Station plan does not identify the pinned runtime"); + } + + const localSnapshot = snapshotPath(plan.local.home); + const peerSnapshot = snapshotPath(plan.peer.home); + return { + localModelRoot: path.posix.dirname(path.posix.dirname(localSnapshot)), + localSnapshot, + peerSnapshot, + peerStaging: path.posix.join( + path.posix.dirname(peerSnapshot), + `.nemoclaw-staging-${stagingTransactionId(plan)}`, + ), + }; +} + +const LOCAL_MANIFEST_SCRIPT = String.raw` +import hashlib +import json +import os +from pathlib import Path +import re +import stat +import sys + +if len(sys.argv) != 3: + raise SystemExit("expected snapshot and model-root paths") + +snapshot = Path(sys.argv[1]) +model_root = Path(sys.argv[2]) +safe_component = re.compile(r"^[A-Za-z0-9._-]+$") + +def relative_name(candidate): + relative = candidate.relative_to(snapshot) + if not relative.parts or any(not safe_component.fullmatch(part) for part in relative.parts): + raise SystemExit("snapshot contains an unsafe relative path") + return relative.as_posix() + +def resolved_regular_file(candidate): + metadata = candidate.lstat() + resolved = candidate.resolve(strict=True) if stat.S_ISLNK(metadata.st_mode) else candidate + try: + common = os.path.commonpath((str(model_root.resolve(strict=True)), str(resolved.resolve(strict=True)))) + except (OSError, ValueError): + raise SystemExit("snapshot file could not be resolved") + if common != str(model_root.resolve(strict=True)): + raise SystemExit("snapshot symlink escapes the pinned model cache") + if not resolved.is_file(): + raise SystemExit("snapshot contains a non-regular file") + return resolved + +if snapshot.is_symlink() or not snapshot.is_dir(): + raise SystemExit("pinned local snapshot directory is missing or unsafe") +if model_root.is_symlink() or not model_root.is_dir(): + raise SystemExit("pinned local model cache root is missing or unsafe") + +try: + index = json.loads((snapshot / "model.safetensors.index.json").read_text(encoding="utf-8")) + weight_map = index.get("weight_map", {}) + shards = sorted(set(weight_map.values())) if isinstance(weight_map, dict) else [] +except (OSError, UnicodeError, json.JSONDecodeError): + raise SystemExit("pinned local weight index is missing or malformed") +if len(shards) != 113 or any( + not isinstance(item, str) + or Path(item).name != item + or not safe_component.fullmatch(item) + or not item.endswith(".safetensors") + for item in shards +): + raise SystemExit("pinned local weight index has an unexpected shard set") +if not (snapshot / "config.json").is_file(): + raise SystemExit("pinned local config.json is missing") +if not any((snapshot / name).is_file() for name in ("tokenizer.json", "tokenizer.model", "vocab.json")): + raise SystemExit("pinned local tokenizer assets are missing") + +files = [] +directories = [] +total_bytes = 0 +entry_count = 0 +for current, dirnames, filenames in os.walk(snapshot, topdown=True, followlinks=False): + current_path = Path(current) + dirnames.sort() + filenames.sort() + for dirname in dirnames: + directory = current_path / dirname + relative = relative_name(directory) + mode = directory.lstat().st_mode + if stat.S_ISLNK(mode) or not stat.S_ISDIR(mode): + raise SystemExit("snapshot contains an unsafe directory") + directories.append(relative) + entry_count += 1 + for filename in filenames: + candidate = current_path / filename + relative = relative_name(candidate) + resolved = resolved_regular_file(candidate) + digest = hashlib.sha256() + size = 0 + with resolved.open("rb") as handle: + for chunk in iter(lambda: handle.read(8 * 1024 * 1024), b""): + size += len(chunk) + digest.update(chunk) + files.append({"path": relative, "size": size, "sha256": digest.hexdigest()}) + total_bytes += size + entry_count += 1 + if entry_count > 4096 or total_bytes > 1024 * 1024 * 1024 * 1024: + raise SystemExit("snapshot exceeds the staging safety bounds") + +if not files or any(not (snapshot / shard).exists() for shard in shards): + raise SystemExit("pinned local snapshot is incomplete") +print(json.dumps({ + "schemaVersion": 1, + "files": files, + "directories": directories, + "totalBytes": total_bytes, +}, separators=(",", ":"))) +`; + +function parseManifest(result: ModelStagingCommandResult): SnapshotManifest { + if (result.status !== 0 || result.timedOut || result.error) { + throw new Error(commandFailure("local pinned snapshot audit", result)); + } + if (Buffer.byteLength(result.stdout, "utf8") > MAX_MANIFEST_BYTES) { + throw new Error("local pinned snapshot manifest exceeded the safety bound"); + } + let parsed: unknown; + try { + parsed = JSON.parse(result.stdout.trim()); + } catch { + throw new Error("local pinned snapshot audit returned invalid JSON"); + } + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { + throw new Error("local pinned snapshot audit returned an invalid manifest"); + } + const record = parsed as Record; + if ( + record.schemaVersion !== MANIFEST_SCHEMA_VERSION || + !Array.isArray(record.files) || + !Array.isArray(record.directories) || + !Number.isSafeInteger(record.totalBytes) || + Number(record.totalBytes) <= 0 || + record.files.length === 0 || + record.files.length + record.directories.length > 4096 + ) { + throw new Error("local pinned snapshot audit returned an invalid manifest"); + } + const files = record.files.map((value) => { + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error("local pinned snapshot manifest contains an invalid file"); + } + const file = value as Record; + if ( + typeof file.path !== "string" || + !/^(?:[A-Za-z0-9._-]+\/)*[A-Za-z0-9._-]+$/.test(file.path) || + !Number.isSafeInteger(file.size) || + Number(file.size) < 0 || + typeof file.sha256 !== "string" || + !/^[a-f0-9]{64}$/.test(file.sha256) + ) { + throw new Error("local pinned snapshot manifest contains an invalid file"); + } + return { path: file.path, size: Number(file.size), sha256: file.sha256 }; + }); + const directories = record.directories.map((value) => { + if (typeof value !== "string" || !/^(?:[A-Za-z0-9._-]+\/)*[A-Za-z0-9._-]+$/.test(value)) { + throw new Error("local pinned snapshot manifest contains an invalid directory"); + } + return value; + }); + const filePaths = files.map((file) => file.path); + if ( + new Set(filePaths).size !== files.length || + new Set(directories).size !== directories.length || + directories.some((directory) => filePaths.includes(directory)) || + files.reduce((total, file) => total + file.size, 0) !== Number(record.totalBytes) + ) { + throw new Error("local pinned snapshot manifest is internally inconsistent"); + } + return { + schemaVersion: 1, + files, + directories, + totalBytes: Number(record.totalBytes), + }; +} + +function remoteScript( + plan: DualStationVllmPlan, + paths: StagingPaths, + manifest: SnapshotManifest, + operation: "prepare" | "finalize", +): string { + const manifestBase64 = Buffer.from(JSON.stringify(manifest), "utf8").toString("base64"); + return String.raw` +import base64 +import hashlib +import json +import os +from pathlib import Path +import re +import shutil +import stat + +EXPECTED_HOME = Path(${JSON.stringify(plan.peer.home)}) +FINAL = Path(${JSON.stringify(paths.peerSnapshot)}) +STAGING = Path(${JSON.stringify(paths.peerStaging)}) +OPERATION = ${JSON.stringify(operation)} +EXPECTED = json.loads(base64.b64decode(${JSON.stringify(manifestBase64)}, validate=True)) +SAFE_COMPONENT = re.compile(r"^[A-Za-z0-9._-]+$") + +def fail(message): + raise SystemExit(message) + +def relative_name(root, candidate): + relative = candidate.relative_to(root) + if not relative.parts or any(not SAFE_COMPONENT.fullmatch(part) for part in relative.parts): + fail("peer snapshot contains an unsafe relative path") + return relative.as_posix() + +def manifest(root): + if root.is_symlink() or not root.is_dir(): + fail("peer snapshot path is missing or unsafe") + files = [] + directories = [] + total_bytes = 0 + entry_count = 0 + for current, dirnames, filenames in os.walk(root, topdown=True, followlinks=False): + current_path = Path(current) + dirnames.sort() + filenames.sort() + for dirname in dirnames: + directory = current_path / dirname + relative = relative_name(root, directory) + mode = directory.lstat().st_mode + if stat.S_ISLNK(mode) or not stat.S_ISDIR(mode): + fail("peer snapshot contains an unsafe directory") + directories.append(relative) + entry_count += 1 + for filename in filenames: + candidate = current_path / filename + relative = relative_name(root, candidate) + mode = candidate.lstat().st_mode + if not stat.S_ISREG(mode): + fail("peer snapshot contains a symlink or non-regular file") + digest = hashlib.sha256() + size = 0 + with candidate.open("rb") as handle: + for chunk in iter(lambda: handle.read(8 * 1024 * 1024), b""): + size += len(chunk) + digest.update(chunk) + files.append({"path": relative, "size": size, "sha256": digest.hexdigest()}) + total_bytes += size + entry_count += 1 + if entry_count > 4096 or total_bytes > 1024 * 1024 * 1024 * 1024: + fail("peer snapshot exceeds the staging safety bounds") + return {"schemaVersion": 1, "files": files, "directories": directories, "totalBytes": total_bytes} + +def path_exists(candidate): + return os.path.lexists(candidate) + +def verify_parent_chain(): + observed_home = Path.home() + if observed_home != EXPECTED_HOME or EXPECTED_HOME.is_symlink() or not EXPECTED_HOME.is_dir(): + fail("peer home identity changed during model staging") + current = EXPECTED_HOME + relative_parent = FINAL.parent.relative_to(EXPECTED_HOME) + for component in relative_parent.parts: + if not SAFE_COMPONENT.fullmatch(component): + fail("peer snapshot parent contains an unsafe component") + current = current / component + if path_exists(current): + mode = current.lstat().st_mode + if stat.S_ISLNK(mode) or not stat.S_ISDIR(mode): + fail("peer snapshot parent is a symlink or non-directory") + else: + current.mkdir(mode=0o700) + +def verify_partial(root): + if root.is_symlink() or not root.is_dir(): + fail("peer staging path is unsafe") + expected_files = {item["path"]: item["size"] for item in EXPECTED["files"]} + expected_directories = set(EXPECTED["directories"]) + entry_count = 0 + reusable_bytes = 0 + for current, dirnames, filenames in os.walk(root, topdown=True, followlinks=False): + current_path = Path(current) + for name in sorted(dirnames): + candidate = current_path / name + relative = relative_name(root, candidate) + mode = candidate.lstat().st_mode + if stat.S_ISLNK(mode) or not stat.S_ISDIR(mode) or relative not in expected_directories: + fail("peer staging path contains an unsafe entry") + entry_count += 1 + if entry_count > 8192: + fail("peer staging path exceeds the safety bound") + for name in sorted(filenames): + candidate = current_path / name + relative = relative_name(root, candidate) + mode = candidate.lstat().st_mode + size = candidate.stat().st_size + if ( + stat.S_ISLNK(mode) + or not stat.S_ISREG(mode) + or relative not in expected_files + or size > expected_files[relative] + ): + fail("peer staging path contains an unsafe entry") + reusable_bytes += size + entry_count += 1 + if entry_count > 8192: + fail("peer staging path exceeds the safety bound") + return reusable_bytes + +verify_parent_chain() +if OPERATION == "prepare": + if path_exists(FINAL): + if manifest(FINAL) != EXPECTED: + fail("peer pinned snapshot already exists with different content") + print(json.dumps({"state": "ready"}, separators=(",", ":"))) + else: + if path_exists(STAGING): + reusable_bytes = verify_partial(STAGING) + else: + STAGING.mkdir(mode=0o700) + reusable_bytes = 0 + remaining_bytes = max(0, EXPECTED["totalBytes"] - reusable_bytes) + headroom_bytes = max(5 * 1024 * 1024 * 1024, EXPECTED["totalBytes"] // 20) + if shutil.disk_usage(STAGING.parent).free < remaining_bytes + headroom_bytes: + fail("peer model cache does not have enough free space for the pinned snapshot") + print(json.dumps({"state": "transfer"}, separators=(",", ":"))) +elif OPERATION == "finalize": + if path_exists(FINAL): + if manifest(FINAL) != EXPECTED: + fail("peer pinned snapshot appeared with different content") + if path_exists(STAGING): + verify_partial(STAGING) + shutil.rmtree(STAGING) + else: + if manifest(STAGING) != EXPECTED: + fail("peer staged snapshot failed byte-integrity verification") + staged_identity = (STAGING.stat().st_dev, STAGING.stat().st_ino) + os.rename(STAGING, FINAL) + installed_identity = (FINAL.stat().st_dev, FINAL.stat().st_ino) + if installed_identity != staged_identity: + fail("peer pinned snapshot identity changed during atomic install") + print(json.dumps({"state": "ready"}, separators=(",", ":"))) +else: + fail("unsupported model staging operation") +`; +} + +function commandFailure(label: string, result: ModelStagingCommandResult): string { + if (result.timedOut) return `${label} timed out`; + const detail = ( + result.error || + result.stderr.trim().split("\n").at(-1) || + "command failed" + ).slice(0, 512); + return `${label} failed: ${detail}`; +} + +function parseRemoteState( + label: string, + result: ModelStagingCommandResult, + expected: "ready" | "transfer", +): void { + if (result.status !== 0 || result.timedOut || result.error) { + throw new Error(commandFailure(label, result)); + } + let parsed: unknown; + try { + parsed = JSON.parse(result.stdout.trim()); + } catch { + throw new Error(`${label} returned invalid JSON`); + } + if ( + !parsed || + typeof parsed !== "object" || + Array.isArray(parsed) || + (parsed as Record).state !== expected + ) { + throw new Error(`${label} returned an unexpected state`); + } +} + +function sshArgs(plan: DualStationVllmPlan): string[] { + return [...strictStationSshTransportArgs(), "--", plan.peerSshTarget, "python3 -"]; +} + +function rsyncRsh(): string { + return ["ssh", ...strictStationSshTransportArgs()].join(" "); +} + +/** + * Materialize only the pinned snapshot on the pre-trusted peer. Hugging Face + * snapshot symlinks are copied as regular files after an escape check, so no + * token or unrelated blob cache crosses the SSH boundary. + */ +export async function stageDualStationModelSnapshot( + plan: DualStationVllmPlan, + deps: DualStationModelStagingDeps = defaultDeps, +): Promise { + let paths: StagingPaths; + try { + paths = stagingPaths(plan); + } catch (err) { + return { ok: false, reason: (err as Error).message }; + } + const env = buildVllmSshTransportEnv({ LC_ALL: "C" }); + + let manifest: SnapshotManifest; + try { + const audit = await deps.runCommand( + "python3", + ["-", paths.localSnapshot, paths.localModelRoot], + { + env, + input: LOCAL_MANIFEST_SCRIPT, + timeoutMs: SNAPSHOT_AUDIT_TIMEOUT_MS, + }, + ); + manifest = parseManifest(audit); + } catch (err) { + return { ok: false, reason: (err as Error).message }; + } + + const prepare = await deps.runCommand("ssh", sshArgs(plan), { + env, + input: remoteScript(plan, paths, manifest, "prepare"), + timeoutMs: SNAPSHOT_AUDIT_TIMEOUT_MS, + }); + try { + if (prepare.status === 0 && JSON.parse(prepare.stdout.trim()).state === "ready") { + parseRemoteState("peer snapshot preflight", prepare, "ready"); + return { ok: true, transferred: false }; + } + parseRemoteState("peer snapshot preflight", prepare, "transfer"); + } catch (err) { + return { ok: false, reason: (err as Error).message }; + } + + const rsync = await deps.runCommand( + "rsync", + [ + "--recursive", + "--times", + "--copy-links", + "--checksum", + "--partial", + "--protect-args", + "--no-owner", + "--no-group", + "--chmod=Du=rwx,Dgo=,Fu=rw,Fgo=", + "--info=progress2", + `--rsh=${rsyncRsh()}`, + "--", + `${paths.localSnapshot}/`, + `${plan.peerSshTarget}:${paths.peerStaging}/`, + ], + { + env, + timeoutMs: RSYNC_TIMEOUT_MS, + idleTimeoutMs: RSYNC_IDLE_TIMEOUT_MS, + streamOutput: true, + }, + ); + if (rsync.status !== 0 || rsync.timedOut || rsync.error) { + return { ok: false, reason: commandFailure("peer snapshot transfer", rsync) }; + } + + const finalize = await deps.runCommand("ssh", sshArgs(plan), { + env, + input: remoteScript(plan, paths, manifest, "finalize"), + timeoutMs: SNAPSHOT_AUDIT_TIMEOUT_MS, + }); + try { + parseRemoteState("peer snapshot verification", finalize, "ready"); + } catch (err) { + return { ok: false, reason: (err as Error).message }; + } + return { ok: true, transferred: true }; +} diff --git a/src/lib/inference/vllm.test.ts b/src/lib/inference/vllm.test.ts index e93a208d82..ad73e54b46 100644 --- a/src/lib/inference/vllm.test.ts +++ b/src/lib/inference/vllm.test.ts @@ -58,7 +58,6 @@ import { buildVllmRunArgs, detectVllmProfile, installVllm, - isNemoClawManagedVllmRunning, NEMOCLAW_VLLM_CONTAINER_NAME, NEMOCLAW_VLLM_MANAGED_LABEL, pullImage, @@ -119,9 +118,16 @@ const MANAGED_CONTAINER_ID = "a".repeat(64); function vllmContainerRow( containerName: string, - { id = MANAGED_CONTAINER_ID, label = "true", state = "exited" } = {}, + { + id = MANAGED_CONTAINER_ID, + label = "true", + state = "exited", + dualRole = "", + dualEndpoint = "", + dualCluster = "", + } = {}, ): string { - return `${id}|${containerName}|${state}|${label}`; + return [id, containerName, state, label, dualRole, dualEndpoint, dualCluster].join("|"); } function mockSuccessfulVllmInstall( @@ -511,52 +517,6 @@ describe("vLLM run command", () => { }); }); -describe("managed vLLM ownership", () => { - beforeEach(() => { - vi.clearAllMocks(); - }); - - it("recognizes only the exact running container with the managed label", () => { - mocks.dockerCapture.mockReturnValue( - vllmContainerRow(NEMOCLAW_VLLM_CONTAINER_NAME, { state: "running" }), - ); - - expect(isNemoClawManagedVllmRunning()).toBe(true); - expect(mocks.dockerCapture).toHaveBeenCalledWith( - [ - "container", - "ls", - "--all", - "--no-trunc", - "--filter", - `name=^/${NEMOCLAW_VLLM_CONTAINER_NAME}$`, - "--format", - `{{.ID}}|{{.Names}}|{{.State}}|{{.Label "${NEMOCLAW_VLLM_MANAGED_LABEL}"}}`, - ], - expect.objectContaining({ timeout: 10_000 }), - ); - }); - - it.each([ - vllmContainerRow(NEMOCLAW_VLLM_CONTAINER_NAME), - vllmContainerRow(NEMOCLAW_VLLM_CONTAINER_NAME, { label: "" }), - vllmContainerRow(NEMOCLAW_VLLM_CONTAINER_NAME, { label: "false", state: "running" }), - "", - "malformed", - `${vllmContainerRow(NEMOCLAW_VLLM_CONTAINER_NAME)}\n${vllmContainerRow(NEMOCLAW_VLLM_CONTAINER_NAME)}`, - ])("fails closed for inspect output %j", (output) => { - mocks.dockerCapture.mockReturnValue(output); - expect(isNemoClawManagedVllmRunning()).toBe(false); - }); - - it("fails closed when Docker inspection throws", () => { - mocks.dockerCapture.mockImplementation(() => { - throw new Error("docker unavailable"); - }); - expect(isNemoClawManagedVllmRunning()).toBe(false); - }); -}); - describe("installVllm model resolution", () => { let logSpy: ReturnType; let errSpy: ReturnType; @@ -572,6 +532,7 @@ describe("installVllm model resolution", () => { stdoutWrite = vi.spyOn(process.stdout, "write").mockImplementation(() => true); delete process.env.NEMOCLAW_VLLM_MODEL; delete process.env.NEMOCLAW_VLLM_EXTRA_ARGS_JSON; + delete process.env.NEMOCLAW_DGX_STATION_PEER; delete process.env.HF_TOKEN; delete process.env.HUGGING_FACE_HUB_TOKEN; // Fail dockerPrereqsOk so the function returns before any docker work, @@ -647,6 +608,7 @@ describe("installVllm model resolution", () => { it("rejects a Station-only runtime override before side effects on generic Linux", async () => { process.env.NEMOCLAW_VLLM_MODEL = "nemotron-3-ultra-550b-a55b"; + process.env.HF_TOKEN = "hf_test"; const profile = detectVllmProfile({ platform: "linux", type: "nvidia" })!; const beforeInstall = vi.fn(); @@ -670,6 +632,7 @@ describe("installVllm model resolution", () => { it("installs the complete Nemotron Ultra Station recipe without another selection", async () => { process.env.NEMOCLAW_VLLM_MODEL = "nemotron-3-ultra-550b-a55b"; + process.env.HF_TOKEN = "hf_test"; const profile = detectVllmProfile({ platform: "station", type: "nvidia" })!; const beforeInstall = vi.fn(); const promptFn = vi.fn<(q: string) => Promise>(); @@ -851,6 +814,7 @@ describe("installVllm model resolution", () => { it("continues a non-interactive Ultra download when the HF cache is too small", async () => { process.env.NEMOCLAW_VLLM_MODEL = "nemotron-3-ultra-550b-a55b"; + process.env.HF_TOKEN = "hf_test"; const profile = detectVllmProfile({ platform: "station", type: "nvidia" })!; mockSuccessfulVllmInstall(profile.containerName); mocks.dockerImageInspectFormat.mockReturnValue("sha256:cached-image"); @@ -896,6 +860,7 @@ describe("installVllm model resolution", () => { downloads, }) => { process.env.NEMOCLAW_VLLM_MODEL = "nemotron-3-ultra-550b-a55b"; + process.env.HF_TOKEN = "hf_test"; const profile = detectVllmProfile({ platform: "station", type: "nvidia" })!; mockSuccessfulVllmInstall(profile.containerName); mocks.dockerImageInspectFormat.mockReturnValue("sha256:cached-image"); @@ -925,6 +890,7 @@ describe("installVllm model resolution", () => { it("fails closed before downloads when non-interactive model-cache capacity is inconclusive", async () => { process.env.NEMOCLAW_VLLM_MODEL = "nemotron-3-ultra-550b-a55b"; + process.env.HF_TOKEN = "hf_test"; const profile = detectVllmProfile({ platform: "station", type: "nvidia" })!; mockSuccessfulVllmInstall(profile.containerName); mocks.probeHostStorage.mockReturnValue(inconclusiveModelStorage()); @@ -961,6 +927,7 @@ describe("installVllm model resolution", () => { downloads, }) => { process.env.NEMOCLAW_VLLM_MODEL = "nemotron-3-ultra-550b-a55b"; + process.env.HF_TOKEN = "hf_test"; const profile = detectVllmProfile({ platform: "station", type: "nvidia" })!; mockSuccessfulVllmInstall(profile.containerName); mocks.dockerImageInspectFormat.mockReturnValue("sha256:cached-image"); @@ -983,6 +950,7 @@ describe("installVllm model resolution", () => { it("re-probes after a cold image pull before continuing past a storage warning", async () => { process.env.NEMOCLAW_VLLM_MODEL = "nemotron-3-ultra-550b-a55b"; + process.env.HF_TOKEN = "hf_test"; const profile = detectVllmProfile({ platform: "station", type: "nvidia" })!; mockSuccessfulVllmInstall(profile.containerName); mocks.probeHostStorage @@ -1028,6 +996,7 @@ describe("installVllm model resolution", () => { it("stops when the post-pull model-cache capacity re-probe is inconclusive", async () => { process.env.NEMOCLAW_VLLM_MODEL = "nemotron-3-ultra-550b-a55b"; + process.env.HF_TOKEN = "hf_test"; const profile = detectVllmProfile({ platform: "station", type: "nvidia" })!; mockSuccessfulVllmInstall(profile.containerName); mocks.probeHostStorage @@ -1059,6 +1028,7 @@ describe("installVllm model resolution", () => { it("requires only missing snapshot bytes plus headroom for a partial Ultra cache", async () => { process.env.NEMOCLAW_VLLM_MODEL = "nemotron-3-ultra-550b-a55b"; + process.env.HF_TOKEN = "hf_test"; const profile = detectVllmProfile({ platform: "station", type: "nvidia" })!; mockSuccessfulVllmInstall(profile.containerName); mocks.dockerImageInspectFormat.mockReturnValue("sha256:cached-image"); @@ -1088,6 +1058,7 @@ describe("installVllm model resolution", () => { it("skips the capacity gate for a complete pinned Ultra snapshot", async () => { process.env.NEMOCLAW_VLLM_MODEL = "nemotron-3-ultra-550b-a55b"; + process.env.HF_TOKEN = "hf_test"; const profile = detectVllmProfile({ platform: "station", type: "nvidia" })!; mockSuccessfulVllmInstall(profile.containerName); mocks.dockerImageInspectFormat.mockReturnValue("sha256:cached-image"); @@ -1376,6 +1347,28 @@ describe("installVllm model resolution", () => { expect(mocks.dockerRunDetached).toHaveBeenCalledTimes(1); }); + it("refuses single-host replacement of a managed dual head so the peer is not orphaned", async () => { + const profile = detectVllmProfile({ platform: "spark", type: "nvidia" })!; + const dualHead = vllmContainerRow(profile.containerName, { + state: "running", + dualRole: "head", + dualEndpoint: "http://192.168.100.1:8000", + dualCluster: "f".repeat(64), + }); + mockSuccessfulVllmInstall(profile.containerName, [() => dualHead]); + + const result = await installVllm(profile, { + hasImage: true, + nonInteractive: true, + promptFn: vi.fn(), + }); + + expect(result).toEqual({ ok: false }); + expect(mocks.dockerForceRm).not.toHaveBeenCalled(); + expect(mocks.dockerSpawn).not.toHaveBeenCalled(); + expect(errSpy).toHaveBeenCalledWith(expect.stringContaining("orphan the peer worker")); + }); + it.each([ "", "false", diff --git a/src/lib/inference/vllm.ts b/src/lib/inference/vllm.ts index 36a1dde93d..6d47ee977c 100644 --- a/src/lib/inference/vllm.ts +++ b/src/lib/inference/vllm.ts @@ -8,6 +8,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; +import { isDeepStrictEqual } from "node:util"; import { dockerCapture, dockerForceRm, @@ -17,14 +18,21 @@ import { dockerSpawn, dockerStop, } from "../adapters/docker"; +import { createBearerAuthConfig } from "../adapters/http/auth-config"; import { buildValidatedCurlCommandArgs } from "../adapters/http/curl-args"; +import { runCurlProbe } from "../adapters/http/probe"; import { VLLM_PORT } from "../core/ports"; import { shellQuote } from "../core/shell-quote"; import { isAffirmativeAnswer } from "../onboard/prompt-helpers"; import { runCapture } from "../runner"; import { isSafeModelId } from "../validation"; import { getGpuIndicesByName } from "./nim"; -import { buildVllmDockerEnv } from "./vllm-docker-env"; +import { ensureDualStationVllmApiKey, loadDualStationVllmApiKey } from "./vllm-api-key"; +import { + buildLocalDualStationDockerEnv, + buildRemoteVllmDockerEnv, + buildVllmDockerEnv, +} from "./vllm-docker-env"; import { buildVllmServeCommand, NEMOTRON_ULTRA_STATION_IMAGE, @@ -35,6 +43,24 @@ import { type VllmPlatform, } from "./vllm-models"; import { resolveVllmInstallModel } from "./vllm-prompt"; +import { + type DualStationVllmPlan, + NEMOCLAW_DGX_STATION_PEER_ENV, + probeDualStationVllmCapability, +} from "./vllm-station-cluster"; +import { + areDualStationManagedVllmContainersRunning, + cleanupDualStationManagedVllm, + DUAL_STATION_VLLM_CLUSTER_LABEL, + DUAL_STATION_VLLM_ENDPOINT_LABEL, + DUAL_STATION_VLLM_ROLE_LABEL, + getDualStationManagedVllmBaseUrl, + preflightDualStationGpuRuntime, + preflightDualStationManagedVllm, + startDualStationManagedVllm, + withDualStationManagedVllmLifecycle, +} from "./vllm-station-cluster-lifecycle"; +import { stageDualStationModelSnapshot } from "./vllm-station-model-staging"; import { findUnwritableTreePath, formatStorageBytes, @@ -331,7 +357,10 @@ function dockerPrereqsOk(): { ok: boolean; reason?: string } { return { ok: true }; } -export async function pullImage(profile: VllmProfile): Promise<{ ok: boolean; reason?: string }> { +export async function pullImage( + profile: VllmProfile, + dockerEnv: Record = buildVllmDockerEnv(), +): Promise<{ ok: boolean; reason?: string }> { try { assertVllmRegistryDigestRef(profile.image); } catch (err) { @@ -342,7 +371,7 @@ export async function pullImage(profile: VllmProfile): Promise<{ ok: boolean; re // profile, so all profiles intentionally share the 15-minute stall default. // The profile-specific maximum still bounds the complete pull operation. const result = await dockerPullWithProgressWatchdog(profile.image, { - env: buildVllmDockerEnv(), + env: dockerEnv, maxTimeoutMs: profile.pullTimeoutSec * 1000, logLine: emit, }); @@ -365,6 +394,7 @@ export async function pullImage(profile: VllmProfile): Promise<{ ok: boolean; re function downloadModel( profile: VllmProfile, model: VllmModelDef, + dockerEnv: Record = buildVllmDockerEnv(), ): Promise<{ ok: boolean; reason?: string }> { emit(`Pre-downloading model with hf: ${model.id}`); return new Promise((resolve) => { @@ -388,7 +418,7 @@ function downloadModel( ...(model.revision ? ["--revision", model.revision] : []), ], { - env: buildVllmDockerEnv(buildHfTokenForwardEnv()), + env: { ...dockerEnv, ...buildHfTokenForwardEnv() }, stdio: ["ignore", "pipe", "pipe"], }, ); @@ -566,12 +596,21 @@ export function assertVllmRegistryDigestRef(image: string): void { type VllmContainerOwnership = | { kind: "absent" } + | { kind: "dual-managed"; containerId: string; running: boolean } | { kind: "foreign" } | { kind: "managed"; containerId: string; running: boolean } | { kind: "unknown" }; function inspectVllmContainerOwnership(containerName: string): VllmContainerOwnership { - const format = `{{.ID}}|{{.Names}}|{{.State}}|{{.Label "${NEMOCLAW_VLLM_MANAGED_LABEL}"}}`; + const format = [ + "{{.ID}}", + "{{.Names}}", + "{{.State}}", + `{{.Label "${NEMOCLAW_VLLM_MANAGED_LABEL}"}}`, + `{{.Label "${DUAL_STATION_VLLM_ROLE_LABEL}"}}`, + `{{.Label "${DUAL_STATION_VLLM_ENDPOINT_LABEL}"}}`, + `{{.Label "${DUAL_STATION_VLLM_CLUSTER_LABEL}"}}`, + ].join("|"); try { const output = dockerCapture( [ @@ -591,12 +630,25 @@ function inspectVllmContainerOwnership(containerName: string): VllmContainerOwne const rows = output.split(/\r?\n/); if (rows.length !== 1) return { kind: "unknown" }; const fields = rows[0].split("|"); - if (fields.length !== 4) return { kind: "unknown" }; - const [containerId, observedName, state, managedLabel] = fields; + if (fields.length !== 7) return { kind: "unknown" }; + const [containerId, observedName, state, managedLabel, dualRole, dualEndpoint, dualCluster] = + fields; if (observedName !== containerName || !DOCKER_CONTAINER_ID_PATTERN.test(containerId)) { return { kind: "unknown" }; } if (managedLabel !== "true") return { kind: "foreign" }; + const hasAnyDualLabel = Boolean(dualRole || dualEndpoint || dualCluster); + if (hasAnyDualLabel) { + const exactDualHead = + dualRole === "head" && + /^http:\/\/192\.168\.|^http:\/\/10\.|^http:\/\/172\.(?:1[6-9]|2[0-9]|3[01])\./.test( + dualEndpoint, + ) && + /^[a-f0-9]{64}$/.test(dualCluster); + return exactDualHead + ? { kind: "dual-managed", containerId, running: state === "running" } + : { kind: "unknown" }; + } return { kind: "managed", containerId, running: state === "running" }; } catch { return { kind: "unknown" }; @@ -619,6 +671,14 @@ function vllmContainerReplacementTarget( reason: `Could not verify ownership of Docker container "${containerName}". NemoClaw will not remove it. Check Docker access and retry.`, }; } + if (ownership.kind === "dual-managed") { + return { + ok: false, + reason: + `Container "${containerName}" is the head of a managed dual-Station deployment. ` + + `Refusing single-host replacement because it would orphan the peer worker. Restore ${NEMOCLAW_DGX_STATION_PEER_ENV} and select Nemotron Ultra to manage the pair.`, + }; + } return ownership.kind === "managed" ? { ok: true, containerId: ownership.containerId } : { ok: true }; @@ -626,7 +686,7 @@ function vllmContainerReplacementTarget( export function isNemoClawManagedVllmRunning(): boolean { const ownership = inspectVllmContainerOwnership(NEMOCLAW_VLLM_CONTAINER_NAME); - return ownership.kind === "managed" && ownership.running; + return (ownership.kind === "managed" || ownership.kind === "dual-managed") && ownership.running; } function startContainer( @@ -667,11 +727,17 @@ function startContainer( return { ok: true }; } -function vllmModelsEndpoint(): string { - return `http://127.0.0.1:${String(VLLM_PORT)}/v1/models`; -} - -function vllmEndpointReady(): boolean { +function vllmEndpointReady(baseUrl?: string): boolean { + if (baseUrl) { + // The dual-Station /v1 surface is bearer-protected. vLLM deliberately + // leaves /health outside its auth middleware, so readiness can stay + // secret-free while onboarding separately validates model inventory with + // the persisted key. + return runCurlProbe( + ["-sS", "--connect-timeout", "2", "--max-time", "5", `${baseUrl.replace(/\/+$/, "")}/health`], + { pinnedAddresses: [] }, + ).ok; + } const response = runCapture( [ "curl", @@ -681,7 +747,7 @@ function vllmEndpointReady(): boolean { "2", "--max-time", "5", - vllmModelsEndpoint(), + `http://127.0.0.1:${String(VLLM_PORT)}/v1/models`, ]), ], { ignoreError: true }, @@ -695,27 +761,102 @@ function vllmEndpointReady(): boolean { } } -function readContainerLogTail(profile: VllmProfile, lineCount = 80): string[] { +function verifyDualStationVllmAuthBoundary( + baseUrl: string, + apiKey: string, + expectedModelId: string, +): { ok: true } | { ok: false; reason: string } { + const modelsUrl = `${baseUrl.replace(/\/+$/, "")}/v1/models`; + const unauthenticated = runCurlProbe( + ["-sS", "--connect-timeout", "3", "--max-time", "5", modelsUrl], + { pinnedAddresses: [] }, + ); + if (unauthenticated.httpStatus !== 401) { + return { + ok: false, + reason: + `unauthenticated model inventory returned HTTP ${String(unauthenticated.httpStatus)}; ` + + "expected vLLM to reject it with HTTP 401", + }; + } + + let authConfig: ReturnType | undefined; + try { + authConfig = createBearerAuthConfig(apiKey, { prefix: "nemoclaw-vllm-install-auth" }); + const authenticated = runCurlProbe( + ["-sS", "--connect-timeout", "3", "--max-time", "5", ...authConfig.args, modelsUrl], + { + trustedConfigFiles: authConfig.trustedConfigFiles, + pinnedAddresses: [], + }, + ); + if (!authenticated.ok) { + return { + ok: false, + reason: `authenticated model inventory failed: ${authenticated.message}`, + }; + } + let parsed: unknown; + try { + parsed = JSON.parse(authenticated.body); + } catch { + return { ok: false, reason: "authenticated model inventory returned malformed JSON" }; + } + const data = (parsed as { data?: unknown } | null)?.data; + const ids = (Array.isArray(data) ? data : []).flatMap((entry) => { + if (typeof entry !== "object" || entry === null) return []; + const id = (entry as { id?: unknown }).id; + return typeof id === "string" ? [id] : []; + }); + if (ids.length !== 1 || ids[0] !== expectedModelId) { + return { + ok: false, + reason: `authenticated model inventory did not expose exactly '${expectedModelId}'`, + }; + } + return { ok: true }; + } catch (error) { + return { + ok: false, + reason: `authenticated model inventory failed: ${(error as Error).message}`, + }; + } finally { + authConfig?.cleanup(); + } +} + +function readContainerLogTail( + profile: VllmProfile, + lineCount = 80, + dockerEnv: Record = buildVllmDockerEnv(), +): string[] { const output = dockerCapture(["logs", "--tail", String(lineCount), profile.containerName], { - env: buildVllmDockerEnv(), + env: dockerEnv, ignoreError: true, }).trim(); if (!output) return []; return output.split(/\r?\n/).slice(-lineCount); } -function printContainerLogTail(profile: VllmProfile): void { - const tail = readContainerLogTail(profile); +function printContainerLogTail( + profile: VllmProfile, + dockerEnv: Record = buildVllmDockerEnv(), +): void { + const tail = readContainerLogTail(profile, 80, dockerEnv); if (tail.length === 0) return; process.stderr.write(` --- Last ${String(tail.length)} vLLM log lines: ---\n`); for (const line of tail) process.stderr.write(` ${line}\n`); process.stderr.write(" ---\n"); } -// Poll the real OpenAI-compatible models endpoint instead of interpreting -// vLLM startup logs. Logs stay quiet on the happy path and print only on -// failure. -function waitForVllmReady(profile: VllmProfile): Promise<{ ok: boolean; reason?: string }> { +// Poll the real OpenAI-compatible models endpoint for the legacy local path, +// or the secret-free vLLM health endpoint for authenticated dual-Station +// serving. Logs stay quiet on the happy path and print only on failure. +function waitForVllmReady( + profile: VllmProfile, + baseUrl?: string, + dockerEnv: Record = buildVllmDockerEnv(), +): Promise<{ ok: boolean; reason?: string }> { return new Promise((resolve) => { let resolved = false; const start = Date.now(); @@ -735,7 +876,7 @@ function waitForVllmReady(profile: VllmProfile): Promise<{ ok: boolean; reason?: function poll(): void { if (resolved) return; - if (vllmEndpointReady()) { + if (vllmEndpointReady(baseUrl)) { emit(`vLLM is serving on :${String(VLLM_PORT)}`); done({ ok: true }); return; @@ -748,7 +889,7 @@ function waitForVllmReady(profile: VllmProfile): Promise<{ ok: boolean; reason?: }); return; } - if (!containerStillRunning(profile)) { + if (!containerStillRunning(profile, dockerEnv)) { done({ ok: false, reason: "vLLM container exited before readiness" }); return; } @@ -763,10 +904,13 @@ function waitForVllmReady(profile: VllmProfile): Promise<{ ok: boolean; reason?: }); } -function containerStillRunning(profile: VllmProfile): boolean { +function containerStillRunning( + profile: VllmProfile, + dockerEnv: Record = buildVllmDockerEnv(), +): boolean { const out = dockerCapture( ["ps", "--filter", `name=${profile.containerName}`, "--format", "{{.Names}}"], - { env: buildVllmDockerEnv(), ignoreError: true }, + { env: dockerEnv, ignoreError: true }, ).trim(); return out === profile.containerName; } @@ -847,8 +991,18 @@ function printModelStorageWarning( async function imageStorageAccepted( profile: VllmProfile, opts: InstallVllmOptions, + dockerEnv: Record = buildVllmDockerEnv(), ): Promise { - const probe = probeDockerStorage(); + const probe = probeDockerStorage({ + dockerContext: dockerEnv.DOCKER_CONTEXT, + dockerHost: dockerEnv.DOCKER_HOST, + dockerInfo: () => + dockerCapture(["info", "--format", "{{json .}}"], { + env: dockerEnv, + ignoreError: true, + timeout: 10_000, + }), + }); const requiredBytes = imageStorageRequirementBytes(profile.imageDownloadSizeBytes); if (probe.ok && probe.capacity.availableBytes >= requiredBytes) { return true; @@ -933,10 +1087,13 @@ interface InstallVllmOptions { beforeInstall?: (modelId: string) => void; } -function imageIsCached(profile: VllmProfile): boolean { +function imageIsCached( + profile: VllmProfile, + dockerEnv: Record = buildVllmDockerEnv(), +): boolean { return Boolean( dockerImageInspectFormat("{{.Id}}", profile.image, { - env: buildVllmDockerEnv(), + env: dockerEnv, ignoreError: true, timeout: 10_000, }).trim(), @@ -971,13 +1128,51 @@ export async function installVllm( profile: VllmProfile, opts: InstallVllmOptions, ): Promise<{ ok: boolean }> { + let dualStationPlan: DualStationVllmPlan | null = null; + let peerModelSnapshot: "ready" | "staging-required" | null = null; + const explicitModel = String(process.env.NEMOCLAW_VLLM_MODEL ?? "").trim(); + const configuredPeer = String(process.env[NEMOCLAW_DGX_STATION_PEER_ENV] ?? "").trim(); + // Model selection lives in `resolveVllmInstallModel` so this entry point // stays focused on the docker side effects. Gated-model access is checked // there before any docker work happens. - const resolved = await resolveVllmInstallModel(profile, { - nonInteractive: opts.nonInteractive, - promptFn: opts.promptFn, - }); + let resolved: Awaited>; + if (profile.platform === "station" && configuredPeer && !explicitModel) { + const capability = probeDualStationVllmCapability(); + if (capability.kind !== "ready") { + const reason = + capability.kind === "unavailable" + ? capability.reason + : "the explicit peer configuration disappeared"; + console.error(` Dual DGX Station setup unavailable: ${reason}`); + return { ok: false }; + } + const ultra = VLLM_MODELS.find( + (candidate) => candidate.envValue === "nemotron-3-ultra-550b-a55b", + ); + if (!ultra) { + console.error(" vLLM install failed: Nemotron Ultra is missing from the model registry"); + return { ok: false }; + } + resolved = await resolveVllmInstallModel( + { ...profile, defaultModel: ultra }, + { + // A qualified explicit peer is the model-selection signal. The normal + // resolver still owns access validation, but no second model choice is + // presented after hardware qualification. + nonInteractive: true, + promptFn: opts.promptFn, + }, + ); + if (!resolved) return { ok: false }; + dualStationPlan = capability.plan; + peerModelSnapshot = capability.peerModelSnapshot; + } else { + resolved = await resolveVllmInstallModel(profile, { + nonInteractive: opts.nonInteractive, + promptFn: opts.promptFn, + }); + } if (!resolved) return { ok: false }; const { model, source: modelSource } = resolved; if (model.runtime && !model.platforms.includes(profile.platform)) { @@ -1001,6 +1196,35 @@ export async function installVllm( console.error(` vLLM install failed: ${(err as Error).message}`); return { ok: false }; } + + if (profile.platform === "station" && model.envValue === "nemotron-3-ultra-550b-a55b") { + if (!dualStationPlan) { + const capability = probeDualStationVllmCapability(); + if (capability.kind === "unavailable") { + console.error(` Dual DGX Station setup unavailable: ${capability.reason}`); + return { ok: false }; + } + if (capability.kind === "ready") { + dualStationPlan = capability.plan; + peerModelSnapshot = capability.peerModelSnapshot; + } + } + if (dualStationPlan) { + if (VLLM_PORT !== 8000) { + console.error( + " Dual DGX Station setup requires the default vLLM port 8000; unset NEMOCLAW_VLLM_PORT and retry.", + ); + return { ok: false }; + } + if (extraServeArgs.length > 0) { + console.error( + ` Dual DGX Station setup does not accept ${VLLM_EXTRA_ARGS_ENV}; the verified distributed launch is fixed.`, + ); + return { ok: false }; + } + } + } + const localDockerEnv = dualStationPlan ? buildLocalDualStationDockerEnv() : buildVllmDockerEnv(); opts.beforeInstall?.(servedModelId); console.log(""); @@ -1014,6 +1238,14 @@ export async function installVllm( ` Extra serve args: ${String(extraServeArgs.length)} token(s) from ${VLLM_EXTRA_ARGS_ENV}`, ); } + if (dualStationPlan) { + console.log( + ` Topology: 2× DGX Station (${dualStationPlan.local.hostname} + ${dualStationPlan.peer.hostname})`, + ); + console.log( + ` Fabric: ${dualStationPlan.rails.map((rail) => rail.subnet).join(", ")} (RoCEv2 GID ${String(dualStationPlan.roceGidIndex)})`, + ); + } if (!opts.hasImage) console.log(" Image download on first run, cached after"); console.log(" Model download on first run, cached after"); console.log(""); @@ -1032,12 +1264,21 @@ export async function installVllm( return { ok: false }; } - // Fail before large downloads when the fixed name belongs to another - // operator. startContainer repeats this check to close the teardown race. - const replacement = vllmContainerReplacementTarget(runtimeProfile.containerName); - if (!replacement.ok) { - console.error(` vLLM install failed: ${replacement.reason}`); - return { ok: false }; + // Fail before large downloads when either daemon has an ambiguous or + // foreign fixed-name container. Each launch path repeats this ownership + // check immediately before teardown to close the name-transfer race. + if (dualStationPlan) { + const preflight = preflightDualStationManagedVllm(dualStationPlan); + if (!preflight.ok) { + console.error(` vLLM install failed: ${preflight.reason}`); + return { ok: false }; + } + } else { + const replacement = vllmContainerReplacementTarget(runtimeProfile.containerName); + if (!replacement.ok) { + console.error(` vLLM install failed: ${replacement.reason}`); + return { ok: false }; + } } // Guard the host filesystem before an image pull or model-download @@ -1047,8 +1288,8 @@ export async function installVllm( return { ok: false }; } - const hasImage = imageIsCached(runtimeProfile); - if (!hasImage && !(await imageStorageAccepted(runtimeProfile, opts))) { + const hasImage = imageIsCached(runtimeProfile, localDockerEnv); + if (!hasImage && !(await imageStorageAccepted(runtimeProfile, opts, localDockerEnv))) { return { ok: false }; } @@ -1058,12 +1299,33 @@ export async function installVllm( return { ok: false }; } - const pull = await pullImage(runtimeProfile); + const pull = await pullImage(runtimeProfile, localDockerEnv); if (!pull.ok) { console.error(` vLLM install failed: ${String(pull.reason)}`); return { ok: false }; } + if (dualStationPlan) { + let peerDockerEnv: Record; + try { + peerDockerEnv = buildRemoteVllmDockerEnv(dualStationPlan.peerDockerHost); + } catch (err) { + console.error(` vLLM install failed: ${(err as Error).message}`); + return { ok: false }; + } + emit(`Pulling the pinned vLLM image on peer ${dualStationPlan.peer.hostname}`); + const peerPull = await pullImage(runtimeProfile, peerDockerEnv); + if (!peerPull.ok) { + console.error(` vLLM install failed on peer: ${String(peerPull.reason)}`); + return { ok: false }; + } + const gpuPreflight = await preflightDualStationGpuRuntime(dualStationPlan); + if (!gpuPreflight.ok) { + console.error(` vLLM install failed: ${gpuPreflight.reason}`); + return { ok: false }; + } + } + // A cold image pull can consume the same host filesystem that backs the // Hugging Face cache. Re-probe after the pull so two independently passing // capacity checks cannot overcommit shared storage before `hf download`. @@ -1071,12 +1333,146 @@ export async function installVllm( return { ok: false }; } - const modelDownload = await downloadModel(runtimeProfile, model); + const modelDownload = await downloadModel(runtimeProfile, model, localDockerEnv); if (!modelDownload.ok) { console.error(` vLLM install failed: ${String(modelDownload.reason)}`); return { ok: false }; } + if (dualStationPlan) { + const stagingPlan = dualStationPlan; + try { + const verification = await withDualStationManagedVllmLifecycle(async () => { + emit( + peerModelSnapshot === "staging-required" + ? `Staging the pinned model snapshot on peer ${stagingPlan.peer.hostname}` + : `Verifying the pinned model snapshot on peer ${stagingPlan.peer.hostname}`, + ); + const staging = await stageDualStationModelSnapshot(stagingPlan); + if (!staging.ok) return { ok: false as const, reason: staging.reason }; + + const refreshedCapability = probeDualStationVllmCapability(); + if (refreshedCapability.kind !== "ready") { + const reason = + refreshedCapability.kind === "unavailable" + ? refreshedCapability.reason + : "the explicit peer configuration disappeared"; + return { + ok: false as const, + reason: `dual-Station capability changed: ${reason}`, + }; + } + if (!isDeepStrictEqual(refreshedCapability.plan, stagingPlan)) { + return { + ok: false as const, + reason: + "dual-Station topology changed during download; rerun setup against a stable pair.", + }; + } + if (refreshedCapability.peerModelSnapshot !== "ready") { + return { + ok: false as const, + reason: "peer pinned model snapshot was not verified after staging.", + }; + } + return { ok: true as const, plan: refreshedCapability.plan }; + }); + if (!verification.ok) { + console.error(` vLLM install failed: ${verification.reason}`); + return { ok: false }; + } + dualStationPlan = verification.plan; + } catch (error) { + console.error( + ` vLLM install failed: dual-Station lifecycle lock failed during model verification: ${(error as Error).message}`, + ); + return { ok: false }; + } + } + + let dualStationApiKey: string | null = null; + if (dualStationPlan) { + try { + const existingManagedBaseUrl = getDualStationManagedVllmBaseUrl(); + const existingApiKey = existingManagedBaseUrl ? loadDualStationVllmApiKey() : null; + // If the key file alone was lost, create a new host-global key. The + // lifecycle fingerprint then forces a coordinated pair replacement + // under its lock instead of reusing containers bound to an unknown key. + dualStationApiKey = existingApiKey ?? ensureDualStationVllmApiKey(); + } catch (err) { + console.error(` vLLM install failed: ${(err as Error).message}`); + return { ok: false }; + } + } + + if (dualStationPlan) { + if (!dualStationApiKey) { + console.error(" vLLM install failed: dual-Station API key was not provisioned"); + return { ok: false }; + } + try { + return await withDualStationManagedVllmLifecycle(async () => { + const start = await startDualStationManagedVllm(dualStationPlan, { + apiKey: dualStationApiKey, + }); + if (!start.ok) { + console.error(` vLLM install failed: ${start.reason}`); + for (const rollbackError of start.rollbackErrors) { + console.error(` vLLM rollback warning: ${rollbackError}`); + } + return { ok: false }; + } + + emit("Launching vLLM"); + emit( + `Launch can take 5 minutes to ${String(Math.ceil(runtimeProfile.loadTimeoutSec / 60))} minutes`, + ); + + const ready = await waitForVllmReady(runtimeProfile, start.baseUrl, localDockerEnv); + if (!ready.ok) { + printContainerLogTail(runtimeProfile, localDockerEnv); + if (!start.reusedExisting) { + const cleanup = await cleanupDualStationManagedVllm(dualStationPlan); + if (!cleanup.ok) console.error(` vLLM rollback warning: ${cleanup.reason}`); + } + console.error(` vLLM install failed: ${String(ready.reason)}`); + return { ok: false }; + } + + const authBoundary = verifyDualStationVllmAuthBoundary( + start.baseUrl, + dualStationApiKey, + servedModelId, + ); + if (!authBoundary.ok) { + if (!start.reusedExisting) { + const cleanup = await cleanupDualStationManagedVllm(dualStationPlan); + if (!cleanup.ok) console.error(` vLLM rollback warning: ${cleanup.reason}`); + } + console.error(` vLLM install failed: ${authBoundary.reason}`); + return { ok: false }; + } + + if (!areDualStationManagedVllmContainersRunning(dualStationPlan)) { + if (!start.reusedExisting) { + const cleanup = await cleanupDualStationManagedVllm(dualStationPlan); + if (!cleanup.ok) console.error(` vLLM rollback warning: ${cleanup.reason}`); + } + console.error(" vLLM distributed containers exited unexpectedly after readiness"); + return { ok: false }; + } + + console.log(` ✓ vLLM ready across two DGX Stations at ${start.baseUrl}`); + return { ok: true }; + }); + } catch (error) { + console.error( + ` vLLM install failed: dual-Station lifecycle lock failed: ${(error as Error).message}`, + ); + return { ok: false }; + } + } + const start = startContainer(runtimeProfile, model); if (!start.ok) { console.error(` vLLM install failed: ${String(start.reason)}`); @@ -1088,9 +1484,9 @@ export async function installVllm( `Launch can take 5 minutes to ${String(Math.ceil(runtimeProfile.loadTimeoutSec / 60))} minutes`, ); - const ready = await waitForVllmReady(runtimeProfile); + const ready = await waitForVllmReady(runtimeProfile, undefined, localDockerEnv); if (!ready.ok) { - printContainerLogTail(runtimeProfile); + printContainerLogTail(runtimeProfile, localDockerEnv); dockerStop(runtimeProfile.containerName, { env: buildVllmDockerEnv(), ignoreError: true, @@ -1100,7 +1496,7 @@ export async function installVllm( return { ok: false }; } - if (!containerStillRunning(runtimeProfile)) { + if (!containerStillRunning(runtimeProfile, localDockerEnv)) { console.error(" vLLM container exited unexpectedly after readiness"); return { ok: false }; } diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 035d7686d5..8ef5119b43 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -1086,7 +1086,7 @@ const { // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. const handleVllmSelection = createSetupNimVllmHandler({ VLLM_PORT, runCapture, getLocalProviderBaseUrl, getLocalProviderValidationBaseUrl, - isSafeModelId, requireValue, validateOpenAiLikeSelection, + getManagedVllmProviderBinding: localInference.getManagedDualStationVllmProviderBinding, queryVllmModels: (baseUrl, apiKey) => { const result = localInference.probeVllmModels(baseUrl, apiKey); return result.ok ? result.body : ""; }, isSafeModelId, requireValue, validateOpenAiLikeSelection, applyVllmRuntimeContextWindow: localInference.applyVllmRuntimeContextWindow, isDgxSparkHost: () => nim.detectNvidiaPlatform() === "spark", isNemoClawManagedVllmRunning, exitProcess: (code) => process.exit(code), }); diff --git a/src/lib/onboard/inference-providers/types.ts b/src/lib/onboard/inference-providers/types.ts index c3c312f5df..5ec725290f 100644 --- a/src/lib/onboard/inference-providers/types.ts +++ b/src/lib/onboard/inference-providers/types.ts @@ -226,6 +226,7 @@ export type VllmDeps = CommonDeps & { applyLocalInferenceRoute: (provider: string, model: string) => Promise; run: RunFn; VLLM_LOCAL_CREDENTIAL_ENV: string; + getManagedVllmProviderBinding: () => { baseUrl: string; apiKey: string } | null; }; export type OllamaDeps = CommonDeps & { diff --git a/src/lib/onboard/inference-providers/vllm-local.test.ts b/src/lib/onboard/inference-providers/vllm-local.test.ts new file mode 100644 index 0000000000..ab87a6d7b3 --- /dev/null +++ b/src/lib/onboard/inference-providers/vllm-local.test.ts @@ -0,0 +1,104 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; +import type { VllmDeps } from "./types"; +import { setupVllmLocalInference } from "./vllm-local"; + +const CREDENTIAL_ENV = "NEMOCLAW_VLLM_LOCAL_TOKEN"; + +function deps(overrides: Partial = {}): VllmDeps { + return { + runOpenshell: vi.fn(() => ({ status: 0 })), + upsertProvider: vi.fn(() => ({ ok: true })), + verifyInferenceRoute: vi.fn(), + verifyOnboardInferenceSmoke: vi.fn(), + isNonInteractive: () => true, + registry: { updateSandbox: vi.fn() as VllmDeps["registry"]["updateSandbox"] }, + exitProcess: (code) => { + throw new Error(`exit ${code}`); + }, + error: vi.fn(), + log: vi.fn(), + validateLocalProvider: () => ({ ok: true }), + getLocalProviderHealthCheck: () => ["curl", "-sf", "http://127.0.0.1:8000/v1/models"], + getLocalProviderBaseUrl: () => "http://host.openshell.internal:8000/v1", + applyLocalInferenceRoute: async () => false, + run: vi.fn(() => ({ status: 0 })), + VLLM_LOCAL_CREDENTIAL_ENV: CREDENTIAL_ENV, + getManagedVllmProviderBinding: () => null, + ...overrides, + }; +} + +describe("vLLM local provider credential", () => { + it("preserves the literal dummy credential for legacy single-host vLLM", async () => { + const upsertProvider = vi.fn(() => ({ ok: true })); + + await expect( + setupVllmLocalInference( + { model: "served/model", provider: "vllm-local" }, + deps({ upsertProvider }), + ), + ).resolves.toEqual({ done: false }); + + expect(upsertProvider).toHaveBeenCalledWith( + "vllm-local", + "openai", + CREDENTIAL_ENV, + "http://host.openshell.internal:8000/v1", + { [CREDENTIAL_ENV]: "dummy" }, + ); + }); + + it("registers the persisted managed key through provider env, never as an argv field", async () => { + const apiKey = "c".repeat(64); + const upsertProvider = vi.fn(() => ({ ok: true })); + + await expect( + setupVllmLocalInference( + { model: "served/model", provider: "vllm-local" }, + deps({ + upsertProvider, + getManagedVllmProviderBinding: () => ({ + baseUrl: "http://10.40.0.1:8000/v1", + apiKey, + }), + }), + ), + ).resolves.toEqual({ done: false }); + + expect(upsertProvider).toHaveBeenCalledWith( + "vllm-local", + "openai", + CREDENTIAL_ENV, + "http://10.40.0.1:8000/v1", + { [CREDENTIAL_ENV]: apiKey }, + ); + }); + + it("fails closed without rendering credential-loader details", async () => { + const leaked = "d".repeat(64); + const error = vi.fn(); + const upsertProvider = vi.fn(() => ({ ok: true })); + + await expect( + setupVllmLocalInference( + { model: "served/model", provider: "vllm-local" }, + deps({ + error, + upsertProvider, + getManagedVllmProviderBinding: () => { + throw new Error(`unsafe ${leaked}`); + }, + }), + ), + ).rejects.toThrow("exit 1"); + + expect(upsertProvider).not.toHaveBeenCalled(); + expect(error).toHaveBeenCalledWith( + " Managed vLLM authentication state is unsafe or unreadable.", + ); + expect(error.mock.calls.flat().join("\n")).not.toContain(leaked); + }); +}); diff --git a/src/lib/onboard/inference-providers/vllm-local.ts b/src/lib/onboard/inference-providers/vllm-local.ts index 5dfc4581be..cb1c7d1fcc 100644 --- a/src/lib/onboard/inference-providers/vllm-local.ts +++ b/src/lib/onboard/inference-providers/vllm-local.ts @@ -19,6 +19,7 @@ export async function setupVllmLocalInference( applyLocalInferenceRoute, run, VLLM_LOCAL_CREDENTIAL_ENV, + getManagedVllmProviderBinding, exitProcess, error, } = deps; @@ -49,17 +50,26 @@ export async function setupVllmLocalInference( return exitProcess(1); } } - const baseUrl = getLocalProviderBaseUrl(provider); + let managedBinding: ReturnType; + try { + managedBinding = getManagedVllmProviderBinding(); + } catch { + error(" Managed vLLM authentication state is unsafe or unreadable."); + return exitProcess(1); + } + const baseUrl = managedBinding?.baseUrl ?? getLocalProviderBaseUrl(provider); + const providerToken = managedBinding?.apiKey ?? "dummy"; // Use a dedicated internal credential env so the gateway does not pick // up the user's host OPENAI_API_KEY for local vLLM. vLLM does not enforce - // the bearer at runtime, but a dedicated env name prevents accidental - // hijacking. See GH #2519. + // the bearer for legacy single-host installs; managed dual-Station vLLM + // uses the private persisted key. The dedicated env name prevents + // accidental hijacking by a host OPENAI_API_KEY. See GH #2519. const providerResult = upsertProvider( "vllm-local", "openai", VLLM_LOCAL_CREDENTIAL_ENV, baseUrl, - { [VLLM_LOCAL_CREDENTIAL_ENV]: "dummy" }, + { [VLLM_LOCAL_CREDENTIAL_ENV]: providerToken }, ); if (!providerResult.ok) { error(` ${providerResult.message}`); diff --git a/src/lib/onboard/inference-selection-validation.test.ts b/src/lib/onboard/inference-selection-validation.test.ts index bd6f5a0a35..2fce1fd48b 100644 --- a/src/lib/onboard/inference-selection-validation.test.ts +++ b/src/lib/onboard/inference-selection-validation.test.ts @@ -10,6 +10,45 @@ import { OnboardInferenceCapabilityCache } from "./inference-capability-cache"; import { createInferenceSelectionValidationHelpers } from "./inference-selection-validation"; describe("inference selection validation", () => { + it("uses an explicit managed key without forwarding it as a probe option", async () => { + const apiKey = "f".repeat(64); + const getCredential = vi.fn(() => "ambient-key"); + const probeOpenAiLikeEndpoint = vi.fn(() => ({ + ok: true, + api: "openai-completions", + label: "Chat Completions API", + })); + const helpers = createInferenceSelectionValidationHelpers({ + isNonInteractive: () => false, + agentProductName: () => "OpenClaw", + getCredential, + probeOpenAiLikeEndpoint, + promptValidationRecovery: vi.fn(async () => "selection" as const), + }); + const log = vi.spyOn(console, "log").mockImplementation(() => {}); + + await expect( + helpers.validateOpenAiLikeSelection( + "Local vLLM", + "http://10.40.0.1:8000/v1", + "served/model", + null, + undefined, + undefined, + { apiKey, pinnedAddresses: [] }, + ), + ).resolves.toEqual({ ok: true, api: "openai-completions" }); + expect(getCredential).not.toHaveBeenCalled(); + expect(probeOpenAiLikeEndpoint).toHaveBeenCalledWith( + "http://10.40.0.1:8000/v1", + "served/model", + apiKey, + { pinnedAddresses: [], calibrateTimeouts: true }, + ); + expect(log.mock.calls.flat().join("\n")).not.toContain(apiKey); + log.mockRestore(); + }); + it("records a completed Chat Completions selection for the matching smoke check", async () => { const capabilityCache = new OnboardInferenceCapabilityCache(); const helpers = createInferenceSelectionValidationHelpers({ diff --git a/src/lib/onboard/inference-selection-validation.ts b/src/lib/onboard/inference-selection-validation.ts index a6ecab0788..a8f447a43c 100644 --- a/src/lib/onboard/inference-selection-validation.ts +++ b/src/lib/onboard/inference-selection-validation.ts @@ -82,6 +82,11 @@ export interface InferenceSelectionValidationHelpers { retryMessage?: string, helpUrl?: string | null, options?: { + /** In-memory credential for managed local endpoints; never read from ambient env. */ + apiKey?: string | null; + /** Approved no-DNS endpoint pin; [] also disables ambient proxies for managed IP URLs. */ + pinnedAddresses?: readonly string[]; + trustedPrivateCapability?: TrustedPrivateEndpointCapability; authMode?: "bearer" | "query-param"; extraHeaders?: readonly string[]; requireResponsesToolCalling?: boolean; @@ -245,6 +250,9 @@ export function createInferenceSelectionValidationHelpers( retryMessage = "Please choose a provider/model again.", helpUrl: string | null = null, options: { + apiKey?: string | null; + pinnedAddresses?: readonly string[]; + trustedPrivateCapability?: TrustedPrivateEndpointCapability; authMode?: "bearer" | "query-param"; extraHeaders?: readonly string[]; requireResponsesToolCalling?: boolean; @@ -255,13 +263,19 @@ export function createInferenceSelectionValidationHelpers( capabilityCache?: OnboardInferenceCapabilityCache; } = {}, ): Promise { - const apiKey = credentialEnv ? resolveCredential(credentialEnv) : ""; + const { apiKey: explicitApiKey, ...probeOptions } = options; + const apiKey = + explicitApiKey !== undefined + ? explicitApiKey + : credentialEnv + ? resolveCredential(credentialEnv) + : ""; const probe = await runOpenAiLikeProbe(endpointUrl, model, apiKey, { - ...options, + ...probeOptions, calibrateTimeouts: true, }); if (!probe.ok) { - options.capabilityCache?.invalidate(); + probeOptions.capabilityCache?.invalidate(); printValidationFailure(label, probe); if (deps.isNonInteractive()) { exitNonInteractiveValidationFailure(); @@ -285,12 +299,12 @@ export function createInferenceSelectionValidationHelpers( } const api = probe.api ?? "openai-completions"; if (api === "openai-completions" && probe.validated !== false) { - options.capabilityCache?.rememberCompletedOpenAiChat({ + probeOptions.capabilityCache?.rememberCompletedOpenAiChat({ endpointUrl, model, - authMode: options.authMode, - requireChatCompletionsToolCalling: options.requireChatCompletionsToolCalling, - extraHeaders: options.extraHeaders, + authMode: probeOptions.authMode, + requireChatCompletionsToolCalling: probeOptions.requireChatCompletionsToolCalling, + extraHeaders: probeOptions.extraHeaders, }); } return { ok: true, api }; diff --git a/src/lib/onboard/provider-host-state.ts b/src/lib/onboard/provider-host-state.ts index fcbd75f02c..64b274dd67 100644 --- a/src/lib/onboard/provider-host-state.ts +++ b/src/lib/onboard/provider-host-state.ts @@ -2,8 +2,12 @@ // SPDX-License-Identifier: Apache-2.0 import { dockerCapture as defaultDockerCapture } from "../adapters/docker"; -import { OLLAMA_PORT, VLLM_PORT } from "../core/ports"; -import { findReachableOllamaHost, OLLAMA_HOST_DOCKER_INTERNAL } from "../inference/local"; +import { OLLAMA_PORT } from "../core/ports"; +import { + findReachableOllamaHost, + getLocalProviderAvailabilityEndpoint, + OLLAMA_HOST_DOCKER_INTERNAL, +} from "../inference/local"; import type { NvidiaPlatform } from "../inference/nim"; import { detectVllmProfile, type VllmProfile } from "../inference/vllm"; import { buildVllmDockerEnv } from "../inference/vllm-docker-env"; @@ -112,10 +116,14 @@ function buildDeps( } function probeVllmRunning(runCapture: RunCapture): boolean { - return !!runCapture( - ["curl", "-sf", ...LOCAL_PROVIDER_PROBE_CURL_ARGS, `http://127.0.0.1:${VLLM_PORT}/v1/models`], - { ignoreError: true }, - ); + const endpoint = getLocalProviderAvailabilityEndpoint("vllm-local"); + if (!endpoint) return false; + const writeOut = endpoint.endsWith("/health") + ? ["--noproxy", "*", "--write-out", "%{http_code}"] + : []; + return !!runCapture(["curl", "-sf", ...LOCAL_PROVIDER_PROBE_CURL_ARGS, ...writeOut, endpoint], { + ignoreError: true, + }); } function probeWindowsOllamaReachable(input: { diff --git a/src/lib/onboard/setup-inference.ts b/src/lib/onboard/setup-inference.ts index 3026fdf565..7f35c65551 100644 --- a/src/lib/onboard/setup-inference.ts +++ b/src/lib/onboard/setup-inference.ts @@ -14,6 +14,7 @@ import { isAdvisoryGatewayRouteConflict, } from "../inference/gateway-route-compatibility"; import { withGatewayRouteMutationLock } from "../inference/gateway-route-mutation-lock"; +import { getManagedDualStationVllmProviderBinding } from "../inference/local"; import { assertNoExplicitOpenShellGatewayEndpoint, assertNoOpenShellGatewayEndpointOverride, @@ -105,6 +106,7 @@ export type SetupInferenceDeps = ProviderBranchDeps & { updateSandbox: typeof import("../state/registry").reserveSandboxInferenceRoute; localInferenceTimeoutSecs: number; vllmLocalCredentialEnv: string; + getManagedVllmProviderBinding?: () => { baseUrl: string; apiKey: string } | null; ollamaProxyCredentialEnv: string; isRoutedInferenceProvider: (provider: string) => boolean; applyLocalInferenceRoute?: VllmDeps["applyLocalInferenceRoute"]; @@ -418,6 +420,8 @@ export function createSetupInference( ), run: deps.run, VLLM_LOCAL_CREDENTIAL_ENV: deps.vllmLocalCredentialEnv, + getManagedVllmProviderBinding: + deps.getManagedVllmProviderBinding ?? getManagedDualStationVllmProviderBinding, }, ); if (outcome.done) return outcome.result; diff --git a/src/lib/onboard/setup-nim-flow.test.ts b/src/lib/onboard/setup-nim-flow.test.ts index 47db126cf8..1ef02d32b8 100644 --- a/src/lib/onboard/setup-nim-flow.test.ts +++ b/src/lib/onboard/setup-nim-flow.test.ts @@ -895,7 +895,7 @@ describe("createSetupNim", () => { }), ); - await expect(setupNim(null)).rejects.toThrow("vLLM is already running on localhost:8000"); + await expect(setupNim(null)).rejects.toThrow("vLLM is already running on the managed endpoint"); expect(error).toHaveBeenCalledWith(expect.stringContaining("Select Local vLLM")); expect(error).toHaveBeenCalledWith(expect.stringContaining("stop the existing server")); diff --git a/src/lib/onboard/setup-nim-flow.ts b/src/lib/onboard/setup-nim-flow.ts index 6495aec12e..ec4ee81d81 100644 --- a/src/lib/onboard/setup-nim-flow.ts +++ b/src/lib/onboard/setup-nim-flow.ts @@ -566,7 +566,7 @@ export function createSetupNim( } if (vllmRunning) { const message = - `vLLM is already running on localhost:${String(deps.vllmPort)}. ` + + "vLLM is already running on the managed endpoint. " + "Select Local vLLM, or stop the existing server before selecting the managed install path."; deps.error(` ${message}`); if (deps.isNonInteractive()) { diff --git a/src/lib/onboard/setup-nim-vllm.test.ts b/src/lib/onboard/setup-nim-vllm.test.ts index 9f45d705e7..8037928adf 100644 --- a/src/lib/onboard/setup-nim-vllm.test.ts +++ b/src/lib/onboard/setup-nim-vllm.test.ts @@ -30,6 +30,10 @@ function deps(overrides: Partial = {}): SetupNimVllmDeps { runCapture: () => JSON.stringify({ data: [{ id: "served/model" }] }), getLocalProviderBaseUrl: () => "http://host.openshell.internal:8000/v1", getLocalProviderValidationBaseUrl: () => "http://127.0.0.1:8000/v1", + getManagedVllmProviderBinding: () => null, + queryVllmModels: () => { + throw new Error("unexpected authenticated vLLM query"); + }, isSafeModelId: () => true, requireValue, validateOpenAiLikeSelection: async () => ({ ok: true, api: "openai-completions" }), @@ -76,6 +80,122 @@ describe("setupNim vLLM route containment", () => { expect(events).toEqual(["preflight", "probe", "exact", "validate"]); }); + it("authenticates managed model discovery and OpenAI validation without exposing the key", async () => { + const apiKey = "a".repeat(64); + const runCapture = vi.fn(() => ""); + const queryVllmModels = vi.fn(() => JSON.stringify({ data: [{ id: "served/model" }] })); + const validateOpenAiLikeSelection = vi.fn(async () => ({ + ok: true, + api: "openai-completions", + })); + const handler = createSetupNimVllmHandler( + deps({ + runCapture, + getLocalProviderBaseUrl: () => "http://10.40.0.1:8000/v1", + getLocalProviderValidationBaseUrl: () => "http://10.40.0.1:8000/v1", + getManagedVllmProviderBinding: () => ({ + baseUrl: "http://10.40.0.1:8000/v1", + apiKey, + }), + queryVllmModels, + validateOpenAiLikeSelection, + }), + ); + + await expect(handler(state(null))).resolves.toBe("selected"); + expect(runCapture).not.toHaveBeenCalled(); + expect(queryVllmModels).toHaveBeenCalledWith("http://10.40.0.1:8000/v1", apiKey); + expect(validateOpenAiLikeSelection).toHaveBeenCalledWith( + "Local vLLM", + "http://10.40.0.1:8000/v1", + "served/model", + null, + undefined, + undefined, + expect.objectContaining({ + apiKey, + pinnedAddresses: [], + trustedPrivateCapability: expect.objectContaining({ addresses: ["10.40.0.1"] }), + }), + ); + const renderedOutput = [ + ...vi.mocked(console.log).mock.calls, + ...vi.mocked(console.error).mock.calls, + ...vi.mocked(console.warn).mock.calls, + ] + .flat() + .join("\n"); + expect(renderedOutput).not.toContain(apiKey); + expect(renderedOutput).toContain("Using managed dual-Station vLLM endpoint"); + expect(renderedOutput).not.toContain("localhost:8000"); + }); + + it("uses topology-neutral mismatch recovery for a managed dual endpoint", async () => { + const selection = state("required/model"); + const validateOpenAiLikeSelection = vi.fn(async () => ({ ok: true })); + const handler = createSetupNimVllmHandler( + deps({ + getLocalProviderBaseUrl: () => "http://10.40.0.1:8000/v1", + getLocalProviderValidationBaseUrl: () => "http://10.40.0.1:8000/v1", + getManagedVllmProviderBinding: () => ({ + baseUrl: "http://10.40.0.1:8000/v1", + apiKey: "a".repeat(64), + }), + queryVllmModels: () => JSON.stringify({ data: [{ id: "served/model" }] }), + validateOpenAiLikeSelection, + }), + ); + + await expect(handler(selection)).rejects.toThrow("exit 1"); + expect(validateOpenAiLikeSelection).not.toHaveBeenCalled(); + expect(console.error).toHaveBeenCalledWith( + " To install 'required/model', stop the managed dual-Station vLLM deployment, then rerun the original install/onboard command.", + ); + expect(vi.mocked(console.error).mock.calls.flat().join("\n")).not.toContain("localhost"); + }); + + it("fails closed before probing a managed dual endpoint whose key is missing", async () => { + const runCapture = vi.fn(() => ""); + const queryVllmModels = vi.fn(() => ""); + const handler = createSetupNimVllmHandler( + deps({ + runCapture, + getManagedVllmProviderBinding: () => { + throw new Error("Managed dual-Station vLLM authentication is missing."); + }, + queryVllmModels, + }), + ); + + await expect(handler(state(null))).rejects.toThrow("exit 1"); + expect(runCapture).not.toHaveBeenCalled(); + expect(queryVllmModels).not.toHaveBeenCalled(); + expect(console.error).toHaveBeenCalledWith( + " Managed vLLM authentication state is unsafe or unreadable.", + ); + }); + + it("fails closed before endpoint probes when managed auth state is unsafe", async () => { + const queryVllmModels = vi.fn(() => ""); + const validateOpenAiLikeSelection = vi.fn(async () => ({ ok: true })); + const handler = createSetupNimVllmHandler( + deps({ + getManagedVllmProviderBinding: () => { + throw new Error(`unsafe ${"b".repeat(64)}`); + }, + queryVllmModels, + validateOpenAiLikeSelection, + }), + ); + + await expect(handler(state(null))).rejects.toThrow("exit 1"); + expect(queryVllmModels).not.toHaveBeenCalled(); + expect(validateOpenAiLikeSelection).not.toHaveBeenCalled(); + expect(console.error).toHaveBeenCalledWith( + " Managed vLLM authentication state is unsafe or unreadable.", + ); + }); + it("rejects a detected model that differs from the durable shared route before validation", async () => { const validate = vi.fn(async () => ({ ok: true })); const selection = state("required/model"); diff --git a/src/lib/onboard/setup-nim-vllm.ts b/src/lib/onboard/setup-nim-vllm.ts index ead86ae713..32ab4f83b8 100644 --- a/src/lib/onboard/setup-nim-vllm.ts +++ b/src/lib/onboard/setup-nim-vllm.ts @@ -1,6 +1,10 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { + assertEndpointResolvesPublic, + type TrustedPrivateEndpointCapability, +} from "../inference/endpoint-ssrf-preflight"; import { cliName } from "./branding"; import type { SetupNimSelectionResult, SetupNimSelectionState } from "./setup-nim-flow"; @@ -25,6 +29,8 @@ export interface SetupNimVllmDeps { runCapture(args: string[], options: { ignoreError: boolean }): string; getLocalProviderBaseUrl(provider: string): string | null; getLocalProviderValidationBaseUrl(provider: string): string | null; + getManagedVllmProviderBinding(): { baseUrl: string; apiKey: string } | null; + queryVllmModels(baseUrl: string, apiKey: string): string; isSafeModelId(model: string): boolean; requireValue(value: T | null | undefined, message: string): T; validateOpenAiLikeSelection( @@ -32,6 +38,13 @@ export interface SetupNimVllmDeps { endpointUrl: string, model: string, credentialEnv: string | null, + retryMessage?: string, + helpUrl?: string | null, + options?: { + apiKey?: string | null; + pinnedAddresses?: readonly string[]; + trustedPrivateCapability?: TrustedPrivateEndpointCapability; + }, ): Promise<{ ok: boolean; retry?: string; api?: string | null }>; applyVllmRuntimeContextWindow(models: VllmModels, model: string): void; isDgxSparkHost?: () => boolean; @@ -48,6 +61,21 @@ const NO_QUANTIZATION_VALUES = new Set(["", "false", "none", "null", "unquantize type ModelSizeClass = "large" | "small" | "unknown"; +async function managedVllmValidationOptions(baseUrl: string, apiKey: string) { + const hostname = new URL(baseUrl).hostname.replace(/^\[|\]$/g, ""); + const preflight = await assertEndpointResolvesPublic(baseUrl, undefined, { + trustedPrivateHosts: [hostname], + }); + if (!preflight.ok || !preflight.trustedPrivateCapability) { + throw new Error("Managed dual-Station vLLM endpoint authorization failed."); + } + return { + apiKey, + pinnedAddresses: preflight.addresses ?? [], + trustedPrivateCapability: preflight.trustedPrivateCapability, + }; +} + /** Parse positive integer metadata reported by vLLM model endpoints. */ function parsePositiveInteger(value: unknown): number | null { const normalized = typeof value === "number" ? value : Number(String(value ?? "").trim()); @@ -155,10 +183,16 @@ export function createSetupNimVllmHandler( state: SetupNimSelectionState, options: SetupNimVllmSelectionOptions = {}, ): Promise { - console.log(` ✓ Using existing vLLM on localhost:${deps.VLLM_PORT}`); state.provider = "vllm-local"; state.credentialEnv = null; - state.endpointUrl = deps.getLocalProviderBaseUrl(state.provider); + let managedBinding: ReturnType; + try { + managedBinding = deps.getManagedVllmProviderBinding(); + } catch { + console.error(" Managed vLLM authentication state is unsafe or unreadable."); + deps.exitProcess(1); + } + state.endpointUrl = managedBinding?.baseUrl ?? deps.getLocalProviderBaseUrl(state.provider); if (!state.endpointUrl) { console.error(" Local vLLM base URL could not be determined."); deps.exitProcess(1); @@ -167,15 +201,33 @@ export function createSetupNimVllmHandler( state.assertRouteCompatible?.(); const requiredModel = typeof state.model === "string" ? state.model : null; - const raw = deps.runCapture(["curl", "-sf", `http://127.0.0.1:${deps.VLLM_PORT}/v1/models`], { - ignoreError: true, - }); + const validationBaseUrl = + managedBinding?.baseUrl ?? deps.getLocalProviderValidationBaseUrl(state.provider); + if (!validationBaseUrl) { + console.error(" Local vLLM validation URL could not be determined."); + deps.exitProcess(1); + } + + const apiKey = managedBinding?.apiKey ?? null; + const managedDualEndpoint = managedBinding !== null; + console.log( + managedDualEndpoint + ? " ✓ Using managed dual-Station vLLM endpoint" + : ` ✓ Using existing vLLM on localhost:${deps.VLLM_PORT}`, + ); + const raw = apiKey + ? deps.queryVllmModels(validationBaseUrl, apiKey) + : deps.runCapture(["curl", "-sf", `${validationBaseUrl}/models`], { + ignoreError: true, + }); let models: VllmModels; try { models = JSON.parse(raw); } catch { console.error( - ` Could not query vLLM models endpoint. Is vLLM running on localhost:${deps.VLLM_PORT}?`, + managedDualEndpoint + ? " Could not query the managed dual-Station vLLM models endpoint. Is the deployment running and reachable?" + : ` Could not query vLLM models endpoint. Is vLLM running on localhost:${deps.VLLM_PORT}?`, ); deps.exitProcess(1); } @@ -196,7 +248,9 @@ export function createSetupNimVllmHandler( ` Detected vLLM model '${detectedModel}' does not match the shared gateway route '${requiredModel}'.`, ); console.error( - ` To install '${requiredModel}', stop the existing vLLM server on localhost:${deps.VLLM_PORT}, then rerun the original install/onboard command.`, + managedDualEndpoint + ? ` To install '${requiredModel}', stop the managed dual-Station vLLM deployment, then rerun the original install/onboard command.` + : ` To install '${requiredModel}', stop the existing vLLM server on localhost:${deps.VLLM_PORT}, then rerun the original install/onboard command.`, ); console.error(` To keep '${detectedModel}' instead, start detailed setup:`); console.error(" unset NEMOCLAW_PROVIDER NEMOCLAW_MODEL NEMOCLAW_VLLM_MODEL"); @@ -220,17 +274,36 @@ export function createSetupNimVllmHandler( } } - const validationBaseUrl = deps.getLocalProviderValidationBaseUrl(state.provider); - if (!validationBaseUrl) { - console.error(" Local vLLM validation URL could not be determined."); - deps.exitProcess(1); + const validationModel = deps.requireValue(state.model, "Expected a detected vLLM model"); + let managedValidationOptions: Awaited> | null = + null; + if (apiKey) { + try { + managedValidationOptions = await managedVllmValidationOptions(validationBaseUrl, apiKey); + } catch { + console.error(" Managed vLLM endpoint authorization could not be verified."); + deps.exitProcess(1); + } } - const validation = await deps.validateOpenAiLikeSelection( - "Local vLLM", - validationBaseUrl, - deps.requireValue(state.model, "Expected a detected vLLM model"), - null, - ); + const validation = apiKey + ? await deps.validateOpenAiLikeSelection( + "Local vLLM", + validationBaseUrl, + validationModel, + null, + undefined, + undefined, + deps.requireValue( + managedValidationOptions, + "Expected managed vLLM validation authorization", + ), + ) + : await deps.validateOpenAiLikeSelection( + "Local vLLM", + validationBaseUrl, + validationModel, + null, + ); if (validation.retry === "selection" || validation.retry === "model" || !validation.ok) { return "retry-selection"; } From c5e6ab8e28704460819bd4da6f12c90fe1d350d9 Mon Sep 17 00:00:00 2001 From: Senthil Kumar Ravichandran Date: Thu, 16 Jul 2026 11:22:32 -0700 Subject: [PATCH 16/74] fix(installer): tighten Station host safety gates Signed-off-by: Senthil Kumar Ravichandran --- docs/get-started/prerequisites.mdx | 2 + scripts/install.sh | 1 + scripts/prepare-dgx-station-host.sh | 70 +++++++- test/install-station-host-preparation.test.ts | 161 +++++++++++++++++- 4 files changed, 227 insertions(+), 7 deletions(-) diff --git a/docs/get-started/prerequisites.mdx b/docs/get-started/prerequisites.mdx index 65cfa80c0f..6ed5617aa8 100644 --- a/docs/get-started/prerequisites.mdx +++ b/docs/get-started/prerequisites.mdx @@ -53,6 +53,8 @@ If the `docker --gpus all` acceptance probe fails, preparation registers the NVI If registration or a post-change acceptance probe fails, preparation restores the prior Docker daemon configuration; if restoration fails, it reports the backup path and stops. After successful registration, the runtime remains configured until the same acceptance probe succeeds through a replacement Docker and NVIDIA runtime integration. It requires Secure Boot to be disabled, matching headers for the running kernel, at least 20 GiB free on the root filesystem, and no active agent, inference, or Docker workloads. +It also stops when systemd reports a failed unit unless the unit matches an exact, condition-qualified state from the generic Station image: the pinned OEM `cloud-init` telemetry failure, a network-wait failure while current network health is established, masked `fwupd`, or an SSSD socket on a host without SSSD configuration. +Any other failed unit blocks preparation for administrator review. It does not install a host CUDA toolkit or Docker Compose. If any other existing prerequisite version differs, preparation stops instead of changing it automatically. After changing pinned packages, the installer exits with status `10`; reboot, sign in, and run the printed command, which pins the exact accepted NemoClaw commit before resuming express setup. diff --git a/scripts/install.sh b/scripts/install.sh index 66e49b5178..c9b6779397 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -3168,6 +3168,7 @@ describe_express_install() { inference_disclosure="Managed vLLM pulls the pinned Station image and approximately 352 GB model, then runs a local inference container." fi printf " Station host setup reuses exact prerequisite versions, applies the reviewed factory DKMS transition when present, installs missing pinned driver, Docker, and NVIDIA Container Toolkit packages, and may require one reboot.\n" + printf " Host setup may add this trusted local account to the docker group, which grants root-equivalent control. This flow is only for trusted single-user development hosts; shared or managed hosts require an organization-approved Docker access path.\n" printf " DGX Station remains Deferred; this recipe has not completed end-to-end validation on physical hardware.\n" sandbox_summary="${NEMOCLAW_SANDBOX_NAME:-my-assistant}" ;; diff --git a/scripts/prepare-dgx-station-host.sh b/scripts/prepare-dgx-station-host.sh index ec54744f2a..5633e203b1 100755 --- a/scripts/prepare-dgx-station-host.sh +++ b/scripts/prepare-dgx-station-host.sh @@ -5,9 +5,15 @@ set -Eeuo pipefail umask 077 -readonly SCRIPT_VERSION="2026-07-16.1" +readonly SCRIPT_VERSION="2026-07-16.2" readonly REBOOT_REQUIRED_EXIT=10 readonly MIN_FREE_KIB=$((20 * 1024 * 1024)) +# The qualified generic image currently ships this OEM telemetry bootcmd. Its +# exception disappears automatically when the file changes or the bootcmd +# failure is fixed; update the pin only with a newly audited image. +readonly FACTORY_CLOUD_INIT_TELEMETRY="/etc/cloud/telemetry-bootcmd-event.py" +readonly FACTORY_CLOUD_INIT_RESULT="/run/cloud-init/result.json" +readonly FACTORY_CLOUD_INIT_TELEMETRY_SHA256="09a526c73fcbbe238db56f0ba4ce90a5a0634bab14b5122b016089d581f07275" readonly CUDA_KEYRING_URL="https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/sbsa/cuda-keyring_1.1-1_all.deb" readonly CUDA_KEYRING_SHA256="6ea7d2737648936820e85677177957a0f6521b840d98eb0bbae0a4f003fa7249" @@ -47,6 +53,7 @@ MODE="" LOG_FILE="" DOCKER_GROUP_ADDED=0 CDI_LIFECYCLE_READY=0 +NETWORK_VALIDATED=0 info() { printf '[station-prepare] %s %s\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$*" @@ -107,6 +114,57 @@ is_driver_transitional_unit() { [[ "${1:-}" == "nvidia-persistenced.service" ]] } +root_owned_file_is_not_writable_by_group_or_other() { + local metadata kind uid gid mode + metadata="$(stat -Lc '%F|%u|%g|%a' "$1" 2>/dev/null)" || return 1 + IFS='|' read -r kind uid gid mode <<<"$metadata" + [[ "$kind" == "regular file" && "$uid" == "0" && "$gid" == "0" && "$mode" =~ ^[0-7]{3,4}$ ]] \ + || return 1 + (((8#$mode & 0022) == 0)) +} + +cloud_init_failure_is_qualified() { + local actual_sha + ((NETWORK_VALIDATED == 1)) || return 1 + root_owned_file_is_not_writable_by_group_or_other "$FACTORY_CLOUD_INIT_TELEMETRY" \ + || return 1 + root_owned_file_is_not_writable_by_group_or_other "$FACTORY_CLOUD_INIT_RESULT" || return 1 + actual_sha="$(sha256sum "$FACTORY_CLOUD_INIT_TELEMETRY" 2>/dev/null | awk '{print $1}')" + [[ "$actual_sha" == "$FACTORY_CLOUD_INIT_TELEMETRY_SHA256" ]] || return 1 + grep -Fq "\"('bootcmd', ProcessExecutionError(" "$FACTORY_CLOUD_INIT_RESULT" +} + +network_wait_failure_is_qualified() { + ((NETWORK_VALIDATED == 1)) \ + && systemctl is-active --quiet NetworkManager.service \ + && systemctl is-active --quiet network-online.target +} + +fwupd_refresh_failure_is_qualified() { + local state + ((NETWORK_VALIDATED == 1)) || return 1 + state="$(systemctl is-enabled fwupd.service 2>/dev/null)" || true + [[ "$state" == "masked" ]] +} + +sssd_socket_failure_is_qualified() { + [[ ! -e /etc/sssd/sssd.conf && ! -L /etc/sssd/sssd.conf ]] +} + +is_qualified_factory_failed_unit() { + case "${1:-}" in + cloud-init.service) cloud_init_failure_is_qualified ;; + NetworkManager-wait-online.service | systemd-networkd-wait-online.service) + network_wait_failure_is_qualified + ;; + fwupd-refresh.service) fwupd_refresh_failure_is_qualified ;; + sssd-autofs.socket | sssd-nss.socket | sssd-pam.socket | sssd-pam-priv.socket) + sssd_socket_failure_is_qualified + ;; + *) return 1 ;; + esac +} + package_name() { printf '%s\n' "${1%%=*}" } @@ -246,6 +304,7 @@ check_network() { for host in developer.download.nvidia.com download.docker.com registry-1.docker.io; do getent ahosts "$host" >/dev/null 2>&1 || fatal "DNS resolution failed for ${host}" done + NETWORK_VALIDATED=1 info "network=required_vendor_hosts_resolve" } @@ -274,11 +333,14 @@ check_failed_units() { elif is_preparation_critical_unit "$unit"; then warn "failed preparation-critical unit: ${unit}" blocking=1 + elif is_qualified_factory_failed_unit "$unit"; then + warn "condition-qualified generic-image failed unit: ${unit}" else - warn "pre-existing failed unit outside the Station preparation transaction: ${unit}" + warn "unqualified failed unit: ${unit}" + blocking=1 fi done - ((blocking == 0)) || fatal "Failed driver or container runtime units block Station preparation" + ((blocking == 0)) || fatal "Unqualified failed system units block Station preparation" } check_no_workloads() { @@ -398,7 +460,9 @@ common_preflight() { require_command df require_command dpkg-query require_command getent + require_command grep require_command ps + require_command sha256sum require_command ss require_command stat require_command systemctl diff --git a/test/install-station-host-preparation.test.ts b/test/install-station-host-preparation.test.ts index 6dcecc400d..402dca36ad 100644 --- a/test/install-station-host-preparation.test.ts +++ b/test/install-station-host-preparation.test.ts @@ -141,16 +141,30 @@ assert_no_package_mismatches expect(output).toMatch(/refusing to change them automatically/); }); - it("warns about unrelated failed units but blocks failed runtime units", () => { - const unrelated = runSourced( + it("allows only condition-qualified factory failures and blocks other failed units", () => { + const qualified = runSourced( STATION_PREPARE, ` systemctl() { printf 'cloud-init.service loaded failed failed Cloud init\n'; } +cloud_init_failure_is_qualified() { return 0; } +check_failed_units +`, + ); + expect(qualified.result.status, qualified.output).toBe(0); + expect(qualified.output).toMatch( + /condition-qualified generic-image failed unit: cloud-init.service/, + ); + + const unrelated = runSourced( + STATION_PREPARE, + ` +systemctl() { printf 'ssh.service loaded failed failed SSH\n'; } check_failed_units `, ); - expect(unrelated.result.status, unrelated.output).toBe(0); - expect(unrelated.output).toMatch(/outside the Station preparation transaction/); + expect(unrelated.result.status, unrelated.output).not.toBe(0); + expect(unrelated.output).toMatch(/unqualified failed unit: ssh.service/); + expect(unrelated.output).toMatch(/Unqualified failed system units block Station preparation/); const critical = runSourced( STATION_PREPARE, @@ -163,6 +177,145 @@ check_failed_units expect(critical.output).toMatch(/failed preparation-critical unit: docker.service/); }); + it("qualifies network-wait failures only after current network health is established", () => { + const healthy = runSourced( + STATION_PREPARE, + ` +NETWORK_VALIDATED=1 +systemctl() { + case "$*" in + 'is-active --quiet NetworkManager.service'|'is-active --quiet network-online.target') return 0 ;; + *) return 1 ;; + esac +} +network_wait_failure_is_qualified +`, + ); + expect(healthy.result.status, healthy.output).toBe(0); + + const unvalidated = runSourced( + STATION_PREPARE, + ` +NETWORK_VALIDATED=0 +systemctl() { return 0; } +network_wait_failure_is_qualified +`, + ); + expect(unvalidated.result.status, unvalidated.output).not.toBe(0); + + const inactiveManager = runSourced( + STATION_PREPARE, + ` +NETWORK_VALIDATED=1 +systemctl() { return 1; } +network_wait_failure_is_qualified +`, + ); + expect(inactiveManager.result.status, inactiveManager.output).not.toBe(0); + }); + + it("qualifies only the pinned OEM cloud-init bootcmd failure", () => { + const qualified = runSourced( + STATION_PREPARE, + ` +NETWORK_VALIDATED=1 +stat() { printf 'regular file|0|0|755\n'; } +sha256sum() { printf '%s %s\n' "$FACTORY_CLOUD_INIT_TELEMETRY_SHA256" "$1"; } +grep() { return 0; } +cloud_init_failure_is_qualified +`, + ); + expect(qualified.result.status, qualified.output).toBe(0); + + const changedTelemetry = runSourced( + STATION_PREPARE, + ` +NETWORK_VALIDATED=1 +stat() { printf 'regular file|0|0|755\n'; } +sha256sum() { printf '%064d %s\n' 0 "$1"; } +grep() { return 0; } +cloud_init_failure_is_qualified +`, + ); + expect(changedTelemetry.result.status, changedTelemetry.output).not.toBe(0); + + const unsafeEvidence = runSourced( + STATION_PREPARE, + ` +NETWORK_VALIDATED=1 +stat() { printf 'regular file|0|0|777\n'; } +sha256sum() { printf '%s %s\n' "$FACTORY_CLOUD_INIT_TELEMETRY_SHA256" "$1"; } +grep() { return 0; } +cloud_init_failure_is_qualified +`, + ); + expect(unsafeEvidence.result.status, unsafeEvidence.output).not.toBe(0); + }); + + it("requires exact conditions for auxiliary factory-image failures", () => { + const maskedFwupd = runSourced( + STATION_PREPARE, + ` +NETWORK_VALIDATED=1 +systemctl() { printf 'masked\n'; } +fwupd_refresh_failure_is_qualified +`, + ); + expect(maskedFwupd.result.status, maskedFwupd.output).toBe(0); + + const enabledFwupd = runSourced( + STATION_PREPARE, + ` +NETWORK_VALIDATED=1 +systemctl() { printf 'enabled\n'; } +fwupd_refresh_failure_is_qualified +`, + ); + expect(enabledFwupd.result.status, enabledFwupd.output).not.toBe(0); + + const exactUnits = runSourced( + STATION_PREPARE, + ` +cloud_init_failure_is_qualified() { return 0; } +network_wait_failure_is_qualified() { return 0; } +fwupd_refresh_failure_is_qualified() { return 0; } +sssd_socket_failure_is_qualified() { return 0; } +for unit in \ + cloud-init.service \ + NetworkManager-wait-online.service \ + systemd-networkd-wait-online.service \ + fwupd-refresh.service \ + sssd-autofs.socket \ + sssd-nss.socket \ + sssd-pam.socket \ + sssd-pam-priv.socket; do + is_qualified_factory_failed_unit "$unit" || exit 1 +done +is_qualified_factory_failed_unit ssh.service && exit 1 +exit 0 +`, + ); + expect(exactUnits.result.status, exactUnits.output).toBe(0); + }); + + it("discloses Docker-group root-equivalent access before Station express consent", () => { + const { result, output } = runSourced( + INSTALLER_PAYLOAD, + ` +STATION_DEEPSEEK=0 +NEMOCLAW_VLLM_MODEL='' +describe_express_install 'DGX Station' +`, + ); + + expect(result.status, output).toBe(0); + expect(output).toContain("docker group, which grants root-equivalent control"); + expect(output).toContain("only for trusted single-user development hosts"); + expect(output).toContain( + "shared or managed hosts require an organization-approved Docker access path", + ); + }); + it("fails closed when failed-service inspection is unavailable", () => { const { result, output } = runSourced( STATION_PREPARE, From 541d7879140e1838cec7589934ed7b2c16415720 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 16 Jul 2026 11:14:16 -0700 Subject: [PATCH 17/74] fix(vllm): address dual-station CodeQL findings Signed-off-by: Aaron Erickson (cherry picked from commit 586a60d9b9d66e0abcb17c90ecf1a13a51948e10) --- src/lib/inference/vllm-api-key.ts | 39 ++++++++----------- .../vllm-station-cluster-lifecycle.test.ts | 20 ++++++++++ .../vllm-station-cluster-lifecycle.ts | 12 +++--- 3 files changed, 44 insertions(+), 27 deletions(-) diff --git a/src/lib/inference/vllm-api-key.ts b/src/lib/inference/vllm-api-key.ts index a2e5e98c26..f8f44fa3ff 100644 --- a/src/lib/inference/vllm-api-key.ts +++ b/src/lib/inference/vllm-api-key.ts @@ -48,34 +48,29 @@ export function loadDualStationVllmApiKey( options: Pick = {}, ): string | null { const filePath = dualStationVllmApiKeyPath(options.stateDir ?? defaultStateDir()); - let before: fs.Stats; - try { - before = fs.lstatSync(filePath); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; - throw error; - } - if (before.isSymbolicLink()) { - throw new Error( - `Refusing to read dual-Station vLLM API key through a symbolic link: ${filePath}`, - ); + const noFollow = fs.constants.O_NOFOLLOW; + if (typeof noFollow !== "number") { + throw new Error("Secure no-follow file opens are unavailable on this platform"); } - assertPrivateRegularFile(before, filePath); - if (before.size < 64 || before.size > 65) { - throw new Error(`Dual-Station vLLM API key file is malformed: ${filePath}`); - } - - const noFollow = fs.constants.O_NOFOLLOW ?? 0; const nonBlock = fs.constants.O_NONBLOCK ?? 0; let fd: number | undefined; try { - fd = fs.openSync(filePath, fs.constants.O_RDONLY | noFollow | nonBlock); + try { + fd = fs.openSync(filePath, fs.constants.O_RDONLY | noFollow | nonBlock); + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + if (code === "ENOENT") return null; + if (code === "ELOOP") { + throw new Error( + `Refusing to read dual-Station vLLM API key through a symbolic link: ${filePath}`, + ); + } + throw error; + } const opened = fs.fstatSync(fd); assertPrivateRegularFile(opened, filePath); - if (opened.dev !== before.dev || opened.ino !== before.ino) { - throw new Error( - `Dual-Station vLLM API key file changed while it was being opened: ${filePath}`, - ); + if (opened.size < 64 || opened.size > 65) { + throw new Error(`Dual-Station vLLM API key file is malformed: ${filePath}`); } const value = fs.readFileSync(fd, "utf8").trim(); if (!DUAL_STATION_VLLM_API_KEY_PATTERN.test(value)) { diff --git a/src/lib/inference/vllm-station-cluster-lifecycle.test.ts b/src/lib/inference/vllm-station-cluster-lifecycle.test.ts index 8da461a585..22e9cb30d6 100644 --- a/src/lib/inference/vllm-station-cluster-lifecycle.test.ts +++ b/src/lib/inference/vllm-station-cluster-lifecycle.test.ts @@ -402,6 +402,26 @@ function harness( } describe("dual-Station managed vLLM run argv", () => { + it("derives stable, distinct service-key fingerprints", () => { + expect(API_KEY_FINGERPRINT).toMatch(/^[a-f0-9]{64}$/u); + expect(dualStationVllmApiKeyFingerprint(API_KEY)).toBe(API_KEY_FINGERPRINT); + expect(dualStationVllmApiKeyFingerprint("f".repeat(64))).not.toBe(API_KEY_FINGERPRINT); + }); + + it("rejects a long slash-heavy runtime image", () => { + const plan = { + ...fixturePlan(), + runtime: { + ...DUAL_STATION_VLLM_RUNTIME, + image: `${"!/".repeat(10_000)}image@sha256:${"a".repeat(64)}`, + }, + } as unknown as DualStationVllmPlan; + + expect(() => + buildDualStationVllmRunArgs(plan, "head", TRANSACTION_ID, API_KEY_FINGERPRINT), + ).toThrow("exact pinned runtime contract"); + }); + it.each(["head", "worker"] as const)("builds the exact %s launch contract", (role) => { const plan = fixturePlan(); const args = buildDualStationVllmRunArgs(plan, role, TRANSACTION_ID, API_KEY_FINGERPRINT); diff --git a/src/lib/inference/vllm-station-cluster-lifecycle.ts b/src/lib/inference/vllm-station-cluster-lifecycle.ts index 9eb4a1b528..5eab1c5e93 100644 --- a/src/lib/inference/vllm-station-cluster-lifecycle.ts +++ b/src/lib/inference/vllm-station-cluster-lifecycle.ts @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { createHash, randomBytes } from "node:crypto"; +import { createHash, createHmac, randomBytes } from "node:crypto"; import net from "node:net"; import os from "node:os"; import path from "node:path"; @@ -39,7 +39,7 @@ const DOCKER_LATE_CREATE_RECONCILE_INTERVAL_MS = 250; const DOCKER_CONTAINER_ID_PATTERN = /^[a-f0-9]{64}$/; const CLUSTER_ID_PATTERN = /^[a-f0-9]{64}$/; const SHA256_HEX_PATTERN = /^[a-f0-9]{64}$/; -const IMMUTABLE_IMAGE_PATTERN = /^[^\s@]+(?:\/[^\s@]+)+@sha256:[a-f0-9]{64}$/; +const IMMUTABLE_IMAGE_PATTERN = /^(?:[^\s/@]+\/)+[^\s/@]+@sha256:[a-f0-9]{64}$/; const IMAGE_ID_PATTERN = /^sha256:[a-f0-9]{64}$/; const SAFE_DEVICE_NAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_.:-]{0,63}$/; const SAFE_UVERBS_DEVICE_PATTERN = /^\/dev\/infiniband\/uverbs[0-9]+$/; @@ -48,7 +48,7 @@ const GPU_SMOKE_NONCE_PATTERN = /^[a-f0-9]{32}$/; const TRANSACTION_ID_PATTERN = /^[a-f0-9]{32}$/; const GPU_SMOKE_CONTAINER_PREFIX = "nemoclaw-vllm-gpu-smoke"; const DUAL_STATION_VLLM_LAUNCH_SCHEMA = "1"; -const API_KEY_FINGERPRINT_DOMAIN = "nemoclaw-dual-station-vllm-api-key\0"; +const VLLM_FINGERPRINT_CONTEXT = "nemoclaw-dual-station-vllm-api-key\0"; export type DualStationVllmRole = "head" | "worker"; @@ -271,14 +271,16 @@ function assertSafeStartConfig(config: DualStationVllmStartConfig): void { } } -/** Domain-separated, non-secret binding for the host-persisted service key. */ +/** Domain-separated, non-secret binding for the host-persisted high-entropy service key. */ export function dualStationVllmApiKeyFingerprint(apiKey: string): string { if (!DUAL_STATION_VLLM_API_KEY_PATTERN.test(apiKey)) { throw new Error( "Dual-Station vLLM API key must be exactly 64 lowercase hexadecimal characters.", ); } - return createHash("sha256").update(API_KEY_FINGERPRINT_DOMAIN).update(apiKey).digest("hex"); + return createHmac("sha256", Buffer.from(apiKey, "hex")) + .update(VLLM_FINGERPRINT_CONTEXT) + .digest("hex"); } function withoutVllmApiKey(env: Record): Record { From 81c80f35a72dfd6ffe00a24ab17ed67e8ec5aa9b Mon Sep 17 00:00:00 2001 From: Senthil Kumar Ravichandran Date: Thu, 16 Jul 2026 11:49:09 -0700 Subject: [PATCH 18/74] fix(installer): fail closed on Station CDI refresh Signed-off-by: Senthil Kumar Ravichandran --- docs/get-started/prerequisites.mdx | 4 +- docs/get-started/quickstart.mdx | 6 +- docs/reference/troubleshooting.mdx | 5 +- scripts/prepare-dgx-station-host.sh | 60 ++------- test/install-station-host-preparation.test.ts | 117 ++---------------- 5 files changed, 25 insertions(+), 167 deletions(-) diff --git a/docs/get-started/prerequisites.mdx b/docs/get-started/prerequisites.mdx index 6ed5617aa8..ca69d5ba40 100644 --- a/docs/get-started/prerequisites.mdx +++ b/docs/get-started/prerequisites.mdx @@ -47,8 +47,8 @@ On those systems, set `NEMOCLAW_PROVIDER` or `NEMOCLAW_NO_EXPRESS=1` explicitly The preparation probes package and runtime state first, reuses exact matches, and installs only missing pinned packages, including the NVIDIA Container Toolkit libraries and `nvidia-ctk` CLI. It permits only the reviewed factory transition from `dkms` `3.0.11-1ubuntu13` to `1:3.4.0-1ubuntu1`. After reboot, preparation enables NVIDIA's packaged CDI refresh path and service, requires the `nvidia.com/gpu=all` device, and verifies it with a real container launch. -If the packaged refresh does not produce that device, it uses NVIDIA's documented transient generation fallback at `/var/run/cdi/nvidia.yaml`; it does not create a persistent manual CDI configuration under `/etc`. -If an administrator has customized the packaged CDI refresh environment, preparation stops instead of bypassing that policy with default direct generation. +If the packaged refresh fails or does not produce that device, preparation prints service diagnostics and stops for administrator repair. +It does not bypass the packaged lifecycle with direct CDI generation. If the `docker --gpus all` acceptance probe fails, preparation registers the NVIDIA Docker runtime only when `docker info` diagnoses that runtime as absent; any other launch failure stops without changing daemon configuration. If registration or a post-change acceptance probe fails, preparation restores the prior Docker daemon configuration; if restoration fails, it reports the backup path and stops. After successful registration, the runtime remains configured until the same acceptance probe succeeds through a replacement Docker and NVIDIA runtime integration. diff --git a/docs/get-started/quickstart.mdx b/docs/get-started/quickstart.mdx index 9123c6d01f..5273d8ce94 100644 --- a/docs/get-started/quickstart.mdx +++ b/docs/get-started/quickstart.mdx @@ -124,7 +124,8 @@ Use these details when your first-run path needs more control. DGX OS, NVIDIA BaseOS images, and other Station generations stop before host preparation. Set `NEMOCLAW_PROVIDER` or `NEMOCLAW_NO_EXPRESS=1` explicitly to continue on an unqualified Station without host automation. It probes the pinned driver, Docker, Buildx, and NVIDIA Container Toolkit versions, reuses exact matches, installs missing pins, and permits only the reviewed factory `dkms` transition. - It establishes NVIDIA CDI through the packaged refresh service, with NVIDIA's transient `/var/run/cdi/nvidia.yaml` generation command as a fallback, then proves both CDI and `--gpus all` with real container launches. + It establishes NVIDIA CDI through the packaged refresh service, then proves both CDI and `--gpus all` with real container launches. + If the packaged refresh fails or does not advertise `nvidia.com/gpu=all`, preparation prints service diagnostics and stops for administrator repair instead of generating CDI configuration directly. After it changes pinned packages, the installer exits with status `10` at the required reboot boundary; reboot, sign in, and run the printed exact-commit command to resume the accepted recipe without another prompt. This automation does not change Station's Deferred support status; physical end-to-end validation remains open. Pass `--station-deepseek` to use DeepSeek V4 Flash for a Station demo instead. @@ -200,7 +201,8 @@ Use these details when your first-run path needs more control. DGX OS, NVIDIA BaseOS images, and other Station generations are outside this automatic preparation boundary. Set `NEMOCLAW_PROVIDER` or `NEMOCLAW_NO_EXPRESS=1` to continue without Station host automation. Preparation reuses exact versions, installs missing pinned packages, permits only the reviewed `dkms` transition from `3.0.11-1ubuntu13` to `1:3.4.0-1ubuntu1`, and refuses every other mismatched version. - It requires the NVIDIA CDI device `nvidia.com/gpu=all`, using the packaged refresh service or NVIDIA's transient fallback, and verifies CDI and `--gpus all` with real container launches. + It requires the packaged NVIDIA CDI refresh service to advertise `nvidia.com/gpu=all`, and verifies CDI and `--gpus all` with real container launches. + If the packaged refresh fails or omits that device, preparation prints service diagnostics and stops for administrator repair instead of generating CDI configuration directly. If NVIDIA Docker runtime registration or a post-change acceptance probe fails, preparation restores the prior Docker daemon configuration. Changing pinned packages exits with status `10` for a reboot; after you sign in and run the printed exact-commit command, the accepted express recipe resumes without another prompt. This automation does not change Station's Deferred support status; physical end-to-end validation remains open. diff --git a/docs/reference/troubleshooting.mdx b/docs/reference/troubleshooting.mdx index aa13c44801..32368fe9b6 100644 --- a/docs/reference/troubleshooting.mdx +++ b/docs/reference/troubleshooting.mdx @@ -2271,7 +2271,8 @@ When shared gateway cleanup would be unsafe, follow the targeted destroy or gate Recent NVIDIA Container Toolkit installs configure the Docker daemon for Container Device Interface (CDI) device injection, which OpenShell's `gateway start --gpu` then auto-selects. If no `nvidia.com/gpu` CDI spec has been generated on the host yet, gateway start fails with `Docker responded with status code 500: CDI device injection failed: unresolvable CDI devices nvidia.com/gpu=all`. -The standard NemoClaw installer detects this gap before onboarding, first tries to enable the NVIDIA CDI refresh systemd units, and falls back to generating the spec directly with `nvidia-ctk`. +Outside Station Express, the standard NemoClaw installer detects this gap before onboarding, first tries to enable the NVIDIA CDI refresh systemd units, and can fall back to generating the spec directly with `nvidia-ctk`. +Station Express requires the packaged refresh lifecycle to work; if it fails or omits `nvidia.com/gpu=all`, inspect `nvidia-cdi-refresh.service`, repair it, and rerun the printed exact-commit install command. If you run `$$nemoclaw onboard` directly, preflight prints the manual remediation instead. The native Linux fix is the same on Docker hosts whose `docker info` advertises a non-empty `CDISpecDirs`. On WSL with Docker Desktop, Docker may advertise CDI directories even though `--device nvidia.com/gpu=all` is not usable from the WSL distro. @@ -2286,7 +2287,7 @@ nvidia-ctk cdi list $$nemoclaw onboard ``` -If the refresh units are unavailable or do not generate CDI devices, generate the spec directly: +For other native Linux installations, if the refresh units are unavailable or do not generate CDI devices, generate the spec directly: ```bash sudo mkdir -p /etc/cdi diff --git a/scripts/prepare-dgx-station-host.sh b/scripts/prepare-dgx-station-host.sh index 5633e203b1..a5d4b35109 100755 --- a/scripts/prepare-dgx-station-host.sh +++ b/scripts/prepare-dgx-station-host.sh @@ -5,7 +5,7 @@ set -Eeuo pipefail umask 077 -readonly SCRIPT_VERSION="2026-07-16.2" +readonly SCRIPT_VERSION="2026-07-16.3" readonly REBOOT_REQUIRED_EXIT=10 readonly MIN_FREE_KIB=$((20 * 1024 * 1024)) # The qualified generic image currently ships this OEM telemetry bootcmd. Its @@ -31,9 +31,6 @@ readonly TARGET_DKMS_VERSION="1:3.4.0-1ubuntu1" readonly ACCEPTANCE_IMAGE="ubuntu@sha256:7f622ca8766bccb22f04242ecb6f19f770b2f08827dc4b8c707de5e78a6da7ab" readonly STATE_DIR="${HOME}/.local/state/station-bootstrap" readonly INSTALL_BOOT_MARKER="${STATE_DIR}/install-boot-id" -readonly CDI_REFRESH_ENV_FILE="/etc/nvidia-container-toolkit/nvidia-cdi-refresh.env" -readonly TRANSIENT_CDI_SPEC_PATH="/var/run/cdi/nvidia.yaml" -readonly TRANSIENT_CDI_SPEC_CANONICAL_PATH="/run/cdi/nvidia.yaml" readonly -a PACKAGE_SPECS=( "dkms=${TARGET_DKMS_VERSION}" @@ -653,18 +650,6 @@ ensure_docker_group() { fi } -cdi_refresh_has_custom_environment() { - local grep_status - sudo test -e "$CDI_REFRESH_ENV_FILE" || return 1 - if sudo grep -Eq '^[[:space:]]*[^#[:space:]]' "$CDI_REFRESH_ENV_FILE"; then - return 0 - else - grep_status=$? - fi - ((grep_status == 1)) && return 1 - fatal "Could not inspect ${CDI_REFRESH_ENV_FILE} before CDI fallback" -} - ensure_cdi_refresh_lifecycle() { ((CDI_LIFECYCLE_READY == 0)) || return 0 check_no_workloads @@ -686,53 +671,22 @@ verify_cdi_refresh_lifecycle() { info "cdi_refresh_lifecycle=verified" } -assert_transient_cdi_output_safe() { - local require_file=${1:-0} resolved_directory - ensure_root_directory_safe /run/cdi /run 0755 "NVIDIA CDI runtime directory" - resolved_directory="$(readlink -f /var/run/cdi)" \ - || fatal "Could not resolve NVIDIA's transient CDI runtime directory" - [[ "$resolved_directory" == "/run/cdi" ]] \ - || fatal "Expected /var/run/cdi to resolve to /run/cdi, found ${resolved_directory}" - sudo test ! -L "$TRANSIENT_CDI_SPEC_CANONICAL_PATH" \ - || fatal "Transient NVIDIA CDI specification must not be a symbolic link: ${TRANSIENT_CDI_SPEC_CANONICAL_PATH}" - if sudo test -e "$TRANSIENT_CDI_SPEC_CANONICAL_PATH"; then - assert_root_regular_file_safe "$TRANSIENT_CDI_SPEC_CANONICAL_PATH" "" "Transient NVIDIA CDI specification" - elif [[ "$require_file" == "1" ]]; then - fatal "Transient NVIDIA CDI generation did not create ${TRANSIENT_CDI_SPEC_CANONICAL_PATH}" - fi -} - refresh_cdi() { check_no_workloads ensure_cdi_refresh_lifecycle if ! sudo systemctl restart nvidia-cdi-refresh.service; then - warn "Packaged CDI refresh failed; collecting diagnostics before the NVIDIA transient fallback" + warn "Packaged CDI refresh failed; collecting diagnostics" sudo systemctl status nvidia-cdi-refresh.service --no-pager || true sudo journalctl -u nvidia-cdi-refresh.service --no-pager -n 50 || true - elif nvidia-ctk cdi list | grep -Fxq 'nvidia.com/gpu=all'; then - info "cdi=nvidia.com/gpu=all source=packaged_refresh_service" - return 0 - else + fatal "Packaged CDI refresh failed; repair nvidia-cdi-refresh.service before rerunning preparation" + fi + if ! nvidia-ctk cdi list | grep -Fxq 'nvidia.com/gpu=all'; then warn "Packaged CDI refresh completed without advertising nvidia.com/gpu=all" sudo systemctl status nvidia-cdi-refresh.service --no-pager || true sudo journalctl -u nvidia-cdi-refresh.service --no-pager -n 50 || true + fatal "Packaged CDI refresh did not advertise nvidia.com/gpu=all; direct CDI generation is not permitted" fi - - # This is NVIDIA's documented fallback and writes the same transient /var/run - # specification as nvidia-cdi-refresh.service. It does not create a persistent - # manual CDI configuration under /etc. - if cdi_refresh_has_custom_environment; then - fatal "${CDI_REFRESH_ENV_FILE} contains administrator overrides; refusing to bypass them with default direct CDI generation" - fi - warn "Generating NVIDIA's transient CDI specification at /var/run/cdi/nvidia.yaml" - check_no_workloads - assert_transient_cdi_output_safe 0 - sudo nvidia-ctk cdi generate --output="$TRANSIENT_CDI_SPEC_PATH" \ - || fatal "Direct transient CDI generation failed; inspect the packaged service diagnostics" - assert_transient_cdi_output_safe 1 - nvidia-ctk cdi list | grep -Fxq 'nvidia.com/gpu=all' \ - || fatal "CDI device nvidia.com/gpu=all is unavailable after direct transient generation" - info "cdi=nvidia.com/gpu=all source=direct_transient_fallback" + info "cdi=nvidia.com/gpu=all source=packaged_refresh_service" } ensure_acceptance_image() { diff --git a/test/install-station-host-preparation.test.ts b/test/install-station-host-preparation.test.ts index 402dca36ad..e5f6916a68 100644 --- a/test/install-station-host-preparation.test.ts +++ b/test/install-station-host-preparation.test.ts @@ -650,148 +650,49 @@ run_apply expect(output).toMatch(/new login before onboarding/); }); - it("falls back to NVIDIA's transient CDI generation when the packaged refresh fails", () => { + it("fails closed when the packaged CDI refresh service fails", () => { const { result, output } = runSourced( STATION_PREPARE, ` -generated=0 check_no_workloads() { printf 'RECHECK_ALL_WORKLOADS\n'; } -assert_transient_cdi_output_safe() { printf 'CDI_OUTPUT_SAFE %s\n' "$1"; } sudo() { printf 'SUDO %s\n' "$*" if [[ "$*" == "systemctl restart nvidia-cdi-refresh.service" ]]; then return 1; fi - if [[ "$*" == "test -e $CDI_REFRESH_ENV_FILE" ]]; then return 1; fi - if [[ "$*" == "nvidia-ctk cdi generate --output=/var/run/cdi/nvidia.yaml" ]]; then generated=1; fi return 0 } -nvidia-ctk() { - if [[ "$*" == "cdi list" && "$generated" == "1" ]]; then printf 'nvidia.com/gpu=all\n'; fi -} refresh_cdi `, ); - expect(result.status, output).toBe(0); + expect(result.status, output).not.toBe(0); expect(output).toContain("systemctl status nvidia-cdi-refresh.service --no-pager"); expect(output).toContain("journalctl -u nvidia-cdi-refresh.service --no-pager -n 50"); expect(output).toContain("RECHECK_ALL_WORKLOADS"); - expect(output).toContain("nvidia-ctk cdi generate --output=/var/run/cdi/nvidia.yaml"); - expect(output).toContain("cdi=nvidia.com/gpu=all source=direct_transient_fallback"); - expect(output).not.toContain("/etc/cdi"); + expect(output).toMatch(/repair nvidia-cdi-refresh\.service/); + expect(output).not.toContain("nvidia-ctk cdi generate"); }); - it("uses the transient CDI fallback when the packaged refresh produces no GPU device", () => { + it("fails closed when the packaged CDI refresh produces no GPU device", () => { const { result, output } = runSourced( STATION_PREPARE, ` -generated=0 check_no_workloads() { printf 'RECHECK_ALL_WORKLOADS\n'; } -assert_transient_cdi_output_safe() { printf 'CDI_OUTPUT_SAFE %s\n' "$1"; } sudo() { printf 'SUDO %s\n' "$*" - if [[ "$*" == "test -e $CDI_REFRESH_ENV_FILE" ]]; then return 1; fi - if [[ "$*" == "nvidia-ctk cdi generate --output=/var/run/cdi/nvidia.yaml" ]]; then generated=1; fi return 0 } -nvidia-ctk() { - if [[ "$*" == "cdi list" && "$generated" == "1" ]]; then printf 'nvidia.com/gpu=all\n'; fi -} +nvidia-ctk() { :; } refresh_cdi `, ); - expect(result.status, output).toBe(0); + expect(result.status, output).not.toBe(0); expect(output).toContain("RECHECK_ALL_WORKLOADS"); expect(output).toMatch(/completed without advertising nvidia\.com\/gpu=all/); - expect(output).toContain("nvidia-ctk cdi generate --output=/var/run/cdi/nvidia.yaml"); - expect(output).toContain("cdi=nvidia.com/gpu=all source=direct_transient_fallback"); - }); - - it("fails closed when direct transient CDI generation also fails", () => { - const { result, output } = runSourced( - STATION_PREPARE, - ` -check_no_workloads() { printf 'RECHECK_ALL_WORKLOADS\n'; } -assert_transient_cdi_output_safe() { printf 'CDI_OUTPUT_SAFE %s\n' "$1"; } -sudo() { - printf 'SUDO %s\n' "$*" - if [[ "$*" == "systemctl restart nvidia-cdi-refresh.service" ]]; then return 1; fi - if [[ "$*" == "test -e $CDI_REFRESH_ENV_FILE" ]]; then return 1; fi - if [[ "$*" == "nvidia-ctk cdi generate --output=/var/run/cdi/nvidia.yaml" ]]; then return 1; fi - return 0 -} -refresh_cdi -`, - ); - - expect(result.status, output).not.toBe(0); expect(output).toContain("systemctl status nvidia-cdi-refresh.service --no-pager"); expect(output).toContain("journalctl -u nvidia-cdi-refresh.service --no-pager -n 50"); - expect(output).toContain("RECHECK_ALL_WORKLOADS"); - expect(output).toContain("nvidia-ctk cdi generate --output=/var/run/cdi/nvidia.yaml"); - expect(output).toMatch(/Direct transient CDI generation failed/); - }); - - it("fails closed when direct CDI generation produces no GPU device", () => { - const { result, output } = runSourced( - STATION_PREPARE, - ` -check_no_workloads() { printf 'RECHECK_ALL_WORKLOADS\n'; } -assert_transient_cdi_output_safe() { printf 'CDI_OUTPUT_SAFE %s\n' "$1"; } -sudo() { - printf 'SUDO %s\n' "$*" - if [[ "$*" == "systemctl restart nvidia-cdi-refresh.service" ]]; then return 1; fi - if [[ "$*" == "test -e $CDI_REFRESH_ENV_FILE" ]]; then return 1; fi - return 0 -} -nvidia-ctk() { :; } -refresh_cdi -`, - ); - - expect(result.status, output).not.toBe(0); - expect(output).toContain("nvidia-ctk cdi generate --output=/var/run/cdi/nvidia.yaml"); - expect(output).toMatch(/unavailable after direct transient generation/); - }); - - it("does not bypass an administrator-customized CDI refresh environment", () => { - const { result, output } = runSourced( - STATION_PREPARE, - ` -check_no_workloads() { printf 'RECHECK_ALL_WORKLOADS\n'; } -sudo() { - printf 'SUDO %s\n' "$*" - if [[ "$*" == "systemctl restart nvidia-cdi-refresh.service" ]]; then return 1; fi - return 0 -} -refresh_cdi -`, - ); - - expect(result.status, output).not.toBe(0); - expect(output).toMatch(/contains administrator overrides/); - expect(output).toMatch(/refusing to bypass/); - expect(output).not.toContain("nvidia-ctk cdi generate --output=/var/run/cdi/nvidia.yaml"); - }); - - it("rejects a symbolic link at the privileged transient CDI output", () => { - const { result, output } = runSourced( - STATION_PREPARE, - ` -ensure_root_directory_safe() { printf 'ENSURE_ROOT_DIRECTORY_SAFE\n'; } -readlink() { printf '/run/cdi\n'; } -sudo() { - printf 'SUDO %s\n' "$*" - if [[ "$*" == "test ! -L /run/cdi/nvidia.yaml" ]]; then return 1; fi - return 0 -} -assert_transient_cdi_output_safe 0 -`, - ); - - expect(result.status, output).not.toBe(0); - expect(output).toContain("ENSURE_ROOT_DIRECTORY_SAFE"); - expect(output).toMatch(/must not be a symbolic link/); + expect(output).toMatch(/direct CDI generation is not permitted/); + expect(output).not.toContain("nvidia-ctk cdi generate"); }); it("rechecks every workload gate immediately before Docker runtime mutation", () => { From 01ddd1f7232bbacc508469310746c7496df29666 Mon Sep 17 00:00:00 2001 From: Senthil Kumar Ravichandran Date: Thu, 16 Jul 2026 12:29:32 -0700 Subject: [PATCH 19/74] fix(installer): avoid Docker restart on workload race --- scripts/prepare-dgx-station-host.sh | 14 ++++--- test/install-station-host-preparation.test.ts | 39 +++++++++++++++++++ 2 files changed, 47 insertions(+), 6 deletions(-) diff --git a/scripts/prepare-dgx-station-host.sh b/scripts/prepare-dgx-station-host.sh index a5d4b35109..9824429d0c 100755 --- a/scripts/prepare-dgx-station-host.sh +++ b/scripts/prepare-dgx-station-host.sh @@ -5,7 +5,7 @@ set -Eeuo pipefail umask 077 -readonly SCRIPT_VERSION="2026-07-16.3" +readonly SCRIPT_VERSION="2026-07-16.4" readonly REBOOT_REQUIRED_EXIT=10 readonly MIN_FREE_KIB=$((20 * 1024 * 1024)) # The qualified generic image currently ships this OEM telemetry bootcmd. Its @@ -763,7 +763,7 @@ configure_docker_runtime_if_needed() { fail_after_docker_runtime_rollback "$backup_dir" "$previous_daemon" "NVIDIA runtime registration produced an unsafe Docker daemon configuration" fi if ! (check_no_workloads); then - fail_after_docker_runtime_rollback "$backup_dir" "$previous_daemon" "A workload appeared before Docker restart" + fail_after_docker_runtime_rollback "$backup_dir" "$previous_daemon" "A workload appeared before Docker restart" 0 fi if ! sudo systemctl restart docker.service; then fail_after_docker_runtime_rollback "$backup_dir" "$previous_daemon" "Docker restart failed after NVIDIA runtime registration" @@ -778,7 +778,7 @@ configure_docker_runtime_if_needed() { } rollback_docker_runtime_config() { - local backup_dir=$1 previous_daemon=$2 + local backup_dir=$1 previous_daemon=$2 restart_after_restore=${3:-1} warn "Restoring the Docker daemon configuration from ${backup_dir}" if [[ "$previous_daemon" == "1" ]]; then root_regular_file_is_safe "${backup_dir}/daemon.json" "" || return 1 @@ -788,12 +788,14 @@ rollback_docker_runtime_config() { else sudo rm -f -- /etc/docker/daemon.json || return 1 fi - sudo systemctl restart docker.service + if [[ "$restart_after_restore" == "1" ]]; then + sudo systemctl restart docker.service + fi } fail_after_docker_runtime_rollback() { - local backup_dir=$1 previous_daemon=$2 reason=$3 - if rollback_docker_runtime_config "$backup_dir" "$previous_daemon"; then + local backup_dir=$1 previous_daemon=$2 reason=$3 restart_after_restore=${4:-1} + if rollback_docker_runtime_config "$backup_dir" "$previous_daemon" "$restart_after_restore"; then fatal "${reason}; the prior Docker daemon configuration was restored" fi fatal "${reason}; automatic Docker daemon rollback failed, restore from ${backup_dir} before retrying" diff --git a/test/install-station-host-preparation.test.ts b/test/install-station-host-preparation.test.ts index e5f6916a68..f167858792 100644 --- a/test/install-station-host-preparation.test.ts +++ b/test/install-station-host-preparation.test.ts @@ -770,6 +770,45 @@ configure_docker_runtime_if_needed expect(output).toContain("docker_gpus_contract=pass"); }); + it("restores configuration without restarting Docker when a workload appears at the restart boundary", () => { + const { result, output } = runSourced( + STATION_PREPARE, + ` +runtime_configured=0 +run_gpus_test_sudo() { return 1; } +run_cdi_test_sudo() { return 0; } +docker_has_nvidia_runtime_sudo() { return 1; } +check_no_workloads() { + printf 'RECHECK_ALL_WORKLOADS configured=%s\n' "$runtime_configured" + [[ "$runtime_configured" == "0" ]] +} +ensure_root_directory_safe() { :; } +assert_root_directory_safe() { :; } +assert_root_regular_file_safe() { :; } +root_regular_file_is_safe() { return 0; } +sudo() { + if [[ "$*" == "mktemp -d /var/backups/station-bootstrap/docker-runtime.XXXXXXXXXX" ]]; then + printf '/var/backups/station-bootstrap/docker-runtime.TEST' + return 0 + fi + [[ "$*" == "test -e /etc/docker/daemon.json" || "$*" == "test -L /etc/docker/daemon.json" ]] && return 1 + if [[ "$*" == "nvidia-ctk runtime configure --runtime=docker" ]]; then + runtime_configured=1 + fi + printf 'SUDO %s\n' "$*" +} +configure_docker_runtime_if_needed +`, + ); + + expect(result.status, output).not.toBe(0); + expect(output).toContain("RECHECK_ALL_WORKLOADS configured=1"); + expect(output).toContain("rm -f -- /etc/docker/daemon.json"); + expect(output).toMatch(/A workload appeared before Docker restart/); + expect(output).toMatch(/prior Docker daemon configuration was restored/); + expect(output).not.toContain("systemctl restart docker.service"); + }); + it("restores the prior Docker configuration when a post-mutation launch probe fails", () => { const { result, output } = runSourced( STATION_PREPARE, From cf46723fb0091b6237326e83cb723388ace5b7db Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 16 Jul 2026 14:08:07 -0700 Subject: [PATCH 20/74] feat(installer): prepare trusted dual DGX Stations Signed-off-by: Aaron Erickson --- ci/platform-matrix.json | 4 +- docs/get-started/prerequisites.mdx | 6 + docs/get-started/quickstart.mdx | 21 +- docs/inference/choose-inference-provider.mdx | 2 +- docs/inference/set-up-vllm.mdx | 14 +- docs/reference/commands.mdx | 3 +- docs/reference/platform-support.mdx | 4 +- docs/resources/starter-prompt.md | 16 +- scripts/checks/vitest-project-overlap.ts | 1 + scripts/install.sh | 385 +++++- scripts/lib/dgx-station-peer.mts | 825 ++++++++++++ scripts/prepare-dgx-station-host.sh | 11 + scripts/prepare-dual-dgx-station.mts | 1030 +++++++++++++++ test/install-express-prompt.test.ts | 18 +- test/install-station-host-preparation.test.ts | 181 ++- test/install-station-pair-preparation.test.ts | 1106 +++++++++++++++++ test/starter-prompt-docs.test.ts | 7 +- test/test-boundary-guards.test.ts | 1 + vitest.config.ts | 2 + 19 files changed, 3577 insertions(+), 60 deletions(-) create mode 100644 scripts/lib/dgx-station-peer.mts create mode 100755 scripts/prepare-dual-dgx-station.mts create mode 100644 test/install-station-pair-preparation.test.ts diff --git a/ci/platform-matrix.json b/ci/platform-matrix.json index 71387c2530..6a04c1039a 100644 --- a/ci/platform-matrix.json +++ b/ci/platform-matrix.json @@ -64,7 +64,7 @@ "status": "deferred", "prd_priority": "P1", "ci_tested": false, - "notes": "The PRD marks this platform as P1. Workstation form-factor with NVIDIA GPUs and the same Docker + NVIDIA Container Toolkit + CDI requirements as DGX Spark. The installer detects DGX Station and offers express install with the pinned `nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4` recipe, including an approximately 352 GB model download, without follow-up provider, model, policy, or sandbox-name choices. Pass `--station-deepseek` to use `deepseek-ai/DeepSeek-V4-Flash` for a Station demo while retaining the one-confirmation express flow. Direct managed-vLLM onboarding still defaults to `deepseek-ai/DeepSeek-V4-Flash` when no model override is set. The full NemoClaw onboarding path, including the express recipe, has not been validated end-to-end on physical DGX Station hardware and remains `deferred` until that run is signed off." + "notes": "The PRD marks this platform as P1. Workstation form-factor with NVIDIA GPUs and the same Docker + NVIDIA Container Toolkit + CDI requirements as DGX Spark. With no explicit model or peer, installer-managed vLLM setup checks only the deterministic counterpart on each of two configured private `/30` CX-8 rails. At least one derived address must already be trusted; if both are trusted, their host keys must identify one coherent SSH host. NemoClaw automatically selects pinned `nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4` only when the reciprocal pair then qualifies. Otherwise the existing single-Station default remains `deepseek-ai/DeepSeek-V4-Flash`. An explicit model or peer remains authoritative; an explicit peer failure does not fall back. Pass `--station-deepseek` to select DeepSeek explicitly. Rail configuration, SSH trust enrollment, and reboots remain operator-owned; reboot resume uses owner-only state tied to the exact accepted revision and pair identity. Direct managed-vLLM onboarding still defaults to `deepseek-ai/DeepSeek-V4-Flash` when no model override is set. The full NemoClaw onboarding path has not been validated end-to-end on physical DGX Station hardware and remains `deferred` until that run is signed off." }, { "name": "NVIDIA RTX (consumer and Pro workstation GPUs)", @@ -147,7 +147,7 @@ "name": "Local vLLM (managed install/start)", "status": "caveated", "endpoint_type": "Local OpenAI-compatible", - "notes": "Appears by default on DGX Spark and DGX Station. DGX Station remains deferred until its managed-vLLM onboarding path is validated end-to-end on physical hardware. Generic Linux NVIDIA GPU hosts require `NEMOCLAW_EXPERIMENTAL=1` or `NEMOCLAW_PROVIDER=install-vllm`. Host must have the NVIDIA Container Toolkit installed and a CDI spec present (`onboard` asserts CDI presence). NemoClaw pins runtime images to immutable digests. DGX Spark and DGX Station models without a model-specific runtime use the `linux/arm64` digest published under `nvcr.io/nvidia/vllm:26.05.post1-py3`; generic Linux NVIDIA GPU hosts use the matching `linux/arm64` or `linux/amd64` digest published under `nvcr.io/nvidia/vllm:26.03.post1-py3`. The DGX Station express installer selects `nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4` with a pinned Hugging Face revision and the multi-platform index digest published under `vllm/vllm-openai:v0.22.0`; that index resolves to the `linux/arm64` manifest on Station. Direct managed-vLLM profile defaults are listed in `src/lib/inference/vllm-models.ts`: DGX Spark uses `nvidia/Qwen3.6-35B-A3B-NVFP4`, DGX Station uses `deepseek-ai/DeepSeek-V4-Flash`, and Linux NVIDIA GPU uses `nvidia/NVIDIA-Nemotron-3-Nano-4B-FP8`. Image pulls from `nvcr.io` require NGC registry login (`docker login nvcr.io`); onboard prompts for the NGC API key when authentication is missing." + "notes": "Appears by default on DGX Spark and DGX Station. DGX Station remains deferred until its managed-vLLM onboarding path is validated end-to-end on physical hardware. Generic Linux NVIDIA GPU hosts require `NEMOCLAW_EXPERIMENTAL=1` or `NEMOCLAW_PROVIDER=install-vllm`. Host must have the NVIDIA Container Toolkit installed and a CDI spec present (`onboard` asserts CDI presence). NemoClaw pins runtime images to immutable digests. DGX Spark and DGX Station models without a model-specific runtime use the `linux/arm64` digest published under `nvcr.io/nvidia/vllm:26.05.post1-py3`; generic Linux NVIDIA GPU hosts use the matching `linux/arm64` or `linux/amd64` digest published under `nvcr.io/nvidia/vllm:26.03.post1-py3`. DGX Station installer-managed vLLM setup selects pinned `nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4` and its `vllm/vllm-openai:v0.22.0` multi-platform index only after one pretrusted reciprocal dual-Station pair qualifies. With both model and peer unset, no qualifying pair leaves the Station profile default at `deepseek-ai/DeepSeek-V4-Flash`. Explicit model and peer selections remain authoritative. Direct managed-vLLM profile defaults are listed in `src/lib/inference/vllm-models.ts`: DGX Spark uses `nvidia/Qwen3.6-35B-A3B-NVFP4`, DGX Station uses `deepseek-ai/DeepSeek-V4-Flash`, and Linux NVIDIA GPU uses `nvidia/NVIDIA-Nemotron-3-Nano-4B-FP8`. Image pulls from `nvcr.io` require NGC registry login (`docker login nvcr.io`); onboard prompts for the NGC API key when authentication is missing." } ], diff --git a/docs/get-started/prerequisites.mdx b/docs/get-started/prerequisites.mdx index ca69d5ba40..c05e73b51b 100644 --- a/docs/get-started/prerequisites.mdx +++ b/docs/get-started/prerequisites.mdx @@ -58,6 +58,12 @@ Any other failed unit blocks preparation for administrator review. It does not install a host CUDA toolkit or Docker Compose. If any other existing prerequisite version differs, preparation stops instead of changing it automatically. After changing pinned packages, the installer exits with status `10`; reboot, sign in, and run the printed command, which pins the exact accepted NemoClaw commit before resuming express setup. +For automatic dual-Station qualification, configure both Stations before installation with exactly two active 400 Gbit/s Ethernet CX-8 rails, MTU 9000, and one usable private `/30` address per rail. +The reciprocal addresses must have direct link routes, the expected peer MAC neighbors, and jumbo-frame connectivity in both directions. +The SSH target must already have usable host-key trust and non-interactive authentication, and the selected peer account must have passwordless `sudo` for remote host preparation. +NemoClaw checks only the two deterministic `/30` counterpart addresses; it does not scan other addresses, configure rails, or enroll SSH trust. +If either host preparation requires a reboot, the installer stops with status `10` and names the host to reboot manually. +Its owner-only resume state binds the exact NemoClaw revision, preparation helper, SSH host key, GPU identities, and reciprocal rails; the installer never reboots a host automatically. DGX Station remains Deferred. diff --git a/docs/get-started/quickstart.mdx b/docs/get-started/quickstart.mdx index 5273d8ce94..8ff47e04ab 100644 --- a/docs/get-started/quickstart.mdx +++ b/docs/get-started/quickstart.mdx @@ -120,13 +120,17 @@ Use these details when your first-run path needs more control. On macOS, start Docker Desktop or Colima first. DGX Spark, Station GB300 hosts running the generic Ubuntu 24.04 ARM64 image, and Windows WSL offer an interactive express-install path that chooses a managed local inference option for the platform. - On that Station configuration, accepting the express prompt selects the pinned `nemotron-3-ultra-550b-a55b` managed-vLLM recipe and completes onboarding without more provider, model, policy, or sandbox-name choices. + On that Station configuration, accepting the express prompt prepares the local host, then checks for one trusted reciprocal peer only at the deterministic counterpart address on each of the two configured private `/30` CX-8 rails. + At least one derived rail address must already be trusted; if both are trusted, their host keys must identify the same SSH host. The installer prepares both hosts and automatically selects the pinned `nemotron-3-ultra-550b-a55b` managed-vLLM recipe only after the two Stations pass reciprocal identity and connectivity checks across both rails. + If no trusted pair qualifies and no model or peer was selected explicitly, onboarding retains the existing single-Station `deepseek-v4-flash` profile default. + Complete the physical rail configuration and SSH host-key and authentication setup before installation; NemoClaw does not configure the rails or enroll SSH trust. DGX OS, NVIDIA BaseOS images, and other Station generations stop before host preparation. - Set `NEMOCLAW_PROVIDER` or `NEMOCLAW_NO_EXPRESS=1` explicitly to continue on an unqualified Station without host automation. + Set `NEMOCLAW_NO_EXPRESS=1`, or select a provider other than `install-vllm`, to continue on an unqualified Station without host automation. It probes the pinned driver, Docker, Buildx, and NVIDIA Container Toolkit versions, reuses exact matches, installs missing pins, and permits only the reviewed factory `dkms` transition. It establishes NVIDIA CDI through the packaged refresh service, then proves both CDI and `--gpus all` with real container launches. If the packaged refresh fails or does not advertise `nvidia.com/gpu=all`, preparation prints service diagnostics and stops for administrator repair instead of generating CDI configuration directly. - After it changes pinned packages, the installer exits with status `10` at the required reboot boundary; reboot, sign in, and run the printed exact-commit command to resume the accepted recipe without another prompt. + After it changes pinned packages on either Station, the installer exits with status `10` at the required reboot boundary; it never reboots a host automatically. + Reboot the named host, sign in, and run the printed exact-commit command to resume from owner-only state without another prompt. This automation does not change Station's Deferred support status; physical end-to-end validation remains open. Pass `--station-deepseek` to use DeepSeek V4 Flash for a Station demo instead. Refer to [Platform Support](../reference/platform-support) and [Choose an Inference Provider](../inference/learn-and-choose/choose-inference-provider) for the current platform behavior. @@ -196,15 +200,20 @@ Use these details when your first-run path needs more control. On DGX Spark, Station GB300 hosts running the generic Ubuntu 24.04 ARM64 image, and Windows WSL, interactive installation offers express install after you accept the third-party software notice. Express install switches onboarding to non-interactive mode, allows `sudo` password prompts for required host changes, and selects the managed local inference path for that platform. DGX Spark uses managed vLLM with `qwen3.6-35b-a3b-nvfp4` by default. - DGX Station express install explicitly selects `nemotron-3-ultra-550b-a55b` instead of the Station managed-vLLM profile default, `deepseek-v4-flash`, and discloses the approximately `352 GB` model download before confirmation. + With no explicit model selection, DGX Station express install checks the deterministic counterparts on two configured private `/30` CX-8 rails. + At least one derived address must already be trusted; if both are trusted, their host keys must identify one SSH host. It automatically selects `nemotron-3-ultra-550b-a55b` only when reciprocal Station, GPU, rail, MAC, route, neighbor, and jumbo-frame checks then qualify the pair. + Otherwise it retains the single-Station managed-vLLM profile default, `deepseek-v4-flash`. + An explicit `NEMOCLAW_VLLM_MODEL` remains authoritative; when `NEMOCLAW_DGX_STATION_PEER` is set, that exact already-trusted peer must qualify or setup stops instead of falling back. Before onboarding, the Station path requires Station GB300 with the generic Ubuntu 24.04 ARM64 image and checks for NVIDIA open driver `610.43.02`, Docker CE `29.6.1` with Buildx, and NVIDIA Container Toolkit `1.19.1`. DGX OS, NVIDIA BaseOS images, and other Station generations are outside this automatic preparation boundary. - Set `NEMOCLAW_PROVIDER` or `NEMOCLAW_NO_EXPRESS=1` to continue without Station host automation. + Set `NEMOCLAW_NO_EXPRESS=1`, or select a provider other than `install-vllm`, to continue without Station host automation. Preparation reuses exact versions, installs missing pinned packages, permits only the reviewed `dkms` transition from `3.0.11-1ubuntu13` to `1:3.4.0-1ubuntu1`, and refuses every other mismatched version. It requires the packaged NVIDIA CDI refresh service to advertise `nvidia.com/gpu=all`, and verifies CDI and `--gpus all` with real container launches. If the packaged refresh fails or omits that device, preparation prints service diagnostics and stops for administrator repair instead of generating CDI configuration directly. If NVIDIA Docker runtime registration or a post-change acceptance probe fails, preparation restores the prior Docker daemon configuration. - Changing pinned packages exits with status `10` for a reboot; after you sign in and run the printed exact-commit command, the accepted express recipe resumes without another prompt. + The physical rails and SSH host-key and authentication trust must already be configured. + NemoClaw does not change rail configuration, enroll SSH trust, or reboot either host automatically. + Changing pinned packages on either Station exits with status `10` for a manual reboot; after you sign in and run the printed exact-commit command, owner-only resume state continues the accepted recipe without another prompt. This automation does not change Station's Deferred support status; physical end-to-end validation remains open. To select DeepSeek V4 Flash while retaining the one-confirmation Station express flow, run `curl -fsSL https://www.nvidia.com/nemoclaw.sh | bash -s -- --station-deepseek`. Unless `NEMOCLAW_POLICY_TIER` is set, express install applies policy in `suggested` mode with the `balanced` tier, including the base sandbox policy and supported package, model, web-search, and local-inference presets. diff --git a/docs/inference/choose-inference-provider.mdx b/docs/inference/choose-inference-provider.mdx index a87cc05013..a31b6327ec 100644 --- a/docs/inference/choose-inference-provider.mdx +++ b/docs/inference/choose-inference-provider.mdx @@ -32,7 +32,7 @@ Use this status table to distinguish validated provider integrations from adapte | Local Ollama | Tested with limitations | Local Ollama API | Available when Ollama is installed or running on the host. Validated default models: `qwen3.6:35b` (high VRAM), `nemotron-3-nano:30b` (medium VRAM), `qwen3.5:9b` (low VRAM fallback). | | Local NVIDIA NIM | Experimental | Local OpenAI-compatible | Requires `NEMOCLAW_EXPERIMENTAL=1` and a NIM-capable NVIDIA GPU. Host must have the NVIDIA Container Toolkit installed and a CDI spec present (`onboard` asserts CDI presence with `assertCdiNvidiaGpuSpecPresent`, `src/lib/onboard/fatal-runtime-preflight.ts`). NIM images pull from `nvcr.io` and require NGC registry login. NemoClaw gates this path behind the experimental flag because it does not auto-select a NIM image for the host today. You must explicitly pick from the validated image list. On Linux arm64 DGX Spark and DGX Station hosts, onboarding warns that some NIM images may not publish a `linux/arm64` manifest; the warning is advisory, and the selected image pull can still fail when the registry has no matching platform manifest. Managed vLLM has host-specific default models and is not gated on the same boxes. Validated images referenced in `src/lib/inference/config.ts` and `nemoclaw/src/index.ts`: `nvidia/nemotron-3-super-120b-a12b` (default cloud model), `nvidia/nemotron-3-nano-30b-a3b`, `nvidia/llama-3.3-nemotron-super-49b-v1.5`. | | Local vLLM (already running) | Tested with limitations | Local OpenAI-compatible | Appears in the onboarding menu when NemoClaw detects a server already on `localhost:8000`. No flag required. Model is whatever the existing server serves. | -| Local vLLM (managed install/start) | Tested with limitations | Local OpenAI-compatible | Appears by default on DGX Spark and DGX Station. DGX Station remains deferred until its managed-vLLM onboarding path is validated end-to-end on physical hardware. Generic Linux NVIDIA GPU hosts require `NEMOCLAW_EXPERIMENTAL=1` or `NEMOCLAW_PROVIDER=install-vllm`. Host must have the NVIDIA Container Toolkit installed and a CDI spec present (`onboard` asserts CDI presence). NemoClaw pins runtime images to immutable digests. DGX Spark and DGX Station models without a model-specific runtime use the `linux/arm64` digest published under `nvcr.io/nvidia/vllm:26.05.post1-py3`; generic Linux NVIDIA GPU hosts use the matching `linux/arm64` or `linux/amd64` digest published under `nvcr.io/nvidia/vllm:26.03.post1-py3`. The DGX Station express installer selects `nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4` with a pinned Hugging Face revision and the multi-platform index digest published under `vllm/vllm-openai:v0.22.0`; that index resolves to the `linux/arm64` manifest on Station. Direct managed-vLLM profile defaults are listed in `src/lib/inference/vllm-models.ts`: DGX Spark uses `nvidia/Qwen3.6-35B-A3B-NVFP4`, DGX Station uses `deepseek-ai/DeepSeek-V4-Flash`, and Linux NVIDIA GPU uses `nvidia/NVIDIA-Nemotron-3-Nano-4B-FP8`. Image pulls from `nvcr.io` require NGC registry login (`docker login nvcr.io`); onboard prompts for the NGC API key when authentication is missing. | +| Local vLLM (managed install/start) | Tested with limitations | Local OpenAI-compatible | Appears by default on DGX Spark and DGX Station. DGX Station remains deferred until its managed-vLLM onboarding path is validated end-to-end on physical hardware. Generic Linux NVIDIA GPU hosts require `NEMOCLAW_EXPERIMENTAL=1` or `NEMOCLAW_PROVIDER=install-vllm`. Host must have the NVIDIA Container Toolkit installed and a CDI spec present (`onboard` asserts CDI presence). NemoClaw pins runtime images to immutable digests. DGX Spark and DGX Station models without a model-specific runtime use the `linux/arm64` digest published under `nvcr.io/nvidia/vllm:26.05.post1-py3`; generic Linux NVIDIA GPU hosts use the matching `linux/arm64` or `linux/amd64` digest published under `nvcr.io/nvidia/vllm:26.03.post1-py3`. DGX Station installer-managed vLLM setup selects pinned `nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4` and its `vllm/vllm-openai:v0.22.0` multi-platform index only after one pretrusted reciprocal dual-Station pair qualifies. With both model and peer unset, no qualifying pair leaves the Station profile default at `deepseek-ai/DeepSeek-V4-Flash`. Explicit model and peer selections remain authoritative. Direct managed-vLLM profile defaults are listed in `src/lib/inference/vllm-models.ts`: DGX Spark uses `nvidia/Qwen3.6-35B-A3B-NVFP4`, DGX Station uses `deepseek-ai/DeepSeek-V4-Flash`, and Linux NVIDIA GPU uses `nvidia/NVIDIA-Nemotron-3-Nano-4B-FP8`. Image pulls from `nvcr.io` require NGC registry login (`docker login nvcr.io`); onboard prompts for the NGC API key when authentication is missing. | {/* provider-status:end */} ## Hosted Providers diff --git a/docs/inference/set-up-vllm.mdx b/docs/inference/set-up-vllm.mdx index a251f37bdc..c0a928c883 100644 --- a/docs/inference/set-up-vllm.mdx +++ b/docs/inference/set-up-vllm.mdx @@ -145,8 +145,15 @@ When you start managed vLLM outside the installer express flow, NemoClaw uses th | DGX Station | `deepseek-ai/DeepSeek-V4-Flash` | | Linux with an NVIDIA GPU | `nvidia/NVIDIA-Nemotron-3-Nano-4B-FP8` | -On DGX Station, accepting the installer express prompt sets `NEMOCLAW_VLLM_MODEL=nemotron-3-ultra-550b-a55b` and overrides the profile default. -Pass `--station-deepseek` to select the existing `deepseek-v4-flash` recipe while retaining the same one-confirmation express flow. +On DGX Station, accepting the installer express prompt leaves the model selector unset while it checks for a trusted reciprocal pair. +The installer derives exactly one counterpart from each of two configured private `/30` CX-8 rails and consults existing SSH trust only for those two addresses. +At least one derived address must already be trusted; if both are trusted, their host keys must identify one coherent SSH host. After the Stations pass reciprocal GPU, rail, MAC, route, neighbor, and jumbo-frame checks across both rails, the installer prepares both hosts, exports the qualified peer, and automatically selects `nemotron-3-ultra-550b-a55b`. +With both model and peer unset, if no trusted pair qualifies, it retains the single-Station `deepseek-v4-flash` profile default. +Setting `NEMOCLAW_VLLM_MODEL` remains authoritative; setting `NEMOCLAW_DGX_STATION_PEER` requests that exact already-trusted peer and fails closed if it does not qualify. +Pass `--station-deepseek` to select the existing `deepseek-v4-flash` recipe explicitly while retaining the same one-confirmation express flow. +Configure the physical rails and SSH host-key and authentication trust before installation. +NemoClaw does not modify rail configuration, enroll SSH trust, or reboot either host automatically. +If either host requires a reboot after pinned prerequisite changes, the installer stops with status `10` and resumes only after the manual reboot with owner-only state tied to the printed exact revision and reciprocal identity. The registered Ultra recipe tracks the [official DGX Station deployment guide](https://github.com/NVIDIA-NeMo/Nemotron/blob/287ae845639d2ce998998cb8fd1f70a3fa943c0b/usage-cookbook/Nemotron-3-Ultra/StationDeploymentGuide/README.md) and configures the pinned model revision, CPU offload, `16 GB` of shared memory, memory/stack ulimits, MTP speculative decoding, and the Nemotron reasoning and tool-call parsers. NemoClaw intentionally keeps its existing bridge-networked managed-inference topology instead of importing the playbook's host-network setting. The container publishes port `8000` through Docker, so apply the firewall guidance at the top of this page. @@ -174,7 +181,8 @@ NEMOCLAW_PROVIDER=install-vllm \ On DGX Spark and DGX Station, `NEMOCLAW_PROVIDER=install-vllm` is sufficient for a non-interactive run. Add `NEMOCLAW_EXPERIMENTAL=1` on a generic Linux NVIDIA GPU host. Non-interactive runs use the profile default unless you set `NEMOCLAW_VLLM_MODEL`. -On DGX Station, a direct provider-only run therefore selects `deepseek-v4-flash`; the installer express flow sets the Nemotron 3 Ultra override for you. +The commands above invoke `nemoclaw onboard` directly, so a DGX Station run with no model or peer selects the `deepseek-v4-flash` profile default. +When `NEMOCLAW_PROVIDER=install-vllm` is instead supplied to the shell installer, the installer enters the same Station host-preparation and trusted-pair discovery boundary used by express setup and selects Nemotron 3 Ultra only after a reciprocal pair qualifies. ## Select a Managed Model diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index ad3c75cef5..384076b731 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -3284,7 +3284,8 @@ Set them before running `$$nemoclaw onboard`. | `SANDBOX_NAME` | sandbox name | Compatibility spelling used after `NEMOCLAW_SANDBOX_NAME` and `NEMOCLAW_SANDBOX`. | | `NEMOCLAW_INSTALL_REF` | git ref | For internal installer commands: the git ref to install from. Overridden by the `--install-ref` flag. | | `NEMOCLAW_INSTALL_TAG` | release tag | For internal installer commands: the release tag to install. Defaults to the admin-promoted `lkg` tag when unset. Overridden by the `--install-tag` flag. | -| `NEMOCLAW_VLLM_MODEL` | registry slug or Hugging Face model id | Selects the model the managed-vLLM install path serves. Recognised slugs: `qwen3.6-27b`, `qwen3.6-35b-a3b-nvfp4`, `nemotron-3-nano-4b`, `deepseek-v4-flash`, `nemotron-3-ultra-550b-a55b`, `deepseek-r1-distill-70b`. Unset uses the per-platform profile default. The DGX Station express installer sets `nemotron-3-ultra-550b-a55b` explicitly. Gated models (e.g. `deepseek-r1-distill-70b`) require `HF_TOKEN` or `HUGGING_FACE_HUB_TOKEN`. | +| `NEMOCLAW_VLLM_MODEL` | registry slug or Hugging Face model id | Selects the model the managed-vLLM install path serves and remains authoritative during DGX Station installer setup. Recognised slugs: `qwen3.6-27b`, `qwen3.6-35b-a3b-nvfp4`, `nemotron-3-nano-4b`, `deepseek-v4-flash`, `nemotron-3-ultra-550b-a55b`, `deepseek-r1-distill-70b`. Unset uses the per-platform profile default, except that DGX Station installer setup automatically selects `nemotron-3-ultra-550b-a55b` after one pretrusted reciprocal pair qualifies. With both model and peer unset, no qualifying pair leaves the Station `deepseek-v4-flash` default in place. Gated models (e.g. `deepseek-r1-distill-70b`) require `HF_TOKEN` or `HUGGING_FACE_HUB_TOKEN`. | +| `NEMOCLAW_DGX_STATION_PEER` | SSH host or `user@host` | Selects one exact, already-trusted DGX Station peer for Nemotron 3 Ultra pair qualification. The peer must match the reciprocal private `/30` rail and hardware checks; an explicit peer failure stops setup instead of falling back. NemoClaw does not enroll SSH trust or accept a port or SSH option in this value. When unset, DGX Station installer discovery checks only the two deterministic `/30` counterpart addresses. A peer cannot be combined with an explicit non-Ultra model; conflicting explicit selections fail before pair preparation. | | `NEMOCLAW_VLLM_EXTRA_ARGS_JSON` | JSON array of non-blank strings | Appends advanced operator-owned tokens to the managed `vllm serve` command after NemoClaw's registry defaults. Example: `["--max-num-seqs","2"]`. Malformed JSON, non-string tokens, or blank tokens fail before Docker work starts. | | `NEMOCLAW_MINIMAL_BOOTSTRAP` | `1` to enable | Skips default OpenClaw workspace-template seeding for new pristine workspaces. Existing files are not deleted; refer to [Understand Runtime Changes](../manage-sandboxes/configure-sandboxes/understand-runtime-changes). | diff --git a/docs/reference/platform-support.mdx b/docs/reference/platform-support.mdx index 35a31916de..89ae729c8f 100644 --- a/docs/reference/platform-support.mdx +++ b/docs/reference/platform-support.mdx @@ -82,7 +82,7 @@ For the onboarding-time supported set without deferred rows, refer to [Prerequis | macOS (Apple Silicon) | Colima, Docker Desktop | Tested with limitations | P0 | Yes | Start the container runtime (Colima or Docker Desktop) before running the installer. Homebrew Colima users must install both Colima and the Docker CLI (`brew install colima docker`) before `docker info` can work. Xcode Command Line Tools (`xcode-select --install`) are typically required for Node native modules during install. NemoClaw recommends them but does not enforce them during preflight. | | DGX Spark | Docker | Tested | P1 | Yes | Use the standard installer and `$$nemoclaw onboard`. For an end-to-end walkthrough with local inference, see the [NVIDIA Spark playbook](https://build.nvidia.com/spark/nemoclaw). | | Windows WSL2 | Docker Desktop (WSL backend) | Tested with limitations | P1 | No | Requires WSL2 with Docker Desktop backend. | -| DGX Station | Docker | Deferred | P1 | No | The PRD marks this platform as P1. Workstation form-factor with NVIDIA GPUs and the same Docker + NVIDIA Container Toolkit + CDI requirements as DGX Spark. The installer detects DGX Station and offers express install with the pinned `nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4` recipe, including an approximately 352 GB model download, without follow-up provider, model, policy, or sandbox-name choices. Pass `--station-deepseek` to use `deepseek-ai/DeepSeek-V4-Flash` for a Station demo while retaining the one-confirmation express flow. Direct managed-vLLM onboarding still defaults to `deepseek-ai/DeepSeek-V4-Flash` when no model override is set. The full NemoClaw onboarding path, including the express recipe, has not been validated end-to-end on physical DGX Station hardware and remains `deferred` until that run is signed off. | +| DGX Station | Docker | Deferred | P1 | No | The PRD marks this platform as P1. Workstation form-factor with NVIDIA GPUs and the same Docker + NVIDIA Container Toolkit + CDI requirements as DGX Spark. With no explicit model or peer, installer-managed vLLM setup checks only the deterministic counterpart on each of two configured private `/30` CX-8 rails. At least one derived address must already be trusted; if both are trusted, their host keys must identify one coherent SSH host. NemoClaw automatically selects pinned `nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4` only when the reciprocal pair then qualifies. Otherwise the existing single-Station default remains `deepseek-ai/DeepSeek-V4-Flash`. An explicit model or peer remains authoritative; an explicit peer failure does not fall back. Pass `--station-deepseek` to select DeepSeek explicitly. Rail configuration, SSH trust enrollment, and reboots remain operator-owned; reboot resume uses owner-only state tied to the exact accepted revision and pair identity. Direct managed-vLLM onboarding still defaults to `deepseek-ai/DeepSeek-V4-Flash` when no model override is set. The full NemoClaw onboarding path has not been validated end-to-end on physical DGX Station hardware and remains `deferred` until that run is signed off. | | NVIDIA RTX (consumer and Pro workstation GPUs) | Docker | Deferred | P1 | No | The PRD marks this platform as P1. Covers RTX consumer cards and RTX Pro workstation cards on Linux hosts that meet the generic-Linux-GPU requirements (NVIDIA Container Toolkit + CDI present). The provider menu emits managed vLLM behind `NEMOCLAW_EXPERIMENTAL=1` or `NEMOCLAW_PROVIDER=install-vllm` for this host class today; the end-to-end onboard path on this hardware is not yet validated in CI. | {/* platform-matrix-full:end */} @@ -104,7 +104,7 @@ NemoClaw routes inference through the OpenShell gateway. Each row below is a pro | Local Ollama | Tested with limitations | Local Ollama API | Available when Ollama is installed or running on the host. Validated default models: `qwen3.6:35b` (high VRAM), `nemotron-3-nano:30b` (medium VRAM), `qwen3.5:9b` (low VRAM fallback). | | Local NVIDIA NIM | Experimental | Local OpenAI-compatible | Requires `NEMOCLAW_EXPERIMENTAL=1` and a NIM-capable NVIDIA GPU. Host must have the NVIDIA Container Toolkit installed and a CDI spec present (`onboard` asserts CDI presence with `assertCdiNvidiaGpuSpecPresent`, `src/lib/onboard/fatal-runtime-preflight.ts`). NIM images pull from `nvcr.io` and require NGC registry login. NemoClaw gates this path behind the experimental flag because it does not auto-select a NIM image for the host today. You must explicitly pick from the validated image list. On Linux arm64 DGX Spark and DGX Station hosts, onboarding warns that some NIM images may not publish a `linux/arm64` manifest; the warning is advisory, and the selected image pull can still fail when the registry has no matching platform manifest. Managed vLLM has host-specific default models and is not gated on the same boxes. Validated images referenced in `src/lib/inference/config.ts` and `nemoclaw/src/index.ts`: `nvidia/nemotron-3-super-120b-a12b` (default cloud model), `nvidia/nemotron-3-nano-30b-a3b`, `nvidia/llama-3.3-nemotron-super-49b-v1.5`. | | Local vLLM (already running) | Tested with limitations | Local OpenAI-compatible | Appears in the onboarding menu when NemoClaw detects a server already on `localhost:8000`. No flag required. Model is whatever the existing server serves. | -| Local vLLM (managed install/start) | Tested with limitations | Local OpenAI-compatible | Appears by default on DGX Spark and DGX Station. DGX Station remains deferred until its managed-vLLM onboarding path is validated end-to-end on physical hardware. Generic Linux NVIDIA GPU hosts require `NEMOCLAW_EXPERIMENTAL=1` or `NEMOCLAW_PROVIDER=install-vllm`. Host must have the NVIDIA Container Toolkit installed and a CDI spec present (`onboard` asserts CDI presence). NemoClaw pins runtime images to immutable digests. DGX Spark and DGX Station models without a model-specific runtime use the `linux/arm64` digest published under `nvcr.io/nvidia/vllm:26.05.post1-py3`; generic Linux NVIDIA GPU hosts use the matching `linux/arm64` or `linux/amd64` digest published under `nvcr.io/nvidia/vllm:26.03.post1-py3`. The DGX Station express installer selects `nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4` with a pinned Hugging Face revision and the multi-platform index digest published under `vllm/vllm-openai:v0.22.0`; that index resolves to the `linux/arm64` manifest on Station. Direct managed-vLLM profile defaults are listed in `src/lib/inference/vllm-models.ts`: DGX Spark uses `nvidia/Qwen3.6-35B-A3B-NVFP4`, DGX Station uses `deepseek-ai/DeepSeek-V4-Flash`, and Linux NVIDIA GPU uses `nvidia/NVIDIA-Nemotron-3-Nano-4B-FP8`. Image pulls from `nvcr.io` require NGC registry login (`docker login nvcr.io`); onboard prompts for the NGC API key when authentication is missing. | +| Local vLLM (managed install/start) | Tested with limitations | Local OpenAI-compatible | Appears by default on DGX Spark and DGX Station. DGX Station remains deferred until its managed-vLLM onboarding path is validated end-to-end on physical hardware. Generic Linux NVIDIA GPU hosts require `NEMOCLAW_EXPERIMENTAL=1` or `NEMOCLAW_PROVIDER=install-vllm`. Host must have the NVIDIA Container Toolkit installed and a CDI spec present (`onboard` asserts CDI presence). NemoClaw pins runtime images to immutable digests. DGX Spark and DGX Station models without a model-specific runtime use the `linux/arm64` digest published under `nvcr.io/nvidia/vllm:26.05.post1-py3`; generic Linux NVIDIA GPU hosts use the matching `linux/arm64` or `linux/amd64` digest published under `nvcr.io/nvidia/vllm:26.03.post1-py3`. DGX Station installer-managed vLLM setup selects pinned `nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4` and its `vllm/vllm-openai:v0.22.0` multi-platform index only after one pretrusted reciprocal dual-Station pair qualifies. With both model and peer unset, no qualifying pair leaves the Station profile default at `deepseek-ai/DeepSeek-V4-Flash`. Explicit model and peer selections remain authoritative. Direct managed-vLLM profile defaults are listed in `src/lib/inference/vllm-models.ts`: DGX Spark uses `nvidia/Qwen3.6-35B-A3B-NVFP4`, DGX Station uses `deepseek-ai/DeepSeek-V4-Flash`, and Linux NVIDIA GPU uses `nvidia/NVIDIA-Nemotron-3-Nano-4B-FP8`. Image pulls from `nvcr.io` require NGC registry login (`docker login nvcr.io`); onboard prompts for the NGC API key when authentication is missing. | {/* provider-status-full:end */} ## Messaging Integrations diff --git a/docs/resources/starter-prompt.md b/docs/resources/starter-prompt.md index 894194121f..2a4b6f970f 100644 --- a/docs/resources/starter-prompt.md +++ b/docs/resources/starter-prompt.md @@ -79,17 +79,17 @@ If DGX Spark Express is selected: If DGX Station Express is selected: -- Use managed vLLM. -- Explicitly select `nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4`. -- Do not leave the model unset; the ordinary managed-vLLM default can select DeepSeek and would not reproduce Express. -- Set `NEMOCLAW_PROVIDER=install-vllm`. -- Set `NEMOCLAW_VLLM_MODEL=nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4`. -- Disclose that the model download is approximately 352 GB, in addition to the vLLM container and temporary download space. +- Use managed vLLM and set `NEMOCLAW_PROVIDER=install-vllm`. +- Leave `NEMOCLAW_VLLM_MODEL` unset unless the user explicitly selected a model. +- Explain that the installer checks only the deterministic counterpart on each of two configured private `/30` CX-8 rails. At least one derived address must already be trusted; if both are trusted, their host keys must identify one coherent SSH host. It selects pinned Nemotron 3 Ultra only when the reciprocal pair then qualifies; otherwise the existing single-Station DeepSeek default remains. +- If the user supplies `NEMOCLAW_DGX_STATION_PEER`, preserve that exact selection and explain that setup stops if the already-trusted peer does not qualify. +- Confirm that the physical rails and SSH host-key and authentication trust were configured outside NemoClaw. Do not configure the rails, enroll SSH trust, or approve an automatic reboot. +- Explain the container and selected model download before it begins, without assuming the Ultra recipe was selected. - Verify the model-cache filesystem and Docker storage have sufficient capacity. - Warn that DGX Station managed deployment has deferred end-to-end physical-hardware validation. - Describe it as an evaluation path, not a validated production deployment. - Explain that startup may fail despite passing initial checks. -- Ask separately for approval of the approximately 352 GB download. +- If either host requires a reboot, let the installer stop, ask the user to reboot the named host manually, and resume only with the printed exact-revision command and owner-only state. For both Express paths: @@ -200,7 +200,7 @@ Use this provider mapping for non-interactive setup: - Anthropic-compatible: `NEMOCLAW_PROVIDER=anthropicCompatible`, endpoint, model, `COMPATIBLE_ANTHROPIC_API_KEY`. - Ollama: `NEMOCLAW_PROVIDER=ollama`, optional `NEMOCLAW_MODEL`. - Existing vLLM: `NEMOCLAW_PROVIDER=vllm`. -- Managed vLLM: `NEMOCLAW_PROVIDER=install-vllm`; leave `NEMOCLAW_VLLM_MODEL` unset for DGX Spark Express, set it to `nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4` for DGX Station Express, or use an approved optional override for non-Express setup. +- Managed vLLM: `NEMOCLAW_PROVIDER=install-vllm`; leave `NEMOCLAW_VLLM_MODEL` unset for DGX Spark Express and for automatic DGX Station trusted-pair selection, or preserve an approved explicit model override. - Windows WSL Express: `NEMOCLAW_PROVIDER=install-windows-ollama`. Do not offer Hermes Provider for OpenClaw or Deep Agents. diff --git a/scripts/checks/vitest-project-overlap.ts b/scripts/checks/vitest-project-overlap.ts index 9e5415c7ee..cb0bdd0a39 100644 --- a/scripts/checks/vitest-project-overlap.ts +++ b/scripts/checks/vitest-project-overlap.ts @@ -47,6 +47,7 @@ const INSTALLER_INTEGRATION_TESTS = new Set([ "test/install-preflight-docker-bootstrap.test.ts", "test/install-preflight.test.ts", "test/install-station-host-preparation.test.ts", + "test/install-station-pair-preparation.test.ts", ]); function normalizeRepoPath(file: string): string { diff --git a/scripts/install.sh b/scripts/install.sh index 0eb66b3494..64df98f321 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -2897,6 +2897,9 @@ STATION_DEEPSEEK_VLLM_MODEL="deepseek-v4-flash" STATION_DEEPSEEK_SERVED_MODEL="deepseek-ai/DeepSeek-V4-Flash" _SELECTED_EXPRESS_PLATFORM="" _STATION_EXPRESS_RESUME_REVISION="" +_STATION_EXPRESS_MODEL_WAS_EXPLICIT=0 +_STATION_EXPRESS_DEFERRED_MANAGED_PAIR=0 +_STATION_INSTALL_MODE="" normalize_station_vllm_model() { printf "%s" "${1:-}" | tr '[:upper:]' '[:lower:]' | sed 's/^[[:space:]]*//; s/[[:space:]]*$//' @@ -2950,13 +2953,20 @@ preflight_explicit_express_flags() { configure_station_express_model() { local selected_model selected_model="$(normalize_station_vllm_model "${NEMOCLAW_VLLM_MODEL:-}")" + _STATION_EXPRESS_MODEL_WAS_EXPLICIT=0 if [ "${STATION_DEEPSEEK:-}" = "1" ]; then + _STATION_EXPRESS_MODEL_WAS_EXPLICIT=1 NEMOCLAW_VLLM_MODEL="$STATION_DEEPSEEK_VLLM_MODEL" NEMOCLAW_MODEL="$STATION_DEEPSEEK_SERVED_MODEL" elif [ -z "$selected_model" ]; then - NEMOCLAW_VLLM_MODEL="$STATION_ULTRA_VLLM_MODEL" - NEMOCLAW_MODEL="$STATION_ULTRA_SERVED_MODEL" + # An unset selector is intentional. Trusted reciprocal pair discovery below + # is the only automatic signal for Nemotron Ultra; without it, onboarding + # continues to use the existing single-Station profile default. + unset NEMOCLAW_VLLM_MODEL else + [ "$selected_model" != "auto" ] \ + || error "NEMOCLAW_VLLM_MODEL=auto is reserved for DGX Station reboot-resume state. Unset NEMOCLAW_VLLM_MODEL to request automatic selection." + _STATION_EXPRESS_MODEL_WAS_EXPLICIT=1 case "$selected_model" in "$STATION_ULTRA_VLLM_MODEL" | "nvidia/nvidia-nemotron-3-ultra-550b-a55b-nvfp4") NEMOCLAW_MODEL="$STATION_ULTRA_SERVED_MODEL" @@ -2973,12 +2983,27 @@ configure_station_express_model() { ;; esac fi - export NEMOCLAW_VLLM_MODEL + if [ -n "${NEMOCLAW_VLLM_MODEL:-}" ]; then + export NEMOCLAW_VLLM_MODEL + fi if [ -n "${NEMOCLAW_MODEL:-}" ]; then export NEMOCLAW_MODEL fi } +station_dual_model_requested() { + local selected_model + selected_model="$(normalize_station_vllm_model "${NEMOCLAW_VLLM_MODEL:-}")" + case "$selected_model" in + "" | "$STATION_ULTRA_VLLM_MODEL" | "$STATION_ULTRA_SERVED_MODEL" | "nvidia/nvidia-nemotron-3-ultra-550b-a55b-nvfp4") + return 0 + ;; + *) + return 1 + ;; + esac +} + station_express_resume_file() { local state_dir state_dir="$(nemoclaw_state_dir)" || return 1 @@ -2994,6 +3019,10 @@ validate_station_express_resume_revision() { [[ "${1:-}" =~ ^[0-9a-f]{40}$ ]] } +validate_station_install_mode() { + [[ "${1:-}" == "express" || "${1:-}" == "provider" ]] +} + station_installer_revision() { local revision revision="$(git -C "${SCRIPT_DIR}/.." rev-parse --verify 'HEAD^{commit}' 2>/dev/null)" \ @@ -3024,7 +3053,10 @@ assert_station_express_resume_file_safe() { } load_station_express_resume() { - local state_file revision_line model_line line_count saved_revision current_revision + local state_file revision_line model_line mode_line line_count saved_revision saved_model saved_mode current_revision + local requested_model requested_mode + requested_model="${NEMOCLAW_VLLM_MODEL:-}" + requested_mode="${_STATION_INSTALL_MODE:-}" state_file="$(station_express_resume_file)" || return 1 assert_nemoclaw_state_path_safe "$state_file" [[ -e "$state_file" || -L "$state_file" ]] || return 1 @@ -3032,23 +3064,48 @@ load_station_express_resume() { line_count="$(wc -l <"$state_file" | tr -d '[:space:]')" revision_line="$(sed -n '1p' "$state_file")" model_line="$(sed -n '2p' "$state_file")" + mode_line="$(sed -n '3p' "$state_file")" saved_revision="${revision_line#revision=}" - NEMOCLAW_VLLM_MODEL="${model_line#model=}" - if [[ "$line_count" != "2" || "$revision_line" != "revision=${saved_revision}" || "$model_line" != "model=${NEMOCLAW_VLLM_MODEL}" ]] \ + saved_model="${model_line#model=}" + saved_mode="${mode_line#mode=}" + if [[ "$line_count" != "3" || "$revision_line" != "revision=${saved_revision}" || "$model_line" != "model=${saved_model}" || "$mode_line" != "mode=${saved_mode}" ]] \ || ! validate_station_express_resume_revision "$saved_revision" \ - || ! validate_station_express_resume_model "$NEMOCLAW_VLLM_MODEL"; then + || ! validate_station_express_resume_model "$saved_model" \ + || ! validate_station_install_mode "$saved_mode"; then error "DGX Station express resume state is invalid. Remove ${state_file} and rerun the installer." fi current_revision="$(station_installer_revision)" if [[ "$current_revision" != "$saved_revision" ]]; then error "DGX Station express resume requires NemoClaw revision ${saved_revision}, but this installer is ${current_revision}. Rerun with: curl -fsSL https://www.nvidia.com/nemoclaw.sh | NEMOCLAW_INSTALL_TAG=${saved_revision} bash" fi - export NEMOCLAW_VLLM_MODEL + if [ -n "$requested_mode" ] && [ "$requested_mode" != "$saved_mode" ]; then + error "DGX Station resume state was accepted in ${saved_mode} mode; refusing to resume it in ${requested_mode} mode. Rerun the exact accepted installer command." + fi + _STATION_INSTALL_MODE="$saved_mode" + if [ -n "$(normalize_station_vllm_model "$requested_model")" ]; then + # A model explicitly supplied on the rerun remains authoritative. If pair + # preparation has already begun, however, a non-dual model would bypass the + # exact peer revalidation, so fail closed instead of silently falling back. + NEMOCLAW_VLLM_MODEL="$requested_model" + export NEMOCLAW_VLLM_MODEL + if station_dual_pair_resume_pending && ! station_dual_model_requested; then + error "A dual-DGX Station pair resume is pending; refusing to replace its model with '${requested_model}' before the exact pair is revalidated. Rerun with the accepted dual-Station model or remove the override." + fi + return 0 + fi + if [ "$saved_model" = "auto" ]; then + unset NEMOCLAW_VLLM_MODEL + else + NEMOCLAW_VLLM_MODEL="$saved_model" + export NEMOCLAW_VLLM_MODEL + fi } save_station_express_resume() { - local state_file state_dir temp_file revision model="${NEMOCLAW_VLLM_MODEL:-}" + local state_file state_dir temp_file revision model="${NEMOCLAW_VLLM_MODEL:-auto}" + local mode="${_STATION_INSTALL_MODE:-express}" validate_station_express_resume_model "$model" || error "Cannot save an invalid DGX Station express model selector." + validate_station_install_mode "$mode" || error "Cannot save an invalid DGX Station installer resume mode." revision="$(station_installer_revision)" state_file="$(station_express_resume_file)" || error "Could not resolve NemoClaw state for DGX Station express resume." state_dir="$(ensure_nemoclaw_state_dir)" || error "Could not prepare NemoClaw state for DGX Station express resume." @@ -3058,7 +3115,7 @@ save_station_express_resume() { rm -f "$temp_file" error "Could not secure DGX Station express resume state under ${state_dir}." } - if ! printf 'revision=%s\nmodel=%s\n' "$revision" "$model" >"$temp_file"; then + if ! printf 'revision=%s\nmodel=%s\nmode=%s\n' "$revision" "$model" "$mode" >"$temp_file"; then rm -f "$temp_file" error "Could not write DGX Station express resume state under ${state_dir}." fi @@ -3080,9 +3137,21 @@ clear_station_express_resume() { rm -f "$state_file" } +station_resume_install_command() { + local revision="$1" + if [ "${_STATION_INSTALL_MODE:-express}" = "provider" ]; then + printf 'curl -fsSL https://www.nvidia.com/nemoclaw.sh | NEMOCLAW_PROVIDER=install-vllm NEMOCLAW_INSTALL_TAG=%s bash' "$revision" + else + printf 'curl -fsSL https://www.nvidia.com/nemoclaw.sh | NEMOCLAW_INSTALL_TAG=%s bash' "$revision" + fi +} + activate_express_install() { local platform="$1" _SELECTED_EXPRESS_PLATFORM="$platform" + if [ "$platform" = "DGX Station" ]; then + _STATION_INSTALL_MODE="express" + fi NON_INTERACTIVE=1 export NEMOCLAW_NON_INTERACTIVE=1 export NEMOCLAW_NON_INTERACTIVE_SUDO_MODE=prompt @@ -3107,18 +3176,67 @@ activate_express_install() { esac } +resume_loaded_station_install() { + local platform="$1" + if [ "$_STATION_INSTALL_MODE" = "provider" ]; then + _SELECTED_EXPRESS_PLATFORM="$platform" + export NEMOCLAW_PROVIDER=install-vllm + configure_station_express_model + info "Detected DGX Station. Resuming the accepted managed-vLLM provider setup after host preparation." + else + info "Detected DGX Station. Resuming the accepted express install after host preparation." + activate_express_install "$platform" + fi +} + run_station_host_preparation() { # Public curl|bash starts in the root bootstrap, which clones the complete # selected ref before executing this payload. Keep the sibling lookup and # fail-closed check so Station preparation cannot drift from that ref. - local helper="${SCRIPT_DIR}/prepare-dgx-station-host.sh" + local helper="${SCRIPT_DIR}/prepare-dgx-station-host.sh" status=0 [[ -f "$helper" ]] || error "DGX Station host preparation helper is missing: ${helper}" - bash "$helper" --apply + bash "$helper" --check || status=$? + [ "$status" -eq 0 ] || return "$status" + bash "$helper" --apply || status=$? + [ "$status" -eq 0 ] || return "$status" + bash "$helper" --verify +} + +station_managed_dual_head_running() { + command_exists docker || return 1 + + local inspection name running managed role schema cluster launch_contract api_fingerprint transaction + inspection="$( + docker container inspect --format \ + '{{.Name}} {{.State.Running}} {{index .Config.Labels "com.nvidia.nemoclaw.managed-vllm"}} {{index .Config.Labels "com.nvidia.nemoclaw.vllm-role"}} {{index .Config.Labels "com.nvidia.nemoclaw.vllm-launch-schema"}} {{index .Config.Labels "com.nvidia.nemoclaw.vllm-cluster"}} {{index .Config.Labels "com.nvidia.nemoclaw.vllm-launch-contract"}} {{index .Config.Labels "com.nvidia.nemoclaw.vllm-api-key-fingerprint"}} {{index .Config.Labels "com.nvidia.nemoclaw.vllm-transaction"}}' \ + nemoclaw-vllm 2>/dev/null + )" || return 1 + read -r name running managed role schema cluster launch_contract api_fingerprint transaction <<<"$inspection" + [[ "$name" == "/nemoclaw-vllm" && + "$running" == "true" && + "$managed" == "true" && + "$role" == "head" && + "$schema" == "1" && + "$cluster" =~ ^[a-f0-9]{64}$ && + "$launch_contract" =~ ^[a-f0-9]{64}$ && + "$api_fingerprint" =~ ^[a-f0-9]{64}$ && + "$transaction" =~ ^[a-f0-9]{32}$ ]] } ensure_station_express_host() { [[ "${_SELECTED_EXPRESS_PLATFORM:-}" == "DGX Station" ]] || return 0 + _STATION_EXPRESS_DEFERRED_MANAGED_PAIR=0 + if station_dual_model_requested && station_managed_dual_head_running; then + # The host-preparation helper intentionally refuses active workloads. Only + # this complete dual-head ownership contract may defer local preparation; + # the post-Node coordinator revalidates the reciprocal physical pair before + # the existing lifecycle is allowed to reuse it. + _STATION_EXPRESS_DEFERRED_MANAGED_PAIR=1 + info "Found a complete running NemoClaw-managed dual-Station head candidate; deferring host preparation until reciprocal pair and lifecycle validation." + return 0 + fi + info "Checking pinned DGX Station host prerequisites. Exact matches are reused." local status=0 run_station_host_preparation || status=$? @@ -3133,7 +3251,7 @@ ensure_station_express_host() { warn "DGX Station host prerequisites were installed and require a reboot." info "Run: sudo reboot" info "After signing in again, rerun the accepted revision:" - info "curl -fsSL https://www.nvidia.com/nemoclaw.sh | NEMOCLAW_INSTALL_TAG=${revision} bash" + info "$(station_resume_install_command "$revision")" exit 10 ;; *) @@ -3142,8 +3260,198 @@ ensure_station_express_host() { esac } +station_dual_pair_resume_file() { + local state_dir + state_dir="$(nemoclaw_state_dir)" || return 1 + printf '%s/station-dual-pair-resume.json' "$state_dir" +} + +station_dual_pair_resume_pending() { + local state_file + state_file="$(station_dual_pair_resume_file)" || return 1 + [[ -e "$state_file" || -L "$state_file" ]] || return 1 + assert_nemoclaw_state_path_safe "$state_file" + return 0 +} + +validate_station_pair_selection() { + [[ "${_SELECTED_EXPRESS_PLATFORM:-}" == "DGX Station" ]] || return 0 + station_dual_model_requested && return 0 + [ -z "${NEMOCLAW_DGX_STATION_PEER:-}" ] \ + || error "NEMOCLAW_DGX_STATION_PEER requires the DGX Station dual-serving model. Unset NEMOCLAW_VLLM_MODEL or select ${STATION_ULTRA_VLLM_MODEL}; the explicit model override remains authoritative." + station_dual_pair_resume_pending \ + && error "A dual-DGX Station pair resume is pending; refusing to bypass exact pair revalidation with model '${NEMOCLAW_VLLM_MODEL}'." + return 0 +} + +parse_station_dual_pair_result() { + # The single-quoted payload is JavaScript, not shell interpolation. + # shellcheck disable=SC2016 + node -e ' + const fs = require("node:fs"); + const net = require("node:net"); + const fail = () => process.exit(2); + try { + const raw = fs.readFileSync(0, "utf8"); + const value = JSON.parse(raw); + if (!value || typeof value !== "object" || Array.isArray(value)) fail(); + if (value.kind === "single-station") { + if (typeof value.reason !== "string" || value.reason.length === 0 || value.reason.length > 4096) fail(); + process.stdout.write("single-station\n"); + process.exit(0); + } + if (value.kind !== "ready" && value.kind !== "reboot-required") fail(); + const peer = value.peerTarget; + if (typeof peer !== "string" || peer.length === 0 || peer.length > 286 || peer !== peer.trim()) fail(); + const parts = peer.split("@"); + if (parts.length > 2) fail(); + const user = parts.length === 2 ? parts[0] : ""; + const host = parts[parts.length - 1]; + const safeUser = /^[A-Za-z_][A-Za-z0-9._-]*$/; + const safeHost = /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?(?:\.[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?)*$/; + const numericHost = /^[0-9.]+$/.test(host); + if ((user && !safeUser.test(user)) || (net.isIP(host) !== 4 && (numericHost || !safeHost.test(host)))) fail(); + const identity = value.identity; + if (!identity || typeof identity !== "object" || Array.isArray(identity)) fail(); + if (identity.peerTarget !== peer || !/^[a-f0-9]{64}$/.test(identity.hostKeyDigest)) fail(); + if (!/^GPU-[A-Za-z0-9-]+$/.test(identity.localGpuUuid) || !/^GPU-[A-Za-z0-9-]+$/.test(identity.peerGpuUuid)) fail(); + if (!Array.isArray(identity.rails) || identity.rails.length !== 2) fail(); + const mac = /^(?:[0-9a-f]{2}:){5}[0-9a-f]{2}$/; + for (const rail of identity.rails) { + if (!rail || typeof rail !== "object" || Array.isArray(rail)) fail(); + if (net.isIP(rail.localAddress) !== 4 || net.isIP(rail.peerAddress) !== 4) fail(); + if (!mac.test(rail.localMac) || !mac.test(rail.peerMac)) fail(); + } + process.stdout.write(`${value.kind}\n${peer}\n`); + } catch { + fail(); + } + ' +} + +ensure_station_express_pair() { + [[ "${_SELECTED_EXPRESS_PLATFORM:-}" == "DGX Station" ]] || return 0 + validate_station_pair_selection + if ! station_dual_model_requested; then + return 0 + fi + + local coordinator="${SCRIPT_DIR}/prepare-dual-dgx-station.mts" + local helper="${SCRIPT_DIR}/prepare-dgx-station-host.sh" + [[ -f "$coordinator" ]] || error "Dual DGX Station preparation coordinator is missing: ${coordinator}" + [[ -f "$helper" ]] || error "DGX Station host preparation helper is missing: ${helper}" + + local state_dir state_file revision output parsed kind peer_target status=0 + state_dir="$(ensure_nemoclaw_state_dir)" || error "Could not prepare owner-only NemoClaw state for dual DGX Station discovery." + state_file="${state_dir}/station-dual-pair-resume.json" + assert_nemoclaw_state_path_safe "$state_file" + revision="$(station_installer_revision)" + + local -a pair_command=( + node --no-warnings --experimental-strip-types "$coordinator" + --helper "$helper" + --state "$state_file" + --revision "$revision" + ) + if [ -n "${NEMOCLAW_DGX_STATION_PEER:-}" ]; then + pair_command+=(--explicit-peer "$NEMOCLAW_DGX_STATION_PEER") + fi + if [ "${_STATION_EXPRESS_DEFERRED_MANAGED_PAIR:-0}" = "1" ]; then + pair_command+=(--reuse-existing-managed-pair) + fi + + info "Checking for one pretrusted reciprocal dual-DGX Station peer on the two direct /30 rail counterparts." + # Publish the companion mode/model state before the coordinator can persist + # pair identity and mutate the remote host. A crash, mismatch, or later + # onboarding failure must resume this exact accepted setup without returning + # to a prompt or silently falling back. + save_station_express_resume + output="$("${pair_command[@]}")" || status=$? + case "$status" in + 0 | 10) ;; + *) + if ! station_dual_pair_resume_pending; then + clear_station_express_resume + fi + error "Dual DGX Station preparation failed. No serving-model or vLLM-image pull was started; review the station-pair diagnostics above and rerun the same revision." + ;; + esac + if ! parsed="$(printf '%s' "$output" | parse_station_dual_pair_result)"; then + if ! station_dual_pair_resume_pending; then + clear_station_express_resume + fi + error "Dual DGX Station preparation returned invalid result data; refusing to continue." + fi + kind="$(printf '%s\n' "$parsed" | sed -n '1p')" + peer_target="$(printf '%s\n' "$parsed" | sed -n '2p')" + + case "$kind" in + single-station) + [ "$status" -eq 0 ] \ + || error "Dual DGX Station preparation returned an inconsistent reboot result; refusing to continue." + [ "${_STATION_EXPRESS_DEFERRED_MANAGED_PAIR:-0}" != "1" ] \ + || error "The running managed dual-Station head could not be matched to its trusted reciprocal peer; refusing single-Station fallback." + [ -z "${NEMOCLAW_DGX_STATION_PEER:-}" ] \ + || error "The explicit DGX Station peer could not be qualified; refusing single-Station fallback." + station_dual_pair_resume_pending \ + && error "Dual DGX Station preparation returned a single-Station result while exact pair resume state is pending; refusing to discard it." + clear_station_express_resume + if [ "${_STATION_EXPRESS_MODEL_WAS_EXPLICIT:-0}" = "0" ]; then + unset NEMOCLAW_VLLM_MODEL + fi + info "No trusted reciprocal dual-DGX Station pair was detected; using the existing single-Station profile default." + ;; + ready) + [ "$status" -eq 0 ] \ + || error "Dual DGX Station preparation returned an inconsistent ready result; refusing to continue." + station_dual_pair_resume_pending \ + || error "Dual DGX Station preparation returned ready without exact pair resume state; refusing to continue." + NEMOCLAW_DGX_STATION_PEER="$peer_target" + export NEMOCLAW_DGX_STATION_PEER + if [ "${_STATION_EXPRESS_MODEL_WAS_EXPLICIT:-0}" = "0" ]; then + # Leave the vLLM selector unset. The trusted peer is the existing + # installVllm signal that performs the authoritative full capability + # probe and selects Nemotron Ultra without a second prompt. + unset NEMOCLAW_VLLM_MODEL + NEMOCLAW_MODEL="$STATION_ULTRA_SERVED_MODEL" + export NEMOCLAW_MODEL + fi + ok "Trusted reciprocal dual-DGX Station pair is ready (${peer_target})" + ;; + reboot-required) + [ "$status" -eq 10 ] \ + || error "Dual DGX Station preparation returned an inconsistent reboot result; refusing to continue." + station_dual_pair_resume_pending \ + || error "Dual DGX Station preparation requested a reboot without exact pair resume state; refusing to continue." + save_station_express_resume + revision="${_STATION_EXPRESS_RESUME_REVISION}" + warn "Peer DGX Station ${peer_target} was prepared and requires a manual reboot." + info "On peer ${peer_target}, run: sudo reboot" + info "After the peer is back online, rerun the accepted revision on this Station:" + info "$(station_resume_install_command "$revision")" + exit 10 + ;; + *) + error "Dual DGX Station preparation returned an unsupported result; refusing to continue." + ;; + esac +} + +clear_station_dual_pair_resume() { + local state_file coordinator="${SCRIPT_DIR}/prepare-dual-dgx-station.mts" + state_file="$(station_dual_pair_resume_file)" || return 0 + assert_nemoclaw_state_path_safe "$state_file" + [[ -e "$state_file" || -L "$state_file" ]] || return 0 + [[ -f "$coordinator" ]] || error "Dual DGX Station preparation coordinator is missing: ${coordinator}" + node --no-warnings --experimental-strip-types "$coordinator" --state "$state_file" --clear-state >/dev/null \ + || error "Could not safely clear completed dual DGX Station resume state: ${state_file}" +} + prepare_installer_host() { maybe_offer_express_install + # Reject conflicting explicit Station selections and pending-pair bypasses + # before the local host-preparation helper can mutate packages or Docker. + validate_station_pair_selection # Intentional ordering: Station preparation owns the reboot boundary before # generic Docker bootstrap; ensure_station_express_host is a no-op elsewhere. ensure_station_express_host @@ -3181,8 +3489,8 @@ describe_express_install() { inference_summary="managed local vLLM with model ${NEMOCLAW_VLLM_MODEL}" inference_disclosure="Managed vLLM pulls the configured vLLM image/model and runs a local inference container." else - inference_summary="managed local vLLM with NVIDIA Nemotron 3 Ultra 550B" - inference_disclosure="Managed vLLM pulls the pinned Station image and approximately 352 GB model, then runs a local inference container." + inference_summary="managed local vLLM using automatic Station model selection" + inference_disclosure="A pretrusted reciprocal dual-Station pair selects NVIDIA Nemotron 3 Ultra 550B; otherwise the existing single-Station profile default is used. Serving-model and vLLM-image pulls start only after qualification." fi printf " Station host setup reuses exact prerequisite versions, applies the reviewed factory DKMS transition when present, installs missing pinned driver, Docker, and NVIDIA Container Toolkit packages, and may require one reboot.\n" printf " Host setup may add this trusted local account to the docker group, which grants root-equivalent control. This flow is only for trusted single-user development hosts; shared or managed hosts require an organization-approved Docker access path.\n" @@ -3231,6 +3539,25 @@ maybe_offer_express_install() { platform="$(detect_express_platform)" validate_express_platform_boundary "$platform" validate_station_deepseek_override "$platform" + + # Pair state is written before remote mutation. It therefore outranks every + # prompt/skip path and must recover the companion exact-revision mode/model + # state before any generic or single-Station setup can continue. + if station_dual_pair_resume_pending; then + [ "$platform" = "DGX Station" ] \ + || error "A dual-DGX Station pair resume is pending, but this host no longer satisfies the DGX Station preparation boundary. Refusing to continue without exact pair revalidation." + [ "${NEMOCLAW_NO_EXPRESS:-}" != "1" ] \ + || error "A dual-DGX Station pair resume is pending; finish exact pair revalidation before disabling Station setup." + case "${NEMOCLAW_PROVIDER:-}" in + "") ;; + install-vllm) _STATION_INSTALL_MODE="provider" ;; + *) error "A dual-DGX Station pair resume is pending; finish exact pair revalidation before changing providers." ;; + esac + load_station_express_resume \ + || error "Dual-DGX Station pair state exists without its required installer resume state. Refusing to prompt, fall back, or continue; restore the owner-only state from the accepted revision." + resume_loaded_station_install "$platform" + return 0 + fi # Not on a platform we have an express recipe for — say nothing. if [ -z "$platform" ]; then return 0 @@ -3238,18 +3565,36 @@ maybe_offer_express_install() { # On a supported platform but a skip condition applies — explain why so # the user understands they could have gotten express otherwise. if [ "${NEMOCLAW_NO_EXPRESS:-}" = "1" ]; then - if [ "$platform" = "DGX Station" ]; then clear_station_express_resume; fi + if [ "$platform" = "DGX Station" ]; then + station_dual_pair_resume_pending \ + && error "A dual-DGX Station pair resume is pending; finish exact pair revalidation before disabling Station setup." + clear_station_express_resume + fi info "Detected ${platform}. Skipping express prompt (NEMOCLAW_NO_EXPRESS=1)." return 0 fi if [ -n "${NEMOCLAW_PROVIDER:-}" ]; then - if [ "$platform" = "DGX Station" ]; then clear_station_express_resume; fi + if [ "$platform" = "DGX Station" ] && [ "$NEMOCLAW_PROVIDER" = "install-vllm" ]; then + # An explicit managed-vLLM provider selects the same Station host/pair + # preparation boundary without forcing the rest of the express policy. + # Honor an existing exact-revision reboot resume before configuration. + _STATION_INSTALL_MODE="provider" + load_station_express_resume || true + _SELECTED_EXPRESS_PLATFORM="$platform" + configure_station_express_model + info "Detected ${platform}. Using Station preparation for the explicitly selected managed-vLLM provider." + return 0 + fi + if [ "$platform" = "DGX Station" ]; then + station_dual_pair_resume_pending \ + && error "A dual-DGX Station pair resume is pending; finish exact pair revalidation before changing providers." + clear_station_express_resume + fi info "Detected ${platform}. Skipping express prompt (NEMOCLAW_PROVIDER=${NEMOCLAW_PROVIDER} already set)." return 0 fi if [ "$platform" = "DGX Station" ] && load_station_express_resume; then - info "Detected DGX Station. Resuming the accepted express install after host preparation." - activate_express_install "$platform" + resume_loaded_station_install "$platform" return 0 fi if [ "${NON_INTERACTIVE:-}" = "1" ]; then @@ -3395,6 +3740,7 @@ main() { step 1 "Node.js" install_nodejs ensure_supported_runtime + ensure_station_express_pair step 2 "${_CLI_DISPLAY} CLI" # Ollama and vLLM install/upgrade and model pulls are owned by @@ -3457,6 +3803,7 @@ main() { finalize_install if [[ "${_SELECTED_EXPRESS_PLATFORM:-}" == "DGX Station" ]]; then + clear_station_dual_pair_resume clear_station_express_resume fi } diff --git a/scripts/lib/dgx-station-peer.mts b/scripts/lib/dgx-station-peer.mts new file mode 100644 index 0000000000..20477eea56 --- /dev/null +++ b/scripts/lib/dgx-station-peer.mts @@ -0,0 +1,825 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import net from "node:net"; + +export const DUAL_STATION_RESUME_SCHEMA_VERSION = 1; +export const STATION_PREP_REBOOT_REQUIRED_EXIT = 10; + +const DIRECT_RAIL_PREFIX_LENGTH = 30; +const SAFE_TARGET_PATTERN = + /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?(?:\.[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?)*$/; +const SAFE_USERNAME_PATTERN = /^[A-Za-z_][A-Za-z0-9._-]*$/; +const SAFE_DEVICE_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_.:-]{0,63}$/; +const GPU_UUID_PATTERN = /^GPU-[A-Za-z0-9-]+$/; +const HOST_KEY_DIGEST_PATTERN = /^[a-f0-9]{64}$/; +const HOST_KEY_FINGERPRINT_PATTERN = /^SHA256:[A-Za-z0-9+/]{16,86}={0,2}$/; +const MAC_PATTERN = /^(?:[0-9a-f]{2}:){5}[0-9a-f]{2}$/; + +export type StationPrepMode = "--check" | "--apply" | "--verify"; + +export interface StationIpv4Address { + address: string; + prefixLength: number; +} + +export interface StationDiscoveryRail { + netdev: string; + macAddress: string; + pciAddress: string; + pciName: string; + state: string; + linkLayer: string; + speedMbps: number; + mtu: number; + ipv4Addresses: StationIpv4Address[]; +} + +export interface StationDiscoveryGpu { + index: number; + name: string; + uuid: string; +} + +export interface StationDiscoveryHost { + schemaVersion: 1; + hostname: string; + productName: string; + architecture: string; + gpus: StationDiscoveryGpu[]; + rails: StationDiscoveryRail[]; +} + +export interface PretrustedSshTarget { + requestedTarget: string; + sshTarget: string; + resolvedHost: string; + sshUser: string; + port: number; + lookupHost: string; + hostKeyDigest: string; + keyFingerprints: string[]; + knownHostsLines: string[]; +} + +export interface RailConnectivityRequest { + netdev: string; + sourceAddress: string; + peerAddress: string; + expectedPeerMac: string; +} + +export interface DualStationRailIdentity { + localAddress: string; + localMac: string; + peerAddress: string; + peerMac: string; +} + +export interface DualStationPairIdentity { + peerTarget: string; + hostKeyDigest: string; + localGpuUuid: string; + peerGpuUuid: string; + rails: DualStationRailIdentity[]; +} + +export interface DualStationResumeState extends DualStationPairIdentity { + schemaVersion: 1; + revision: string; + helperSha256: string; + phase: "remote-preparation" | "remote-reboot-required" | "ready"; +} + +export type DualStationPreparationResult = + | { kind: "single-station"; reason: string } + | { kind: "ready"; peerTarget: string; identity: DualStationPairIdentity } + | { kind: "reboot-required"; peerTarget: string; identity: DualStationPairIdentity }; + +export interface DualStationPreparationOptions { + revision: string; + helperSha256: string; + explicitPeer?: string; + reuseExistingManagedPair?: boolean; +} + +export interface DualStationPreparationDeps { + runLocalHelper(mode: "--check" | "--verify"): number; + probeLocalHost(): StationDiscoveryHost; + inspectPretrustedTarget(target: string): PretrustedSshTarget | null; + probePeerHost(target: PretrustedSshTarget): StationDiscoveryHost; + probeLocalConnectivity(requests: readonly RailConnectivityRequest[]): boolean; + probePeerConnectivity( + target: PretrustedSshTarget, + requests: readonly RailConnectivityRequest[], + ): boolean; + runRemoteHelper(target: PretrustedSshTarget, mode: StationPrepMode): number; + readResumeState(): DualStationResumeState | null; + writeResumeState(state: DualStationResumeState): void; + clearResumeState(): void; + log(message: string): void; +} + +type QualifiedRail = { + rail: StationDiscoveryRail; + address: string; + peerAddress: string; + subnet: string; +}; + +type DiscoveryPlan = { + identity: DualStationPairIdentity; + localConnectivity: RailConnectivityRequest[]; + peerConnectivity: RailConnectivityRequest[]; +}; + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function requireString(value: unknown, label: string, maxLength: number): string { + if ( + typeof value !== "string" || + value.length === 0 || + value.length > maxLength || + /[\u0000-\u001f\u007f]/.test(value) + ) { + throw new Error(`${label} must be a non-empty printable string`); + } + return value; +} + +function requireArray(value: unknown, label: string, maxLength: number): unknown[] { + if (!Array.isArray(value) || value.length > maxLength) { + throw new Error(`${label} must be an array with at most ${String(maxLength)} entries`); + } + return value; +} + +function requireInteger(value: unknown, label: string, min: number, max: number): number { + if (!Number.isInteger(value) || (value as number) < min || (value as number) > max) { + throw new Error(`${label} must be an integer between ${String(min)} and ${String(max)}`); + } + return value as number; +} + +function normalizeMac(value: unknown, label: string): string { + const mac = requireString(value, label, 17).toLowerCase(); + if (!MAC_PATTERN.test(mac) || mac === "00:00:00:00:00:00") { + throw new Error(`${label} must be a nonzero canonical MAC address`); + } + const firstOctet = Number.parseInt(mac.slice(0, 2), 16); + if ((firstOctet & 1) !== 0) throw new Error(`${label} must be a unicast MAC address`); + return mac; +} + +function requireIpv4(value: unknown, label: string): string { + const address = requireString(value, label, 15); + if (net.isIP(address) !== 4) throw new Error(`${label} must be IPv4`); + return address; +} + +function isSafeTargetHost(hostname: string): boolean { + return ( + net.isIP(hostname) === 4 || (!/^[0-9.]+$/.test(hostname) && SAFE_TARGET_PATTERN.test(hostname)) + ); +} + +export function validateStationPeerTarget(raw: string): string { + if (raw.length === 0 || raw !== raw.trim() || raw.length > 286) { + throw new Error("Station peer must be one canonical SSH host or user@host"); + } + if (/[/,:;`'"\\$(){}[\]<>|&!?*\s\u0000-\u001f\u007f]/.test(raw)) { + throw new Error("Station peer must be one canonical SSH host or user@host"); + } + const parts = raw.split("@"); + if (parts.length > 2) throw new Error("Station peer must be one canonical SSH host or user@host"); + const username = parts.length === 2 ? parts[0] : ""; + const hostname = parts.at(-1) ?? ""; + const validHost = isSafeTargetHost(hostname); + if ( + !validHost || + (parts.length === 2 && username.length === 0) || + (username.length > 0 && !SAFE_USERNAME_PATTERN.test(username)) + ) { + throw new Error("Station peer must be one canonical SSH host or user@host"); + } + return raw; +} + +function ipv4ToNumber(address: string): number { + return address + .split(".") + .map(Number) + .reduce((value, octet) => value * 256 + octet, 0); +} + +function numberToIpv4(value: number): string { + return [24, 16, 8, 0].map((shift) => Math.floor(value / 2 ** shift) % 256).join("."); +} + +function isPrivateIpv4(address: string): boolean { + const value = ipv4ToNumber(address); + return ( + (value >= ipv4ToNumber("10.0.0.0") && value <= ipv4ToNumber("10.255.255.255")) || + (value >= ipv4ToNumber("172.16.0.0") && value <= ipv4ToNumber("172.31.255.255")) || + (value >= ipv4ToNumber("192.168.0.0") && value <= ipv4ToNumber("192.168.255.255")) + ); +} + +export function deriveSlash30Counterpart(address: string, prefixLength = 30): string | null { + if (prefixLength !== DIRECT_RAIL_PREFIX_LENGTH || net.isIP(address) !== 4) return null; + if (!isPrivateIpv4(address)) return null; + const value = ipv4ToNumber(address); + const network = Math.floor(value / 4) * 4; + const host = value - network; + if (host === 1) return numberToIpv4(network + 2); + if (host === 2) return numberToIpv4(network + 1); + return null; +} + +function subnetOfSlash30(address: string): string { + return `${numberToIpv4(Math.floor(ipv4ToNumber(address) / 4) * 4)}/30`; +} + +export function parseStationDiscoveryHost(value: unknown): StationDiscoveryHost { + if (!isRecord(value) || value.schemaVersion !== 1) { + throw new Error("Station discovery probe schema is unsupported"); + } + const gpus = requireArray(value.gpus, "Station discovery GPUs", 16).map((entry, index) => { + if (!isRecord(entry)) throw new Error(`Station discovery GPU ${String(index)} is invalid`); + const uuid = requireString(entry.uuid, `Station discovery GPU ${String(index)} UUID`, 128); + if (!GPU_UUID_PATTERN.test(uuid)) { + throw new Error(`Station discovery GPU ${String(index)} UUID is invalid`); + } + return { + index: requireInteger(entry.index, `Station discovery GPU ${String(index)} index`, 0, 1024), + name: requireString(entry.name, `Station discovery GPU ${String(index)} name`, 256), + uuid, + }; + }); + const rails = requireArray(value.rails, "Station discovery rails", 16).map((entry, index) => { + if (!isRecord(entry)) throw new Error(`Station discovery rail ${String(index)} is invalid`); + const netdev = requireString( + entry.netdev, + `Station discovery rail ${String(index)} netdev`, + 64, + ); + if (!SAFE_DEVICE_PATTERN.test(netdev)) { + throw new Error(`Station discovery rail ${String(index)} netdev is unsafe`); + } + const pciAddress = requireString( + entry.pciAddress, + `Station discovery rail ${String(index)} PCI address`, + 32, + ); + if (!/^[0-9A-Fa-f]{4}:[0-9A-Fa-f]{2}:[0-9A-Fa-f]{2}\.[0-7]$/.test(pciAddress)) { + throw new Error(`Station discovery rail ${String(index)} PCI address is invalid`); + } + return { + netdev, + macAddress: normalizeMac(entry.macAddress, `Station discovery rail ${String(index)} MAC`), + pciAddress, + pciName: requireString( + entry.pciName, + `Station discovery rail ${String(index)} PCI name`, + 512, + ), + state: requireString(entry.state, `Station discovery rail ${String(index)} state`, 64), + linkLayer: requireString( + entry.linkLayer, + `Station discovery rail ${String(index)} link layer`, + 64, + ), + speedMbps: requireInteger( + entry.speedMbps, + `Station discovery rail ${String(index)} speed`, + -1, + 1_000_000, + ), + mtu: requireInteger(entry.mtu, `Station discovery rail ${String(index)} MTU`, -1, 1_000_000), + ipv4Addresses: requireArray( + entry.ipv4Addresses, + `Station discovery rail ${String(index)} IPv4 addresses`, + 16, + ).map((rawAddress, addressIndex) => { + if (!isRecord(rawAddress)) { + throw new Error( + `Station discovery rail ${String(index)} address ${String(addressIndex)} is invalid`, + ); + } + return { + address: requireIpv4( + rawAddress.address, + `Station discovery rail ${String(index)} address ${String(addressIndex)}`, + ), + prefixLength: requireInteger( + rawAddress.prefixLength, + `Station discovery rail ${String(index)} prefix ${String(addressIndex)}`, + 1, + 32, + ), + }; + }), + }; + }); + return { + schemaVersion: 1, + hostname: requireString(value.hostname, "Station discovery hostname", 256), + productName: requireString(value.productName, "Station discovery product", 512), + architecture: requireString(value.architecture, "Station discovery architecture", 64), + gpus, + rails, + }; +} + +function selectedGb300(host: StationDiscoveryHost, label: string): StationDiscoveryGpu { + const matches = host.gpus.filter((gpu) => /\bGB300\b/i.test(gpu.name)); + if (matches.length !== 1) throw new Error(`${label} must expose exactly one GB300 GPU`); + return matches[0]; +} + +function assertStationIdentity(host: StationDiscoveryHost, label: string): void { + if ( + !( + /DGX[_\s-]+Station/i.test(host.productName) || + (/Station/i.test(host.productName) && /GB300/i.test(host.productName)) + ) || + !/^(?:aarch64|arm64)$/i.test(host.architecture) + ) { + throw new Error(`${label} is not a verified arm64 DGX Station GB300`); + } +} + +function qualifyRails(host: StationDiscoveryHost, label: string): QualifiedRail[] { + const cx8 = host.rails.filter((rail) => /ConnectX[- ]?8|\bCX-?8\b/i.test(rail.pciName)); + if (cx8.length !== 2) throw new Error(`${label} must expose exactly two CX-8 rails`); + const result = cx8.map((rail, index): QualifiedRail => { + if ( + !/\bACTIVE\b/i.test(rail.state) || + rail.linkLayer.toLowerCase() !== "ethernet" || + rail.speedMbps !== 400_000 || + rail.mtu !== 9000 + ) { + throw new Error(`${label} rail ${String(index + 1)} is not active 400G Ethernet MTU 9000`); + } + const plausible = rail.ipv4Addresses + .map((entry) => ({ + entry, + peer: deriveSlash30Counterpart(entry.address, entry.prefixLength), + })) + .filter((entry): entry is { entry: StationIpv4Address; peer: string } => entry.peer !== null); + if (plausible.length !== 1) { + throw new Error( + `${label} rail ${String(index + 1)} must have exactly one usable private /30 address`, + ); + } + return { + rail, + address: plausible[0].entry.address, + peerAddress: plausible[0].peer, + subnet: subnetOfSlash30(plausible[0].entry.address), + }; + }); + if ( + new Set(result.map((entry) => entry.rail.netdev)).size !== 2 || + new Set(result.map((entry) => entry.rail.macAddress)).size !== 2 || + new Set(result.map((entry) => entry.rail.pciAddress)).size !== 2 || + new Set(result.map((entry) => entry.subnet)).size !== 2 || + new Set(result.map((entry) => entry.peerAddress)).size !== 2 + ) { + throw new Error(`${label} CX-8 rail identity is ambiguous`); + } + return result.sort((left, right) => left.subnet.localeCompare(right.subnet)); +} + +export function deriveDiscoveryCandidates(host: StationDiscoveryHost): string[] { + assertStationIdentity(host, "Local host"); + selectedGb300(host, "Local host"); + return qualifyRails(host, "Local host").map((entry) => entry.peerAddress); +} + +function peerHostFromTarget(target: string): string { + return target.slice(target.lastIndexOf("@") + 1); +} + +function buildDiscoveryPlan( + binding: PretrustedSshTarget, + local: StationDiscoveryHost, + peer: StationDiscoveryHost, + automatic: boolean, +): DiscoveryPlan { + assertStationIdentity(local, "Local host"); + assertStationIdentity(peer, "Peer host"); + const localGpu = selectedGb300(local, "Local host"); + const peerGpu = selectedGb300(peer, "Peer host"); + if (localGpu.uuid === peerGpu.uuid) { + throw new Error("Peer SSH target resolved back to the local Station GPU"); + } + const localRails = qualifyRails(local, "Local host"); + const peerRails = qualifyRails(peer, "Peer host"); + const matched = localRails.map((localRail) => { + const peers = peerRails.filter( + (peerRail) => + peerRail.subnet === localRail.subnet && + peerRail.address === localRail.peerAddress && + peerRail.peerAddress === localRail.address, + ); + if (peers.length !== 1) { + throw new Error("Peer did not report one reciprocal address and MAC on each /30 rail"); + } + return { local: localRail, peer: peers[0] }; + }); + if (new Set(matched.map((entry) => entry.peer.rail.macAddress)).size !== 2) { + throw new Error("Peer rail MAC identity is ambiguous"); + } + if ( + automatic && + !matched.some((entry) => entry.peer.address === peerHostFromTarget(binding.requestedTarget)) + ) { + throw new Error("Pretrusted discovery target is not one of the reciprocal peer rail addresses"); + } + const rails = matched.map( + (entry): DualStationRailIdentity => ({ + localAddress: entry.local.address, + localMac: entry.local.rail.macAddress, + peerAddress: entry.peer.address, + peerMac: entry.peer.rail.macAddress, + }), + ); + return { + identity: { + peerTarget: binding.sshTarget, + hostKeyDigest: binding.hostKeyDigest, + localGpuUuid: localGpu.uuid, + peerGpuUuid: peerGpu.uuid, + rails, + }, + localConnectivity: matched.map((entry) => ({ + netdev: entry.local.rail.netdev, + sourceAddress: entry.local.address, + peerAddress: entry.peer.address, + expectedPeerMac: entry.peer.rail.macAddress, + })), + peerConnectivity: matched.map((entry) => ({ + netdev: entry.peer.rail.netdev, + sourceAddress: entry.peer.address, + peerAddress: entry.local.address, + expectedPeerMac: entry.local.rail.macAddress, + })), + }; +} + +function validateRailIdentity(value: unknown, label: string): DualStationRailIdentity { + if (!isRecord(value)) throw new Error(`${label} must be an object`); + const localAddress = requireIpv4(value.localAddress, `${label}.localAddress`); + const peerAddress = requireIpv4(value.peerAddress, `${label}.peerAddress`); + if ( + deriveSlash30Counterpart(localAddress) !== peerAddress || + deriveSlash30Counterpart(peerAddress) !== localAddress + ) { + throw new Error(`${label} must contain reciprocal private /30 addresses`); + } + return { + localAddress, + localMac: normalizeMac(value.localMac, `${label}.localMac`), + peerAddress, + peerMac: normalizeMac(value.peerMac, `${label}.peerMac`), + }; +} + +export function parseDualStationResumeState(value: unknown): DualStationResumeState { + if (!isRecord(value) || value.schemaVersion !== DUAL_STATION_RESUME_SCHEMA_VERSION) { + throw new Error("Dual-Station resume state schema is unsupported"); + } + const revision = requireString(value.revision, "Dual-Station resume revision", 40); + if (!/^[a-f0-9]{40}$/.test(revision)) { + throw new Error("Dual-Station resume revision is invalid"); + } + const helperSha256 = requireString(value.helperSha256, "Dual-Station resume helper SHA-256", 64); + if (!HOST_KEY_DIGEST_PATTERN.test(helperSha256)) { + throw new Error("Dual-Station resume helper SHA-256 is invalid"); + } + const peerTarget = validateStationPeerTarget( + requireString(value.peerTarget, "Dual-Station resume peer target", 286), + ); + const hostKeyDigest = requireString( + value.hostKeyDigest, + "Dual-Station resume host-key digest", + 64, + ); + if (!HOST_KEY_DIGEST_PATTERN.test(hostKeyDigest)) { + throw new Error("Dual-Station resume host-key digest is invalid"); + } + const localGpuUuid = requireString(value.localGpuUuid, "Dual-Station local GPU UUID", 128); + const peerGpuUuid = requireString(value.peerGpuUuid, "Dual-Station peer GPU UUID", 128); + if ( + !GPU_UUID_PATTERN.test(localGpuUuid) || + !GPU_UUID_PATTERN.test(peerGpuUuid) || + localGpuUuid === peerGpuUuid + ) { + throw new Error("Dual-Station resume GPU identity is invalid"); + } + const phase = value.phase; + if (phase !== "remote-preparation" && phase !== "remote-reboot-required" && phase !== "ready") { + throw new Error("Dual-Station resume phase is invalid"); + } + const rails = requireArray(value.rails, "Dual-Station resume rails", 2) + .map((entry, index) => validateRailIdentity(entry, `Dual-Station resume rail ${String(index)}`)) + .sort((left, right) => left.localAddress.localeCompare(right.localAddress)); + if ( + rails.length !== 2 || + new Set(rails.map((rail) => rail.localAddress)).size !== 2 || + new Set(rails.map((rail) => rail.peerAddress)).size !== 2 || + new Set(rails.map((rail) => rail.localMac)).size !== 2 || + new Set(rails.map((rail) => rail.peerMac)).size !== 2 + ) { + throw new Error("Dual-Station resume rail identity is ambiguous"); + } + return { + schemaVersion: 1, + revision, + helperSha256, + phase, + peerTarget, + hostKeyDigest, + localGpuUuid, + peerGpuUuid, + rails, + }; +} + +export function validateResumeFileMetadata( + metadata: { isFile: boolean; isSymbolicLink: boolean; uid: number; mode: number; size: number }, + expectedUid: number, +): void { + if (metadata.isSymbolicLink || !metadata.isFile) { + throw new Error("Dual-Station resume state must be a regular file, not a symlink"); + } + if (metadata.uid !== expectedUid) { + throw new Error("Dual-Station resume state is not owned by the current user"); + } + if ((metadata.mode & 0o777) !== 0o600) { + throw new Error("Dual-Station resume state must have mode 0600"); + } + if (metadata.size <= 0 || metadata.size > 16 * 1024) { + throw new Error("Dual-Station resume state size is invalid"); + } +} + +function canonicalPairIdentity(value: DualStationPairIdentity): DualStationPairIdentity { + return { + peerTarget: value.peerTarget, + hostKeyDigest: value.hostKeyDigest, + localGpuUuid: value.localGpuUuid, + peerGpuUuid: value.peerGpuUuid, + rails: [...value.rails].sort((left, right) => + left.localAddress.localeCompare(right.localAddress), + ), + }; +} + +function samePair(left: DualStationPairIdentity, right: DualStationPairIdentity): boolean { + return ( + JSON.stringify(canonicalPairIdentity(left)) === JSON.stringify(canonicalPairIdentity(right)) + ); +} + +function samePhysicalSshIdentity(left: PretrustedSshTarget, right: PretrustedSshTarget): boolean { + return ( + left.sshUser === right.sshUser && + left.port === right.port && + left.hostKeyDigest === right.hostKeyDigest + ); +} + +function validateKnownHostsLookupHost(binding: PretrustedSshTarget): void { + const expected = + binding.port === 22 + ? binding.resolvedHost + : `[${binding.resolvedHost}]:${String(binding.port)}`; + if (binding.lookupHost !== expected) { + throw new Error("Pretrusted SSH target has an invalid known-hosts lookup identity"); + } +} + +function validateBinding(binding: PretrustedSshTarget): void { + validateStationPeerTarget(binding.requestedTarget); + validateStationPeerTarget(binding.sshTarget); + if (!isSafeTargetHost(binding.resolvedHost)) { + throw new Error("Pretrusted SSH target resolved to an unsafe host"); + } + if (!SAFE_USERNAME_PATTERN.test(binding.sshUser) || binding.port < 1 || binding.port > 65535) { + throw new Error("Pretrusted SSH target has an unsafe user or port"); + } + validateKnownHostsLookupHost(binding); + if (!HOST_KEY_DIGEST_PATTERN.test(binding.hostKeyDigest)) { + throw new Error("Pretrusted SSH target has an invalid host-key digest"); + } + if ( + binding.keyFingerprints.length === 0 || + binding.keyFingerprints.some( + (fingerprint) => !HOST_KEY_FINGERPRINT_PATTERN.test(fingerprint), + ) || + binding.knownHostsLines.length === 0 || + binding.knownHostsLines.some( + (line) => line.length === 0 || line.length > 16 * 1024 || /[\u0000\r\n]/.test(line), + ) + ) { + throw new Error("Pretrusted SSH target has invalid known-hosts evidence"); + } +} + +function selectPretrustedTarget( + options: DualStationPreparationOptions, + local: StationDiscoveryHost, + resume: DualStationResumeState | null, + deps: DualStationPreparationDeps, +): { binding: PretrustedSshTarget; automatic: boolean } | DualStationPreparationResult { + const candidates = deriveDiscoveryCandidates(local); + const explicitPeer = options.explicitPeer?.trim() ?? ""; + if (explicitPeer) validateStationPeerTarget(explicitPeer); + + if (resume) { + if (resume.revision !== options.revision) { + throw new Error( + `Dual-Station resume requires NemoClaw revision ${resume.revision}; current revision is ${options.revision}`, + ); + } + if (resume.helperSha256 !== options.helperSha256) { + throw new Error("The reviewed Station host-preparation helper changed during reboot resume"); + } + if (explicitPeer && explicitPeer !== resume.peerTarget) { + throw new Error("Explicit Station peer does not match the reboot-resume pair"); + } + if (!explicitPeer && !candidates.includes(peerHostFromTarget(resume.peerTarget))) { + throw new Error("The reboot-resume peer is no longer a derived local /30 counterpart"); + } + const binding = deps.inspectPretrustedTarget(resume.peerTarget); + if (!binding) throw new Error("The reboot-resume peer is no longer pretrusted"); + validateBinding(binding); + if (binding.hostKeyDigest !== resume.hostKeyDigest) { + throw new Error("The reboot-resume peer host-key identity changed"); + } + return { binding, automatic: !explicitPeer }; + } + + if (explicitPeer) { + const binding = deps.inspectPretrustedTarget(explicitPeer); + if (!binding) + throw new Error("Explicit Station peer is not pretrusted; SSH trust was not changed"); + validateBinding(binding); + return { binding, automatic: false }; + } + + const trusted: PretrustedSshTarget[] = []; + for (const candidate of candidates) { + try { + const binding = deps.inspectPretrustedTarget(candidate); + if (!binding) continue; + validateBinding(binding); + trusted.push(binding); + } catch (error) { + deps.log( + `Ignoring derived peer ${candidate}: pre-existing SSH trust is unusable (${(error as Error).message})`, + ); + } + } + if (trusted.length === 0) { + return { + kind: "single-station", + reason: "No derived dual-rail peer address has pre-existing SSH host-key trust", + }; + } + if (trusted.length === 2 && !samePhysicalSshIdentity(trusted[0], trusted[1])) { + return { + kind: "single-station", + reason: "The two derived rail addresses map to different pretrusted SSH identities", + }; + } + const binding = [...trusted].sort((left, right) => + left.requestedTarget.localeCompare(right.requestedTarget), + )[0]; + return { binding, automatic: true }; +} + +function fallbackOrThrow( + strict: boolean, + reason: string, +): Extract { + if (strict) throw new Error(reason); + return { kind: "single-station", reason }; +} + +export function prepareDualStationPair( + options: DualStationPreparationOptions, + deps: DualStationPreparationDeps, +): DualStationPreparationResult { + if (!/^[a-f0-9]{40}$/.test(options.revision)) { + throw new Error("Exact NemoClaw revision is required for dual-Station preparation"); + } + if (!HOST_KEY_DIGEST_PATTERN.test(options.helperSha256)) { + throw new Error("Exact Station host-preparation helper SHA-256 is required"); + } + + if (!options.reuseExistingManagedPair) { + deps.log("Checking the local Station with the reviewed host-preparation helper"); + if (deps.runLocalHelper("--check") !== 0) { + throw new Error("Local DGX Station host-preparation check failed before peer contact"); + } + if (deps.runLocalHelper("--verify") !== 0) { + throw new Error("Local DGX Station verification failed before peer contact"); + } + } else { + deps.log("Revalidating the exact running managed pair without host mutation"); + } + + const resume = deps.readResumeState(); + let local: StationDiscoveryHost; + try { + local = deps.probeLocalHost(); + deriveDiscoveryCandidates(local); + } catch (error) { + if (resume || options.explicitPeer?.trim()) throw error; + return { + kind: "single-station", + reason: `Local direct-rail discovery is unavailable: ${(error as Error).message}`, + }; + } + + const selected = selectPretrustedTarget(options, local, resume, deps); + if ("kind" in selected) return selected; + const strict = Boolean( + resume || options.explicitPeer?.trim() || options.reuseExistingManagedPair, + ); + const { binding, automatic } = selected; + + let peer: StationDiscoveryHost; + try { + peer = deps.probePeerHost(binding); + } catch (error) { + return fallbackOrThrow( + strict, + `Trusted peer identity probe failed: ${(error as Error).message}`, + ); + } + + let plan: DiscoveryPlan; + try { + plan = buildDiscoveryPlan(binding, local, peer, automatic); + } catch (error) { + return fallbackOrThrow(strict, `Trusted peer was not reciprocal: ${(error as Error).message}`); + } + if (resume && !samePair(resume, plan.identity)) { + throw new Error("The physical dual-Station pair changed during reboot resume"); + } + + let connectivityReady = false; + try { + connectivityReady = + deps.probeLocalConnectivity(plan.localConnectivity) && + deps.probePeerConnectivity(binding, plan.peerConnectivity); + } catch { + connectivityReady = false; + } + if (!connectivityReady) { + return fallbackOrThrow( + strict, + "Trusted peer failed direct-route, neighbor-MAC, or jumbo-frame checks", + ); + } + + const state: DualStationResumeState = { + schemaVersion: 1, + revision: options.revision, + helperSha256: options.helperSha256, + phase: "remote-preparation", + ...plan.identity, + }; + deps.writeResumeState(state); + if (options.reuseExistingManagedPair) { + deps.writeResumeState({ ...state, phase: "ready" }); + return { kind: "ready", peerTarget: binding.sshTarget, identity: plan.identity }; + } + deps.log(`Preparing reciprocal peer ${binding.sshTarget} with the exact reviewed helper`); + + if (deps.runRemoteHelper(binding, "--check") !== 0) { + throw new Error( + "Peer DGX Station host-preparation check failed; the selected pair remains pinned", + ); + } + const applyStatus = deps.runRemoteHelper(binding, "--apply"); + if (applyStatus === STATION_PREP_REBOOT_REQUIRED_EXIT) { + deps.writeResumeState({ ...state, phase: "remote-reboot-required" }); + return { kind: "reboot-required", peerTarget: binding.sshTarget, identity: plan.identity }; + } + if (applyStatus !== 0) { + throw new Error("Peer DGX Station host preparation failed; refusing single-Station fallback"); + } + if (deps.runRemoteHelper(binding, "--verify") !== 0) { + throw new Error("Peer DGX Station verification failed; refusing single-Station fallback"); + } + + deps.writeResumeState({ ...state, phase: "ready" }); + return { kind: "ready", peerTarget: binding.sshTarget, identity: plan.identity }; +} diff --git a/scripts/prepare-dgx-station-host.sh b/scripts/prepare-dgx-station-host.sh index 9824429d0c..2573591394 100755 --- a/scripts/prepare-dgx-station-host.sh +++ b/scripts/prepare-dgx-station-host.sh @@ -73,6 +73,17 @@ on_error() { exit "$rc" } +# Remote pair preparation must never fall back to an interactive sudo prompt. +# Keep the helper's local behavior unchanged unless its caller explicitly opts +# into this strict mode after proving `sudo -n true` succeeds. +sudo() { + if [[ "${NEMOCLAW_STATION_PREP_SUDO_NONINTERACTIVE:-0}" == "1" && "${1:-}" != "-n" ]]; then + command sudo -n "$@" + else + command sudo "$@" + fi +} + usage() { cat <<'EOF' Usage: prepare-dgx-station-host.sh --check|--apply|--verify diff --git a/scripts/prepare-dual-dgx-station.mts b/scripts/prepare-dual-dgx-station.mts new file mode 100755 index 0000000000..1df59c1528 --- /dev/null +++ b/scripts/prepare-dual-dgx-station.mts @@ -0,0 +1,1030 @@ +#!/usr/bin/env -S node --no-warnings --experimental-strip-types +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import { createHash, randomBytes } from "node:crypto"; +import fs from "node:fs"; +import net from "node:net"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { + type DualStationPreparationDeps, + type DualStationResumeState, + type PretrustedSshTarget, + parseDualStationResumeState, + parseStationDiscoveryHost, + prepareDualStationPair, + type RailConnectivityRequest, + type StationDiscoveryHost, + type StationPrepMode, + validateResumeFileMetadata, + validateStationPeerTarget, +} from "./lib/dgx-station-peer.mts"; + +const COMMAND_TIMEOUT_MS = 60_000; +const HELPER_TIMEOUT_MS = 2 * 60 * 60_000; +const MAX_PROBE_OUTPUT_BYTES = 1024 * 1024; + +type CommandResult = { + status: number | null; + stdout: string; + stderr: string; + error?: string; +}; + +type CliOptions = { + helperPath: string; + statePath: string; + revision: string; + explicitPeer?: string; + reuseExistingManagedPair: boolean; + clearState: boolean; +}; + +type SshConfig = Map; + +const SUBPROCESS_ENV_NAMES = new Set([ + "HOME", + "USER", + "LOGNAME", + "SHELL", + "PATH", + "TERM", + "HOSTNAME", + "NODE_ENV", + "TMPDIR", + "TMP", + "TEMP", + "LANG", + "HTTP_PROXY", + "HTTPS_PROXY", + "NO_PROXY", + "http_proxy", + "https_proxy", + "no_proxy", + "SSL_CERT_FILE", + "SSL_CERT_DIR", + "NODE_EXTRA_CA_CERTS", + "GIT_SSL_CAINFO", + "GIT_SSL_CAPATH", + "CURL_CA_BUNDLE", + "REQUESTS_CA_BUNDLE", + "SSH_AUTH_SOCK", +]); + +const STATION_DISCOVERY_PROBE = String.raw` +import csv +import json +from pathlib import Path +import platform +import re +import socket +import subprocess + +def read_text(path): + try: + return Path(path).read_text(encoding="utf-8").rstrip("\x00").strip() + except (OSError, UnicodeError): + return "" + +def run(argv, timeout=5): + try: + result = subprocess.run( + argv, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + timeout=timeout, + check=False, + ) + return result.returncode, result.stdout.strip() + except (FileNotFoundError, OSError, subprocess.TimeoutExpired): + return 127, "" + +def product_name(): + for candidate in ( + "/sys/class/dmi/id/product_name", + "/sys/devices/virtual/dmi/id/product_name", + "/sys/firmware/devicetree/base/model", + ): + value = read_text(candidate) + if value: + return value + return "" + +def gpu_inventory(): + rc, output = run([ + "nvidia-smi", + "--query-gpu=index,name,uuid", + "--format=csv,noheader,nounits", + ]) + if rc != 0: + return [] + result = [] + for row in csv.reader(output.splitlines()): + if len(row) != 3: + continue + try: + index = int(row[0].strip()) + except ValueError: + continue + result.append({"index": index, "name": row[1].strip(), "uuid": row[2].strip()}) + return result + +def ipv4_addresses(netdev): + rc, output = run(["ip", "-j", "-4", "address", "show", "dev", netdev]) + if rc != 0: + return [] + try: + links = json.loads(output) + except json.JSONDecodeError: + return [] + result = [] + for link in links if isinstance(links, list) else []: + for address in link.get("addr_info", []): + if address.get("family") != "inet" or address.get("scope") == "host": + continue + local = address.get("local") + prefix = address.get("prefixlen") + if isinstance(local, str) and isinstance(prefix, int): + result.append({"address": local, "prefixLength": prefix}) + return result + +def rail_inventory(): + rc, output = run(["ibdev2netdev"]) + if rc != 0: + return [] + rails = [] + pattern = re.compile(r"^(\S+)\s+port\s+(\d+)\s+==>\s+(\S+)\s+\(([^)]*)\)") + for line in output.splitlines(): + match = pattern.match(line.strip()) + if not match: + continue + rdma_device, raw_port, netdev, _reported_state = match.groups() + port = int(raw_port) + device_path = Path("/sys/class/net") / netdev / "device" + try: + pci_address = device_path.resolve(strict=True).name + except OSError: + pci_address = "" + rc_lspci, pci_name = run(["lspci", "-D", "-s", pci_address]) if pci_address else (127, "") + if rc_lspci != 0: + pci_name = "" + try: + speed_mbps = int(read_text(Path("/sys/class/net") / netdev / "speed")) + except ValueError: + speed_mbps = -1 + try: + mtu = int(read_text(Path("/sys/class/net") / netdev / "mtu")) + except ValueError: + mtu = -1 + ib_port = Path("/sys/class/infiniband") / rdma_device / "ports" / str(port) + rails.append({ + "netdev": netdev, + "macAddress": read_text(Path("/sys/class/net") / netdev / "address").lower(), + "pciAddress": pci_address, + "pciName": pci_name, + "state": read_text(ib_port / "state"), + "linkLayer": read_text(ib_port / "link_layer"), + "speedMbps": speed_mbps, + "mtu": mtu, + "ipv4Addresses": ipv4_addresses(netdev), + }) + return rails + +print(json.dumps({ + "schemaVersion": 1, + "hostname": socket.gethostname(), + "productName": product_name(), + "architecture": platform.machine(), + "gpus": gpu_inventory(), + "rails": rail_inventory(), +}, separators=(",", ":"))) +`; + +const CONNECTIVITY_PROBE = String.raw` +import ipaddress +import json +import subprocess +import sys + +def run(argv, timeout=5): + try: + result = subprocess.run( + argv, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + timeout=timeout, + check=False, + ) + return result.returncode, result.stdout.strip() + except (FileNotFoundError, OSError, subprocess.TimeoutExpired): + return 127, "" + +if len(sys.argv[1:]) != 6: + raise SystemExit("expected two netdev/source/peer triples") + +checks = [] +for offset in (0, 3): + netdev, source, peer = sys.argv[1 + offset:4 + offset] + route_device = "" + route_source = "" + route_gateway = None + route_scope = "" + peer_mac = "" + peer_neighbor_state = "" + rc, output = run(["ip", "-j", "route", "get", peer, "from", source, "oif", netdev]) + if rc == 0: + try: + routes = json.loads(output) + route = routes[0] if isinstance(routes, list) and routes else {} + route_device = route.get("dev", "") if isinstance(route, dict) else "" + route_source = route.get("prefsrc", route.get("src", "")) if isinstance(route, dict) else "" + route_gateway = route.get("gateway") if isinstance(route, dict) else None + except json.JSONDecodeError: + pass + network = str(ipaddress.ip_network(source + "/30", strict=False)) + link_rc, link_output = run(["ip", "-j", "route", "show", "exact", network, "dev", netdev]) + if link_rc == 0: + try: + link_routes = json.loads(link_output) + link_route = link_routes[0] if isinstance(link_routes, list) and link_routes else {} + if ( + isinstance(link_route, dict) + and link_route.get("dst") == network + and link_route.get("dev") == netdev + and link_route.get("gateway") is None + ): + route_scope = link_route.get("scope", "") + except json.JSONDecodeError: + pass + ping_rc, _ = run([ + "ping", "-4", "-M", "do", "-s", "8972", "-c", "1", "-W", "2", "-I", source, peer, + ]) + neighbor_rc, neighbor_output = run(["ip", "-j", "neighbor", "show", "to", peer, "dev", netdev]) + if neighbor_rc == 0: + try: + neighbors = json.loads(neighbor_output) + neighbor = neighbors[0] if isinstance(neighbors, list) and neighbors else {} + if isinstance(neighbor, dict) and neighbor.get("dst") == peer and neighbor.get("dev") == netdev: + peer_mac = str(neighbor.get("lladdr", "")).lower() + raw_state = neighbor.get("state", "") + peer_neighbor_state = ",".join(raw_state) if isinstance(raw_state, list) else str(raw_state) + except json.JSONDecodeError: + pass + checks.append({ + "netdev": netdev, + "sourceAddress": source, + "peerAddress": peer, + "routeDevice": route_device, + "routeSource": route_source, + "routeGateway": route_gateway, + "routeScope": route_scope, + "peerMac": peer_mac, + "peerNeighborState": peer_neighbor_state, + "jumboPing": ping_rc == 0, + }) + +print(json.dumps({"schemaVersion": 1, "checks": checks}, separators=(",", ":"))) +`; + +export function buildStationPrepSubprocessEnv( + source: NodeJS.ProcessEnv = process.env, +): NodeJS.ProcessEnv { + const env: NodeJS.ProcessEnv = {}; + for (const [name, value] of Object.entries(source)) { + if ( + value !== undefined && + (SUBPROCESS_ENV_NAMES.has(name) || name.startsWith("LC_") || name.startsWith("XDG_")) + ) { + env[name] = value; + } + } + env.LC_ALL = "C"; + env.LANG = "C"; + return env; +} + +function runCommand( + file: string, + args: readonly string[], + input = "", + timeout = COMMAND_TIMEOUT_MS, + maxBuffer = MAX_PROBE_OUTPUT_BYTES, +): CommandResult { + const result = spawnSync(file, [...args], { + encoding: "utf8", + input, + timeout, + maxBuffer, + killSignal: "SIGKILL", + windowsHide: true, + env: buildStationPrepSubprocessEnv(), + }); + return { + status: result.status, + stdout: result.stdout ?? "", + stderr: result.stderr ?? "", + error: result.error?.message, + }; +} + +function commandSucceeded(result: CommandResult, requireOutput = false): boolean { + return ( + result.status === 0 && !result.error && (!requireOutput || result.stdout.trim().length > 0) + ); +} + +function runStreamingCommand( + file: string, + args: readonly string[], + input: string, + timeout = HELPER_TIMEOUT_MS, +): number { + const result = spawnSync(file, [...args], { + input, + timeout, + killSignal: "SIGKILL", + windowsHide: true, + env: buildStationPrepSubprocessEnv(), + // Keep stdout machine-readable for the coordinator result while allowing + // long package and acceptance-image operations to stream without a + // bounded child-process buffer. + stdio: ["pipe", process.stderr.fd, process.stderr.fd], + }); + if (result.error) { + process.stderr.write(`[station-pair] ${result.error.message}\n`); + } + return result.status ?? 1; +} + +export function strictStationPrepSshTransportArgs(): string[] { + return [ + "-T", + "-o", + "BatchMode=yes", + "-o", + "StrictHostKeyChecking=yes", + "-o", + "VerifyHostKeyDNS=no", + "-o", + "NoHostAuthenticationForLocalhost=no", + "-o", + "NumberOfPasswordPrompts=0", + "-o", + "PasswordAuthentication=no", + "-o", + "KbdInteractiveAuthentication=no", + "-o", + "PreferredAuthentications=publickey", + "-o", + "ConnectTimeout=5", + "-o", + "ConnectionAttempts=1", + "-o", + "ServerAliveInterval=5", + "-o", + "ServerAliveCountMax=1", + "-o", + "ClearAllForwardings=yes", + "-o", + "ForwardAgent=no", + "-o", + "ForwardX11=no", + "-o", + "ForwardX11Trusted=no", + "-o", + "Tunnel=no", + "-o", + "UpdateHostKeys=no", + "-o", + "ControlMaster=no", + "-o", + "ControlPath=none", + "-o", + "PermitLocalCommand=no", + "-o", + "RemoteCommand=none", + "-o", + "ProxyCommand=none", + "-o", + "ProxyJump=none", + "-o", + "KnownHostsCommand=none", + "-o", + "LogLevel=ERROR", + ]; +} + +function parseSshConfig(stdout: string): SshConfig { + const values = new Map(); + for (const rawLine of stdout.split(/\r?\n/)) { + const line = rawLine.trim(); + if (!line) continue; + const separator = line.search(/\s/); + if (separator <= 0) throw new Error("ssh -G returned malformed effective configuration"); + const key = line.slice(0, separator).toLowerCase(); + const value = line.slice(separator).trim(); + values.set(key, [...(values.get(key) ?? []), value]); + } + return values; +} + +function assertStrictSshConfig(values: SshConfig): void { + const exactly = (key: string, allowed: readonly string[]): boolean => { + const observed = (values.get(key) ?? []).map((value) => value.toLowerCase()); + return observed.length === 1 && allowed.includes(observed[0]); + }; + const absentOrNone = (key: string): boolean => { + const observed = (values.get(key) ?? []).map((value) => value.toLowerCase()); + return observed.length === 0 || (observed.length === 1 && observed[0] === "none"); + }; + const sendEnv = (values.get("sendenv") ?? []).map((value) => value.toLowerCase()); + if ( + !exactly("batchmode", ["yes"]) || + !exactly("stricthostkeychecking", ["yes", "true"]) || + !exactly("verifyhostkeydns", ["false", "no"]) || + !exactly("nohostauthenticationforlocalhost", ["false", "no"]) || + !exactly("permitlocalcommand", ["no"]) || + !exactly("forwardagent", ["no"]) || + !exactly("forwardx11", ["no"]) || + !exactly("forwardx11trusted", ["no"]) || + !exactly("tunnel", ["false", "no"]) || + !exactly("updatehostkeys", ["false", "no"]) || + !exactly("controlmaster", ["false", "no"]) || + !absentOrNone("controlpath") || + !absentOrNone("remotecommand") || + !absentOrNone("proxycommand") || + !absentOrNone("proxyjump") || + !absentOrNone("localcommand") || + !absentOrNone("knownhostscommand") || + values.has("localforward") || + values.has("remoteforward") || + values.has("dynamicforward") || + values.has("setenv") || + !sendEnv.every((value) => value === "lang" || value === "lc_*") + ) { + throw new Error("Effective SSH configuration is unsafe for Station peer preparation"); + } +} + +function oneSshConfigValue(values: SshConfig, key: string): string { + const entries = values.get(key) ?? []; + if (entries.length !== 1 || entries[0].length === 0) { + throw new Error(`Effective SSH configuration must define exactly one ${key}`); + } + return entries[0]; +} + +function fingerprintKnownHostKey(keyType: string, keyData: string): string | null { + const result = runCommand( + "ssh-keygen", + ["-l", "-E", "sha256", "-f", "-"], + `${keyType} ${keyData}\n`, + ); + if (!commandSucceeded(result, true)) return null; + return result.stdout.match(/\b(SHA256:[A-Za-z0-9+/]{16,86}={0,2})\b/)?.[1] ?? null; +} + +function knownHostEvidence( + lookupHost: string, + files: readonly string[], +): { + lines: string[]; + fingerprints: string[]; + digest: string; +} | null { + const lines = new Set(); + const keys = new Set(); + const fingerprints = new Set(); + for (const file of files) { + if (!path.isAbsolute(file)) continue; + let metadata: fs.Stats; + try { + metadata = fs.lstatSync(file); + } catch { + continue; + } + const uid = process.getuid?.(); + if ( + !metadata.isFile() || + metadata.isSymbolicLink() || + uid === undefined || + (metadata.uid !== uid && metadata.uid !== 0) || + (metadata.mode & 0o022) !== 0 + ) { + continue; + } + const result = runCommand("ssh-keygen", ["-F", lookupHost, "-f", file]); + if (!commandSucceeded(result, true)) continue; + for (const rawLine of result.stdout.split(/\r?\n/)) { + const line = rawLine.trim(); + if (!line || line.startsWith("#") || /[\u0000\r\n]/.test(line)) continue; + const fields = line.split(/\s+/); + const marker = fields[0]?.startsWith("@") ? fields.shift() : ""; + if (fields.length < 3) continue; + const [_hosts, keyType, keyData] = fields; + if (!/^(?:ssh-|ecdsa-|sk-)[A-Za-z0-9@._+-]+$/.test(keyType)) continue; + if (!/^[A-Za-z0-9+/]+={0,3}$/.test(keyData)) continue; + const fingerprint = fingerprintKnownHostKey(keyType, keyData); + if (!fingerprint) continue; + lines.add(line); + keys.add(`${marker ?? ""}|${keyType}|${keyData}`); + // Preserve matching revocations in the private pinned file so the + // subsequent SSH connection cannot resurrect a key the operator or + // system administrator explicitly revoked. A revoked line alone is not + // positive trust evidence. + if (marker === "@revoked") continue; + fingerprints.add(fingerprint); + } + } + if (lines.size === 0 || fingerprints.size === 0) return null; + return { + lines: [...lines].sort(), + fingerprints: [...fingerprints].sort(), + digest: createHash("sha256") + .update([...keys].sort().join("\n")) + .digest("hex"), + }; +} + +export function inspectPretrustedSshTarget(target: string): PretrustedSshTarget | null { + validateStationPeerTarget(target); + const configResult = runCommand( + "ssh", + ["-G", ...strictStationPrepSshTransportArgs(), "--", target], + "", + ); + if (!commandSucceeded(configResult, true)) return null; + const config = parseSshConfig(configResult.stdout); + assertStrictSshConfig(config); + const resolvedHost = oneSshConfigValue(config, "hostname"); + const sshUser = oneSshConfigValue(config, "user"); + const portText = oneSshConfigValue(config, "port"); + const port = Number(portText); + if (!Number.isInteger(port) || port < 1 || port > 65535) { + throw new Error("Effective SSH port is invalid"); + } + const requestedHost = target.slice(target.lastIndexOf("@") + 1); + if (net.isIP(requestedHost) === 4 && resolvedHost !== requestedHost) { + throw new Error("Automatic rail target was remapped by SSH configuration"); + } + const alias = config.get("hostkeyalias")?.[0]; + const baseLookupHost = alias && alias.toLowerCase() !== "none" ? alias : resolvedHost; + validateStationPeerTarget(baseLookupHost); + const lookupHost = port === 22 ? baseLookupHost : `[${baseLookupHost}]:${String(port)}`; + const knownHostFiles = [ + ...(config.get("userknownhostsfile") ?? []), + ...(config.get("globalknownhostsfile") ?? []), + ].flatMap((entry) => entry.split(/\s+/).filter((value) => value && value !== "none")); + const evidence = knownHostEvidence(lookupHost, knownHostFiles); + if (!evidence) return null; + return { + requestedTarget: target, + sshTarget: target, + resolvedHost, + sshUser, + port, + lookupHost, + hostKeyDigest: evidence.digest, + keyFingerprints: evidence.fingerprints, + knownHostsLines: evidence.lines, + }; +} + +function parseHostResult(result: CommandResult, label: string): StationDiscoveryHost { + if (!commandSucceeded(result, true)) { + throw new Error(`${label} failed${result.error ? `: ${result.error}` : ""}`); + } + if (Buffer.byteLength(result.stdout, "utf8") > MAX_PROBE_OUTPUT_BYTES) { + throw new Error(`${label} output is too large`); + } + let value: unknown; + try { + value = JSON.parse(result.stdout); + } catch { + throw new Error(`${label} returned invalid JSON`); + } + return parseStationDiscoveryHost(value); +} + +function validateConnectivityArgs(requests: readonly RailConnectivityRequest[]): string[] { + if (requests.length !== 2) throw new Error("Station connectivity requires exactly two rails"); + const args: string[] = []; + for (const request of requests) { + if (!/^[A-Za-z0-9][A-Za-z0-9_.:-]{0,63}$/.test(request.netdev)) { + throw new Error("Station connectivity netdev is unsafe"); + } + if (net.isIP(request.sourceAddress) !== 4 || net.isIP(request.peerAddress) !== 4) { + throw new Error("Station connectivity addresses are invalid"); + } + if (!/^(?:[0-9a-f]{2}:){5}[0-9a-f]{2}$/.test(request.expectedPeerMac)) { + throw new Error("Station connectivity peer MAC is invalid"); + } + args.push(request.netdev, request.sourceAddress, request.peerAddress); + } + return args; +} + +function connectivityMatches( + result: CommandResult, + requests: readonly RailConnectivityRequest[], +): boolean { + if (!commandSucceeded(result, true)) return false; + let value: unknown; + try { + value = JSON.parse(result.stdout); + } catch { + return false; + } + if ( + typeof value !== "object" || + value === null || + !("schemaVersion" in value) || + value.schemaVersion !== 1 || + !("checks" in value) || + !Array.isArray(value.checks) || + value.checks.length !== requests.length + ) { + return false; + } + const checks = value.checks as unknown[]; + return requests.every((request) => + checks.some( + (check) => + typeof check === "object" && + check !== null && + "netdev" in check && + check.netdev === request.netdev && + "sourceAddress" in check && + check.sourceAddress === request.sourceAddress && + "peerAddress" in check && + check.peerAddress === request.peerAddress && + "routeDevice" in check && + check.routeDevice === request.netdev && + "routeSource" in check && + check.routeSource === request.sourceAddress && + "routeGateway" in check && + check.routeGateway === null && + "routeScope" in check && + typeof check.routeScope === "string" && + check.routeScope.toLowerCase() === "link" && + "peerMac" in check && + check.peerMac === request.expectedPeerMac && + "peerNeighborState" in check && + typeof check.peerNeighborState === "string" && + /^(?:REACHABLE|STALE|DELAY|PROBE|PERMANENT|NOARP)(?:,(?:REACHABLE|STALE|DELAY|PROBE|PERMANENT|NOARP))*$/i.test( + check.peerNeighborState, + ) && + "jumboPing" in check && + check.jumboPing === true, + ), + ); +} + +export function buildRemoteHelperCommand(helperSha256: string, mode: StationPrepMode): string { + if (!/^[a-f0-9]{64}$/.test(helperSha256)) throw new Error("Helper SHA-256 is invalid"); + if (mode !== "--check" && mode !== "--apply" && mode !== "--verify") { + throw new Error("Helper mode is invalid"); + } + return [ + "set -eu", + "umask 077", + 'd=$(mktemp -d "${TMPDIR:-/tmp}/nemoclaw-station-prep.XXXXXX")', + "trap 'rm -rf -- \"$d\"' EXIT HUP INT TERM", + 'f="$d/prepare-dgx-station-host.sh"', + 'cat >"$f"', + `test "$(sha256sum "$f" | awk '{print $1}')" = "${helperSha256}"`, + 'chmod 0600 "$f"', + "sudo -n true", + `NEMOCLAW_STATION_PREP_SUDO_NONINTERACTIVE=1 bash "$f" ${mode}`, + ].join("; "); +} + +function assertSecureStateDirectory(directory: string): void { + let metadata: fs.Stats; + try { + metadata = fs.lstatSync(directory); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + throw new Error("Dual-Station resume directory must already exist"); + } + throw error; + } + const uid = process.getuid?.(); + if ( + !metadata.isDirectory() || + metadata.isSymbolicLink() || + uid === undefined || + metadata.uid !== uid || + (metadata.mode & 0o077) !== 0 + ) { + throw new Error("Dual-Station resume directory must be owner-only and symlink-free"); + } +} + +export function readDualStationResumeState(statePath: string): DualStationResumeState | null { + const directory = path.dirname(statePath); + assertSecureStateDirectory(directory); + const noFollow = fs.constants.O_NOFOLLOW; + if (typeof noFollow !== "number") throw new Error("O_NOFOLLOW is required for resume state"); + let fd: number; + try { + fd = fs.openSync(statePath, fs.constants.O_RDONLY | noFollow | fs.constants.O_NONBLOCK); + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + if (code === "ENOENT") return null; + if (code === "ELOOP") throw new Error("Dual-Station resume state must not be a symlink"); + throw error; + } + try { + const metadata = fs.fstatSync(fd); + const uid = process.getuid?.(); + if (uid === undefined) throw new Error("Current user identity is unavailable"); + validateResumeFileMetadata( + { + isFile: metadata.isFile(), + isSymbolicLink: metadata.isSymbolicLink(), + uid: metadata.uid, + mode: metadata.mode, + size: metadata.size, + }, + uid, + ); + const raw = fs.readFileSync(fd, "utf8"); + let value: unknown; + try { + value = JSON.parse(raw); + } catch { + throw new Error("Dual-Station resume state is malformed JSON"); + } + return parseDualStationResumeState(value); + } finally { + fs.closeSync(fd); + } +} + +export function writeDualStationResumeState( + statePath: string, + state: DualStationResumeState, +): void { + const validated = parseDualStationResumeState(state); + const directory = path.dirname(statePath); + assertSecureStateDirectory(directory); + const noFollow = fs.constants.O_NOFOLLOW; + if (typeof noFollow !== "number") throw new Error("O_NOFOLLOW is required for resume state"); + const temporary = `${statePath}.tmp.${randomBytes(12).toString("hex")}`; + let fd: number | null = null; + try { + fd = fs.openSync( + temporary, + fs.constants.O_WRONLY | fs.constants.O_CREAT | fs.constants.O_EXCL | noFollow, + 0o600, + ); + fs.writeFileSync(fd, `${JSON.stringify(validated)}\n`, "utf8"); + fs.fsyncSync(fd); + fs.closeSync(fd); + fd = null; + fs.renameSync(temporary, statePath); + const directoryFd = fs.openSync(directory, fs.constants.O_RDONLY); + try { + fs.fsyncSync(directoryFd); + } finally { + fs.closeSync(directoryFd); + } + } finally { + if (fd !== null) fs.closeSync(fd); + try { + fs.unlinkSync(temporary); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + } +} + +export function clearDualStationResumeState(statePath: string): void { + const current = readDualStationResumeState(statePath); + if (!current) return; + fs.unlinkSync(statePath); +} + +function parseCliOptions(args: readonly string[]): CliOptions { + const values = new Map(); + let reuseExistingManagedPair = false; + let clearState = false; + for (let index = 0; index < args.length; index += 1) { + const arg = args[index]; + if (arg === "--reuse-existing-managed-pair") { + reuseExistingManagedPair = true; + continue; + } + if (arg === "--clear-state") { + clearState = true; + continue; + } + if (!arg.startsWith("--") || index + 1 >= args.length) { + throw new Error(`Unexpected argument: ${arg}`); + } + if (values.has(arg)) throw new Error(`Duplicate argument: ${arg}`); + values.set(arg, args[(index += 1)]); + } + const helperPath = values.get("--helper") ?? ""; + const statePath = values.get("--state") ?? ""; + const revision = values.get("--revision") ?? ""; + if (!path.isAbsolute(statePath)) throw new Error("--state must be an absolute path"); + if (!clearState && !path.isAbsolute(helperPath)) { + throw new Error("--helper must be an absolute path"); + } + if (!clearState && !/^[a-f0-9]{40}$/.test(revision)) { + throw new Error("--revision must be an exact commit SHA"); + } + const explicitPeer = values.get("--explicit-peer"); + if (explicitPeer !== undefined) validateStationPeerTarget(explicitPeer); + const allowed = new Set(["--helper", "--state", "--revision", "--explicit-peer"]); + for (const key of values.keys()) { + if (!allowed.has(key)) throw new Error(`Unexpected argument: ${key}`); + } + return { + helperPath, + statePath, + revision, + explicitPeer, + reuseExistingManagedPair, + clearState, + }; +} + +function assertHelperFile(helperPath: string): Buffer { + const noFollow = fs.constants.O_NOFOLLOW; + if (typeof noFollow !== "number") throw new Error("O_NOFOLLOW is required for the helper"); + let fd: number; + try { + fd = fs.openSync(helperPath, fs.constants.O_RDONLY | noFollow | fs.constants.O_NONBLOCK); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ELOOP") { + throw new Error("Station host-preparation helper must not be a symlink"); + } + throw error; + } + try { + const metadata = fs.fstatSync(fd); + if (!metadata.isFile() || metadata.size <= 0 || metadata.size > 2 ** 20) { + throw new Error("Station host-preparation helper must be a bounded regular file"); + } + return fs.readFileSync(fd); + } finally { + fs.closeSync(fd); + } +} + +function sshArgs( + binding: PretrustedSshTarget, + pinnedKnownHostsPath: string, + remoteCommand: string, +): string[] { + return [ + ...strictStationPrepSshTransportArgs(), + "-o", + `UserKnownHostsFile=${pinnedKnownHostsPath}`, + "-o", + "GlobalKnownHostsFile=/dev/null", + "-o", + `HostKeyAlias=${binding.lookupHost}`, + "--", + binding.sshTarget, + remoteCommand, + ]; +} + +function createRuntimeDeps(options: CliOptions): { + deps: DualStationPreparationDeps; + helperSha256: string; + cleanup(): void; +} { + const helperBytes = assertHelperFile(options.helperPath); + const helperSha256 = createHash("sha256").update(helperBytes).digest("hex"); + const temporaryDirectory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-station-pair-")); + fs.chmodSync(temporaryDirectory, 0o700); + const pinnedHelperPath = path.join(temporaryDirectory, "prepare-dgx-station-host.sh"); + fs.writeFileSync(pinnedHelperPath, helperBytes, { flag: "wx", mode: 0o600 }); + const pinnedFiles = new Map(); + + const pinnedKnownHosts = (binding: PretrustedSshTarget): string => { + const cached = pinnedFiles.get(binding.hostKeyDigest); + if (cached) return cached; + const file = path.join(temporaryDirectory, `known-hosts-${binding.hostKeyDigest}`); + fs.writeFileSync(file, `${binding.knownHostsLines.join("\n")}\n`, { + encoding: "utf8", + flag: "wx", + mode: 0o600, + }); + pinnedFiles.set(binding.hostKeyDigest, file); + return file; + }; + + const runLocalHelper = (mode: "--check" | "--verify"): number => { + return runStreamingCommand("bash", [pinnedHelperPath, mode], ""); + }; + + const deps: DualStationPreparationDeps = { + runLocalHelper, + probeLocalHost: () => + parseHostResult( + runCommand("python3", ["-"], STATION_DISCOVERY_PROBE), + "Local Station identity probe", + ), + inspectPretrustedTarget: inspectPretrustedSshTarget, + probePeerHost: (binding) => + parseHostResult( + runCommand( + "ssh", + sshArgs(binding, pinnedKnownHosts(binding), "python3 -"), + STATION_DISCOVERY_PROBE, + ), + "Peer Station identity probe", + ), + probeLocalConnectivity: (requests) => { + const args = validateConnectivityArgs(requests); + return connectivityMatches( + runCommand("python3", ["-", ...args], CONNECTIVITY_PROBE), + requests, + ); + }, + probePeerConnectivity: (binding, requests) => { + const args = validateConnectivityArgs(requests); + return connectivityMatches( + runCommand( + "ssh", + sshArgs(binding, pinnedKnownHosts(binding), ["python3", "-", ...args].join(" ")), + CONNECTIVITY_PROBE, + ), + requests, + ); + }, + runRemoteHelper: (binding, mode) => { + return runStreamingCommand( + "ssh", + sshArgs(binding, pinnedKnownHosts(binding), buildRemoteHelperCommand(helperSha256, mode)), + helperBytes.toString("utf8"), + ); + }, + readResumeState: () => readDualStationResumeState(options.statePath), + writeResumeState: (state) => writeDualStationResumeState(options.statePath, state), + clearResumeState: () => clearDualStationResumeState(options.statePath), + log: (message) => process.stderr.write(`[station-pair] ${message}\n`), + }; + + return { + deps, + helperSha256, + cleanup: () => fs.rmSync(temporaryDirectory, { recursive: true, force: true }), + }; +} + +export function runCli(args: readonly string[]): number { + const options = parseCliOptions(args); + if (options.clearState) { + clearDualStationResumeState(options.statePath); + process.stdout.write(`${JSON.stringify({ kind: "cleared" })}\n`); + return 0; + } + const runtime = createRuntimeDeps(options); + try { + const result = prepareDualStationPair( + { + revision: options.revision, + helperSha256: runtime.helperSha256, + explicitPeer: options.explicitPeer, + reuseExistingManagedPair: options.reuseExistingManagedPair, + }, + runtime.deps, + ); + if (result.kind === "single-station") { + runtime.deps.log(`Using the existing single-Station path: ${result.reason}`); + } + process.stdout.write(`${JSON.stringify(result)}\n`); + return result.kind === "reboot-required" ? 10 : 0; + } finally { + runtime.cleanup(); + } +} + +function isMainModule(): boolean { + const invoked = process.argv[1]; + return Boolean(invoked && path.resolve(invoked) === path.resolve(fileURLToPath(import.meta.url))); +} + +if (isMainModule()) { + try { + process.exitCode = runCli(process.argv.slice(2)); + } catch (error) { + process.stderr.write(`[station-pair] ERROR: ${(error as Error).message}\n`); + process.exitCode = 1; + } +} diff --git a/test/install-express-prompt.test.ts b/test/install-express-prompt.test.ts index 74f3600ac3..4d87f2b531 100644 --- a/test/install-express-prompt.test.ts +++ b/test/install-express-prompt.test.ts @@ -244,22 +244,24 @@ detect_express_platform ); }); - it("uses the Nemotron Ultra recipe without follow-up choices on DGX Station", () => { + it("defers automatic Station model selection until reciprocal pair discovery", () => { const result = runExpressPromptWithTty("\n", "pipe", "DGX Station"); const output = `${result.stdout}${result.stderr}`; expect(result.status, output).toBe(0); expect(output).toMatch(/Detected DGX Station/); expect(output).toMatch( - /Express install will configure managed local vLLM with NVIDIA Nemotron 3 Ultra 550B/, + /Express install will configure managed local vLLM using automatic Station model selection/, + ); + expect(output).toMatch( + /pretrusted reciprocal dual-Station pair selects NVIDIA Nemotron 3 Ultra/, ); - expect(output).toMatch(/approximately 352 GB model/); expect(output).toMatch( /installs missing pinned driver, Docker, and NVIDIA Container Toolkit packages/, ); expect(output).toMatch(/DGX Station remains Deferred/); expect(output).toMatch(/Using express install for DGX Station/); expect(output).toMatch( - /RESULT NON_INTERACTIVE=1 SUDO_MODE=prompt PROVIDER=install-vllm MODEL=nvidia\/nemotron-3-ultra-550b-a55b VLLM_MODEL=nemotron-3-ultra-550b-a55b POLICY=suggested YES=1 SANDBOX=my-assistant/, + /RESULT NON_INTERACTIVE=1 SUDO_MODE=prompt PROVIDER=install-vllm MODEL= VLLM_MODEL= POLICY=suggested YES=1 SANDBOX=my-assistant/, ); }); @@ -514,10 +516,12 @@ printf 'NON_EXPRESS_ALLOWED\n' }); const output = `${result.stdout}${result.stderr}`; expect(result.status, output).toBe(0); - expect(output).toMatch(/managed local vLLM with NVIDIA Nemotron 3 Ultra 550B/); - expect(output).toMatch(/approximately 352 GB model/); + expect(output).toMatch(/managed local vLLM using automatic Station model selection/); expect(output).toMatch( - /RESULT NON_INTERACTIVE=1 SUDO_MODE=prompt PROVIDER=install-vllm MODEL=nvidia\/nemotron-3-ultra-550b-a55b VLLM_MODEL=nemotron-3-ultra-550b-a55b POLICY=suggested YES=1 SANDBOX=my-assistant/, + /pretrusted reciprocal dual-Station pair selects NVIDIA Nemotron 3 Ultra/, + ); + expect(output).toMatch( + /RESULT NON_INTERACTIVE=1 SUDO_MODE=prompt PROVIDER=install-vllm MODEL= VLLM_MODEL= POLICY=suggested YES=1 SANDBOX=my-assistant/, ); }); diff --git a/test/install-station-host-preparation.test.ts b/test/install-station-host-preparation.test.ts index f167858792..7c97b3bb9e 100644 --- a/test/install-station-host-preparation.test.ts +++ b/test/install-station-host-preparation.test.ts @@ -970,8 +970,10 @@ PAYLOAD cat > "$target/scripts/prepare-dgx-station-host.sh" <<'HELPER' #!/usr/bin/env bash set -euo pipefail -[ "\${1:-}" = "--apply" ] -printf 'PREPARE_STATION\\n' +case "\${1:-}" in + --check|--apply|--verify) printf 'PREPARE_STATION_%s\\n' "\${1#--}" ;; + *) exit 2 ;; +esac HELPER chmod +x "$target/scripts/install.sh" "$target/scripts/prepare-dgx-station-host.sh" exit 0 @@ -1003,8 +1005,14 @@ exit 0 expect(result.status, output).toBe(0); expect(output).toContain("DGX Station host prerequisites are ready"); - expect(output.indexOf("PREPARE_STATION")).toBeGreaterThanOrEqual(0); - expect(output.indexOf("PREPARE_STATION")).toBeLessThan(output.indexOf("ENSURE_DOCKER")); + expect(output.indexOf("PREPARE_STATION_check")).toBeGreaterThanOrEqual(0); + expect(output.indexOf("PREPARE_STATION_check")).toBeLessThan( + output.indexOf("PREPARE_STATION_apply"), + ); + expect(output.indexOf("PREPARE_STATION_apply")).toBeLessThan( + output.indexOf("PREPARE_STATION_verify"), + ); + expect(output.indexOf("PREPARE_STATION_verify")).toBeLessThan(output.indexOf("ENSURE_DOCKER")); expect(output.indexOf("ENSURE_DOCKER")).toBeLessThan(output.indexOf("ENSURE_BUILD_DEPS")); }); @@ -1064,12 +1072,36 @@ ensure_station_express_host expect(result.status, output).toBe(10); expect(fs.readFileSync(stateFile, "utf-8")).toBe( - `revision=${STATION_REVISION}\nmodel=nemotron-3-ultra-550b-a55b\n`, + `revision=${STATION_REVISION}\nmodel=nemotron-3-ultra-550b-a55b\nmode=express\n`, ); expect(fs.statSync(stateFile).mode & 0o777).toBe(0o600); expect(output).toContain(`NEMOCLAW_INSTALL_TAG=${STATION_REVISION}`); }); + it("persists provider-only origin and prints an exact provider resume command", () => { + const { home, result, output } = runSourced( + INSTALLER_PAYLOAD, + ` +_SELECTED_EXPRESS_PLATFORM='DGX Station' +_STATION_INSTALL_MODE='provider' +NEMOCLAW_PROVIDER='install-vllm' +unset NEMOCLAW_VLLM_MODEL +station_installer_revision() { printf '${STATION_REVISION}'; } +run_station_host_preparation() { return 10; } +ensure_station_express_host +`, + ); + const stateFile = path.join(home, ".nemoclaw", "station-express-resume"); + + expect(result.status, output).toBe(10); + expect(fs.readFileSync(stateFile, "utf-8")).toBe( + `revision=${STATION_REVISION}\nmodel=auto\nmode=provider\n`, + ); + expect(output).toContain( + `NEMOCLAW_PROVIDER=install-vllm NEMOCLAW_INSTALL_TAG=${STATION_REVISION} bash`, + ); + }); + it("rejects a resume-state symlink without loading its target", () => { const { home, result, output } = runSourced( INSTALLER_PAYLOAD, @@ -1120,7 +1152,7 @@ ensure_station_express_host fs.mkdirSync(stateDir, { mode: 0o700 }); fs.writeFileSync( path.join(stateDir, "station-express-resume"), - `revision=${STATION_REVISION}\nmodel=nemotron-3-ultra-550b-a55b\n`, + `revision=${STATION_REVISION}\nmodel=nemotron-3-ultra-550b-a55b\nmode=express\n`, { mode: 0o600 }, ); const result = spawnSync( @@ -1163,6 +1195,137 @@ printf 'RESULT PLATFORM=%s PROVIDER=%s MODEL=%s VLLM_MODEL=%s\n' \ ); }); + it("resumes provider-only Station setup without enabling express policy", () => { + const { result, output } = runSourced( + INSTALLER_PAYLOAD, + ` +mkdir -p "$HOME/.nemoclaw" +chmod 0700 "$HOME/.nemoclaw" +printf 'revision=${STATION_REVISION}\nmodel=auto\nmode=provider\n' >"$HOME/.nemoclaw/station-express-resume" +chmod 0600 "$HOME/.nemoclaw/station-express-resume" +detect_express_platform() { printf 'DGX Station'; } +station_installer_revision() { printf '${STATION_REVISION}'; } +NON_INTERACTIVE='' +NEMOCLAW_PROVIDER='' +NEMOCLAW_NO_EXPRESS='' +unset NEMOCLAW_POLICY_MODE NEMOCLAW_YES NEMOCLAW_SANDBOX_NAME +maybe_offer_express_install +printf 'RESULT MODE=%s PROVIDER=%s POLICY=%s YES=%s SANDBOX=%s\n' \ + "$_STATION_INSTALL_MODE" "$NEMOCLAW_PROVIDER" "\${NEMOCLAW_POLICY_MODE:-}" "\${NEMOCLAW_YES:-}" "\${NEMOCLAW_SANDBOX_NAME:-}" +`, + ); + + expect(result.status, output).toBe(0); + expect(output).toContain("Resuming the accepted managed-vLLM provider setup"); + expect(output).toContain("RESULT MODE=provider PROVIDER=install-vllm POLICY= YES= SANDBOX="); + expect(output).not.toContain("Resuming the accepted express install"); + }); + + it("preserves an explicit model supplied on a pre-discovery reboot rerun", () => { + const { result, output } = runSourced( + INSTALLER_PAYLOAD, + ` +mkdir -p "$HOME/.nemoclaw" +chmod 0700 "$HOME/.nemoclaw" +printf 'revision=${STATION_REVISION}\nmodel=nemotron-3-ultra-550b-a55b\nmode=express\n' >"$HOME/.nemoclaw/station-express-resume" +chmod 0600 "$HOME/.nemoclaw/station-express-resume" +station_installer_revision() { printf '${STATION_REVISION}'; } +NEMOCLAW_VLLM_MODEL='deepseek-v4-flash' +load_station_express_resume +printf 'RESULT MODEL=%s\n' "$NEMOCLAW_VLLM_MODEL" +`, + ); + + expect(result.status, output).toBe(0); + expect(output).toContain("RESULT MODEL=deepseek-v4-flash"); + }); + + it("fails closed when a model rerun would bypass pending pair revalidation", () => { + const { home, result, output } = runSourced( + INSTALLER_PAYLOAD, + ` +mkdir -p "$HOME/.nemoclaw" +chmod 0700 "$HOME/.nemoclaw" +printf 'revision=${STATION_REVISION}\nmodel=auto\nmode=express\n' >"$HOME/.nemoclaw/station-express-resume" +printf '{}\n' >"$HOME/.nemoclaw/station-dual-pair-resume.json" +chmod 0600 "$HOME/.nemoclaw/station-express-resume" "$HOME/.nemoclaw/station-dual-pair-resume.json" +station_installer_revision() { printf '${STATION_REVISION}'; } +NEMOCLAW_VLLM_MODEL='deepseek-v4-flash' +load_station_express_resume +`, + ); + + expect(result.status, output).not.toBe(0); + expect(output).toContain("refusing to replace its model"); + expect(fs.existsSync(path.join(home, ".nemoclaw", "station-dual-pair-resume.json"))).toBe(true); + }); + + it("fails closed when Station setup is disabled with a pending pair resume", () => { + const { home, result, output } = runSourced( + INSTALLER_PAYLOAD, + ` +mkdir -p "$HOME/.nemoclaw" +chmod 0700 "$HOME/.nemoclaw" +printf '{}\n' >"$HOME/.nemoclaw/station-dual-pair-resume.json" +chmod 0600 "$HOME/.nemoclaw/station-dual-pair-resume.json" +detect_express_platform() { printf 'DGX Station'; } +NEMOCLAW_NO_EXPRESS='1' +NEMOCLAW_PROVIDER='' +maybe_offer_express_install +`, + ); + + expect(result.status, output).not.toBe(0); + expect(output).toContain("finish exact pair revalidation before disabling Station setup"); + expect(fs.existsSync(path.join(home, ".nemoclaw", "station-dual-pair-resume.json"))).toBe(true); + }); + + it("forces pending pair recovery before a non-interactive skip path", () => { + const { result, output } = runSourced( + INSTALLER_PAYLOAD, + ` +mkdir -p "$HOME/.nemoclaw" +chmod 0700 "$HOME/.nemoclaw" +printf 'revision=${STATION_REVISION}\nmodel=auto\nmode=express\n' >"$HOME/.nemoclaw/station-express-resume" +printf '{}\n' >"$HOME/.nemoclaw/station-dual-pair-resume.json" +chmod 0600 "$HOME/.nemoclaw/station-express-resume" "$HOME/.nemoclaw/station-dual-pair-resume.json" +detect_express_platform() { printf 'DGX Station'; } +station_installer_revision() { printf '${STATION_REVISION}'; } +NON_INTERACTIVE='1' +NEMOCLAW_PROVIDER='' +NEMOCLAW_NO_EXPRESS='' +maybe_offer_express_install +printf 'RESULT PLATFORM=%s PROVIDER=%s MODE=%s\n' "$_SELECTED_EXPRESS_PLATFORM" "$NEMOCLAW_PROVIDER" "$_STATION_INSTALL_MODE" +`, + ); + + expect(result.status, output).toBe(0); + expect(output).toContain("Resuming the accepted express install"); + expect(output).toContain("RESULT PLATFORM=DGX Station PROVIDER=install-vllm MODE=express"); + expect(output).not.toContain("Skipping express prompt (--non-interactive set)"); + }); + + it("rejects pair-only crash state before prompts or fallback", () => { + const { result, output } = runSourced( + INSTALLER_PAYLOAD, + ` +mkdir -p "$HOME/.nemoclaw" +chmod 0700 "$HOME/.nemoclaw" +printf '{}\n' >"$HOME/.nemoclaw/station-dual-pair-resume.json" +chmod 0600 "$HOME/.nemoclaw/station-dual-pair-resume.json" +detect_express_platform() { printf 'DGX Station'; } +NON_INTERACTIVE='1' +NEMOCLAW_PROVIDER='' +NEMOCLAW_NO_EXPRESS='' +maybe_offer_express_install +`, + ); + + expect(result.status, output).not.toBe(0); + expect(output).toContain("pair state exists without its required installer resume state"); + expect(output).not.toContain("Skipping express prompt"); + }); + it("preserves an explicit provider even when Station resume state exists", () => { const { home, result, output } = runSourced( INSTALLER_PAYLOAD, @@ -1193,7 +1356,7 @@ printf 'RESULT PROVIDER=%s\n' "$NEMOCLAW_PROVIDER" ` mkdir -p "$HOME/.nemoclaw" chmod 0700 "$HOME/.nemoclaw" -printf 'revision=${STATION_REVISION}\nmodel=nemotron-3-ultra-550b-a55b\n' >"$HOME/.nemoclaw/station-express-resume" +printf 'revision=${STATION_REVISION}\nmodel=nemotron-3-ultra-550b-a55b\nmode=express\n' >"$HOME/.nemoclaw/station-express-resume" chmod 0600 "$HOME/.nemoclaw/station-express-resume" detect_express_platform() { printf 'DGX Station'; } NON_INTERACTIVE='' @@ -1239,7 +1402,7 @@ printf 'RESULT MODEL=%s\n' "$NEMOCLAW_VLLM_MODEL" fs.mkdirSync(stateDir, { mode: 0o700 }); fs.writeFileSync( path.join(stateDir, "station-express-resume"), - `revision=${STATION_REVISION}\nmodel=nemotron-3-ultra-550b-a55b\nunexpected\n`, + `revision=${STATION_REVISION}\nmodel=nemotron-3-ultra-550b-a55b\nmode=express\nunexpected\n`, { mode: 0o600 }, ); const result = spawnSync( @@ -1276,7 +1439,7 @@ printf 'RESULT MODEL=%s\n' "$NEMOCLAW_VLLM_MODEL" ` mkdir -p "$HOME/.nemoclaw" chmod 0700 "$HOME/.nemoclaw" -printf 'revision=${savedRevision}\nmodel=nemotron-3-ultra-550b-a55b\n' >"$HOME/.nemoclaw/station-express-resume" +printf 'revision=${savedRevision}\nmodel=nemotron-3-ultra-550b-a55b\nmode=express\n' >"$HOME/.nemoclaw/station-express-resume" chmod 0600 "$HOME/.nemoclaw/station-express-resume" station_installer_revision() { printf '${currentRevision}'; } load_station_express_resume diff --git a/test/install-station-pair-preparation.test.ts b/test/install-station-pair-preparation.test.ts new file mode 100644 index 0000000000..72524c0056 --- /dev/null +++ b/test/install-station-pair-preparation.test.ts @@ -0,0 +1,1106 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { + type DualStationPreparationDeps, + type DualStationResumeState, + deriveDiscoveryCandidates, + deriveSlash30Counterpart, + type PretrustedSshTarget, + parseDualStationResumeState, + prepareDualStationPair, + type StationDiscoveryHost, + validateResumeFileMetadata, + validateStationPeerTarget, +} from "../scripts/lib/dgx-station-peer.mts"; +import { + buildRemoteHelperCommand, + buildStationPrepSubprocessEnv, + clearDualStationResumeState, + inspectPretrustedSshTarget, + readDualStationResumeState, + strictStationPrepSshTransportArgs, + writeDualStationResumeState, +} from "../scripts/prepare-dual-dgx-station.mts"; +import { INSTALLER_PAYLOAD, TEST_SYSTEM_PATH } from "./helpers/installer-sourced-env"; + +const REPO_ROOT = path.resolve(import.meta.dirname, ".."); +const COORDINATOR = path.join(REPO_ROOT, "scripts", "prepare-dual-dgx-station.mts"); +const STATION_HELPER = path.join(REPO_ROOT, "scripts", "prepare-dgx-station-host.sh"); +const REVISION = "a".repeat(40); +const HELPER_SHA256 = "b".repeat(64); +const HOST_KEY_DIGEST = "c".repeat(64); +const HOST_KEY_FINGERPRINT = `SHA256:${"A".repeat(43)}`; + +function stationHost(side: "local" | "peer"): StationDiscoveryHost { + const local = side === "local"; + return { + schemaVersion: 1, + hostname: local ? "station-a" : "station-b", + productName: "NVIDIA DGX Station GB300", + architecture: "aarch64", + gpus: [ + { + index: 0, + name: "NVIDIA GB300", + uuid: local ? "GPU-LOCAL-0001" : "GPU-PEER-0002", + }, + ], + rails: [ + { + netdev: "enp1s0f0np0", + macAddress: local ? "02:00:00:00:00:01" : "02:00:00:00:00:02", + pciAddress: "0000:01:00.0", + pciName: "NVIDIA ConnectX-8 Ethernet Controller", + state: "4: ACTIVE", + linkLayer: "Ethernet", + speedMbps: 400_000, + mtu: 9000, + ipv4Addresses: [{ address: local ? "10.10.0.1" : "10.10.0.2", prefixLength: 30 }], + }, + { + netdev: "enp2s0f0np0", + macAddress: local ? "02:00:00:00:00:05" : "02:00:00:00:00:06", + pciAddress: "0000:02:00.0", + pciName: "NVIDIA ConnectX-8 Ethernet Controller", + state: "4: ACTIVE", + linkLayer: "Ethernet", + speedMbps: 400_000, + mtu: 9000, + ipv4Addresses: [{ address: local ? "10.10.0.5" : "10.10.0.6", prefixLength: 30 }], + }, + ], + }; +} + +function sshBinding(target = "10.10.0.2", hostKeyDigest = HOST_KEY_DIGEST): PretrustedSshTarget { + return { + requestedTarget: target, + sshTarget: target, + resolvedHost: target.slice(target.lastIndexOf("@") + 1), + sshUser: "ubuntu", + port: 22, + lookupHost: target.slice(target.lastIndexOf("@") + 1), + hostKeyDigest, + keyFingerprints: [HOST_KEY_FINGERPRINT], + knownHostsLines: [ + `${target.slice(target.lastIndexOf("@") + 1)} ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAITestKey`, + ], + }; +} + +function preparationOptions() { + return { revision: REVISION, helperSha256: HELPER_SHA256 }; +} + +class PreparationHarness { + readonly calls: string[] = []; + readonly statePhases: DualStationResumeState["phase"][] = []; + readonly trusted = new Map(); + readonly trustErrors = new Map(); + readonly localHelperStatus = new Map(); + readonly remoteHelperStatus = new Map(); + local = stationHost("local"); + peer = stationHost("peer"); + resume: DualStationResumeState | null = null; + localConnectivity = true; + peerConnectivity = true; + peerProbeError: Error | null = null; + + readonly deps: DualStationPreparationDeps = { + runLocalHelper: (mode) => { + this.calls.push(`local:${mode}`); + return this.localHelperStatus.get(mode) ?? 0; + }, + probeLocalHost: () => { + this.calls.push("probe:local"); + return structuredClone(this.local); + }, + inspectPretrustedTarget: (target) => { + this.calls.push(`trust:${target}`); + const error = this.trustErrors.get(target); + if (error) throw error; + return this.trusted.get(target) ?? null; + }, + probePeerHost: (binding) => { + this.calls.push(`probe:peer:${binding.sshTarget}`); + if (this.peerProbeError) throw this.peerProbeError; + return structuredClone(this.peer); + }, + probeLocalConnectivity: () => { + this.calls.push("connectivity:local"); + return this.localConnectivity; + }, + probePeerConnectivity: (binding) => { + this.calls.push(`connectivity:peer:${binding.sshTarget}`); + return this.peerConnectivity; + }, + runRemoteHelper: (binding, mode) => { + this.calls.push(`remote:${binding.sshTarget}:${mode}`); + return this.remoteHelperStatus.get(mode) ?? 0; + }, + readResumeState: () => { + this.calls.push("state:read"); + return this.resume ? structuredClone(this.resume) : null; + }, + writeResumeState: (state) => { + this.calls.push(`state:write:${state.phase}`); + this.resume = structuredClone(state); + this.statePhases.push(state.phase); + }, + clearResumeState: () => { + this.calls.push("state:clear"); + this.resume = null; + }, + log: (message) => this.calls.push(`log:${message}`), + }; +} + +function trustFirstRail(harness: PreparationHarness): void { + harness.trusted.set("10.10.0.2", sshBinding()); +} + +function readyState(): DualStationResumeState { + return { + schemaVersion: 1, + revision: REVISION, + helperSha256: HELPER_SHA256, + phase: "ready", + peerTarget: "10.10.0.2", + hostKeyDigest: HOST_KEY_DIGEST, + localGpuUuid: "GPU-LOCAL-0001", + peerGpuUuid: "GPU-PEER-0002", + rails: [ + { + localAddress: "10.10.0.1", + localMac: "02:00:00:00:00:01", + peerAddress: "10.10.0.2", + peerMac: "02:00:00:00:00:02", + }, + { + localAddress: "10.10.0.5", + localMac: "02:00:00:00:00:05", + peerAddress: "10.10.0.6", + peerMac: "02:00:00:00:00:06", + }, + ], + }; +} + +function runInstallerBody(body: string, extraEnv: Record = {}) { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-pair-installer-")); + const result = spawnSync( + "bash", + ["--noprofile", "--norc", "-c", `source "$INSTALLER_UNDER_TEST" >/dev/null\n${body}`], + { + cwd: REPO_ROOT, + encoding: "utf8", + env: { + HOME: home, + INSTALLER_UNDER_TEST: INSTALLER_PAYLOAD, + PATH: `${path.dirname(process.execPath)}:${TEST_SYSTEM_PATH}`, + ...extraEnv, + }, + timeout: 20_000, + killSignal: "SIGKILL", + }, + ); + return { home, result, output: `${result.stdout}${result.stderr}` }; +} + +function coordinatorResult(kind: "ready" | "reboot-required", peer = "10.10.0.2"): string { + const state = readyState(); + state.peerTarget = peer; + return JSON.stringify({ + kind, + peerTarget: peer, + identity: { + peerTarget: peer, + hostKeyDigest: state.hostKeyDigest, + localGpuUuid: state.localGpuUuid, + peerGpuUuid: state.peerGpuUuid, + rails: state.rails, + }, + }); +} + +describe("deterministic dual-DGX Station peer discovery", () => { + it.each([ + ["10.0.0.1", "10.0.0.2"], + ["10.0.0.2", "10.0.0.1"], + ["172.16.8.5", "172.16.8.6"], + ["172.16.8.6", "172.16.8.5"], + ["192.168.20.1", "192.168.20.2"], + ])("derives only the other usable /30 address: %s -> %s", (address, counterpart) => { + expect(deriveSlash30Counterpart(address)).toBe(counterpart); + }); + + it.each([ + ["10.0.0.0", 30], + ["10.0.0.3", 30], + ["10.0.0.1", 24], + ["8.8.8.1", 30], + ["not-an-ip", 30], + ])("refuses non-usable, non-/30, public, and malformed addresses", (address, prefix) => { + expect(deriveSlash30Counterpart(address, prefix)).toBeNull(); + }); + + it("derives exactly the two reciprocal CX-8 candidates", () => { + expect(deriveDiscoveryCandidates(stationHost("local"))).toEqual(["10.10.0.2", "10.10.0.6"]); + }); + + it("rejects extra rails, duplicate identities, and non-jumbo links", () => { + const extraRail = stationHost("local"); + extraRail.rails.push({ + ...structuredClone(extraRail.rails[1]), + netdev: "enp3s0f0np0", + pciAddress: "0000:03:00.0", + macAddress: "02:00:00:00:00:09", + ipv4Addresses: [{ address: "10.10.0.9", prefixLength: 30 }], + }); + expect(() => deriveDiscoveryCandidates(extraRail)).toThrow(/exactly two CX-8 rails/); + + const duplicate = stationHost("local"); + duplicate.rails[1].macAddress = duplicate.rails[0].macAddress; + expect(() => deriveDiscoveryCandidates(duplicate)).toThrow(/identity is ambiguous/); + + const nonJumbo = stationHost("local"); + nonJumbo.rails[0].mtu = 1500; + expect(() => deriveDiscoveryCandidates(nonJumbo)).toThrow(/MTU 9000/); + }); + + it("inspects only the two mathematically derived candidates and preserves single-Station behavior", () => { + const harness = new PreparationHarness(); + const result = prepareDualStationPair(preparationOptions(), harness.deps); + + expect(result).toEqual({ + kind: "single-station", + reason: "No derived dual-rail peer address has pre-existing SSH host-key trust", + }); + expect(harness.calls.filter((call) => call.startsWith("trust:"))).toEqual([ + "trust:10.10.0.2", + "trust:10.10.0.6", + ]); + expect(harness.calls.some((call) => call.startsWith("probe:peer"))).toBe(false); + expect(harness.calls.some((call) => call.startsWith("remote:"))).toBe(false); + }); + + it("accepts one pretrusted reciprocal peer and runs preparation in order", () => { + const harness = new PreparationHarness(); + trustFirstRail(harness); + + const result = prepareDualStationPair(preparationOptions(), harness.deps); + + expect(result.kind).toBe("ready"); + expect(result.kind === "ready" && result.peerTarget).toBe("10.10.0.2"); + expect(harness.statePhases).toEqual(["remote-preparation", "ready"]); + expect(harness.calls.indexOf("local:--verify")).toBeLessThan( + harness.calls.indexOf("probe:peer:10.10.0.2"), + ); + expect(harness.calls.indexOf("state:write:remote-preparation")).toBeLessThan( + harness.calls.indexOf("remote:10.10.0.2:--check"), + ); + expect(harness.calls.filter((call) => call.startsWith("remote:"))).toEqual([ + "remote:10.10.0.2:--check", + "remote:10.10.0.2:--apply", + "remote:10.10.0.2:--verify", + ]); + }); + + it("accepts two rail aliases only when their exact SSH identity is coherent", () => { + const coherent = new PreparationHarness(); + coherent.trusted.set("10.10.0.2", sshBinding("10.10.0.2")); + coherent.trusted.set("10.10.0.6", sshBinding("10.10.0.6")); + expect(prepareDualStationPair(preparationOptions(), coherent.deps).kind).toBe("ready"); + expect(coherent.calls.filter((call) => call.startsWith("probe:peer:"))).toHaveLength(1); + + const ambiguous = new PreparationHarness(); + ambiguous.trusted.set("10.10.0.2", sshBinding("10.10.0.2")); + ambiguous.trusted.set("10.10.0.6", sshBinding("10.10.0.6", "d".repeat(64))); + const result = prepareDualStationPair(preparationOptions(), ambiguous.deps); + expect(result).toMatchObject({ + kind: "single-station", + reason: expect.stringMatching(/different/), + }); + expect(ambiguous.calls.some((call) => call.startsWith("probe:peer:"))).toBe(false); + }); + + it("treats an unusable automatic trust entry as untrusted without contact", () => { + const harness = new PreparationHarness(); + harness.trustErrors.set("10.10.0.2", new Error("unsafe HostKeyAlias")); + + expect(prepareDualStationPair(preparationOptions(), harness.deps)).toMatchObject({ + kind: "single-station", + }); + expect(harness.calls).toContain( + "log:Ignoring derived peer 10.10.0.2: pre-existing SSH trust is unusable (unsafe HostKeyAlias)", + ); + expect(harness.calls.some((call) => call.startsWith("probe:peer:"))).toBe(false); + }); + + it("requires reciprocal rail addresses and MACs plus a distinct peer GPU", () => { + const nonreciprocal = new PreparationHarness(); + trustFirstRail(nonreciprocal); + nonreciprocal.peer.rails[1].ipv4Addresses = [{ address: "10.10.0.10", prefixLength: 30 }]; + expect(prepareDualStationPair(preparationOptions(), nonreciprocal.deps)).toMatchObject({ + kind: "single-station", + reason: expect.stringMatching(/not reciprocal/), + }); + expect(nonreciprocal.calls.some((call) => call.startsWith("remote:"))).toBe(false); + + const sameGpu = new PreparationHarness(); + trustFirstRail(sameGpu); + sameGpu.peer.gpus[0].uuid = sameGpu.local.gpus[0].uuid; + expect(prepareDualStationPair(preparationOptions(), sameGpu.deps)).toMatchObject({ + kind: "single-station", + reason: expect.stringMatching(/local Station GPU/), + }); + expect(sameGpu.calls.some((call) => call.startsWith("remote:"))).toBe(false); + }); + + it("keeps an explicit peer authoritative and fail-closed", () => { + const explicit = "ubuntu@station-b"; + const harness = new PreparationHarness(); + harness.trusted.set(explicit, sshBinding(explicit)); + + expect( + prepareDualStationPair({ ...preparationOptions(), explicitPeer: explicit }, harness.deps) + .kind, + ).toBe("ready"); + expect(harness.calls.filter((call) => call.startsWith("trust:"))).toEqual([ + `trust:${explicit}`, + ]); + + const untrusted = new PreparationHarness(); + expect(() => + prepareDualStationPair({ ...preparationOptions(), explicitPeer: explicit }, untrusted.deps), + ).toThrow(/not pretrusted/); + expect(untrusted.calls.some((call) => call.startsWith("probe:peer:"))).toBe(false); + }); + + it.each([ + "root@station;reboot", + "station-b -o ProxyCommand=evil", + "station-b/path", + "user@@station-b", + "[10.10.0.2]", + "$(touch pwned)", + "station-b:2222", + "010.010.000.002", + ])("rejects a malicious or noncanonical peer string: %s", (target) => { + expect(() => validateStationPeerTarget(target)).toThrow(/canonical SSH host/); + }); + + it("fails local verification before trust inspection or remote mutation", () => { + const harness = new PreparationHarness(); + trustFirstRail(harness); + harness.localHelperStatus.set("--verify", 1); + + expect(() => prepareDualStationPair(preparationOptions(), harness.deps)).toThrow( + /verification failed before peer contact/, + ); + expect(harness.calls.some((call) => call.startsWith("trust:"))).toBe(false); + expect(harness.calls.some((call) => call.startsWith("remote:"))).toBe(false); + }); + + it("falls back only before mutation and fails closed for explicit connectivity failure", () => { + const automatic = new PreparationHarness(); + trustFirstRail(automatic); + automatic.localConnectivity = false; + expect(prepareDualStationPair(preparationOptions(), automatic.deps)).toMatchObject({ + kind: "single-station", + reason: expect.stringMatching(/jumbo-frame/), + }); + expect(automatic.statePhases).toEqual([]); + + const explicitTarget = "ubuntu@station-b"; + const explicit = new PreparationHarness(); + explicit.trusted.set(explicitTarget, sshBinding(explicitTarget)); + explicit.peerConnectivity = false; + expect(() => + prepareDualStationPair( + { ...preparationOptions(), explicitPeer: explicitTarget }, + explicit.deps, + ), + ).toThrow(/jumbo-frame/); + }); +}); + +describe("dual-DGX Station reboot resume and reuse", () => { + it("persists the exact pair before remote mutation and resumes remote exit 10", () => { + const first = new PreparationHarness(); + trustFirstRail(first); + first.remoteHelperStatus.set("--apply", 10); + + const interrupted = prepareDualStationPair(preparationOptions(), first.deps); + expect(interrupted.kind).toBe("reboot-required"); + expect(first.resume?.phase).toBe("remote-reboot-required"); + expect(first.resume?.helperSha256).toBe(HELPER_SHA256); + expect(first.calls.some((call) => call.endsWith(":--verify"))).toBe(true); + expect(first.calls).not.toContain("remote:10.10.0.2:--verify"); + + const resumed = new PreparationHarness(); + resumed.resume = structuredClone(first.resume); + trustFirstRail(resumed); + expect(prepareDualStationPair(preparationOptions(), resumed.deps).kind).toBe("ready"); + expect(resumed.statePhases).toEqual(["remote-preparation", "ready"]); + expect(resumed.calls.filter((call) => call.startsWith("remote:"))).toEqual([ + "remote:10.10.0.2:--check", + "remote:10.10.0.2:--apply", + "remote:10.10.0.2:--verify", + ]); + }); + + it("rejects revision, helper, host-key, GPU, and rail substitution on resume", () => { + const scenarios: Array<{ + name: string; + configure(harness: PreparationHarness): void; + options?: ReturnType; + expected: RegExp; + }> = [ + { + name: "revision", + configure: () => undefined, + options: { revision: "d".repeat(40), helperSha256: HELPER_SHA256 }, + expected: /requires NemoClaw revision/, + }, + { + name: "helper", + configure: () => undefined, + options: { revision: REVISION, helperSha256: "d".repeat(64) }, + expected: /helper changed/, + }, + { + name: "host key", + configure: (harness) => { + harness.trusted.set("10.10.0.2", sshBinding("10.10.0.2", "d".repeat(64))); + }, + expected: /host-key identity changed/, + }, + { + name: "GPU", + configure: (harness) => { + harness.peer.gpus[0].uuid = "GPU-SUBSTITUTED-0003"; + }, + expected: /physical dual-Station pair changed/, + }, + { + name: "rail", + configure: (harness) => { + harness.peer.rails[0].macAddress = "02:00:00:00:00:12"; + }, + expected: /physical dual-Station pair changed/, + }, + ]; + + for (const scenario of scenarios) { + const harness = new PreparationHarness(); + harness.resume = readyState(); + trustFirstRail(harness); + scenario.configure(harness); + expect( + () => prepareDualStationPair(scenario.options ?? preparationOptions(), harness.deps), + scenario.name, + ).toThrow(scenario.expected); + expect( + harness.calls.some((call) => call.startsWith("remote:")), + scenario.name, + ).toBe(false); + } + }); + + it("preserves a remote mismatch as a pinned fail-closed state", () => { + const harness = new PreparationHarness(); + trustFirstRail(harness); + harness.remoteHelperStatus.set("--apply", 1); + + expect(() => prepareDualStationPair(preparationOptions(), harness.deps)).toThrow( + /refusing single-Station fallback/, + ); + expect(harness.resume?.phase).toBe("remote-preparation"); + expect(harness.calls).not.toContain("remote:10.10.0.2:--verify"); + }); + + it("revalidates but does not rerun either helper for an exact managed pair", () => { + const harness = new PreparationHarness(); + trustFirstRail(harness); + + const result = prepareDualStationPair( + { ...preparationOptions(), reuseExistingManagedPair: true }, + harness.deps, + ); + expect(result.kind).toBe("ready"); + expect(harness.calls.some((call) => call.startsWith("local:"))).toBe(false); + expect(harness.calls.some((call) => call.startsWith("remote:"))).toBe(false); + expect(harness.calls).toContain("connectivity:local"); + expect(harness.calls).toContain("connectivity:peer:10.10.0.2"); + expect(harness.resume?.phase).toBe("ready"); + }); +}); + +describe.sequential("dual-DGX Station trust and resume-state boundaries", () => { + it("validates owner-only regular-file metadata", () => { + expect(() => + validateResumeFileMetadata( + { isFile: true, isSymbolicLink: false, uid: 1000, mode: 0o600, size: 100 }, + 1000, + ), + ).not.toThrow(); + expect(() => + validateResumeFileMetadata( + { isFile: true, isSymbolicLink: false, uid: 1001, mode: 0o600, size: 100 }, + 1000, + ), + ).toThrow(/not owned/); + expect(() => + validateResumeFileMetadata( + { isFile: true, isSymbolicLink: false, uid: 1000, mode: 0o644, size: 100 }, + 1000, + ), + ).toThrow(/0600/); + expect(() => + validateResumeFileMetadata( + { isFile: false, isSymbolicLink: true, uid: 1000, mode: 0o600, size: 100 }, + 1000, + ), + ).toThrow(/symlink/); + }); + + it("writes, fsyncs, reads, and clears canonical owner-only state", () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-pair-state-")); + fs.chmodSync(directory, 0o700); + const statePath = path.join(directory, "resume.json"); + try { + const state = readyState(); + state.rails.reverse(); + writeDualStationResumeState(statePath, state); + expect(fs.statSync(statePath).mode & 0o777).toBe(0o600); + const loaded = readDualStationResumeState(statePath); + expect(loaded?.rails.map((rail) => rail.localAddress)).toEqual(["10.10.0.1", "10.10.0.5"]); + clearDualStationResumeState(statePath); + expect(fs.existsSync(statePath)).toBe(false); + } finally { + fs.rmSync(directory, { recursive: true, force: true }); + } + }); + + it("rejects malformed, permissive, symlinked, and substitution-prone state", () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-pair-state-")); + fs.chmodSync(directory, 0o700); + const statePath = path.join(directory, "resume.json"); + try { + fs.writeFileSync(statePath, "not-json\n", { mode: 0o600 }); + expect(() => readDualStationResumeState(statePath)).toThrow(/malformed JSON/); + + fs.writeFileSync(statePath, `${JSON.stringify(readyState())}\n`, { mode: 0o600 }); + fs.chmodSync(statePath, 0o644); + expect(() => readDualStationResumeState(statePath)).toThrow(/0600/); + + fs.rmSync(statePath); + const target = path.join(directory, "target.json"); + fs.writeFileSync(target, `${JSON.stringify(readyState())}\n`, { mode: 0o600 }); + fs.symlinkSync(target, statePath); + expect(() => readDualStationResumeState(statePath)).toThrow(/symlink/); + + const changed = readyState(); + changed.peerGpuUuid = changed.localGpuUuid; + expect(() => parseDualStationResumeState(changed)).toThrow(/GPU identity/); + } finally { + fs.rmSync(directory, { recursive: true, force: true }); + } + }); + + it("requires an existing owner-only resume directory", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-pair-state-")); + const missingState = path.join(root, "missing", "resume.json"); + try { + expect(() => readDualStationResumeState(missingState)).toThrow(/must already exist/); + fs.chmodSync(root, 0o755); + expect(() => readDualStationResumeState(path.join(root, "resume.json"))).toThrow( + /owner-only/, + ); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } + }); + + it("uses a strict noninteractive SSH argument boundary and exact-byte helper command", () => { + const args = strictStationPrepSshTransportArgs(); + expect(args).toContain("BatchMode=yes"); + expect(args).toContain("StrictHostKeyChecking=yes"); + expect(args).toContain("VerifyHostKeyDNS=no"); + expect(args).toContain("NoHostAuthenticationForLocalhost=no"); + expect(args).toContain("ClearAllForwardings=yes"); + expect(args).toContain("ProxyCommand=none"); + expect(args).toContain("ProxyJump=none"); + const command = buildRemoteHelperCommand(HELPER_SHA256, "--apply"); + expect(command).toContain(HELPER_SHA256); + expect(command).toContain("sudo -n true"); + expect(command).toContain("NEMOCLAW_STATION_PREP_SUDO_NONINTERACTIVE=1"); + expect(command).toContain('bash "$f" --apply'); + }); + + it("does not expose ambient credentials or shell-loader variables to probes and helpers", () => { + const env = buildStationPrepSubprocessEnv({ + HOME: "/home/operator", + PATH: "/usr/bin:/bin", + SSH_AUTH_SOCK: "/run/user/1000/agent", + HTTPS_PROXY: "http://proxy.example:8080", + LC_CTYPE: "en_US.UTF-8", + NVIDIA_API_KEY: "secret", + HF_TOKEN: "secret", + BASH_ENV: "/tmp/evil", + ENV: "/tmp/evil", + LD_PRELOAD: "/tmp/evil.so", + SSH_ASKPASS: "/tmp/evil", + }); + expect(env).toMatchObject({ + HOME: "/home/operator", + PATH: "/usr/bin:/bin", + SSH_AUTH_SOCK: "/run/user/1000/agent", + HTTPS_PROXY: "http://proxy.example:8080", + LC_ALL: "C", + LC_CTYPE: "en_US.UTF-8", + LANG: "C", + }); + for (const forbidden of [ + "NVIDIA_API_KEY", + "HF_TOKEN", + "BASH_ENV", + "ENV", + "LD_PRELOAD", + "SSH_ASKPASS", + ]) { + expect(env).not.toHaveProperty(forbidden); + } + }); + + it("forces every remote helper sudo call through noninteractive mode", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-pair-sudo-")); + const fakeSudo = path.join(root, "sudo"); + fs.writeFileSync(fakeSudo, "#!/usr/bin/env bash\nprintf 'SUDO_ARGS=%s\\n' \"$*\"\n", { + mode: 0o700, + }); + try { + const strict = spawnSync( + "bash", + [ + "--noprofile", + "--norc", + "-c", + 'source "$STATION_HELPER" >/dev/null; NEMOCLAW_STATION_PREP_SUDO_NONINTERACTIVE=1; sudo true', + ], + { + encoding: "utf8", + env: { HOME: root, PATH: `${root}:${TEST_SYSTEM_PATH}`, STATION_HELPER }, + }, + ); + const local = spawnSync( + "bash", + ["--noprofile", "--norc", "-c", 'source "$STATION_HELPER" >/dev/null; sudo true'], + { + encoding: "utf8", + env: { HOME: root, PATH: `${root}:${TEST_SYSTEM_PATH}`, STATION_HELPER }, + }, + ); + expect(strict.status, strict.stderr).toBe(0); + expect(strict.stdout).toContain("SUDO_ARGS=-n true"); + expect(local.status, local.stderr).toBe(0); + expect(local.stdout).toContain("SUDO_ARGS=true"); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } + }); + + it("preserves matching revoked host keys in the pinned trust evidence", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-pair-trust-")); + const bin = path.join(root, "bin"); + const knownHosts = path.join(root, "known_hosts"); + fs.mkdirSync(bin, { mode: 0o700 }); + fs.writeFileSync(knownHosts, "fixture\n", { mode: 0o600 }); + const ssh = path.join(bin, "ssh"); + const keygen = path.join(bin, "ssh-keygen"); + fs.writeFileSync( + ssh, + `#!/usr/bin/env bash +cat <<'EOF' +user ubuntu +hostname 10.10.0.2 +port 22 +batchmode yes +stricthostkeychecking true +verifyhostkeydns false +nohostauthenticationforlocalhost no +permitlocalcommand no +forwardagent no +forwardx11 no +forwardx11trusted no +tunnel false +updatehostkeys false +controlmaster false +controlpath none +remotecommand none +proxycommand none +proxyjump none +localcommand none +knownhostscommand none +userknownhostsfile ${knownHosts} +globalknownhostsfile none +sendenv LANG +sendenv LC_* +EOF +`, + { mode: 0o700 }, + ); + fs.writeFileSync( + keygen, + `#!/usr/bin/env bash +if [[ " $* " == *" -F "* ]]; then + printf '%s\n' '@revoked 10.10.0.2 ssh-ed25519 AAAAC3NzaRevoked' + printf '%s\n' '10.10.0.2 ssh-ed25519 AAAAC3NzaTrusted' +else + printf '%s\n' '256 ${HOST_KEY_FINGERPRINT} fixture (ED25519)' +fi +`, + { mode: 0o700 }, + ); + const originalPath = process.env.PATH; + process.env.PATH = `${bin}:${originalPath ?? ""}`; + try { + const binding = inspectPretrustedSshTarget("10.10.0.2"); + expect(binding?.knownHostsLines).toContain("@revoked 10.10.0.2 ssh-ed25519 AAAAC3NzaRevoked"); + expect(binding?.knownHostsLines).toContain("10.10.0.2 ssh-ed25519 AAAAC3NzaTrusted"); + fs.chmodSync(knownHosts, 0o666); + expect(inspectPretrustedSshTarget("10.10.0.2")).toBeNull(); + } finally { + process.env.PATH = originalPath; + fs.rmSync(root, { recursive: true, force: true }); + } + }); + + it("contains no trust enrollment or general network-discovery mechanism", () => { + const source = fs.readFileSync(COORDINATOR, "utf8").toLowerCase(); + for (const forbidden of [ + "ssh-keyscan", + "arp-scan", + "avahi", + "dns-sd", + "lldp", + "nmap", + "mdns", + ]) { + expect(source, forbidden).not.toContain(forbidden); + } + }); +}); + +describe("dual-DGX Station installer handoff", () => { + it("selects Ultra only after the coordinator returns a validated ready pair", () => { + const argsFile = path.join(os.tmpdir(), `nemoclaw-pair-args-${process.pid}-${Date.now()}`); + const { result, output, home } = runInstallerBody( + ` +node() { + if [[ "\${1:-}" == "--no-warnings" ]]; then + printf '%s\n' "$PAIR_RESULT" + printf '%s\n' "$*" >"$PAIR_ARGS_FILE" + return 0 + fi + command node "$@" +} +station_installer_revision() { printf '%s' "$PAIR_REVISION"; } +station_dual_pair_resume_pending() { return 0; } +_SELECTED_EXPRESS_PLATFORM='DGX Station' +_STATION_EXPRESS_MODEL_WAS_EXPLICIT=0 +unset NEMOCLAW_VLLM_MODEL NEMOCLAW_MODEL NEMOCLAW_DGX_STATION_PEER +ensure_station_express_pair +printf 'RESULT peer=%s model=%s selector=%s\n' "\${NEMOCLAW_DGX_STATION_PEER:-}" "\${NEMOCLAW_MODEL:-}" "\${NEMOCLAW_VLLM_MODEL:-}" +`, + { + PAIR_ARGS_FILE: argsFile, + PAIR_RESULT: coordinatorResult("ready"), + PAIR_REVISION: REVISION, + }, + ); + try { + expect(result.status, output).toBe(0); + expect(output).toContain( + "RESULT peer=10.10.0.2 model=nvidia/nemotron-3-ultra-550b-a55b selector=", + ); + const args = fs.readFileSync(argsFile, "utf8"); + expect(args).toContain("--helper"); + expect(args).toContain("prepare-dgx-station-host.sh"); + expect(args).toContain(`--revision ${REVISION}`); + expect(args).not.toContain("--explicit-peer"); + } finally { + fs.rmSync(argsFile, { force: true }); + fs.rmSync(home, { recursive: true, force: true }); + } + }); + + it("passes explicit peer and exact managed-pair reuse without shell interpolation", () => { + const argsFile = path.join(os.tmpdir(), `nemoclaw-pair-args-${process.pid}-${Date.now()}`); + const explicitPeer = "ubuntu@station-b"; + const { result, output, home } = runInstallerBody( + ` +node() { + if [[ "\${1:-}" == "--no-warnings" ]]; then + printf '%s\n' "$PAIR_RESULT" + printf '%s\n' "$*" >"$PAIR_ARGS_FILE" + return 0 + fi + command node "$@" +} +station_installer_revision() { printf '%s' "$PAIR_REVISION"; } +station_dual_pair_resume_pending() { return 0; } +_SELECTED_EXPRESS_PLATFORM='DGX Station' +_STATION_EXPRESS_MODEL_WAS_EXPLICIT=0 +_STATION_EXPRESS_DEFERRED_MANAGED_PAIR=1 +NEMOCLAW_DGX_STATION_PEER="$PAIR_PEER" +unset NEMOCLAW_VLLM_MODEL NEMOCLAW_MODEL +ensure_station_express_pair +printf 'RESULT peer=%s model=%s\n' "$NEMOCLAW_DGX_STATION_PEER" "$NEMOCLAW_MODEL" +`, + { + PAIR_ARGS_FILE: argsFile, + PAIR_PEER: explicitPeer, + PAIR_RESULT: coordinatorResult("ready", explicitPeer), + PAIR_REVISION: REVISION, + }, + ); + try { + expect(result.status, output).toBe(0); + expect(output).toContain( + `RESULT peer=${explicitPeer} model=nvidia/nemotron-3-ultra-550b-a55b`, + ); + const args = fs.readFileSync(argsFile, "utf8"); + expect(args).toContain(`--explicit-peer ${explicitPeer}`); + expect(args).toContain("--reuse-existing-managed-pair"); + } finally { + fs.rmSync(argsFile, { force: true }); + fs.rmSync(home, { recursive: true, force: true }); + } + }); + + it("preserves the existing single-Station default when discovery finds no peer", () => { + const { result, output, home } = runInstallerBody( + ` +node() { + if [[ "\${1:-}" == "--no-warnings" ]]; then + printf '%s\n' '{"kind":"single-station","reason":"no pretrusted reciprocal peer"}' + return 0 + fi + command node "$@" +} +station_installer_revision() { printf '%s' "$PAIR_REVISION"; } +_SELECTED_EXPRESS_PLATFORM='DGX Station' +_STATION_EXPRESS_MODEL_WAS_EXPLICIT=0 +unset NEMOCLAW_VLLM_MODEL NEMOCLAW_MODEL NEMOCLAW_DGX_STATION_PEER +ensure_station_express_pair +printf 'RESULT peer=%s model=%s selector=%s\n' "\${NEMOCLAW_DGX_STATION_PEER:-}" "\${NEMOCLAW_MODEL:-}" "\${NEMOCLAW_VLLM_MODEL:-}" +`, + { PAIR_REVISION: REVISION }, + ); + try { + expect(result.status, output).toBe(0); + expect(output).toContain("No trusted reciprocal dual-DGX Station pair was detected"); + expect(output).toContain("RESULT peer= model= selector="); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + }); + + it("rejects an explicit peer combined with an explicit non-dual model", () => { + const { result, output, home } = runInstallerBody( + ` +node() { printf 'COORDINATOR_CALLED\n'; return 0; } +_SELECTED_EXPRESS_PLATFORM='DGX Station' +_STATION_EXPRESS_MODEL_WAS_EXPLICIT=1 +NEMOCLAW_VLLM_MODEL='deepseek-v4-flash' +NEMOCLAW_DGX_STATION_PEER='ubuntu@station-b' +ensure_station_express_pair +`, + ); + try { + expect(result.status, output).not.toBe(0); + expect(output).toContain( + "NEMOCLAW_DGX_STATION_PEER requires the DGX Station dual-serving model", + ); + expect(output).not.toContain("COORDINATOR_CALLED"); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + }); + + it("rejects conflicting peer and model selections before local host preparation", () => { + const { result, output, home } = runInstallerBody( + ` +maybe_offer_express_install() { + _SELECTED_EXPRESS_PLATFORM='DGX Station' + _STATION_EXPRESS_MODEL_WAS_EXPLICIT=1 + NEMOCLAW_VLLM_MODEL='deepseek-v4-flash' + NEMOCLAW_DGX_STATION_PEER='ubuntu@station-b' +} +ensure_station_express_host() { printf 'LOCAL_HELPER_CALLED\n'; } +ensure_docker() { printf 'DOCKER_CALLED\n'; } +ensure_openshell_build_deps() { printf 'BUILD_DEPS_CALLED\n'; } +prepare_installer_host +`, + ); + try { + expect(result.status, output).not.toBe(0); + expect(output).toContain( + "NEMOCLAW_DGX_STATION_PEER requires the DGX Station dual-serving model", + ); + expect(output).not.toContain("LOCAL_HELPER_CALLED"); + expect(output).not.toContain("DOCKER_CALLED"); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + }); + + it("propagates peer exit 10 with manual reboot and exact-revision rerun guidance", () => { + const { result, output, home } = runInstallerBody( + ` +node() { + if [[ "\${1:-}" == "--no-warnings" ]]; then + printf '%s\n' "$PAIR_RESULT" + return 10 + fi + command node "$@" +} +station_installer_revision() { printf '%s' "$PAIR_REVISION"; } +save_station_express_resume() { _STATION_EXPRESS_RESUME_REVISION="$PAIR_REVISION"; printf 'SAVED_EXPRESS_RESUME\n'; } +station_dual_pair_resume_pending() { return 0; } +_SELECTED_EXPRESS_PLATFORM='DGX Station' +_STATION_EXPRESS_MODEL_WAS_EXPLICIT=0 +unset NEMOCLAW_VLLM_MODEL NEMOCLAW_MODEL NEMOCLAW_DGX_STATION_PEER +ensure_station_express_pair +`, + { PAIR_RESULT: coordinatorResult("reboot-required"), PAIR_REVISION: REVISION }, + ); + try { + expect(result.status, output).toBe(10); + expect(output).toContain("SAVED_EXPRESS_RESUME"); + expect(output).toContain("requires a manual reboot"); + expect(output).toContain(`NEMOCLAW_INSTALL_TAG=${REVISION}`); + expect(output).not.toMatch(/reboot.*-[a-z]*f/i); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + }); + + it("preserves companion resume state when coordinator failure follows pair-state publication", () => { + const { result, output, home } = runInstallerBody( + ` +node() { + if [[ "\${1:-}" == "--no-warnings" ]]; then + while (( $# > 0 )); do + if [[ "$1" == "--state" ]]; then + printf '{}\n' >"$2" + break + fi + shift + done + return 1 + fi + command node "$@" +} +station_installer_revision() { printf '%s' "$PAIR_REVISION"; } +_SELECTED_EXPRESS_PLATFORM='DGX Station' +_STATION_EXPRESS_MODEL_WAS_EXPLICIT=0 +_STATION_INSTALL_MODE='express' +unset NEMOCLAW_VLLM_MODEL NEMOCLAW_MODEL NEMOCLAW_DGX_STATION_PEER +ensure_station_express_pair +`, + { PAIR_REVISION: REVISION }, + ); + try { + expect(result.status, output).not.toBe(0); + expect(output).toContain("Dual DGX Station preparation failed"); + expect(fs.existsSync(path.join(home, ".nemoclaw", "station-dual-pair-resume.json"))).toBe( + true, + ); + expect(fs.readFileSync(path.join(home, ".nemoclaw", "station-express-resume"), "utf8")).toBe( + `revision=${REVISION}\nmodel=auto\nmode=express\n`, + ); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + }); + + it("defers host preparation only for a complete running managed dual-head candidate", () => { + const valid = [ + "/nemoclaw-vllm", + "true", + "true", + "head", + "1", + "c".repeat(64), + "d".repeat(64), + "e".repeat(64), + "f".repeat(32), + ].join(" "); + const accepted = runInstallerBody( + ` +command_exists() { return 0; } +docker() { printf '%s\n' "$DOCKER_INSPECTION"; } +station_managed_dual_head_running +`, + { DOCKER_INSPECTION: valid }, + ); + const malformed = runInstallerBody( + ` +command_exists() { return 0; } +docker() { printf '%s\n' "$DOCKER_INSPECTION"; } +station_managed_dual_head_running +`, + { DOCKER_INSPECTION: valid.replace(" head ", " worker ") }, + ); + try { + expect(accepted.result.status, accepted.output).toBe(0); + expect(malformed.result.status, malformed.output).not.toBe(0); + } finally { + fs.rmSync(accepted.home, { recursive: true, force: true }); + fs.rmSync(malformed.home, { recursive: true, force: true }); + } + }); + + it("applies Station preparation to an explicitly selected managed-vLLM provider", () => { + const { result, output, home } = runInstallerBody( + ` +detect_express_platform() { printf 'DGX Station'; } +NON_INTERACTIVE='' +NEMOCLAW_NO_EXPRESS='' +NEMOCLAW_PROVIDER='install-vllm' +unset NEMOCLAW_VLLM_MODEL +maybe_offer_express_install +printf 'RESULT selected=%s provider=%s selector=%s\n' "$_SELECTED_EXPRESS_PLATFORM" "$NEMOCLAW_PROVIDER" "\${NEMOCLAW_VLLM_MODEL:-}" +`, + ); + try { + expect(result.status, output).toBe(0); + expect(output).toContain("explicitly selected managed-vLLM provider"); + expect(output).toContain("RESULT selected=DGX Station provider=install-vllm selector="); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + }); + + it("orders pair qualification before CLI/onboarding and clears state only after success", () => { + const source = fs.readFileSync(INSTALLER_PAYLOAD, "utf8"); + const main = source.slice(source.indexOf("main() {"), source.indexOf("finalize_install() {")); + expect(main.indexOf("ensure_station_express_pair")).toBeGreaterThanOrEqual(0); + expect(main.indexOf("ensure_station_express_pair")).toBeLessThan( + main.indexOf("install_nemoclaw"), + ); + expect(main.indexOf("run_onboard")).toBeLessThan(main.indexOf("finalize_install")); + expect(main.indexOf("finalize_install")).toBeLessThan( + main.indexOf("clear_station_dual_pair_resume"), + ); + }); +}); diff --git a/test/starter-prompt-docs.test.ts b/test/starter-prompt-docs.test.ts index ada41527ce..4b02d1cc5f 100644 --- a/test/starter-prompt-docs.test.ts +++ b/test/starter-prompt-docs.test.ts @@ -609,7 +609,10 @@ describe("starter prompt docs CTA", () => { "Windows WSL Express: `NEMOCLAW_PROVIDER=install-windows-ollama`", ); expect(promptSource).toContain( - "NEMOCLAW_VLLM_MODEL=nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4", + "leave `NEMOCLAW_VLLM_MODEL` unset for DGX Spark Express and for automatic DGX Station trusted-pair selection", + ); + expect(promptSource).toContain( + "At least one derived address must already be trusted; if both are trusted, their host keys must identify one coherent SSH host.", ); expect(promptSource).toContain( "Leave `NEMOCLAW_VLLM_MODEL` unset so the installed maintained release selects its current Spark Express model.", @@ -640,7 +643,7 @@ describe("starter prompt docs CTA", () => { ); expect(promptSource).not.toContain("\n- Ask for Balanced, Restricted, or Open policy.\n"); expect(promptSource).toContain( - "Managed vLLM: `NEMOCLAW_PROVIDER=install-vllm`; leave `NEMOCLAW_VLLM_MODEL` unset for DGX Spark Express, set it to `nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4` for DGX Station Express", + "Managed vLLM: `NEMOCLAW_PROVIDER=install-vllm`; leave `NEMOCLAW_VLLM_MODEL` unset for DGX Spark Express and for automatic DGX Station trusted-pair selection, or preserve an approved explicit model override.", ); }); diff --git a/test/test-boundary-guards.test.ts b/test/test-boundary-guards.test.ts index 6f0a5115ae..b41003d3b7 100644 --- a/test/test-boundary-guards.test.ts +++ b/test/test-boundary-guards.test.ts @@ -697,6 +697,7 @@ describe("Vitest project membership boundary", () => { ["test/install-preflight-docker-bootstrap.test.ts", "installer-integration"], ["test/install-preflight.test.ts", "installer-integration"], ["test/install-station-host-preparation.test.ts", "installer-integration"], + ["test/install-station-pair-preparation.test.ts", "installer-integration"], ["test/package-contract/example.test.js", "package-contract"], ["test/e2e/support/example.test.js", "e2e-support"], ["test/e2e/live/example.spec.ts", "e2e-live"], diff --git a/vitest.config.ts b/vitest.config.ts index 6d5509c6b1..e2f439ee04 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -125,6 +125,7 @@ export default defineConfig({ "test/install-preflight.test.ts", "test/install-preflight-docker-bootstrap.test.ts", "test/install-station-host-preparation.test.ts", + "test/install-station-pair-preparation.test.ts", "test/install-openshell-version-check.test.ts", ], }, @@ -144,6 +145,7 @@ export default defineConfig({ "test/install-preflight.test.ts", "test/install-preflight-docker-bootstrap.test.ts", "test/install-station-host-preparation.test.ts", + "test/install-station-pair-preparation.test.ts", "test/install-openshell-version-check.test.ts", ], // Slow tests that spawn real bash install.sh processes. Explicit From 21f906a5b5ca90fd8e44b8f8a8ace96e5988a358 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 16 Jul 2026 14:12:07 -0700 Subject: [PATCH 21/74] test(installer): keep Station fixtures linear Signed-off-by: Aaron Erickson --- test/install-station-pair-preparation.test.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/test/install-station-pair-preparation.test.ts b/test/install-station-pair-preparation.test.ts index 72524c0056..cc7119d300 100644 --- a/test/install-station-pair-preparation.test.ts +++ b/test/install-station-pair-preparation.test.ts @@ -98,6 +98,10 @@ function preparationOptions() { return { revision: REVISION, helperSha256: HELPER_SHA256 }; } +function throwFixtureError(error: Error): never { + throw error; +} + class PreparationHarness { readonly calls: string[] = []; readonly statePhases: DualStationResumeState["phase"][] = []; @@ -124,12 +128,12 @@ class PreparationHarness { inspectPretrustedTarget: (target) => { this.calls.push(`trust:${target}`); const error = this.trustErrors.get(target); - if (error) throw error; + error && throwFixtureError(error); return this.trusted.get(target) ?? null; }, probePeerHost: (binding) => { this.calls.push(`probe:peer:${binding.sshTarget}`); - if (this.peerProbeError) throw this.peerProbeError; + this.peerProbeError && throwFixtureError(this.peerProbeError); return structuredClone(this.peer); }, probeLocalConnectivity: () => { From ee5d6d3f92599be36adc1a70282717c0fa46a81b Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 16 Jul 2026 14:28:34 -0700 Subject: [PATCH 22/74] fix(installer): clarify Station GPU probe contract Signed-off-by: Aaron Erickson --- scripts/prepare-dgx-station-host.sh | 8 ++++++-- test/install-station-host-preparation.test.ts | 19 +++++++++++++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/scripts/prepare-dgx-station-host.sh b/scripts/prepare-dgx-station-host.sh index 9824429d0c..ae14f99bc7 100755 --- a/scripts/prepare-dgx-station-host.sh +++ b/scripts/prepare-dgx-station-host.sh @@ -5,7 +5,7 @@ set -Eeuo pipefail umask 077 -readonly SCRIPT_VERSION="2026-07-16.4" +readonly SCRIPT_VERSION="2026-07-16.5" readonly REBOOT_REQUIRED_EXIT=10 readonly MIN_FREE_KIB=$((20 * 1024 * 1024)) # The qualified generic image currently ships this OEM telemetry bootcmd. Its @@ -28,7 +28,11 @@ readonly DOCKER_VERSION="29.6.1" readonly TOOLKIT_VERSION="1.19.1" readonly FACTORY_DKMS_VERSION="3.0.11-1ubuntu13" readonly TARGET_DKMS_VERSION="1:3.4.0-1ubuntu1" -readonly ACCEPTANCE_IMAGE="ubuntu@sha256:7f622ca8766bccb22f04242ecb6f19f770b2f08827dc4b8c707de5e78a6da7ab" +# Keep this as a plain Ubuntu image: NVIDIA Container Toolkit injects the host +# driver utility when CDI or --gpus is requested. This intentionally exercises +# the documented runtime contract instead of relying on a CUDA image payload: +# https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/sample-workload.html +readonly ACCEPTANCE_IMAGE="docker.io/library/ubuntu@sha256:7f622ca8766bccb22f04242ecb6f19f770b2f08827dc4b8c707de5e78a6da7ab" readonly STATE_DIR="${HOME}/.local/state/station-bootstrap" readonly INSTALL_BOOT_MARKER="${STATE_DIR}/install-boot-id" diff --git a/test/install-station-host-preparation.test.ts b/test/install-station-host-preparation.test.ts index f167858792..67228d531e 100644 --- a/test/install-station-host-preparation.test.ts +++ b/test/install-station-host-preparation.test.ts @@ -61,6 +61,25 @@ describe("DGX Station host preparation", () => { } }); + it("uses the documented plain-Ubuntu driver-injection probe for CDI and --gpus", () => { + const { result, output } = runSourced( + STATION_PREPARE, + ` +sudo() { printf 'SUDO %s\\n' "$*"; } +run_cdi_test_sudo +run_gpus_test_sudo +`, + ); + + const image = + "docker.io/library/ubuntu@sha256:7f622ca8766bccb22f04242ecb6f19f770b2f08827dc4b8c707de5e78a6da7ab"; + expect(result.status, output).toBe(0); + expect(output).toContain( + `SUDO docker run --rm --device nvidia.com/gpu=all ${image} nvidia-smi`, + ); + expect(output).toContain(`SUDO docker run --rm --gpus all ${image} nvidia-smi`); + }); + it.each([ ["", "missing"], ["5:29.6.1-1~ubuntu.24.04~noble", "exact"], From 826b7b8fa37e294b6d6722b8379e3596221e4277 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 16 Jul 2026 15:48:06 -0700 Subject: [PATCH 23/74] fix(inference): pin qualified Station SSH identity Signed-off-by: Aaron Erickson --- docs/reference/commands.mdx | 1 + scripts/install.sh | 28 +- scripts/lib/dgx-station-peer.mts | 74 +- scripts/prepare-dual-dgx-station.mts | 45 +- src/lib/inference/vllm-docker-env.test.ts | 82 +- src/lib/inference/vllm-docker-env.ts | 11 +- src/lib/inference/vllm-dual-station.test.ts | 15 +- .../vllm-station-cluster-lifecycle.test.ts | 38 +- .../vllm-station-cluster-lifecycle.ts | 9 +- .../inference/vllm-station-cluster.test.ts | 156 +++- src/lib/inference/vllm-station-cluster.ts | 149 ++-- .../vllm-station-model-staging.test.ts | 53 +- .../inference/vllm-station-model-staging.ts | 113 +-- .../vllm-station-ssh-binding.test-support.ts | 55 ++ .../vllm-station-ssh-binding.test.ts | 403 +++++++++ src/lib/inference/vllm-station-ssh-binding.ts | 771 ++++++++++++++++++ src/lib/inference/vllm.ts | 2 +- test/install-station-pair-preparation.test.ts | 149 +++- 18 files changed, 1916 insertions(+), 238 deletions(-) create mode 100644 src/lib/inference/vllm-station-ssh-binding.test-support.ts create mode 100644 src/lib/inference/vllm-station-ssh-binding.test.ts create mode 100644 src/lib/inference/vllm-station-ssh-binding.ts diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 384076b731..57c1703ad4 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -3286,6 +3286,7 @@ Set them before running `$$nemoclaw onboard`. | `NEMOCLAW_INSTALL_TAG` | release tag | For internal installer commands: the release tag to install. Defaults to the admin-promoted `lkg` tag when unset. Overridden by the `--install-tag` flag. | | `NEMOCLAW_VLLM_MODEL` | registry slug or Hugging Face model id | Selects the model the managed-vLLM install path serves and remains authoritative during DGX Station installer setup. Recognised slugs: `qwen3.6-27b`, `qwen3.6-35b-a3b-nvfp4`, `nemotron-3-nano-4b`, `deepseek-v4-flash`, `nemotron-3-ultra-550b-a55b`, `deepseek-r1-distill-70b`. Unset uses the per-platform profile default, except that DGX Station installer setup automatically selects `nemotron-3-ultra-550b-a55b` after one pretrusted reciprocal pair qualifies. With both model and peer unset, no qualifying pair leaves the Station `deepseek-v4-flash` default in place. Gated models (e.g. `deepseek-r1-distill-70b`) require `HF_TOKEN` or `HUGGING_FACE_HUB_TOKEN`. | | `NEMOCLAW_DGX_STATION_PEER` | SSH host or `user@host` | Selects one exact, already-trusted DGX Station peer for Nemotron 3 Ultra pair qualification. The peer must match the reciprocal private `/30` rail and hardware checks; an explicit peer failure stops setup instead of falling back. NemoClaw does not enroll SSH trust or accept a port or SSH option in this value. When unset, DGX Station installer discovery checks only the two deterministic `/30` counterpart addresses. A peer cannot be combined with an explicit non-Ultra model; conflicting explicit selections fail before pair preparation. | +| `NEMOCLAW_DGX_STATION_SSH_BINDING` | opaque installer-managed token | Carries the qualified peer endpoint and host-key binding from DGX Station pair preparation into the current managed-vLLM install. The installer creates and clears this token; operators should not set or persist it. Missing, changed, or mismatched binding state fails before peer SSH or Docker work. | | `NEMOCLAW_VLLM_EXTRA_ARGS_JSON` | JSON array of non-blank strings | Appends advanced operator-owned tokens to the managed `vllm serve` command after NemoClaw's registry defaults. Example: `["--max-num-seqs","2"]`. Malformed JSON, non-string tokens, or blank tokens fail before Docker work starts. | | `NEMOCLAW_MINIMAL_BOOTSTRAP` | `1` to enable | Skips default OpenClaw workspace-template seeding for new pristine workspaces. Existing files are not deleted; refer to [Understand Runtime Changes](../manage-sandboxes/configure-sandboxes/understand-runtime-changes). | diff --git a/scripts/install.sh b/scripts/install.sh index 64df98f321..9c35f673cd 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -3290,6 +3290,7 @@ parse_station_dual_pair_result() { node -e ' const fs = require("node:fs"); const net = require("node:net"); + const path = require("node:path"); const fail = () => process.exit(2); try { const raw = fs.readFileSync(0, "utf8"); @@ -3322,7 +3323,23 @@ parse_station_dual_pair_result() { if (net.isIP(rail.localAddress) !== 4 || net.isIP(rail.peerAddress) !== 4) fail(); if (!mac.test(rail.localMac) || !mac.test(rail.peerMac)) fail(); } - process.stdout.write(`${value.kind}\n${peer}\n`); + const token = value.sshBinding; + if (typeof token !== "string" || token.length === 0 || token.length > 8192 || !/^[A-Za-z0-9_-]+$/.test(token)) fail(); + let handoff; + try { + const decoded = Buffer.from(token, "base64url"); + if (decoded.toString("base64url") !== token) fail(); + handoff = JSON.parse(decoded.toString("utf8")); + } catch { + fail(); + } + if (!handoff || typeof handoff !== "object" || Array.isArray(handoff)) fail(); + if (JSON.stringify(Object.keys(handoff).sort()) !== JSON.stringify(["bindingFile", "hostKeyDigest"])) fail(); + const bindingFile = handoff.bindingFile; + if (typeof bindingFile !== "string" || bindingFile.length === 0 || bindingFile.length > 4096 || bindingFile !== bindingFile.trim() || /[\u0000-\u001f\u007f]/.test(bindingFile)) fail(); + if (!path.isAbsolute(bindingFile) || path.normalize(bindingFile) !== bindingFile) fail(); + if (handoff.hostKeyDigest !== identity.hostKeyDigest) fail(); + process.stdout.write(`${value.kind}\n${peer}\n${token}\n`); } catch { fail(); } @@ -3341,7 +3358,7 @@ ensure_station_express_pair() { [[ -f "$coordinator" ]] || error "Dual DGX Station preparation coordinator is missing: ${coordinator}" [[ -f "$helper" ]] || error "DGX Station host preparation helper is missing: ${helper}" - local state_dir state_file revision output parsed kind peer_target status=0 + local state_dir state_file revision output parsed kind peer_target ssh_binding status=0 state_dir="$(ensure_nemoclaw_state_dir)" || error "Could not prepare owner-only NemoClaw state for dual DGX Station discovery." state_file="${state_dir}/station-dual-pair-resume.json" assert_nemoclaw_state_path_safe "$state_file" @@ -3384,6 +3401,7 @@ ensure_station_express_pair() { fi kind="$(printf '%s\n' "$parsed" | sed -n '1p')" peer_target="$(printf '%s\n' "$parsed" | sed -n '2p')" + ssh_binding="$(printf '%s\n' "$parsed" | sed -n '3p')" case "$kind" in single-station) @@ -3399,6 +3417,7 @@ ensure_station_express_pair() { if [ "${_STATION_EXPRESS_MODEL_WAS_EXPLICIT:-0}" = "0" ]; then unset NEMOCLAW_VLLM_MODEL fi + unset NEMOCLAW_DGX_STATION_SSH_BINDING info "No trusted reciprocal dual-DGX Station pair was detected; using the existing single-Station profile default." ;; ready) @@ -3407,7 +3426,8 @@ ensure_station_express_pair() { station_dual_pair_resume_pending \ || error "Dual DGX Station preparation returned ready without exact pair resume state; refusing to continue." NEMOCLAW_DGX_STATION_PEER="$peer_target" - export NEMOCLAW_DGX_STATION_PEER + NEMOCLAW_DGX_STATION_SSH_BINDING="$ssh_binding" + export NEMOCLAW_DGX_STATION_PEER NEMOCLAW_DGX_STATION_SSH_BINDING if [ "${_STATION_EXPRESS_MODEL_WAS_EXPLICIT:-0}" = "0" ]; then # Leave the vLLM selector unset. The trusted peer is the existing # installVllm signal that performs the authoritative full capability @@ -3441,7 +3461,7 @@ clear_station_dual_pair_resume() { local state_file coordinator="${SCRIPT_DIR}/prepare-dual-dgx-station.mts" state_file="$(station_dual_pair_resume_file)" || return 0 assert_nemoclaw_state_path_safe "$state_file" - [[ -e "$state_file" || -L "$state_file" ]] || return 0 + [[ -e "$state_file" || -L "$state_file" || -e "${state_file}.ssh-binding" || -L "${state_file}.ssh-binding" ]] || return 0 [[ -f "$coordinator" ]] || error "Dual DGX Station preparation coordinator is missing: ${coordinator}" node --no-warnings --experimental-strip-types "$coordinator" --state "$state_file" --clear-state >/dev/null \ || error "Could not safely clear completed dual DGX Station resume state: ${state_file}" diff --git a/scripts/lib/dgx-station-peer.mts b/scripts/lib/dgx-station-peer.mts index 20477eea56..cd73fd0bd2 100644 --- a/scripts/lib/dgx-station-peer.mts +++ b/scripts/lib/dgx-station-peer.mts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import net from "node:net"; +import { stationKnownHostsDigest } from "../../src/lib/inference/vllm-station-ssh-binding.ts"; export const DUAL_STATION_RESUME_SCHEMA_VERSION = 1; export const STATION_PREP_REBOOT_REQUIRED_EXIT = 10; @@ -15,6 +16,8 @@ const GPU_UUID_PATTERN = /^GPU-[A-Za-z0-9-]+$/; const HOST_KEY_DIGEST_PATTERN = /^[a-f0-9]{64}$/; const HOST_KEY_FINGERPRINT_PATTERN = /^SHA256:[A-Za-z0-9+/]{16,86}={0,2}$/; const MAC_PATTERN = /^(?:[0-9a-f]{2}:){5}[0-9a-f]{2}$/; +const MAX_KNOWN_HOSTS_BYTES = 64 * 1024; +const MAX_KNOWN_HOSTS_LINE_BYTES = 16 * 1024; export type StationPrepMode = "--check" | "--apply" | "--verify"; @@ -93,8 +96,18 @@ export interface DualStationResumeState extends DualStationPairIdentity { export type DualStationPreparationResult = | { kind: "single-station"; reason: string } - | { kind: "ready"; peerTarget: string; identity: DualStationPairIdentity } - | { kind: "reboot-required"; peerTarget: string; identity: DualStationPairIdentity }; + | { + kind: "ready"; + peerTarget: string; + identity: DualStationPairIdentity; + binding: PretrustedSshTarget; + } + | { + kind: "reboot-required"; + peerTarget: string; + identity: DualStationPairIdentity; + binding: PretrustedSshTarget; + }; export interface DualStationPreparationOptions { revision: string; @@ -604,12 +617,22 @@ function validateKnownHostsLookupHost(binding: PretrustedSshTarget): void { } function validateBinding(binding: PretrustedSshTarget): void { - validateStationPeerTarget(binding.requestedTarget); - validateStationPeerTarget(binding.sshTarget); + const requestedTarget = validateStationPeerTarget(binding.requestedTarget); + const sshTarget = validateStationPeerTarget(binding.sshTarget); + if (requestedTarget !== sshTarget) { + throw new Error("Pretrusted SSH target changed after configuration resolution"); + } if (!isSafeTargetHost(binding.resolvedHost)) { throw new Error("Pretrusted SSH target resolved to an unsafe host"); } - if (!SAFE_USERNAME_PATTERN.test(binding.sshUser) || binding.port < 1 || binding.port > 65535) { + const explicitUser = sshTarget.includes("@") ? sshTarget.slice(0, sshTarget.indexOf("@")) : null; + if ( + !SAFE_USERNAME_PATTERN.test(binding.sshUser) || + (explicitUser !== null && explicitUser !== binding.sshUser) || + !Number.isInteger(binding.port) || + binding.port < 1 || + binding.port > 65535 + ) { throw new Error("Pretrusted SSH target has an unsafe user or port"); } validateKnownHostsLookupHost(binding); @@ -617,17 +640,33 @@ function validateBinding(binding: PretrustedSshTarget): void { throw new Error("Pretrusted SSH target has an invalid host-key digest"); } if ( + !Array.isArray(binding.keyFingerprints) || binding.keyFingerprints.length === 0 || binding.keyFingerprints.some( - (fingerprint) => !HOST_KEY_FINGERPRINT_PATTERN.test(fingerprint), + (fingerprint) => + typeof fingerprint !== "string" || !HOST_KEY_FINGERPRINT_PATTERN.test(fingerprint), ) || + !Array.isArray(binding.knownHostsLines) || binding.knownHostsLines.length === 0 || binding.knownHostsLines.some( - (line) => line.length === 0 || line.length > 16 * 1024 || /[\u0000\r\n]/.test(line), + (line) => + typeof line !== "string" || + line.length === 0 || + Buffer.byteLength(line, "utf8") > MAX_KNOWN_HOSTS_LINE_BYTES || + line !== line.trim() || + line.startsWith("#") || + /[\u0000\r\n]/.test(line), ) ) { throw new Error("Pretrusted SSH target has invalid known-hosts evidence"); } + const knownHosts = `${[...new Set(binding.knownHostsLines)].sort().join("\n")}\n`; + if ( + Buffer.byteLength(knownHosts, "utf8") > MAX_KNOWN_HOSTS_BYTES || + stationKnownHostsDigest(knownHosts) !== binding.hostKeyDigest + ) { + throw new Error("Pretrusted SSH target known-hosts evidence does not match its digest"); + } } function selectPretrustedTarget( @@ -799,7 +838,12 @@ export function prepareDualStationPair( deps.writeResumeState(state); if (options.reuseExistingManagedPair) { deps.writeResumeState({ ...state, phase: "ready" }); - return { kind: "ready", peerTarget: binding.sshTarget, identity: plan.identity }; + return { + kind: "ready", + peerTarget: binding.sshTarget, + identity: plan.identity, + binding, + }; } deps.log(`Preparing reciprocal peer ${binding.sshTarget} with the exact reviewed helper`); @@ -811,7 +855,12 @@ export function prepareDualStationPair( const applyStatus = deps.runRemoteHelper(binding, "--apply"); if (applyStatus === STATION_PREP_REBOOT_REQUIRED_EXIT) { deps.writeResumeState({ ...state, phase: "remote-reboot-required" }); - return { kind: "reboot-required", peerTarget: binding.sshTarget, identity: plan.identity }; + return { + kind: "reboot-required", + peerTarget: binding.sshTarget, + identity: plan.identity, + binding, + }; } if (applyStatus !== 0) { throw new Error("Peer DGX Station host preparation failed; refusing single-Station fallback"); @@ -821,5 +870,10 @@ export function prepareDualStationPair( } deps.writeResumeState({ ...state, phase: "ready" }); - return { kind: "ready", peerTarget: binding.sshTarget, identity: plan.identity }; + return { + kind: "ready", + peerTarget: binding.sshTarget, + identity: plan.identity, + binding, + }; } diff --git a/scripts/prepare-dual-dgx-station.mts b/scripts/prepare-dual-dgx-station.mts index 1df59c1528..1c6d671785 100755 --- a/scripts/prepare-dual-dgx-station.mts +++ b/scripts/prepare-dual-dgx-station.mts @@ -9,7 +9,12 @@ import net from "node:net"; import os from "node:os"; import path from "node:path"; import { fileURLToPath } from "node:url"; - +import { + clearDualStationSshBinding, + encodeDualStationSshBindingHandoff, + stationKnownHostsDigest, + writeDualStationSshBinding, +} from "../src/lib/inference/vllm-station-ssh-binding.ts"; import { type DualStationPreparationDeps, type DualStationResumeState, @@ -501,7 +506,6 @@ function knownHostEvidence( digest: string; } | null { const lines = new Set(); - const keys = new Set(); const fingerprints = new Set(); for (const file of files) { if (!path.isAbsolute(file)) continue; @@ -535,7 +539,6 @@ function knownHostEvidence( const fingerprint = fingerprintKnownHostKey(keyType, keyData); if (!fingerprint) continue; lines.add(line); - keys.add(`${marker ?? ""}|${keyType}|${keyData}`); // Preserve matching revocations in the private pinned file so the // subsequent SSH connection cannot resurrect a key the operator or // system administrator explicitly revoked. A revoked line alone is not @@ -545,12 +548,11 @@ function knownHostEvidence( } } if (lines.size === 0 || fingerprints.size === 0) return null; + const retainedLines = [...lines].sort(); return { - lines: [...lines].sort(), + lines: retainedLines, fingerprints: [...fingerprints].sort(), - digest: createHash("sha256") - .update([...keys].sort().join("\n")) - .digest("hex"), + digest: stationKnownHostsDigest(`${retainedLines.join("\n")}\n`), }; } @@ -810,8 +812,8 @@ export function writeDualStationResumeState( export function clearDualStationResumeState(statePath: string): void { const current = readDualStationResumeState(statePath); - if (!current) return; - fs.unlinkSync(statePath); + clearDualStationSshBinding(statePath); + if (current) fs.unlinkSync(statePath); } function parseCliOptions(args: readonly string[]): CliOptions { @@ -1007,8 +1009,31 @@ export function runCli(args: readonly string[]): number { ); if (result.kind === "single-station") { runtime.deps.log(`Using the existing single-Station path: ${result.reason}`); + // A single-Station result is possible only without pair resume state. + // Remove any owner-only binding orphan left by an interrupted earlier + // cleanup so it cannot linger without the pair identity that owns it. + if (readDualStationResumeState(options.statePath)) { + throw new Error("Single-Station fallback cannot discard exact pair resume state"); + } + clearDualStationSshBinding(options.statePath); + process.stdout.write(`${JSON.stringify(result)}\n`); + return 0; + } + if ( + result.binding.sshTarget !== result.peerTarget || + result.binding.hostKeyDigest !== result.identity.hostKeyDigest + ) { + throw new Error("Qualified Station SSH binding does not match the prepared pair identity"); } - process.stdout.write(`${JSON.stringify(result)}\n`); + const binding = writeDualStationSshBinding(options.statePath, result.binding); + process.stdout.write( + `${JSON.stringify({ + kind: result.kind, + peerTarget: result.peerTarget, + identity: result.identity, + sshBinding: encodeDualStationSshBindingHandoff(binding), + })}\n`, + ); return result.kind === "reboot-required" ? 10 : 0; } finally { runtime.cleanup(); diff --git a/src/lib/inference/vllm-docker-env.test.ts b/src/lib/inference/vllm-docker-env.test.ts index cf0395559c..20497497a3 100644 --- a/src/lib/inference/vllm-docker-env.test.ts +++ b/src/lib/inference/vllm-docker-env.test.ts @@ -1,15 +1,34 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { afterEach, describe, expect, it, vi } from "vitest"; +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { buildLocalDualStationDockerEnv, buildRemoteVllmDockerEnv, buildVllmDockerEnv, } from "./vllm-docker-env"; +import { + clearDualStationSshBinding, + stationKnownHostsDigest, + writeDualStationSshBinding, +} from "./vllm-station-ssh-binding"; +import { + createDualStationSshBindingFixture, + type DualStationSshBindingFixture, +} from "./vllm-station-ssh-binding.test-support"; + +let sshFixture: DualStationSshBindingFixture; + +beforeEach(() => { + sshFixture = createDualStationSshBindingFixture("station@dgx-peer.example.test"); +}); afterEach(() => { vi.unstubAllEnvs(); + sshFixture.cleanup(); }); describe("managed vLLM Docker client environment", () => { @@ -55,14 +74,15 @@ describe("managed vLLM Docker client environment", () => { vi.stubEnv("OPENSHELL_GATEWAY_AUTH_TOKEN", "must-not-cross-ssh"); vi.stubEnv("UNRELATED_SECRET", "do-not-forward"); - const env = buildRemoteVllmDockerEnv("ssh://station@dgx-peer.example.test:22"); + const env = buildRemoteVllmDockerEnv(sshFixture.binding); expect(env).toEqual( expect.objectContaining({ - DOCKER_HOST: "ssh://station@dgx-peer.example.test:22", + DOCKER_HOST: "ssh://station@192.168.50.20", SSH_AUTH_SOCK: "/tmp/ssh-agent.sock", }), ); + expect(env.PATH).toBe(sshFixture.binding.sshWrapperDirectory); expect(env.DOCKER_API_VERSION).toBeUndefined(); expect(env.DOCKER_CERT_PATH).toBeUndefined(); expect(env.DOCKER_CONFIG).toBeUndefined(); @@ -89,16 +109,52 @@ describe("managed vLLM Docker client environment", () => { expect(env.DOCKER_CONFIG).toBeUndefined(); }); - it.each([ - "tcp://dgx-peer.example.test:2376", - "ssh://station:secret@dgx-peer.example.test", - "ssh://dgx-peer.example.test/", - "ssh://dgx-peer.example.test?context=other", - "ssh://Dgx-Peer.example.test", - " ssh://dgx-peer.example.test", - ])("rejects a non-canonical or unsafe remote Docker target: %s", (sshUri) => { - expect(() => buildRemoteVllmDockerEnv(sshUri)).toThrow( - "Remote Docker host must be a canonical ssh://[user@]host[:port] URI", + it("rejects a changed qualified host-key pin before constructing a Docker environment", () => { + fs.appendFileSync(sshFixture.binding.knownHostsFile, "changed\n"); + + expect(() => buildRemoteVllmDockerEnv(sshFixture.binding)).toThrow( + "Station SSH known-hosts binding changed after qualification", ); }); + + it("keeps an existing environment pinned when a later qualification writes a new version", () => { + const first = sshFixture.binding; + const firstEnv = buildRemoteVllmDockerEnv(first); + const replacementHost = "192.168.50.21"; + const replacementLines = [ + `${replacementHost} ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIcmVwbGFjZW1lbnQ=`, + ]; + const second = writeDualStationSshBinding( + sshFixture.resumeStatePath, + { + ...sshFixture.identity, + resolvedHost: replacementHost, + lookupHost: replacementHost, + hostKeyDigest: stationKnownHostsDigest(`${replacementLines.join("\n")}\n`), + knownHostsLines: replacementLines, + }, + { dockerCliFile: sshFixture.dockerCliFile }, + ); + const secondEnv = buildRemoteVllmDockerEnv(second); + + expect(second.sshWrapperDirectory).not.toBe(first.sshWrapperDirectory); + expect(firstEnv.PATH).toBe(first.sshWrapperDirectory); + expect(secondEnv.PATH).toBe(second.sshWrapperDirectory); + expect(firstEnv.DOCKER_HOST).toBe("ssh://station@192.168.50.20"); + expect(secondEnv.DOCKER_HOST).toBe(`ssh://station@${replacementHost}`); + expect(buildRemoteVllmDockerEnv(first)).toEqual(firstEnv); + expect(fs.existsSync(first.dockerShimFile)).toBe(true); + expect(fs.existsSync(second.dockerShimFile)).toBe(true); + }); + + it("fails closed instead of falling through to ambient Docker after cleanup", () => { + const env = buildRemoteVllmDockerEnv(sshFixture.binding); + + clearDualStationSshBinding(sshFixture.resumeStatePath); + + const result = spawnSync("docker", ["version"], { env, encoding: "utf8" }); + expect(result.status).toBeNull(); + expect(result.error).toMatchObject({ code: "ENOENT" }); + expect(() => buildRemoteVllmDockerEnv(sshFixture.binding)).toThrow(); + }); }); diff --git a/src/lib/inference/vllm-docker-env.ts b/src/lib/inference/vllm-docker-env.ts index 3ffe0f2f69..6dcf3f14f9 100644 --- a/src/lib/inference/vllm-docker-env.ts +++ b/src/lib/inference/vllm-docker-env.ts @@ -4,6 +4,11 @@ import { isIP } from "node:net"; import { buildSubprocessEnv } from "../subprocess-env"; +import { + assertDualStationSshBindingFiles, + type DualStationSshBinding, + dualStationDockerSshUri, +} from "./vllm-station-ssh-binding"; const DOCKER_CLIENT_ENV_NAMES = [ "DOCKER_API_VERSION", @@ -135,11 +140,13 @@ export function buildVllmSshTransportEnv( * ambient context, client config, API pin, or TCP/TLS settings to influence it. */ export function buildRemoteVllmDockerEnv( - sshUri: string, + binding: DualStationSshBinding, source: NodeJS.ProcessEnv = process.env, ): Record { - const remoteHost = validateRemoteDockerSshUri(sshUri); + assertDualStationSshBindingFiles(binding); + const remoteHost = validateRemoteDockerSshUri(dualStationDockerSshUri(binding)); const env = buildVllmSshTransportEnv({ DOCKER_HOST: remoteHost }, source); for (const name of REMOTE_DOCKER_INCOMPATIBLE_ENV_NAMES) delete env[name]; + env.PATH = binding.sshWrapperDirectory; return env; } diff --git a/src/lib/inference/vllm-dual-station.test.ts b/src/lib/inference/vllm-dual-station.test.ts index e967ac1fc4..323b0273a1 100644 --- a/src/lib/inference/vllm-dual-station.test.ts +++ b/src/lib/inference/vllm-dual-station.test.ts @@ -93,6 +93,10 @@ vi.mock("./vllm-api-key", () => ({ import { detectVllmProfile, installVllm } from "./vllm"; import { DUAL_STATION_VLLM_RUNTIME, type DualStationVllmPlan } from "./vllm-station-cluster"; +import { + createDualStationSshBindingFixture, + type DualStationSshBindingFixture, +} from "./vllm-station-ssh-binding.test-support"; const API_KEY = "ab".repeat(32); const HEAD_ID = "a".repeat(64); @@ -101,8 +105,7 @@ const HEAD_BASE_URL = "http://192.168.100.1:8000"; function plan(): DualStationVllmPlan { return { - peerSshTarget: "nvidia@station-b", - peerDockerHost: "ssh://nvidia@station-b", + peerSshBinding: sshFixture.binding, runtime: DUAL_STATION_VLLM_RUNTIME, local: { hostname: "station-a", @@ -179,9 +182,11 @@ let logSpy: ReturnType; let errorSpy: ReturnType; let mkdirSpy: ReturnType; let stdoutSpy: ReturnType; +let sshFixture: DualStationSshBindingFixture; beforeEach(() => { vi.clearAllMocks(); + sshFixture = createDualStationSshBindingFixture(); process.env.NEMOCLAW_VLLM_MODEL = "nemotron-3-ultra-550b-a55b"; process.env.NEMOCLAW_DGX_STATION_PEER = "nvidia@station-b"; process.env.HF_TOKEN = "hf_test"; @@ -268,6 +273,7 @@ afterEach(() => { errorSpy.mockRestore(); mkdirSpy.mockRestore(); stdoutSpy.mockRestore(); + sshFixture.cleanup(); process.env = { ...originalEnv }; }); @@ -316,7 +322,10 @@ describe("dual DGX Station vLLM install orchestration", () => { expect(mocks.dockerPullWithProgressWatchdog).toHaveBeenCalledTimes(2); const localPullOptions = mocks.dockerPullWithProgressWatchdog.mock.calls[0][1]; const peerPullOptions = mocks.dockerPullWithProgressWatchdog.mock.calls[1][1]; - expect(peerPullOptions.env.DOCKER_HOST).toBe("ssh://nvidia@station-b"); + expect(peerPullOptions.env.DOCKER_HOST).toBe("ssh://nvidia@192.168.50.20"); + expect(peerPullOptions.env.PATH.split(":")[0]).toBe( + clusterPlan.peerSshBinding.sshWrapperDirectory, + ); expect(localPullOptions.env.DOCKER_HOST).not.toBe(peerPullOptions.env.DOCKER_HOST); expect(localPullOptions.env.DOCKER_CONTEXT).toBe("default"); expect(peerPullOptions.env.DOCKER_CONTEXT).toBeUndefined(); diff --git a/src/lib/inference/vllm-station-cluster-lifecycle.test.ts b/src/lib/inference/vllm-station-cluster-lifecycle.test.ts index 22e9cb30d6..6b7ef28f18 100644 --- a/src/lib/inference/vllm-station-cluster-lifecycle.test.ts +++ b/src/lib/inference/vllm-station-cluster-lifecycle.test.ts @@ -2,7 +2,8 @@ // SPDX-License-Identifier: Apache-2.0 import { AsyncLocalStorage } from "node:async_hooks"; -import { describe, expect, it, vi } from "vitest"; +import fs from "node:fs"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { DUAL_STATION_VLLM_RUNTIME, type DualStationVllmPlan } from "./vllm-station-cluster"; import { areDualStationManagedVllmContainersRunning, @@ -32,6 +33,11 @@ import { startDualStationManagedVllm, withDualStationManagedVllmLifecycle, } from "./vllm-station-cluster-lifecycle"; +import type { DualStationSshBinding } from "./vllm-station-ssh-binding"; +import { + createDualStationSshBindingFixture, + type DualStationSshBindingFixture, +} from "./vllm-station-ssh-binding.test-support"; const WORKER_ID = "a".repeat(64); const HEAD_ID = "b".repeat(64); @@ -41,11 +47,19 @@ const API_KEY = "e".repeat(64); const START_CONFIG = { apiKey: API_KEY }; const API_KEY_FINGERPRINT = dualStationVllmApiKeyFingerprint(API_KEY); const TRANSACTION_ID = "1".repeat(32); +let sshFixture: DualStationSshBindingFixture; + +beforeEach(() => { + sshFixture = createDualStationSshBindingFixture(); +}); + +afterEach(() => { + sshFixture.cleanup(); +}); function fixturePlan(): DualStationVllmPlan { return { - peerSshTarget: "nvidia@station-b", - peerDockerHost: "ssh://nvidia@station-b", + peerSshBinding: sshFixture.binding, runtime: DUAL_STATION_VLLM_RUNTIME, local: { hostname: "station-a", @@ -199,9 +213,9 @@ function harness( args: readonly string[]; options: DualStationDockerOptions | undefined; }> = []; - const buildRemoteDockerEnv = vi.fn((sshUri: string) => ({ + const buildRemoteDockerEnv = vi.fn((binding: DualStationSshBinding) => ({ TARGET: "peer", - DOCKER_HOST: sshUri, + DOCKER_HOST: `ssh://${binding.sshUser}@${binding.resolvedHost}`, VLLM_API_KEY: "ambient-must-be-stripped", })); let nonceCounter = 0; @@ -554,6 +568,18 @@ describe("dual-Station managed vLLM lifecycle", () => { expect(fake.operations.some((operation) => operation.kind === "rm")).toBe(false); }); + it("rejects a changed host-key pin before reading or mutating either Docker daemon", () => { + const fake = harness(); + fs.appendFileSync(sshFixture.binding.knownHostsFile, "changed\n"); + + expect(preflightDualStationManagedVllm(fixturePlan(), fake.deps)).toMatchObject({ + ok: false, + reason: expect.stringContaining("known-hosts binding changed"), + }); + expect(fake.operations).toEqual([]); + expect(fake.buildRemoteDockerEnv).not.toHaveBeenCalled(); + }); + it("proves exact-image GPU execution on both daemons and removes both exact probe IDs", async () => { const fake = harness(); @@ -653,7 +679,7 @@ describe("dual-Station managed vLLM lifecycle", () => { for (const options of [...fake.captureOptions, ...fake.rmOptions]) { expect(options?.env?.VLLM_API_KEY).toBeUndefined(); } - expect(fake.buildRemoteDockerEnv).toHaveBeenCalledWith("ssh://nvidia@station-b"); + expect(fake.buildRemoteDockerEnv).toHaveBeenCalledWith(sshFixture.binding); }); it("reuses an already-running exact pair without tearing down the working service", async () => { diff --git a/src/lib/inference/vllm-station-cluster-lifecycle.ts b/src/lib/inference/vllm-station-cluster-lifecycle.ts index 5eab1c5e93..874d29b1a5 100644 --- a/src/lib/inference/vllm-station-cluster-lifecycle.ts +++ b/src/lib/inference/vllm-station-cluster-lifecycle.ts @@ -12,6 +12,10 @@ import { buildLocalDualStationDockerEnv, buildRemoteVllmDockerEnv } from "./vllm import { buildNemotronUltraDistributedServeCommand } from "./vllm-models"; import { DUAL_STATION_VLLM_RUNTIME, type DualStationVllmPlan } from "./vllm-station-cluster"; import { withDualStationVllmLifecycleLock } from "./vllm-station-lifecycle-lock"; +import { + assertDualStationSshBindingFiles, + type DualStationSshBinding, +} from "./vllm-station-ssh-binding"; export const DUAL_STATION_VLLM_HEAD_CONTAINER_NAME = "nemoclaw-vllm"; export const DUAL_STATION_VLLM_WORKER_CONTAINER_NAME = "nemoclaw-vllm-worker"; @@ -75,7 +79,7 @@ export interface DualStationVllmLifecycleDeps { options?: DualStationDockerOptions, ): DualStationDockerResult; buildLocalDockerEnv(): Record; - buildRemoteDockerEnv(sshUri: string): Record; + buildRemoteDockerEnv(binding: DualStationSshBinding): Record; createProbeNonce(): string; createTransactionId(): string; waitBeforeReconcile(ms: number): Promise; @@ -180,6 +184,7 @@ function isRfc1918Ipv4(address: string): boolean { } function assertSafePlan(plan: DualStationVllmPlan): void { + assertDualStationSshBindingFiles(plan.peerSshBinding); if ( !IMMUTABLE_IMAGE_PATTERN.test(plan.runtime.image) || plan.runtime.image !== DUAL_STATION_VLLM_RUNTIME.image || @@ -508,7 +513,7 @@ function specsForPlan( image: plan.runtime.image, launchContract: dualStationVllmLaunchContract(plan, "worker"), apiKeyFingerprint, - env: withoutVllmApiKey(deps.buildRemoteDockerEnv(plan.peerDockerHost)), + env: withoutVllmApiKey(deps.buildRemoteDockerEnv(plan.peerSshBinding)), }, }; } diff --git a/src/lib/inference/vllm-station-cluster.test.ts b/src/lib/inference/vllm-station-cluster.test.ts index 463809d7c3..c32c49f1c2 100644 --- a/src/lib/inference/vllm-station-cluster.test.ts +++ b/src/lib/inference/vllm-station-cluster.test.ts @@ -2,8 +2,9 @@ // SPDX-License-Identifier: Apache-2.0 import type { SpawnSyncOptionsWithStringEncoding } from "node:child_process"; +import fs from "node:fs"; -import { describe, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { buildRemoteVllmDockerEnv } from "./vllm-docker-env"; import { @@ -16,24 +17,52 @@ import { type StationHostProbe, type StationProbeCommandResult, type StationRailConnectivityRequest, + validatePeerTarget, } from "./vllm-station-cluster"; +import { + type DualStationSshBinding, + loadDualStationSshBindingHandoff, + NEMOCLAW_DGX_STATION_SSH_BINDING_ENV, +} from "./vllm-station-ssh-binding"; +import { + createDualStationSshBindingFixture, + type DualStationSshBindingFixture, +} from "./vllm-station-ssh-binding.test-support"; const LOCAL_HOME = "/home/local"; const PEER_HOME = "/home/nvidia"; -const STRICT_DOCKER_SSH_CONFIG = [ - "batchmode yes", - "stricthostkeychecking true", - "permitlocalcommand no", - "forwardagent no", - "forwardx11 no", - "forwardx11trusted no", - "tunnel false", - "updatehostkeys no", - "controlmaster false", - "controlpersist no", - "sendenv LANG", - "sendenv LC_*", -].join("\n"); +function strictDockerSshConfig(binding: DualStationSshBinding): string { + return [ + `hostname ${binding.resolvedHost}`, + `user ${binding.sshUser}`, + `port ${String(binding.port)}`, + `hostkeyalias ${binding.lookupHost}`, + `userknownhostsfile ${binding.knownHostsFile}`, + "globalknownhostsfile /dev/null", + "batchmode yes", + "stricthostkeychecking true", + "permitlocalcommand no", + "forwardagent no", + "forwardx11 no", + "forwardx11trusted no", + "tunnel false", + "updatehostkeys no", + "controlmaster false", + "controlpersist no", + "sendenv LANG", + "sendenv LC_*", + ].join("\n"); +} + +let sshFixture: DualStationSshBindingFixture; + +beforeEach(() => { + sshFixture = createDualStationSshBindingFixture(); +}); + +afterEach(() => { + sshFixture.cleanup(); +}); function snapshotPath(home: string): string { return [ @@ -174,15 +203,18 @@ function fixtureDeps( ): FixtureDeps { const localHost = vi.fn(() => command(local)); const peerHost = vi.fn(() => command(peer)); - const sshConfig = vi.fn(() => command(STRICT_DOCKER_SSH_CONFIG)); + const sshConfig = vi.fn((binding: DualStationSshBinding) => + command(strictDockerSshConfig(binding)), + ); const localConnectivity = vi.fn((requests: readonly StationRailConnectivityRequest[]) => connectivityResponse(requests, options.localConnectivityAlter), ); const peerConnectivity = vi.fn( - (_target: string, requests: readonly StationRailConnectivityRequest[]) => + (_binding: DualStationSshBinding, requests: readonly StationRailConnectivityRequest[]) => connectivityResponse(requests, options.peerConnectivityAlter), ); return { + loadPeerSshBinding: loadDualStationSshBindingHandoff, probePeerSshConfig: sshConfig, probeLocalHost: localHost, probePeerHost: peerHost, @@ -193,8 +225,15 @@ function fixtureDeps( } function runWith(deps: StationClusterProbeDeps, target = "nvidia@station-b") { + if (validatePeerTarget(target).ok && sshFixture.binding.peerTarget !== target) { + sshFixture.cleanup(); + sshFixture = createDualStationSshBindingFixture(target); + } return probeDualStationVllmCapability({ - env: { [NEMOCLAW_DGX_STATION_PEER_ENV]: target }, + env: { + [NEMOCLAW_DGX_STATION_PEER_ENV]: target, + [NEMOCLAW_DGX_STATION_SSH_BINDING_ENV]: sshFixture.token, + }, deps, }); } @@ -241,11 +280,38 @@ describe("probeDualStationVllmCapability", () => { expect(deps.calls.peerHost).not.toHaveBeenCalled(); }); + it("requires the installer-qualified SSH binding before any peer probe", () => { + const deps = fixtureDeps(); + + expect( + probeDualStationVllmCapability({ + env: { [NEMOCLAW_DGX_STATION_PEER_ENV]: "nvidia@station-b" }, + deps, + }), + ).toMatchObject({ kind: "unavailable", code: "peer-ssh-config-unsafe" }); + expect(deps.calls.sshConfig).not.toHaveBeenCalled(); + expect(deps.calls.localHost).not.toHaveBeenCalled(); + expect(deps.calls.peerHost).not.toHaveBeenCalled(); + }); + + it("rejects a changed qualified host-key pin before any peer probe", () => { + const deps = fixtureDeps(); + fs.appendFileSync(sshFixture.binding.knownHostsFile, "changed\n"); + + expect(runWith(deps)).toMatchObject({ + kind: "unavailable", + code: "peer-ssh-config-unsafe", + }); + expect(deps.calls.sshConfig).not.toHaveBeenCalled(); + expect(deps.calls.localHost).not.toHaveBeenCalled(); + expect(deps.calls.peerHost).not.toHaveBeenCalled(); + }); + it("rejects Docker-over-SSH when the effective operator config weakens peer trust", () => { const deps = fixtureDeps(); - deps.probePeerSshConfig = () => + deps.probePeerSshConfig = (binding) => command( - STRICT_DOCKER_SSH_CONFIG.replace( + strictDockerSshConfig(binding).replace( "stricthostkeychecking true", "stricthostkeychecking false", ), @@ -261,7 +327,7 @@ describe("probeDualStationVllmCapability", () => { it("rejects an SSH config that can SendEnv arbitrary NemoClaw secrets", () => { const deps = fixtureDeps(); - deps.probePeerSshConfig = () => command(`${STRICT_DOCKER_SSH_CONFIG}\nsendenv *`); + deps.probePeerSshConfig = (binding) => command(`${strictDockerSshConfig(binding)}\nsendenv *`); expect(runWith(deps)).toMatchObject({ kind: "unavailable", @@ -277,6 +343,7 @@ describe("probeDualStationVllmCapability", () => { probeDualStationVllmCapability({ env: { [NEMOCLAW_DGX_STATION_PEER_ENV]: "nvidia@station-b", + [NEMOCLAW_DGX_STATION_SSH_BINDING_ENV]: sshFixture.token, DOCKER_CONTEXT: "remote-builder", }, deps, @@ -290,13 +357,17 @@ describe("probeDualStationVllmCapability", () => { "station-b", "nvidia@station-b", "_svc@192.168.50.20", - ])("returns a downstream-compatible canonical Docker-over-SSH URI for %s", (target) => { - expect(runWith(fixtureDeps(), target)).toMatchObject({ + ])("returns the qualified binding for %s", (target) => { + const result = runWith(fixtureDeps(), target); + expect(result).toMatchObject({ kind: "ready", peerModelSnapshot: "ready", - plan: { peerSshTarget: target, peerDockerHost: `ssh://${target}` }, + plan: { peerSshBinding: { peerTarget: target } }, }); - expect(buildRemoteVllmDockerEnv(`ssh://${target}`, {}).DOCKER_HOST).toBe(`ssh://${target}`); + if (result.kind !== "ready") throw new Error("expected ready fixture"); + expect(buildRemoteVllmDockerEnv(result.plan.peerSshBinding, {}).DOCKER_HOST).toBe( + `ssh://${result.plan.peerSshBinding.sshUser}@${result.plan.peerSshBinding.resolvedHost}`, + ); }); it("returns a deterministic two-rail TP2 plan and permits one auxiliary non-GB300 GPU", () => { @@ -307,8 +378,11 @@ describe("probeDualStationVllmCapability", () => { expect(result).toMatchObject({ kind: "ready", plan: { - peerSshTarget: "nvidia@station-b", - peerDockerHost: "ssh://nvidia@station-b", + peerSshBinding: { + peerTarget: "nvidia@station-b", + resolvedHost: "192.168.50.20", + sshUser: "nvidia", + }, runtime: { image: "vllm/vllm-openai@sha256:0fec7ec5f3e6bc168e54899935fb0557da908a4832a1dbc88e2debcf2f889416", @@ -358,7 +432,7 @@ describe("probeDualStationVllmCapability", () => { ], }, }); - expect(deps.calls.peerHost).toHaveBeenCalledWith("nvidia@station-b"); + expect(deps.calls.peerHost).toHaveBeenCalledWith(sshFixture.binding); expect(deps.calls.localConnectivity).toHaveBeenCalledWith([ { netdev: "cx8a0", @@ -373,7 +447,7 @@ describe("probeDualStationVllmCapability", () => { expectedPeerMac: "02:00:00:bb:00:01", }, ]); - expect(deps.calls.peerConnectivity).toHaveBeenCalledWith("nvidia@station-b", [ + expect(deps.calls.peerConnectivity).toHaveBeenCalledWith(sshFixture.binding, [ { netdev: "cx8b0", sourceAddress: "192.168.240.2", @@ -687,15 +761,27 @@ describe("probe command boundary", () => { _file: string, _args: readonly string[], _options: SpawnSyncOptionsWithStringEncoding, - ): StationProbeCommandResult => command(STRICT_DOCKER_SSH_CONFIG), + ): StationProbeCommandResult => command(strictDockerSshConfig(sshFixture.binding)), ); const deps = createStationClusterProbeDeps(spawn); - deps.probePeerSshConfig("nvidia@station-b"); + deps.probePeerSshConfig(sshFixture.binding); const [file, args, options] = spawn.mock.calls[0]; expect(file).toBe("ssh"); - expect(args).toEqual(["-G", "-T", "-o", "ConnectTimeout=30", "--", "nvidia@station-b"]); + expect(args).toEqual( + expect.arrayContaining([ + "-G", + "BatchMode=yes", + `UserKnownHostsFile=${sshFixture.binding.knownHostsFile}`, + `HostKeyAlias=${sshFixture.binding.lookupHost}`, + `Hostname=${sshFixture.binding.resolvedHost}`, + "User=nvidia", + "Port=22", + "--", + "nvidia@station-b", + ]), + ); expect(options.input).toBe(""); expect(options.timeout).toBe(20_000); }); @@ -710,7 +796,7 @@ describe("probe command boundary", () => { ); const deps = createStationClusterProbeDeps(spawn); - deps.probePeerHost("nvidia@station-b"); + deps.probePeerHost(sshFixture.binding); const [file, args, options] = spawn.mock.calls[0]; expect(file).toBe("ssh"); @@ -755,7 +841,7 @@ describe("probe command boundary", () => { ); const deps = createStationClusterProbeDeps(spawn); - deps.probePeerHost("nvidia@station-b"); + deps.probePeerHost(sshFixture.binding); const [, , options] = spawn.mock.calls[0]; expect(options.env?.OPENSHELL_GATEWAY_AUTH_TOKEN).toBeUndefined(); @@ -806,7 +892,7 @@ describe("probe command boundary", () => { }, ]; - deps.probePeerConnectivity("station-b", requests); + deps.probePeerConnectivity(sshFixture.binding, requests); const [, args, options] = spawn.mock.calls[0]; expect(args.at(-1)).toBe( diff --git a/src/lib/inference/vllm-station-cluster.ts b/src/lib/inference/vllm-station-cluster.ts index 58749346b1..a7afb37ab0 100644 --- a/src/lib/inference/vllm-station-cluster.ts +++ b/src/lib/inference/vllm-station-cluster.ts @@ -8,13 +8,18 @@ import path from "node:path"; import { buildSubprocessEnv } from "../subprocess-env"; import { buildVllmSshTransportEnv } from "./vllm-docker-env"; import { NEMOTRON_ULTRA_STATION_IMAGE, VLLM_MODELS } from "./vllm-models"; +import { + type DualStationSshBinding, + dualStationPinnedSshArgs, + loadDualStationSshBindingHandoff, + NEMOCLAW_DGX_STATION_SSH_BINDING_ENV, +} from "./vllm-station-ssh-binding"; export const NEMOCLAW_DGX_STATION_PEER_ENV = "NEMOCLAW_DGX_STATION_PEER"; const HOST_PROBE_SCHEMA_VERSION = 1; const CONNECTIVITY_PROBE_SCHEMA_VERSION = 1; const COMMAND_TIMEOUT_MS = 20_000; -const CONNECT_TIMEOUT_SECONDS = 5; const MAX_PROBE_OUTPUT_BYTES = 1024 * 1024; const PREFERRED_ROCE_GID_INDEX = 3; const EXPECTED_ULTRA_WEIGHT_SHARDS = 113; @@ -134,14 +139,15 @@ export interface StationProbeCommandResult { } export interface StationClusterProbeDeps { - probePeerSshConfig(peerTarget: string): StationProbeCommandResult; + loadPeerSshBinding(token: string, expectedPeerTarget: string): DualStationSshBinding; + probePeerSshConfig(binding: DualStationSshBinding): StationProbeCommandResult; probeLocalHost(): StationProbeCommandResult; - probePeerHost(peerTarget: string): StationProbeCommandResult; + probePeerHost(binding: DualStationSshBinding): StationProbeCommandResult; probeLocalConnectivity( requests: readonly StationRailConnectivityRequest[], ): StationProbeCommandResult; probePeerConnectivity( - peerTarget: string, + binding: DualStationSshBinding, requests: readonly StationRailConnectivityRequest[], ): StationProbeCommandResult; } @@ -192,8 +198,7 @@ export interface DualStationPlanRail { } export interface DualStationVllmPlan { - peerSshTarget: string; - peerDockerHost: string; + peerSshBinding: DualStationSshBinding; runtime: typeof DUAL_STATION_VLLM_RUNTIME; local: DualStationPlanNode; peer: DualStationPlanNode; @@ -672,50 +677,8 @@ function sshCommandOptions(input: string): SpawnSyncOptionsWithStringEncoding { }; } -export function strictStationSshTransportArgs(): string[] { - return [ - "-T", - "-o", - "BatchMode=yes", - "-o", - "StrictHostKeyChecking=yes", - "-o", - "NumberOfPasswordPrompts=0", - "-o", - `ConnectTimeout=${String(CONNECT_TIMEOUT_SECONDS)}`, - "-o", - "ConnectionAttempts=1", - "-o", - "ServerAliveInterval=5", - "-o", - "ServerAliveCountMax=1", - "-o", - "ClearAllForwardings=yes", - "-o", - "ForwardAgent=no", - "-o", - "ForwardX11=no", - "-o", - "ForwardX11Trusted=no", - "-o", - "Tunnel=no", - "-o", - "UpdateHostKeys=no", - "-o", - "ControlMaster=no", - "-o", - "ControlPath=none", - "-o", - "PermitLocalCommand=no", - "-o", - "RemoteCommand=none", - "-o", - "LogLevel=ERROR", - ]; -} - -function strictSshArgs(peerTarget: string, remoteCommand: string): string[] { - return [...strictStationSshTransportArgs(), "--", peerTarget, remoteCommand]; +function strictSshArgs(binding: DualStationSshBinding, remoteCommand: string): string[] { + return [...dualStationPinnedSshArgs(binding), "--", binding.peerTarget, remoteCommand]; } function connectivityArgv(requests: readonly StationRailConnectivityRequest[]): string[] { @@ -744,22 +707,19 @@ export function createStationClusterProbeDeps( spawn: StationProbeSpawn = defaultSpawn, ): StationClusterProbeDeps { return { - probePeerSshConfig: (peerTarget) => { - const validated = validatePeerTarget(peerTarget); - if (!validated.ok) throw new Error(validated.reason); + loadPeerSshBinding: loadDualStationSshBindingHandoff, + probePeerSshConfig: (binding) => { return spawn( "ssh", - ["-G", "-T", "-o", "ConnectTimeout=30", "--", validated.target], + ["-G", ...dualStationPinnedSshArgs(binding), "--", binding.peerTarget], sshCommandOptions(""), ); }, probeLocalHost: () => spawn("python3", ["-"], localProbeCommandOptions(HOST_PROBE_SCRIPT)), - probePeerHost: (peerTarget) => { - const validated = validatePeerTarget(peerTarget); - if (!validated.ok) throw new Error(validated.reason); + probePeerHost: (binding) => { return spawn( "ssh", - strictSshArgs(validated.target, "python3 -"), + strictSshArgs(binding, "python3 -"), sshCommandOptions(HOST_PROBE_SCRIPT), ); }, @@ -767,14 +727,12 @@ export function createStationClusterProbeDeps( const args = connectivityArgv(requests); return spawn("python3", ["-", ...args], localProbeCommandOptions(CONNECTIVITY_PROBE_SCRIPT)); }, - probePeerConnectivity: (peerTarget, requests) => { - const validated = validatePeerTarget(peerTarget); - if (!validated.ok) throw new Error(validated.reason); + probePeerConnectivity: (binding, requests) => { const args = connectivityArgv(requests); const remoteCommand = ["python3", "-", ...args].join(" "); return spawn( "ssh", - strictSshArgs(validated.target, remoteCommand), + strictSshArgs(binding, remoteCommand), sshCommandOptions(CONNECTIVITY_PROBE_SCRIPT), ); }, @@ -1084,7 +1042,10 @@ function unavailable(code: StationClusterFailureCode, reason: string): PlanFailu return { kind: "unavailable", code, reason }; } -function dockerSshConfigIsStrict(result: StationProbeCommandResult): boolean { +function dockerSshConfigIsStrict( + result: StationProbeCommandResult, + binding: DualStationSshBinding, +): boolean { if (!commandSucceeded(result)) return false; const values = new Map(); for (const rawLine of result.stdout.split(/\r?\n/)) { @@ -1093,19 +1054,29 @@ function dockerSshConfigIsStrict(result: StationProbeCommandResult): boolean { const separator = line.search(/\s/); if (separator <= 0) return false; const key = line.slice(0, separator).toLowerCase(); - const value = line.slice(separator).trim().toLowerCase(); + const value = line.slice(separator).trim(); values.set(key, [...(values.get(key) ?? []), value]); } const exactly = (key: string, allowed: readonly string[]): boolean => { - const observed = values.get(key) ?? []; + const observed = (values.get(key) ?? []).map((value) => value.toLowerCase()); return observed.length === 1 && allowed.includes(observed[0]); }; - const absentOrNone = (key: string): boolean => { + const exactlyValue = (key: string, expected: string): boolean => { const observed = values.get(key) ?? []; + return observed.length === 1 && observed[0] === expected; + }; + const absentOrNone = (key: string): boolean => { + const observed = (values.get(key) ?? []).map((value) => value.toLowerCase()); return observed.length === 0 || (observed.length === 1 && observed[0] === "none"); }; - const sendEnv = values.get("sendenv") ?? []; + const sendEnv = (values.get("sendenv") ?? []).map((value) => value.toLowerCase()); return ( + exactlyValue("hostname", binding.resolvedHost) && + exactlyValue("user", binding.sshUser) && + exactlyValue("port", String(binding.port)) && + exactlyValue("hostkeyalias", binding.lookupHost) && + exactlyValue("userknownhostsfile", binding.knownHostsFile) && + exactlyValue("globalknownhostsfile", "/dev/null") && exactly("batchmode", ["yes"]) && exactly("stricthostkeychecking", ["yes", "true"]) && exactly("permitlocalcommand", ["no"]) && @@ -1124,7 +1095,7 @@ function dockerSshConfigIsStrict(result: StationProbeCommandResult): boolean { !values.has("localforward") && !values.has("remoteforward") && !values.has("dynamicforward") && - !values.has("knownhostscommand") && + absentOrNone("knownhostscommand") && !values.has("setenv") && sendEnv.every((value) => value === "lang" || value === "lc_*") ); @@ -1302,7 +1273,7 @@ function expectedPeerSnapshotPath(peer: StationHostProbe): string { } function buildStaticPlan( - peerTarget: string, + peerSshBinding: DualStationSshBinding, local: StationHostProbe, peer: StationHostProbe, ): StaticPlan | PlanFailure { @@ -1435,8 +1406,7 @@ function buildStaticPlan( return { plan: { - peerSshTarget: peerTarget, - peerDockerHost: `ssh://${peerTarget}`, + peerSshBinding, runtime: DUAL_STATION_VLLM_RUNTIME, local: { hostname: local.hostname, @@ -1527,7 +1497,7 @@ export interface ProbeDualStationVllmOptions { * Read-only, fail-closed capability probe for the explicit two-Station path. * * An unset/blank NEMOCLAW_DGX_STATION_PEER returns before touching deps. A - * configured peer is contacted exactly by that pretrusted SSH destination; + * configured peer is contacted only through the installer-qualified binding; * the probe never discovers hosts, changes known_hosts, prompts, or mutates * either machine. */ @@ -1541,6 +1511,13 @@ export function probeDualStationVllmCapability( } const peerValidation = validatePeerTarget(rawPeer); if (!peerValidation.ok) return unavailable("invalid-peer", peerValidation.reason); + const rawBinding = env[NEMOCLAW_DGX_STATION_SSH_BINDING_ENV]; + if (rawBinding === undefined || rawBinding === "" || rawBinding.trim() === "") { + return unavailable( + "peer-ssh-config-unsafe", + `${NEMOCLAW_DGX_STATION_SSH_BINDING_ENV} must identify the installer-qualified peer`, + ); + } const localDockerOverride = DUAL_STATION_LOCAL_DOCKER_OVERRIDE_ENV_NAMES.find( (name) => env[name] !== undefined && String(env[name]).trim() !== "", ); @@ -1552,9 +1529,24 @@ export function probeDualStationVllmCapability( } const deps = options.deps ?? defaultStationClusterProbeDeps; + let peerSshBinding: DualStationSshBinding; + try { + peerSshBinding = deps.loadPeerSshBinding(rawBinding, peerValidation.target); + } catch { + return unavailable( + "peer-ssh-config-unsafe", + "installer-qualified Station SSH binding is invalid or changed", + ); + } + if (peerSshBinding.peerTarget !== peerValidation.target) { + return unavailable( + "peer-ssh-config-unsafe", + "qualified Station SSH binding does not match the configured peer", + ); + } try { - const sshConfig = deps.probePeerSshConfig(peerValidation.target); - if (!dockerSshConfigIsStrict(sshConfig)) { + const sshConfig = deps.probePeerSshConfig(peerSshBinding); + if (!dockerSshConfigIsStrict(sshConfig, peerSshBinding)) { return unavailable( "peer-ssh-config-unsafe", "configured peer SSH options are not safe for Docker transport; require BatchMode=yes, StrictHostKeyChecking=yes, no forwarding/proxy/local commands, and no connection sharing", @@ -1562,13 +1554,10 @@ export function probeDualStationVllmCapability( } const localResult = parseHostCommand(deps.probeLocalHost(), "local-probe-failed"); if ("kind" in localResult) return localResult; - const peerResult = parseHostCommand( - deps.probePeerHost(peerValidation.target), - "peer-probe-failed", - ); + const peerResult = parseHostCommand(deps.probePeerHost(peerSshBinding), "peer-probe-failed"); if ("kind" in peerResult) return peerResult; - const staticPlan = buildStaticPlan(peerValidation.target, localResult, peerResult); + const staticPlan = buildStaticPlan(peerSshBinding, localResult, peerResult); if ("kind" in staticPlan) return staticPlan; const localConnectivityResult = deps.probeLocalConnectivity(staticPlan.localConnectivity); @@ -1595,7 +1584,7 @@ export function probeDualStationVllmCapability( } const peerConnectivityResult = deps.probePeerConnectivity( - peerValidation.target, + peerSshBinding, staticPlan.peerConnectivity, ); if (!commandSucceeded(peerConnectivityResult)) { diff --git a/src/lib/inference/vllm-station-model-staging.test.ts b/src/lib/inference/vllm-station-model-staging.test.ts index e6a482dcde..9065f9da39 100644 --- a/src/lib/inference/vllm-station-model-staging.test.ts +++ b/src/lib/inference/vllm-station-model-staging.test.ts @@ -1,18 +1,29 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { afterEach, describe, expect, it, vi } from "vitest"; +import fs from "node:fs"; + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { DUAL_STATION_VLLM_RUNTIME, type DualStationVllmPlan } from "./vllm-station-cluster"; import { type ModelStagingCommandResult, stageDualStationModelSnapshot, } from "./vllm-station-model-staging"; +import { + createDualStationSshBindingFixture, + type DualStationSshBindingFixture, +} from "./vllm-station-ssh-binding.test-support"; + +let sshFixture: DualStationSshBindingFixture; + +beforeEach(() => { + sshFixture = createDualStationSshBindingFixture(); +}); function plan(): DualStationVllmPlan { return { - peerSshTarget: "nvidia@station-b", - peerDockerHost: "ssh://nvidia@station-b", + peerSshBinding: sshFixture.binding, runtime: DUAL_STATION_VLLM_RUNTIME, local: { hostname: "station-a", @@ -102,6 +113,7 @@ function stagingSuffix(runCommand: ReturnType): afterEach(() => { vi.unstubAllEnvs(); + sshFixture.cleanup(); }); describe("dual-Station pinned model staging", () => { @@ -138,6 +150,12 @@ describe("dual-Station pinned model staging", () => { "StrictHostKeyChecking=yes", "ClearAllForwardings=yes", "ControlMaster=no", + `UserKnownHostsFile=${sshFixture.binding.knownHostsFile}`, + "GlobalKnownHostsFile=/dev/null", + `HostKeyAlias=${sshFixture.binding.lookupHost}`, + `Hostname=${sshFixture.binding.resolvedHost}`, + "User=nvidia", + "Port=22", "--", "nvidia@station-b", "python3 -", @@ -187,19 +205,23 @@ describe("dual-Station pinned model staging", () => { const reversedLocal = reversed.local; reversed.local = reversed.peer; reversed.peer = reversedLocal; - reversed.peerSshTarget = "nvidia@station-a"; - reversed.peerDockerHost = "ssh://nvidia@station-a"; + const reversedSshFixture = createDualStationSshBindingFixture("nvidia@station-a"); + reversed.peerSshBinding = reversedSshFixture.binding; const differentHead = plan(); differentHead.local.gpu.uuid = "GPU-c"; const originalRunner = successfulTransferRunner(); const reversedRunner = successfulTransferRunner(); const differentHeadRunner = successfulTransferRunner(); - await Promise.all([ - stageDualStationModelSnapshot(original, { runCommand: originalRunner }), - stageDualStationModelSnapshot(reversed, { runCommand: reversedRunner }), - stageDualStationModelSnapshot(differentHead, { runCommand: differentHeadRunner }), - ]); + try { + await Promise.all([ + stageDualStationModelSnapshot(original, { runCommand: originalRunner }), + stageDualStationModelSnapshot(reversed, { runCommand: reversedRunner }), + stageDualStationModelSnapshot(differentHead, { runCommand: differentHeadRunner }), + ]); + } finally { + reversedSshFixture.cleanup(); + } const suffixes = [ stagingSuffix(originalRunner), @@ -246,6 +268,17 @@ describe("dual-Station pinned model staging", () => { expect(runCommand).not.toHaveBeenCalled(); }); + it("fails before any command when the qualified host-key pin changes", async () => { + fs.appendFileSync(sshFixture.binding.knownHostsFile, "changed\n"); + const runCommand = vi.fn(); + + await expect(stageDualStationModelSnapshot(plan(), { runCommand })).resolves.toMatchObject({ + ok: false, + reason: expect.stringContaining("known-hosts binding changed"), + }); + expect(runCommand).not.toHaveBeenCalled(); + }); + it("fails before SSH when the local manifest is malformed", async () => { const runCommand = vi.fn().mockResolvedValueOnce(result('{"schemaVersion":1}')); diff --git a/src/lib/inference/vllm-station-model-staging.ts b/src/lib/inference/vllm-station-model-staging.ts index c3e9511cd8..53dc0f697e 100644 --- a/src/lib/inference/vllm-station-model-staging.ts +++ b/src/lib/inference/vllm-station-model-staging.ts @@ -6,12 +6,8 @@ import { createHash } from "node:crypto"; import path from "node:path"; import { buildVllmSshTransportEnv } from "./vllm-docker-env"; -import { - DUAL_STATION_VLLM_RUNTIME, - type DualStationVllmPlan, - strictStationSshTransportArgs, - validatePeerTarget, -} from "./vllm-station-cluster"; +import { DUAL_STATION_VLLM_RUNTIME, type DualStationVllmPlan } from "./vllm-station-cluster"; +import { dualStationPinnedSshArgs } from "./vllm-station-ssh-binding"; const MANIFEST_SCHEMA_VERSION = 1; const MAX_MANIFEST_BYTES = 2 * 1024 * 1024; @@ -182,10 +178,7 @@ function stagingPaths(plan: DualStationVllmPlan): StagingPaths { throw new Error(`${label} home is unsafe for model staging`); } } - const validatedPeer = validatePeerTarget(plan.peerSshTarget); - if (!validatedPeer.ok || validatedPeer.target !== plan.peerSshTarget) { - throw new Error("peer SSH target is unsafe for model staging"); - } + dualStationPinnedSshArgs(plan.peerSshBinding); if ( plan.runtime.modelId !== DUAL_STATION_VLLM_RUNTIME.modelId || plan.runtime.modelRevision !== DUAL_STATION_VLLM_RUNTIME.modelRevision || @@ -576,11 +569,20 @@ function parseRemoteState( } function sshArgs(plan: DualStationVllmPlan): string[] { - return [...strictStationSshTransportArgs(), "--", plan.peerSshTarget, "python3 -"]; + return [ + ...dualStationPinnedSshArgs(plan.peerSshBinding), + "--", + plan.peerSshBinding.peerTarget, + "python3 -", + ]; +} + +function shellQuote(value: string): string { + return `'${value.replaceAll("'", `'"'"'`)}'`; } -function rsyncRsh(): string { - return ["ssh", ...strictStationSshTransportArgs()].join(" "); +function rsyncRsh(plan: DualStationVllmPlan): string { + return ["ssh", ...dualStationPinnedSshArgs(plan.peerSshBinding)].map(shellQuote).join(" "); } /** @@ -616,11 +618,16 @@ export async function stageDualStationModelSnapshot( return { ok: false, reason: (err as Error).message }; } - const prepare = await deps.runCommand("ssh", sshArgs(plan), { - env, - input: remoteScript(plan, paths, manifest, "prepare"), - timeoutMs: SNAPSHOT_AUDIT_TIMEOUT_MS, - }); + let prepare: ModelStagingCommandResult; + try { + prepare = await deps.runCommand("ssh", sshArgs(plan), { + env, + input: remoteScript(plan, paths, manifest, "prepare"), + timeoutMs: SNAPSHOT_AUDIT_TIMEOUT_MS, + }); + } catch (err) { + return { ok: false, reason: (err as Error).message }; + } try { if (prepare.status === 0 && JSON.parse(prepare.stdout.trim()).state === "ready") { parseRemoteState("peer snapshot preflight", prepare, "ready"); @@ -631,40 +638,50 @@ export async function stageDualStationModelSnapshot( return { ok: false, reason: (err as Error).message }; } - const rsync = await deps.runCommand( - "rsync", - [ - "--recursive", - "--times", - "--copy-links", - "--checksum", - "--partial", - "--protect-args", - "--no-owner", - "--no-group", - "--chmod=Du=rwx,Dgo=,Fu=rw,Fgo=", - "--info=progress2", - `--rsh=${rsyncRsh()}`, - "--", - `${paths.localSnapshot}/`, - `${plan.peerSshTarget}:${paths.peerStaging}/`, - ], - { - env, - timeoutMs: RSYNC_TIMEOUT_MS, - idleTimeoutMs: RSYNC_IDLE_TIMEOUT_MS, - streamOutput: true, - }, - ); + let rsync: ModelStagingCommandResult; + try { + rsync = await deps.runCommand( + "rsync", + [ + "--recursive", + "--times", + "--copy-links", + "--checksum", + "--partial", + "--protect-args", + "--no-owner", + "--no-group", + "--chmod=Du=rwx,Dgo=,Fu=rw,Fgo=", + "--info=progress2", + `--rsh=${rsyncRsh(plan)}`, + "--", + `${paths.localSnapshot}/`, + `${plan.peerSshBinding.peerTarget}:${paths.peerStaging}/`, + ], + { + env, + timeoutMs: RSYNC_TIMEOUT_MS, + idleTimeoutMs: RSYNC_IDLE_TIMEOUT_MS, + streamOutput: true, + }, + ); + } catch (err) { + return { ok: false, reason: (err as Error).message }; + } if (rsync.status !== 0 || rsync.timedOut || rsync.error) { return { ok: false, reason: commandFailure("peer snapshot transfer", rsync) }; } - const finalize = await deps.runCommand("ssh", sshArgs(plan), { - env, - input: remoteScript(plan, paths, manifest, "finalize"), - timeoutMs: SNAPSHOT_AUDIT_TIMEOUT_MS, - }); + let finalize: ModelStagingCommandResult; + try { + finalize = await deps.runCommand("ssh", sshArgs(plan), { + env, + input: remoteScript(plan, paths, manifest, "finalize"), + timeoutMs: SNAPSHOT_AUDIT_TIMEOUT_MS, + }); + } catch (err) { + return { ok: false, reason: (err as Error).message }; + } try { parseRemoteState("peer snapshot verification", finalize, "ready"); } catch (err) { diff --git a/src/lib/inference/vllm-station-ssh-binding.test-support.ts b/src/lib/inference/vllm-station-ssh-binding.test-support.ts new file mode 100644 index 0000000000..4213d50dcb --- /dev/null +++ b/src/lib/inference/vllm-station-ssh-binding.test-support.ts @@ -0,0 +1,55 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { + type DualStationSshBinding, + encodeDualStationSshBindingHandoff, + type QualifiedStationSshIdentity, + stationKnownHostsDigest, + writeDualStationSshBinding, +} from "./vllm-station-ssh-binding"; + +export interface DualStationSshBindingFixture { + binding: DualStationSshBinding; + dockerCliFile: string; + identity: QualifiedStationSshIdentity; + resumeStatePath: string; + token: string; + cleanup(): void; +} + +export function createDualStationSshBindingFixture( + peerTarget = "nvidia@station-b", +): DualStationSshBindingFixture { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-station-ssh-test-")); + fs.chmodSync(root, 0o700); + const dockerCliFile = path.join(root, "docker-cli"); + fs.writeFileSync(dockerCliFile, "#!/bin/bash\nexit 0\n", { mode: 0o700 }); + fs.chmodSync(dockerCliFile, 0o700); + const knownHostsLines = ["192.168.50.20 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGZpeHR1cmU="]; + const hostKeyDigest = stationKnownHostsDigest(`${knownHostsLines.join("\n")}\n`); + const resumeStatePath = path.join(root, "pair.json"); + const identity: QualifiedStationSshIdentity = { + requestedTarget: peerTarget, + sshTarget: peerTarget, + resolvedHost: "192.168.50.20", + sshUser: peerTarget.includes("@") ? peerTarget.split("@", 1)[0] : "nvidia", + port: 22, + lookupHost: "192.168.50.20", + hostKeyDigest, + knownHostsLines, + }; + const binding = writeDualStationSshBinding(resumeStatePath, identity, { dockerCliFile }); + return { + binding, + dockerCliFile, + identity, + resumeStatePath, + token: encodeDualStationSshBindingHandoff(binding), + cleanup: () => fs.rmSync(root, { recursive: true, force: true }), + }; +} diff --git a/src/lib/inference/vllm-station-ssh-binding.test.ts b/src/lib/inference/vllm-station-ssh-binding.test.ts new file mode 100644 index 0000000000..79fa41f347 --- /dev/null +++ b/src/lib/inference/vllm-station-ssh-binding.test.ts @@ -0,0 +1,403 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +import { + assertDualStationSshBindingFiles, + clearDualStationSshBinding, + type DualStationSshBinding, + dualStationDockerSshUri, + dualStationPinnedSshArgs, + dualStationSshBindingDirectory, + encodeDualStationSshBindingHandoff, + loadDualStationSshBinding, + loadDualStationSshBindingHandoff, + type QualifiedStationSshIdentity, + stationKnownHostsDigest, + strictStationSshTransportArgs, + writeDualStationSshBinding, +} from "./vllm-station-ssh-binding"; + +const PEER_TARGET = "station@10.10.0.2"; +const PEER_HOST = "10.10.0.2"; +const PEER_PORT = 2222; +const ED25519_KEY = "AAAAC3NzaC1lZDI1NTE5AAAAIFirstQualifiedStationKey"; +const RSA_KEY = "AAAAB3NzaC1yc2EAAAADAQABAAABAQCRevokedStationKey"; +const KNOWN_HOSTS_LINES = [ + `[${PEER_HOST}]:${String(PEER_PORT)} ssh-ed25519 ${ED25519_KEY}`, + `@revoked [${PEER_HOST}]:${String(PEER_PORT)} ssh-rsa ${RSA_KEY}`, +] as const; + +function sha256(value: string): string { + return createHash("sha256").update(value).digest("hex"); +} + +function fileMode(filePath: string): number { + return fs.lstatSync(filePath).mode & 0o777; +} + +describe("qualified dual-Station SSH binding", () => { + let dockerCliFile: string; + let root: string; + let resumeStatePath: string; + + beforeEach(() => { + root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-station-ssh-binding-")); + fs.chmodSync(root, 0o700); + dockerCliFile = path.join(root, "docker-cli"); + fs.writeFileSync(dockerCliFile, "#!/bin/bash\nexit 0\n", { mode: 0o700 }); + fs.chmodSync(dockerCliFile, 0o700); + resumeStatePath = path.join(root, "station-dual-pair-resume.json"); + }); + + afterEach(() => { + fs.rmSync(root, { force: true, recursive: true }); + }); + + function identity(overrides: Partial = {}) { + const knownHostsLines = overrides.knownHostsLines ?? KNOWN_HOSTS_LINES; + return { + requestedTarget: PEER_TARGET, + sshTarget: PEER_TARGET, + resolvedHost: PEER_HOST, + sshUser: "station", + port: PEER_PORT, + lookupHost: `[${PEER_HOST}]:${String(PEER_PORT)}`, + hostKeyDigest: stationKnownHostsDigest(knownHostsLines.join("\n")), + knownHostsLines, + ...overrides, + } satisfies QualifiedStationSshIdentity; + } + + function writeBinding( + overrides: Partial = {}, + ): DualStationSshBinding { + return writeDualStationSshBinding(resumeStatePath, identity(overrides), { + dockerCliFile, + }); + } + + it("matches the coordinator digest over sorted unique key identities", () => { + const raw = [ + "# ignored comment", + KNOWN_HOSTS_LINES[1], + KNOWN_HOSTS_LINES[0], + KNOWN_HOSTS_LINES[0], + "", + ].join("\n"); + const expected = sha256( + [`@revoked|ssh-rsa|${RSA_KEY}`, `|ssh-ed25519|${ED25519_KEY}`].sort().join("\n"), + ); + + expect(stationKnownHostsDigest(raw)).toBe(expected); + expect(() => stationKnownHostsDigest(`@revoked ${PEER_HOST} ssh-rsa ${RSA_KEY}`)).toThrow( + "no trusted key", + ); + }); + + it("persists owner-only evidence and reloads one canonical handoff", () => { + const binding = writeBinding(); + const runtimeDirectory = dualStationSshBindingDirectory(resumeStatePath); + const versionDirectory = path.dirname(binding.bindingFile); + const knownHosts = `${[...KNOWN_HOSTS_LINES].sort().join("\n")}\n`; + + expect(binding).toEqual( + expect.objectContaining({ + schemaVersion: 2, + peerTarget: PEER_TARGET, + resolvedHost: PEER_HOST, + sshUser: "station", + port: PEER_PORT, + lookupHost: `[${PEER_HOST}]:${String(PEER_PORT)}`, + hostKeyDigest: stationKnownHostsDigest(knownHosts), + bindingFile: path.join(versionDirectory, "binding.json"), + dockerCliFile: fs.realpathSync(dockerCliFile), + dockerShimFile: path.join(versionDirectory, "bin", "docker"), + knownHostsFile: path.join(versionDirectory, "known_hosts"), + sshWrapperDirectory: path.join(versionDirectory, "bin"), + sshWrapperFile: path.join(versionDirectory, "bin", "ssh"), + }), + ); + expect(path.dirname(versionDirectory)).toBe(runtimeDirectory); + expect(path.basename(versionDirectory)).toMatch(/^v2-[a-f0-9]{32}$/); + expect(fs.readFileSync(binding.knownHostsFile, "utf8")).toBe(knownHosts); + expect(fileMode(runtimeDirectory)).toBe(0o700); + expect(fileMode(versionDirectory)).toBe(0o700); + expect(fileMode(binding.sshWrapperDirectory)).toBe(0o700); + expect(fileMode(binding.bindingFile)).toBe(0o600); + expect(fileMode(binding.dockerShimFile)).toBe(0o700); + expect(fileMode(binding.knownHostsFile)).toBe(0o600); + expect(fileMode(binding.sshWrapperFile)).toBe(0o700); + + const token = encodeDualStationSshBindingHandoff(binding); + expect(loadDualStationSshBindingHandoff(token, PEER_TARGET)).toEqual(binding); + expect( + loadDualStationSshBinding(binding.bindingFile, PEER_TARGET, binding.hostKeyDigest), + ).toEqual(binding); + }); + + it("pins direct SSH and Docker-over-SSH to the qualified endpoint", () => { + const binding = writeBinding(); + const args = dualStationPinnedSshArgs(binding); + + expect(args.slice(0, strictStationSshTransportArgs().length)).toEqual( + strictStationSshTransportArgs(), + ); + expect(args).toEqual( + expect.arrayContaining([ + `UserKnownHostsFile=${binding.knownHostsFile}`, + "GlobalKnownHostsFile=/dev/null", + `HostKeyAlias=${binding.lookupHost}`, + `Hostname=${PEER_HOST}`, + "User=station", + `Port=${String(PEER_PORT)}`, + ]), + ); + expect(dualStationDockerSshUri(binding)).toBe( + `ssh://station@${PEER_HOST}:${String(PEER_PORT)}`, + ); + + const wrapper = fs.readFileSync(binding.sshWrapperFile, "utf8"); + expect(wrapper).toMatch(/^#!\/bin\/bash\n/); + expect(wrapper).toContain(`/usr/bin/sha256sum < "$known_hosts"`); + expect(wrapper).toContain("exec /usr/bin/ssh '-T'"); + expect(wrapper).toContain(`'UserKnownHostsFile=${binding.knownHostsFile}'`); + expect(wrapper).toContain(`'HostKeyAlias=${binding.lookupHost}'`); + expect(wrapper).toContain(`'Hostname=${PEER_HOST}'`); + expect(wrapper).toContain('"$@"'); + expect(spawnSync("/bin/bash", ["-n", binding.sshWrapperFile]).status).toBe(0); + + const dockerShim = fs.readFileSync(binding.dockerShimFile, "utf8"); + expect(dockerShim).toMatch(/^#!\/bin\/bash\n/); + expect(dockerShim).toContain(`exec '${binding.dockerCliFile}' "$@"`); + expect(spawnSync("/bin/bash", ["-n", binding.dockerShimFile]).status).toBe(0); + }); + + it("omits only the default SSH port from the Docker URI", () => { + const binding = writeDualStationSshBinding( + resumeStatePath, + identity({ port: 22, lookupHost: PEER_HOST }), + { dockerCliFile }, + ); + + expect(dualStationDockerSshUri(binding)).toBe(`ssh://station@${PEER_HOST}`); + }); + + it("keeps an earlier accepted host-key set in its immutable version", () => { + const first = writeBinding(); + const replacementKey = "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"; + const replacementLines = [`[${PEER_HOST}]:${String(PEER_PORT)} ssh-ed25519 ${replacementKey}`]; + const second = writeDualStationSshBinding( + resumeStatePath, + identity({ + knownHostsLines: replacementLines, + hostKeyDigest: stationKnownHostsDigest(replacementLines.join("\n")), + }), + { dockerCliFile }, + ); + + expect(second.hostKeyDigest).not.toBe(first.hostKeyDigest); + expect(path.dirname(second.bindingFile)).not.toBe(path.dirname(first.bindingFile)); + expect(() => assertDualStationSshBindingFiles(first)).not.toThrow(); + expect(() => assertDualStationSshBindingFiles(second)).not.toThrow(); + expect(fs.readFileSync(first.knownHostsFile, "utf8")).toContain(ED25519_KEY); + expect(fs.readFileSync(second.knownHostsFile, "utf8")).toContain(replacementKey); + }); + + it.each([ + { + name: "mismatched requested target", + override: { requestedTarget: "station@10.10.0.3" }, + }, + { name: "non-canonical resolved host", override: { resolvedHost: "999.999.999.999" } }, + { name: "unsafe SSH user", override: { sshUser: "station root" } }, + { name: "mismatched explicit SSH user", override: { sshUser: "other-user" } }, + { name: "out-of-range port", override: { port: 65_536 } }, + { name: "mismatched lookup host", override: { lookupHost: PEER_HOST } }, + { name: "mismatched digest", override: { hostKeyDigest: "0".repeat(64) } }, + { name: "comment evidence", override: { knownHostsLines: ["# not evidence"] } }, + { + name: "multiline evidence", + override: { knownHostsLines: [`${KNOWN_HOSTS_LINES[0]}\n${KNOWN_HOSTS_LINES[1]}`] }, + }, + ])("rejects $name before persisting it", ({ override }) => { + expect(() => + writeDualStationSshBinding( + resumeStatePath, + identity(override as Partial), + { dockerCliFile }, + ), + ).toThrow(); + expect(fs.existsSync(dualStationSshBindingDirectory(resumeStatePath))).toBe(false); + }); + + it("rejects a state path that cannot be one Docker helper PATH entry", () => { + const incompatiblePath = path.join(root, `pair${path.delimiter}resume.json`); + + expect(() => + writeDualStationSshBinding(incompatiblePath, identity(), { dockerCliFile }), + ).toThrow("normalized absolute path"); + }); + + it.each([ + ["binding file", (binding: DualStationSshBinding) => binding.bindingFile, 0o644], + ["Docker CLI", (binding: DualStationSshBinding) => binding.dockerCliFile, 0o722], + ["Docker shim", (binding: DualStationSshBinding) => binding.dockerShimFile, 0o755], + ["known-hosts file", (binding: DualStationSshBinding) => binding.knownHostsFile, 0o644], + ["wrapper file", (binding: DualStationSshBinding) => binding.sshWrapperFile, 0o755], + ["wrapper directory", (binding: DualStationSshBinding) => binding.sshWrapperDirectory, 0o755], + [ + "binding version", + (binding: DualStationSshBinding) => path.dirname(binding.bindingFile), + 0o755, + ], + [ + "binding root", + (binding: DualStationSshBinding) => path.dirname(path.dirname(binding.bindingFile)), + 0o755, + ], + ["binding parent", (_binding: DualStationSshBinding) => root, 0o755], + ] as const)("rejects unsafe %s permissions", (_name, target, mode) => { + const binding = writeBinding(); + fs.chmodSync(target(binding), mode); + + expect(() => + loadDualStationSshBinding(binding.bindingFile, PEER_TARGET, binding.hostKeyDigest), + ).toThrow(); + }); + + it.each([ + ["binding file", (binding: DualStationSshBinding) => binding.bindingFile], + ["Docker shim", (binding: DualStationSshBinding) => binding.dockerShimFile], + ["known-hosts file", (binding: DualStationSshBinding) => binding.knownHostsFile], + ["wrapper file", (binding: DualStationSshBinding) => binding.sshWrapperFile], + ] as const)("refuses a symbolic-link %s", (_name, selectedPath) => { + const binding = writeBinding(); + const filePath = selectedPath(binding); + const copyPath = path.join(root, `copy-${path.basename(filePath)}`); + fs.copyFileSync(filePath, copyPath); + fs.chmodSync(copyPath, fileMode(filePath)); + fs.unlinkSync(filePath); + fs.symlinkSync(copyPath, filePath); + + expect(() => + loadDualStationSshBinding(binding.bindingFile, PEER_TARGET, binding.hostKeyDigest), + ).toThrow(); + }); + + it("rejects known-hosts and wrapper tampering before returning transport data", () => { + const binding = writeBinding(); + fs.appendFileSync(binding.knownHostsFile, "# changed\n"); + + expect(() => dualStationPinnedSshArgs(binding)).toThrow( + "known-hosts binding changed after qualification", + ); + expect(() => dualStationDockerSshUri(binding)).toThrow( + "known-hosts binding changed after qualification", + ); + + const restored = writeBinding(); + fs.appendFileSync(restored.sshWrapperFile, "# changed\n"); + expect(() => dualStationPinnedSshArgs(restored)).toThrow("wrapper changed after qualification"); + + const dockerTampered = writeBinding(); + fs.appendFileSync(dockerTampered.dockerShimFile, "# changed\n"); + expect(() => dualStationDockerSshUri(dockerTampered)).toThrow( + "Docker shim changed after qualification", + ); + }); + + it.each([ + ["binding file", (binding: DualStationSshBinding) => binding.bindingFile, 16 * 1024 + 1], + ["known-hosts file", (binding: DualStationSshBinding) => binding.knownHostsFile, 64 * 1024 + 1], + ["wrapper file", (binding: DualStationSshBinding) => binding.sshWrapperFile, 16 * 1024 + 1], + ["Docker shim", (binding: DualStationSshBinding) => binding.dockerShimFile, 16 * 1024 + 1], + ] as const)("rejects an oversized %s", (_name, selectedPath, size) => { + const binding = writeBinding(); + fs.writeFileSync(selectedPath(binding), "x".repeat(size)); + + expect(() => + loadDualStationSshBinding(binding.bindingFile, PEER_TARGET, binding.hostKeyDigest), + ).toThrow(); + }); + + it.each([ + ["unexpected field", (value: Record): void => void (value.extra = true)], + [ + "unsupported schema", + (value: Record): void => void (value.schemaVersion = 1), + ], + [ + "string port", + (value: Record): void => void (value.port = String(PEER_PORT)), + ], + [ + "changed peer", + (value: Record): void => void (value.peerTarget = "station@10.10.0.3"), + ], + ] as const)("rejects a binding JSON %s", (_name, mutate) => { + const binding = writeBinding(); + const value = JSON.parse(fs.readFileSync(binding.bindingFile, "utf8")) as Record< + string, + unknown + >; + mutate(value); + fs.writeFileSync(binding.bindingFile, `${JSON.stringify(value)}\n`); + + expect(() => + loadDualStationSshBinding(binding.bindingFile, PEER_TARGET, binding.hostKeyDigest), + ).toThrow(); + }); + + it("rejects forged handoffs and in-memory endpoint fields", () => { + const binding = writeBinding(); + const token = encodeDualStationSshBindingHandoff(binding); + const extraFieldToken = Buffer.from( + JSON.stringify({ + bindingFile: binding.bindingFile, + hostKeyDigest: binding.hostKeyDigest, + peerTarget: PEER_TARGET, + }), + "utf8", + ).toString("base64url"); + + expect(() => loadDualStationSshBindingHandoff(`${token}=`, PEER_TARGET)).toThrow( + "NEMOCLAW_DGX_STATION_SSH_BINDING is invalid", + ); + expect(() => loadDualStationSshBindingHandoff(extraFieldToken, PEER_TARGET)).toThrow( + "unexpected fields", + ); + expect(() => loadDualStationSshBindingHandoff(token, "station@10.10.0.3")).toThrow( + "does not match the qualified peer identity", + ); + expect(() => + dualStationPinnedSshArgs({ + ...binding, + resolvedHost: "10.10.0.2 ProxyCommand=attacker", + }), + ).toThrow("identity is invalid"); + }); + + it("clears only an owner-only regular binding tree", () => { + writeBinding(); + const runtimeDirectory = dualStationSshBindingDirectory(resumeStatePath); + clearDualStationSshBinding(resumeStatePath); + clearDualStationSshBinding(resumeStatePath); + expect(fs.existsSync(runtimeDirectory)).toBe(false); + + const outside = path.join(root, "outside"); + fs.mkdirSync(outside, { mode: 0o700 }); + const marker = path.join(outside, "keep"); + fs.writeFileSync(marker, "keep", { mode: 0o600 }); + fs.symlinkSync(outside, runtimeDirectory); + expect(() => clearDualStationSshBinding(resumeStatePath)).toThrow("unsafe to remove"); + expect(fs.readFileSync(marker, "utf8")).toBe("keep"); + }); +}); diff --git a/src/lib/inference/vllm-station-ssh-binding.ts b/src/lib/inference/vllm-station-ssh-binding.ts new file mode 100644 index 0000000000..6cdd608aaa --- /dev/null +++ b/src/lib/inference/vllm-station-ssh-binding.ts @@ -0,0 +1,771 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash, randomBytes } from "node:crypto"; +import fs from "node:fs"; +import net from "node:net"; +import path from "node:path"; + +export const NEMOCLAW_DGX_STATION_SSH_BINDING_ENV = "NEMOCLAW_DGX_STATION_SSH_BINDING"; + +const BINDING_SCHEMA_VERSION = 2; +const SHA256_HEX_PATTERN = /^[a-f0-9]{64}$/; +const SSH_USERNAME_PATTERN = /^[A-Za-z_][A-Za-z0-9._-]*$/; +const SSH_HOST_PATTERN = + /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?(?:\.[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?)*$/; +const SSH_KEY_TYPE_PATTERN = /^(?:ssh-|ecdsa-|sk-)[A-Za-z0-9@._+-]+$/; +const SSH_KEY_DATA_PATTERN = /^[A-Za-z0-9+/]+={0,3}$/; +const MAX_KNOWN_HOSTS_LINE_BYTES = 16 * 1024; +const MAX_BINDING_BYTES = 16 * 1024; +const MAX_KNOWN_HOSTS_BYTES = 64 * 1024; +const MAX_WRAPPER_BYTES = 16 * 1024; +const VERSION_DIRECTORY_PATTERN = /^v2-[a-f0-9]{32}$/; +const BINDING_KEYS = [ + "bindingFile", + "dockerCliFile", + "dockerShimFile", + "dockerShimSha256", + "hostKeyDigest", + "knownHostsFile", + "knownHostsSha256", + "lookupHost", + "peerTarget", + "port", + "resolvedHost", + "schemaVersion", + "sshUser", + "sshWrapperDirectory", + "sshWrapperFile", + "sshWrapperSha256", +] as const; + +export interface QualifiedStationSshIdentity { + requestedTarget: string; + sshTarget: string; + resolvedHost: string; + sshUser: string; + port: number; + lookupHost: string; + hostKeyDigest: string; + knownHostsLines: readonly string[]; +} + +export interface DualStationSshBinding { + schemaVersion: 2; + peerTarget: string; + resolvedHost: string; + sshUser: string; + port: number; + lookupHost: string; + hostKeyDigest: string; + bindingFile: string; + dockerCliFile: string; + dockerShimFile: string; + dockerShimSha256: string; + knownHostsFile: string; + knownHostsSha256: string; + sshWrapperDirectory: string; + sshWrapperFile: string; + sshWrapperSha256: string; +} + +interface BindingHandoff { + bindingFile: string; + hostKeyDigest: string; +} + +export interface WriteDualStationSshBindingOptions { + /** Test seam; production resolves the effective Docker CLI from PATH. */ + dockerCliFile?: string; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function requireString(value: unknown, label: string, maxLength: number): string { + if ( + typeof value !== "string" || + value.length === 0 || + value.length > maxLength || + value !== value.trim() || + /[\u0000-\u001f\u007f]/.test(value) + ) { + throw new Error(`${label} is invalid`); + } + return value; +} + +function isCanonicalHost(value: string): boolean { + return net.isIP(value) === 4 || (!/^[0-9.]+$/.test(value) && SSH_HOST_PATTERN.test(value)); +} + +function validatePeerTarget(value: string): string { + requireString(value, "Station SSH peer target", 286); + if (/[/,:;`'"\\$(){}[\]<>|&!?*\s]/.test(value)) { + throw new Error("Station SSH peer target is invalid"); + } + const parts = value.split("@"); + if (parts.length > 2) throw new Error("Station SSH peer target is invalid"); + const user = parts.length === 2 ? parts[0] : ""; + const host = parts.at(-1) ?? ""; + if ((user && !SSH_USERNAME_PATTERN.test(user)) || !isCanonicalHost(host)) { + throw new Error("Station SSH peer target is invalid"); + } + const canonical = user ? `${user}@${host}` : host; + if (canonical !== value) throw new Error("Station SSH peer target is not canonical"); + return canonical; +} + +function explicitPeerUser(peerTarget: string): string | null { + const separator = peerTarget.indexOf("@"); + return separator === -1 ? null : peerTarget.slice(0, separator); +} + +function requireAbsolutePath(value: unknown, label: string): string { + const parsed = requireString(value, label, 4096); + if ( + !path.isAbsolute(parsed) || + path.normalize(parsed) !== parsed || + parsed.includes(path.delimiter) + ) { + throw new Error(`${label} must be a normalized absolute path`); + } + return parsed; +} + +function requireInteger(value: unknown, label: string, minimum: number, maximum: number): number { + if (typeof value !== "number" || !Number.isInteger(value) || value < minimum || value > maximum) { + throw new Error(`${label} is invalid`); + } + return value; +} + +function requireExactKeys( + value: Record, + expected: readonly string[], + label: string, +): void { + const actual = Object.keys(value).sort(); + if (actual.length !== expected.length || actual.some((key, index) => key !== expected[index])) { + throw new Error(`${label} has unexpected fields`); + } +} + +function currentUid(): number { + const uid = process.getuid?.(); + if (uid === undefined) throw new Error("Station SSH binding requires a POSIX user identity"); + return uid; +} + +function assertDirectory(filePath: string, mode: number, label: string): void { + const metadata = fs.lstatSync(filePath); + if ( + metadata.isSymbolicLink() || + !metadata.isDirectory() || + metadata.uid !== currentUid() || + (metadata.mode & 0o777) !== mode + ) { + throw new Error(`${label} must be an owner-only directory`); + } +} + +function readBoundedFile(filePath: string, mode: number, maxBytes: number, label: string): Buffer { + if (typeof fs.constants.O_NOFOLLOW !== "number") { + throw new Error("Station SSH binding requires O_NOFOLLOW support"); + } + const flags = fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW | fs.constants.O_NONBLOCK; + let fd: number; + try { + fd = fs.openSync(filePath, flags); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ELOOP") { + throw new Error(`${label} must not be a symbolic link`); + } + throw error; + } + try { + const metadata = fs.fstatSync(fd); + if ( + !metadata.isFile() || + metadata.uid !== currentUid() || + (metadata.mode & 0o777) !== mode || + metadata.size <= 0 || + metadata.size > maxBytes + ) { + throw new Error(`${label} metadata is invalid`); + } + const content = fs.readFileSync(fd); + if (content.length !== metadata.size || content.length > maxBytes) { + throw new Error(`${label} changed while it was being read`); + } + return content; + } finally { + fs.closeSync(fd); + } +} + +function fsyncDirectory(directory: string): void { + const fd = fs.openSync(directory, fs.constants.O_RDONLY); + try { + fs.fsyncSync(fd); + } finally { + fs.closeSync(fd); + } +} + +function sha256(value: Buffer | string): string { + return createHash("sha256").update(value).digest("hex"); +} + +function validateDockerCliFile(filePath: string): string { + const requested = requireAbsolutePath(filePath, "Station Docker CLI"); + let canonical: string; + try { + canonical = fs.realpathSync(requested); + } catch { + throw new Error("Station Docker CLI could not be resolved"); + } + requireAbsolutePath(canonical, "Station Docker CLI"); + const metadata = fs.lstatSync(canonical); + const uid = currentUid(); + if ( + metadata.isSymbolicLink() || + !metadata.isFile() || + (metadata.uid !== 0 && metadata.uid !== uid) || + (metadata.mode & 0o111) === 0 || + (metadata.mode & 0o022) !== 0 || + metadata.size <= 0 + ) { + throw new Error("Station Docker CLI is not a safe executable file"); + } + return canonical; +} + +function resolveDockerCliFile(explicit?: string): string { + if (explicit !== undefined) return validateDockerCliFile(explicit); + for (const directory of (process.env.PATH ?? "").split(path.delimiter)) { + if (!directory || !path.isAbsolute(directory) || path.normalize(directory) !== directory) { + continue; + } + const candidate = path.join(directory, "docker"); + try { + return validateDockerCliFile(candidate); + } catch { + // Continue through the effective PATH until one safe Docker CLI is found. + } + } + throw new Error("Station Docker CLI could not be resolved to a safe absolute executable"); +} + +export function stationKnownHostsDigest(raw: string): string { + const keys = new Set(); + let positiveKeys = 0; + for (const rawLine of raw.split(/\r?\n/)) { + const line = rawLine.trim(); + if (!line || line.startsWith("#") || /[\u0000\r\n]/.test(line)) continue; + const fields = line.split(/\s+/); + const marker = fields[0]?.startsWith("@") ? (fields.shift() ?? "") : ""; + if (fields.length < 3) throw new Error("Pinned Station known-hosts data is invalid"); + const [_hosts, keyType, keyData] = fields; + if (!SSH_KEY_TYPE_PATTERN.test(keyType) || !SSH_KEY_DATA_PATTERN.test(keyData)) { + throw new Error("Pinned Station known-hosts key is invalid"); + } + keys.add(`${marker}|${keyType}|${keyData}`); + if (marker !== "@revoked") positiveKeys += 1; + } + if (keys.size === 0 || positiveKeys === 0) { + throw new Error("Pinned Station known-hosts data has no trusted key"); + } + return sha256([...keys].sort().join("\n")); +} + +function shellQuote(value: string): string { + return `'${value.replaceAll("'", `'"'"'`)}'`; +} + +export function strictStationSshTransportArgs(): string[] { + return [ + "-T", + "-o", + "BatchMode=yes", + "-o", + "StrictHostKeyChecking=yes", + "-o", + "VerifyHostKeyDNS=no", + "-o", + "NoHostAuthenticationForLocalhost=no", + "-o", + "NumberOfPasswordPrompts=0", + "-o", + "PasswordAuthentication=no", + "-o", + "KbdInteractiveAuthentication=no", + "-o", + "PreferredAuthentications=publickey", + "-o", + "ConnectTimeout=5", + "-o", + "ConnectionAttempts=1", + "-o", + "ServerAliveInterval=5", + "-o", + "ServerAliveCountMax=1", + "-o", + "ClearAllForwardings=yes", + "-o", + "ForwardAgent=no", + "-o", + "ForwardX11=no", + "-o", + "ForwardX11Trusted=no", + "-o", + "Tunnel=no", + "-o", + "UpdateHostKeys=no", + "-o", + "ControlMaster=no", + "-o", + "ControlPath=none", + "-o", + "PermitLocalCommand=no", + "-o", + "RemoteCommand=none", + "-o", + "ProxyCommand=none", + "-o", + "ProxyJump=none", + "-o", + "KnownHostsCommand=none", + "-o", + "LogLevel=ERROR", + ]; +} + +type PinnedStationEndpoint = Pick< + DualStationSshBinding, + "knownHostsFile" | "lookupHost" | "port" | "resolvedHost" | "sshUser" +>; + +function pinnedOptionArgs(binding: PinnedStationEndpoint): string[] { + return [ + ...strictStationSshTransportArgs(), + "-o", + `UserKnownHostsFile=${binding.knownHostsFile}`, + "-o", + "GlobalKnownHostsFile=/dev/null", + "-o", + `HostKeyAlias=${binding.lookupHost}`, + "-o", + `Hostname=${binding.resolvedHost}`, + "-o", + `User=${binding.sshUser}`, + "-o", + `Port=${String(binding.port)}`, + ]; +} + +function renderSshWrapper(binding: PinnedStationEndpoint & { knownHostsSha256: string }): string { + const args = pinnedOptionArgs(binding).map(shellQuote).join(" "); + return `#!/bin/bash +set -Eeuo pipefail +readonly known_hosts=${shellQuote(binding.knownHostsFile)} +readonly expected_sha256=${shellQuote(binding.knownHostsSha256)} +actual_sha256="$(/usr/bin/sha256sum < "$known_hosts")" || exit 255 +actual_sha256="${"${actual_sha256%% *}"}" +if [[ "$actual_sha256" != "$expected_sha256" ]]; then + printf '%s\n' 'NemoClaw refused a changed dual-Station SSH host-key pin.' >&2 + exit 255 +fi +exec /usr/bin/ssh ${args} "$@" +`; +} + +function renderDockerShim(binding: Pick): string { + return `#!/bin/bash +set -Eeuo pipefail +exec ${shellQuote(binding.dockerCliFile)} "$@" +`; +} + +function writeExclusive(filePath: string, content: string, mode: number): void { + if (typeof fs.constants.O_NOFOLLOW !== "number") { + throw new Error("Station SSH binding requires O_NOFOLLOW support"); + } + const flags = + fs.constants.O_WRONLY | fs.constants.O_CREAT | fs.constants.O_EXCL | fs.constants.O_NOFOLLOW; + const fd = fs.openSync(filePath, flags, mode); + try { + fs.writeFileSync(fd, content, "utf8"); + fs.fsyncSync(fd); + fs.fchmodSync(fd, mode); + } finally { + fs.closeSync(fd); + } +} + +function assertRemovableTree(root: string): void { + const metadata = fs.lstatSync(root); + if ( + metadata.isSymbolicLink() || + !metadata.isDirectory() || + metadata.uid !== currentUid() || + (metadata.mode & 0o777) !== 0o700 + ) { + throw new Error("Station SSH binding directory is unsafe to remove"); + } + for (const entry of fs.readdirSync(root, { withFileTypes: true })) { + const child = path.join(root, entry.name); + const childMetadata = fs.lstatSync(child); + if (childMetadata.isSymbolicLink() || childMetadata.uid !== currentUid()) { + throw new Error("Station SSH binding tree contains an unsafe entry"); + } + if (childMetadata.isDirectory()) assertRemovableTree(child); + else if (!childMetadata.isFile()) { + throw new Error("Station SSH binding tree contains a non-file entry"); + } + } +} + +export function dualStationSshBindingDirectory(resumeStatePath: string): string { + const statePath = requireAbsolutePath(resumeStatePath, "Dual-Station resume-state path"); + return `${statePath}.ssh-binding`; +} + +export function clearDualStationSshBinding(resumeStatePath: string): void { + const runtimeDirectory = dualStationSshBindingDirectory(resumeStatePath); + try { + fs.lstatSync(runtimeDirectory); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return; + throw error; + } + assertRemovableTree(runtimeDirectory); + fs.rmSync(runtimeDirectory, { recursive: true, force: false }); + fsyncDirectory(path.dirname(runtimeDirectory)); +} + +function ensureBindingRoot(runtimeDirectory: string, parent: string): void { + try { + fs.mkdirSync(runtimeDirectory, { mode: 0o700 }); + fs.chmodSync(runtimeDirectory, 0o700); + fsyncDirectory(parent); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; + } + assertDirectory(runtimeDirectory, 0o700, "Station SSH binding root"); +} + +export function writeDualStationSshBinding( + resumeStatePath: string, + identity: QualifiedStationSshIdentity, + options: WriteDualStationSshBindingOptions = {}, +): DualStationSshBinding { + const runtimeDirectory = dualStationSshBindingDirectory(resumeStatePath); + const parent = path.dirname(runtimeDirectory); + assertDirectory(parent, 0o700, "Station SSH binding parent"); + const peerTarget = validatePeerTarget(identity.sshTarget); + if (identity.requestedTarget !== peerTarget || !isCanonicalHost(identity.resolvedHost)) { + throw new Error("Qualified Station SSH endpoint is invalid"); + } + if (!SSH_USERNAME_PATTERN.test(identity.sshUser)) { + throw new Error("Qualified Station SSH user is invalid"); + } + const requestedUser = explicitPeerUser(peerTarget); + if (requestedUser !== null && requestedUser !== identity.sshUser) { + throw new Error("Qualified Station SSH target and resolved user do not match"); + } + if (!Number.isInteger(identity.port) || identity.port < 1 || identity.port > 65535) { + throw new Error("Qualified Station SSH port is invalid"); + } + const expectedLookup = + identity.port === 22 + ? identity.resolvedHost + : `[${identity.resolvedHost}]:${String(identity.port)}`; + if (identity.lookupHost !== expectedLookup || !SHA256_HEX_PATTERN.test(identity.hostKeyDigest)) { + throw new Error("Qualified Station SSH host-key identity is invalid"); + } + if ( + !Array.isArray(identity.knownHostsLines) || + identity.knownHostsLines.length === 0 || + identity.knownHostsLines.some( + (line) => + typeof line !== "string" || + line.length === 0 || + Buffer.byteLength(line, "utf8") > MAX_KNOWN_HOSTS_LINE_BYTES || + line !== line.trim() || + line.startsWith("#") || + /[\u0000\r\n]/.test(line), + ) + ) { + throw new Error("Qualified Station known-hosts evidence is invalid"); + } + const knownHosts = `${[...new Set(identity.knownHostsLines)].sort().join("\n")}\n`; + if (Buffer.byteLength(knownHosts, "utf8") > MAX_KNOWN_HOSTS_BYTES) { + throw new Error("Qualified Station known-hosts evidence is too large"); + } + if (stationKnownHostsDigest(knownHosts) !== identity.hostKeyDigest) { + throw new Error("Qualified Station known-hosts evidence does not match its accepted digest"); + } + + const dockerCliFile = resolveDockerCliFile(options.dockerCliFile); + ensureBindingRoot(runtimeDirectory, parent); + const versionDirectory = path.join(runtimeDirectory, `v2-${randomBytes(16).toString("hex")}`); + fs.mkdirSync(versionDirectory, { mode: 0o700 }); + fs.chmodSync(versionDirectory, 0o700); + try { + const wrapperDirectory = path.join(versionDirectory, "bin"); + fs.mkdirSync(wrapperDirectory, { mode: 0o700 }); + const finalKnownHostsFile = path.join(versionDirectory, "known_hosts"); + const finalBindingFile = path.join(versionDirectory, "binding.json"); + const finalWrapperDirectory = path.join(versionDirectory, "bin"); + const finalWrapperFile = path.join(finalWrapperDirectory, "ssh"); + const finalDockerShimFile = path.join(finalWrapperDirectory, "docker"); + const knownHostsSha256 = sha256(knownHosts); + const persisted = { + schemaVersion: 2, + peerTarget, + resolvedHost: identity.resolvedHost, + sshUser: identity.sshUser, + port: identity.port, + lookupHost: identity.lookupHost, + hostKeyDigest: identity.hostKeyDigest, + bindingFile: finalBindingFile, + dockerCliFile, + dockerShimFile: finalDockerShimFile, + knownHostsFile: finalKnownHostsFile, + knownHostsSha256, + sshWrapperDirectory: finalWrapperDirectory, + sshWrapperFile: finalWrapperFile, + } satisfies Omit; + const wrapper = renderSshWrapper(persisted); + const dockerShim = renderDockerShim(persisted); + const binding: DualStationSshBinding = { + ...persisted, + dockerShimSha256: sha256(dockerShim), + sshWrapperSha256: sha256(wrapper), + }; + writeExclusive(path.join(versionDirectory, "known_hosts"), knownHosts, 0o600); + writeExclusive(path.join(wrapperDirectory, "ssh"), wrapper, 0o700); + writeExclusive(path.join(wrapperDirectory, "docker"), dockerShim, 0o700); + writeExclusive( + path.join(versionDirectory, "binding.json"), + `${JSON.stringify(binding)}\n`, + 0o600, + ); + fsyncDirectory(wrapperDirectory); + fsyncDirectory(versionDirectory); + fsyncDirectory(runtimeDirectory); + return loadDualStationSshBinding(binding.bindingFile, peerTarget, identity.hostKeyDigest); + } catch (error) { + try { + fs.rmSync(versionDirectory, { recursive: true, force: true }); + fsyncDirectory(runtimeDirectory); + } catch { + // Preserve the original validation or persistence failure. + } + throw error; + } +} + +function parseBinding(value: unknown): DualStationSshBinding { + if (!isRecord(value) || value.schemaVersion !== BINDING_SCHEMA_VERSION) { + throw new Error("Station SSH binding schema is unsupported"); + } + requireExactKeys(value, BINDING_KEYS, "Station SSH binding"); + const binding: DualStationSshBinding = { + schemaVersion: 2, + peerTarget: validatePeerTarget(requireString(value.peerTarget, "Station SSH peer", 286)), + resolvedHost: requireString(value.resolvedHost, "Station SSH resolved host", 253), + sshUser: requireString(value.sshUser, "Station SSH user", 64), + port: requireInteger(value.port, "Station SSH port", 1, 65535), + lookupHost: requireString(value.lookupHost, "Station SSH lookup host", 300), + hostKeyDigest: requireString(value.hostKeyDigest, "Station SSH host-key digest", 64), + bindingFile: requireAbsolutePath(value.bindingFile, "Station SSH binding file"), + dockerCliFile: requireAbsolutePath(value.dockerCliFile, "Station Docker CLI"), + dockerShimFile: requireAbsolutePath(value.dockerShimFile, "Station Docker shim"), + dockerShimSha256: requireString(value.dockerShimSha256, "Station Docker shim SHA-256", 64), + knownHostsFile: requireAbsolutePath(value.knownHostsFile, "Station SSH known-hosts file"), + knownHostsSha256: requireString(value.knownHostsSha256, "Station SSH known-hosts SHA-256", 64), + sshWrapperDirectory: requireAbsolutePath( + value.sshWrapperDirectory, + "Station SSH wrapper directory", + ), + sshWrapperFile: requireAbsolutePath(value.sshWrapperFile, "Station SSH wrapper file"), + sshWrapperSha256: requireString(value.sshWrapperSha256, "Station SSH wrapper SHA-256", 64), + }; + if ( + !isCanonicalHost(binding.resolvedHost) || + !SSH_USERNAME_PATTERN.test(binding.sshUser) || + (explicitPeerUser(binding.peerTarget) !== null && + explicitPeerUser(binding.peerTarget) !== binding.sshUser) || + !SHA256_HEX_PATTERN.test(binding.hostKeyDigest) || + !SHA256_HEX_PATTERN.test(binding.dockerShimSha256) || + !SHA256_HEX_PATTERN.test(binding.knownHostsSha256) || + !SHA256_HEX_PATTERN.test(binding.sshWrapperSha256) + ) { + throw new Error("Station SSH binding identity is invalid"); + } + const expectedLookup = + binding.port === 22 + ? binding.resolvedHost + : `[${binding.resolvedHost}]:${String(binding.port)}`; + if (binding.lookupHost !== expectedLookup) { + throw new Error("Station SSH binding lookup host is invalid"); + } + return binding; +} + +function canonicalBinding(value: unknown): DualStationSshBinding { + return parseBinding(value); +} + +function validateDualStationSshBindingFiles(binding: DualStationSshBinding): DualStationSshBinding { + const canonical = canonicalBinding(binding); + const versionDirectory = path.dirname(canonical.bindingFile); + const runtimeDirectory = path.dirname(versionDirectory); + if ( + !path.basename(runtimeDirectory).endsWith(".ssh-binding") || + !VERSION_DIRECTORY_PATTERN.test(path.basename(versionDirectory)) || + canonical.bindingFile !== path.join(versionDirectory, "binding.json") || + canonical.knownHostsFile !== path.join(versionDirectory, "known_hosts") || + canonical.sshWrapperDirectory !== path.join(versionDirectory, "bin") || + canonical.sshWrapperFile !== path.join(versionDirectory, "bin", "ssh") || + canonical.dockerShimFile !== path.join(versionDirectory, "bin", "docker") + ) { + throw new Error("Station SSH binding paths are inconsistent"); + } + assertDirectory(path.dirname(runtimeDirectory), 0o700, "Station SSH binding parent"); + assertDirectory(runtimeDirectory, 0o700, "Station SSH binding root"); + assertDirectory(versionDirectory, 0o700, "Station SSH binding version"); + assertDirectory(canonical.sshWrapperDirectory, 0o700, "Station SSH wrapper directory"); + if (validateDockerCliFile(canonical.dockerCliFile) !== canonical.dockerCliFile) { + throw new Error("Station Docker CLI changed after qualification"); + } + const dockerShim = readBoundedFile( + canonical.dockerShimFile, + 0o700, + MAX_WRAPPER_BYTES, + "Station Docker shim", + ); + if ( + sha256(dockerShim) !== canonical.dockerShimSha256 || + dockerShim.toString("utf8") !== renderDockerShim(canonical) + ) { + throw new Error("Station Docker shim changed after qualification"); + } + const knownHosts = readBoundedFile( + canonical.knownHostsFile, + 0o600, + MAX_KNOWN_HOSTS_BYTES, + "Station SSH known-hosts file", + ); + if ( + sha256(knownHosts) !== canonical.knownHostsSha256 || + stationKnownHostsDigest(knownHosts.toString("utf8")) !== canonical.hostKeyDigest + ) { + throw new Error("Station SSH known-hosts binding changed after qualification"); + } + const wrapper = readBoundedFile( + canonical.sshWrapperFile, + 0o700, + MAX_WRAPPER_BYTES, + "Station SSH wrapper", + ); + if ( + sha256(wrapper) !== canonical.sshWrapperSha256 || + wrapper.toString("utf8") !== renderSshWrapper(canonical) + ) { + throw new Error("Station SSH wrapper changed after qualification"); + } + return canonical; +} + +export function assertDualStationSshBindingFiles(binding: DualStationSshBinding): void { + validateDualStationSshBindingFiles(binding); +} + +export function loadDualStationSshBinding( + bindingFile: string, + expectedPeerTarget: string, + expectedHostKeyDigest: string, +): DualStationSshBinding { + const expectedPeer = validatePeerTarget(expectedPeerTarget); + if (!SHA256_HEX_PATTERN.test(expectedHostKeyDigest)) { + throw new Error("Expected Station SSH host-key digest is invalid"); + } + const normalizedBindingFile = requireAbsolutePath(bindingFile, "Station SSH binding file"); + const raw = readBoundedFile( + normalizedBindingFile, + 0o600, + MAX_BINDING_BYTES, + "Station SSH binding file", + ); + let value: unknown; + try { + value = JSON.parse(raw.toString("utf8")); + } catch { + throw new Error("Station SSH binding file contains invalid JSON"); + } + const binding = parseBinding(value); + if ( + binding.bindingFile !== normalizedBindingFile || + binding.peerTarget !== expectedPeer || + binding.hostKeyDigest !== expectedHostKeyDigest + ) { + throw new Error("Station SSH binding does not match the qualified peer identity"); + } + return validateDualStationSshBindingFiles(binding); +} + +export function encodeDualStationSshBindingHandoff(binding: DualStationSshBinding): string { + const canonical = validateDualStationSshBindingFiles(binding); + const handoff: BindingHandoff = { + bindingFile: canonical.bindingFile, + hostKeyDigest: canonical.hostKeyDigest, + }; + return Buffer.from(JSON.stringify(handoff), "utf8").toString("base64url"); +} + +export function loadDualStationSshBindingHandoff( + token: string, + expectedPeerTarget: string, +): DualStationSshBinding { + if ( + typeof token !== "string" || + token.length === 0 || + token.length > 8192 || + !/^[A-Za-z0-9_-]+$/.test(token) + ) { + throw new Error(`${NEMOCLAW_DGX_STATION_SSH_BINDING_ENV} is invalid`); + } + let value: unknown; + try { + const decoded = Buffer.from(token, "base64url"); + if (decoded.toString("base64url") !== token) throw new Error("non-canonical base64url"); + value = JSON.parse(decoded.toString("utf8")); + } catch { + throw new Error(`${NEMOCLAW_DGX_STATION_SSH_BINDING_ENV} is invalid`); + } + if (!isRecord(value)) { + throw new Error(`${NEMOCLAW_DGX_STATION_SSH_BINDING_ENV} is invalid`); + } + requireExactKeys(value, ["bindingFile", "hostKeyDigest"], NEMOCLAW_DGX_STATION_SSH_BINDING_ENV); + const bindingFile = requireAbsolutePath(value.bindingFile, "Station SSH binding handoff file"); + const hostKeyDigest = requireString( + value.hostKeyDigest, + "Station SSH binding handoff digest", + 64, + ); + return loadDualStationSshBinding(bindingFile, expectedPeerTarget, hostKeyDigest); +} + +export function dualStationPinnedSshArgs(binding: DualStationSshBinding): string[] { + return pinnedOptionArgs(validateDualStationSshBindingFiles(binding)); +} + +export function dualStationDockerSshUri(binding: DualStationSshBinding): string { + const canonical = validateDualStationSshBindingFiles(binding); + const port = canonical.port === 22 ? "" : `:${String(canonical.port)}`; + return `ssh://${canonical.sshUser}@${canonical.resolvedHost}${port}`; +} diff --git a/src/lib/inference/vllm.ts b/src/lib/inference/vllm.ts index 6d47ee977c..1bf03bd3c0 100644 --- a/src/lib/inference/vllm.ts +++ b/src/lib/inference/vllm.ts @@ -1308,7 +1308,7 @@ export async function installVllm( if (dualStationPlan) { let peerDockerEnv: Record; try { - peerDockerEnv = buildRemoteVllmDockerEnv(dualStationPlan.peerDockerHost); + peerDockerEnv = buildRemoteVllmDockerEnv(dualStationPlan.peerSshBinding); } catch (err) { console.error(` vLLM install failed: ${(err as Error).message}`); return { ok: false }; diff --git a/test/install-station-pair-preparation.test.ts b/test/install-station-pair-preparation.test.ts index cc7119d300..9864d510d1 100644 --- a/test/install-station-pair-preparation.test.ts +++ b/test/install-station-pair-preparation.test.ts @@ -27,6 +27,10 @@ import { strictStationPrepSshTransportArgs, writeDualStationResumeState, } from "../scripts/prepare-dual-dgx-station.mts"; +import { + stationKnownHostsDigest, + strictStationSshTransportArgs, +} from "../src/lib/inference/vllm-station-ssh-binding.ts"; import { INSTALLER_PAYLOAD, TEST_SYSTEM_PATH } from "./helpers/installer-sourced-env"; const REPO_ROOT = path.resolve(import.meta.dirname, ".."); @@ -34,7 +38,8 @@ const COORDINATOR = path.join(REPO_ROOT, "scripts", "prepare-dual-dgx-station.mt const STATION_HELPER = path.join(REPO_ROOT, "scripts", "prepare-dgx-station-host.sh"); const REVISION = "a".repeat(40); const HELPER_SHA256 = "b".repeat(64); -const HOST_KEY_DIGEST = "c".repeat(64); +const HOST_KEY_DATA = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; +const HOST_KEY_DIGEST = stationKnownHostsDigest(`10.10.0.2 ssh-ed25519 ${HOST_KEY_DATA}\n`); const HOST_KEY_FINGERPRINT = `SHA256:${"A".repeat(43)}`; function stationHost(side: "local" | "peer"): StationDiscoveryHost { @@ -78,7 +83,8 @@ function stationHost(side: "local" | "peer"): StationDiscoveryHost { }; } -function sshBinding(target = "10.10.0.2", hostKeyDigest = HOST_KEY_DIGEST): PretrustedSshTarget { +function sshBinding(target = "10.10.0.2", keyData = HOST_KEY_DATA): PretrustedSshTarget { + const knownHostsLine = `${target.slice(target.lastIndexOf("@") + 1)} ssh-ed25519 ${keyData}`; return { requestedTarget: target, sshTarget: target, @@ -86,11 +92,9 @@ function sshBinding(target = "10.10.0.2", hostKeyDigest = HOST_KEY_DIGEST): Pret sshUser: "ubuntu", port: 22, lookupHost: target.slice(target.lastIndexOf("@") + 1), - hostKeyDigest, + hostKeyDigest: stationKnownHostsDigest(`${knownHostsLine}\n`), keyFingerprints: [HOST_KEY_FINGERPRINT], - knownHostsLines: [ - `${target.slice(target.lastIndexOf("@") + 1)} ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAITestKey`, - ], + knownHostsLines: [knownHostsLine], }; } @@ -220,6 +224,13 @@ function runInstallerBody(body: string, extraEnv: Record = {}) { function coordinatorResult(kind: "ready" | "reboot-required", peer = "10.10.0.2"): string { const state = readyState(); state.peerTarget = peer; + const sshBinding = Buffer.from( + JSON.stringify({ + bindingFile: "/tmp/nemoclaw-station-pair/resume.json.ssh-binding/binding.json", + hostKeyDigest: state.hostKeyDigest, + }), + "utf8", + ).toString("base64url"); return JSON.stringify({ kind, peerTarget: peer, @@ -230,6 +241,7 @@ function coordinatorResult(kind: "ready" | "reboot-required", peer = "10.10.0.2" peerGpuUuid: state.peerGpuUuid, rails: state.rails, }, + sshBinding, }); } @@ -302,6 +314,7 @@ describe("deterministic dual-DGX Station peer discovery", () => { expect(result.kind).toBe("ready"); expect(result.kind === "ready" && result.peerTarget).toBe("10.10.0.2"); + expect(result.kind === "ready" && result.binding).toEqual(sshBinding()); expect(harness.statePhases).toEqual(["remote-preparation", "ready"]); expect(harness.calls.indexOf("local:--verify")).toBeLessThan( harness.calls.indexOf("probe:peer:10.10.0.2"), @@ -325,7 +338,7 @@ describe("deterministic dual-DGX Station peer discovery", () => { const ambiguous = new PreparationHarness(); ambiguous.trusted.set("10.10.0.2", sshBinding("10.10.0.2")); - ambiguous.trusted.set("10.10.0.6", sshBinding("10.10.0.6", "d".repeat(64))); + ambiguous.trusted.set("10.10.0.6", sshBinding("10.10.0.6", "AAAAC3NzaChangedKey")); const result = prepareDualStationPair(preparationOptions(), ambiguous.deps); expect(result).toMatchObject({ kind: "single-station", @@ -347,6 +360,93 @@ describe("deterministic dual-DGX Station peer discovery", () => { expect(harness.calls.some((call) => call.startsWith("probe:peer:"))).toBe(false); }); + it("rejects altered endpoint, port, and known-hosts evidence before peer contact", () => { + const oversizedLines = Array.from( + { length: 5 }, + (_, index) => `10.10.0.${String(index + 2)} ssh-ed25519 A${"B".repeat(14_000)}`, + ); + const scenarios: Array<{ + name: string; + mutate(binding: PretrustedSshTarget): void; + }> = [ + { + name: "requested target substitution", + mutate: (binding) => { + binding.requestedTarget = "10.10.0.6"; + }, + }, + { + name: "fractional port", + mutate: (binding) => { + binding.port = 22.5; + }, + }, + { + name: "blank known-hosts line", + mutate: (binding) => { + binding.knownHostsLines = [""]; + }, + }, + { + name: "untrimmed known-hosts line", + mutate: (binding) => { + binding.knownHostsLines = [`${binding.knownHostsLines[0]} `]; + }, + }, + { + name: "comment known-hosts line", + mutate: (binding) => { + binding.knownHostsLines = ["# no trust evidence"]; + }, + }, + { + name: "oversized known-hosts evidence", + mutate: (binding) => { + binding.knownHostsLines = oversizedLines; + }, + }, + { + name: "known-hosts digest mismatch", + mutate: (binding) => { + binding.knownHostsLines = ["10.10.0.2 ssh-ed25519 AAAAC3NzaChangedKey"]; + }, + }, + ]; + + for (const scenario of scenarios) { + const harness = new PreparationHarness(); + const binding = sshBinding(); + scenario.mutate(binding); + harness.trusted.set("10.10.0.2", binding); + expect( + prepareDualStationPair(preparationOptions(), harness.deps), + scenario.name, + ).toMatchObject({ kind: "single-station" }); + expect( + harness.calls.some((call) => call.startsWith("probe:peer:")), + scenario.name, + ).toBe(false); + expect( + harness.calls.some((call) => call.startsWith("remote:")), + scenario.name, + ).toBe(false); + } + }); + + it("rejects an explicit target whose resolved SSH user changed before peer contact", () => { + const explicit = "ubuntu@station-b"; + const harness = new PreparationHarness(); + const binding = sshBinding(explicit); + binding.sshUser = "root"; + harness.trusted.set(explicit, binding); + + expect(() => + prepareDualStationPair({ ...preparationOptions(), explicitPeer: explicit }, harness.deps), + ).toThrow(/unsafe user or port/); + expect(harness.calls.some((call) => call.startsWith("probe:peer:"))).toBe(false); + expect(harness.calls.some((call) => call.startsWith("remote:"))).toBe(false); + }); + it("requires reciprocal rail addresses and MACs plus a distinct peer GPU", () => { const nonreciprocal = new PreparationHarness(); trustFirstRail(nonreciprocal); @@ -443,6 +543,7 @@ describe("dual-DGX Station reboot resume and reuse", () => { const interrupted = prepareDualStationPair(preparationOptions(), first.deps); expect(interrupted.kind).toBe("reboot-required"); + expect(interrupted.kind === "reboot-required" && interrupted.binding).toEqual(sshBinding()); expect(first.resume?.phase).toBe("remote-reboot-required"); expect(first.resume?.helperSha256).toBe(HELPER_SHA256); expect(first.calls.some((call) => call.endsWith(":--verify"))).toBe(true); @@ -482,7 +583,7 @@ describe("dual-DGX Station reboot resume and reuse", () => { { name: "host key", configure: (harness) => { - harness.trusted.set("10.10.0.2", sshBinding("10.10.0.2", "d".repeat(64))); + harness.trusted.set("10.10.0.2", sshBinding("10.10.0.2", "AAAAC3NzaChangedKey")); }, expected: /host-key identity changed/, }, @@ -593,6 +694,21 @@ describe.sequential("dual-DGX Station trust and resume-state boundaries", () => } }); + it("clears an owner-only SSH binding orphan even when pair state is absent", () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-pair-state-")); + fs.chmodSync(directory, 0o700); + const statePath = path.join(directory, "resume.json"); + const bindingDirectory = `${statePath}.ssh-binding`; + try { + fs.mkdirSync(bindingDirectory, { mode: 0o700 }); + fs.writeFileSync(path.join(bindingDirectory, "orphan"), "binding\n", { mode: 0o600 }); + clearDualStationResumeState(statePath); + expect(fs.existsSync(bindingDirectory)).toBe(false); + } finally { + fs.rmSync(directory, { recursive: true, force: true }); + } + }); + it("rejects malformed, permissive, symlinked, and substitution-prone state", () => { const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-pair-state-")); fs.chmodSync(directory, 0o700); @@ -635,6 +751,7 @@ describe.sequential("dual-DGX Station trust and resume-state boundaries", () => it("uses a strict noninteractive SSH argument boundary and exact-byte helper command", () => { const args = strictStationPrepSshTransportArgs(); + expect(args).toEqual(strictStationSshTransportArgs()); expect(args).toContain("BatchMode=yes"); expect(args).toContain("StrictHostKeyChecking=yes"); expect(args).toContain("VerifyHostKeyDNS=no"); @@ -820,9 +937,9 @@ station_installer_revision() { printf '%s' "$PAIR_REVISION"; } station_dual_pair_resume_pending() { return 0; } _SELECTED_EXPRESS_PLATFORM='DGX Station' _STATION_EXPRESS_MODEL_WAS_EXPLICIT=0 -unset NEMOCLAW_VLLM_MODEL NEMOCLAW_MODEL NEMOCLAW_DGX_STATION_PEER +unset NEMOCLAW_VLLM_MODEL NEMOCLAW_MODEL NEMOCLAW_DGX_STATION_PEER NEMOCLAW_DGX_STATION_SSH_BINDING ensure_station_express_pair -printf 'RESULT peer=%s model=%s selector=%s\n' "\${NEMOCLAW_DGX_STATION_PEER:-}" "\${NEMOCLAW_MODEL:-}" "\${NEMOCLAW_VLLM_MODEL:-}" +printf 'RESULT peer=%s model=%s selector=%s binding=%s\n' "\${NEMOCLAW_DGX_STATION_PEER:-}" "\${NEMOCLAW_MODEL:-}" "\${NEMOCLAW_VLLM_MODEL:-}" "\${NEMOCLAW_DGX_STATION_SSH_BINDING:-}" `, { PAIR_ARGS_FILE: argsFile, @@ -835,6 +952,8 @@ printf 'RESULT peer=%s model=%s selector=%s\n' "\${NEMOCLAW_DGX_STATION_PEER:-}" expect(output).toContain( "RESULT peer=10.10.0.2 model=nvidia/nemotron-3-ultra-550b-a55b selector=", ); + const expectedToken = JSON.parse(coordinatorResult("ready")).sshBinding; + expect(output).toContain(`binding=${expectedToken}`); const args = fs.readFileSync(argsFile, "utf8"); expect(args).toContain("--helper"); expect(args).toContain("prepare-dgx-station-host.sh"); @@ -865,9 +984,9 @@ _SELECTED_EXPRESS_PLATFORM='DGX Station' _STATION_EXPRESS_MODEL_WAS_EXPLICIT=0 _STATION_EXPRESS_DEFERRED_MANAGED_PAIR=1 NEMOCLAW_DGX_STATION_PEER="$PAIR_PEER" -unset NEMOCLAW_VLLM_MODEL NEMOCLAW_MODEL +unset NEMOCLAW_VLLM_MODEL NEMOCLAW_MODEL NEMOCLAW_DGX_STATION_SSH_BINDING ensure_station_express_pair -printf 'RESULT peer=%s model=%s\n' "$NEMOCLAW_DGX_STATION_PEER" "$NEMOCLAW_MODEL" +printf 'RESULT peer=%s model=%s binding=%s\n' "$NEMOCLAW_DGX_STATION_PEER" "$NEMOCLAW_MODEL" "$NEMOCLAW_DGX_STATION_SSH_BINDING" `, { PAIR_ARGS_FILE: argsFile, @@ -881,6 +1000,7 @@ printf 'RESULT peer=%s model=%s\n' "$NEMOCLAW_DGX_STATION_PEER" "$NEMOCLAW_MODEL expect(output).toContain( `RESULT peer=${explicitPeer} model=nvidia/nemotron-3-ultra-550b-a55b`, ); + expect(output).toContain(`binding=${JSON.parse(coordinatorResult("ready")).sshBinding}`); const args = fs.readFileSync(argsFile, "utf8"); expect(args).toContain(`--explicit-peer ${explicitPeer}`); expect(args).toContain("--reuse-existing-managed-pair"); @@ -904,15 +1024,16 @@ station_installer_revision() { printf '%s' "$PAIR_REVISION"; } _SELECTED_EXPRESS_PLATFORM='DGX Station' _STATION_EXPRESS_MODEL_WAS_EXPLICIT=0 unset NEMOCLAW_VLLM_MODEL NEMOCLAW_MODEL NEMOCLAW_DGX_STATION_PEER +NEMOCLAW_DGX_STATION_SSH_BINDING='stale' ensure_station_express_pair -printf 'RESULT peer=%s model=%s selector=%s\n' "\${NEMOCLAW_DGX_STATION_PEER:-}" "\${NEMOCLAW_MODEL:-}" "\${NEMOCLAW_VLLM_MODEL:-}" +printf 'RESULT peer=%s model=%s selector=%s binding=%s\n' "\${NEMOCLAW_DGX_STATION_PEER:-}" "\${NEMOCLAW_MODEL:-}" "\${NEMOCLAW_VLLM_MODEL:-}" "\${NEMOCLAW_DGX_STATION_SSH_BINDING:-}" `, { PAIR_REVISION: REVISION }, ); try { expect(result.status, output).toBe(0); expect(output).toContain("No trusted reciprocal dual-DGX Station pair was detected"); - expect(output).toContain("RESULT peer= model= selector="); + expect(output).toContain("RESULT peer= model= selector= binding="); } finally { fs.rmSync(home, { recursive: true, force: true }); } From 9e0cdbdd68655013eb79a264f944fc2fa7e6d5b4 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 16 Jul 2026 15:57:13 -0700 Subject: [PATCH 24/74] fix(installer): preserve provider help parity Signed-off-by: Aaron Erickson --- scripts/install.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/install.sh b/scripts/install.sh index 9c35f673cd..8298919d15 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -3140,7 +3140,8 @@ clear_station_express_resume() { station_resume_install_command() { local revision="$1" if [ "${_STATION_INSTALL_MODE:-express}" = "provider" ]; then - printf 'curl -fsSL https://www.nvidia.com/nemoclaw.sh | NEMOCLAW_PROVIDER=install-vllm NEMOCLAW_INSTALL_TAG=%s bash' "$revision" + local provider_assignment="NEMOCLAW_PROVIDER=install-vllm " + printf 'curl -fsSL https://www.nvidia.com/nemoclaw.sh | %sNEMOCLAW_INSTALL_TAG=%s bash' "$provider_assignment" "$revision" else printf 'curl -fsSL https://www.nvidia.com/nemoclaw.sh | NEMOCLAW_INSTALL_TAG=%s bash' "$revision" fi From b5bb20f6083a1a70f076a1d0f059471b9bd90f4d Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 16 Jul 2026 17:07:38 -0700 Subject: [PATCH 25/74] test(inference): add isolated dual Station simulator Signed-off-by: Aaron Erickson --- CONTRIBUTING.md | 34 ++ package.json | 1 + scripts/simulate-dual-station.mts | 216 +++++++++ ...llm-dual-station-simulator-command.test.ts | 106 +++++ ...llm-dual-station-simulator.test-support.ts | 432 ++++++++++++++++++ .../vllm-dual-station-simulator.test.ts | 182 ++++++++ .../inference/vllm-station-cluster.test.ts | 13 +- .../vllm-station-ssh-binding.test-support.ts | 10 + 8 files changed, 989 insertions(+), 5 deletions(-) create mode 100644 scripts/simulate-dual-station.mts create mode 100644 src/lib/inference/vllm-dual-station-simulator-command.test.ts create mode 100644 src/lib/inference/vllm-dual-station-simulator.test-support.ts create mode 100644 src/lib/inference/vllm-dual-station-simulator.test.ts diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6a8fc23ee7..4600174931 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -206,6 +206,7 @@ These are the primary npm scripts for day-to-day development: | `npm run test:watch` | Watch the CLI, plugin, and E2E-support projects and rerun affected tests | | `npm run test:shuffle` | Shuffle test order in the focused source projects without collecting coverage | | `npm run test:diagnose:leaks` | Report async-resource leaks and diagnose a Vitest process that hangs during shutdown | +| `npm run test:dual-station-sim` | Run the guarded, fixture-backed dual-DGX Station planner and lifecycle simulator | | `npm run test:integration` | Clean-build the CLI and run root integration and installer tests | | `npm run test:package` | Clean-build CLI/plugin artifacts and run compiled-package contracts | | `npm run test:live-e2e` | Opt into live E2E scenarios (mutates real external state) | @@ -227,6 +228,39 @@ npx vitest run --project e2e-support This project is fast and does not run live targets. Live E2E remains opt-in through `npm run test:live-e2e` or the applicable GitHub Actions workflow. +### Dual-Station Simulation + +Run the dual-Station simulator when changing DGX Station topology qualification, strict peer SSH +binding, model staging, distributed launch arguments, lifecycle ownership, rollback, or managed-vLLM +source orchestration: + +```bash +npm run test:dual-station-sim +``` + +The command selects only the repository's audited non-live source suites, gives the Vitest child a +private temporary home, cache, and temp directory, passes a minimal allowlist of local process +variables, forces live projects off, and prepends fail-fast shims for Docker, SSH, networking, GPU, +Python, and download commands. Its planner fixtures +model two GB300 systems, two 400 Gbit/s MTU-9000 CX-8 rails, reciprocal private `/30` links, direct +routes and neighbors, jumbo frames, and a common RoCEv2 GID. Its in-memory Docker adapters exercise +the exact production launch contracts, worker-first ordering, ownership, transaction, rollback, +reconciliation, secret placement, and cleanup logic. A connected contract test passes the real +planner output into those lifecycle functions and models registration-gated readiness, bearer auth, +worker loss, and recovery in memory. + +The simulator does not execute or cover `install.sh` pair-preparation integration; its audited suites +begin at the managed-vLLM source contracts. + +The selected suites use in-memory adapters rather than a real Docker daemon, SSH target, network +interface, GPU, image pull, or model download. They may execute local fixture-only shell syntax +checks and therefore require a POSIX development host. The runner is bounded to two minutes. This +guarding is defense in depth around a narrowly reviewed suite list, not an OS network sandbox. The +simulated service is not a real vLLM server. This simulator does not prove NCCL, RoCE, GPUDirect +RDMA, physical link behavior, or real TP=2 inference. +Use an explicitly authorized live two-Station smoke for those claims; do not add production +`SKIP_GPU`, `SKIP_RDMA`, or assumed-ready switches to make this simulator pass. + ### Test Declarative Behavior Do not read a shipped YAML, JSON, manifest, workflow, or E2E runtime file only to assert its keys, diff --git a/package.json b/package.json index 707a0ef64d..a80ec75b81 100644 --- a/package.json +++ b/package.json @@ -30,6 +30,7 @@ "test:watch": "vitest watch --project cli --project plugin --project e2e-support", "test:shuffle": "vitest run --project cli --project plugin --project e2e-support --sequence.shuffle.tests --coverage=false", "test:diagnose:leaks": "vitest run --project cli --project plugin --project e2e-support --detectAsyncLeaks --coverage=false --reporter=default --reporter=hanging-process", + "test:dual-station-sim": "tsx scripts/simulate-dual-station.mts", "test:integration": "npm run clean:cli && npm run build:cli && vitest run --project integration --project installer-integration", "test:package": "npm run clean:cli && npm --prefix nemoclaw run clean && npm run build:cli && npm --prefix nemoclaw run build && vitest run --project package-contract", "test:coverage:cli": "npm run clean:cli && npm run build:cli && tsx scripts/check-dist-sourcemaps.mts dist && vitest run --project cli --project integration --coverage --coverage.reporter=text-summary --coverage.reporter=json-summary --coverage.reportsDirectory=coverage/cli --coverage.include=\"bin/**/*.js\" --coverage.include=\"src/**/*.ts\" --coverage.exclude=\"test/**/*.js\" --coverage.exclude=\"test/**/*.ts\" && tsx scripts/check-coverage-ratchet.mts coverage/cli/coverage-summary.json ci/coverage-threshold-cli.json \"CLI coverage\"", diff --git a/scripts/simulate-dual-station.mts b/scripts/simulate-dual-station.mts new file mode 100644 index 0000000000..0f6f64b0be --- /dev/null +++ b/scripts/simulate-dual-station.mts @@ -0,0 +1,216 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import process from "node:process"; +import { fileURLToPath } from "node:url"; + +export const DUAL_STATION_SIMULATION_SUITES = Object.freeze([ + "src/lib/inference/vllm-models.test.ts", + "src/lib/inference/vllm-station-cluster.test.ts", + "src/lib/inference/vllm-station-cluster-lifecycle.test.ts", + "src/lib/inference/vllm-dual-station.test.ts", + "src/lib/inference/vllm-station-model-staging.test.ts", + "src/lib/inference/vllm-station-ssh-binding.test.ts", + "src/lib/inference/vllm-dual-station-simulator.test.ts", + "src/lib/inference/vllm-dual-station-simulator-command.test.ts", +]); + +export const DUAL_STATION_SIMULATION_POISON_EXECUTABLES = Object.freeze([ + "curl", + "docker", + "ibstat", + "ibv_devinfo", + "ip", + "mpirun", + "nccl-tests", + "nvidia-smi", + "ping", + "python", + "python3", + "rdma", + "rsync", + "scp", + "sftp", + "ssh", + "wget", +]); + +export const DUAL_STATION_SIMULATION_TIMEOUT_MS = 120_000; + +const INHERITED_ENV_KEYS = Object.freeze([ + "CI", + "COLORTERM", + "FORCE_COLOR", + "LANG", + "LC_ALL", + "LC_CTYPE", + "NO_COLOR", + "PATH", + "TERM", +]); + +export interface DualStationSimulationInvocation { + command: string; + args: string[]; + env: NodeJS.ProcessEnv; +} + +interface SimulationPoisonBin { + directory: string; + cacheDirectory: string; + homeDirectory: string; + tempDirectory: string; + cleanup(): void; +} + +export function createSimulationPoisonBin(): SimulationPoisonBin { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-dual-station-sim-")); + fs.chmodSync(directory, 0o700); + const cacheDirectory = path.join(directory, "cache"); + const homeDirectory = path.join(directory, "home"); + const tempDirectory = path.join(directory, "tmp"); + for (const child of [cacheDirectory, homeDirectory, tempDirectory]) { + fs.mkdirSync(child, { mode: 0o700 }); + } + for (const executable of DUAL_STATION_SIMULATION_POISON_EXECUTABLES) { + const unixFile = path.join(directory, executable); + fs.writeFileSync( + unixFile, + `#!/bin/sh\necho "dual-Station simulator blocked external command: ${executable}" >&2\nexit 97\n`, + { mode: 0o700 }, + ); + fs.chmodSync(unixFile, 0o700); + fs.writeFileSync( + `${unixFile}.cmd`, + `@echo off\r\necho dual-Station simulator blocked external command: ${executable} 1>&2\r\nexit /b 97\r\n`, + { mode: 0o600 }, + ); + } + return { + directory, + cacheDirectory, + homeDirectory, + tempDirectory, + cleanup: () => fs.rmSync(directory, { recursive: true, force: true }), + }; +} + +export function dualStationSimulationEnvironment( + source: NodeJS.ProcessEnv = process.env, +): NodeJS.ProcessEnv { + const env: NodeJS.ProcessEnv = {}; + for (const key of INHERITED_ENV_KEYS) { + if (source[key] !== undefined) env[key] = source[key]; + } + env.NEMOCLAW_RUN_BRANCH_VALIDATION_E2E = "0"; + env.NEMOCLAW_RUN_LIVE_E2E = "0"; + return env; +} + +export function buildDualStationSimulationInvocation( + repositoryRoot: string, + sourceEnv: NodeJS.ProcessEnv = process.env, +): DualStationSimulationInvocation { + const vitestEntry = path.join(repositoryRoot, "node_modules", "vitest", "vitest.mjs"); + if (!fs.existsSync(vitestEntry)) { + throw new Error(`Repository-local Vitest entry point is missing: ${vitestEntry}`); + } + return { + command: process.execPath, + args: [ + vitestEntry, + "run", + "--project", + "cli", + ...DUAL_STATION_SIMULATION_SUITES, + "--reporter=dot", + ], + env: dualStationSimulationEnvironment(sourceEnv), + }; +} + +export function assertDualStationSimulationPlatform(platform = process.platform): void { + if (platform === "win32") { + throw new Error( + "The dual-Station simulator requires a POSIX host because its SSH-binding fixtures validate POSIX modes and shell syntax.", + ); + } +} + +function repositoryRoot(): string { + return path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +} + +function describeSimulation(): void { + process.stdout.write( + `${JSON.stringify( + { + mode: "simulation-only", + localProcesses: ["repository-local Vitest worker", "fixture-only shell syntax checks"], + liveTargets: [], + externalCommandShims: DUAL_STATION_SIMULATION_POISON_EXECUTABLES, + suites: DUAL_STATION_SIMULATION_SUITES, + }, + null, + 2, + )}\n`, + ); +} + +export function main(argv: readonly string[] = process.argv.slice(2)): number { + if (argv.length === 1 && argv[0] === "--describe") { + describeSimulation(); + return 0; + } + if (argv.length > 0) { + throw new Error(`Unknown dual-Station simulator argument: ${argv.join(" ")}`); + } + + assertDualStationSimulationPlatform(); + const invocation = buildDualStationSimulationInvocation(repositoryRoot()); + const poisonBin = createSimulationPoisonBin(); + invocation.env.PATH = [poisonBin.directory, invocation.env.PATH] + .filter((entry): entry is string => Boolean(entry)) + .join(path.delimiter); + invocation.env.HOME = poisonBin.homeDirectory; + invocation.env.XDG_CACHE_HOME = poisonBin.cacheDirectory; + invocation.env.TEMP = poisonBin.tempDirectory; + invocation.env.TMP = poisonBin.tempDirectory; + invocation.env.TMPDIR = poisonBin.tempDirectory; + delete invocation.env.XDG_RUNTIME_DIR; + process.stdout.write( + "Running the guarded, fixture-backed dual-Station simulator. " + + "The audited suites use in-memory Docker adapters and no live Station target.\n", + ); + let result: ReturnType; + try { + result = spawnSync(invocation.command, invocation.args, { + cwd: repositoryRoot(), + env: invocation.env, + stdio: "inherit", + timeout: DUAL_STATION_SIMULATION_TIMEOUT_MS, + }); + } finally { + poisonBin.cleanup(); + } + if (result.error) throw result.error; + if (result.signal) { + process.stderr.write(`Dual-Station simulator terminated by signal ${result.signal}.\n`); + return 1; + } + return result.status ?? 1; +} + +const invokedFile = process.argv[1] ? path.resolve(process.argv[1]) : ""; +if (invokedFile === fileURLToPath(import.meta.url)) { + try { + process.exitCode = main(); + } catch (error) { + process.stderr.write(`${(error as Error).message}\n`); + process.exitCode = 1; + } +} diff --git a/src/lib/inference/vllm-dual-station-simulator-command.test.ts b/src/lib/inference/vllm-dual-station-simulator-command.test.ts new file mode 100644 index 0000000000..c5e2c2b857 --- /dev/null +++ b/src/lib/inference/vllm-dual-station-simulator-command.test.ts @@ -0,0 +1,106 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { + assertDualStationSimulationPlatform, + buildDualStationSimulationInvocation, + createSimulationPoisonBin, + DUAL_STATION_SIMULATION_POISON_EXECUTABLES, + DUAL_STATION_SIMULATION_SUITES, + dualStationSimulationEnvironment, + main, +} from "../../../scripts/simulate-dual-station.mts"; + +describe("dual-Station simulator command", () => { + it("selects only the audited non-live source simulation suites", () => { + expect(DUAL_STATION_SIMULATION_SUITES).toHaveLength(8); + expect(DUAL_STATION_SIMULATION_SUITES).toContain( + "src/lib/inference/vllm-dual-station-simulator.test.ts", + ); + expect( + DUAL_STATION_SIMULATION_SUITES.every( + (suite) => + suite.startsWith("src/lib/inference/") && + suite.endsWith(".test.ts") && + !suite.includes("/e2e/") && + !suite.includes("/live/"), + ), + ).toBe(true); + }); + + it("poisons commands that could reach a live host or external service", () => { + expect(DUAL_STATION_SIMULATION_POISON_EXECUTABLES).toEqual( + expect.arrayContaining(["curl", "docker", "nvidia-smi", "ping", "python3", "ssh"]), + ); + + const poisonBin = createSimulationPoisonBin(); + try { + expect(fs.readFileSync(path.join(poisonBin.directory, "docker"), "utf8")).toContain( + "exit 97", + ); + expect(fs.statSync(poisonBin.homeDirectory).mode & 0o777).toBe(0o700); + expect(fs.statSync(poisonBin.cacheDirectory).mode & 0o777).toBe(0o700); + expect(fs.statSync(poisonBin.tempDirectory).mode & 0o777).toBe(0o700); + } finally { + poisonBin.cleanup(); + } + expect(fs.existsSync(poisonBin.directory)).toBe(false); + }); + + it("inherits only local process basics and forces live projects off", () => { + const env = dualStationSimulationEnvironment({ + PATH: "/fixture/bin", + HOME: "/real/home", + XDG_CACHE_HOME: "/real/cache", + XDG_RUNTIME_DIR: "/real/runtime", + ACME_SECRET: "should-not-leak", + DOCKER_HOST: "ssh://should-not-run", + GIT_ASKPASS: "/should/not/run", + GITHUB_TOKEN: "should-not-leak", + HF_TOKEN: "should-not-leak", + NEMOCLAW_DGX_STATION_PEER: "should-not-run", + NEMOCLAW_DGX_STATION_SSH_BINDING: "should-not-run", + NEMOCLAW_RUN_BRANCH_VALIDATION_E2E: "1", + NEMOCLAW_RUN_LIVE_E2E: "1", + UNRELATED_AMBIENT_VALUE: "should-not-inherit", + }); + + expect(env).toEqual({ + PATH: "/fixture/bin", + NEMOCLAW_RUN_BRANCH_VALIDATION_E2E: "0", + NEMOCLAW_RUN_LIVE_E2E: "0", + }); + }); + + it("pins the repository-local Vitest CLI and the non-live project", () => { + const root = path.resolve(import.meta.dirname, "../../.."); + const invocation = buildDualStationSimulationInvocation(root, {}); + + expect(invocation.command).toBe(process.execPath); + expect(invocation.args).toEqual([ + path.join(root, "node_modules", "vitest", "vitest.mjs"), + "run", + "--project", + "cli", + ...DUAL_STATION_SIMULATION_SUITES, + "--reporter=dot", + ]); + }); + + it("fails clearly on native Windows instead of weakening POSIX fixture checks", () => { + expect(() => assertDualStationSimulationPlatform("win32")).toThrow("requires a POSIX host"); + expect(() => assertDualStationSimulationPlatform("darwin")).not.toThrow(); + expect(() => assertDualStationSimulationPlatform("linux")).not.toThrow(); + }); + + it("rejects arguments instead of forwarding them to another test project", () => { + expect(() => main(["--project", "e2e-live"])).toThrow( + "Unknown dual-Station simulator argument", + ); + }); +}); diff --git a/src/lib/inference/vllm-dual-station-simulator.test-support.ts b/src/lib/inference/vllm-dual-station-simulator.test-support.ts new file mode 100644 index 0000000000..8e2e6c0260 --- /dev/null +++ b/src/lib/inference/vllm-dual-station-simulator.test-support.ts @@ -0,0 +1,432 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + DUAL_STATION_VLLM_RUNTIME, + NEMOCLAW_DGX_STATION_PEER_ENV, + probeDualStationVllmCapability, + type StationClusterProbeDeps, + type StationHostProbe, + type StationProbeCommandResult, + type StationRailConnectivityRequest, +} from "./vllm-station-cluster"; +import { + DUAL_STATION_VLLM_API_KEY_FINGERPRINT_LABEL, + DUAL_STATION_VLLM_CLUSTER_LABEL, + DUAL_STATION_VLLM_ENDPOINT_LABEL, + DUAL_STATION_VLLM_GPU_LABEL, + DUAL_STATION_VLLM_GPU_SMOKE_LABEL, + DUAL_STATION_VLLM_HEAD_CONTAINER_NAME, + DUAL_STATION_VLLM_LAUNCH_CONTRACT_LABEL, + DUAL_STATION_VLLM_LAUNCH_SCHEMA_LABEL, + DUAL_STATION_VLLM_MANAGED_LABEL, + DUAL_STATION_VLLM_ROLE_LABEL, + DUAL_STATION_VLLM_TRANSACTION_LABEL, + DUAL_STATION_VLLM_WORKER_CONTAINER_NAME, + type DualStationDockerOptions, + type DualStationVllmLifecycleDeps, +} from "./vllm-station-cluster-lifecycle"; +import { NEMOCLAW_DGX_STATION_SSH_BINDING_ENV } from "./vllm-station-ssh-binding"; +import { + createDualStationSshBindingFixture, + type DualStationSshBindingFixture, +} from "./vllm-station-ssh-binding.test-support"; + +export const DUAL_STATION_SIMULATOR_API_KEY = "a".repeat(64); +export const DUAL_STATION_SIMULATOR_LOCAL_GPU = "GPU-aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee"; +export const DUAL_STATION_SIMULATOR_PEER_GPU = "GPU-11111111-2222-3333-4444-555555555555"; + +const PEER_TARGET = "nvidia@station-b"; +const BINDING_TOKEN = "simulated-qualified-binding"; +const activeSshFixtures = new Set(); + +function snapshotPath(home: string): string { + return [ + home, + ".cache/huggingface/hub", + `models--${DUAL_STATION_VLLM_RUNTIME.modelId.replace("/", "--")}`, + "snapshots", + DUAL_STATION_VLLM_RUNTIME.modelRevision, + ].join("/"); +} + +function stationRail( + rdmaDevice: string, + netdev: string, + pciAddress: string, + macAddress: string, + address: string, +) { + return { + rdmaDevice, + port: 1, + netdev, + macAddress, + uverbsDevice: `/dev/infiniband/uverbs${rdmaDevice.endsWith("0") ? "0" : "1"}`, + pciAddress, + pciName: `${pciAddress} Ethernet controller: NVIDIA ConnectX-8 SuperNIC`, + state: "4: ACTIVE", + linkLayer: "Ethernet", + speedMbps: 400_000, + mtu: 9000, + ipv4Addresses: [{ address, prefixLength: 30 }], + roceV2Ipv4Gids: [{ index: 3, address }], + }; +} + +function stationHost(side: "local" | "peer"): StationHostProbe { + const local = side === "local"; + const home = local ? "/home/local" : "/home/peer"; + return { + schemaVersion: 1, + hostname: local ? "station-a" : "station-b", + productName: "NVIDIA DGX Station GB300", + architecture: "aarch64", + home, + uid: local ? 1000 : 1001, + gpus: [ + { + index: 0, + name: "NVIDIA GB300 Grace Blackwell Superchip", + uuid: local ? DUAL_STATION_SIMULATOR_LOCAL_GPU : DUAL_STATION_SIMULATOR_PEER_GPU, + }, + ], + docker: { reachable: true, nvidiaRuntime: true }, + rsyncAvailable: true, + nvidiaPeermemLoaded: true, + rails: local + ? [ + stationRail("mlx5_0", "cx8a0", "0001:03:00.0", "02:00:00:aa:00:00", "192.168.240.1"), + stationRail("mlx5_1", "cx8a1", "0001:03:00.1", "02:00:00:aa:00:01", "192.168.240.5"), + ] + : [ + // Inventory is intentionally reversed; the production planner must match by subnet. + stationRail("mlx5_1", "cx8b1", "0002:03:00.1", "02:00:00:bb:00:01", "192.168.240.6"), + stationRail("mlx5_0", "cx8b0", "0002:03:00.0", "02:00:00:bb:00:00", "192.168.240.2"), + ], + modelSnapshot: { + modelId: DUAL_STATION_VLLM_RUNTIME.modelId, + revision: DUAL_STATION_VLLM_RUNTIME.modelRevision, + path: snapshotPath(home), + directoryExists: true, + complete: true, + shardCount: 113, + reason: "", + }, + }; +} + +function command(stdout: unknown): StationProbeCommandResult { + return { + status: 0, + stdout: typeof stdout === "string" ? stdout : JSON.stringify(stdout), + }; +} + +function strictSshConfig(fixture: DualStationSshBindingFixture): string { + const binding = fixture.binding; + return [ + `hostname ${binding.resolvedHost}`, + `user ${binding.sshUser}`, + `port ${String(binding.port)}`, + `hostkeyalias ${binding.lookupHost}`, + `userknownhostsfile ${binding.knownHostsFile}`, + "globalknownhostsfile /dev/null", + "batchmode yes", + "stricthostkeychecking true", + "permitlocalcommand no", + "forwardagent no", + "forwardx11 no", + "forwardx11trusted no", + "tunnel false", + "updatehostkeys no", + "controlmaster false", + "controlpersist no", + "sendenv LANG", + "sendenv LC_*", + ].join("\n"); +} + +function connectivity(requests: readonly StationRailConnectivityRequest[]) { + return command({ + schemaVersion: 1, + checks: requests.map((request) => ({ + ...request, + routeDevice: request.netdev, + routeSource: request.sourceAddress, + routeGateway: null, + routeScope: "link", + peerMac: request.expectedPeerMac, + peerNeighborState: "REACHABLE", + jumboPing: true, + })), + }); +} + +function probeDeps(fixture: DualStationSshBindingFixture): StationClusterProbeDeps { + return { + loadPeerSshBinding: (token, expectedPeerTarget) => { + if (token !== BINDING_TOKEN) throw new Error("unexpected simulated SSH binding token"); + if (expectedPeerTarget !== PEER_TARGET) throw new Error("unexpected simulated peer target"); + return fixture.binding; + }, + probePeerSshConfig: () => command(strictSshConfig(fixture)), + probeLocalHost: () => command(stationHost("local")), + probePeerHost: () => command(stationHost("peer")), + probeLocalConnectivity: connectivity, + probePeerConnectivity: (_binding, requests) => connectivity(requests), + }; +} + +export function createDualStationSimulationPlan() { + const fixture = createDualStationSshBindingFixture(PEER_TARGET); + activeSshFixtures.add(fixture); + const capability = probeDualStationVllmCapability({ + env: { + [NEMOCLAW_DGX_STATION_PEER_ENV]: PEER_TARGET, + [NEMOCLAW_DGX_STATION_SSH_BINDING_ENV]: BINDING_TOKEN, + }, + deps: probeDeps(fixture), + }); + if (capability.kind !== "ready") throw new Error("synthetic topology did not qualify"); + return capability.plan; +} + +export function cleanupDualStationSimulationFixtures(): void { + for (const fixture of activeSshFixtures) fixture.cleanup(); + activeSshFixtures.clear(); +} + +type SimulatedContainer = { + id: string; + target: "local" | "peer"; + name: string; + state: "running" | "exited"; + image: string; + labels: Record; + visibleGpu: string | null; + command: string; + environment: NodeJS.ProcessEnv; +}; + +export type DualStationSimulatorMutation = { + kind: "run" | "rm"; + target: "local" | "peer"; + nameOrId: string; + args?: readonly string[]; + options?: DualStationDockerOptions; +}; + +function dockerValues(args: readonly string[], flag: string): string[] { + return args.flatMap((arg, index) => + arg === flag && index < args.length - 1 ? [args[index + 1]] : [], + ); +} + +export function createDualStationLifecycleSimulator() { + const containers = new Map(); + const mutations: DualStationSimulatorMutation[] = []; + const healthDuringManagedLaunch: number[] = []; + let idCounter = 0; + let nonceCounter = 0; + let transactionCounter = 0; + let registeredWorkerId: string | null = null; + + const targetFor = (options?: DualStationDockerOptions): "local" | "peer" => + options?.env?.SIMULATED_DOCKER_TARGET === "peer" ? "peer" : "local"; + const keyFor = (target: "local" | "peer", name: string) => `${target}:${name}`; + const nextId = () => (++idCounter).toString(16).padStart(64, "0"); + const getById = (id: string) => [...containers.values()].find((item) => item.id === id); + const managedPair = () => ({ + head: containers.get(keyFor("local", DUAL_STATION_VLLM_HEAD_CONTAINER_NAME)), + worker: containers.get(keyFor("peer", DUAL_STATION_VLLM_WORKER_CONTAINER_NAME)), + }); + const managedReady = () => { + const { head, worker } = managedPair(); + return ( + head?.state === "running" && worker?.state === "running" && registeredWorkerId === worker.id + ); + }; + + const deps = { + buildLocalDockerEnv: () => ({ + SIMULATED_DOCKER_TARGET: "local", + VLLM_API_KEY: "ambient-secret-must-be-stripped", + }), + buildRemoteDockerEnv: () => ({ + SIMULATED_DOCKER_TARGET: "peer", + VLLM_API_KEY: "ambient-secret-must-be-stripped", + }), + createProbeNonce: () => (++nonceCounter).toString(16).padStart(32, "0"), + createTransactionId: () => (++transactionCounter).toString(16).padStart(32, "0"), + waitBeforeReconcile: async () => undefined, + withLifecycleLock: async (operation: () => Promise | T): Promise => await operation(), + loadApiKey: () => DUAL_STATION_SIMULATOR_API_KEY, + localInterfaceAddresses: () => ["192.168.240.1"], + dockerCapture: (args: readonly string[], options?: DualStationDockerOptions): string => { + const target = targetFor(options); + if (args[0] === "image") return `sha256:${"f".repeat(64)}\n`; + if (args[0] === "wait") return getById(args[1]) ? "0\n" : "1\n"; + if (args[0] === "logs") return `${getById(args[1])?.visibleGpu ?? ""}\n`; + + const nameFilter = dockerValues(args, "--filter")[0] ?? ""; + const name = nameFilter.replace(/^name=\^\//u, "").replace(/\$$/u, ""); + const item = containers.get(keyFor(target, name)); + if (!item) return ""; + const format = dockerValues(args, "--format")[0] ?? ""; + if (format.includes(DUAL_STATION_VLLM_GPU_SMOKE_LABEL)) { + return [ + item.id, + item.name, + item.image, + item.labels[DUAL_STATION_VLLM_GPU_SMOKE_LABEL] ?? "", + item.labels[DUAL_STATION_VLLM_ROLE_LABEL] ?? "", + ].join("\t"); + } + return [ + item.id, + item.name, + item.state, + item.image, + item.labels[DUAL_STATION_VLLM_MANAGED_LABEL] ?? "", + item.labels[DUAL_STATION_VLLM_ROLE_LABEL] ?? "", + item.labels[DUAL_STATION_VLLM_ENDPOINT_LABEL] ?? "", + item.labels[DUAL_STATION_VLLM_CLUSTER_LABEL] ?? "", + item.labels[DUAL_STATION_VLLM_GPU_LABEL] ?? "", + item.labels[DUAL_STATION_VLLM_LAUNCH_SCHEMA_LABEL] ?? "", + item.labels[DUAL_STATION_VLLM_LAUNCH_CONTRACT_LABEL] ?? "", + item.labels[DUAL_STATION_VLLM_API_KEY_FINGERPRINT_LABEL] ?? "", + item.labels[DUAL_STATION_VLLM_TRANSACTION_LABEL] ?? "", + ].join("\t"); + }, + dockerRunDetached: (args: readonly string[], options?: DualStationDockerOptions) => { + const target = targetFor(options); + const name = dockerValues(args, "--name")[0]; + const labels = Object.fromEntries( + dockerValues(args, "--label").map((entry) => { + const separator = entry.indexOf("="); + return [entry.slice(0, separator), entry.slice(separator + 1)]; + }), + ); + const visibleGpu = (dockerValues(args, "--gpus")[0] ?? "").replace(/^device=/u, "") || null; + const item: SimulatedContainer = { + id: nextId(), + target, + name, + state: name.startsWith("nemoclaw-vllm-gpu-smoke-") ? "exited" : "running", + image: DUAL_STATION_VLLM_RUNTIME.image, + labels, + visibleGpu, + command: args.at(-1) ?? "", + environment: { ...options?.env }, + }; + containers.set(keyFor(target, name), item); + mutations.push({ + kind: "run", + target, + nameOrId: name, + args: [...args], + options, + }); + if ( + name === DUAL_STATION_VLLM_WORKER_CONTAINER_NAME || + name === DUAL_STATION_VLLM_HEAD_CONTAINER_NAME + ) { + healthDuringManagedLaunch.push(managedReady() ? 200 : 503); + } + return { status: 0, stdout: `${item.id}\n` }; + }, + dockerForceRm: (containerId: string, options?: DualStationDockerOptions) => { + const target = targetFor(options); + const item = getById(containerId); + if (!item || item.target !== target) return { status: 1 }; + if (registeredWorkerId === item.id) registeredWorkerId = null; + containers.delete(keyFor(item.target, item.name)); + mutations.push({ kind: "rm", target, nameOrId: containerId, options }); + return { status: 0 }; + }, + } satisfies DualStationVllmLifecycleDeps; + + function serviceRequest(path: string, authorization?: string) { + // This is an in-memory API contract simulator, not inference or a network/RDMA test. + if (path === "/health") return { status: managedReady() ? 200 : 503, body: null }; + const { head } = managedPair(); + if ( + !head?.environment.VLLM_API_KEY || + authorization !== `Bearer ${head.environment.VLLM_API_KEY}` + ) + return { status: 401, body: { error: "unauthorized" } }; + if (!managedReady()) return { status: 503, body: { error: "worker unavailable" } }; + if (path === "/v1/models") { + return { + status: 200, + body: { data: [{ id: DUAL_STATION_VLLM_RUNTIME.servedModelId }] }, + }; + } + if (path === "/v1/chat/completions") { + return { + status: 200, + body: { + choices: [{ message: { role: "assistant", content: "SIMULATED_OK" } }], + }, + }; + } + return { status: 404, body: null }; + } + + function registerWorker(): void { + const { head, worker } = managedPair(); + if (head?.state !== "running" || worker?.state !== "running") { + throw new Error("both simulated roles must be running before registration"); + } + if ( + !worker.command.includes("--node-rank 1") || + !worker.command.includes("--headless") || + !worker.command.includes("--master-addr 192.168.240.1") || + !head.command.includes("--node-rank 0") || + head.command.includes("--headless") || + worker.labels[DUAL_STATION_VLLM_TRANSACTION_LABEL] !== + head.labels[DUAL_STATION_VLLM_TRANSACTION_LABEL] + ) { + throw new Error("simulated worker launch contract cannot register with the head"); + } + registeredWorkerId = worker.id; + } + + function loseWorker(): void { + const worker = containers.get(keyFor("peer", DUAL_STATION_VLLM_WORKER_CONTAINER_NAME)); + if (!worker) throw new Error("simulated worker was not running"); + worker.state = "exited"; + registeredWorkerId = null; + } + + return { + containers, + deps, + healthDuringManagedLaunch, + loseWorker, + mutations, + registerWorker, + serviceRequest, + }; +} + +export function dualStationManagedRuns(mutations: readonly DualStationSimulatorMutation[]) { + return mutations.filter( + (mutation) => + mutation.kind === "run" && + (mutation.nameOrId === DUAL_STATION_VLLM_WORKER_CONTAINER_NAME || + mutation.nameOrId === DUAL_STATION_VLLM_HEAD_CONTAINER_NAME), + ); +} + +export function dualStationSimulatorLabels( + mutation: DualStationSimulatorMutation, +): Record { + return Object.fromEntries( + dockerValues(mutation.args ?? [], "--label").map((entry) => { + const separator = entry.indexOf("="); + return [entry.slice(0, separator), entry.slice(separator + 1)]; + }), + ); +} diff --git a/src/lib/inference/vllm-dual-station-simulator.test.ts b/src/lib/inference/vllm-dual-station-simulator.test.ts new file mode 100644 index 0000000000..dd8e9adf0f --- /dev/null +++ b/src/lib/inference/vllm-dual-station-simulator.test.ts @@ -0,0 +1,182 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { afterEach, describe, expect, it } from "vitest"; + +import { + DUAL_STATION_SIMULATOR_API_KEY as API_KEY, + cleanupDualStationSimulationFixtures, + createDualStationLifecycleSimulator, + createDualStationSimulationPlan, + dualStationManagedRuns, + dualStationSimulatorLabels, + DUAL_STATION_SIMULATOR_LOCAL_GPU as LOCAL_GPU, + DUAL_STATION_SIMULATOR_PEER_GPU as PEER_GPU, +} from "./vllm-dual-station-simulator.test-support"; +import { DUAL_STATION_VLLM_RUNTIME } from "./vllm-station-cluster"; +import { + areDualStationManagedVllmContainersRunning, + cleanupDualStationManagedVllm, + DUAL_STATION_VLLM_API_KEY_FINGERPRINT_LABEL, + DUAL_STATION_VLLM_CLUSTER_LABEL, + DUAL_STATION_VLLM_ENDPOINT_LABEL, + DUAL_STATION_VLLM_GPU_LABEL, + DUAL_STATION_VLLM_HEAD_CONTAINER_NAME, + DUAL_STATION_VLLM_LAUNCH_CONTRACT_LABEL, + DUAL_STATION_VLLM_LAUNCH_SCHEMA_LABEL, + DUAL_STATION_VLLM_MANAGED_LABEL, + DUAL_STATION_VLLM_ROLE_LABEL, + DUAL_STATION_VLLM_TRANSACTION_LABEL, + DUAL_STATION_VLLM_WORKER_CONTAINER_NAME, + type DualStationVllmRole, + dualStationVllmApiKeyFingerprint, + dualStationVllmClusterId, + dualStationVllmLaunchContract, + preflightDualStationGpuRuntime, + preflightDualStationManagedVllm, + startDualStationManagedVllm, +} from "./vllm-station-cluster-lifecycle"; + +afterEach(cleanupDualStationSimulationFixtures); + +describe("connected dual-Station planner-to-lifecycle simulator", () => { + it("passes a synthetic plan through launch, simulated registration, loss, recovery, and cleanup", async () => { + const plan = createDualStationSimulationPlan(); + expect(plan).toMatchObject({ + masterAddress: "192.168.240.1", + roceGidIndex: 3, + runtime: DUAL_STATION_VLLM_RUNTIME, + rails: [ + { local: { netdev: "cx8a0" }, peer: { netdev: "cx8b0" } }, + { local: { netdev: "cx8a1" }, peer: { netdev: "cx8b1" } }, + ], + }); + + const sim = createDualStationLifecycleSimulator(); + expect(sim.serviceRequest("/health")).toEqual({ status: 503, body: null }); + expect(preflightDualStationManagedVllm(plan, sim.deps)).toEqual({ + ok: true, + }); + expect(await preflightDualStationGpuRuntime(plan, sim.deps)).toEqual({ + ok: true, + }); + expect(areDualStationManagedVllmContainersRunning(plan, sim.deps)).toBe(false); + + const firstStart = await startDualStationManagedVllm(plan, { apiKey: API_KEY }, sim.deps); + expect(firstStart).toMatchObject({ ok: true, reusedExisting: false }); + expect(areDualStationManagedVllmContainersRunning(plan, sim.deps)).toBe(true); + expect(sim.healthDuringManagedLaunch.slice(0, 2)).toEqual([503, 503]); + expect(sim.serviceRequest("/health")).toEqual({ status: 503, body: null }); + sim.registerWorker(); + expect(sim.serviceRequest("/health")).toEqual({ status: 200, body: null }); + expect(sim.serviceRequest("/v1/models").status).toBe(401); + expect(sim.serviceRequest("/v1/models", `Bearer ${API_KEY}`)).toEqual({ + status: 200, + body: { data: [{ id: DUAL_STATION_VLLM_RUNTIME.servedModelId }] }, + }); + expect(sim.serviceRequest("/v1/chat/completions", `Bearer ${API_KEY}`)).toEqual({ + status: 200, + body: { + choices: [{ message: { role: "assistant", content: "SIMULATED_OK" } }], + }, + }); + + const firstRuns = dualStationManagedRuns(sim.mutations); + expect(firstRuns.map(({ target, nameOrId }) => [target, nameOrId])).toEqual([ + ["peer", DUAL_STATION_VLLM_WORKER_CONTAINER_NAME], + ["local", DUAL_STATION_VLLM_HEAD_CONTAINER_NAME], + ]); + const expectedFingerprint = dualStationVllmApiKeyFingerprint(API_KEY); + for (const [index, run] of firstRuns.entries()) { + const role: DualStationVllmRole = index === 0 ? "worker" : "head"; + const labels = dualStationSimulatorLabels(run); + expect(labels).toEqual({ + [DUAL_STATION_VLLM_MANAGED_LABEL]: "true", + [DUAL_STATION_VLLM_ROLE_LABEL]: role, + [DUAL_STATION_VLLM_ENDPOINT_LABEL]: + role === "head" ? "http://192.168.240.1:8000" : "headless", + [DUAL_STATION_VLLM_CLUSTER_LABEL]: dualStationVllmClusterId(plan), + [DUAL_STATION_VLLM_GPU_LABEL]: role === "head" ? LOCAL_GPU : PEER_GPU, + [DUAL_STATION_VLLM_LAUNCH_SCHEMA_LABEL]: "1", + [DUAL_STATION_VLLM_LAUNCH_CONTRACT_LABEL]: dualStationVllmLaunchContract(plan, role), + [DUAL_STATION_VLLM_API_KEY_FINGERPRINT_LABEL]: expectedFingerprint, + [DUAL_STATION_VLLM_TRANSACTION_LABEL]: "1".padStart(32, "0"), + }); + expect(JSON.stringify(run.args)).not.toContain(API_KEY); + expect(JSON.stringify(labels)).not.toContain(API_KEY); + expect(run.options?.env?.VLLM_API_KEY).toBe(role === "head" ? API_KEY : undefined); + } + for (const mutation of sim.mutations.filter((item) => !firstRuns.includes(item))) { + expect(JSON.stringify(mutation.args ?? [])).not.toContain(API_KEY); + expect(mutation.options?.env?.VLLM_API_KEY).toBeUndefined(); + } + + sim.loseWorker(); + expect(areDualStationManagedVllmContainersRunning(plan, sim.deps)).toBe(false); + expect(sim.serviceRequest("/health")).toEqual({ status: 503, body: null }); + expect(sim.serviceRequest("/v1/chat/completions", `Bearer ${API_KEY}`).status).toBe(503); + + const recovered = await startDualStationManagedVllm(plan, { apiKey: API_KEY }, sim.deps); + expect(recovered).toMatchObject({ ok: true, reusedExisting: false }); + expect(areDualStationManagedVllmContainersRunning(plan, sim.deps)).toBe(true); + expect(sim.serviceRequest("/health")).toEqual({ status: 503, body: null }); + expect(sim.serviceRequest("/v1/chat/completions", `Bearer ${API_KEY}`).status).toBe(503); + sim.registerWorker(); + expect(sim.serviceRequest("/health")).toEqual({ status: 200, body: null }); + expect(sim.serviceRequest("/v1/chat/completions", `Bearer ${API_KEY}`).status).toBe(200); + + const allRuns = dualStationManagedRuns(sim.mutations); + expect(allRuns.map(({ target, nameOrId }) => [target, nameOrId])).toEqual([ + ["peer", DUAL_STATION_VLLM_WORKER_CONTAINER_NAME], + ["local", DUAL_STATION_VLLM_HEAD_CONTAINER_NAME], + ["peer", DUAL_STATION_VLLM_WORKER_CONTAINER_NAME], + ["local", DUAL_STATION_VLLM_HEAD_CONTAINER_NAME], + ]); + expect( + allRuns + .slice(2) + .map((run) => dualStationSimulatorLabels(run)[DUAL_STATION_VLLM_TRANSACTION_LABEL]), + ).toEqual(["2".padStart(32, "0"), "2".padStart(32, "0")]); + expect( + allRuns.map( + (run) => dualStationSimulatorLabels(run)[DUAL_STATION_VLLM_LAUNCH_CONTRACT_LABEL], + ), + ).toEqual([ + dualStationVllmLaunchContract(plan, "worker"), + dualStationVllmLaunchContract(plan, "head"), + dualStationVllmLaunchContract(plan, "worker"), + dualStationVllmLaunchContract(plan, "head"), + ]); + expect(sim.healthDuringManagedLaunch).toEqual([503, 503, 503, 503]); + for (const [index, run] of allRuns.entries()) { + const role: DualStationVllmRole = index % 2 === 0 ? "worker" : "head"; + const transactionId = index < 2 ? "1".padStart(32, "0") : "2".padStart(32, "0"); + expect(dualStationSimulatorLabels(run)).toEqual({ + [DUAL_STATION_VLLM_MANAGED_LABEL]: "true", + [DUAL_STATION_VLLM_ROLE_LABEL]: role, + [DUAL_STATION_VLLM_ENDPOINT_LABEL]: + role === "head" ? "http://192.168.240.1:8000" : "headless", + [DUAL_STATION_VLLM_CLUSTER_LABEL]: dualStationVllmClusterId(plan), + [DUAL_STATION_VLLM_GPU_LABEL]: role === "head" ? LOCAL_GPU : PEER_GPU, + [DUAL_STATION_VLLM_LAUNCH_SCHEMA_LABEL]: "1", + [DUAL_STATION_VLLM_LAUNCH_CONTRACT_LABEL]: dualStationVllmLaunchContract(plan, role), + [DUAL_STATION_VLLM_API_KEY_FINGERPRINT_LABEL]: expectedFingerprint, + [DUAL_STATION_VLLM_TRANSACTION_LABEL]: transactionId, + }); + } + for (const mutation of sim.mutations) { + expect(JSON.stringify(mutation.args ?? [])).not.toContain(API_KEY); + expect(JSON.stringify(dualStationSimulatorLabels(mutation))).not.toContain(API_KEY); + const isManagedHeadRun = + mutation.kind === "run" && mutation.nameOrId === DUAL_STATION_VLLM_HEAD_CONTAINER_NAME; + expect(mutation.options?.env?.VLLM_API_KEY).toBe(isManagedHeadRun ? API_KEY : undefined); + } + + const cleanup = await cleanupDualStationManagedVllm(plan, sim.deps); + expect(cleanup).toMatchObject({ ok: true }); + expect(cleanup.ok && cleanup.removedContainerIds).toHaveLength(2); + expect(areDualStationManagedVllmContainersRunning(plan, sim.deps)).toBe(false); + expect(sim.serviceRequest("/health")).toEqual({ status: 503, body: null }); + expect(sim.containers.size).toBe(0); + }); +}); diff --git a/src/lib/inference/vllm-station-cluster.test.ts b/src/lib/inference/vllm-station-cluster.test.ts index c32c49f1c2..93db07124f 100644 --- a/src/lib/inference/vllm-station-cluster.test.ts +++ b/src/lib/inference/vllm-station-cluster.test.ts @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import assert from "node:assert/strict"; import type { SpawnSyncOptionsWithStringEncoding } from "node:child_process"; import fs from "node:fs"; @@ -27,6 +28,7 @@ import { import { createDualStationSshBindingFixture, type DualStationSshBindingFixture, + retargetDualStationSshBindingFixture, } from "./vllm-station-ssh-binding.test-support"; const LOCAL_HOME = "/home/local"; @@ -225,10 +227,11 @@ function fixtureDeps( } function runWith(deps: StationClusterProbeDeps, target = "nvidia@station-b") { - if (validatePeerTarget(target).ok && sshFixture.binding.peerTarget !== target) { - sshFixture.cleanup(); - sshFixture = createDualStationSshBindingFixture(target); - } + sshFixture = retargetDualStationSshBindingFixture( + sshFixture, + target, + validatePeerTarget(target).ok, + ); return probeDualStationVllmCapability({ env: { [NEMOCLAW_DGX_STATION_PEER_ENV]: target, @@ -364,7 +367,7 @@ describe("probeDualStationVllmCapability", () => { peerModelSnapshot: "ready", plan: { peerSshBinding: { peerTarget: target } }, }); - if (result.kind !== "ready") throw new Error("expected ready fixture"); + assert(result.kind === "ready", "expected ready fixture"); expect(buildRemoteVllmDockerEnv(result.plan.peerSshBinding, {}).DOCKER_HOST).toBe( `ssh://${result.plan.peerSshBinding.sshUser}@${result.plan.peerSshBinding.resolvedHost}`, ); diff --git a/src/lib/inference/vllm-station-ssh-binding.test-support.ts b/src/lib/inference/vllm-station-ssh-binding.test-support.ts index 4213d50dcb..b431fc83f2 100644 --- a/src/lib/inference/vllm-station-ssh-binding.test-support.ts +++ b/src/lib/inference/vllm-station-ssh-binding.test-support.ts @@ -53,3 +53,13 @@ export function createDualStationSshBindingFixture( cleanup: () => fs.rmSync(root, { recursive: true, force: true }), }; } + +export function retargetDualStationSshBindingFixture( + fixture: DualStationSshBindingFixture, + peerTarget: string, + enabled = true, +): DualStationSshBindingFixture { + if (!enabled || fixture.binding.peerTarget === peerTarget) return fixture; + fixture.cleanup(); + return createDualStationSshBindingFixture(peerTarget); +} From 9e47986e0222cf0a917477837b73164466c31141 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 16 Jul 2026 17:49:08 -0700 Subject: [PATCH 26/74] fix(inference): harden dual Station runtime Signed-off-by: Aaron Erickson --- ...llm-dual-station-simulator.test-support.ts | 1 + src/lib/inference/vllm-dual-station.test.ts | 2 + .../vllm-station-cluster-lifecycle.test.ts | 97 ++++++++++++-- .../vllm-station-cluster-lifecycle.ts | 119 +++++++++++++----- .../inference/vllm-station-cluster.test.ts | 20 ++- src/lib/inference/vllm-station-cluster.ts | 8 +- .../vllm-station-model-staging.test.ts | 2 + 7 files changed, 206 insertions(+), 43 deletions(-) diff --git a/src/lib/inference/vllm-dual-station-simulator.test-support.ts b/src/lib/inference/vllm-dual-station-simulator.test-support.ts index 8e2e6c0260..039297a68f 100644 --- a/src/lib/inference/vllm-dual-station-simulator.test-support.ts +++ b/src/lib/inference/vllm-dual-station-simulator.test-support.ts @@ -84,6 +84,7 @@ function stationHost(side: "local" | "peer"): StationHostProbe { architecture: "aarch64", home, uid: local ? 1000 : 1001, + gid: local ? 1000 : 1001, gpus: [ { index: 0, diff --git a/src/lib/inference/vllm-dual-station.test.ts b/src/lib/inference/vllm-dual-station.test.ts index 323b0273a1..1e5162ded9 100644 --- a/src/lib/inference/vllm-dual-station.test.ts +++ b/src/lib/inference/vllm-dual-station.test.ts @@ -111,12 +111,14 @@ function plan(): DualStationVllmPlan { hostname: "station-a", home: "/home/nvidia", uid: 1000, + gid: 1000, gpu: { index: 0, name: "NVIDIA GB300", uuid: "GPU-aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa" }, }, peer: { hostname: "station-b", home: "/home/nvidia", uid: 1000, + gid: 1000, gpu: { index: 0, name: "NVIDIA GB300", uuid: "GPU-bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb" }, }, rails: [ diff --git a/src/lib/inference/vllm-station-cluster-lifecycle.test.ts b/src/lib/inference/vllm-station-cluster-lifecycle.test.ts index 6b7ef28f18..25323c3638 100644 --- a/src/lib/inference/vllm-station-cluster-lifecycle.test.ts +++ b/src/lib/inference/vllm-station-cluster-lifecycle.test.ts @@ -65,6 +65,7 @@ function fixturePlan(): DualStationVllmPlan { hostname: "station-a", home: "/home/local", uid: 1000, + gid: 1000, gpu: { index: 0, name: "NVIDIA GB300", @@ -75,6 +76,7 @@ function fixturePlan(): DualStationVllmPlan { hostname: "station-b", home: "/home/nvidia", uid: 1001, + gid: 1001, gpu: { index: 1, name: "NVIDIA GB300 Grace Blackwell Superchip", @@ -446,20 +448,19 @@ describe("dual-Station managed vLLM run argv", () => { expect(args).toEqual( expect.arrayContaining(["--network", "host", "--shm-size", "16g", "--read-only"]), ); + expect(dockerValues(args, "--workdir")).toEqual(["/home/vllm"]); expect(dockerValues(args, "--tmpfs")).toEqual([ "/tmp:rw,nosuid,nodev,size=17179869184", - "/root/.cache:rw,nosuid,nodev,size=68719476736", + `/home/vllm:rw,nosuid,nodev,uid=${String(expectedNode.uid)},gid=${String(expectedNode.gid)},mode=0700,size=68719476736`, ]); - expect(args).toEqual( - expect.arrayContaining([ - "--cap-drop", - "ALL", - "--cap-add", - "IPC_LOCK", - "--cap-add", - "DAC_READ_SEARCH", - ]), - ); + expect(dockerValues(args, "--user")).toEqual([ + `${String(expectedNode.uid)}:${String(expectedNode.gid)}`, + ]); + expect(dockerValues(args, "--security-opt")).toEqual(["no-new-privileges:true"]); + expect(dockerValues(args, "--cap-drop")).toEqual(["ALL"]); + expect(dockerValues(args, "--cap-add")).toEqual([]); + expect(args).not.toContain("DAC_READ_SEARCH"); + expect(args).not.toContain("IPC_LOCK"); expect(dockerValues(args, "--ulimit")).toEqual([ "memlock=-1", "stack=67108864", @@ -471,7 +472,7 @@ describe("dual-Station managed vLLM run argv", () => { "--device=/dev/infiniband/uverbs1", ]); expect(dockerValues(args, "--volume")).toEqual([ - `${expectedNode.home}/.cache/huggingface/hub:/root/.cache/huggingface/hub:ro`, + `${expectedNode.home}/.cache/huggingface/hub:/model-cache:ro`, ]); expect(dockerValues(args, "--volume").join("\n")).not.toContain("/huggingface/token"); expect(env).toEqual( @@ -489,6 +490,12 @@ describe("dual-Station managed vLLM run argv", () => { "UCX_IB_GID_INDEX=3", "HF_HUB_OFFLINE=1", "TRANSFORMERS_OFFLINE=1", + "HF_HOME=/home/vllm/.cache/huggingface", + "HF_HUB_CACHE=/model-cache", + "HUGGINGFACE_HUB_CACHE=/model-cache", + "HOME=/home/vllm", + "USER=vllm", + "LOGNAME=vllm", ]), ); expect(args).toContain(plan.runtime.image); @@ -531,7 +538,32 @@ describe("dual-Station managed vLLM run argv", () => { `device=${role === "head" ? fixturePlan().local.gpu.uuid : fixturePlan().peer.gpu.uuid}`, ]); expect(dockerValues(args, "--network")).toEqual(["none"]); + const expectedNode = role === "head" ? fixturePlan().local : fixturePlan().peer; + const expectedDevices = fixturePlan().rails.map((rail) => + role === "head" ? rail.local.uverbsDevice : rail.peer.uverbsDevice, + ); + expect(args).toContain("--read-only"); + expect(dockerValues(args, "--workdir")).toEqual(["/home/vllm"]); + expect(dockerValues(args, "--user")).toEqual([ + `${String(expectedNode.uid)}:${String(expectedNode.gid)}`, + ]); + expect(dockerValues(args, "--security-opt")).toEqual(["no-new-privileges:true"]); expect(dockerValues(args, "--cap-drop")).toEqual(["ALL"]); + expect(dockerValues(args, "--cap-add")).toEqual([]); + expect(dockerValues(args, "--ulimit")).toEqual(["memlock=-1"]); + expect(dockerValues(args, "--device")).toEqual(expectedDevices); + expect(dockerValues(args, "--tmpfs")).toEqual([ + "/tmp:rw,nosuid,nodev,size=17179869184", + `/home/vllm:rw,nosuid,nodev,uid=${String(expectedNode.uid)},gid=${String(expectedNode.gid)},mode=0700,size=68719476736`, + ]); + expect(dockerValues(args, "--env")).toEqual( + expect.arrayContaining([ + "HF_HOME=/home/vllm/.cache/huggingface", + "HOME=/home/vllm", + "USER=vllm", + "LOGNAME=vllm", + ]), + ); expect(dockerValues(args, "--label")).toEqual( expect.arrayContaining([ `${DUAL_STATION_VLLM_GPU_SMOKE_LABEL}=${nonce}`, @@ -540,11 +572,50 @@ describe("dual-Station managed vLLM run argv", () => { ); expect(args).toContain("--pull=never"); expect(args).toContain(DUAL_STATION_VLLM_RUNTIME.image); - expect(args).toEqual(expect.arrayContaining(["--entrypoint", "nvidia-smi"])); + expect(args).toEqual(expect.arrayContaining(["--entrypoint", "/bin/bash"])); + expect(args.slice(-3, -1)).toEqual([DUAL_STATION_VLLM_RUNTIME.image, "-c"]); + const command = args.at(-1) ?? ""; + expect(command).toContain("NoNewPrivs"); + expect(command).toContain("Cap(Inh|Prm|Eff|Bnd|Amb)"); + expect(command).toContain('test "$(ulimit -l)" = "unlimited"'); + expect(command).toContain( + `for device in ${expectedDevices.join(" ")}; do test -c "$device"; test -r "$device"; test -w "$device"; exec 3<>"$device"; exec 3>&-; done`, + ); + expect(command).toContain("$HOME/.cache/torch/.nemoclaw-write-probe"); + expect(command).toContain("$HF_HOME/.nemoclaw-write-probe"); + expect(command).toContain("exec nvidia-smi --query-gpu=uuid --format=csv,noheader"); + expect(dockerValues(args, "--volume")).toEqual([]); expect(args).not.toContain("--rm"); expect(args.join("\n")).not.toContain("VLLM_API_KEY"); }); + it("binds the managed launch contract to both runtime owner IDs", () => { + const baseline = fixturePlan(); + const changedUid = fixturePlan(); + const changedGid = fixturePlan(); + changedUid.local.uid += 1; + changedGid.local.gid += 1; + + expect(dualStationVllmLaunchContract(changedUid, "head")).not.toBe( + dualStationVllmLaunchContract(baseline, "head"), + ); + expect(dualStationVllmLaunchContract(changedGid, "head")).not.toBe( + dualStationVllmLaunchContract(baseline, "head"), + ); + }); + + it.each([ + ["root uid", (plan: DualStationVllmPlan) => (plan.local.uid = 0)], + ["root gid", (plan: DualStationVllmPlan) => (plan.peer.gid = 0)], + ])("rejects an unsafe %s runtime identity", (_label, mutate) => { + const plan = fixturePlan(); + mutate(plan); + + expect(() => + buildDualStationVllmRunArgs(plan, "head", TRANSACTION_ID, API_KEY_FINGERPRINT), + ).toThrow("runtime identity must use non-root UID and GID values"); + }); + it.each([ "/dev/infiniband/rdma_cm", "/dev/infiniband/uverbs0", diff --git a/src/lib/inference/vllm-station-cluster-lifecycle.ts b/src/lib/inference/vllm-station-cluster-lifecycle.ts index 874d29b1a5..86fa4f2519 100644 --- a/src/lib/inference/vllm-station-cluster-lifecycle.ts +++ b/src/lib/inference/vllm-station-cluster-lifecycle.ts @@ -33,8 +33,11 @@ export const DUAL_STATION_VLLM_GPU_SMOKE_LABEL = "com.nvidia.nemoclaw.gpu-smoke" export const DUAL_STATION_VLLM_MASTER_PORT = 29501; const HEAD_API_PORT = 8000; -const HF_CACHE_CONTAINER_DIR = "/root/.cache/huggingface"; -const HF_HUB_CACHE_CONTAINER_DIR = `${HF_CACHE_CONTAINER_DIR}/hub`; +// The pinned vLLM 0.22 image ships a non-root-ready /home/vllm. Each Station +// mounts an owner-only tmpfs there and runs as the probed model-cache owner. +const VLLM_RUNTIME_HOME = "/home/vllm"; +const HF_CACHE_CONTAINER_DIR = `${VLLM_RUNTIME_HOME}/.cache/huggingface`; +const HF_HUB_CACHE_CONTAINER_DIR = "/model-cache"; const DOCKER_INSPECT_TIMEOUT_MS = 10_000; const DOCKER_MUTATION_TIMEOUT_MS = 60_000; const DOCKER_GPU_SMOKE_TIMEOUT_MS = 30_000; @@ -213,6 +216,18 @@ function assertSafePlan(plan: DualStationVllmPlan): void { ) { throw new Error(`Dual-Station ${role} home must be a normalized absolute POSIX path.`); } + if ( + !Number.isInteger(node.uid) || + node.uid <= 0 || + node.uid > 2_147_483_647 || + !Number.isInteger(node.gid) || + node.gid <= 0 || + node.gid > 2_147_483_647 + ) { + throw new Error( + `Dual-Station ${role} runtime identity must use non-root UID and GID values.`, + ); + } if (!SAFE_GPU_UUID_PATTERN.test(node.gpu.uuid)) { throw new Error(`Dual-Station ${role} GB300 UUID is invalid.`); } @@ -308,6 +323,20 @@ function appendEnv(args: string[], name: string, value: string): void { args.push("--env", `${name}=${value}`); } +function runtimeUser(node: DualStationVllmPlan["local"]): string { + return `${String(node.uid)}:${String(node.gid)}`; +} + +function runtimeHomeTmpfs(node: DualStationVllmPlan["local"]): string { + return `${VLLM_RUNTIME_HOME}:rw,nosuid,nodev,uid=${String(node.uid)},gid=${String(node.gid)},mode=0700,size=68719476736`; +} + +function appendRuntimeHomeEnv(args: string[]): void { + appendEnv(args, "HOME", VLLM_RUNTIME_HOME); + appendEnv(args, "USER", "vllm"); + appendEnv(args, "LOGNAME", "vllm"); +} + /** Build the deterministic shell-free launch argv before per-operation labels. */ function buildDualStationVllmBaseRunArgs( plan: DualStationVllmPlan, @@ -327,16 +356,20 @@ function buildDualStationVllmBaseRunArgs( "--shm-size", "16g", "--read-only", + "--workdir", + VLLM_RUNTIME_HOME, + "--user", + runtimeUser(node), + "--security-opt", + "no-new-privileges:true", "--tmpfs", "/tmp:rw,nosuid,nodev,size=17179869184", "--tmpfs", - "/root/.cache:rw,nosuid,nodev,size=68719476736", + runtimeHomeTmpfs(node), "--cap-drop", "ALL", - "--cap-add", - "IPC_LOCK", - "--cap-add", - "DAC_READ_SEARCH", + // Non-root GPU/RDMA memory registration is bounded by this explicit rlimit; + // no Linux capabilities remain in the effective or permitted set. "--ulimit", "memlock=-1", "--ulimit", @@ -363,6 +396,9 @@ function buildDualStationVllmBaseRunArgs( ]; appendEnv(args, "HF_HOME", HF_CACHE_CONTAINER_DIR); + appendEnv(args, "HF_HUB_CACHE", HF_HUB_CACHE_CONTAINER_DIR); + appendEnv(args, "HUGGINGFACE_HUB_CACHE", HF_HUB_CACHE_CONTAINER_DIR); + appendRuntimeHomeEnv(args); appendEnv(args, "HF_HUB_OFFLINE", "1"); appendEnv(args, "TRANSFORMERS_OFFLINE", "1"); appendEnv(args, "VLLM_HOST_IP", endpoints[0].address); @@ -447,7 +483,7 @@ export function buildDualStationVllmRunArgs( return args; } -/** Build a no-network, no-pull GPU runtime probe that only executes nvidia-smi. */ +/** Build a no-network, no-pull GPU/RDMA runtime probe that finishes with nvidia-smi. */ export function buildDualStationGpuSmokeRunArgs( plan: DualStationVllmPlan, role: DualStationVllmRole, @@ -458,29 +494,56 @@ export function buildDualStationGpuSmokeRunArgs( throw new Error("Dual-Station GPU smoke nonce is invalid."); } const node = role === "head" ? plan.local : plan.peer; + const endpoints = plan.rails.map((rail) => (role === "head" ? rail.local : rail.peer)); const containerName = `${GPU_SMOKE_CONTAINER_PREFIX}-${role}-${nonce}`; + const command = [ + "set -euo pipefail", + `test \"$(id -u)\" = \"${String(node.uid)}\"`, + `test \"$(id -g)\" = \"${String(node.gid)}\"`, + "grep -Eq '^NoNewPrivs:[[:space:]]+1$' /proc/self/status", + "! grep -Eq '^Cap(Inh|Prm|Eff|Bnd|Amb):[[:space:]]+[0-9a-fA-F]*[1-9a-fA-F][0-9a-fA-F]*$' /proc/self/status", + 'test "$(ulimit -l)" = "unlimited"', + `for device in ${endpoints.map((endpoint) => endpoint.uverbsDevice).join(" ")}; do test -c "$device"; test -r "$device"; test -w "$device"; exec 3<>"$device"; exec 3>&-; done`, + 'mkdir -p "$HOME/.cache/torch" "$HF_HOME"', + 'probe="$HOME/.cache/torch/.nemoclaw-write-probe"; : > "$probe"; rm -f "$probe"', + 'probe="$HF_HOME/.nemoclaw-write-probe"; : > "$probe"; rm -f "$probe"', + "exec nvidia-smi --query-gpu=uuid --format=csv,noheader", + ].join("; "); + const args = [ + "--pull=never", + "--network", + "none", + "--read-only", + "--workdir", + VLLM_RUNTIME_HOME, + "--user", + runtimeUser(node), + "--security-opt", + "no-new-privileges:true", + "--tmpfs", + "/tmp:rw,nosuid,nodev,size=17179869184", + "--tmpfs", + runtimeHomeTmpfs(node), + "--cap-drop", + "ALL", + "--ulimit", + "memlock=-1", + "--gpus", + `device=${node.gpu.uuid}`, + ...endpoints.flatMap((endpoint) => ["--device", endpoint.uverbsDevice]), + "--label", + `${DUAL_STATION_VLLM_GPU_SMOKE_LABEL}=${nonce}`, + "--label", + `${DUAL_STATION_VLLM_ROLE_LABEL}=${role}`, + "--name", + containerName, + ]; + appendEnv(args, "HF_HOME", HF_CACHE_CONTAINER_DIR); + appendRuntimeHomeEnv(args); + args.push("--entrypoint", "/bin/bash", plan.runtime.image, "-c", command); return { containerName, - args: [ - "--pull=never", - "--network", - "none", - "--cap-drop", - "ALL", - "--gpus", - `device=${node.gpu.uuid}`, - "--label", - `${DUAL_STATION_VLLM_GPU_SMOKE_LABEL}=${nonce}`, - "--label", - `${DUAL_STATION_VLLM_ROLE_LABEL}=${role}`, - "--name", - containerName, - "--entrypoint", - "nvidia-smi", - plan.runtime.image, - "--query-gpu=uuid", - "--format=csv,noheader", - ], + args, }; } diff --git a/src/lib/inference/vllm-station-cluster.test.ts b/src/lib/inference/vllm-station-cluster.test.ts index 93db07124f..a6c19553c5 100644 --- a/src/lib/inference/vllm-station-cluster.test.ts +++ b/src/lib/inference/vllm-station-cluster.test.ts @@ -121,6 +121,7 @@ function hostFixture(side: "local" | "peer"): StationHostProbe { architecture: "aarch64", home, uid: isLocal ? 1000 : 1001, + gid: isLocal ? 1000 : 1001, gpus: isLocal ? [{ index: 0, name: "NVIDIA GB300", uuid: "GPU-aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee" }] : [ @@ -393,10 +394,16 @@ describe("probeDualStationVllmCapability", () => { tensorParallelSize: 2, nodeCount: 2, }, - local: { home: LOCAL_HOME, uid: 1000, gpu: { index: 0, name: "NVIDIA GB300" } }, + local: { + home: LOCAL_HOME, + uid: 1000, + gid: 1000, + gpu: { index: 0, name: "NVIDIA GB300" }, + }, peer: { home: PEER_HOME, uid: 1001, + gid: 1001, gpu: { index: 1, name: "NVIDIA GB300 Grace Blackwell Superchip" }, }, masterAddress: "192.168.240.1", @@ -869,6 +876,7 @@ describe("probe command boundary", () => { expect(options.env?.DOCKER_CONTEXT).toBe("default"); expect(options.env?.DOCKER_HOST).toBeUndefined(); expect(options.env?.DOCKER_CONFIG).toBeUndefined(); + expect(options.input).toEqual(expect.stringContaining('"gid": os.getgid()')); }); it("passes only validated discovered rail values to the fixed peer connectivity script", () => { @@ -928,4 +936,14 @@ describe("parseStationHostProbe", () => { unsafeUverbs.rails[0].uverbsDevice = "/dev/infiniband/../mem"; expect(() => parseStationHostProbe(JSON.stringify(unsafeUverbs))).toThrow(/uverbs/); }); + + it("rejects root or invalid runtime cache-owner identities", () => { + const root = hostFixture("local"); + root.uid = 0; + expect(() => parseStationHostProbe(JSON.stringify(root))).toThrow(/host probe\.uid/); + + const rootGroup = hostFixture("peer"); + rootGroup.gid = 0; + expect(() => parseStationHostProbe(JSON.stringify(rootGroup))).toThrow(/host probe\.gid/); + }); }); diff --git a/src/lib/inference/vllm-station-cluster.ts b/src/lib/inference/vllm-station-cluster.ts index a7afb37ab0..9c80ffd461 100644 --- a/src/lib/inference/vllm-station-cluster.ts +++ b/src/lib/inference/vllm-station-cluster.ts @@ -100,6 +100,7 @@ export interface StationHostProbe { architecture: string; home: string; uid: number; + gid: number; gpus: StationGpuProbe[]; docker: { reachable: boolean; @@ -178,6 +179,7 @@ export interface DualStationPlanNode { hostname: string; home: string; uid: number; + gid: number; gpu: StationGpuProbe; } @@ -535,6 +537,7 @@ payload = { "architecture": platform.machine(), "home": str(Path.home()), "uid": os.getuid(), + "gid": os.getgid(), "gpus": gpu_inventory(), "docker": docker_state(), "rsyncAvailable": shutil.which("rsync") is not None, @@ -955,7 +958,8 @@ export function parseStationHostProbe(stdout: string): StationHostProbe { productName: requireString(record.productName, "host probe.productName", 512), architecture: requireString(record.architecture, "host probe.architecture", 64), home, - uid: requireInteger(record.uid, "host probe.uid", 0, 2_147_483_647), + uid: requireInteger(record.uid, "host probe.uid", 1, 2_147_483_647), + gid: requireInteger(record.gid, "host probe.gid", 1, 2_147_483_647), gpus: requireArray(record.gpus, "host probe.gpus", 64).map((entry, index) => parseGpu(entry, `host probe.gpus[${String(index)}]`), ), @@ -1412,12 +1416,14 @@ function buildStaticPlan( hostname: local.hostname, home: local.home, uid: local.uid, + gid: local.gid, gpu: localGpu, }, peer: { hostname: peer.hostname, home: peer.home, uid: peer.uid, + gid: peer.gid, gpu: peerGpu, }, rails, diff --git a/src/lib/inference/vllm-station-model-staging.test.ts b/src/lib/inference/vllm-station-model-staging.test.ts index 9065f9da39..a39cee77ef 100644 --- a/src/lib/inference/vllm-station-model-staging.test.ts +++ b/src/lib/inference/vllm-station-model-staging.test.ts @@ -29,12 +29,14 @@ function plan(): DualStationVllmPlan { hostname: "station-a", home: "/home/nvidia", uid: 1000, + gid: 1000, gpu: { index: 0, name: "NVIDIA GB300", uuid: "GPU-a" }, }, peer: { hostname: "station-b", home: "/home/nvidia", uid: 1000, + gid: 1000, gpu: { index: 0, name: "NVIDIA GB300", uuid: "GPU-b" }, }, rails: [ From e447a845d11fd8d7a7efb82cb6f2a2d376522a2f Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 16 Jul 2026 20:49:16 -0700 Subject: [PATCH 27/74] fix(vllm): harden dual Station lifecycle Signed-off-by: Aaron Erickson --- docs/get-started/prerequisites.mdx | 3 + docs/get-started/quickstart.mdx | 3 + docs/resources/starter-prompt.md | 7 +- scripts/checks/vitest-project-overlap.ts | 1 + scripts/install.sh | 49 +- scripts/lib/dgx-station-peer.mts | 29 +- scripts/prepare-dgx-station-host.sh | 136 +++- scripts/prepare-dual-dgx-station.mts | 60 +- scripts/simulate-dual-station.mts | 56 +- src/lib/inference/local-vllm-auth.test.ts | 104 ++- src/lib/inference/local.ts | 114 +++- src/lib/inference/vllm-contracts.test.ts | 97 +++ ...llm-dual-station-simulator-command.test.ts | 37 ++ ...llm-dual-station-simulator.test-support.ts | 2 + src/lib/inference/vllm-dual-station.test.ts | 84 +++ src/lib/inference/vllm-ownership.test.ts | 32 + .../vllm-station-cluster-lifecycle.test.ts | 412 +++++++++++- .../vllm-station-cluster-lifecycle.ts | 368 ++++++++++- .../inference/vllm-station-cluster.test.ts | 151 ++++- .../vllm-station-lifecycle-lock.test.ts | 100 +++ .../inference/vllm-station-lifecycle-lock.ts | 137 +++- .../vllm-station-model-staging.test.ts | 618 +++++++++++++++++- .../inference/vllm-station-model-staging.ts | 515 ++++++++++++--- .../vllm-station-ssh-binding.test.ts | 53 +- src/lib/inference/vllm-station-ssh-binding.ts | 14 +- src/lib/inference/vllm.test.ts | 179 ++--- src/lib/inference/vllm.ts | 73 ++- ...install-station-controller-binding.test.ts | 411 ++++++++++++ test/install-station-host-preparation.test.ts | 6 + test/install-station-pair-preparation.test.ts | 288 +++++++- ...pare-dual-dgx-station-resume-state.test.ts | 144 ++++ test/test-boundary-guards.test.ts | 1 + vitest.config.ts | 2 + 33 files changed, 3929 insertions(+), 357 deletions(-) create mode 100644 src/lib/inference/vllm-contracts.test.ts create mode 100644 src/lib/inference/vllm-station-lifecycle-lock.test.ts create mode 100644 test/install-station-controller-binding.test.ts create mode 100644 test/prepare-dual-dgx-station-resume-state.test.ts diff --git a/docs/get-started/prerequisites.mdx b/docs/get-started/prerequisites.mdx index c05e73b51b..517b08e9d5 100644 --- a/docs/get-started/prerequisites.mdx +++ b/docs/get-started/prerequisites.mdx @@ -62,6 +62,9 @@ For automatic dual-Station qualification, configure both Stations before install The reciprocal addresses must have direct link routes, the expected peer MAC neighbors, and jumbo-frame connectivity in both directions. The SSH target must already have usable host-key trust and non-interactive authentication, and the selected peer account must have passwordless `sudo` for remote host preparation. NemoClaw checks only the two deterministic `/30` counterpart addresses; it does not scan other addresses, configure rails, or enroll SSH trust. +Station host preparation records the preparing non-root account's UID in root-owned `/etc/nemoclaw/dual-station-controller-uid`. +That binding prevents a different local account from managing the fixed dual-Station containers. +Rebinding requires an administrator to remove the file explicitly before preparation is rerun as the replacement account. If either host preparation requires a reboot, the installer stops with status `10` and names the host to reboot manually. Its owner-only resume state binds the exact NemoClaw revision, preparation helper, SSH host key, GPU identities, and reciprocal rails; the installer never reboots a host automatically. diff --git a/docs/get-started/quickstart.mdx b/docs/get-started/quickstart.mdx index 029f9981de..e7f89b7ec4 100644 --- a/docs/get-started/quickstart.mdx +++ b/docs/get-started/quickstart.mdx @@ -124,6 +124,9 @@ Use these details when your first-run path needs more control. At least one derived rail address must already be trusted; if both are trusted, their host keys must identify the same SSH host. The installer prepares the peer with the same reviewed helper and automatically selects the pinned `nemotron-3-ultra-550b-a55b` managed-vLLM recipe only after the two Stations pass reciprocal identity and connectivity checks across both rails. If no trusted pair qualifies and no model or peer was selected explicitly, onboarding retains the existing single-Station `deepseek-v4-flash` profile default. Complete the physical rail configuration and SSH host-key and authentication setup before installation; NemoClaw does not configure the rails or enroll SSH trust. + Station host preparation records the preparing non-root account's UID in root-owned `/etc/nemoclaw/dual-station-controller-uid`. + That binding prevents a different local account from managing the fixed dual-Station containers. + Rebinding requires an administrator to remove the file explicitly before preparation is rerun as the replacement account. DGX OS, NVIDIA BaseOS images, and other Station generations stop before host preparation. Set `NEMOCLAW_NO_EXPRESS=1`, or select a provider other than `install-vllm`, to continue on an unqualified Station without host automation. It probes the pinned driver, Docker, Buildx, and NVIDIA Container Toolkit versions, reuses exact matches, installs missing pins, and permits only the reviewed factory `dkms` transition. diff --git a/docs/resources/starter-prompt.md b/docs/resources/starter-prompt.md index 2a4b6f970f..697892d3f1 100644 --- a/docs/resources/starter-prompt.md +++ b/docs/resources/starter-prompt.md @@ -81,9 +81,12 @@ If DGX Station Express is selected: - Use managed vLLM and set `NEMOCLAW_PROVIDER=install-vllm`. - Leave `NEMOCLAW_VLLM_MODEL` unset unless the user explicitly selected a model. -- Explain that the installer checks only the deterministic counterpart on each of two configured private `/30` CX-8 rails. At least one derived address must already be trusted; if both are trusted, their host keys must identify one coherent SSH host. It selects pinned Nemotron 3 Ultra only when the reciprocal pair then qualifies; otherwise the existing single-Station DeepSeek default remains. +- Explain that the installer checks only the deterministic counterpart on each of two configured private `/30` CX-8 rails. + At least one derived address must already be trusted; if both are trusted, their host keys must identify one coherent SSH host. + It selects pinned Nemotron 3 Ultra only when the reciprocal pair then qualifies; otherwise the existing single-Station DeepSeek default remains. - If the user supplies `NEMOCLAW_DGX_STATION_PEER`, preserve that exact selection and explain that setup stops if the already-trusted peer does not qualify. -- Confirm that the physical rails and SSH host-key and authentication trust were configured outside NemoClaw. Do not configure the rails, enroll SSH trust, or approve an automatic reboot. +- Confirm that the physical rails and SSH host-key and authentication trust were configured outside NemoClaw. + Do not configure the rails, enroll SSH trust, or approve an automatic reboot. - Explain the container and selected model download before it begins, without assuming the Ultra recipe was selected. - Verify the model-cache filesystem and Docker storage have sufficient capacity. - Warn that DGX Station managed deployment has deferred end-to-end physical-hardware validation. diff --git a/scripts/checks/vitest-project-overlap.ts b/scripts/checks/vitest-project-overlap.ts index cb0bdd0a39..1191daeab0 100644 --- a/scripts/checks/vitest-project-overlap.ts +++ b/scripts/checks/vitest-project-overlap.ts @@ -46,6 +46,7 @@ const INSTALLER_INTEGRATION_TESTS = new Set([ "test/install-openshell-version-check.test.ts", "test/install-preflight-docker-bootstrap.test.ts", "test/install-preflight.test.ts", + "test/install-station-controller-binding.test.ts", "test/install-station-host-preparation.test.ts", "test/install-station-pair-preparation.test.ts", ]); diff --git a/scripts/install.sh b/scripts/install.sh index a7d44ab5be..d3b0e3ed2d 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -2893,12 +2893,14 @@ validate_express_platform_boundary() { STATION_ULTRA_VLLM_MODEL="nemotron-3-ultra-550b-a55b" STATION_ULTRA_SERVED_MODEL="nvidia/nemotron-3-ultra-550b-a55b" +STATION_ULTRA_LEGACY_VLLM_IMAGE="vllm/vllm-openai@sha256:0fec7ec5f3e6bc168e54899935fb0557da908a4832a1dbc88e2debcf2f889416" STATION_DEEPSEEK_VLLM_MODEL="deepseek-v4-flash" STATION_DEEPSEEK_SERVED_MODEL="deepseek-ai/DeepSeek-V4-Flash" _SELECTED_EXPRESS_PLATFORM="" _STATION_EXPRESS_RESUME_REVISION="" _STATION_EXPRESS_MODEL_WAS_EXPLICIT=0 _STATION_EXPRESS_DEFERRED_MANAGED_PAIR=0 +_STATION_EXPRESS_MIGRATING_LEGACY_HEAD=0 _STATION_INSTALL_MODE="" normalize_station_vllm_model() { @@ -3230,12 +3232,19 @@ run_station_host_preparation() { bash "$helper" --verify } +station_local_default_docker() { + ( + unset DOCKER_HOST DOCKER_CONTEXT + docker --context default "$@" + ) +} + station_managed_dual_head_running() { command_exists docker || return 1 local inspection name running managed role schema cluster launch_contract api_fingerprint transaction inspection="$( - docker container inspect --format \ + station_local_default_docker container inspect --format \ '{{.Name}} {{.State.Running}} {{index .Config.Labels "com.nvidia.nemoclaw.managed-vllm"}} {{index .Config.Labels "com.nvidia.nemoclaw.vllm-role"}} {{index .Config.Labels "com.nvidia.nemoclaw.vllm-launch-schema"}} {{index .Config.Labels "com.nvidia.nemoclaw.vllm-cluster"}} {{index .Config.Labels "com.nvidia.nemoclaw.vllm-launch-contract"}} {{index .Config.Labels "com.nvidia.nemoclaw.vllm-api-key-fingerprint"}} {{index .Config.Labels "com.nvidia.nemoclaw.vllm-transaction"}}' \ nemoclaw-vllm 2>/dev/null )" || return 1 @@ -3251,10 +3260,35 @@ station_managed_dual_head_running() { "$transaction" =~ ^[a-f0-9]{32}$ ]] } +station_migratable_legacy_single_head_running() { + command_exists docker || return 1 + + local inspection name running image managed role endpoint cluster gpu schema launch_contract api_fingerprint transaction + inspection="$( + station_local_default_docker container inspect --format \ + '{{.Name}}|{{.State.Running}}|{{.Config.Image}}|{{with index .Config.Labels "com.nvidia.nemoclaw.managed-vllm"}}{{.}}{{else}}-{{end}}|{{with index .Config.Labels "com.nvidia.nemoclaw.vllm-role"}}{{.}}{{else}}-{{end}}|{{with index .Config.Labels "com.nvidia.nemoclaw.vllm-endpoint"}}{{.}}{{else}}-{{end}}|{{with index .Config.Labels "com.nvidia.nemoclaw.vllm-cluster"}}{{.}}{{else}}-{{end}}|{{with index .Config.Labels "com.nvidia.nemoclaw.vllm-gpu"}}{{.}}{{else}}-{{end}}|{{with index .Config.Labels "com.nvidia.nemoclaw.vllm-launch-schema"}}{{.}}{{else}}-{{end}}|{{with index .Config.Labels "com.nvidia.nemoclaw.vllm-launch-contract"}}{{.}}{{else}}-{{end}}|{{with index .Config.Labels "com.nvidia.nemoclaw.vllm-api-key-fingerprint"}}{{.}}{{else}}-{{end}}|{{with index .Config.Labels "com.nvidia.nemoclaw.vllm-transaction"}}{{.}}{{else}}-{{end}}' \ + nemoclaw-vllm 2>/dev/null + )" || return 1 + IFS='|' read -r name running image managed role endpoint cluster gpu schema launch_contract api_fingerprint transaction <<<"$inspection" + [[ "$name" == "/nemoclaw-vllm" && + "$running" == "true" && + "$image" == "$STATION_ULTRA_LEGACY_VLLM_IMAGE" && + "$managed" == "true" && + "$role" == "-" && + "$endpoint" == "-" && + "$cluster" == "-" && + "$gpu" == "-" && + "$schema" == "-" && + "$launch_contract" == "-" && + "$api_fingerprint" == "-" && + "$transaction" == "-" ]] +} + ensure_station_express_host() { [[ "${_SELECTED_EXPRESS_PLATFORM:-}" == "DGX Station" ]] || return 0 _STATION_EXPRESS_DEFERRED_MANAGED_PAIR=0 + _STATION_EXPRESS_MIGRATING_LEGACY_HEAD=0 if station_dual_model_requested && station_managed_dual_head_running; then # The host-preparation helper intentionally refuses active workloads. Only # this complete dual-head ownership contract may defer local preparation; @@ -3264,6 +3298,14 @@ ensure_station_express_host() { info "Found a complete running NemoClaw-managed dual-Station head candidate; deferring host preparation until reciprocal pair and lifecycle validation." return 0 fi + if station_dual_model_requested && station_migratable_legacy_single_head_running; then + # This exact frozen legacy head is the only single-host workload that the + # dual lifecycle can migrate. Bind its controller without running the + # workload-rejecting host probes, then prepare the newly qualified peer. + _STATION_EXPRESS_MIGRATING_LEGACY_HEAD=1 + info "Found the exact running legacy single-Station Ultra head; deferring workload-safe controller binding and reciprocal peer preparation." + return 0 + fi info "Checking pinned DGX Station host prerequisites. Exact matches are reused." local status=0 @@ -3404,6 +3446,9 @@ ensure_station_express_pair() { if [ "${_STATION_EXPRESS_DEFERRED_MANAGED_PAIR:-0}" = "1" ]; then pair_command+=(--reuse-existing-managed-pair) fi + if [ "${_STATION_EXPRESS_MIGRATING_LEGACY_HEAD:-0}" = "1" ]; then + pair_command+=(--migrate-legacy-single-head) + fi info "Checking for one pretrusted reciprocal dual-DGX Station peer on the two direct /30 rail counterparts." # Publish the companion mode/model state before the coordinator can persist @@ -3437,6 +3482,8 @@ ensure_station_express_pair() { || error "Dual DGX Station preparation returned an inconsistent reboot result; refusing to continue." [ "${_STATION_EXPRESS_DEFERRED_MANAGED_PAIR:-0}" != "1" ] \ || error "The running managed dual-Station head could not be matched to its trusted reciprocal peer; refusing single-Station fallback." + [ "${_STATION_EXPRESS_MIGRATING_LEGACY_HEAD:-0}" != "1" ] \ + || error "The running legacy single-Station head could not be matched to a trusted reciprocal peer; refusing migration and single-Station fallback." [ -z "${NEMOCLAW_DGX_STATION_PEER:-}" ] \ || error "The explicit DGX Station peer could not be qualified; refusing single-Station fallback." station_dual_pair_resume_pending \ diff --git a/scripts/lib/dgx-station-peer.mts b/scripts/lib/dgx-station-peer.mts index cd73fd0bd2..186cf3e924 100644 --- a/scripts/lib/dgx-station-peer.mts +++ b/scripts/lib/dgx-station-peer.mts @@ -19,7 +19,7 @@ const MAC_PATTERN = /^(?:[0-9a-f]{2}:){5}[0-9a-f]{2}$/; const MAX_KNOWN_HOSTS_BYTES = 64 * 1024; const MAX_KNOWN_HOSTS_LINE_BYTES = 16 * 1024; -export type StationPrepMode = "--check" | "--apply" | "--verify"; +export type StationPrepMode = "--check" | "--apply" | "--verify" | "--bind-controller"; export interface StationIpv4Address { address: string; @@ -114,10 +114,11 @@ export interface DualStationPreparationOptions { helperSha256: string; explicitPeer?: string; reuseExistingManagedPair?: boolean; + migrateLegacySingleStationHead?: boolean; } export interface DualStationPreparationDeps { - runLocalHelper(mode: "--check" | "--verify"): number; + runLocalHelper(mode: StationPrepMode): number; probeLocalHost(): StationDiscoveryHost; inspectPretrustedTarget(target: string): PretrustedSshTarget | null; probePeerHost(target: PretrustedSshTarget): StationDiscoveryHost; @@ -760,8 +761,11 @@ export function prepareDualStationPair( if (!HOST_KEY_DIGEST_PATTERN.test(options.helperSha256)) { throw new Error("Exact Station host-preparation helper SHA-256 is required"); } + if (options.reuseExistingManagedPair && options.migrateLegacySingleStationHead) { + throw new Error("Managed-pair reuse and legacy single-head migration are mutually exclusive"); + } - if (!options.reuseExistingManagedPair) { + if (!options.reuseExistingManagedPair && !options.migrateLegacySingleStationHead) { deps.log("Checking the local Station with the reviewed host-preparation helper"); if (deps.runLocalHelper("--check") !== 0) { throw new Error("Local DGX Station host-preparation check failed before peer contact"); @@ -769,8 +773,10 @@ export function prepareDualStationPair( if (deps.runLocalHelper("--verify") !== 0) { throw new Error("Local DGX Station verification failed before peer contact"); } + } else if (options.reuseExistingManagedPair) { + deps.log("Revalidating the exact running managed pair without disrupting its workloads"); } else { - deps.log("Revalidating the exact running managed pair without host mutation"); + deps.log("Revalidating the exact running legacy single-Station head before migration"); } const resume = deps.readResumeState(); @@ -789,7 +795,10 @@ export function prepareDualStationPair( const selected = selectPretrustedTarget(options, local, resume, deps); if ("kind" in selected) return selected; const strict = Boolean( - resume || options.explicitPeer?.trim() || options.reuseExistingManagedPair, + resume || + options.explicitPeer?.trim() || + options.reuseExistingManagedPair || + options.migrateLegacySingleStationHead, ); const { binding, automatic } = selected; @@ -836,7 +845,17 @@ export function prepareDualStationPair( ...plan.identity, }; deps.writeResumeState(state); + if (options.reuseExistingManagedPair || options.migrateLegacySingleStationHead) { + deps.log("Binding the local Station controller account without disrupting managed inference"); + if (deps.runLocalHelper("--bind-controller") !== 0) { + throw new Error("Local DGX Station controller UID binding failed"); + } + } if (options.reuseExistingManagedPair) { + deps.log("Binding the reciprocal peer controller account without disrupting managed inference"); + if (deps.runRemoteHelper(binding, "--bind-controller") !== 0) { + throw new Error("Peer DGX Station controller UID binding failed"); + } deps.writeResumeState({ ...state, phase: "ready" }); return { kind: "ready", diff --git a/scripts/prepare-dgx-station-host.sh b/scripts/prepare-dgx-station-host.sh index 6ae639a67c..777e741cde 100755 --- a/scripts/prepare-dgx-station-host.sh +++ b/scripts/prepare-dgx-station-host.sh @@ -5,7 +5,7 @@ set -Eeuo pipefail umask 077 -readonly SCRIPT_VERSION="2026-07-16.5" +readonly SCRIPT_VERSION="2026-07-16.7" readonly REBOOT_REQUIRED_EXIT=10 readonly MIN_FREE_KIB=$((20 * 1024 * 1024)) # The qualified generic image currently ships this OEM telemetry bootcmd. Its @@ -35,6 +35,9 @@ readonly TARGET_DKMS_VERSION="1:3.4.0-1ubuntu1" readonly ACCEPTANCE_IMAGE="docker.io/library/ubuntu@sha256:7f622ca8766bccb22f04242ecb6f19f770b2f08827dc4b8c707de5e78a6da7ab" readonly STATE_DIR="${HOME}/.local/state/station-bootstrap" readonly INSTALL_BOOT_MARKER="${STATE_DIR}/install-boot-id" +readonly NEMOCLAW_CONFIG_DIR="/etc/nemoclaw" +readonly DUAL_STATION_CONTROLLER_UID_FILE="${NEMOCLAW_CONFIG_DIR}/dual-station-controller-uid" +readonly DUAL_STATION_CONTROLLER_UID_MODE="0644" readonly -a PACKAGE_SPECS=( "dkms=${TARGET_DKMS_VERSION}" @@ -90,11 +93,14 @@ sudo() { usage() { cat <<'EOF' -Usage: prepare-dgx-station-host.sh --check|--apply|--verify +Usage: prepare-dgx-station-host.sh --check|--apply|--verify|--bind-controller --check Read-only eligibility and current-state report. --apply Install exact prerequisites or finish post-reboot runtime setup. --verify Read-only host verification plus ephemeral GPU container tests. + --bind-controller + Bind only the current non-root controller UID without inspecting + or disrupting an already-running managed inference workload. Exit 10 from --apply means an operator-controlled reboot is required. After the reboot, run --apply once more, followed by --verify. @@ -103,7 +109,7 @@ EOF is_valid_mode() { case "${1:-}" in - --check | --apply | --verify) return 0 ;; + --check | --apply | --verify | --bind-controller) return 0 ;; *) return 1 ;; esac } @@ -589,6 +595,112 @@ install_exact_file_or_reuse() { info "${label}=installed path=${target}" } +preparation_controller_uid_for() { + local effective_uid=${1:-} sudo_uid=${2:-} uid + if [[ "$effective_uid" == "0" ]]; then + uid="$sudo_uid" + else + uid="$effective_uid" + fi + if ! [[ "$uid" =~ ^[1-9][0-9]*$ && ${#uid} -le 10 ]] || ((10#$uid > 4294967295)); then + fatal "Station preparation must be run by a non-root controller account" + fi + printf '%s\n' "$uid" +} + +preparation_controller_uid() { + preparation_controller_uid_for "$EUID" "${SUDO_UID:-}" +} + +root_directory_is_safe_unprivileged() { + local path=$1 expected_mode=${2:-} metadata uid gid mode + test ! -L "$path" || return 1 + test -d "$path" || return 1 + metadata="$(stat -c '%u %g %a' -- "$path")" || return 1 + read -r uid gid mode <<<"$metadata" + [[ "$uid" == "0" && "$gid" == "0" && "$mode" =~ ^[0-7]{3,4}$ ]] || return 1 + (((8#$mode & 0022) == 0)) || return 1 + [[ -z "$expected_mode" || "$mode" == "${expected_mode#0}" ]] +} + +root_regular_file_is_safe_unprivileged() { + local path=$1 expected_mode=$2 metadata uid gid mode + test ! -L "$path" || return 1 + test -f "$path" || return 1 + metadata="$(stat -c '%u %g %a' -- "$path")" || return 1 + read -r uid gid mode <<<"$metadata" + [[ "$uid" == "0" && "$gid" == "0" && "$mode" == "${expected_mode#0}" ]] +} + +verify_dual_station_controller_uid_binding() { + local expected_uid=${1:-} config_dir=${2:-$NEMOCLAW_CONFIG_DIR} + local binding_file=${3:-$DUAL_STATION_CONTROLLER_UID_FILE} + [[ -n "$expected_uid" ]] || expected_uid="$(preparation_controller_uid)" + root_directory_is_safe_unprivileged "$config_dir" 0755 \ + || fatal "NemoClaw configuration directory must be root-owned with mode 0755 so the prepared controller can read its binding: ${config_dir}" + root_regular_file_is_safe_unprivileged "$binding_file" "$DUAL_STATION_CONTROLLER_UID_MODE" \ + || fatal "Dual-Station controller UID binding must be a root-owned regular file with mode ${DUAL_STATION_CONTROLLER_UID_MODE}: ${binding_file}" + if ! printf '%s\n' "$expected_uid" | cmp -s - "$binding_file"; then + fatal "Dual-Station is already bound to a different controller UID; an administrator must remove ${binding_file} before rebinding" + fi + info "dual_station_controller_uid=verified uid=${expected_uid} path=${binding_file}" +} + +ensure_dual_station_controller_uid_binding() { + local config_dir=${1:-$NEMOCLAW_CONFIG_DIR} + local binding_file=${2:-$DUAL_STATION_CONTROLLER_UID_FILE} + local expected_uid parent tmp published=0 + expected_uid="$(preparation_controller_uid)" + parent="$(dirname "$config_dir")" + ensure_root_directory_safe \ + "$config_dir" "$parent" 0755 "NemoClaw configuration directory" + root_directory_is_safe_unprivileged "$config_dir" 0755 \ + || fatal "NemoClaw configuration directory must be root-owned with mode 0755 before binding the controller: ${config_dir}" + sudo test ! -L "$binding_file" \ + || fatal "Dual-Station controller UID binding must not be a symbolic link: ${binding_file}" + if sudo test -e "$binding_file"; then + verify_dual_station_controller_uid_binding "$expected_uid" "$config_dir" "$binding_file" + return 0 + fi + + tmp="$(sudo mktemp "${config_dir}/.dual-station-controller-uid.XXXXXXXXXX")" \ + || fatal "Could not create the root-owned Dual-Station controller UID candidate" + if [[ "$tmp" != "${config_dir}/.dual-station-controller-uid."* || "$tmp" == *$'\n'* ]]; then + sudo rm -f -- "$tmp" || true + fatal "Root-owned Dual-Station controller UID candidate path was invalid" + fi + if ! printf '%s\n' "$expected_uid" | sudo tee "$tmp" >/dev/null; then + sudo rm -f -- "$tmp" || true + fatal "Could not write the Dual-Station controller UID candidate" + fi + if ! sudo chown root:root "$tmp" || ! sudo chmod "$DUAL_STATION_CONTROLLER_UID_MODE" "$tmp"; then + sudo rm -f -- "$tmp" || true + fatal "Could not secure the Dual-Station controller UID candidate" + fi + if ! root_regular_file_is_safe "$tmp" "$DUAL_STATION_CONTROLLER_UID_MODE"; then + sudo rm -f -- "$tmp" || true + fatal "Dual-Station controller UID candidate metadata was unsafe" + fi + if ! printf '%s\n' "$expected_uid" | sudo cmp -s - "$tmp"; then + sudo rm -f -- "$tmp" || true + fatal "Dual-Station controller UID candidate content was invalid" + fi + if sudo ln -- "$tmp" "$binding_file" 2>/dev/null; then + published=1 + fi + sudo rm -f -- "$tmp" \ + || fatal "Could not remove the Dual-Station controller UID candidate" + if ((published == 0)); then + sudo test ! -L "$binding_file" \ + || fatal "Dual-Station controller UID binding must not be a symbolic link: ${binding_file}" + sudo test -e "$binding_file" \ + || fatal "Could not publish the Dual-Station controller UID binding without replacement" + else + info "dual_station_controller_uid=installed uid=${expected_uid} path=${binding_file}" + fi + verify_dual_station_controller_uid_binding "$expected_uid" "$config_dir" "$binding_file" +} + configure_repositories() { local tmp cuda_deb docker_asc docker_gpg docker_list tmp="$(mktemp -d)" @@ -840,6 +952,7 @@ verify_apply_state() { nvidia-ctk cdi list | grep -Fxq 'nvidia.com/gpu=all' || fatal "CDI verification failed" sudo docker image inspect "$ACCEPTANCE_IMAGE" >/dev/null 2>&1 || fatal "Digest-pinned acceptance image is missing" [[ -z "$(sudo docker ps -aq)" ]] || fatal "Verification found a leftover Docker container" + verify_dual_station_controller_uid_binding info "STATION_HOST_READY" } @@ -877,6 +990,7 @@ verify_host() { docker run --rm --device nvidia.com/gpu=all "$ACCEPTANCE_IMAGE" nvidia-smi >/dev/null docker run --rm --gpus all "$ACCEPTANCE_IMAGE" nvidia-smi >/dev/null [[ -z "$(docker ps -aq)" ]] || fatal "Verification left a Docker container behind" + verify_dual_station_controller_uid_binding info "docker=$(docker version --format '{{.Server.Version}}') expected_docker=${DOCKER_VERSION} toolkit=$(nvidia-ctk --version | head -n1) expected_toolkit=${TOOLKIT_VERSION}" info "STATION_HOST_READY" } @@ -921,6 +1035,8 @@ run_apply() { fatal "An unrelated reboot is already pending" fi + ensure_dual_station_controller_uid_binding + if ! all_packages_exact; then assert_no_package_mismatches install_packages @@ -961,6 +1077,7 @@ run_apply() { run_verify() { common_preflight + require_command cmp require_command docker require_command nvidia-ctk require_command nvidia-smi @@ -969,6 +1086,16 @@ run_verify() { verify_host } +run_bind_controller() { + require_command cmp + require_command stat + require_command sudo + check_platform + acquire_sudo + ensure_dual_station_controller_uid_binding + info "CONTROLLER_UID_BINDING_READY" +} + main() { if (($# != 1)) || ! is_valid_mode "${1:-}"; then usage >&2 @@ -977,6 +1104,8 @@ main() { MODE=$1 if [[ "$MODE" == "--apply" ]]; then setup_log + elif [[ "$MODE" == "--bind-controller" ]]; then + info "version=${SCRIPT_VERSION} mode=${MODE} log=disabled_binding_only" else info "version=${SCRIPT_VERSION} mode=${MODE} log=disabled_read_only" fi @@ -985,6 +1114,7 @@ main() { --check) run_check ;; --apply) run_apply ;; --verify) run_verify ;; + --bind-controller) run_bind_controller ;; esac } diff --git a/scripts/prepare-dual-dgx-station.mts b/scripts/prepare-dual-dgx-station.mts index 1c6d671785..b9d53e23c1 100755 --- a/scripts/prepare-dual-dgx-station.mts +++ b/scripts/prepare-dual-dgx-station.mts @@ -46,6 +46,7 @@ type CliOptions = { revision: string; explicitPeer?: string; reuseExistingManagedPair: boolean; + migrateLegacySingleStationHead: boolean; clearState: boolean; }; @@ -692,7 +693,12 @@ function connectivityMatches( export function buildRemoteHelperCommand(helperSha256: string, mode: StationPrepMode): string { if (!/^[a-f0-9]{64}$/.test(helperSha256)) throw new Error("Helper SHA-256 is invalid"); - if (mode !== "--check" && mode !== "--apply" && mode !== "--verify") { + if ( + mode !== "--check" && + mode !== "--apply" && + mode !== "--verify" && + mode !== "--bind-controller" + ) { throw new Error("Helper mode is invalid"); } return [ @@ -783,6 +789,7 @@ export function writeDualStationResumeState( if (typeof noFollow !== "number") throw new Error("O_NOFOLLOW is required for resume state"); const temporary = `${statePath}.tmp.${randomBytes(12).toString("hex")}`; let fd: number | null = null; + let failure: { error: unknown } | null = null; try { fd = fs.openSync( temporary, @@ -795,30 +802,61 @@ export function writeDualStationResumeState( fd = null; fs.renameSync(temporary, statePath); const directoryFd = fs.openSync(directory, fs.constants.O_RDONLY); + let directoryFailure: { error: unknown } | null = null; try { fs.fsyncSync(directoryFd); - } finally { + } catch (error) { + directoryFailure = { error }; + } + try { fs.closeSync(directoryFd); + } catch (error) { + directoryFailure ??= { error }; } - } finally { - if (fd !== null) fs.closeSync(fd); + if (directoryFailure) throw directoryFailure.error; + } catch (error) { + failure = { error }; + } + if (fd !== null) { try { - fs.unlinkSync(temporary); + fs.closeSync(fd); } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + failure ??= { error }; } } + try { + fs.unlinkSync(temporary); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") failure ??= { error }; + } + if (failure) throw failure.error; } export function clearDualStationResumeState(statePath: string): void { const current = readDualStationResumeState(statePath); clearDualStationSshBinding(statePath); - if (current) fs.unlinkSync(statePath); + if (!current) return; + + fs.unlinkSync(statePath); + const directoryFd = fs.openSync(path.dirname(statePath), fs.constants.O_RDONLY); + let failure: { error: unknown } | null = null; + try { + fs.fsyncSync(directoryFd); + } catch (error) { + failure = { error }; + } + try { + fs.closeSync(directoryFd); + } catch (error) { + failure ??= { error }; + } + if (failure) throw failure.error; } function parseCliOptions(args: readonly string[]): CliOptions { const values = new Map(); let reuseExistingManagedPair = false; + let migrateLegacySingleStationHead = false; let clearState = false; for (let index = 0; index < args.length; index += 1) { const arg = args[index]; @@ -826,6 +864,10 @@ function parseCliOptions(args: readonly string[]): CliOptions { reuseExistingManagedPair = true; continue; } + if (arg === "--migrate-legacy-single-head") { + migrateLegacySingleStationHead = true; + continue; + } if (arg === "--clear-state") { clearState = true; continue; @@ -858,6 +900,7 @@ function parseCliOptions(args: readonly string[]): CliOptions { revision, explicitPeer, reuseExistingManagedPair, + migrateLegacySingleStationHead, clearState, }; } @@ -930,7 +973,7 @@ function createRuntimeDeps(options: CliOptions): { return file; }; - const runLocalHelper = (mode: "--check" | "--verify"): number => { + const runLocalHelper = (mode: StationPrepMode): number => { return runStreamingCommand("bash", [pinnedHelperPath, mode], ""); }; @@ -1004,6 +1047,7 @@ export function runCli(args: readonly string[]): number { helperSha256: runtime.helperSha256, explicitPeer: options.explicitPeer, reuseExistingManagedPair: options.reuseExistingManagedPair, + migrateLegacySingleStationHead: options.migrateLegacySingleStationHead, }, runtime.deps, ); diff --git a/scripts/simulate-dual-station.mts b/scripts/simulate-dual-station.mts index 0f6f64b0be..407fc9036b 100644 --- a/scripts/simulate-dual-station.mts +++ b/scripts/simulate-dual-station.mts @@ -40,6 +40,8 @@ export const DUAL_STATION_SIMULATION_POISON_EXECUTABLES = Object.freeze([ ]); export const DUAL_STATION_SIMULATION_TIMEOUT_MS = 120_000; +export const DUAL_STATION_SIMULATION_FIXTURE_PYTHON_ENV = + "NEMOCLAW_TEST_DUAL_STATION_FIXTURE_PYTHON"; const INHERITED_ENV_KEYS = Object.freeze([ "CI", @@ -111,6 +113,52 @@ export function dualStationSimulationEnvironment( return env; } +function canonicalExecutable(candidate: string): string | null { + try { + const canonical = fs.realpathSync(candidate); + if (!path.isAbsolute(canonical) || path.normalize(canonical) !== canonical) return null; + const metadata = fs.statSync(canonical); + if (!metadata.isFile()) return null; + fs.accessSync(canonical, fs.constants.X_OK); + return canonical; + } catch { + return null; + } +} + +/** + * Capture one absolute Python interpreter for behavioral fixtures before PATH + * is poisoned. The simulator passes only this canonical path to its test + * worker; production probes and every PATH-based Python invocation remain + * guarded by the exit-97 shims. + */ +export function resolveDualStationSimulationFixturePython( + sourceEnv: NodeJS.ProcessEnv = process.env, +): string { + const captured = sourceEnv[DUAL_STATION_SIMULATION_FIXTURE_PYTHON_ENV]; + if (captured !== undefined) { + if (!path.isAbsolute(captured) || path.normalize(captured) !== captured) { + throw new Error("The dual-Station simulator fixture Python path must be absolute"); + } + const canonical = canonicalExecutable(captured); + if (!canonical || canonical !== captured) { + throw new Error("The dual-Station simulator fixture Python path is not canonical executable"); + } + return canonical; + } + + for (const directory of (sourceEnv.PATH ?? "").split(path.delimiter)) { + if (!directory || !path.isAbsolute(directory) || path.normalize(directory) !== directory) { + continue; + } + const canonical = canonicalExecutable(path.join(directory, "python3")); + if (canonical) return canonical; + } + throw new Error( + "The dual-Station simulator requires python3 for its local embedded-script fixtures", + ); +} + export function buildDualStationSimulationInvocation( repositoryRoot: string, sourceEnv: NodeJS.ProcessEnv = process.env, @@ -150,7 +198,11 @@ function describeSimulation(): void { `${JSON.stringify( { mode: "simulation-only", - localProcesses: ["repository-local Vitest worker", "fixture-only shell syntax checks"], + localProcesses: [ + "repository-local Vitest worker", + "fixture-only shell syntax checks", + "captured absolute Python interpreter for embedded-script fixtures", + ], liveTargets: [], externalCommandShims: DUAL_STATION_SIMULATION_POISON_EXECUTABLES, suites: DUAL_STATION_SIMULATION_SUITES, @@ -172,6 +224,7 @@ export function main(argv: readonly string[] = process.argv.slice(2)): number { assertDualStationSimulationPlatform(); const invocation = buildDualStationSimulationInvocation(repositoryRoot()); + const fixturePython = resolveDualStationSimulationFixturePython(invocation.env); const poisonBin = createSimulationPoisonBin(); invocation.env.PATH = [poisonBin.directory, invocation.env.PATH] .filter((entry): entry is string => Boolean(entry)) @@ -181,6 +234,7 @@ export function main(argv: readonly string[] = process.argv.slice(2)): number { invocation.env.TEMP = poisonBin.tempDirectory; invocation.env.TMP = poisonBin.tempDirectory; invocation.env.TMPDIR = poisonBin.tempDirectory; + invocation.env[DUAL_STATION_SIMULATION_FIXTURE_PYTHON_ENV] = fixturePython; delete invocation.env.XDG_RUNTIME_DIR; process.stdout.write( "Running the guarded, fixture-backed dual-Station simulator. " + diff --git a/src/lib/inference/local-vllm-auth.test.ts b/src/lib/inference/local-vllm-auth.test.ts index e8a24ac705..d097fc3df9 100644 --- a/src/lib/inference/local-vllm-auth.test.ts +++ b/src/lib/inference/local-vllm-auth.test.ts @@ -2,10 +2,17 @@ // SPDX-License-Identifier: Apache-2.0 import fs from "node:fs"; -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; + +import { DUAL_STATION_VLLM_RUNTIME } from "./vllm-station-cluster"; + +interface ManagedBaseUrlOverrides { + loadApiKey?: () => string | null; + onManagedHeadObserved?: () => void; +} const lifecycle = vi.hoisted(() => ({ - baseUrl: vi.fn<() => string | null>(), + baseUrl: vi.fn<(overrides?: ManagedBaseUrlOverrides) => string | null>(), })); vi.mock("./vllm-station-cluster-lifecycle", () => ({ @@ -19,6 +26,7 @@ import { getLocalProviderHealthCheck, getLocalProviderHealthEndpoint, getManagedDualStationVllmProviderBinding, + getManagedDualStationVllmProviderState, LOCAL_INFERENCE_SANDBOX_HOST_URL_ENV, probeLocalProviderHealth, probeVllmModels, @@ -27,11 +35,51 @@ import { const BASE_URL = "http://10.40.0.1:8000"; const API_KEY = "e".repeat(64); +const OTHER_API_KEY = "f".repeat(64); + +let actualLifecycle: typeof import("./vllm-station-cluster-lifecycle"); + +beforeAll(async () => { + actualLifecycle = await vi.importActual("./vllm-station-cluster-lifecycle"); +}); + +function productionManagedBaseUrlResolver( + expectedApiKey = API_KEY, + apiKeyFingerprint = actualLifecycle.dualStationVllmApiKeyFingerprint(expectedApiKey), +) { + const row = [ + "a".repeat(64), + actualLifecycle.DUAL_STATION_VLLM_HEAD_CONTAINER_NAME, + "running", + DUAL_STATION_VLLM_RUNTIME.image, + "true", + "head", + BASE_URL, + "b".repeat(64), + "GPU-12345678", + "1", + "c".repeat(64), + apiKeyFingerprint, + "d".repeat(32), + ].join("\t"); + + return (overrides: ManagedBaseUrlOverrides = {}): string | null => + actualLifecycle.getDualStationManagedVllmBaseUrl({ + dockerCapture: () => row, + buildLocalDockerEnv: () => ({}), + loadApiKey: overrides.loadApiKey ?? (() => null), + onManagedHeadObserved: overrides.onManagedHeadObserved ?? (() => undefined), + localInterfaceAddresses: () => ["10.40.0.1"], + }); +} beforeEach(() => { vi.stubEnv(LOCAL_INFERENCE_SANDBOX_HOST_URL_ENV, undefined); lifecycle.baseUrl.mockReset(); - lifecycle.baseUrl.mockReturnValue(BASE_URL); + lifecycle.baseUrl.mockImplementation((overrides) => { + if (!overrides?.loadApiKey) return BASE_URL; + return overrides.loadApiKey() === API_KEY ? BASE_URL : null; + }); }); afterEach(() => vi.unstubAllEnvs()); @@ -98,6 +146,11 @@ describe("managed dual-Station vLLM authentication", () => { baseUrl: `${BASE_URL}/v1`, apiKey: API_KEY, }); + expect(getManagedDualStationVllmProviderState({ loadApiKeyImpl: () => API_KEY })).toEqual({ + kind: "ready", + baseUrl: `${BASE_URL}/v1`, + apiKey: API_KEY, + }); }); it("uses /health only for unauthenticated availability checks", () => { @@ -203,9 +256,10 @@ describe("managed dual-Station vLLM authentication", () => { expect(result?.detail).toContain("different/model"); }); - it("fails closed before model inventory when the managed key is absent", () => { + it("fails closed through production lifecycle recovery when the managed key is absent", () => { const runCurlProbeImpl = vi.fn(); const result = probeLocalProviderHealth("vllm-local", { + getManagedVllmBaseUrlImpl: productionManagedBaseUrlResolver(), model: "required/model", loadVllmApiKeyImpl: () => null, runCurlProbeImpl, @@ -216,4 +270,46 @@ describe("managed dual-Station vLLM authentication", () => { expect(result?.failureLabel).toBe("unauthorized"); expect(result?.detail).not.toContain(API_KEY); }); + + it("fails closed through production lifecycle recovery when managed key state is unsafe", () => { + const runCurlProbeImpl = vi.fn(); + const result = probeLocalProviderHealth("vllm-local", { + getManagedVllmBaseUrlImpl: productionManagedBaseUrlResolver(), + model: "required/model", + loadVllmApiKeyImpl: () => { + throw new Error(`unsafe ${API_KEY}`); + }, + runCurlProbeImpl, + }); + + expect(runCurlProbeImpl).not.toHaveBeenCalled(); + expect(result?.ok).toBe(false); + expect(result?.failureLabel).toBe("unhealthy"); + expect(result?.detail).not.toContain(API_KEY); + }); + + it("returns invalid-auth when the private key does not match the managed lifecycle", () => { + expect( + getManagedDualStationVllmProviderState({ + getManagedBaseUrlImpl: productionManagedBaseUrlResolver(OTHER_API_KEY), + loadApiKeyImpl: () => API_KEY, + }), + ).toEqual({ kind: "invalid-auth", reason: "mismatched" }); + }); + + it("fails closed when an owned managed head has invalid auth fingerprint metadata", () => { + const runCurlProbeImpl = vi.fn(); + const result = probeLocalProviderHealth("vllm-local", { + getManagedVllmBaseUrlImpl: productionManagedBaseUrlResolver(API_KEY, ""), + loadVllmApiKeyImpl: () => API_KEY, + runCurlProbeImpl, + }); + + expect(runCurlProbeImpl).not.toHaveBeenCalled(); + expect(result).toMatchObject({ + ok: false, + endpoint: "managed dual-Station vLLM", + failureLabel: "unhealthy", + }); + }); }); diff --git a/src/lib/inference/local.ts b/src/lib/inference/local.ts index b611acc9d3..a6428dc285 100644 --- a/src/lib/inference/local.ts +++ b/src/lib/inference/local.ts @@ -254,6 +254,8 @@ export interface LocalProviderHealthProbeOptions { loadOllamaProxyTokenImpl?: () => string | null; /** Reads the managed dual-Station vLLM key. Injectable so tests stay deterministic. */ loadVllmApiKeyImpl?: () => string | null; + /** Recovers an owned managed endpoint while validating the injected key in one lifecycle read. */ + getManagedVllmBaseUrlImpl?: ManagedDualStationVllmBaseUrlResolver; } function defaultLoadOllamaProxyToken(): string | null { @@ -415,26 +417,84 @@ export interface ManagedDualStationVllmProviderBinding { apiKey: string; } +type ManagedDualStationVllmBaseUrlResolver = (overrides?: { + loadApiKey?: () => string | null; + onManagedHeadObserved?: () => void; +}) => string | null; + +export type ManagedDualStationVllmProviderState = + | { kind: "absent" } + | { kind: "invalid-auth"; reason: "missing" | "unsafe" | "mismatched" } + | ({ kind: "ready" } & ManagedDualStationVllmProviderBinding); + export interface ManagedDualStationVllmProviderBindingOptions { hostUrl?: string | null; - getManagedBaseUrlImpl?: () => string | null; + getManagedBaseUrlImpl?: ManagedDualStationVllmBaseUrlResolver; loadApiKeyImpl?: () => string | null; } -/** Recover one endpoint+credential pair only from an owned managed-dual container. */ -export function getManagedDualStationVllmProviderBinding( +/** Recover endpoint and credential as one lifecycle-validated state. */ +export function getManagedDualStationVllmProviderState( options: ManagedDualStationVllmProviderBindingOptions = {}, -): ManagedDualStationVllmProviderBinding | null { +): ManagedDualStationVllmProviderState { const configuredHostUrl = configuredLocalInferenceHostUrl(options.hostUrl); - if (configuredHostUrl) return null; + if (configuredHostUrl) return { kind: "absent" }; + + const loadApiKey = options.loadApiKeyImpl ?? loadDualStationVllmApiKey; + let keyRead = false; + let managedHeadObserved = false; + let apiKey: string | null = null; + let authFailure: "missing" | "unsafe" | null = null; + let managedBaseUrl: string | null; + try { + managedBaseUrl = (options.getManagedBaseUrlImpl ?? getDualStationManagedVllmBaseUrl)({ + onManagedHeadObserved: () => { + managedHeadObserved = true; + }, + loadApiKey: () => { + keyRead = true; + try { + apiKey = loadApiKey(); + if (!apiKey) authFailure = "missing"; + return apiKey; + } catch { + authFailure = "unsafe"; + return null; + } + }, + }); + } catch (error) { + // A malformed key can make lifecycle fingerprint validation throw. Once + // key recovery began, treat every such failure as unsafe authentication; + // unrelated endpoint-inspection failures retain their existing behavior. + if (keyRead) return { kind: "invalid-auth", reason: "unsafe" }; + throw error; + } - const managedBaseUrl = (options.getManagedBaseUrlImpl ?? getDualStationManagedVllmBaseUrl)(); - if (!managedBaseUrl) return null; - const apiKey = (options.loadApiKeyImpl ?? loadDualStationVllmApiKey)(); - if (!apiKey) { + // Production recovery reports a structurally owned managed head before it + // validates fingerprint metadata. Test resolvers may still signal ownership + // by invoking the key reader, so only the absence of both signals permits the + // legacy single-host path. + if (!managedHeadObserved && !keyRead) return { kind: "absent" }; + if (!managedBaseUrl || !apiKey) { + return { kind: "invalid-auth", reason: authFailure ?? "mismatched" }; + } + return { kind: "ready", baseUrl: `${managedBaseUrl}/v1`, apiKey }; +} + +/** Compatibility binding for onboarding and context-window callers. */ +export function getManagedDualStationVllmProviderBinding( + options: ManagedDualStationVllmProviderBindingOptions = {}, +): ManagedDualStationVllmProviderBinding | null { + const state = getManagedDualStationVllmProviderState(options); + if (state.kind === "absent") return null; + if (state.kind === "invalid-auth") { + if (state.reason !== "missing") { + throw new Error("Managed dual-Station vLLM authentication is unsafe or mismatched."); + } throw new Error("Managed dual-Station vLLM authentication is missing."); } - return { baseUrl: `${managedBaseUrl}/v1`, apiKey }; + return { baseUrl: state.baseUrl, apiKey: state.apiKey }; } export function getLocalProviderBaseUrl( @@ -673,29 +733,39 @@ export function probeLocalProviderHealth( const providerLabel = getLocalProviderLabel(provider); if (!providerLabel) return null; - let managedBinding: ManagedDualStationVllmProviderBinding | null = null; + let managedState: ManagedDualStationVllmProviderState = { kind: "absent" }; if (provider === "vllm-local") { try { - managedBinding = getManagedDualStationVllmProviderBinding({ + managedState = getManagedDualStationVllmProviderState({ + getManagedBaseUrlImpl: options.getManagedVllmBaseUrlImpl, loadApiKeyImpl: options.loadVllmApiKeyImpl, }); - } catch (error) { - const missingAuth = - error instanceof Error && - error.message === "Managed dual-Station vLLM authentication is missing."; + } catch { return { ok: false, providerLabel, - endpoint: - getLocalProviderHealthEndpoint(provider) ?? `http://127.0.0.1:${VLLM_PORT}/v1/models`, - failureLabel: missingAuth ? "unauthorized" : "unhealthy", + endpoint: "managed dual-Station vLLM", + failureLabel: "unhealthy", probeLabel: "vllm backend", - detail: missingAuth - ? "Local vLLM requires its managed bearer credential, but no private key is available. Re-run `nemoclaw onboard` to repair the dual-Station provider." - : "Local vLLM authentication state is unsafe or unreadable. Re-run `nemoclaw onboard` to repair the managed dual-Station provider.", + detail: + "Local vLLM authentication state could not be inspected safely. Re-run `nemoclaw onboard` to repair the managed dual-Station provider.", }; } } + if (managedState.kind === "invalid-auth") { + const missingAuth = managedState.reason === "missing"; + return { + ok: false, + providerLabel, + endpoint: "managed dual-Station vLLM", + failureLabel: missingAuth ? "unauthorized" : "unhealthy", + probeLabel: "vllm backend", + detail: missingAuth + ? "Local vLLM requires its managed bearer credential, but no private key is available. Re-run `nemoclaw onboard` to repair the dual-Station provider." + : "Local vLLM authentication state is unsafe or does not match the managed service. Re-run `nemoclaw onboard` to repair the managed dual-Station provider.", + }; + } + const managedBinding = managedState.kind === "ready" ? managedState : null; const endpoint = managedBinding ? `${managedBinding.baseUrl}/models` : provider === "vllm-local" diff --git a/src/lib/inference/vllm-contracts.test.ts b/src/lib/inference/vllm-contracts.test.ts new file mode 100644 index 0000000000..c63334e0e1 --- /dev/null +++ b/src/lib/inference/vllm-contracts.test.ts @@ -0,0 +1,97 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + dockerPullWithProgressWatchdog: vi.fn(), +})); + +vi.mock("../adapters/docker", async (importOriginal) => ({ + ...(await importOriginal()), + dockerPullWithProgressWatchdog: mocks.dockerPullWithProgressWatchdog, +})); + +import { + assertVllmRegistryDigestRef, + detectVllmProfile, + pullImage, + resolveVllmServedModelId, + VLLM_IMAGES, +} from "./vllm"; +import { VLLM_MODELS } from "./vllm-models"; + +beforeEach(() => vi.clearAllMocks()); + +describe("vLLM served route identity", () => { + it("uses one safe served-model override and rejects ambiguous aliases (#6315)", () => { + expect(resolveVllmServedModelId("catalog/model", [])).toBe("catalog/model"); + expect(resolveVllmServedModelId("catalog/model", ["--served-model-name", "served/model"])).toBe( + "served/model", + ); + expect(() => + resolveVllmServedModelId("catalog/model", [ + "--served-model-name", + "served/one", + "served/two", + ]), + ).toThrow("exactly one safe model ID"); + }); +}); + +describe("managed vLLM image distribution boundary", () => { + const digest = `sha256:${"a".repeat(64)}`; + + it("accepts repository-qualified immutable registry digests", () => { + expect(() => assertVllmRegistryDigestRef(`vllm/vllm-openai@${digest}`)).not.toThrow(); + expect(() => + assertVllmRegistryDigestRef(`registry.example.test:5000/team/runtime@${digest}`), + ).not.toThrow(); + }); + + it.each([ + `sha256:${"a".repeat(64)}`, + "vllm/vllm-openai:latest", + `ubuntu@${digest}`, + `vllm/vllm-openai@sha256:${"A".repeat(64)}`, + `vllm/vllm-openai@${digest}suffix`, + ` vllm/vllm-openai@${digest}`, + `vllm/vllm-openai@${digest} `, + ])("rejects an unpullable or mutable product image reference %j", (image) => { + expect(() => assertVllmRegistryDigestRef(image)).toThrow( + /pullable immutable registry reference/, + ); + }); + + it("keeps every shipped managed-vLLM image on a registry digest", () => { + const platformRefs = Object.values(VLLM_IMAGES).flatMap((imageSet) => + Object.values(imageSet) + .map((value) => + typeof value === "object" && value !== null && "ref" in value ? String(value.ref) : null, + ) + .filter((ref): ref is string => ref !== null), + ); + const runtimeRefs = VLLM_MODELS.map((model) => model.runtime?.image).filter( + (ref): ref is string => typeof ref === "string", + ); + const refs = new Set([...platformRefs, ...runtimeRefs]); + + expect(refs.size).toBeGreaterThan(0); + for (const ref of refs) { + expect(() => assertVllmRegistryDigestRef(ref), ref).not.toThrow(); + } + }); + + it("refuses a local image ID before invoking Docker pull", async () => { + const profile = { + ...detectVllmProfile({ platform: "station", type: "nvidia" })!, + image: `sha256:${"a".repeat(64)}`, + }; + + await expect(pullImage(profile)).resolves.toEqual({ + ok: false, + reason: expect.stringContaining("Local image IDs"), + }); + expect(mocks.dockerPullWithProgressWatchdog).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/inference/vllm-dual-station-simulator-command.test.ts b/src/lib/inference/vllm-dual-station-simulator-command.test.ts index c5e2c2b857..1da2697a94 100644 --- a/src/lib/inference/vllm-dual-station-simulator-command.test.ts +++ b/src/lib/inference/vllm-dual-station-simulator-command.test.ts @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { spawnSync } from "node:child_process"; import fs from "node:fs"; import path from "node:path"; @@ -10,10 +11,12 @@ import { assertDualStationSimulationPlatform, buildDualStationSimulationInvocation, createSimulationPoisonBin, + DUAL_STATION_SIMULATION_FIXTURE_PYTHON_ENV, DUAL_STATION_SIMULATION_POISON_EXECUTABLES, DUAL_STATION_SIMULATION_SUITES, dualStationSimulationEnvironment, main, + resolveDualStationSimulationFixturePython, } from "../../../scripts/simulate-dual-station.mts"; describe("dual-Station simulator command", () => { @@ -52,6 +55,39 @@ describe("dual-Station simulator command", () => { expect(fs.existsSync(poisonBin.directory)).toBe(false); }); + it("uses one captured absolute Python only for embedded fixtures", () => { + const fixturePython = resolveDualStationSimulationFixturePython(); + const poisonBin = createSimulationPoisonBin(); + const env = dualStationSimulationEnvironment(process.env); + env.PATH = [poisonBin.directory, env.PATH] + .filter((entry): entry is string => Boolean(entry)) + .join(path.delimiter); + env.HOME = poisonBin.homeDirectory; + env.TMPDIR = poisonBin.tempDirectory; + env[DUAL_STATION_SIMULATION_FIXTURE_PYTHON_ENV] = fixturePython; + + try { + expect(path.isAbsolute(fixturePython)).toBe(true); + expect(resolveDualStationSimulationFixturePython(env)).toBe(fixturePython); + + const guarded = spawnSync("python3", ["-c", "print('must-not-run')"], { + encoding: "utf8", + env, + }); + expect(guarded.status, guarded.stderr).toBe(97); + expect(guarded.stderr).toContain("simulator blocked external command: python3"); + + const fixture = spawnSync(fixturePython, ["-c", "print('fixture-ok')"], { + encoding: "utf8", + env, + }); + expect(fixture.status, fixture.stderr).toBe(0); + expect(fixture.stdout.trim()).toBe("fixture-ok"); + } finally { + poisonBin.cleanup(); + } + }); + it("inherits only local process basics and forces live projects off", () => { const env = dualStationSimulationEnvironment({ PATH: "/fixture/bin", @@ -65,6 +101,7 @@ describe("dual-Station simulator command", () => { HF_TOKEN: "should-not-leak", NEMOCLAW_DGX_STATION_PEER: "should-not-run", NEMOCLAW_DGX_STATION_SSH_BINDING: "should-not-run", + [DUAL_STATION_SIMULATION_FIXTURE_PYTHON_ENV]: "/should/not/inherit/python3", NEMOCLAW_RUN_BRANCH_VALIDATION_E2E: "1", NEMOCLAW_RUN_LIVE_E2E: "1", UNRELATED_AMBIENT_VALUE: "should-not-inherit", diff --git a/src/lib/inference/vllm-dual-station-simulator.test-support.ts b/src/lib/inference/vllm-dual-station-simulator.test-support.ts index 039297a68f..500bd9ce5e 100644 --- a/src/lib/inference/vllm-dual-station-simulator.test-support.ts +++ b/src/lib/inference/vllm-dual-station-simulator.test-support.ts @@ -260,6 +260,8 @@ export function createDualStationLifecycleSimulator() { }), createProbeNonce: () => (++nonceCounter).toString(16).padStart(32, "0"), createTransactionId: () => (++transactionCounter).toString(16).padStart(32, "0"), + effectiveControllerUid: () => stationHost("local").uid, + readControllerUid: () => stationHost("local").uid, waitBeforeReconcile: async () => undefined, withLifecycleLock: async (operation: () => Promise | T): Promise => await operation(), loadApiKey: () => DUAL_STATION_SIMULATOR_API_KEY, diff --git a/src/lib/inference/vllm-dual-station.test.ts b/src/lib/inference/vllm-dual-station.test.ts index 1e5162ded9..b8ccf398b2 100644 --- a/src/lib/inference/vllm-dual-station.test.ts +++ b/src/lib/inference/vllm-dual-station.test.ts @@ -11,6 +11,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; const mocks = vi.hoisted(() => ({ areContainersRunning: vi.fn(), cleanup: vi.fn(), + commitLegacyMigration: vi.fn(), dockerCapture: vi.fn(), dockerForceRm: vi.fn(), dockerImageInspectFormat: vi.fn(), @@ -31,6 +32,7 @@ const mocks = vi.hoisted(() => ({ probeHostStorage: vi.fn(), runCapture: vi.fn(), runCurlProbe: vi.fn(), + rollbackLegacyMigration: vi.fn(), startManaged: vi.fn(), stageModelSnapshot: vi.fn(), withLifecycle: vi.fn(), @@ -79,9 +81,11 @@ vi.mock("./vllm-station-model-staging", () => ({ vi.mock("./vllm-station-cluster-lifecycle", () => ({ areDualStationManagedVllmContainersRunning: mocks.areContainersRunning, cleanupDualStationManagedVllm: mocks.cleanup, + commitDualStationLegacyMigration: mocks.commitLegacyMigration, getDualStationManagedVllmBaseUrl: mocks.getManagedBaseUrl, preflightDualStationGpuRuntime: mocks.preflightGpuRuntime, preflightDualStationManagedVllm: mocks.preflightOwnership, + rollbackDualStationLegacyMigration: mocks.rollbackLegacyMigration, startDualStationManagedVllm: mocks.startManaged, withDualStationManagedVllmLifecycle: mocks.withLifecycle, })); @@ -102,6 +106,13 @@ const API_KEY = "ab".repeat(32); const HEAD_ID = "a".repeat(64); const WORKER_ID = "b".repeat(64); const HEAD_BASE_URL = "http://192.168.100.1:8000"; +const LEGACY_MIGRATION = { + backupContainerName: `nemoclaw-vllm-legacy-${"1".repeat(32)}`, + legacyContainerId: "c".repeat(64), + transactionId: "1".repeat(32), + headContainerId: HEAD_ID, + workerContainerId: WORKER_ID, +}; function plan(): DualStationVllmPlan { return { @@ -221,6 +232,8 @@ beforeEach(() => { mocks.withLifecycle.mockImplementation(async (operation) => await operation()); mocks.areContainersRunning.mockReturnValue(true); mocks.cleanup.mockReturnValue({ ok: true, removedContainerIds: [] }); + mocks.commitLegacyMigration.mockResolvedValue({ ok: true, cleanupWarnings: [] }); + mocks.rollbackLegacyMigration.mockResolvedValue({ ok: true }); mocks.findUnwritableTreePath.mockReturnValue(null); mocks.measureDirectorySizeBytes.mockReturnValue(0n); mocks.probeDockerStorage.mockReturnValue({ @@ -570,6 +583,77 @@ describe("dual DGX Station vLLM install orchestration", () => { ); }); + it("commits legacy retirement only after readiness, auth, and final pair validation", async () => { + mocks.startManaged.mockReturnValue({ + ok: true, + baseUrl: HEAD_BASE_URL, + headContainerId: HEAD_ID, + workerContainerId: WORKER_ID, + reusedExisting: false, + legacyMigration: LEGACY_MIGRATION, + }); + const profile = detectVllmProfile({ platform: "station", type: "nvidia" }); + + await expect( + installVllm(profile!, { hasImage: true, nonInteractive: true, promptFn: vi.fn() }), + ).resolves.toEqual({ ok: true }); + + expect(mocks.commitLegacyMigration).toHaveBeenCalledWith(plan(), LEGACY_MIGRATION); + expect(mocks.commitLegacyMigration.mock.invocationCallOrder[0]).toBeGreaterThan( + mocks.areContainersRunning.mock.invocationCallOrder[0], + ); + expect(mocks.commitLegacyMigration.mock.invocationCallOrder[0]).toBeGreaterThan( + mocks.runCurlProbe.mock.invocationCallOrder.at(-1) ?? 0, + ); + expect(mocks.rollbackLegacyMigration).not.toHaveBeenCalled(); + expect(mocks.cleanup).not.toHaveBeenCalled(); + }); + + it.each([ + "readiness", + "authentication", + "final running check", + "commit validation", + ])("restores legacy state when external %s fails", async (failure) => { + mocks.startManaged.mockReturnValue({ + ok: true, + baseUrl: HEAD_BASE_URL, + headContainerId: HEAD_ID, + workerContainerId: WORKER_ID, + reusedExisting: false, + legacyMigration: LEGACY_MIGRATION, + }); + if (failure === "readiness") { + mocks.runCurlProbe.mockReturnValue({ ok: false, httpStatus: 503, message: "loading" }); + mocks.dockerCapture.mockReturnValue(""); + } else if (failure === "authentication") { + mocks.runCurlProbe.mockImplementation((args: string[]) => ({ + ok: true, + httpStatus: 200, + message: "ok", + body: args.at(-1)?.endsWith("/v1/models") + ? JSON.stringify({ data: [{ id: "nvidia/nemotron-3-ultra-550b-a55b" }] }) + : "", + })); + } else if (failure === "final running check") { + mocks.areContainersRunning.mockReturnValue(false); + } else { + mocks.commitLegacyMigration.mockResolvedValue({ + ok: false, + reason: "new dual-Station transaction changed before commit", + }); + } + const profile = detectVllmProfile({ platform: "station", type: "nvidia" }); + + await expect( + installVllm(profile!, { hasImage: true, nonInteractive: true, promptFn: vi.fn() }), + ).resolves.toEqual({ ok: false }); + + expect(mocks.rollbackLegacyMigration).toHaveBeenCalledWith(plan(), LEGACY_MIGRATION); + expect(mocks.cleanup).not.toHaveBeenCalled(); + expect(mocks.commitLegacyMigration.mock.calls.length > 0).toBe(failure === "commit validation"); + }); + it("rolls back a new pair when unauthenticated model inventory is exposed", async () => { mocks.runCurlProbe.mockImplementation((args: string[]) => ({ ok: true, diff --git a/src/lib/inference/vllm-ownership.test.ts b/src/lib/inference/vllm-ownership.test.ts index 744f0d295d..41e92c6902 100644 --- a/src/lib/inference/vllm-ownership.test.ts +++ b/src/lib/inference/vllm-ownership.test.ts @@ -84,6 +84,38 @@ describe("managed vLLM ownership", () => { expect(isNemoClawManagedVllmRunning()).toBe(true); }); + it("checks the canonical local daemon before an ambient remote Docker host", () => { + const previousDockerHost = process.env.DOCKER_HOST; + const previousDockerContext = process.env.DOCKER_CONTEXT; + process.env.DOCKER_HOST = "ssh://builder.example.test"; + delete process.env.DOCKER_CONTEXT; + mocks.dockerCapture.mockImplementation( + (_args: readonly string[], options?: { env?: NodeJS.ProcessEnv }) => + options?.env?.DOCKER_CONTEXT === "default" + ? vllmContainerRow(NEMOCLAW_VLLM_CONTAINER_NAME, { + state: "running", + dualRole: "head", + dualEndpoint: "http://192.168.100.1:8000", + dualCluster: "f".repeat(64), + }) + : "", + ); + + try { + expect(isNemoClawManagedVllmRunning()).toBe(true); + expect(mocks.dockerCapture).toHaveBeenCalledTimes(1); + expect(mocks.dockerCapture.mock.calls[0]?.[1]?.env).toMatchObject({ + DOCKER_CONTEXT: "default", + }); + expect(mocks.dockerCapture.mock.calls[0]?.[1]?.env).not.toHaveProperty("DOCKER_HOST"); + } finally { + if (previousDockerHost === undefined) delete process.env.DOCKER_HOST; + else process.env.DOCKER_HOST = previousDockerHost; + if (previousDockerContext === undefined) delete process.env.DOCKER_CONTEXT; + else process.env.DOCKER_CONTEXT = previousDockerContext; + } + }); + it.each([ vllmContainerRow(NEMOCLAW_VLLM_CONTAINER_NAME), vllmContainerRow(NEMOCLAW_VLLM_CONTAINER_NAME, { label: "" }), diff --git a/src/lib/inference/vllm-station-cluster-lifecycle.test.ts b/src/lib/inference/vllm-station-cluster-lifecycle.test.ts index 25323c3638..3401ee32ab 100644 --- a/src/lib/inference/vllm-station-cluster-lifecycle.test.ts +++ b/src/lib/inference/vllm-station-cluster-lifecycle.test.ts @@ -3,6 +3,8 @@ import { AsyncLocalStorage } from "node:async_hooks"; import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { DUAL_STATION_VLLM_RUNTIME, type DualStationVllmPlan } from "./vllm-station-cluster"; import { @@ -10,6 +12,7 @@ import { buildDualStationGpuSmokeRunArgs, buildDualStationVllmRunArgs, cleanupDualStationManagedVllm, + commitDualStationLegacyMigration, DUAL_STATION_VLLM_API_KEY_FINGERPRINT_LABEL, DUAL_STATION_VLLM_CLUSTER_LABEL, DUAL_STATION_VLLM_ENDPOINT_LABEL, @@ -30,9 +33,11 @@ import { getDualStationManagedVllmBaseUrl, preflightDualStationGpuRuntime, preflightDualStationManagedVllm, + rollbackDualStationLegacyMigration, startDualStationManagedVllm, withDualStationManagedVllmLifecycle, } from "./vllm-station-cluster-lifecycle"; +import { withDualStationVllmLifecycleLock } from "./vllm-station-lifecycle-lock"; import type { DualStationSshBinding } from "./vllm-station-ssh-binding"; import { createDualStationSshBindingFixture, @@ -41,6 +46,7 @@ import { const WORKER_ID = "a".repeat(64); const HEAD_ID = "b".repeat(64); +const LEGACY_HEAD_ID = "9".repeat(64); const WORKER_SMOKE_ID = "c".repeat(64); const HEAD_SMOKE_ID = "d".repeat(64); const API_KEY = "e".repeat(64); @@ -195,20 +201,26 @@ function fakeContainer( }; } -function harness( - options: { - failRole?: "head" | "worker"; - invalidIdRole?: "head" | "worker"; - failSmokeTarget?: "local" | "peer"; - failSmokeCleanupTarget?: "local" | "peer"; - missingImageTarget?: "local" | "peer"; - smokeGpuOutput?: Partial>; - lateCreateRole?: "head" | "worker"; - failedRoleForeignTransaction?: "head" | "worker"; - } = {}, -) { +type HarnessOptions = { + failRole?: "head" | "worker"; + invalidIdRole?: "head" | "worker"; + failSmokeTarget?: "local" | "peer"; + failSmokeCleanupTarget?: "local" | "peer"; + missingImageTarget?: "local" | "peer"; + smokeGpuOutput?: Partial>; + lateCreateRole?: "head" | "worker"; + failedRoleForeignTransaction?: "head" | "worker"; + failFinalInspectionRole?: "head" | "worker"; + failLegacyBackupRemoval?: boolean; +}; + +function harness(options: HarnessOptions = {}) { const containers = new Map(); - const operations: Array<{ kind: "capture" | "rm" | "run"; target: string; value: string }> = []; + const operations: Array<{ + kind: "capture" | "rename" | "rm" | "run" | "start" | "stop"; + target: string; + value: string; + }> = []; const captureOptions: Array = []; const rmOptions: Array = []; const runCalls: Array<{ @@ -227,6 +239,9 @@ function harness( let lifecycleLockTail = Promise.resolve(); const lifecycleLockContext = new AsyncLocalStorage(); let lateContainer: { targetName: string; container: FakeContainer } | null = null; + const launchedRoles = new Set<"head" | "worker">(); + const managedInspectionCounts = { head: 0, worker: 0 }; + let finalInspectionFailureInjected = false; async function acquireLifecycleLock(operation: () => Promise | T): Promise { const previous = lifecycleLockTail; @@ -253,6 +268,18 @@ function harness( return `${targetName}:${name}`; } + function exactContainerById( + targetName: string, + containerId: string, + ): { containerKey: string; entries: FakeContainer[]; container: FakeContainer } | null { + for (const [containerKey, entries] of containers.entries()) { + if (!containerKey.startsWith(`${targetName}:`)) continue; + const container = entries.find((entry) => entry.id === containerId); + if (container) return { containerKey, entries, container }; + } + return null; + } + const deps: DualStationVllmLifecycleDeps = { buildLocalDockerEnv: () => ({ TARGET: "local", @@ -267,6 +294,8 @@ function harness( transactionCounter += 1; return transactionCounter.toString(16).padStart(32, "0"); }, + effectiveControllerUid: () => fixturePlan().local.uid, + readControllerUid: () => fixturePlan().local.uid, loadApiKey: () => API_KEY, localInterfaceAddresses: () => [fixturePlan().masterAddress], waitBeforeReconcile: async () => { @@ -281,6 +310,35 @@ function harness( dockerCapture: (args, optionsArg) => { captureOptions.push(optionsArg); const targetName = target(optionsArg); + if (args[0] === "container" && args[1] === "rename") { + const containerId = args[2]; + const newName = args[3]; + operations.push({ + kind: "rename", + target: targetName, + value: `${containerId}:${newName}`, + }); + const located = exactContainerById(targetName, containerId); + if (!located || (containers.get(key(targetName, newName)) ?? []).length > 0) { + return raise("rename failed"); + } + containers.set( + located.containerKey, + located.entries.filter((entry) => entry.id !== containerId), + ); + located.container.name = newName; + containers.set(key(targetName, newName), [located.container]); + return ""; + } + if (args[0] === "container" && (args[1] === "start" || args[1] === "stop")) { + const action = args[1]; + const containerId = args.at(-1) ?? ""; + operations.push({ kind: action, target: targetName, value: containerId }); + const located = exactContainerById(targetName, containerId); + if (!located) return raise(`${action} failed`); + located.container.state = action === "start" ? "running" : "exited"; + return containerId; + } switch (args[0]) { case "image": { operations.push({ kind: "capture", target: targetName, value: `image:${args.at(-1)}` }); @@ -305,7 +363,25 @@ function harness( operations.push({ kind: "capture", target: targetName, value: name }); const isSmokeInspection = dockerValues(args, "--format")[0]?.includes(DUAL_STATION_VLLM_GPU_SMOKE_LABEL) ?? false; - return (containers.get(key(targetName, name)) ?? []) + let inspected = containers.get(key(targetName, name)) ?? []; + const inspectedRole = + name === DUAL_STATION_VLLM_HEAD_CONTAINER_NAME + ? "head" + : name === DUAL_STATION_VLLM_WORKER_CONTAINER_NAME + ? "worker" + : null; + if (!isSmokeInspection && inspectedRole && launchedRoles.has(inspectedRole)) { + managedInspectionCounts[inspectedRole] += 1; + if ( + options.failFinalInspectionRole === inspectedRole && + managedInspectionCounts[inspectedRole] === 2 && + !finalInspectionFailureInjected + ) { + finalInspectionFailureInjected = true; + inspected = inspected.map((container) => ({ ...container, state: "exited" })); + } + } + return inspected .map((container) => isSmokeInspection ? [ @@ -349,6 +425,7 @@ function harness( break; } const role = name === DUAL_STATION_VLLM_HEAD_CONTAINER_NAME ? "head" : "worker"; + launchedRoles.add(role); const imageIndex = args.indexOf("/bin/bash") + 1; const container = fakeContainer(role, { image: args[imageIndex], @@ -381,8 +458,9 @@ function harness( const targetName = target(optionsArg); operations.push({ kind: "rm", target: targetName, value: containerId }); const shouldFail = - options.failSmokeCleanupTarget === targetName && - (containerId === WORKER_SMOKE_ID || containerId === HEAD_SMOKE_ID); + (options.failSmokeCleanupTarget === targetName && + (containerId === WORKER_SMOKE_ID || containerId === HEAD_SMOKE_ID)) || + (options.failLegacyBackupRemoval && containerId === LEGACY_HEAD_ID); const match = [...containers.entries()].find( ([containerKey, entries]) => containerKey.startsWith(`${targetName}:`) && @@ -417,6 +495,29 @@ function harness( }; } +type LifecycleHarness = ReturnType; + +function seedLegacyHead(fake: LifecycleHarness): void { + fake.seed( + "local", + fakeContainer("head", { + id: LEGACY_HEAD_ID, + labels: { [DUAL_STATION_VLLM_MANAGED_LABEL]: "true" }, + }), + ); +} + +function expectRestoredLegacyHead(fake: LifecycleHarness): void { + expect(fake.containers.get(`local:${DUAL_STATION_VLLM_HEAD_CONTAINER_NAME}`)).toEqual([ + expect.objectContaining({ + id: LEGACY_HEAD_ID, + name: DUAL_STATION_VLLM_HEAD_CONTAINER_NAME, + state: "running", + }), + ]); + expect(fake.containers.get(`peer:${DUAL_STATION_VLLM_WORKER_CONTAINER_NAME}`) ?? []).toEqual([]); +} + describe("dual-Station managed vLLM run argv", () => { it("derives stable, distinct service-key fingerprints", () => { expect(API_KEY_FINGERPRINT).toMatch(/^[a-f0-9]{64}$/u); @@ -630,6 +731,70 @@ describe("dual-Station managed vLLM run argv", () => { }); describe("dual-Station managed vLLM lifecycle", () => { + it("rejects an effective account that differs from the prepared controller", () => { + const fake = harness(); + + expect( + preflightDualStationManagedVllm(fixturePlan(), { + ...fake.deps, + effectiveControllerUid: () => fixturePlan().local.uid + 1, + }), + ).toEqual({ + ok: false, + reason: + "Dual-Station lifecycle effective UID 1001 does not match prepared controller UID 1000", + }); + expect(fake.operations).toEqual([]); + }); + + it("rejects a prepared controller account that does not own the probed local Station plan", () => { + const fake = harness(); + + expect( + preflightDualStationManagedVllm(fixturePlan(), { + ...fake.deps, + effectiveControllerUid: () => fixturePlan().local.uid + 1, + readControllerUid: () => fixturePlan().local.uid + 1, + }), + ).toEqual({ + ok: false, + reason: "Dual-Station lifecycle controller UID must match probed local UID 1000", + }); + expect(fake.operations).toEqual([]); + }); + + it("anchors the default lock under the effective account home instead of mutable HOME", async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-station-lock-home-")); + const accountHome = path.join(root, "account-home"); + const ambientHome = path.join(root, "ambient-home"); + fs.mkdirSync(accountHome, { mode: 0o700 }); + const userInfo = os.userInfo(); + const userInfoSpy = vi.spyOn(os, "userInfo").mockReturnValue({ + ...userInfo, + homedir: accountHome, + }); + vi.stubEnv("HOME", ambientHome); + try { + await withDualStationVllmLifecycleLock( + () => { + expect( + fs.existsSync(path.join(accountHome, ".nemoclaw", "state", "mcp-lifecycle-locks")), + ).toBe(true); + expect(fs.existsSync(ambientHome)).toBe(false); + }, + { pollIntervalMs: 5, timeoutMs: 250, corruptLockGraceMs: 5 }, + { + readControllerUid: () => userInfo.uid, + effectiveControllerUid: () => userInfo.uid, + }, + ); + } finally { + vi.unstubAllEnvs(); + userInfoSpy.mockRestore(); + fs.rmSync(root, { recursive: true, force: true }); + } + }); + it("provides a read-only ownership preflight before download work", () => { const fake = harness(); @@ -905,26 +1070,157 @@ describe("dual-Station managed vLLM lifecycle", () => { expect(fake.containers.get(`peer:${DUAL_STATION_VLLM_WORKER_CONTAINER_NAME}`)).toHaveLength(1); }); - it("migrates only the exact legacy single-Station head when the peer name is absent", async () => { + it("migrates only the exact running legacy single-Station head from the rollback window", async () => { + const fake = harness(); + seedLegacyHead(fake); + const started = await startDualStationManagedVllm(fixturePlan(), START_CONFIG, fake.deps); + expect(started).toMatchObject({ + ok: true, + reusedExisting: false, + legacyMigration: expect.objectContaining({ legacyContainerId: LEGACY_HEAD_ID }), + }); + if (!started.ok || !started.legacyMigration) + throw new Error("expected legacy migration handle"); + expect( + fake.operations.some(({ kind, value }) => kind === "rm" && value === LEGACY_HEAD_ID), + ).toBe(false); + await expect( + commitDualStationLegacyMigration(fixturePlan(), started.legacyMigration, fake.deps), + ).resolves.toEqual({ ok: true, cleanupWarnings: [] }); + const cutoverOrder = fake.operations + .filter( + ({ kind, value }) => + (kind === "run" && + (value === DUAL_STATION_VLLM_WORKER_CONTAINER_NAME || + value === DUAL_STATION_VLLM_HEAD_CONTAINER_NAME)) || + kind === "rename" || + ((kind === "stop" || kind === "rm") && value === LEGACY_HEAD_ID), + ) + .map(({ kind, value }) => `${kind}:${value}`); + expect(cutoverOrder).toEqual([ + `run:${DUAL_STATION_VLLM_WORKER_CONTAINER_NAME}`, + expect.stringMatching(`^rename:${LEGACY_HEAD_ID}:nemoclaw-vllm-legacy-`), + `stop:${LEGACY_HEAD_ID}`, + `run:${DUAL_STATION_VLLM_HEAD_CONTAINER_NAME}`, + `rm:${LEGACY_HEAD_ID}`, + ]); + }); + + it("restores the preserved legacy head when external validation rolls back", async () => { const fake = harness(); + seedLegacyHead(fake); + const started = await startDualStationManagedVllm(fixturePlan(), START_CONFIG, fake.deps); + if (!started.ok || !started.legacyMigration) + throw new Error("expected legacy migration handle"); + + await expect( + rollbackDualStationLegacyMigration(fixturePlan(), started.legacyMigration, fake.deps), + ).resolves.toEqual({ ok: true }); + expectRestoredLegacyHead(fake); + }); + + it("keeps the running legacy head untouched when the peer worker cannot start", async () => { + const fake = harness({ failRole: "worker" }); + seedLegacyHead(fake); + expect(await startDualStationManagedVllm(fixturePlan(), START_CONFIG, fake.deps)).toEqual({ + ok: false, + reason: "worker container failed to start", + rollbackErrors: [], + }); + expectRestoredLegacyHead(fake); + expect(fake.operations.some(({ kind }) => ["rename", "start", "stop"].includes(kind))).toBe( + false, + ); + expect( + fake.operations.some(({ kind, value }) => kind === "rm" && value === LEGACY_HEAD_ID), + ).toBe(false); + }); + + it.each([ + ["new head launch", { failRole: "head" }, "head container failed to start", false], + [ + "final pair verification", + { failFinalInspectionRole: "head" }, + "dual-Station containers did not remain running", + true, + ], + ] as const)("restores the exact legacy head after %s failure", async (_case, options, reason, ranHead) => { + const fake = harness(options); + seedLegacyHead(fake); + expect(await startDualStationManagedVllm(fixturePlan(), START_CONFIG, fake.deps)).toEqual({ + ok: false, + reason, + rollbackErrors: [], + }); + expectRestoredLegacyHead(fake); + expect(fake.operations).toEqual( + expect.arrayContaining([ + { kind: "rm", target: "peer", value: WORKER_ID }, + { kind: "start", target: "local", value: LEGACY_HEAD_ID }, + ]), + ); + expect(fake.operations.some(({ kind, value }) => kind === "rm" && value === HEAD_ID)).toBe( + ranHead, + ); + expect( + fake.operations.some(({ kind, value }) => kind === "rm" && value === LEGACY_HEAD_ID), + ).toBe(false); + }); + + it("keeps the validated new pair when legacy backup removal is ambiguous", async () => { + const fake = harness({ failLegacyBackupRemoval: true }); + seedLegacyHead(fake); + const started = await startDualStationManagedVllm(fixturePlan(), START_CONFIG, fake.deps); + if (!started.ok || !started.legacyMigration) + throw new Error("expected legacy migration handle"); + await expect( + commitDualStationLegacyMigration(fixturePlan(), started.legacyMigration, fake.deps), + ).resolves.toMatchObject({ + ok: true, + cleanupWarnings: [expect.stringContaining("legacy backup")], + }); + expect(fake.containers.get(`local:${DUAL_STATION_VLLM_HEAD_CONTAINER_NAME}`)).toEqual([ + expect.objectContaining({ id: HEAD_ID, state: "running" }), + ]); + expect(fake.containers.get(`peer:${DUAL_STATION_VLLM_WORKER_CONTAINER_NAME}`)).toEqual([ + expect.objectContaining({ id: WORKER_ID, state: "running" }), + ]); + const preservedBackup = [...fake.containers.entries()].flatMap(([containerKey, entries]) => + containerKey.startsWith(`local:${DUAL_STATION_VLLM_HEAD_CONTAINER_NAME}-legacy-`) + ? entries + : [], + ); + expect(preservedBackup).toEqual([ + expect.objectContaining({ id: LEGACY_HEAD_ID, state: "exited" }), + ]); + }); + + it.each([ + ["stopped", { state: "exited" }], + [ + "outside the frozen image window", + { + image: + "vllm/vllm-openai@sha256:2222222222222222222222222222222222222222222222222222222222222222", + }, + ], + ])("refuses a schema-less managed head that is %s", async (_case, override) => { + const fake = harness(); + const plan = fixturePlan(); fake.seed( "local", fakeContainer("head", { + ...override, labels: { [DUAL_STATION_VLLM_MANAGED_LABEL]: "true" }, }), ); - expect(await startDualStationManagedVllm(fixturePlan(), START_CONFIG, fake.deps)).toMatchObject( - { - ok: true, - reusedExisting: false, - }, - ); - expect(fake.operations.filter((operation) => operation.kind === "rm")).toContainEqual({ - kind: "rm", - target: "local", - value: HEAD_ID, + expect(await startDualStationManagedVllm(plan, START_CONFIG, fake.deps)).toMatchObject({ + ok: false, + reason: expect.stringContaining("foreign"), }); + expect(fake.operations.some((operation) => operation.kind === "rm")).toBe(false); + expect(fake.operations.some((operation) => operation.kind === "run")).toBe(false); }); it("refuses a same-image worker that belongs to another physical cluster plan", () => { @@ -978,6 +1274,45 @@ describe("dual-Station managed vLLM lifecycle", () => { ]); }); + it("uses the real reentrant lease across an outer lifecycle and start rollback", async () => { + const fake = harness({ failRole: "head" }); + const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-station-real-lock-")); + const withRealLock: DualStationVllmLifecycleDeps["withLifecycleLock"] = (operation) => + withDualStationVllmLifecycleLock( + operation, + { + stateDir, + pollIntervalMs: 5, + timeoutMs: 250, + corruptLockGraceMs: 5, + }, + { + readControllerUid: () => fixturePlan().local.uid, + effectiveControllerUid: () => fixturePlan().local.uid, + }, + ); + const deps = { ...fake.deps, withLifecycleLock: withRealLock }; + try { + expect( + await withDualStationManagedVllmLifecycle( + () => startDualStationManagedVllm(fixturePlan(), START_CONFIG, deps), + deps, + ), + ).toEqual({ + ok: false, + reason: "head container failed to start", + rollbackErrors: [], + }); + expect(fake.operations.filter((operation) => operation.kind === "rm")).toContainEqual({ + kind: "rm", + target: "peer", + value: WORKER_ID, + }); + } finally { + fs.rmSync(stateDir, { recursive: true, force: true }); + } + }, 2_000); + it("rejects an invalid docker-run ID and recovers only its exact owned ID", async () => { const fake = harness({ invalidIdRole: "worker" }); @@ -1042,6 +1377,25 @@ describe("managed dual-Station base URL recovery", () => { expect(fake.captureOptions.at(-1)?.env?.VLLM_API_KEY).toBeUndefined(); }); + it("reports a structurally managed running head before API-key fingerprint validation", () => { + const fake = harness(); + const head = fakeContainer("head"); + head.labels[DUAL_STATION_VLLM_API_KEY_FINGERPRINT_LABEL] = "invalid"; + fake.seed("local", head); + const onManagedHeadObserved = vi.fn(); + const loadApiKey = vi.fn(() => API_KEY); + + expect( + getDualStationManagedVllmBaseUrl({ + ...fake.deps, + onManagedHeadObserved, + loadApiKey, + }), + ).toBeNull(); + expect(onManagedHeadObserved).toHaveBeenCalledOnce(); + expect(loadApiKey).not.toHaveBeenCalled(); + }); + it.each([ ["missing persisted key", { loadApiKey: () => null }], ["mismatched persisted key", { loadApiKey: () => "a".repeat(64) }], @@ -1075,6 +1429,7 @@ describe("managed dual-Station base URL recovery", () => { it("rejects an owned-looking head that does not use the pinned runtime image", () => { const fake = harness(); + const onManagedHeadObserved = vi.fn(); fake.seed( "local", fakeContainer("head", { @@ -1083,6 +1438,7 @@ describe("managed dual-Station base URL recovery", () => { }), ); - expect(getDualStationManagedVllmBaseUrl(fake.deps)).toBeNull(); + expect(getDualStationManagedVllmBaseUrl({ ...fake.deps, onManagedHeadObserved })).toBeNull(); + expect(onManagedHeadObserved).not.toHaveBeenCalled(); }); }); diff --git a/src/lib/inference/vllm-station-cluster-lifecycle.ts b/src/lib/inference/vllm-station-cluster-lifecycle.ts index 86fa4f2519..70cfedcafe 100644 --- a/src/lib/inference/vllm-station-cluster-lifecycle.ts +++ b/src/lib/inference/vllm-station-cluster-lifecycle.ts @@ -11,7 +11,11 @@ import { DUAL_STATION_VLLM_API_KEY_PATTERN, loadDualStationVllmApiKey } from "./ import { buildLocalDualStationDockerEnv, buildRemoteVllmDockerEnv } from "./vllm-docker-env"; import { buildNemotronUltraDistributedServeCommand } from "./vllm-models"; import { DUAL_STATION_VLLM_RUNTIME, type DualStationVllmPlan } from "./vllm-station-cluster"; -import { withDualStationVllmLifecycleLock } from "./vllm-station-lifecycle-lock"; +import { + assertDualStationControllerAccount, + readDualStationControllerUid, + withDualStationVllmLifecycleLock, +} from "./vllm-station-lifecycle-lock"; import { assertDualStationSshBindingFiles, type DualStationSshBinding, @@ -56,6 +60,11 @@ const TRANSACTION_ID_PATTERN = /^[a-f0-9]{32}$/; const GPU_SMOKE_CONTAINER_PREFIX = "nemoclaw-vllm-gpu-smoke"; const DUAL_STATION_VLLM_LAUNCH_SCHEMA = "1"; const VLLM_FINGERPRINT_CONTEXT = "nemoclaw-dual-station-vllm-api-key\0"; +// Compatibility bridge for schema-less single-Station Ultra containers from +// the v0.0.86 rollback window before dual launch schema 1. +// Do not add future image digests; retire this branch when v0.0.86 support ends. +const LEGACY_SINGLE_STATION_MIGRATION_IMAGE = + "vllm/vllm-openai@sha256:0fec7ec5f3e6bc168e54899935fb0557da908a4832a1dbc88e2debcf2f889416"; export type DualStationVllmRole = "head" | "worker"; @@ -85,6 +94,9 @@ export interface DualStationVllmLifecycleDeps { buildRemoteDockerEnv(binding: DualStationSshBinding): Record; createProbeNonce(): string; createTransactionId(): string; + effectiveControllerUid(): number | null; + readControllerUid(): number; + onManagedHeadObserved?(): void; waitBeforeReconcile(ms: number): Promise; withLifecycleLock(operation: () => Promise | T): Promise; loadApiKey(): string | null; @@ -96,6 +108,14 @@ export interface DualStationVllmStartConfig { apiKey: string; } +export interface DualStationLegacyMigration { + backupContainerName: string; + legacyContainerId: string; + transactionId: string; + headContainerId: string; + workerContainerId: string; +} + export type StartDualStationVllmResult = | { ok: true; @@ -104,6 +124,8 @@ export type StartDualStationVllmResult = workerContainerId: string; /** True when an already-running exact owned pair was left untouched. */ reusedExisting: boolean; + /** Present until the caller commits or rolls back external validation. */ + legacyMigration?: DualStationLegacyMigration; } | { ok: false; reason: string; rollbackErrors: string[] }; @@ -113,6 +135,14 @@ export type CleanupDualStationVllmResult = export type PreflightDualStationVllmResult = { ok: true } | { ok: false; reason: string }; +export type CommitDualStationLegacyMigrationResult = + | { ok: true; cleanupWarnings: string[] } + | { ok: false; reason: string }; + +export type RollbackDualStationLegacyMigrationResult = + | { ok: true } + | { ok: false; rollbackErrors: string[] }; + type ManagedContainerSpec = { role: DualStationVllmRole; name: string; @@ -137,6 +167,12 @@ type ManagedContainerInspection = | { kind: "legacy-managed"; containerId: string; running: boolean } | { kind: "foreign" | "ambiguous" | "unknown" }; +type LegacyHeadCutover = { + originalSpec: ManagedContainerSpec; + backupSpec: ManagedContainerSpec; + containerId: string; +}; + type GpuSmokeSpec = { role: DualStationVllmRole; containerName: string; @@ -159,6 +195,9 @@ const DEFAULT_DEPS: DualStationVllmLifecycleDeps = { buildRemoteDockerEnv: buildRemoteVllmDockerEnv, createProbeNonce: () => randomBytes(16).toString("hex"), createTransactionId: () => randomBytes(16).toString("hex"), + effectiveControllerUid: () => process.getuid?.() ?? null, + readControllerUid: readDualStationControllerUid, + onManagedHeadObserved: () => undefined, waitBeforeReconcile: (ms) => new Promise((resolve) => setTimeout(resolve, ms)), withLifecycleLock: withDualStationVllmLifecycleLock, loadApiKey: loadDualStationVllmApiKey, @@ -553,6 +592,15 @@ function specsForPlan( config?: DualStationVllmStartConfig, ): { head: ManagedContainerSpec; worker: ManagedContainerSpec } { assertSafePlan(plan); + const controllerUid = assertDualStationControllerAccount( + deps.readControllerUid, + deps.effectiveControllerUid, + ); + if (controllerUid !== plan.local.uid) { + throw new Error( + `Dual-Station lifecycle controller UID must match probed local UID ${String(plan.local.uid)}`, + ); + } const clusterId = clusterIdForPlan(plan); const apiKeyFingerprint = config ? dualStationVllmApiKeyFingerprint(config.apiKey) : null; return { @@ -629,6 +677,7 @@ function inspectRows( function inspectManagedContainer( spec: ManagedContainerSpec, deps: DualStationVllmLifecycleDeps, + options: { allowStoppedLegacy?: boolean } = {}, ): ManagedContainerInspection { const rows = inspectRows(spec.name, spec.env, deps); if (rows === null) return { kind: "unknown" }; @@ -656,6 +705,8 @@ function inspectManagedContainer( spec.role === "head" && name === spec.name && image === spec.image && + image === LEGACY_SINGLE_STATION_MIGRATION_IMAGE && + (state === "running" || (options.allowStoppedLegacy && state === "exited")) && managed === "true" && !role && !endpoint && @@ -918,6 +969,189 @@ function removeExact( } } +function runExactContainerMutation( + spec: ManagedContainerSpec, + args: readonly string[], + deps: DualStationVllmLifecycleDeps, +): void { + deps.dockerCapture(["container", ...args], { + env: spec.env, + timeout: DOCKER_MUTATION_TIMEOUT_MS, + }); +} + +function exactLegacyGeneration( + spec: ManagedContainerSpec, + containerId: string, + deps: DualStationVllmLifecycleDeps, +): Extract | null { + const inspection = inspectManagedContainer(spec, deps, { allowStoppedLegacy: true }); + return inspection.kind === "legacy-managed" && inspection.containerId === containerId + ? inspection + : null; +} + +function restoreLegacyHead( + cutover: LegacyHeadCutover, + deps: DualStationVllmLifecycleDeps, +): string | null { + let original = exactLegacyGeneration(cutover.originalSpec, cutover.containerId, deps); + let backup = exactLegacyGeneration(cutover.backupSpec, cutover.containerId, deps); + + if (original && backup) { + return "legacy head restoration found the exact container ID under two names"; + } + if (!original) { + if (!backup) return "legacy head restoration could not find the exact owned container ID"; + if (inspectManagedContainer(cutover.originalSpec, deps).kind !== "absent") { + return "legacy head restoration found the original name occupied"; + } + try { + runExactContainerMutation( + cutover.backupSpec, + ["rename", cutover.containerId, cutover.originalSpec.name], + deps, + ); + } catch { + // Docker may commit a rename before its client reports failure. Reconcile + // the exact generation below instead of trusting the command response. + } + original = exactLegacyGeneration(cutover.originalSpec, cutover.containerId, deps); + backup = exactLegacyGeneration(cutover.backupSpec, cutover.containerId, deps); + if (!original || backup) { + return "legacy head restoration could not restore the exact original container name"; + } + } + + if (!original.running) { + try { + runExactContainerMutation(cutover.originalSpec, ["start", cutover.containerId], deps); + } catch { + // As with rename, validate the exact postcondition rather than the Docker + // client's response so a committed start is not reported as lost. + } + original = exactLegacyGeneration(cutover.originalSpec, cutover.containerId, deps); + } + return original?.running + ? null + : "legacy head restoration could not restart the exact owned container ID"; +} + +function prepareLegacyHeadCutover( + spec: ManagedContainerSpec, + containerId: string, + transactionId: string, + deps: DualStationVllmLifecycleDeps, +): + | { ok: true; cutover: LegacyHeadCutover } + | { ok: false; reason: string; rollbackErrors: string[] } { + const cutover: LegacyHeadCutover = { + originalSpec: spec, + backupSpec: { ...spec, name: `${spec.name}-legacy-${transactionId}` }, + containerId, + }; + const fail = (reason: string) => { + const restoreError = restoreLegacyHead(cutover, deps); + return { + ok: false as const, + reason, + rollbackErrors: restoreError ? [restoreError] : [], + }; + }; + + const original = exactLegacyGeneration(spec, containerId, deps); + if (!original?.running) { + return { + ok: false, + reason: "legacy head changed before transactional cutover", + rollbackErrors: [], + }; + } + if (inspectManagedContainer(cutover.backupSpec, deps).kind !== "absent") { + return { + ok: false, + reason: "legacy head backup name is not absent", + rollbackErrors: [], + }; + } + + try { + runExactContainerMutation(spec, ["rename", containerId, cutover.backupSpec.name], deps); + } catch { + // Reconcile below: rename can succeed even when the client times out. + } + const renamed = exactLegacyGeneration(cutover.backupSpec, containerId, deps); + if (!renamed?.running || inspectManagedContainer(cutover.originalSpec, deps).kind !== "absent") { + return fail("failed to preserve the exact legacy head under its transaction backup name"); + } + + try { + runExactContainerMutation(cutover.backupSpec, ["stop", "--time", "30", containerId], deps); + } catch { + // Reconcile and restore below if the exact generation is not stopped. + } + const stopped = exactLegacyGeneration(cutover.backupSpec, containerId, deps); + if (!stopped || stopped.running) { + return fail("failed to stop the preserved exact legacy head for cutover"); + } + return { ok: true, cutover }; +} + +function rollbackNewPairAndRestoreLegacy( + entries: readonly { spec: ManagedContainerSpec; containerId: string }[], + transactionId: string, + cutover: LegacyHeadCutover, + deps: DualStationVllmLifecycleDeps, +): string[] { + const errors = rollbackExact(entries, transactionId, deps); + const restoreError = restoreLegacyHead(cutover, deps); + if (restoreError) errors.push(restoreError); + return errors; +} + +function finalizeLegacyHeadCutover( + cutover: LegacyHeadCutover, + deps: DualStationVllmLifecycleDeps, +): string | null { + const initial = inspectManagedContainer(cutover.backupSpec, deps, { + allowStoppedLegacy: true, + }); + if (initial.kind === "absent") return null; + const backup = exactLegacyGeneration(cutover.backupSpec, cutover.containerId, deps); + if (!backup || backup.running) { + return "could not verify the stopped exact legacy backup for post-commit cleanup"; + } + const removed = removeExact(cutover.backupSpec, cutover.containerId, deps); + const after = inspectManagedContainer(cutover.backupSpec, deps); + // A Docker client error can race a committed removal. Accept either the + // successful exact-ID mutation or an exact-name absence on reconciliation. + return removed || after.kind === "absent" + ? null + : "could not remove the stopped exact legacy backup after the new pair committed"; +} + +function legacyCutoverFromMigration( + specs: ReturnType, + migration: DualStationLegacyMigration, +): LegacyHeadCutover | null { + if ( + !TRANSACTION_ID_PATTERN.test(migration.transactionId) || + !DOCKER_CONTAINER_ID_PATTERN.test(migration.legacyContainerId) || + !DOCKER_CONTAINER_ID_PATTERN.test(migration.headContainerId) || + !DOCKER_CONTAINER_ID_PATTERN.test(migration.workerContainerId) || + new Set([migration.legacyContainerId, migration.headContainerId, migration.workerContainerId]) + .size !== 3 || + migration.backupContainerName !== `${specs.head.name}-legacy-${migration.transactionId}` + ) { + return null; + } + return { + originalSpec: specs.head, + backupSpec: { ...specs.head, name: migration.backupContainerName }, + containerId: migration.legacyContainerId, + }; +} + function removeTransactionExact( spec: ManagedContainerSpec, containerId: string, @@ -1198,10 +1432,9 @@ export async function startDualStationManagedVllm( [specs.head, existingHead], [specs.worker, existingWorker], ] as const) { - if ( - (inspection.kind === "managed" || inspection.kind === "legacy-managed") && - !removeExact(spec, inspection.containerId, deps) - ) { + // A legacy head remains live until the newly created peer worker is + // proven running. Its cutover is handled transactionally below. + if (inspection.kind === "managed" && !removeExact(spec, inspection.containerId, deps)) { return { ok: false, reason: `failed to remove existing owned ${spec.role} container`, @@ -1222,6 +1455,31 @@ export async function startDualStationManagedVllm( return { ok: false, reason: "worker container failed to start", rollbackErrors }; } + let legacyCutover: LegacyHeadCutover | null = null; + if (existingHead.kind === "legacy-managed") { + const prepared = prepareLegacyHeadCutover( + specs.head, + existingHead.containerId, + transactionId, + deps, + ); + if (!prepared.ok) { + return { + ok: false, + reason: prepared.reason, + rollbackErrors: [ + ...rollbackExact( + [{ spec: specs.worker, containerId: worker.containerId }], + transactionId, + deps, + ), + ...prepared.rollbackErrors, + ], + }; + } + legacyCutover = prepared.cutover; + } + const head = await startOne(plan, specs.head, config, transactionId, deps); if (!head.ok) { const rollback = [ @@ -1231,7 +1489,9 @@ export async function startDualStationManagedVllm( return { ok: false, reason: "head container failed to start", - rollbackErrors: rollbackExact(rollback, transactionId, deps), + rollbackErrors: legacyCutover + ? rollbackNewPairAndRestoreLegacy(rollback, transactionId, legacyCutover, deps) + : rollbackExact(rollback, transactionId, deps), }; } @@ -1249,17 +1509,16 @@ export async function startDualStationManagedVllm( finalWorker.transactionId !== transactionId || finalWorker.containerId !== worker.containerId ) { + const rollback = [ + { spec: specs.head, containerId: head.containerId }, + { spec: specs.worker, containerId: worker.containerId }, + ]; return { ok: false, reason: "dual-Station containers did not remain running", - rollbackErrors: rollbackExact( - [ - { spec: specs.head, containerId: head.containerId }, - { spec: specs.worker, containerId: worker.containerId }, - ], - transactionId, - deps, - ), + rollbackErrors: legacyCutover + ? rollbackNewPairAndRestoreLegacy(rollback, transactionId, legacyCutover, deps) + : rollbackExact(rollback, transactionId, deps), }; } @@ -1269,6 +1528,17 @@ export async function startDualStationManagedVllm( headContainerId: head.containerId, workerContainerId: worker.containerId, reusedExisting: false, + ...(legacyCutover + ? { + legacyMigration: { + backupContainerName: legacyCutover.backupSpec.name, + legacyContainerId: legacyCutover.containerId, + transactionId, + headContainerId: head.containerId, + workerContainerId: worker.containerId, + }, + } + : {}), }; }); } catch (error) { @@ -1280,6 +1550,73 @@ export async function startDualStationManagedVllm( } } +/** Retire the preserved legacy generation only after external readiness/auth commit. */ +export async function commitDualStationLegacyMigration( + plan: DualStationVllmPlan, + migration: DualStationLegacyMigration, + overrides: Partial = {}, +): Promise { + const deps = depsWith(overrides); + try { + const specs = specsForPlan(plan, deps); + const cutover = legacyCutoverFromMigration(specs, migration); + if (!cutover) return { ok: false, reason: "legacy migration handle is invalid" }; + return await deps.withLifecycleLock(() => { + const head = inspectManagedContainer(specs.head, deps); + const worker = inspectManagedContainer(specs.worker, deps); + if ( + head.kind !== "managed" || + !head.running || + !head.reusable || + head.containerId !== migration.headContainerId || + head.transactionId !== migration.transactionId || + worker.kind !== "managed" || + !worker.running || + !worker.reusable || + worker.containerId !== migration.workerContainerId || + worker.transactionId !== migration.transactionId + ) { + return { ok: false, reason: "new dual-Station transaction changed before commit" }; + } + const warning = finalizeLegacyHeadCutover(cutover, deps); + return { ok: true, cleanupWarnings: warning ? [warning] : [] }; + }); + } catch (error) { + return { ok: false, reason: `legacy migration commit failed: ${(error as Error).message}` }; + } +} + +/** Remove only the new transaction and restore its exact preserved legacy generation. */ +export async function rollbackDualStationLegacyMigration( + plan: DualStationVllmPlan, + migration: DualStationLegacyMigration, + overrides: Partial = {}, +): Promise { + const deps = depsWith(overrides); + try { + const specs = specsForPlan(plan, deps); + const cutover = legacyCutoverFromMigration(specs, migration); + if (!cutover) return { ok: false, rollbackErrors: ["legacy migration handle is invalid"] }; + return await deps.withLifecycleLock(() => { + const rollbackErrors = rollbackNewPairAndRestoreLegacy( + [ + { spec: specs.head, containerId: migration.headContainerId }, + { spec: specs.worker, containerId: migration.workerContainerId }, + ], + migration.transactionId, + cutover, + deps, + ); + return rollbackErrors.length === 0 ? { ok: true } : { ok: false, rollbackErrors }; + }); + } catch (error) { + return { + ok: false, + rollbackErrors: [`legacy migration rollback failed: ${(error as Error).message}`], + }; + } +} + /** Remove only containers whose complete dual-Station ownership tuple matches. */ function cleanupDualStationManagedVllmUnlocked( plan: DualStationVllmPlan, @@ -1412,11 +1749,12 @@ export function getDualStationManagedVllmBaseUrl( !SAFE_GPU_UUID_PATTERN.test(gpuUuid) || launchSchema !== DUAL_STATION_VLLM_LAUNCH_SCHEMA || !SHA256_HEX_PATTERN.test(launchContract) || - !SHA256_HEX_PATTERN.test(apiKeyFingerprint) || !TRANSACTION_ID_PATTERN.test(transactionId) ) { return null; } + deps.onManagedHeadObserved?.(); + if (!SHA256_HEX_PATTERN.test(apiKeyFingerprint)) return null; let apiKey: string | null; try { apiKey = deps.loadApiKey(); diff --git a/src/lib/inference/vllm-station-cluster.test.ts b/src/lib/inference/vllm-station-cluster.test.ts index a6c19553c5..fa929b94db 100644 --- a/src/lib/inference/vllm-station-cluster.test.ts +++ b/src/lib/inference/vllm-station-cluster.test.ts @@ -2,11 +2,15 @@ // SPDX-License-Identifier: Apache-2.0 import assert from "node:assert/strict"; -import type { SpawnSyncOptionsWithStringEncoding } from "node:child_process"; +import { type SpawnSyncOptionsWithStringEncoding, spawnSync } from "node:child_process"; import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { resolveDualStationSimulationFixturePython } from "../../../scripts/simulate-dual-station.mts"; + import { buildRemoteVllmDockerEnv } from "./vllm-docker-env"; import { createStationClusterProbeDeps, @@ -63,6 +67,7 @@ beforeEach(() => { }); afterEach(() => { + vi.unstubAllEnvs(); sshFixture.cleanup(); }); @@ -825,21 +830,142 @@ describe("probe command boundary", () => { expect(args.join(" ")).not.toMatch(/keyscan|accept-new|StrictHostKeyChecking=no/); expect(options.input).toEqual(expect.stringContaining('docker", "info')); expect(options.input).toEqual(expect.stringContaining("/sys/firmware/devicetree/base/model")); - expect(options.input).toEqual(expect.stringContaining("shard_path.is_absolute()")); - expect(options.input).toEqual(expect.stringContaining('shutil.which("rsync")')); - expect(options.input).toEqual(expect.stringContaining('"directoryExists"')); - expect(options.input).toEqual(expect.stringContaining("len(shards) != 113")); - expect(options.input).toEqual( - expect.stringContaining("observed_tensor_size != expected_total_size"), - ); - expect(options.input).toEqual(expect.stringContaining('"infiniband_verbs"')); - expect(options.input).toEqual(expect.stringContaining('re.fullmatch(r"uverbs[0-9]+"')); - expect(options.input).toEqual(expect.stringContaining("len(names) != 1")); - expect(options.input).toEqual(expect.stringContaining("stat.S_ISCHR")); expect(options.timeout).toBe(20_000); expect(options.maxBuffer).toBe(1024 * 1024); }); + it("executes the host probe and reports malformed weights plus missing staging tools", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-station-probe-fixture-")); + const home = path.join(root, "home"); + const bin = path.join(root, "bin"); + const snapshot = snapshotPath(home); + fs.mkdirSync(snapshot, { mode: 0o700, recursive: true }); + fs.mkdirSync(bin, { mode: 0o700 }); + fs.writeFileSync(path.join(snapshot, "config.json"), "{}"); + fs.writeFileSync(path.join(snapshot, "tokenizer.json"), "{}"); + const shards = Array.from( + { length: 113 }, + (_, index) => `model-${String(index + 1).padStart(5, "0")}-of-00113.safetensors`, + ); + fs.writeFileSync( + path.join(snapshot, "model.safetensors.index.json"), + JSON.stringify({ + metadata: { total_size: 1 }, + weight_map: Object.fromEntries( + shards.map((shard, index) => [`model.layers.${String(index)}.weight`, shard]), + ), + }), + ); + fs.writeFileSync(path.join(snapshot, shards[0]), "malformed"); + + let probeScript = ""; + const recordingSpawn = vi.fn( + ( + _file: string, + _args: readonly string[], + options: SpawnSyncOptionsWithStringEncoding, + ): StationProbeCommandResult => { + probeScript = typeof options.input === "string" ? options.input : ""; + return command({}); + }, + ); + createStationClusterProbeDeps(recordingSpawn).probeLocalHost(); + const python = resolveDualStationSimulationFixturePython(); + + try { + const executed = spawnSync(python, ["-"], { + encoding: "utf8", + env: { ...process.env, HOME: home, PATH: bin }, + input: probeScript, + timeout: 20_000, + }); + expect(executed.status, executed.stderr).toBe(0); + const observed = JSON.parse(executed.stdout) as StationHostProbe; + expect(observed.modelSnapshot).toMatchObject({ + complete: false, + shardCount: 113, + reason: expect.stringContaining("weight shards are unreadable or malformed"), + }); + expect(observed.rsyncAvailable).toBe(false); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } + }); + + it("executes the host probe and refuses a non-character uverbs device", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-station-verbs-fixture-")); + const home = path.join(root, "home"); + const bin = path.join(root, "bin"); + fs.mkdirSync(home, { mode: 0o700 }); + fs.mkdirSync(bin, { mode: 0o700 }); + + let probeScript = ""; + const recordingSpawn = vi.fn( + ( + _file: string, + _args: readonly string[], + options: SpawnSyncOptionsWithStringEncoding, + ): StationProbeCommandResult => { + probeScript = typeof options.input === "string" ? options.input : ""; + return command({}); + }, + ); + createStationClusterProbeDeps(recordingSpawn).probeLocalHost(); + const python = resolveDualStationSimulationFixturePython(); + const fixturePrelude = String.raw` +import pathlib +import stat as fixture_stat +import subprocess + +class FixtureResult: + def __init__(self, returncode, stdout=""): + self.returncode = returncode + self.stdout = stdout + self.stderr = "" + +original_iterdir = pathlib.Path.iterdir +original_stat = pathlib.Path.stat + +def fixture_iterdir(candidate): + if str(candidate) == "/sys/class/infiniband/mlx5_0/device/infiniband_verbs": + return iter([candidate / "uverbs0"]) + return original_iterdir(candidate) + +def fixture_path_stat(candidate, *args, **kwargs): + if str(candidate) == "/dev/infiniband/uverbs0": + return type("FixtureStat", (), {"st_mode": fixture_stat.S_IFREG | 0o600})() + return original_stat(candidate, *args, **kwargs) + +def fixture_run(argv, **_kwargs): + if argv and argv[0] == "ibdev2netdev": + return FixtureResult(0, "mlx5_0 port 1 ==> cx8p0 (Up)") + return FixtureResult(127) + +pathlib.Path.iterdir = fixture_iterdir +pathlib.Path.stat = fixture_path_stat +subprocess.run = fixture_run +`; + + try { + const executed = spawnSync(python, ["-"], { + encoding: "utf8", + env: { ...process.env, HOME: home, PATH: bin }, + input: `${fixturePrelude}\n${probeScript}`, + timeout: 20_000, + }); + expect(executed.status, executed.stderr).toBe(0); + const observed = JSON.parse(executed.stdout) as StationHostProbe; + expect(observed.rails).toHaveLength(1); + expect(observed.rails[0]).toMatchObject({ + rdmaDevice: "mlx5_0", + netdev: "cx8p0", + uverbsDevice: "", + }); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } + }); + it("removes OPENSHELL secrets from the direct SSH probe environment", () => { vi.stubEnv("OPENSHELL_GATEWAY_AUTH_TOKEN", "must-not-cross-ssh"); const spawn = vi.fn( @@ -856,7 +982,6 @@ describe("probe command boundary", () => { const [, , options] = spawn.mock.calls[0]; expect(options.env?.OPENSHELL_GATEWAY_AUTH_TOKEN).toBeUndefined(); expect(options.env?.PATH).toBeTruthy(); - vi.unstubAllEnvs(); }); it("pins the local hardware probe to Docker's physical default context", () => { diff --git a/src/lib/inference/vllm-station-lifecycle-lock.test.ts b/src/lib/inference/vllm-station-lifecycle-lock.test.ts new file mode 100644 index 0000000000..da3ccc6f8d --- /dev/null +++ b/src/lib/inference/vllm-station-lifecycle-lock.test.ts @@ -0,0 +1,100 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, it, vi } from "vitest"; +import { + DUAL_STATION_CONTROLLER_UID_FILE, + type DualStationControllerUidFileStat, + readDualStationControllerUid, + withDualStationVllmLifecycleLock, +} from "./vllm-station-lifecycle-lock"; + +function controllerUidStat( + kind: "directory" | "file", + overrides: Partial = {}, +): DualStationControllerUidFileStat { + return { + uid: 0, + gid: 0, + mode: kind === "directory" ? 0o40755 : 0o100644, + size: kind === "directory" ? 0 : 5, + isDirectory: () => kind === "directory", + isFile: () => kind === "file", + ...overrides, + }; +} + +describe("dual-Station controller UID binding", () => { + it("reads one root-owned UID through an O_NOFOLLOW descriptor and fstat", () => { + const close = vi.fn(); + const open = vi.fn((_pathname: string, _flags: number) => 17); + const fstat = vi.fn(() => controllerUidStat("file")); + + expect( + readDualStationControllerUid({ + lstat: (pathname) => { + expect(pathname).toBe(path.dirname(DUAL_STATION_CONTROLLER_UID_FILE)); + return controllerUidStat("directory"); + }, + open, + fstat, + read: (fd) => { + expect(fd).toBe(17); + return "1001\n"; + }, + close, + }), + ).toBe(1001); + expect(open).toHaveBeenCalledWith(DUAL_STATION_CONTROLLER_UID_FILE, expect.any(Number)); + const flags = open.mock.calls[0]?.[1] ?? 0; + expect(flags & fs.constants.O_NOFOLLOW).toBe(fs.constants.O_NOFOLLOW); + expect(fstat).toHaveBeenCalledWith(17); + expect(close).toHaveBeenCalledWith(17); + }); + + it.each([ + ["group-writable parent", controllerUidStat("directory", { mode: 0o40775 }), null, "1001\n"], + ["non-traversable parent", controllerUidStat("directory", { mode: 0o40700 }), null, "1001\n"], + ["non-root-owned file", controllerUidStat("directory"), { uid: 1001 }, "1001\n"], + ["wrong file mode", controllerUidStat("directory"), { mode: 0o100664 }, "1001\n"], + ["root UID content", controllerUidStat("directory"), null, "0\n"], + ["multiple UID lines", controllerUidStat("directory"), { size: 10 }, "1001\n1002\n"], + ])("rejects an unsafe controller binding: %s", (_case, directory, fileOverride, contents) => { + const close = vi.fn(); + expect(() => + readDualStationControllerUid({ + lstat: () => directory, + open: () => 19, + fstat: () => controllerUidStat("file", fileOverride ?? {}), + read: () => contents, + close, + }), + ).toThrow(/Dual-Station controller/u); + if ((directory.mode & 0o7777) === 0o755) expect(close).toHaveBeenCalledWith(19); + }); + + it("refuses a direct lock call from an account other than the prepared controller", () => { + const stateDir = path.join(os.tmpdir(), `nemoclaw-station-refused-lock-${String(process.pid)}`); + fs.rmSync(stateDir, { recursive: true, force: true }); + const effectiveUid = process.getuid?.() ?? 0; + const operation = vi.fn(); + + expect(() => + withDualStationVllmLifecycleLock( + operation, + { stateDir, pollIntervalMs: 5, timeoutMs: 250, corruptLockGraceMs: 5 }, + { + readControllerUid: () => (effectiveUid > 0 ? effectiveUid + 1 : 1), + effectiveControllerUid: () => effectiveUid, + }, + ), + ).toThrow( + /requires a non-root effective controller UID|does not match prepared controller UID/u, + ); + expect(operation).not.toHaveBeenCalled(); + expect(fs.existsSync(stateDir)).toBe(false); + }); +}); diff --git a/src/lib/inference/vllm-station-lifecycle-lock.ts b/src/lib/inference/vllm-station-lifecycle-lock.ts index c1e7b856c4..1db4076f21 100644 --- a/src/lib/inference/vllm-station-lifecycle-lock.ts +++ b/src/lib/inference/vllm-station-lifecycle-lock.ts @@ -1,25 +1,152 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import fs from "node:fs"; +import os from "node:os"; import path from "node:path"; import { type McpLifecycleLockOptions, withMcpLifecycleLock } from "../state/mcp-lifecycle-lock"; -import { resolveHome, STATE_DIR_NAME } from "../state/state-root"; +import { STATE_DIR_NAME } from "../state/state-root"; const DUAL_STATION_VLLM_LIFECYCLE_LOCK = "dual-station-vllm:host-global"; +const DUAL_STATION_CONTROLLER_CONFIG_DIR = "/etc/nemoclaw"; +export const DUAL_STATION_CONTROLLER_UID_FILE = path.join( + DUAL_STATION_CONTROLLER_CONFIG_DIR, + "dual-station-controller-uid", +); +const DUAL_STATION_CONTROLLER_UID_FILE_MODE = 0o644; +const MAX_POSIX_UID = 0xffff_ffff; + +/** @internal Filesystem seam for security-focused unit tests. */ +export interface DualStationControllerUidFileStat { + uid: number; + gid: number; + mode: number; + size: number; + isDirectory(): boolean; + isFile(): boolean; +} + +/** @internal Filesystem seam for security-focused unit tests. */ +export interface DualStationControllerUidReaderDeps { + lstat(pathname: string): DualStationControllerUidFileStat; + open(pathname: string, flags: number): number; + fstat(fd: number): DualStationControllerUidFileStat; + read(fd: number): string; + close(fd: number): void; +} + +/** @internal Identity seam for tests that cannot use the prepared host account. */ +export interface DualStationControllerIdentityDeps { + readControllerUid(): number; + effectiveControllerUid(): number | null; +} + +const DEFAULT_CONTROLLER_UID_READER_DEPS: DualStationControllerUidReaderDeps = { + lstat: (pathname) => fs.lstatSync(pathname), + open: (pathname, flags) => fs.openSync(pathname, flags), + fstat: (fd) => fs.fstatSync(fd), + read: (fd) => fs.readFileSync(fd, "utf8"), + close: (fd) => fs.closeSync(fd), +}; + +function isNonRootUid(value: number | null): value is number { + return Number.isSafeInteger(value) && value !== null && value > 0 && value <= MAX_POSIX_UID; +} + +/** + * Read the host-preparation controller binding without following the file's + * final path component. Metadata and contents are consumed from the same open + * descriptor so a replacement cannot change what is authorized. + */ +export function readDualStationControllerUid( + deps: DualStationControllerUidReaderDeps = DEFAULT_CONTROLLER_UID_READER_DEPS, +): number { + const directory = deps.lstat(DUAL_STATION_CONTROLLER_CONFIG_DIR); + if ( + !directory.isDirectory() || + directory.uid !== 0 || + directory.gid !== 0 || + (directory.mode & 0o7777) !== 0o755 + ) { + throw new Error( + `Dual-Station controller directory must be root-owned with mode 0755: ${DUAL_STATION_CONTROLLER_CONFIG_DIR}`, + ); + } + + const fd = deps.open( + DUAL_STATION_CONTROLLER_UID_FILE, + fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW, + ); + try { + const binding = deps.fstat(fd); + if ( + !binding.isFile() || + binding.uid !== 0 || + binding.gid !== 0 || + (binding.mode & 0o7777) !== DUAL_STATION_CONTROLLER_UID_FILE_MODE || + binding.size < 2 || + binding.size > 11 + ) { + throw new Error( + `Dual-Station controller UID binding must be a root-owned regular file with mode 0644: ${DUAL_STATION_CONTROLLER_UID_FILE}`, + ); + } + const match = /^([1-9][0-9]*)\n$/.exec(deps.read(fd)); + const controllerUid = match ? Number(match[1]) : Number.NaN; + if (!isNonRootUid(controllerUid)) { + throw new Error( + `Dual-Station controller UID binding must contain exactly one non-root UID: ${DUAL_STATION_CONTROLLER_UID_FILE}`, + ); + } + return controllerUid; + } finally { + deps.close(fd); + } +} + +export function assertDualStationControllerAccount( + readControllerUid: () => number = readDualStationControllerUid, + effectiveControllerUid: () => number | null = () => process.getuid?.() ?? null, +): number { + const controllerUid = readControllerUid(); + if (!isNonRootUid(controllerUid)) { + throw new Error("Dual-Station host preparation returned an invalid controller UID"); + } + const effectiveUid = effectiveControllerUid(); + if (!isNonRootUid(effectiveUid)) { + throw new Error("Dual-Station lifecycle requires a non-root effective controller UID"); + } + if (effectiveUid !== controllerUid) { + throw new Error( + `Dual-Station lifecycle effective UID ${String(effectiveUid)} does not match prepared controller UID ${String(controllerUid)}`, + ); + } + return controllerUid; +} /** * Serialize the host-managed dual-Station service across gateway instances. * - * This deliberately anchors the lease at the default ~/.nemoclaw root instead - * of a gateway-specific state root: every local NemoClaw process controls the - * same two fixed Docker container names on the same pair of daemons. + * Dual-Station lifecycle supports one effective controller account per host. + * This anchors every supported caller at that account's passwd home instead of + * mutable HOME or a gateway-specific root. Host preparation binds that account + * in root-owned state before the lease can be acquired. */ export function withDualStationVllmLifecycleLock( operation: () => Promise | T, options: McpLifecycleLockOptions = {}, + /** @internal Explicit identity injection keeps test storage overrides authorized. */ + identityDeps: DualStationControllerIdentityDeps = { + readControllerUid: readDualStationControllerUid, + effectiveControllerUid: () => process.getuid?.() ?? null, + }, ): Promise { - const stateDir = options.stateDir ?? path.join(resolveHome(), STATE_DIR_NAME, "state"); + assertDualStationControllerAccount( + identityDeps.readControllerUid, + identityDeps.effectiveControllerUid, + ); + const stateDir = options.stateDir ?? path.join(os.userInfo().homedir, STATE_DIR_NAME, "state"); return withMcpLifecycleLock(DUAL_STATION_VLLM_LIFECYCLE_LOCK, operation, { ...options, stateDir, diff --git a/src/lib/inference/vllm-station-model-staging.test.ts b/src/lib/inference/vllm-station-model-staging.test.ts index a39cee77ef..b9511d5cef 100644 --- a/src/lib/inference/vllm-station-model-staging.test.ts +++ b/src/lib/inference/vllm-station-model-staging.test.ts @@ -1,12 +1,19 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { resolveDualStationSimulationFixturePython } from "../../../scripts/simulate-dual-station.mts"; + import { DUAL_STATION_VLLM_RUNTIME, type DualStationVllmPlan } from "./vllm-station-cluster"; import { + type ModelStagingCommandOptions, type ModelStagingCommandResult, stageDualStationModelSnapshot, } from "./vllm-station-model-staging"; @@ -16,9 +23,30 @@ import { } from "./vllm-station-ssh-binding.test-support"; let sshFixture: DualStationSshBindingFixture; +let mockedLocalRoot: string; +let mockedLocalHome: string; +let mockedLocalModelRoot: string; + +function modelRootForHome(home: string): string { + return path.join( + home, + ".cache", + "huggingface", + "hub", + `models--${DUAL_STATION_VLLM_RUNTIME.modelId.replace("/", "--")}`, + ); +} + +function snapshotForHome(home: string): string { + return path.join(modelRootForHome(home), "snapshots", DUAL_STATION_VLLM_RUNTIME.modelRevision); +} beforeEach(() => { sshFixture = createDualStationSshBindingFixture(); + mockedLocalRoot = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-model-staging-test-")); + mockedLocalHome = path.join(mockedLocalRoot, "home"); + mockedLocalModelRoot = modelRootForHome(mockedLocalHome); + fs.mkdirSync(mockedLocalModelRoot, { mode: 0o700, recursive: true }); }); function plan(): DualStationVllmPlan { @@ -27,7 +55,7 @@ function plan(): DualStationVllmPlan { runtime: DUAL_STATION_VLLM_RUNTIME, local: { hostname: "station-a", - home: "/home/nvidia", + home: mockedLocalHome, uid: 1000, gid: 1000, gpu: { index: 0, name: "NVIDIA GB300", uuid: "GPU-a" }, @@ -93,29 +121,109 @@ function result(stdout = "", status = 0): ModelStagingCommandResult { function manifest(): string { return JSON.stringify({ schemaVersion: 1, - files: [{ path: "config.json", size: 2, sha256: "a".repeat(64) }], + files: [ + { + path: "config.json", + size: 2, + sha256: createHash("sha256").update("{}").digest("hex"), + }, + ], directories: [], totalBytes: 2, }); } +function peerStagingForPlan(value: DualStationVllmPlan): string { + const identity = [ + "nemoclaw-dual-station-model-staging-v1", + DUAL_STATION_VLLM_RUNTIME.image, + DUAL_STATION_VLLM_RUNTIME.modelId, + DUAL_STATION_VLLM_RUNTIME.modelRevision, + DUAL_STATION_VLLM_RUNTIME.servedModelId, + String(DUAL_STATION_VLLM_RUNTIME.tensorParallelSize), + String(DUAL_STATION_VLLM_RUNTIME.nodeCount), + value.local.gpu.uuid, + value.peer.gpu.uuid, + ].join("\0"); + const transaction = createHash("sha256").update(identity, "utf8").digest("hex").slice(0, 32); + return path.join( + path.dirname(snapshotForHome(value.peer.home)), + `.nemoclaw-staging-${transaction}`, + ); +} + +function sufficientStatfs() { + return vi.fn().mockResolvedValue({ bavail: 1024n * 1024n * 1024n, bsize: 4096n }); +} + +function runPython( + args: readonly string[], + options: ModelStagingCommandOptions, +): ModelStagingCommandResult { + const completed = spawnSync(resolveDualStationSimulationFixturePython(), [...args], { + encoding: "utf8", + env: options.env, + input: options.input, + timeout: options.timeoutMs, + }); + return { + status: completed.status, + stdout: completed.stdout ?? "", + stderr: completed.stderr ?? "", + error: completed.error?.message, + timedOut: (completed.error as NodeJS.ErrnoException | undefined)?.code === "ETIMEDOUT", + }; +} + +function createLocalSnapshotFixture(): { root: string; home: string; snapshot: string } { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-model-snapshot-")); + const home = path.join(root, "home"); + const snapshot = snapshotForHome(home); + fs.mkdirSync(snapshot, { mode: 0o700, recursive: true }); + const shards = Array.from( + { length: 113 }, + (_, index) => `model-${String(index + 1).padStart(5, "0")}-of-00113.safetensors`, + ); + for (const [index, shard] of shards.entries()) { + fs.writeFileSync(path.join(snapshot, shard), `shard-${String(index)}`); + } + fs.writeFileSync(path.join(snapshot, "config.json"), "{}"); + fs.writeFileSync(path.join(snapshot, "tokenizer.json"), "{}"); + fs.writeFileSync( + path.join(snapshot, "model.safetensors.index.json"), + JSON.stringify({ + weight_map: Object.fromEntries( + shards.map((shard, index) => [`model.layers.${String(index)}.weight`, shard]), + ), + }), + ); + return { root, home, snapshot }; +} + function successfulTransferRunner() { return vi .fn() .mockResolvedValueOnce(result(manifest())) .mockResolvedValueOnce(result('{"state":"transfer"}')) + .mockResolvedValueOnce(result(manifest())) + .mockResolvedValueOnce(result('{"state":"transfer"}')) .mockResolvedValueOnce(result()) .mockResolvedValueOnce(result('{"state":"ready"}')); } function stagingSuffix(runCommand: ReturnType): string { - const destination = String(runCommand.mock.calls[2][1].at(-1)); + const destination = String(runCommand.mock.calls[4][1].at(-1)); return destination.match(/\.nemoclaw-staging-[a-f0-9]{32}/)?.[0] ?? ""; } afterEach(() => { + const stagingLeftovers = fs + .readdirSync(mockedLocalModelRoot) + .filter((entry) => entry.startsWith(".nemoclaw-vllm-model-staging-")); vi.unstubAllEnvs(); sshFixture.cleanup(); + fs.rmSync(mockedLocalRoot, { force: true, recursive: true }); + expect(stagingLeftovers).toEqual([]); }); describe("dual-Station pinned model staging", () => { @@ -123,27 +231,29 @@ describe("dual-Station pinned model staging", () => { vi.stubEnv("OPENSHELL_GATEWAY_AUTH_TOKEN", "must-not-cross-ssh"); vi.stubEnv("HF_TOKEN", "must-not-cross-ssh"); const runCommand = successfulTransferRunner(); + const statfs = sufficientStatfs(); - await expect(stageDualStationModelSnapshot(plan(), { runCommand })).resolves.toEqual({ + await expect(stageDualStationModelSnapshot(plan(), { runCommand, statfs })).resolves.toEqual({ ok: true, transferred: true, }); - expect(runCommand).toHaveBeenCalledTimes(4); + expect(runCommand).toHaveBeenCalledTimes(6); expect(runCommand.mock.calls.map((call) => call[0])).toEqual([ + "python3", + "ssh", "python3", "ssh", "rsync", "ssh", ]); - const localArgs = runCommand.mock.calls[0][1] as string[]; - expect(localArgs).toEqual([ - "-", - `/home/nvidia/.cache/huggingface/hub/models--${DUAL_STATION_VLLM_RUNTIME.modelId.replace("/", "--")}/snapshots/${DUAL_STATION_VLLM_RUNTIME.modelRevision}`, - `/home/nvidia/.cache/huggingface/hub/models--${DUAL_STATION_VLLM_RUNTIME.modelId.replace("/", "--")}`, - ]); - expect(runCommand.mock.calls[0][2].input).toContain("snapshot symlink escapes"); - expect(runCommand.mock.calls[0][2].input).toContain("len(shards) != 113"); + const auditArgs = runCommand.mock.calls[0][1] as string[]; + expect(auditArgs).toEqual(["-", snapshotForHome(mockedLocalHome), mockedLocalModelRoot]); + const materializeArgs = runCommand.mock.calls[2][1] as string[]; + expect(materializeArgs.slice(0, 3)).toEqual(auditArgs); + expect(materializeArgs[3]).toMatch(/nemoclaw-vllm-model-staging-[^/]+\/snapshot$/); + expect(path.dirname(path.dirname(materializeArgs[3]))).toBe(mockedLocalModelRoot); + expect(statfs).toHaveBeenCalledWith(mockedLocalModelRoot); const sshArgs = runCommand.mock.calls[1][1] as string[]; expect(sshArgs).toEqual( @@ -163,28 +273,170 @@ describe("dual-Station pinned model staging", () => { "python3 -", ]), ); - expect(runCommand.mock.calls[1][2].input).toContain("peer pinned snapshot already exists"); - expect(runCommand.mock.calls[1][2].input).toContain("shutil.disk_usage(STAGING.parent).free"); - expect(runCommand.mock.calls[3][2].input).toContain("os.rename(STAGING, FINAL)"); - expect(runCommand.mock.calls[3][2].input).toContain("installed_identity != staged_identity"); - - const rsyncArgs = runCommand.mock.calls[2][1] as string[]; + const rsyncArgs = runCommand.mock.calls[4][1] as string[]; expect(rsyncArgs).toEqual( - expect.arrayContaining(["--copy-links", "--checksum", "--partial", "--protect-args", "--"]), + expect.arrayContaining(["--checksum", "--partial", "--protect-args", "--"]), ); + expect(rsyncArgs).not.toContain("--copy-links"); expect(rsyncArgs).not.toContain("--delete"); + expect(rsyncArgs.at(-2)).toBe(`${materializeArgs[3]}/`); expect(rsyncArgs.at(-1)).toMatch( new RegExp( `^nvidia@station-b:/home/nvidia/\\.cache/huggingface/hub/models--${DUAL_STATION_VLLM_RUNTIME.modelId.replace("/", "--")}/snapshots/\\.nemoclaw-staging-[a-f0-9]{32}/$`, ), ); expect(rsyncArgs.join(" ")).toContain("StrictHostKeyChecking=yes"); - expect(runCommand.mock.calls[2][2]).toMatchObject({ + expect(runCommand.mock.calls[4][2]).toMatchObject({ idleTimeoutMs: 30 * 60 * 1000, streamOutput: true, }); - expect(runCommand.mock.calls[2][2].env.OPENSHELL_GATEWAY_AUTH_TOKEN).toBeUndefined(); - expect(runCommand.mock.calls[2][2].env.HF_TOKEN).toBeUndefined(); + expect(runCommand.mock.calls[4][2].env.OPENSHELL_GATEWAY_AUTH_TOKEN).toBeUndefined(); + expect(runCommand.mock.calls[4][2].env.HF_TOKEN).toBeUndefined(); + }); + + it("transfers the materialized bytes if the source changes after the second audit", async () => { + const fixture = createLocalSnapshotFixture(); + const fixturePlan = plan(); + fixturePlan.local.home = fixture.home; + let pythonCall = 0; + let sshCall = 0; + let transferSource = ""; + let transferredConfig = ""; + let snapshotMode = 0; + let configMode = 0; + const runCommand = vi.fn( + async ( + file: string, + args: readonly string[], + options: ModelStagingCommandOptions, + ): Promise => { + if (file === "python3") { + pythonCall += 1; + const audit = runPython(args, options); + if (pythonCall === 2 && audit.status === 0) { + fs.writeFileSync(path.join(fixture.snapshot, "config.json"), "changed-after-audit"); + } + return audit; + } + if (file === "ssh") { + sshCall += 1; + return result(sshCall <= 2 ? '{"state":"transfer"}' : '{"state":"ready"}'); + } + if (file === "rsync") { + transferSource = String(args.at(-2)).replace(/\/$/, ""); + transferredConfig = fs.readFileSync(path.join(transferSource, "config.json"), "utf8"); + snapshotMode = fs.statSync(transferSource).mode & 0o777; + configMode = fs.statSync(path.join(transferSource, "config.json")).mode & 0o777; + return result(); + } + throw new Error(`unexpected command: ${file}`); + }, + ); + const statfs = sufficientStatfs(); + + try { + await expect( + stageDualStationModelSnapshot(fixturePlan, { runCommand, statfs }), + ).resolves.toEqual({ ok: true, transferred: true }); + expect(transferredConfig).toBe("{}"); + expect(snapshotMode).toBe(0o500); + expect(configMode).toBe(0o400); + expect(transferSource).not.toBe(fixture.snapshot); + expect(path.dirname(path.dirname(transferSource))).toBe(modelRootForHome(fixture.home)); + expect(fs.existsSync(transferSource)).toBe(false); + } finally { + fs.rmSync(fixture.root, { force: true, recursive: true }); + } + }); + + it("rejects source mutation between audits and cleans both staging roots", async () => { + const fixture = createLocalSnapshotFixture(); + const fixturePlan = plan(); + fixturePlan.local.home = fixture.home; + let pythonCall = 0; + let materializedSnapshot = ""; + const runCommand = vi.fn( + async ( + file: string, + args: readonly string[], + options: ModelStagingCommandOptions, + ): Promise => { + if (file === "python3") { + pythonCall += 1; + if (args[3] !== undefined) materializedSnapshot = String(args[3]); + const audit = runPython(args, options); + if (pythonCall === 1 && audit.status === 0) { + fs.writeFileSync(path.join(fixture.snapshot, "config.json"), "changed-between-audits"); + } + return audit; + } + if (file === "ssh") { + return result(pythonCall === 1 ? '{"state":"transfer"}' : '{"state":"cleaned"}'); + } + throw new Error(`unexpected command: ${file}`); + }, + ); + + try { + await expect( + stageDualStationModelSnapshot(fixturePlan, { + runCommand, + statfs: sufficientStatfs(), + }), + ).resolves.toEqual({ + ok: false, + reason: "local pinned snapshot changed between audit and materialization", + }); + expect(runCommand.mock.calls.map((call) => call[0])).toEqual([ + "python3", + "ssh", + "python3", + "ssh", + ]); + expect(fs.existsSync(materializedSnapshot)).toBe(false); + expect(runCommand).not.toHaveBeenCalledWith("rsync", expect.anything(), expect.anything()); + } finally { + fs.rmSync(fixture.root, { force: true, recursive: true }); + } + }); + + it("rejects a snapshot symlink escape through the public staging boundary", async () => { + const fixture = createLocalSnapshotFixture(); + const fixturePlan = plan(); + fixturePlan.local.home = fixture.home; + const outside = path.join(fixture.root, "outside-tokenizer.json"); + const tokenizer = path.join(fixture.snapshot, "tokenizer.json"); + fs.writeFileSync(outside, "{}"); + fs.unlinkSync(tokenizer); + fs.symlinkSync(outside, tokenizer); + const runCommand = vi.fn( + async ( + file: string, + args: readonly string[], + options: ModelStagingCommandOptions, + ): Promise => { + if (file !== "python3") throw new Error(`unexpected command: ${file}`); + return runPython(args, options); + }, + ); + + try { + await expect( + stageDualStationModelSnapshot(fixturePlan, { runCommand }), + ).resolves.toMatchObject({ + ok: false, + reason: expect.stringContaining("snapshot symlink escapes the pinned model cache"), + }); + expect(runCommand).toHaveBeenCalledTimes(1); + expect(runCommand.mock.calls[0][1] as string[]).toHaveLength(3); + expect( + fs + .readdirSync(modelRootForHome(fixture.home)) + .some((entry) => entry.startsWith(".nemoclaw-vllm-model-staging-")), + ).toBe(false); + } finally { + fs.rmSync(fixture.root, { force: true, recursive: true }); + } }); it("does no transfer when the peer already has the exact byte manifest", async () => { @@ -192,13 +444,137 @@ describe("dual-Station pinned model staging", () => { .fn() .mockResolvedValueOnce(result(manifest())) .mockResolvedValueOnce(result('{"state":"ready"}')); + const statfs = sufficientStatfs(); - await expect(stageDualStationModelSnapshot(plan(), { runCommand })).resolves.toEqual({ + await expect(stageDualStationModelSnapshot(plan(), { runCommand, statfs })).resolves.toEqual({ ok: true, transferred: false, }); expect(runCommand).toHaveBeenCalledTimes(2); + expect(statfs).not.toHaveBeenCalled(); + expect(runCommand.mock.calls[0][1] as string[]).toHaveLength(3); + expect(runCommand).not.toHaveBeenCalledWith("rsync", expect.anything(), expect.anything()); + expect( + fs + .readdirSync(mockedLocalModelRoot) + .some((entry) => entry.startsWith(".nemoclaw-vllm-model-staging-")), + ).toBe(false); + }); + + it("removes deterministic peer staging when the final snapshot is already exact", async () => { + const remoteRoot = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-model-peer-ready-")); + const peerHome = path.join(remoteRoot, "peer-home"); + fs.mkdirSync(peerHome, { mode: 0o700 }); + const fixturePlan = plan(); + fixturePlan.peer.home = peerHome; + const finalSnapshot = snapshotForHome(peerHome); + const peerStaging = peerStagingForPlan(fixturePlan); + fs.mkdirSync(finalSnapshot, { mode: 0o700, recursive: true }); + fs.writeFileSync(path.join(finalSnapshot, "config.json"), "{}"); + fs.mkdirSync(peerStaging, { mode: 0o700 }); + fs.writeFileSync(path.join(peerStaging, "config.json"), "{"); + const statfs = sufficientStatfs(); + const runCommand = vi.fn( + async ( + file: string, + _args: readonly string[], + options: ModelStagingCommandOptions, + ): Promise => { + if (file === "python3") return result(manifest()); + if (file === "ssh") { + return runPython(["-"], { + ...options, + env: { ...options.env, HOME: peerHome }, + }); + } + throw new Error(`unexpected command: ${file}`); + }, + ); + + try { + await expect( + stageDualStationModelSnapshot(fixturePlan, { runCommand, statfs }), + ).resolves.toEqual({ ok: true, transferred: false }); + expect(runCommand.mock.calls.map((call) => call[0])).toEqual(["python3", "ssh"]); + expect(statfs).not.toHaveBeenCalled(); + expect(fs.existsSync(finalSnapshot)).toBe(true); + expect(fs.existsSync(peerStaging)).toBe(false); + expect(runCommand).not.toHaveBeenCalledWith("rsync", expect.anything(), expect.anything()); + } finally { + fs.rmSync(remoteRoot, { force: true, recursive: true }); + } + }); + + it("cleans peer staging without materializing when local capacity is insufficient", async () => { + const runCommand = vi + .fn() + .mockResolvedValueOnce(result(manifest())) + .mockResolvedValueOnce(result('{"state":"transfer"}')) + .mockResolvedValueOnce(result('{"state":"cleaned"}')); + const statfs = vi.fn().mockResolvedValue({ bavail: 1n, bsize: 4096n }); + + await expect(stageDualStationModelSnapshot(plan(), { runCommand, statfs })).resolves.toEqual({ + ok: false, + reason: "local model cache does not have enough free space for the audited snapshot copy", + }); + expect(statfs).toHaveBeenCalledWith(mockedLocalModelRoot); + expect(runCommand.mock.calls.map((call) => call[0])).toEqual(["python3", "ssh", "ssh"]); expect(runCommand).not.toHaveBeenCalledWith("rsync", expect.anything(), expect.anything()); + expect( + fs + .readdirSync(mockedLocalModelRoot) + .some((entry) => entry.startsWith(".nemoclaw-vllm-model-staging-")), + ).toBe(false); + }); + + it("fails remote capacity preflight before creating either staging copy", async () => { + const remoteRoot = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-model-peer-capacity-")); + const peerHome = path.join(remoteRoot, "peer-home"); + fs.mkdirSync(peerHome, { mode: 0o700 }); + const fixturePlan = plan(); + fixturePlan.peer.home = peerHome; + const statfs = sufficientStatfs(); + const runCommand = vi.fn( + async ( + file: string, + _args: readonly string[], + options: ModelStagingCommandOptions, + ): Promise => { + if (file === "python3") return result(manifest()); + if (file === "ssh") { + const noCapacity = `import shutil +class _NemoClawDiskUsage: + free = 0 +shutil.disk_usage = lambda _path: _NemoClawDiskUsage() +`; + return runPython(["-"], { + ...options, + env: { ...options.env, HOME: peerHome }, + input: `${noCapacity}${options.input ?? ""}`, + }); + } + throw new Error(`unexpected command: ${file}`); + }, + ); + + try { + await expect( + stageDualStationModelSnapshot(fixturePlan, { runCommand, statfs }), + ).resolves.toEqual({ + ok: false, + reason: + "peer snapshot preflight failed: peer model cache does not have enough free space for the pinned snapshot", + }); + expect(runCommand.mock.calls.map((call) => call[0])).toEqual(["python3", "ssh"]); + expect(statfs).not.toHaveBeenCalled(); + expect( + fs + .readdirSync(path.dirname(snapshotForHome(peerHome))) + .some((entry) => entry.startsWith(".nemoclaw-staging-")), + ).toBe(false); + } finally { + fs.rmSync(remoteRoot, { force: true, recursive: true }); + } }); it("gives concurrent reversed and different local heads disjoint retry-safe staging paths", async () => { @@ -207,6 +583,8 @@ describe("dual-Station pinned model staging", () => { const reversedLocal = reversed.local; reversed.local = reversed.peer; reversed.peer = reversedLocal; + reversed.local.home = mockedLocalHome; + reversed.peer.home = "/home/nvidia"; const reversedSshFixture = createDualStationSshBindingFixture("nvidia@station-a"); reversed.peerSshBinding = reversedSshFixture.binding; const differentHead = plan(); @@ -217,9 +595,18 @@ describe("dual-Station pinned model staging", () => { try { await Promise.all([ - stageDualStationModelSnapshot(original, { runCommand: originalRunner }), - stageDualStationModelSnapshot(reversed, { runCommand: reversedRunner }), - stageDualStationModelSnapshot(differentHead, { runCommand: differentHeadRunner }), + stageDualStationModelSnapshot(original, { + runCommand: originalRunner, + statfs: sufficientStatfs(), + }), + stageDualStationModelSnapshot(reversed, { + runCommand: reversedRunner, + statfs: sufficientStatfs(), + }), + stageDualStationModelSnapshot(differentHead, { + runCommand: differentHeadRunner, + statfs: sufficientStatfs(), + }), ]); } finally { reversedSshFixture.cleanup(); @@ -238,24 +625,183 @@ describe("dual-Station pinned model staging", () => { const firstRunner = successfulTransferRunner(); const retryRunner = successfulTransferRunner(); - await stageDualStationModelSnapshot(plan(), { runCommand: firstRunner }); - await stageDualStationModelSnapshot(plan(), { runCommand: retryRunner }); + await stageDualStationModelSnapshot(plan(), { + runCommand: firstRunner, + statfs: sufficientStatfs(), + }); + await stageDualStationModelSnapshot(plan(), { + runCommand: retryRunner, + statfs: sufficientStatfs(), + }); expect(stagingSuffix(firstRunner)).toBe(stagingSuffix(retryRunner)); }); - it("leaves the private staging tree for a safe retry when rsync fails", async () => { + it("cleans the private peer staging tree when rsync fails", async () => { const runCommand = vi .fn() .mockResolvedValueOnce(result(manifest())) .mockResolvedValueOnce(result('{"state":"transfer"}')) - .mockResolvedValueOnce({ status: 23, stdout: "", stderr: "partial transfer" }); + .mockResolvedValueOnce(result(manifest())) + .mockResolvedValueOnce(result('{"state":"transfer"}')) + .mockResolvedValueOnce({ status: 23, stdout: "", stderr: "partial transfer" }) + .mockResolvedValueOnce(result('{"state":"cleaned"}')); - await expect(stageDualStationModelSnapshot(plan(), { runCommand })).resolves.toEqual({ - ok: false, - reason: "peer snapshot transfer failed: partial transfer", - }); - expect(runCommand).toHaveBeenCalledTimes(3); + await expect( + stageDualStationModelSnapshot(plan(), { runCommand, statfs: sufficientStatfs() }), + ).resolves.toEqual({ ok: false, reason: "peer snapshot transfer failed: partial transfer" }); + expect(runCommand).toHaveBeenCalledTimes(6); + expect(runCommand.mock.calls.map((call) => call[0])).toEqual([ + "python3", + "ssh", + "python3", + "ssh", + "rsync", + "ssh", + ]); + }); + + it("removes peer bytes that fail real manifest verification", async () => { + const remoteRoot = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-model-peer-")); + const peerHome = path.join(remoteRoot, "peer-home"); + fs.mkdirSync(peerHome, { mode: 0o700 }); + const fixturePlan = plan(); + fixturePlan.peer.home = peerHome; + let stagingPath = ""; + const runCommand = vi.fn( + async ( + file: string, + args: readonly string[], + options: ModelStagingCommandOptions, + ): Promise => { + if (file === "python3") return result(manifest()); + if (file === "ssh") { + return runPython(["-"], { + ...options, + env: { ...options.env, HOME: peerHome }, + }); + } + if (file === "rsync") { + const destination = String(args.at(-1)); + stagingPath = destination.slice(destination.indexOf(":") + 1).replace(/\/$/, ""); + fs.mkdirSync(stagingPath, { mode: 0o700, recursive: true }); + fs.writeFileSync(path.join(stagingPath, "config.json"), "xx"); + return result(); + } + throw new Error(`unexpected command: ${file}`); + }, + ); + const statfs = sufficientStatfs(); + + try { + await expect( + stageDualStationModelSnapshot(fixturePlan, { runCommand, statfs }), + ).resolves.toEqual({ + ok: false, + reason: + "peer snapshot verification failed: peer staged snapshot failed byte-integrity verification", + }); + expect(runCommand.mock.calls.map((call) => call[0])).toEqual([ + "python3", + "ssh", + "python3", + "ssh", + "rsync", + "ssh", + "ssh", + ]); + expect(fs.existsSync(stagingPath)).toBe(false); + } finally { + fs.rmSync(remoteRoot, { force: true, recursive: true }); + } + }); + + it("fails closed when atomic install resolves to a different directory identity", async () => { + const fixture = createLocalSnapshotFixture(); + const remoteRoot = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-model-peer-identity-")); + const peerHome = path.join(remoteRoot, "peer-home"); + fs.mkdirSync(peerHome, { mode: 0o700 }); + const fixturePlan = plan(); + fixturePlan.local.home = fixture.home; + fixturePlan.peer.home = peerHome; + let sshCall = 0; + let materializedSnapshot = ""; + const runCommand = vi.fn( + async ( + file: string, + args: readonly string[], + options: ModelStagingCommandOptions, + ): Promise => { + if (file === "python3") { + if (args[3] !== undefined) materializedSnapshot = String(args[3]); + return runPython(args, options); + } + if (file === "ssh") { + sshCall += 1; + const ampleCapacity = `import shutil +class _NemoClawDiskUsage: + free = 1 << 50 +shutil.disk_usage = lambda _path: _NemoClawDiskUsage() +`; + const replaceInstalledIdentity = + sshCall === 3 + ? `import os +_nemoclaw_original_rename = os.rename +def _nemoclaw_replace_after_rename(source, destination): + _nemoclaw_original_rename(source, destination) + _nemoclaw_original_rename(destination, str(destination) + ".nemoclaw-test-original") + os.mkdir(destination, 0o700) +os.rename = _nemoclaw_replace_after_rename +` + : ""; + return runPython(["-"], { + ...options, + env: { ...options.env, HOME: peerHome }, + input: `${ampleCapacity}${replaceInstalledIdentity}${options.input ?? ""}`, + }); + } + if (file === "rsync") { + const source = String(args.at(-2)).replace(/\/$/, ""); + const destination = String(args.at(-1)); + const stagingPath = destination.slice(destination.indexOf(":") + 1).replace(/\/$/, ""); + for (const entry of fs.readdirSync(source)) { + fs.cpSync(path.join(source, entry), path.join(stagingPath, entry), { + recursive: true, + }); + } + return result(); + } + throw new Error(`unexpected command: ${file}`); + }, + ); + + try { + await expect( + stageDualStationModelSnapshot(fixturePlan, { + runCommand, + statfs: sufficientStatfs(), + }), + ).resolves.toEqual({ + ok: false, + reason: + "peer snapshot verification failed: peer pinned snapshot identity changed during atomic install", + }); + expect(runCommand.mock.calls.map((call) => call[0])).toEqual([ + "python3", + "ssh", + "python3", + "ssh", + "rsync", + "ssh", + "ssh", + ]); + expect(fs.existsSync(materializedSnapshot)).toBe(false); + expect(fs.existsSync(snapshotForHome(peerHome))).toBe(true); + expect(fs.existsSync(`${snapshotForHome(peerHome)}.nemoclaw-test-original`)).toBe(true); + } finally { + fs.rmSync(fixture.root, { force: true, recursive: true }); + fs.rmSync(remoteRoot, { force: true, recursive: true }); + } }); it("rejects a peer home that cannot be represented without remote shell syntax", async () => { diff --git a/src/lib/inference/vllm-station-model-staging.ts b/src/lib/inference/vllm-station-model-staging.ts index 53dc0f697e..9a20c17d1d 100644 --- a/src/lib/inference/vllm-station-model-staging.ts +++ b/src/lib/inference/vllm-station-model-staging.ts @@ -3,6 +3,7 @@ import { type ChildProcess, spawn } from "node:child_process"; import { createHash } from "node:crypto"; +import fs from "node:fs"; import path from "node:path"; import { buildVllmSshTransportEnv } from "./vllm-docker-env"; @@ -15,6 +16,8 @@ const MAX_COMMAND_OUTPUT_BYTES = 256 * 1024; const SNAPSHOT_AUDIT_TIMEOUT_MS = 6 * 60 * 60 * 1000; const RSYNC_TIMEOUT_MS = 12 * 60 * 60 * 1000; const RSYNC_IDLE_TIMEOUT_MS = 30 * 60 * 1000; +const LOCAL_STAGING_MIN_HEADROOM_BYTES = 5n * 1024n * 1024n * 1024n; +const MAX_SNAPSHOT_BYTES = 1024 * 1024 * 1024 * 1024; const SAFE_POSIX_PATH_PATTERN = /^\/(?:[A-Za-z0-9._-]+\/)*[A-Za-z0-9._-]+$/; export interface ModelStagingCommandOptions { @@ -39,6 +42,9 @@ export interface DualStationModelStagingDeps { args: readonly string[], options: ModelStagingCommandOptions, ): Promise; + statfs?( + filePath: string, + ): Promise<{ bavail: bigint; bsize: bigint }> | { bavail: bigint; bsize: bigint }; } export type DualStationModelStagingResult = @@ -136,7 +142,13 @@ function defaultRunCommand( }); } -const defaultDeps: DualStationModelStagingDeps = { runCommand: defaultRunCommand }; +const defaultDeps: DualStationModelStagingDeps = { + runCommand: defaultRunCommand, + statfs(filePath) { + const stats = fs.statfsSync(filePath, { bigint: true }); + return { bavail: stats.bavail, bsize: stats.bsize }; + }, +}; function snapshotPath(home: string): string { return path.posix.join( @@ -209,12 +221,18 @@ import re import stat import sys -if len(sys.argv) != 3: - raise SystemExit("expected snapshot and model-root paths") +if len(sys.argv) not in (3, 4): + raise SystemExit("expected snapshot and model-root paths, plus optional materialized-snapshot path") snapshot = Path(sys.argv[1]) model_root = Path(sys.argv[2]) +materialized_snapshot = Path(sys.argv[3]) if len(sys.argv) == 4 else None safe_component = re.compile(r"^[A-Za-z0-9._-]+$") +weight_index_name = "model.safetensors.index.json" +max_weight_index_bytes = 64 * 1024 * 1024 + +def fail(message): + raise SystemExit(message) def relative_name(candidate): relative = candidate.relative_to(snapshot) @@ -224,45 +242,88 @@ def relative_name(candidate): def resolved_regular_file(candidate): metadata = candidate.lstat() - resolved = candidate.resolve(strict=True) if stat.S_ISLNK(metadata.st_mode) else candidate + resolved = candidate.resolve(strict=True) try: - common = os.path.commonpath((str(model_root.resolve(strict=True)), str(resolved.resolve(strict=True)))) + common = os.path.commonpath((str(canonical_model_root), str(resolved))) except (OSError, ValueError): raise SystemExit("snapshot file could not be resolved") - if common != str(model_root.resolve(strict=True)): + if common != str(canonical_model_root): raise SystemExit("snapshot symlink escapes the pinned model cache") if not resolved.is_file(): raise SystemExit("snapshot contains a non-regular file") return resolved +def create_materialized_directory(relative): + if materialized_snapshot is None: + return + destination = materialized_snapshot / relative + try: + destination.mkdir(mode=0o700) + except FileExistsError: + fail("materialized snapshot path changed during audit") + +def audit_regular_file(source, relative): + if not hasattr(os, "O_NOFOLLOW"): + fail("snapshot audit requires O_NOFOLLOW support") + source_flags = os.O_RDONLY | os.O_NOFOLLOW + destination_flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW + digest = hashlib.sha256() + index_bytes = bytearray() if relative == weight_index_name else None + size = 0 + try: + source_handle = os.fdopen(os.open(source, source_flags), "rb") + except OSError: + fail("snapshot file changed while it was being opened") + with source_handle: + if not stat.S_ISREG(os.fstat(source_handle.fileno()).st_mode): + fail("snapshot contains a non-regular file") + destination_handle = None + if materialized_snapshot is not None: + destination = materialized_snapshot / relative + try: + destination_handle = os.fdopen( + os.open(destination, destination_flags, 0o400), + "wb", + ) + except OSError: + fail("materialized snapshot path changed during audit") + try: + for chunk in iter(lambda: source_handle.read(8 * 1024 * 1024), b""): + size += len(chunk) + digest.update(chunk) + if index_bytes is not None: + if len(index_bytes) + len(chunk) > max_weight_index_bytes: + fail("pinned local weight index exceeds the safety bound") + index_bytes.extend(chunk) + if destination_handle is not None: + destination_handle.write(chunk) + if destination_handle is not None: + destination_handle.flush() + os.fsync(destination_handle.fileno()) + os.fchmod(destination_handle.fileno(), 0o400) + finally: + if destination_handle is not None: + destination_handle.close() + return size, digest.hexdigest(), index_bytes + if snapshot.is_symlink() or not snapshot.is_dir(): raise SystemExit("pinned local snapshot directory is missing or unsafe") if model_root.is_symlink() or not model_root.is_dir(): raise SystemExit("pinned local model cache root is missing or unsafe") - -try: - index = json.loads((snapshot / "model.safetensors.index.json").read_text(encoding="utf-8")) - weight_map = index.get("weight_map", {}) - shards = sorted(set(weight_map.values())) if isinstance(weight_map, dict) else [] -except (OSError, UnicodeError, json.JSONDecodeError): - raise SystemExit("pinned local weight index is missing or malformed") -if len(shards) != 113 or any( - not isinstance(item, str) - or Path(item).name != item - or not safe_component.fullmatch(item) - or not item.endswith(".safetensors") - for item in shards -): - raise SystemExit("pinned local weight index has an unexpected shard set") -if not (snapshot / "config.json").is_file(): - raise SystemExit("pinned local config.json is missing") -if not any((snapshot / name).is_file() for name in ("tokenizer.json", "tokenizer.model", "vocab.json")): - raise SystemExit("pinned local tokenizer assets are missing") +canonical_model_root = model_root.resolve(strict=True) +if materialized_snapshot is not None: + if os.path.lexists(materialized_snapshot): + fail("private materialized snapshot path already exists") + try: + materialized_snapshot.mkdir(mode=0o700) + except OSError: + fail("private materialized snapshot could not be created") files = [] directories = [] total_bytes = 0 entry_count = 0 +weight_index_bytes = None for current, dirnames, filenames in os.walk(snapshot, topdown=True, followlinks=False): current_path = Path(current) dirnames.sort() @@ -273,26 +334,50 @@ for current, dirnames, filenames in os.walk(snapshot, topdown=True, followlinks= mode = directory.lstat().st_mode if stat.S_ISLNK(mode) or not stat.S_ISDIR(mode): raise SystemExit("snapshot contains an unsafe directory") + create_materialized_directory(relative) directories.append(relative) entry_count += 1 for filename in filenames: candidate = current_path / filename relative = relative_name(candidate) resolved = resolved_regular_file(candidate) - digest = hashlib.sha256() - size = 0 - with resolved.open("rb") as handle: - for chunk in iter(lambda: handle.read(8 * 1024 * 1024), b""): - size += len(chunk) - digest.update(chunk) - files.append({"path": relative, "size": size, "sha256": digest.hexdigest()}) + size, digest, audited_index_bytes = audit_regular_file(resolved, relative) + if audited_index_bytes is not None: + weight_index_bytes = audited_index_bytes + files.append({"path": relative, "size": size, "sha256": digest}) total_bytes += size entry_count += 1 - if entry_count > 4096 or total_bytes > 1024 * 1024 * 1024 * 1024: + if entry_count > 4096 or total_bytes > ${String(MAX_SNAPSHOT_BYTES)}: raise SystemExit("snapshot exceeds the staging safety bounds") -if not files or any(not (snapshot / shard).exists() for shard in shards): +file_paths = {item["path"] for item in files} +try: + if weight_index_bytes is None: + fail("pinned local weight index is missing or malformed") + index = json.loads(weight_index_bytes.decode("utf-8")) + weight_map = index.get("weight_map", {}) + shards = sorted(set(weight_map.values())) if isinstance(weight_map, dict) else [] +except (OSError, UnicodeError, json.JSONDecodeError): + fail("pinned local weight index is missing or malformed") +if len(shards) != 113 or any( + not isinstance(item, str) + or Path(item).name != item + or not safe_component.fullmatch(item) + or not item.endswith(".safetensors") + for item in shards +): + fail("pinned local weight index has an unexpected shard set") +if "config.json" not in file_paths: + fail("pinned local config.json is missing") +if not any(name in file_paths for name in ("tokenizer.json", "tokenizer.model", "vocab.json")): + fail("pinned local tokenizer assets are missing") +if not files or any(shard not in file_paths for shard in shards): raise SystemExit("pinned local snapshot is incomplete") +if materialized_snapshot is not None: + for current, dirnames, _filenames in os.walk(materialized_snapshot, topdown=False): + for dirname in dirnames: + os.chmod(Path(current) / dirname, 0o500, follow_symlinks=False) + os.chmod(materialized_snapshot, 0o500, follow_symlinks=False) print(json.dumps({ "schemaVersion": 1, "files": files, @@ -324,6 +409,7 @@ function parseManifest(result: ModelStagingCommandResult): SnapshotManifest { !Array.isArray(record.directories) || !Number.isSafeInteger(record.totalBytes) || Number(record.totalBytes) <= 0 || + Number(record.totalBytes) > MAX_SNAPSHOT_BYTES || record.files.length === 0 || record.files.length + record.directories.length > 4096 ) { @@ -373,7 +459,7 @@ function remoteScript( plan: DualStationVllmPlan, paths: StagingPaths, manifest: SnapshotManifest, - operation: "prepare" | "finalize", + operation: "prepare" | "finalize" | "cleanup", ): string { const manifestBase64 = Buffer.from(JSON.stringify(manifest), "utf8").toString("base64"); return String.raw` @@ -501,17 +587,21 @@ if OPERATION == "prepare": if path_exists(FINAL): if manifest(FINAL) != EXPECTED: fail("peer pinned snapshot already exists with different content") + if path_exists(STAGING): + verify_partial(STAGING) + shutil.rmtree(STAGING) print(json.dumps({"state": "ready"}, separators=(",", ":"))) else: if path_exists(STAGING): reusable_bytes = verify_partial(STAGING) else: - STAGING.mkdir(mode=0o700) reusable_bytes = 0 remaining_bytes = max(0, EXPECTED["totalBytes"] - reusable_bytes) headroom_bytes = max(5 * 1024 * 1024 * 1024, EXPECTED["totalBytes"] // 20) if shutil.disk_usage(STAGING.parent).free < remaining_bytes + headroom_bytes: fail("peer model cache does not have enough free space for the pinned snapshot") + if not path_exists(STAGING): + STAGING.mkdir(mode=0o700) print(json.dumps({"state": "transfer"}, separators=(",", ":"))) elif OPERATION == "finalize": if path_exists(FINAL): @@ -529,6 +619,11 @@ elif OPERATION == "finalize": if installed_identity != staged_identity: fail("peer pinned snapshot identity changed during atomic install") print(json.dumps({"state": "ready"}, separators=(",", ":"))) +elif OPERATION == "cleanup": + if path_exists(STAGING): + verify_partial(STAGING) + shutil.rmtree(STAGING) + print(json.dumps({"state": "cleaned"}, separators=(",", ":"))) else: fail("unsupported model staging operation") `; @@ -547,7 +642,7 @@ function commandFailure(label: string, result: ModelStagingCommandResult): strin function parseRemoteState( label: string, result: ModelStagingCommandResult, - expected: "ready" | "transfer", + expected: "cleaned" | "ready" | "transfer", ): void { if (result.status !== 0 || result.timedOut || result.error) { throw new Error(commandFailure(label, result)); @@ -568,6 +663,26 @@ function parseRemoteState( } } +function parseRemotePrepareState(result: ModelStagingCommandResult): "ready" | "transfer" { + if (result.status !== 0 || result.timedOut || result.error) { + throw new Error(commandFailure("peer snapshot preflight", result)); + } + let parsed: unknown; + try { + parsed = JSON.parse(result.stdout.trim()); + } catch { + throw new Error("peer snapshot preflight returned invalid JSON"); + } + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { + throw new Error("peer snapshot preflight returned an unexpected state"); + } + const state = (parsed as Record).state; + if (state !== "ready" && state !== "transfer") { + throw new Error("peer snapshot preflight returned an unexpected state"); + } + return state; +} + function sshArgs(plan: DualStationVllmPlan): string[] { return [ ...dualStationPinnedSshArgs(plan.peerSshBinding), @@ -585,57 +700,203 @@ function rsyncRsh(plan: DualStationVllmPlan): string { return ["ssh", ...dualStationPinnedSshArgs(plan.peerSshBinding)].map(shellQuote).join(" "); } -/** - * Materialize only the pinned snapshot on the pre-trusted peer. Hugging Face - * snapshot symlinks are copied as regular files after an escape check, so no - * token or unrelated blob cache crosses the SSH boundary. - */ -export async function stageDualStationModelSnapshot( - plan: DualStationVllmPlan, - deps: DualStationModelStagingDeps = defaultDeps, -): Promise { - let paths: StagingPaths; +function createLocalStagingRoot(modelRoot: string): { root: string; snapshot: string } { + const modelRootMetadata = fs.lstatSync(modelRoot); + if (modelRootMetadata.isSymbolicLink() || !modelRootMetadata.isDirectory()) { + throw new Error("pinned local model cache root is missing or unsafe"); + } + // Keep the potentially large immutable copy on the model-cache filesystem; + // the OS temporary filesystem is commonly too small for the Ultra snapshot. + const root = fs.mkdtempSync(path.join(modelRoot, ".nemoclaw-vllm-model-staging-")); try { - paths = stagingPaths(plan); + fs.chmodSync(root, 0o700); } catch (err) { - return { ok: false, reason: (err as Error).message }; + try { + fs.rmSync(root, { force: false, recursive: true }); + } catch (cleanupError) { + throw new Error( + `${(err as Error).message}; local staging setup cleanup failed: ${(cleanupError as Error).message}`, + ); + } + throw err; } - const env = buildVllmSshTransportEnv({ LC_ALL: "C" }); + return { root, snapshot: path.join(root, "snapshot") }; +} - let manifest: SnapshotManifest; - try { - const audit = await deps.runCommand( - "python3", - ["-", paths.localSnapshot, paths.localModelRoot], - { - env, - input: LOCAL_MANIFEST_SCRIPT, - timeoutMs: SNAPSHOT_AUDIT_TIMEOUT_MS, - }, - ); - manifest = parseManifest(audit); - } catch (err) { - return { ok: false, reason: (err as Error).message }; +function makeLocalStagingTreeRemovable(candidate: string): void { + const metadata = fs.lstatSync(candidate); + if (metadata.isSymbolicLink()) { + throw new Error("local audited snapshot cleanup encountered a symbolic link"); + } + if (metadata.isDirectory()) { + fs.chmodSync(candidate, 0o700); + for (const entry of fs.readdirSync(candidate)) { + makeLocalStagingTreeRemovable(path.join(candidate, entry)); + } + return; + } + if (!metadata.isFile()) { + throw new Error("local audited snapshot cleanup encountered a non-file entry"); } +} - let prepare: ModelStagingCommandResult; +function clearLocalStagingRoot(root: string): void { + makeLocalStagingTreeRemovable(root); + fs.rmSync(root, { force: false, recursive: true }); +} + +async function cleanupPeerStaging( + plan: DualStationVllmPlan, + paths: StagingPaths, + manifest: SnapshotManifest, + deps: DualStationModelStagingDeps, + env: Record, +): Promise { try { - prepare = await deps.runCommand("ssh", sshArgs(plan), { + const cleanup = await deps.runCommand("ssh", sshArgs(plan), { env, - input: remoteScript(plan, paths, manifest, "prepare"), + input: remoteScript(plan, paths, manifest, "cleanup"), timeoutMs: SNAPSHOT_AUDIT_TIMEOUT_MS, }); + parseRemoteState("peer snapshot cleanup", cleanup, "cleaned"); + return null; } catch (err) { - return { ok: false, reason: (err as Error).message }; + return (err as Error).message; + } +} + +async function failAfterPeerCleanup( + reason: string, + plan: DualStationVllmPlan, + paths: StagingPaths, + manifest: SnapshotManifest, + deps: DualStationModelStagingDeps, + env: Record, +): Promise { + const cleanupFailure = await cleanupPeerStaging(plan, paths, manifest, deps, env); + return { + ok: false, + reason: cleanupFailure ? `${reason}; ${cleanupFailure}` : reason, + }; +} + +async function auditLocalSnapshot( + paths: StagingPaths, + deps: DualStationModelStagingDeps, + env: Record, + localMaterializedSnapshot?: string, +): Promise { + const args = ["-", paths.localSnapshot, paths.localModelRoot]; + if (localMaterializedSnapshot !== undefined) args.push(localMaterializedSnapshot); + const audit = await deps.runCommand("python3", args, { + env, + input: LOCAL_MANIFEST_SCRIPT, + timeoutMs: SNAPSHOT_AUDIT_TIMEOUT_MS, + }); + return parseManifest(audit); +} + +async function preparePeerSnapshot( + plan: DualStationVllmPlan, + paths: StagingPaths, + manifest: SnapshotManifest, + deps: DualStationModelStagingDeps, + env: Record, +): Promise<"ready" | "transfer"> { + const prepare = await deps.runCommand("ssh", sshArgs(plan), { + env, + input: remoteScript(plan, paths, manifest, "prepare"), + timeoutMs: SNAPSHOT_AUDIT_TIMEOUT_MS, + }); + return parseRemotePrepareState(prepare); +} + +function manifestsEqual(left: SnapshotManifest, right: SnapshotManifest): boolean { + return ( + left.schemaVersion === right.schemaVersion && + left.totalBytes === right.totalBytes && + left.directories.length === right.directories.length && + left.directories.every((directory, index) => directory === right.directories[index]) && + left.files.length === right.files.length && + left.files.every((file, index) => { + const other = right.files[index]; + return ( + other !== undefined && + file.path === other.path && + file.size === other.size && + file.sha256 === other.sha256 + ); + }) + ); +} + +async function requireLocalStagingCapacity( + modelRoot: string, + manifest: SnapshotManifest, + deps: DualStationModelStagingDeps, +): Promise { + let stats: { bavail: bigint; bsize: bigint }; + try { + const statfs = deps.statfs ?? defaultDeps.statfs; + if (!statfs) throw new Error("statfs dependency is unavailable"); + stats = await statfs(modelRoot); + } catch (err) { + throw new Error(`local model cache capacity check failed: ${(err as Error).message}`); + } + if ( + typeof stats.bavail !== "bigint" || + typeof stats.bsize !== "bigint" || + stats.bavail < 0n || + stats.bsize <= 0n + ) { + throw new Error("local model cache capacity check returned invalid filesystem data"); + } + const snapshotBytes = BigInt(manifest.totalBytes); + const proportionalHeadroom = (snapshotBytes + 19n) / 20n; + const headroom = + proportionalHeadroom > LOCAL_STAGING_MIN_HEADROOM_BYTES + ? proportionalHeadroom + : LOCAL_STAGING_MIN_HEADROOM_BYTES; + if (stats.bavail * stats.bsize < snapshotBytes + headroom) { + throw new Error( + "local model cache does not have enough free space for the audited snapshot copy", + ); } +} + +async function stagePreparedSnapshot( + plan: DualStationVllmPlan, + paths: StagingPaths, + auditedManifest: SnapshotManifest, + localMaterializedSnapshot: string, + deps: DualStationModelStagingDeps, + env: Record, +): Promise { + let transferManifest: SnapshotManifest; try { - if (prepare.status === 0 && JSON.parse(prepare.stdout.trim()).state === "ready") { - parseRemoteState("peer snapshot preflight", prepare, "ready"); - return { ok: true, transferred: false }; - } - parseRemoteState("peer snapshot preflight", prepare, "transfer"); + transferManifest = await auditLocalSnapshot(paths, deps, env, localMaterializedSnapshot); } catch (err) { - return { ok: false, reason: (err as Error).message }; + return failAfterPeerCleanup((err as Error).message, plan, paths, auditedManifest, deps, env); + } + if (!manifestsEqual(auditedManifest, transferManifest)) { + return failAfterPeerCleanup( + "local pinned snapshot changed between audit and materialization", + plan, + paths, + auditedManifest, + deps, + env, + ); + } + + let prepareState: "ready" | "transfer"; + try { + prepareState = await preparePeerSnapshot(plan, paths, transferManifest, deps, env); + } catch (err) { + return failAfterPeerCleanup((err as Error).message, plan, paths, transferManifest, deps, env); + } + if (prepareState === "ready") { + return { ok: true, transferred: false }; } let rsync: ModelStagingCommandResult; @@ -645,7 +906,6 @@ export async function stageDualStationModelSnapshot( [ "--recursive", "--times", - "--copy-links", "--checksum", "--partial", "--protect-args", @@ -655,7 +915,7 @@ export async function stageDualStationModelSnapshot( "--info=progress2", `--rsh=${rsyncRsh(plan)}`, "--", - `${paths.localSnapshot}/`, + `${localMaterializedSnapshot}/`, `${plan.peerSshBinding.peerTarget}:${paths.peerStaging}/`, ], { @@ -666,26 +926,119 @@ export async function stageDualStationModelSnapshot( }, ); } catch (err) { - return { ok: false, reason: (err as Error).message }; + return failAfterPeerCleanup((err as Error).message, plan, paths, transferManifest, deps, env); } if (rsync.status !== 0 || rsync.timedOut || rsync.error) { - return { ok: false, reason: commandFailure("peer snapshot transfer", rsync) }; + return failAfterPeerCleanup( + commandFailure("peer snapshot transfer", rsync), + plan, + paths, + transferManifest, + deps, + env, + ); } let finalize: ModelStagingCommandResult; try { finalize = await deps.runCommand("ssh", sshArgs(plan), { env, - input: remoteScript(plan, paths, manifest, "finalize"), + input: remoteScript(plan, paths, transferManifest, "finalize"), timeoutMs: SNAPSHOT_AUDIT_TIMEOUT_MS, }); } catch (err) { - return { ok: false, reason: (err as Error).message }; + return failAfterPeerCleanup((err as Error).message, plan, paths, transferManifest, deps, env); } try { parseRemoteState("peer snapshot verification", finalize, "ready"); } catch (err) { - return { ok: false, reason: (err as Error).message }; + return failAfterPeerCleanup((err as Error).message, plan, paths, transferManifest, deps, env); } return { ok: true, transferred: true }; } + +/** + * Audit before peer preflight so an already-ready peer avoids the large local + * copy. Transfers use only a second, immutable audit that exactly matches the + * first manifest, so no token or unrelated blob cache crosses the SSH boundary. + */ +export async function stageDualStationModelSnapshot( + plan: DualStationVllmPlan, + deps: DualStationModelStagingDeps = defaultDeps, +): Promise { + let paths: StagingPaths; + try { + paths = stagingPaths(plan); + } catch (err) { + return { ok: false, reason: (err as Error).message }; + } + const env = buildVllmSshTransportEnv({ LC_ALL: "C" }); + + let auditedManifest: SnapshotManifest; + try { + auditedManifest = await auditLocalSnapshot(paths, deps, env); + } catch (err) { + return { ok: false, reason: (err as Error).message }; + } + + let prepareState: "ready" | "transfer"; + try { + prepareState = await preparePeerSnapshot(plan, paths, auditedManifest, deps, env); + } catch (err) { + return { ok: false, reason: (err as Error).message }; + } + if (prepareState === "ready") { + return { ok: true, transferred: false }; + } + + try { + await requireLocalStagingCapacity(paths.localModelRoot, auditedManifest, deps); + } catch (err) { + return failAfterPeerCleanup((err as Error).message, plan, paths, auditedManifest, deps, env); + } + + let localStaging: { root: string; snapshot: string }; + try { + localStaging = createLocalStagingRoot(paths.localModelRoot); + } catch (err) { + return failAfterPeerCleanup( + `local audited snapshot setup failed: ${(err as Error).message}`, + plan, + paths, + auditedManifest, + deps, + env, + ); + } + + let result: DualStationModelStagingResult; + try { + result = await stagePreparedSnapshot( + plan, + paths, + auditedManifest, + localStaging.snapshot, + deps, + env, + ); + } catch (err) { + result = await failAfterPeerCleanup( + (err as Error).message, + plan, + paths, + auditedManifest, + deps, + env, + ); + } + try { + clearLocalStagingRoot(localStaging.root); + } catch (err) { + const cleanupFailure = `local audited snapshot cleanup failed: ${(err as Error).message}`; + return { + ok: false, + reason: result.ok ? cleanupFailure : `${result.reason}; ${cleanupFailure}`, + }; + } + return result; +} diff --git a/src/lib/inference/vllm-station-ssh-binding.test.ts b/src/lib/inference/vllm-station-ssh-binding.test.ts index 79fa41f347..5d548b7da9 100644 --- a/src/lib/inference/vllm-station-ssh-binding.test.ts +++ b/src/lib/inference/vllm-station-ssh-binding.test.ts @@ -52,7 +52,15 @@ describe("qualified dual-Station SSH binding", () => { root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-station-ssh-binding-")); fs.chmodSync(root, 0o700); dockerCliFile = path.join(root, "docker-cli"); - fs.writeFileSync(dockerCliFile, "#!/bin/bash\nexit 0\n", { mode: 0o700 }); + fs.writeFileSync( + dockerCliFile, + `#!/bin/bash +set -Eeuo pipefail +printf '%s\\0' "$@" > "${"${NEMOCLAW_TEST_DOCKER_RECORD:?}"}" +exit "${"${NEMOCLAW_TEST_DOCKER_EXIT:-0}"}" +`, + { mode: 0o700 }, + ); fs.chmodSync(dockerCliFile, 0o700); resumeStatePath = path.join(root, "station-dual-pair-resume.json"); }); @@ -100,6 +108,13 @@ describe("qualified dual-Station SSH binding", () => { expect(() => stationKnownHostsDigest(`@revoked ${PEER_HOST} ssh-rsa ${RSA_KEY}`)).toThrow( "no trusted key", ); + expect(() => + stationKnownHostsDigest( + [`@cert-authority ${PEER_HOST} ssh-ed25519 ${ED25519_KEY}`, KNOWN_HOSTS_LINES[0]].join( + "\n", + ), + ), + ).toThrow("marker is not allowed"); }); it("persists owner-only evidence and reloads one canonical handoff", () => { @@ -164,20 +179,32 @@ describe("qualified dual-Station SSH binding", () => { `ssh://station@${PEER_HOST}:${String(PEER_PORT)}`, ); - const wrapper = fs.readFileSync(binding.sshWrapperFile, "utf8"); - expect(wrapper).toMatch(/^#!\/bin\/bash\n/); - expect(wrapper).toContain(`/usr/bin/sha256sum < "$known_hosts"`); - expect(wrapper).toContain("exec /usr/bin/ssh '-T'"); - expect(wrapper).toContain(`'UserKnownHostsFile=${binding.knownHostsFile}'`); - expect(wrapper).toContain(`'HostKeyAlias=${binding.lookupHost}'`); - expect(wrapper).toContain(`'Hostname=${PEER_HOST}'`); - expect(wrapper).toContain('"$@"'); expect(spawnSync("/bin/bash", ["-n", binding.sshWrapperFile]).status).toBe(0); - - const dockerShim = fs.readFileSync(binding.dockerShimFile, "utf8"); - expect(dockerShim).toMatch(/^#!\/bin\/bash\n/); - expect(dockerShim).toContain(`exec '${binding.dockerCliFile}' "$@"`); expect(spawnSync("/bin/bash", ["-n", binding.dockerShimFile]).status).toBe(0); + + const dockerRecord = path.join(root, "docker-args"); + const dockerResult = spawnSync( + binding.dockerShimFile, + ["context", "inspect", "value with spaces"], + { + env: { + ...process.env, + NEMOCLAW_TEST_DOCKER_EXIT: "37", + NEMOCLAW_TEST_DOCKER_RECORD: dockerRecord, + }, + }, + ); + expect(dockerResult.status).toBe(37); + expect(fs.readFileSync(dockerRecord).toString("utf8").split("\0").slice(0, -1)).toEqual([ + "context", + "inspect", + "value with spaces", + ]); + + fs.appendFileSync(binding.knownHostsFile, "# tampered\n"); + const sshResult = spawnSync(binding.sshWrapperFile, ["-V"], { encoding: "utf8" }); + expect(sshResult.status).toBe(255); + expect(sshResult.stderr).toContain("refused a changed dual-Station SSH host-key pin"); }); it("omits only the default SSH port from the Docker URI", () => { diff --git a/src/lib/inference/vllm-station-ssh-binding.ts b/src/lib/inference/vllm-station-ssh-binding.ts index 6cdd608aaa..ceb413ec27 100644 --- a/src/lib/inference/vllm-station-ssh-binding.ts +++ b/src/lib/inference/vllm-station-ssh-binding.ts @@ -266,13 +266,16 @@ export function stationKnownHostsDigest(raw: string): string { if (!line || line.startsWith("#") || /[\u0000\r\n]/.test(line)) continue; const fields = line.split(/\s+/); const marker = fields[0]?.startsWith("@") ? (fields.shift() ?? "") : ""; + if (marker !== "" && marker !== "@revoked") { + throw new Error("Pinned Station known-hosts marker is not allowed"); + } if (fields.length < 3) throw new Error("Pinned Station known-hosts data is invalid"); const [_hosts, keyType, keyData] = fields; if (!SSH_KEY_TYPE_PATTERN.test(keyType) || !SSH_KEY_DATA_PATTERN.test(keyData)) { throw new Error("Pinned Station known-hosts key is invalid"); } keys.add(`${marker}|${keyType}|${keyData}`); - if (marker !== "@revoked") positiveKeys += 1; + if (marker === "") positiveKeys += 1; } if (keys.size === 0 || positiveKeys === 0) { throw new Error("Pinned Station known-hosts data has no trusted key"); @@ -371,7 +374,14 @@ function renderSshWrapper(binding: PinnedStationEndpoint & { knownHostsSha256: s set -Eeuo pipefail readonly known_hosts=${shellQuote(binding.knownHostsFile)} readonly expected_sha256=${shellQuote(binding.knownHostsSha256)} -actual_sha256="$(/usr/bin/sha256sum < "$known_hosts")" || exit 255 +if [[ -x /usr/bin/sha256sum ]]; then + actual_sha256="$(/usr/bin/sha256sum < "$known_hosts")" || exit 255 +elif [[ -x /usr/bin/shasum ]]; then + actual_sha256="$(/usr/bin/shasum -a 256 < "$known_hosts")" || exit 255 +else + printf '%s\n' 'NemoClaw could not verify the dual-Station SSH host-key pin.' >&2 + exit 255 +fi actual_sha256="${"${actual_sha256%% *}"}" if [[ "$actual_sha256" != "$expected_sha256" ]]; then printf '%s\n' 'NemoClaw refused a changed dual-Station SSH host-key pin.' >&2 diff --git a/src/lib/inference/vllm.test.ts b/src/lib/inference/vllm.test.ts index ad73e54b46..6bd8a7289a 100644 --- a/src/lib/inference/vllm.test.ts +++ b/src/lib/inference/vllm.test.ts @@ -54,7 +54,6 @@ vi.mock("./vllm-storage", async (importOriginal) => { }); import { - assertVllmRegistryDigestRef, buildVllmRunArgs, detectVllmProfile, installVllm, @@ -62,8 +61,6 @@ import { NEMOCLAW_VLLM_MANAGED_LABEL, pullImage, resolveVllmRuntimeProfile, - resolveVllmServedModelId, - VLLM_IMAGES, } from "./vllm"; import { buildVllmServeCommand, VLLM_MODELS } from "./vllm-models"; @@ -155,8 +152,13 @@ function mockSuccessfulVllmInstall( ["container", () => (ownershipQueue.shift() ?? (() => ""))()], ["ps", () => `${containerName}\n`], ]); - mocks.dockerCapture.mockImplementation((args: readonly string[]) => - (dockerCaptureByCommand.get(args[0] ?? "") ?? (() => ""))(), + mocks.dockerCapture.mockImplementation( + (args: readonly string[], options?: { env?: NodeJS.ProcessEnv }) => { + if (args[0] === "container" && options?.env?.DOCKER_CONTEXT === "default") { + return ""; + } + return (dockerCaptureByCommand.get(args[0] ?? "") ?? (() => ""))(); + }, ); } @@ -167,80 +169,6 @@ function mockInconclusiveDockerStorage(): void { }); } -describe("vLLM served route identity", () => { - it("uses one safe served-model override and rejects ambiguous aliases (#6315)", () => { - expect(resolveVllmServedModelId("catalog/model", [])).toBe("catalog/model"); - expect(resolveVllmServedModelId("catalog/model", ["--served-model-name", "served/model"])).toBe( - "served/model", - ); - expect(() => - resolveVllmServedModelId("catalog/model", [ - "--served-model-name", - "served/one", - "served/two", - ]), - ).toThrow("exactly one safe model ID"); - }); -}); - -describe("managed vLLM image distribution boundary", () => { - const digest = `sha256:${"a".repeat(64)}`; - - it("accepts repository-qualified immutable registry digests", () => { - expect(() => assertVllmRegistryDigestRef(`vllm/vllm-openai@${digest}`)).not.toThrow(); - expect(() => - assertVllmRegistryDigestRef(`registry.example.test:5000/team/runtime@${digest}`), - ).not.toThrow(); - }); - - it.each([ - `sha256:${"a".repeat(64)}`, - "vllm/vllm-openai:latest", - `ubuntu@${digest}`, - `vllm/vllm-openai@sha256:${"A".repeat(64)}`, - `vllm/vllm-openai@${digest}suffix`, - ` vllm/vllm-openai@${digest}`, - `vllm/vllm-openai@${digest} `, - ])("rejects an unpullable or mutable product image reference %j", (image) => { - expect(() => assertVllmRegistryDigestRef(image)).toThrow( - /pullable immutable registry reference/, - ); - }); - - it("keeps every shipped managed-vLLM image on a registry digest", () => { - const platformRefs = Object.values(VLLM_IMAGES).flatMap((imageSet) => - Object.values(imageSet) - .map((value) => - typeof value === "object" && value !== null && "ref" in value ? String(value.ref) : null, - ) - .filter((ref): ref is string => ref !== null), - ); - const runtimeRefs = VLLM_MODELS.map((model) => model.runtime?.image).filter( - (ref): ref is string => typeof ref === "string", - ); - const refs = new Set([...platformRefs, ...runtimeRefs]); - - expect(refs.size).toBeGreaterThan(0); - for (const ref of refs) { - expect(() => assertVllmRegistryDigestRef(ref), ref).not.toThrow(); - } - }); - - it("refuses a local image ID before invoking Docker pull", async () => { - mocks.dockerPullWithProgressWatchdog.mockClear(); - const profile = { - ...detectVllmProfile({ platform: "station", type: "nvidia" })!, - image: `sha256:${"a".repeat(64)}`, - }; - - await expect(pullImage(profile)).resolves.toEqual({ - ok: false, - reason: expect.stringContaining("Local image IDs"), - }); - expect(mocks.dockerPullWithProgressWatchdog).not.toHaveBeenCalled(); - }); -}); - describe("vLLM profile detection", () => { beforeEach(() => { vi.clearAllMocks(); @@ -1220,13 +1148,24 @@ describe("installVllm model resolution", () => { expect(result).toEqual({ ok: true }); expect(mocks.probeDockerStorage).toHaveBeenCalledTimes(1); + const canonicalOwnershipInspections = mocks.dockerCapture.mock.calls.filter( + (call) => call[0][0] === "container" && call[1]?.env?.DOCKER_CONTEXT === "default", + ); + expect(canonicalOwnershipInspections).toHaveLength(2); + for (const call of canonicalOwnershipInspections) { + expect(call[1]?.env).not.toHaveProperty("DOCKER_HOST"); + } + + const ambientDockerCaptureOptions = mocks.dockerCapture.mock.calls + .filter((call) => call[1]?.env?.DOCKER_CONTEXT !== "default") + .map((call) => call[1]); const dockerAdapterOptions = [ ...mocks.dockerImageInspectFormat.mock.calls.map((call) => call[2]), ...mocks.dockerPullWithProgressWatchdog.mock.calls.map((call) => call[1]), ...mocks.dockerSpawn.mock.calls.map((call) => call[1]), ...mocks.dockerForceRm.mock.calls.map((call) => call[1]), ...mocks.dockerRunDetached.mock.calls.map((call) => call[1]), - ...mocks.dockerCapture.mock.calls.map((call) => call[1]), + ...ambientDockerCaptureOptions, ]; expect(dockerAdapterOptions).toHaveLength(7); for (const options of dockerAdapterOptions) { @@ -1348,6 +1287,8 @@ describe("installVllm model resolution", () => { }); it("refuses single-host replacement of a managed dual head so the peer is not orphaned", async () => { + process.env.DOCKER_HOST = "ssh://builder.example.test"; + delete process.env.DOCKER_CONTEXT; const profile = detectVllmProfile({ platform: "spark", type: "nvidia" })!; const dualHead = vllmContainerRow(profile.containerName, { state: "running", @@ -1355,7 +1296,11 @@ describe("installVllm model resolution", () => { dualEndpoint: "http://192.168.100.1:8000", dualCluster: "f".repeat(64), }); - mockSuccessfulVllmInstall(profile.containerName, [() => dualHead]); + mockSuccessfulVllmInstall(profile.containerName); + mocks.dockerCapture.mockImplementation( + (args: readonly string[], options?: { env?: NodeJS.ProcessEnv }) => + args[0] === "container" && options?.env?.DOCKER_CONTEXT === "default" ? dualHead : "", + ); const result = await installVllm(profile, { hasImage: true, @@ -1365,10 +1310,82 @@ describe("installVllm model resolution", () => { expect(result).toEqual({ ok: false }); expect(mocks.dockerForceRm).not.toHaveBeenCalled(); + expect(mocks.dockerPullWithProgressWatchdog).not.toHaveBeenCalled(); expect(mocks.dockerSpawn).not.toHaveBeenCalled(); + expect(mocks.dockerCapture).toHaveBeenCalledTimes(1); + expect(mocks.dockerCapture.mock.calls[0]?.[1]?.env).toMatchObject({ + DOCKER_CONTEXT: "default", + }); + expect(mocks.dockerCapture.mock.calls[0]?.[1]?.env).not.toHaveProperty("DOCKER_HOST"); expect(errSpy).toHaveBeenCalledWith(expect.stringContaining("orphan the peer worker")); }); + it("fails closed when canonical Docker ownership inspection errors", async () => { + process.env.DOCKER_HOST = "ssh://builder.example.test"; + delete process.env.DOCKER_CONTEXT; + const profile = detectVllmProfile({ platform: "spark", type: "nvidia" })!; + mockSuccessfulVllmInstall(profile.containerName); + mocks.dockerCapture.mockImplementation( + (args: readonly string[], options?: { env?: NodeJS.ProcessEnv }) => { + if (args[0] === "container" && options?.env?.DOCKER_CONTEXT === "default") { + throw new Error("canonical Docker unavailable"); + } + return vllmContainerRow(profile.containerName); + }, + ); + + const result = await installVllm(profile, { + hasImage: true, + nonInteractive: true, + promptFn: vi.fn(), + }); + + expect(result).toEqual({ ok: false }); + expect(mocks.dockerCapture).toHaveBeenCalledTimes(1); + expect(mocks.dockerForceRm).not.toHaveBeenCalled(); + expect(mocks.dockerRunDetached).not.toHaveBeenCalled(); + expect(mocks.dockerPullWithProgressWatchdog).not.toHaveBeenCalled(); + expect(mocks.dockerSpawn).not.toHaveBeenCalled(); + expect(errSpy).toHaveBeenCalledWith( + expect.stringContaining("Could not verify ownership of Docker container"), + ); + }); + + it("fails closed on malformed canonical dual ownership instead of using ambient Docker", async () => { + process.env.DOCKER_HOST = "ssh://builder.example.test"; + delete process.env.DOCKER_CONTEXT; + const profile = detectVllmProfile({ platform: "spark", type: "nvidia" })!; + const malformedDualHead = vllmContainerRow(profile.containerName, { + state: "running", + dualRole: "head", + dualEndpoint: "http://192.168.100.1:8000", + dualCluster: "malformed", + }); + mockSuccessfulVllmInstall(profile.containerName); + mocks.dockerCapture.mockImplementation( + (args: readonly string[], options?: { env?: NodeJS.ProcessEnv }) => + args[0] === "container" && options?.env?.DOCKER_CONTEXT === "default" + ? malformedDualHead + : vllmContainerRow(profile.containerName), + ); + + const result = await installVllm(profile, { + hasImage: true, + nonInteractive: true, + promptFn: vi.fn(), + }); + + expect(result).toEqual({ ok: false }); + expect(mocks.dockerCapture).toHaveBeenCalledTimes(1); + expect(mocks.dockerForceRm).not.toHaveBeenCalled(); + expect(mocks.dockerRunDetached).not.toHaveBeenCalled(); + expect(mocks.dockerPullWithProgressWatchdog).not.toHaveBeenCalled(); + expect(mocks.dockerSpawn).not.toHaveBeenCalled(); + expect(errSpy).toHaveBeenCalledWith( + expect.stringContaining("Could not verify ownership of Docker container"), + ); + }); + it.each([ "", "false", diff --git a/src/lib/inference/vllm.ts b/src/lib/inference/vllm.ts index 1bf03bd3c0..37ae43cf03 100644 --- a/src/lib/inference/vllm.ts +++ b/src/lib/inference/vllm.ts @@ -51,12 +51,14 @@ import { import { areDualStationManagedVllmContainersRunning, cleanupDualStationManagedVllm, + commitDualStationLegacyMigration, DUAL_STATION_VLLM_CLUSTER_LABEL, DUAL_STATION_VLLM_ENDPOINT_LABEL, DUAL_STATION_VLLM_ROLE_LABEL, getDualStationManagedVllmBaseUrl, preflightDualStationGpuRuntime, preflightDualStationManagedVllm, + rollbackDualStationLegacyMigration, startDualStationManagedVllm, withDualStationManagedVllmLifecycle, } from "./vllm-station-cluster-lifecycle"; @@ -601,7 +603,10 @@ type VllmContainerOwnership = | { kind: "managed"; containerId: string; running: boolean } | { kind: "unknown" }; -function inspectVllmContainerOwnership(containerName: string): VllmContainerOwnership { +function inspectVllmContainerOwnershipInDockerEnv( + containerName: string, + env: Record, +): VllmContainerOwnership { const format = [ "{{.ID}}", "{{.Names}}", @@ -623,7 +628,7 @@ function inspectVllmContainerOwnership(containerName: string): VllmContainerOwne "--format", format, ], - { env: buildVllmDockerEnv(), timeout: 10_000 }, + { env, timeout: 10_000 }, ).trim(); if (!output) return { kind: "absent" }; @@ -655,6 +660,22 @@ function inspectVllmContainerOwnership(containerName: string): VllmContainerOwne } } +function inspectVllmContainerOwnership(containerName: string): VllmContainerOwnership { + // A managed dual-Station head always lives on the physical host's default + // daemon. Inspect it before following ambient single-host Docker routing so + // DOCKER_HOST, DOCKER_CONTEXT, or Docker's persisted currentContext cannot + // hide the pair from running-state detection or replacement guards. + const canonicalOwnership = inspectVllmContainerOwnershipInDockerEnv( + containerName, + buildLocalDualStationDockerEnv(), + ); + if (canonicalOwnership.kind === "dual-managed" || canonicalOwnership.kind === "unknown") { + return canonicalOwnership; + } + + return inspectVllmContainerOwnershipInDockerEnv(containerName, buildVllmDockerEnv()); +} + function vllmContainerReplacementTarget( containerName: string, ): { ok: true; containerId?: string } | { ok: false; reason: string } { @@ -1423,6 +1444,24 @@ export async function installVllm( return { ok: false }; } + const rollbackStartedPair = async (): Promise => { + if (start.reusedExisting) return; + if (start.legacyMigration) { + const rollback = await rollbackDualStationLegacyMigration( + dualStationPlan, + start.legacyMigration, + ); + if (!rollback.ok) { + for (const rollbackError of rollback.rollbackErrors) { + console.error(` vLLM rollback warning: ${rollbackError}`); + } + } + return; + } + const cleanup = await cleanupDualStationManagedVllm(dualStationPlan); + if (!cleanup.ok) console.error(` vLLM rollback warning: ${cleanup.reason}`); + }; + emit("Launching vLLM"); emit( `Launch can take 5 minutes to ${String(Math.ceil(runtimeProfile.loadTimeoutSec / 60))} minutes`, @@ -1431,10 +1470,7 @@ export async function installVllm( const ready = await waitForVllmReady(runtimeProfile, start.baseUrl, localDockerEnv); if (!ready.ok) { printContainerLogTail(runtimeProfile, localDockerEnv); - if (!start.reusedExisting) { - const cleanup = await cleanupDualStationManagedVllm(dualStationPlan); - if (!cleanup.ok) console.error(` vLLM rollback warning: ${cleanup.reason}`); - } + await rollbackStartedPair(); console.error(` vLLM install failed: ${String(ready.reason)}`); return { ok: false }; } @@ -1445,23 +1481,32 @@ export async function installVllm( servedModelId, ); if (!authBoundary.ok) { - if (!start.reusedExisting) { - const cleanup = await cleanupDualStationManagedVllm(dualStationPlan); - if (!cleanup.ok) console.error(` vLLM rollback warning: ${cleanup.reason}`); - } + await rollbackStartedPair(); console.error(` vLLM install failed: ${authBoundary.reason}`); return { ok: false }; } if (!areDualStationManagedVllmContainersRunning(dualStationPlan)) { - if (!start.reusedExisting) { - const cleanup = await cleanupDualStationManagedVllm(dualStationPlan); - if (!cleanup.ok) console.error(` vLLM rollback warning: ${cleanup.reason}`); - } + await rollbackStartedPair(); console.error(" vLLM distributed containers exited unexpectedly after readiness"); return { ok: false }; } + if (start.legacyMigration) { + const commit = await commitDualStationLegacyMigration( + dualStationPlan, + start.legacyMigration, + ); + if (!commit.ok) { + await rollbackStartedPair(); + console.error(` vLLM install failed: ${commit.reason}`); + return { ok: false }; + } + for (const warning of commit.cleanupWarnings) { + console.error(` vLLM cleanup warning: ${warning}`); + } + } + console.log(` ✓ vLLM ready across two DGX Stations at ${start.baseUrl}`); return { ok: true }; }); diff --git a/test/install-station-controller-binding.test.ts b/test/install-station-controller-binding.test.ts new file mode 100644 index 0000000000..1b82073b16 --- /dev/null +++ b/test/install-station-controller-binding.test.ts @@ -0,0 +1,411 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { INSTALLER_PAYLOAD, TEST_SYSTEM_PATH } from "./helpers/installer-sourced-env"; + +const REPO_ROOT = path.resolve(import.meta.dirname, ".."); +const STATION_PREPARE = path.join(REPO_ROOT, "scripts", "prepare-dgx-station-host.sh"); + +function runSourced(body: string) { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-station-controller-")); + const result = spawnSync( + "bash", + ["--noprofile", "--norc", "-c", `source "$SCRIPT_UNDER_TEST" >/dev/null\n${body}`], + { + cwd: REPO_ROOT, + encoding: "utf8", + env: { HOME: home, PATH: TEST_SYSTEM_PATH, SCRIPT_UNDER_TEST: STATION_PREPARE }, + timeout: 15_000, + killSignal: "SIGKILL", + }, + ); + return { home, result, output: `${result.stdout}${result.stderr}` }; +} + +function runInstallerBody(body: string, extraEnv: Record = {}) { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-station-migration-")); + const result = spawnSync( + "bash", + ["--noprofile", "--norc", "-c", `source "$INSTALLER_UNDER_TEST" >/dev/null\n${body}`], + { + cwd: REPO_ROOT, + encoding: "utf8", + env: { + HOME: home, + PATH: `${path.dirname(process.execPath)}:${TEST_SYSTEM_PATH}`, + INSTALLER_UNDER_TEST: INSTALLER_PAYLOAD, + ...extraEnv, + }, + timeout: 15_000, + killSignal: "SIGKILL", + }, + ); + return { home, result, output: `${result.stdout}${result.stderr}` }; +} + +function runInstallerOrderHarness(onboardStatus: 0 | 1) { + return runInstallerBody( + ` +record_order() { printf '%s\n' "$1" >>"$HOME/order.trace"; } +resolve_nemoclaw_gateway_port() { printf '18789'; } +preflight_explicit_express_flags() { :; } +print_banner() { :; } +preflight_usage_notice_prompt() { :; } +prepare_installer_host() { _SELECTED_EXPRESS_PLATFORM='DGX Station'; } +bash() { :; } +step() { :; } +install_nodejs() { :; } +ensure_supported_runtime() { :; } +ensure_station_express_pair() { record_order qualify; } +fix_npm_permissions() { :; } +preinstall_backup_and_retire_legacy_gateway() { :; } +install_nemoclaw() { record_order install; } +verify_nemoclaw() { :; } +command_exists() { return 0; } +registered_sandbox_count() { printf '0\n'; } +run_installer_host_preflight() { return 0; } +recover_preexisting_sandboxes_before_onboard() { return 0; } +run_onboard() { record_order onboard; return "$ONBOARD_STATUS"; } +restore_onboard_forward_after_post_checks() { return 0; } +finalize_install() { record_order finalize; } +clear_station_dual_pair_resume() { record_order clear; } +clear_station_express_resume() { :; } +main --non-interactive --yes-i-accept-third-party-software +`, + { ONBOARD_STATUS: String(onboardStatus) }, + ); +} + +describe("DGX Station controller UID binding", () => { + it("runs binding-only preparation without workload inspection and retains sudo acquisition", () => { + const { home, result, output } = runSourced(` +require_command() { :; } +acquire_sudo() { sudo_mode='acquired'; printf 'ACQUIRE_SUDO\\n'; } +check_platform() { printf 'CHECK_PLATFORM\\n'; } +check_no_workloads() { printf 'WORKLOAD_CHECK_MUST_NOT_RUN\\n'; return 1; } +ensure_dual_station_controller_uid_binding() { printf 'ENSURE_CONTROLLER_BINDING sudo=%s\\n' "$sudo_mode"; } +run_bind_controller +`); + try { + expect(result.status, output).toBe(0); + expect(output).toContain("ACQUIRE_SUDO"); + expect(output).toContain("CHECK_PLATFORM"); + expect(output).toContain("ENSURE_CONTROLLER_BINDING sudo=acquired"); + expect(output).toContain("CONTROLLER_UID_BINDING_READY"); + expect(output).not.toContain("WORKLOAD_CHECK_MUST_NOT_RUN"); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + }); + + it("creates one exact binding, reuses it, and requires administrator removal to rebind", () => { + const { home, result, output } = runSourced(` +config_dir="$HOME/etc/nemoclaw" +binding_file="$config_dir/dual-station-controller-uid" +mkdir -p "$HOME/etc" +controller_uid=1001 +preparation_controller_uid() { printf '%s\\n' "$controller_uid"; } +ensure_root_directory_safe() { mkdir -p "$1"; } +assert_root_directory_safe() { [[ -d "$1" ]]; } +root_regular_file_is_safe() { [[ -f "$1" ]]; } +root_directory_is_safe_unprivileged() { [[ -d "$1" ]]; } +root_regular_file_is_safe_unprivileged() { [[ -f "$1" ]]; } +sudo() { if [[ "$1" == "chown" ]]; then return 0; fi; "$@"; } +ensure_dual_station_controller_uid_binding "$config_dir" "$binding_file" +ensure_dual_station_controller_uid_binding "$config_dir" "$binding_file" +controller_uid=1002 +ensure_dual_station_controller_uid_binding "$config_dir" "$binding_file" +`); + + try { + expect(result.status, output).not.toBe(0); + expect(output.match(/dual_station_controller_uid=installed/g)).toHaveLength(1); + expect(output).toMatch(/administrator must remove .* before rebinding/); + const binding = path.join(home, "etc/nemoclaw/dual-station-controller-uid"); + expect(fs.readFileSync(binding, "utf8")).toBe("1001\n"); + expect(fs.statSync(binding).mode & 0o777).toBe(0o644); + expect( + fs + .readdirSync(path.dirname(binding)) + .filter((name) => name.startsWith(".dual-station-controller-uid.")), + ).toEqual([]); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + }); + + it("never lets an atomic publication loser replace the winner", () => { + const { home, result, output } = runSourced(` +config_dir="$HOME/etc/nemoclaw" +binding_file="$config_dir/dual-station-controller-uid" +mkdir -p "$HOME/etc" +preparation_controller_uid() { printf '1001\\n'; } +ensure_root_directory_safe() { mkdir -p "$1"; } +assert_root_directory_safe() { [[ -d "$1" ]]; } +root_regular_file_is_safe() { [[ -f "$1" ]]; } +root_directory_is_safe_unprivileged() { [[ -d "$1" ]]; } +root_regular_file_is_safe_unprivileged() { [[ -f "$1" ]]; } +sudo() { + if [[ "$1" == "chown" ]]; then return 0; fi + if [[ "$1" == "ln" ]]; then + printf '1002\\n' >"$binding_file" + chmod 0644 "$binding_file" + return 1 + fi + "$@" +} +ensure_dual_station_controller_uid_binding "$config_dir" "$binding_file" +`); + + try { + expect(result.status, output).not.toBe(0); + expect(output).toMatch(/administrator must remove .* before rebinding/); + expect( + fs.readFileSync(path.join(home, "etc/nemoclaw/dual-station-controller-uid"), "utf8"), + ).toBe("1002\n"); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + }); + + it("rejects a symlink without modifying its target", () => { + const { home, result, output } = runSourced(` +config_dir="$HOME/etc/nemoclaw" +binding_file="$config_dir/dual-station-controller-uid" +mkdir -p "$config_dir" +printf 'preserve\\n' >"$HOME/target" +ln -s "$HOME/target" "$binding_file" +preparation_controller_uid() { printf '1001\\n'; } +ensure_root_directory_safe() { :; } +root_directory_is_safe_unprivileged() { return 0; } +sudo() { "$@"; } +ensure_dual_station_controller_uid_binding "$config_dir" "$binding_file" +`); + + try { + expect(result.status, output).not.toBe(0); + expect(output).toMatch(/must not be a symbolic link/); + expect(fs.readFileSync(path.join(home, "target"), "utf8")).toBe("preserve\n"); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + }); + + it("rejects an existing mode-0700 configuration directory before creating a binding", () => { + const { home, result, output } = runSourced(` +config_dir="$HOME/etc/nemoclaw" +binding_file="$config_dir/dual-station-controller-uid" +mkdir -p "$config_dir" +chmod 0700 "$config_dir" +preparation_controller_uid() { printf '1001\\n'; } +ensure_root_directory_safe() { :; } +root_directory_is_safe_unprivileged() { return 1; } +sudo() { printf 'SUDO_AFTER_MODE_CHECK\\n' >&2; return 97; } +ensure_dual_station_controller_uid_binding "$config_dir" "$binding_file" +`); + try { + expect(result.status, output).not.toBe(0); + expect(output).toMatch(/must be root-owned with mode 0755 before binding/); + expect(output).not.toContain("SUDO_AFTER_MODE_CHECK"); + expect(fs.existsSync(path.join(home, "etc/nemoclaw/dual-station-controller-uid"))).toBe( + false, + ); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + }); + + it("fails verification on unsafe binding metadata", () => { + const { home, result, output } = runSourced(` +root_directory_is_safe_unprivileged() { return 0; } +root_regular_file_is_safe_unprivileged() { return 1; } +verify_dual_station_controller_uid_binding 1001 /etc/nemoclaw /etc/nemoclaw/dual-station-controller-uid +`); + try { + expect(result.status, output).not.toBe(0); + expect(output).toMatch(/root-owned regular file with mode 0644/); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + }); + + it("verifies readable root metadata and exact content without invoking sudo", () => { + const { home, result, output } = runSourced(` +config_dir="$HOME/etc/nemoclaw" +binding_file="$config_dir/dual-station-controller-uid" +mkdir -p "$config_dir" +printf '1001\\n' >"$binding_file" +stat() { + if [[ "\${@: -1}" == "$config_dir" ]]; then printf '0 0 755\\n'; else printf '0 0 644\\n'; fi +} +sudo() { printf 'SUDO_MUST_NOT_RUN\\n' >&2; return 97; } +verify_dual_station_controller_uid_binding 1001 "$config_dir" "$binding_file" +`); + try { + expect(result.status, output).toBe(0); + expect(output).toContain("dual_station_controller_uid=verified uid=1001"); + expect(output).not.toContain("SUDO_MUST_NOT_RUN"); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + }); + + it("rejects root preparation without an original non-root sudo UID", () => { + const { home, result, output } = runSourced(`preparation_controller_uid_for 0 ''`); + try { + expect(result.status, output).not.toBe(0); + expect(output).toMatch(/must be run by a non-root controller account/); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + }); + + it("recognizes only the frozen legacy head and routes it around workload preparation", () => { + const digest = + "vllm/vllm-openai@sha256:0fec7ec5f3e6bc168e54899935fb0557da908a4832a1dbc88e2debcf2f889416"; + const inspection = ["/nemoclaw-vllm", "true", digest, "true", ...Array(8).fill("-")].join("|"); + const detected = runInstallerBody( + `command_exists() { return 0; }; docker() { printf '%s\\n' "$INSPECTION"; }; station_migratable_legacy_single_head_running`, + { INSPECTION: inspection }, + ); + const routed = runInstallerBody(` +_SELECTED_EXPRESS_PLATFORM='DGX Station' +station_dual_model_requested() { return 0; } +station_managed_dual_head_running() { return 1; } +station_migratable_legacy_single_head_running() { return 0; } +run_station_host_preparation() { printf 'FULL_PREP_MUST_NOT_RUN\\n'; return 1; } +ensure_station_express_host +printf 'MIGRATE=%s REUSE=%s\\n' "$_STATION_EXPRESS_MIGRATING_LEGACY_HEAD" "$_STATION_EXPRESS_DEFERRED_MANAGED_PAIR" +`); + try { + expect(detected.result.status, detected.output).toBe(0); + expect(routed.result.status, routed.output).toBe(0); + expect(routed.output).toContain("MIGRATE=1 REUSE=0"); + expect(routed.output).not.toContain("FULL_PREP_MUST_NOT_RUN"); + } finally { + fs.rmSync(detected.home, { recursive: true, force: true }); + fs.rmSync(routed.home, { recursive: true, force: true }); + } + }); + + it.each([ + [ + "managed dual", + [ + "/nemoclaw-vllm", + "true", + "true", + "head", + "1", + "c".repeat(64), + "d".repeat(64), + "e".repeat(64), + "f".repeat(32), + ].join(" "), + "REUSE=1 MIGRATE=0", + 1, + ], + [ + "legacy single", + [ + "/nemoclaw-vllm", + "true", + "vllm/vllm-openai@sha256:0fec7ec5f3e6bc168e54899935fb0557da908a4832a1dbc88e2debcf2f889416", + "true", + ...Array(8).fill("-"), + ].join("|"), + "REUSE=0 MIGRATE=1", + 2, + ], + ] as const)("uses canonical local Docker to preserve a hidden %s head", (_kind, localInspection, expectedFlags, expectedInspections) => { + const { result, output, home } = runInstallerBody( + ` +command_exists() { return 0; } +docker() { + printf 'host=%s context=%s args=%s,%s\n' "\${DOCKER_HOST-unset}" "\${DOCKER_CONTEXT-unset}" "$1" "$2" >>"$HOME/docker.trace" + if [[ "\${DOCKER_HOST+x}" != x && "\${DOCKER_CONTEXT+x}" != x && "$1" == --context && "$2" == default ]]; then + printf '%s\n' "$LOCAL_DOCKER_INSPECTION" + fi +} +station_dual_model_requested() { return 0; } +run_station_host_preparation() { printf 'FULL_PREP_MUST_NOT_RUN\n'; return 97; } +_SELECTED_EXPRESS_PLATFORM='DGX Station' +ensure_station_express_host +printf 'REUSE=%s MIGRATE=%s\n' "$_STATION_EXPRESS_DEFERRED_MANAGED_PAIR" "$_STATION_EXPRESS_MIGRATING_LEGACY_HEAD" +`, + { + DOCKER_CONTEXT: "ambient-remote", + DOCKER_HOST: "ssh://remote-builder.example.test", + LOCAL_DOCKER_INSPECTION: localInspection, + }, + ); + try { + expect(result.status, output).toBe(0); + expect(output).toContain(expectedFlags); + expect(output).not.toContain("FULL_PREP_MUST_NOT_RUN"); + expect(fs.readFileSync(path.join(home, "docker.trace"), "utf8")).toBe( + "host=unset context=unset args=--context,default\n".repeat(expectedInspections), + ); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + }); + + it("passes legacy migration to the coordinator without managed-pair reuse", () => { + const argsFile = path.join(os.tmpdir(), `nemoclaw-legacy-args-${process.pid}-${Date.now()}`); + const { home, result, output } = runInstallerBody( + ` +node() { + if [[ "\${1:-}" == "--no-warnings" ]]; then + printf '%s\\n' "$*" >"$ARGS_FILE" + printf '%s\\n' '{"kind":"single-station","reason":"fixture"}' + return 0 + fi + command node "$@" +} +station_installer_revision() { printf '%040d' 0; } +_SELECTED_EXPRESS_PLATFORM='DGX Station' +_STATION_EXPRESS_MODEL_WAS_EXPLICIT=0 +_STATION_EXPRESS_DEFERRED_MANAGED_PAIR=0 +_STATION_EXPRESS_MIGRATING_LEGACY_HEAD=1 +unset NEMOCLAW_VLLM_MODEL NEMOCLAW_DGX_STATION_PEER +ensure_station_express_pair +`, + { ARGS_FILE: argsFile }, + ); + try { + expect(result.status, output).not.toBe(0); + const args = fs.readFileSync(argsFile, "utf8"); + expect(args).toContain("--migrate-legacy-single-head"); + expect(args).not.toContain("--reuse-existing-managed-pair"); + expect(output).toMatch(/legacy single-Station head.*refusing migration/u); + } finally { + fs.rmSync(argsFile, { force: true }); + fs.rmSync(home, { recursive: true, force: true }); + } + }); + + it("qualifies before install/onboarding and clears pair state only after success", () => { + const success = runInstallerOrderHarness(0); + const failure = runInstallerOrderHarness(1); + try { + expect(success.result.status, success.output).toBe(0); + expect(fs.readFileSync(path.join(success.home, "order.trace"), "utf8")).toBe( + "qualify\ninstall\nonboard\nfinalize\nclear\n", + ); + + expect(failure.result.status, failure.output).not.toBe(0); + expect(fs.readFileSync(path.join(failure.home, "order.trace"), "utf8")).toBe( + "qualify\ninstall\nonboard\n", + ); + } finally { + fs.rmSync(success.home, { recursive: true, force: true }); + fs.rmSync(failure.home, { recursive: true, force: true }); + } + }); +}); diff --git a/test/install-station-host-preparation.test.ts b/test/install-station-host-preparation.test.ts index 930eac49f5..eaf7f9cc58 100644 --- a/test/install-station-host-preparation.test.ts +++ b/test/install-station-host-preparation.test.ts @@ -41,6 +41,7 @@ function runSourced(script: string, body: string, extraEnv: Record { it("keeps documented Station pins and Deferred status aligned", () => { const helper = fs.readFileSync(STATION_PREPARE, "utf-8"); + expect(helper).toContain('readonly SCRIPT_VERSION="2026-07-16.7"'); const docs = STATION_DOCS.map((doc) => fs.readFileSync(doc, "utf-8")); const pinnedValues = [ "DRIVER_VERSION", @@ -358,6 +359,7 @@ acquire_sudo() { :; } all_packages_exact() { return 0; } install_boot_marker_matches_current_boot() { return 1; } driver_loaded_exact() { return 0; } +ensure_dual_station_controller_uid_binding() { printf 'ENSURE_CONTROLLER_UID\n'; } install_packages() { printf 'INSTALL_PACKAGES\n'; } finish_runtime() { printf 'FINISH_RUNTIME\n'; } verify_apply_state() { printf 'VERIFY_APPLY_STATE\n'; } @@ -366,6 +368,7 @@ run_apply ); expect(result.status, output).toBe(0); + expect(output).toContain("ENSURE_CONTROLLER_UID"); expect(output).toContain("FINISH_RUNTIME"); expect(output).toContain("VERIFY_APPLY_STATE"); expect(output).not.toContain("INSTALL_PACKAGES"); @@ -380,6 +383,7 @@ common_preflight() { :; } require_command() { :; } acquire_sudo() { :; } all_packages_exact() { return 1; } +ensure_dual_station_controller_uid_binding() { printf 'ENSURE_CONTROLLER_UID\n'; } installed_version() { if [[ "$1" == "dkms" ]]; then printf '3.0.11-1ubuntu13'; fi } @@ -394,6 +398,7 @@ run_apply expect(result.status, output).toBe(10); expect(output).toContain("package=dkms status=approved_transition"); + expect(output).toContain("ENSURE_CONTROLLER_UID"); expect(output).toContain("INSTALL_PACKAGES"); expect(output).toContain("ENSURE_DOCKER_GROUP"); expect(output).toContain("RECHECK_ALL_WORKLOADS"); @@ -657,6 +662,7 @@ acquire_sudo() { :; } all_packages_exact() { return 0; } install_boot_marker_matches_current_boot() { return 1; } driver_loaded_exact() { return 0; } +ensure_dual_station_controller_uid_binding() { printf 'ENSURE_CONTROLLER_UID\n'; } finish_runtime() { DOCKER_GROUP_ADDED=1; printf 'FINISH_RUNTIME\n'; } verify_apply_state() { printf 'VERIFY_APPLY_STATE\n'; } run_apply diff --git a/test/install-station-pair-preparation.test.ts b/test/install-station-pair-preparation.test.ts index 9864d510d1..d2695afa07 100644 --- a/test/install-station-pair-preparation.test.ts +++ b/test/install-station-pair-preparation.test.ts @@ -83,6 +83,26 @@ function stationHost(side: "local" | "peer"): StationDiscoveryHost { }; } +function stationConnectivity(side: "local" | "peer"): string { + const source = stationHost(side); + const destination = stationHost(side === "local" ? "peer" : "local"); + return JSON.stringify({ + schemaVersion: 1, + checks: source.rails.map((rail, index) => ({ + netdev: rail.netdev, + sourceAddress: rail.ipv4Addresses[0].address, + peerAddress: destination.rails[index].ipv4Addresses[0].address, + routeDevice: rail.netdev, + routeSource: rail.ipv4Addresses[0].address, + routeGateway: null, + routeScope: "link", + peerMac: destination.rails[index].macAddress, + peerNeighborState: "REACHABLE", + jumboPing: true, + })), + }); +} + function sshBinding(target = "10.10.0.2", keyData = HOST_KEY_DATA): PretrustedSshTarget { const knownHostsLine = `${target.slice(target.lastIndexOf("@") + 1)} ssh-ed25519 ${keyData}`; return { @@ -631,7 +651,7 @@ describe("dual-DGX Station reboot resume and reuse", () => { expect(harness.calls).not.toContain("remote:10.10.0.2:--verify"); }); - it("revalidates but does not rerun either helper for an exact managed pair", () => { + it("revalidates an exact managed pair and binds both controllers without workload probes", () => { const harness = new PreparationHarness(); trustFirstRail(harness); @@ -640,12 +660,36 @@ describe("dual-DGX Station reboot resume and reuse", () => { harness.deps, ); expect(result.kind).toBe("ready"); - expect(harness.calls.some((call) => call.startsWith("local:"))).toBe(false); - expect(harness.calls.some((call) => call.startsWith("remote:"))).toBe(false); + expect(harness.calls.filter((call) => call.startsWith("local:"))).toEqual([ + "local:--bind-controller", + ]); + expect(harness.calls.filter((call) => call.startsWith("remote:"))).toEqual([ + "remote:10.10.0.2:--bind-controller", + ]); expect(harness.calls).toContain("connectivity:local"); expect(harness.calls).toContain("connectivity:peer:10.10.0.2"); expect(harness.resume?.phase).toBe("ready"); }); + + it("binds only the active local controller before preparing a peer for legacy migration", () => { + const harness = new PreparationHarness(); + trustFirstRail(harness); + + expect( + prepareDualStationPair( + { ...preparationOptions(), migrateLegacySingleStationHead: true }, + harness.deps, + ).kind, + ).toBe("ready"); + expect(harness.calls.filter((call) => call.startsWith("local:"))).toEqual([ + "local:--bind-controller", + ]); + expect(harness.calls.filter((call) => call.startsWith("remote:"))).toEqual([ + "remote:10.10.0.2:--check", + "remote:10.10.0.2:--apply", + "remote:10.10.0.2:--verify", + ]); + }); }); describe.sequential("dual-DGX Station trust and resume-state boundaries", () => { @@ -764,6 +808,9 @@ describe.sequential("dual-DGX Station trust and resume-state boundaries", () => expect(command).toContain("sudo -n true"); expect(command).toContain("NEMOCLAW_STATION_PREP_SUDO_NONINTERACTIVE=1"); expect(command).toContain('bash "$f" --apply'); + expect(buildRemoteHelperCommand(HELPER_SHA256, "--bind-controller")).toContain( + 'bash "$f" --bind-controller', + ); }); it("does not expose ambient credentials or shell-loader variables to probes and helpers", () => { @@ -904,18 +951,226 @@ fi } }); - it("contains no trust enrollment or general network-discovery mechanism", () => { - const source = fs.readFileSync(COORDINATOR, "utf8").toLowerCase(); - for (const forbidden of [ + it("uses only deterministic rail candidates without trust enrollment or network discovery", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-pair-command-boundary-")); + const bin = path.join(root, "bin"); + const stateDirectory = path.join(root, "state"); + const helper = path.join(root, "prepare-dgx-station-host.sh"); + const state = path.join(stateDirectory, "resume.json"); + const forbiddenLog = path.join(root, "forbidden.log"); + fs.mkdirSync(bin, { mode: 0o700 }); + fs.mkdirSync(stateDirectory, { mode: 0o700 }); + fs.writeFileSync(helper, "#!/usr/bin/env bash\nexit 0\n", { mode: 0o700 }); + fs.writeFileSync( + path.join(bin, "python3"), + `#!/usr/bin/env bash\ncat <<'JSON'\n${JSON.stringify(stationHost("local"))}\nJSON\n`, + { mode: 0o700 }, + ); + fs.writeFileSync(path.join(bin, "ssh"), "#!/usr/bin/env bash\nexit 1\n", { mode: 0o700 }); + for (const command of [ "ssh-keyscan", "arp-scan", - "avahi", + "avahi-browse", "dns-sd", - "lldp", + "lldpctl", "nmap", - "mdns", + "mdns-scan", ]) { - expect(source, forbidden).not.toContain(forbidden); + fs.writeFileSync( + path.join(bin, command), + `#!/usr/bin/env bash\nprintf '%s\\n' ${JSON.stringify(command)} >>${JSON.stringify(forbiddenLog)}\nexit 97\n`, + { mode: 0o700 }, + ); + } + + try { + const result = spawnSync( + process.execPath, + [ + "--no-warnings", + "--experimental-strip-types", + COORDINATOR, + "--helper", + helper, + "--state", + state, + "--revision", + REVISION, + ], + { + cwd: REPO_ROOT, + encoding: "utf8", + env: { + ...process.env, + HOME: root, + PATH: `${bin}:${TEST_SYSTEM_PATH}`, + }, + timeout: 20_000, + killSignal: "SIGKILL", + }, + ); + + expect(result.status, `${result.stdout}${result.stderr}`).toBe(0); + expect(JSON.parse(result.stdout)).toMatchObject({ kind: "single-station" }); + expect(fs.existsSync(forbiddenLog)).toBe(false); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } + }); + + it("keeps forbidden discovery and trust enrollment unreachable through pair qualification", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-pair-ready-boundary-")); + const bin = path.join(root, "bin"); + const stateDirectory = path.join(root, "state"); + const helper = path.join(root, "prepare-dgx-station-host.sh"); + const state = path.join(stateDirectory, "resume.json"); + const knownHosts = path.join(root, "known_hosts"); + const forbiddenLog = path.join(root, "forbidden.log"); + fs.mkdirSync(bin, { mode: 0o700 }); + fs.mkdirSync(stateDirectory, { mode: 0o700 }); + fs.writeFileSync(helper, "#!/usr/bin/env bash\nexit 0\n", { mode: 0o700 }); + fs.writeFileSync(knownHosts, "fixture\n", { mode: 0o600 }); + fs.writeFileSync(path.join(bin, "docker"), "#!/usr/bin/env bash\nexit 0\n", { + mode: 0o700, + }); + fs.writeFileSync( + path.join(bin, "python3"), + `#!/usr/bin/env bash +set -Eeuo pipefail +cat >/dev/null +if (($# == 1)); then + cat <<'JSON' +${JSON.stringify(stationHost("local"))} +JSON +else + cat <<'JSON' +${stationConnectivity("local")} +JSON +fi +`, + { mode: 0o700 }, + ); + fs.writeFileSync( + path.join(bin, "ssh-keygen"), + `#!/usr/bin/env bash +set -Eeuo pipefail +if [[ " $* " == *" -F 10.10.0.2 "* ]]; then + printf '%s\n' '10.10.0.2 ssh-ed25519 ${HOST_KEY_DATA}' + exit 0 +fi +if [[ " $* " == *" -F "* ]]; then + exit 1 +fi +printf '%s\n' '256 ${HOST_KEY_FINGERPRINT} fixture (ED25519)' +`, + { mode: 0o700 }, + ); + fs.writeFileSync( + path.join(bin, "ssh"), + `#!/usr/bin/env bash +set -Eeuo pipefail +if [[ " $* " == *" -G "* ]]; then + target='' + for value in "$@"; do target="$value"; done + [[ "$target" == '10.10.0.2' ]] || exit 1 + cat <<'EOF' +user ubuntu +hostname 10.10.0.2 +port 22 +batchmode yes +stricthostkeychecking true +verifyhostkeydns false +nohostauthenticationforlocalhost no +permitlocalcommand no +forwardagent no +forwardx11 no +forwardx11trusted no +tunnel false +updatehostkeys false +controlmaster false +controlpath none +remotecommand none +proxycommand none +proxyjump none +localcommand none +knownhostscommand none +userknownhostsfile ${knownHosts} +globalknownhostsfile none +sendenv LANG +sendenv LC_* +EOF + exit 0 +fi +if [[ " $* " == *'python3 - enp1s0f0np0'* ]]; then + cat >/dev/null + cat <<'JSON' +${stationConnectivity("peer")} +JSON + exit 0 +fi +if [[ " $* " == *'python3 -'* ]]; then + cat >/dev/null + cat <<'JSON' +${JSON.stringify(stationHost("peer"))} +JSON + exit 0 +fi +if [[ " $* " == *'prepare-dgx-station-host.sh'* ]]; then + cat >/dev/null + exit 0 +fi +exit 96 +`, + { mode: 0o700 }, + ); + for (const command of [ + "ssh-keyscan", + "arp-scan", + "avahi-browse", + "dns-sd", + "lldpctl", + "nmap", + "mdns-scan", + ]) { + fs.writeFileSync( + path.join(bin, command), + `#!/usr/bin/env bash\nprintf '%s\\n' ${JSON.stringify(command)} >>${JSON.stringify(forbiddenLog)}\nexit 97\n`, + { mode: 0o700 }, + ); + } + + try { + const result = spawnSync( + process.execPath, + [ + "--no-warnings", + "--experimental-strip-types", + COORDINATOR, + "--helper", + helper, + "--state", + state, + "--revision", + REVISION, + ], + { + cwd: REPO_ROOT, + encoding: "utf8", + env: { + ...process.env, + HOME: root, + PATH: `${bin}:${TEST_SYSTEM_PATH}`, + }, + timeout: 20_000, + killSignal: "SIGKILL", + }, + ); + + expect(result.status, `${result.stdout}${result.stderr}`).toBe(0); + expect(JSON.parse(result.stdout)).toMatchObject({ kind: "ready", peerTarget: "10.10.0.2" }); + expect(fs.existsSync(forbiddenLog)).toBe(false); + } finally { + fs.rmSync(root, { recursive: true, force: true }); } }); }); @@ -1215,17 +1470,4 @@ printf 'RESULT selected=%s provider=%s selector=%s\n' "$_SELECTED_EXPRESS_PLATFO fs.rmSync(home, { recursive: true, force: true }); } }); - - it("orders pair qualification before CLI/onboarding and clears state only after success", () => { - const source = fs.readFileSync(INSTALLER_PAYLOAD, "utf8"); - const main = source.slice(source.indexOf("main() {"), source.indexOf("finalize_install() {")); - expect(main.indexOf("ensure_station_express_pair")).toBeGreaterThanOrEqual(0); - expect(main.indexOf("ensure_station_express_pair")).toBeLessThan( - main.indexOf("install_nemoclaw"), - ); - expect(main.indexOf("run_onboard")).toBeLessThan(main.indexOf("finalize_install")); - expect(main.indexOf("finalize_install")).toBeLessThan( - main.indexOf("clear_station_dual_pair_resume"), - ); - }); }); diff --git a/test/prepare-dual-dgx-station-resume-state.test.ts b/test/prepare-dual-dgx-station-resume-state.test.ts new file mode 100644 index 0000000000..5858cd7b26 --- /dev/null +++ b/test/prepare-dual-dgx-station-resume-state.test.ts @@ -0,0 +1,144 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { expect, it, vi } from "vitest"; +import type { DualStationResumeState } from "../scripts/lib/dgx-station-peer.mts"; +import { + clearDualStationResumeState, + writeDualStationResumeState, +} from "../scripts/prepare-dual-dgx-station.mts"; + +function readyState(): DualStationResumeState { + return { + schemaVersion: 1, + revision: "a".repeat(40), + helperSha256: "b".repeat(64), + phase: "ready", + peerTarget: "10.10.0.2", + hostKeyDigest: "c".repeat(64), + localGpuUuid: "GPU-LOCAL-0001", + peerGpuUuid: "GPU-PEER-0002", + rails: [ + { + localAddress: "10.10.0.1", + localMac: "02:00:00:00:00:01", + peerAddress: "10.10.0.2", + peerMac: "02:00:00:00:00:02", + }, + { + localAddress: "10.10.0.5", + localMac: "02:00:00:00:00:05", + peerAddress: "10.10.0.6", + peerMac: "02:00:00:00:00:06", + }, + ], + }; +} + +function captureThrown(operation: () => void): unknown { + try { + operation(); + } catch (error) { + return error; + } + return null; +} + +it("preserves the primary resume-state write error when temporary cleanup also fails", () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-pair-state-cleanup-")); + fs.chmodSync(directory, 0o700); + const statePath = path.join(directory, "resume.json"); + const primaryError = new Error("primary write failure"); + const cleanupError = Object.assign(new Error("temporary unlink failure"), { code: "EACCES" }); + const writeSpy = vi.spyOn(fs, "writeFileSync").mockImplementationOnce(() => { + throw primaryError; + }); + const unlinkSpy = vi.spyOn(fs, "unlinkSync").mockImplementation(() => { + throw cleanupError; + }); + try { + expect(captureThrown(() => writeDualStationResumeState(statePath, readyState()))).toBe( + primaryError, + ); + expect(unlinkSpy).toHaveBeenCalledTimes(1); + } finally { + writeSpy.mockRestore(); + unlinkSpy.mockRestore(); + fs.rmSync(directory, { recursive: true, force: true }); + } +}); + +it("surfaces a non-ENOENT temporary cleanup error when the state write succeeded", () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-pair-state-cleanup-")); + fs.chmodSync(directory, 0o700); + const statePath = path.join(directory, "resume.json"); + const cleanupError = Object.assign(new Error("temporary unlink failure"), { code: "EACCES" }); + const unlinkSpy = vi.spyOn(fs, "unlinkSync").mockImplementation(() => { + throw cleanupError; + }); + try { + expect(captureThrown(() => writeDualStationResumeState(statePath, readyState()))).toBe( + cleanupError, + ); + expect(fs.existsSync(statePath)).toBe(true); + } finally { + unlinkSpy.mockRestore(); + fs.rmSync(directory, { recursive: true, force: true }); + } +}); + +it("fsyncs the parent directory after deleting resume state", () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-pair-state-clear-")); + fs.chmodSync(directory, 0o700); + const statePath = path.join(directory, "resume.json"); + writeDualStationResumeState(statePath, readyState()); + const openSpy = vi.spyOn(fs, "openSync"); + const fsyncSpy = vi.spyOn(fs, "fsyncSync"); + const closeSpy = vi.spyOn(fs, "closeSync"); + try { + clearDualStationResumeState(statePath); + + const directoryOpenIndex = openSpy.mock.calls.findIndex(([target]) => target === directory); + expect(directoryOpenIndex).toBeGreaterThanOrEqual(0); + const directoryFd = openSpy.mock.results[directoryOpenIndex]?.value; + expect(fsyncSpy).toHaveBeenCalledWith(directoryFd); + expect(closeSpy).toHaveBeenCalledWith(directoryFd); + expect(fs.existsSync(statePath)).toBe(false); + } finally { + openSpy.mockRestore(); + fsyncSpy.mockRestore(); + closeSpy.mockRestore(); + fs.rmSync(directory, { recursive: true, force: true }); + } +}); + +it("preserves the directory fsync error when closing the directory also fails", () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-pair-state-clear-")); + fs.chmodSync(directory, 0o700); + const statePath = path.join(directory, "resume.json"); + writeDualStationResumeState(statePath, readyState()); + const primaryError = new Error("directory fsync failure"); + const cleanupError = new Error("directory close failure"); + const closeSync = fs.closeSync.bind(fs); + const fsyncSpy = vi.spyOn(fs, "fsyncSync").mockImplementationOnce(() => { + throw primaryError; + }); + const closeSpy = vi + .spyOn(fs, "closeSync") + .mockImplementationOnce((fd) => closeSync(fd)) + .mockImplementationOnce((fd) => { + closeSync(fd); + throw cleanupError; + }); + try { + expect(captureThrown(() => clearDualStationResumeState(statePath))).toBe(primaryError); + expect(fs.existsSync(statePath)).toBe(false); + } finally { + fsyncSpy.mockRestore(); + closeSpy.mockRestore(); + fs.rmSync(directory, { recursive: true, force: true }); + } +}); diff --git a/test/test-boundary-guards.test.ts b/test/test-boundary-guards.test.ts index b41003d3b7..d0468e2e63 100644 --- a/test/test-boundary-guards.test.ts +++ b/test/test-boundary-guards.test.ts @@ -696,6 +696,7 @@ describe("Vitest project membership boundary", () => { ["test/install-openshell-version-check.test.ts", "installer-integration"], ["test/install-preflight-docker-bootstrap.test.ts", "installer-integration"], ["test/install-preflight.test.ts", "installer-integration"], + ["test/install-station-controller-binding.test.ts", "installer-integration"], ["test/install-station-host-preparation.test.ts", "installer-integration"], ["test/install-station-pair-preparation.test.ts", "installer-integration"], ["test/package-contract/example.test.js", "package-contract"], diff --git a/vitest.config.ts b/vitest.config.ts index e2f439ee04..afc39a0a0f 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -124,6 +124,7 @@ export default defineConfig({ "test/install-clone-ref.test.ts", "test/install-preflight.test.ts", "test/install-preflight-docker-bootstrap.test.ts", + "test/install-station-controller-binding.test.ts", "test/install-station-host-preparation.test.ts", "test/install-station-pair-preparation.test.ts", "test/install-openshell-version-check.test.ts", @@ -144,6 +145,7 @@ export default defineConfig({ "test/install-clone-ref.test.ts", "test/install-preflight.test.ts", "test/install-preflight-docker-bootstrap.test.ts", + "test/install-station-controller-binding.test.ts", "test/install-station-host-preparation.test.ts", "test/install-station-pair-preparation.test.ts", "test/install-openshell-version-check.test.ts", From 152ebd9e02fb6649b384b5fc141c9fb26fd2e9e8 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 16 Jul 2026 21:15:23 -0700 Subject: [PATCH 28/74] test(vllm): satisfy conditional growth guardrail Signed-off-by: Aaron Erickson --- src/lib/inference/local-vllm-auth.test.ts | 12 +- src/lib/inference/vllm-dual-station.test.ts | 68 ++-- src/lib/inference/vllm-ownership.test.ts | 28 +- ...-station-cluster-lifecycle.test-support.ts | 384 ++++++++++++++++++ .../vllm-station-cluster-lifecycle.test.ts | 361 ++-------------- .../vllm-station-lifecycle-lock.test.ts | 28 +- ...vllm-station-model-staging.test-support.ts | 204 ++++++++++ .../vllm-station-model-staging.test.ts | 242 ++--------- src/lib/inference/vllm.test.ts | 38 +- 9 files changed, 752 insertions(+), 613 deletions(-) create mode 100644 src/lib/inference/vllm-station-cluster-lifecycle.test-support.ts create mode 100644 src/lib/inference/vllm-station-model-staging.test-support.ts diff --git a/src/lib/inference/local-vllm-auth.test.ts b/src/lib/inference/local-vllm-auth.test.ts index d097fc3df9..a8b5fdcb3a 100644 --- a/src/lib/inference/local-vllm-auth.test.ts +++ b/src/lib/inference/local-vllm-auth.test.ts @@ -36,6 +36,11 @@ import { const BASE_URL = "http://10.40.0.1:8000"; const API_KEY = "e".repeat(64); const OTHER_API_KEY = "f".repeat(64); +const MANAGED_BASE_URL_BY_API_KEY = new Map([ + [undefined, BASE_URL], + [API_KEY, BASE_URL], + [null, null], +]); let actualLifecycle: typeof import("./vllm-station-cluster-lifecycle"); @@ -76,10 +81,9 @@ function productionManagedBaseUrlResolver( beforeEach(() => { vi.stubEnv(LOCAL_INFERENCE_SANDBOX_HOST_URL_ENV, undefined); lifecycle.baseUrl.mockReset(); - lifecycle.baseUrl.mockImplementation((overrides) => { - if (!overrides?.loadApiKey) return BASE_URL; - return overrides.loadApiKey() === API_KEY ? BASE_URL : null; - }); + lifecycle.baseUrl.mockImplementation( + (overrides) => MANAGED_BASE_URL_BY_API_KEY.get(overrides?.loadApiKey?.()) ?? null, + ); }); afterEach(() => vi.unstubAllEnvs()); diff --git a/src/lib/inference/vllm-dual-station.test.ts b/src/lib/inference/vllm-dual-station.test.ts index b8ccf398b2..e3600fb90a 100644 --- a/src/lib/inference/vllm-dual-station.test.ts +++ b/src/lib/inference/vllm-dual-station.test.ts @@ -610,11 +610,46 @@ describe("dual DGX Station vLLM install orchestration", () => { }); it.each([ - "readiness", - "authentication", - "final running check", - "commit validation", - ])("restores legacy state when external %s fails", async (failure) => { + { + failure: "readiness", + configureFailure: () => { + mocks.runCurlProbe.mockReturnValue({ ok: false, httpStatus: 503, message: "loading" }); + mocks.dockerCapture.mockReturnValue(""); + }, + expectedCommitCalls: 0, + }, + { + failure: "authentication", + configureFailure: () => { + mocks.runCurlProbe.mockImplementation((args: string[]) => ({ + ok: true, + httpStatus: 200, + message: "ok", + body: args.at(-1)?.endsWith("/v1/models") + ? JSON.stringify({ data: [{ id: "nvidia/nemotron-3-ultra-550b-a55b" }] }) + : "", + })); + }, + expectedCommitCalls: 0, + }, + { + failure: "final running check", + configureFailure: () => mocks.areContainersRunning.mockReturnValue(false), + expectedCommitCalls: 0, + }, + { + failure: "commit validation", + configureFailure: () => + mocks.commitLegacyMigration.mockResolvedValue({ + ok: false, + reason: "new dual-Station transaction changed before commit", + }), + expectedCommitCalls: 1, + }, + ])("restores legacy state when external $failure fails", async ({ + configureFailure, + expectedCommitCalls, + }) => { mocks.startManaged.mockReturnValue({ ok: true, baseUrl: HEAD_BASE_URL, @@ -623,26 +658,7 @@ describe("dual DGX Station vLLM install orchestration", () => { reusedExisting: false, legacyMigration: LEGACY_MIGRATION, }); - if (failure === "readiness") { - mocks.runCurlProbe.mockReturnValue({ ok: false, httpStatus: 503, message: "loading" }); - mocks.dockerCapture.mockReturnValue(""); - } else if (failure === "authentication") { - mocks.runCurlProbe.mockImplementation((args: string[]) => ({ - ok: true, - httpStatus: 200, - message: "ok", - body: args.at(-1)?.endsWith("/v1/models") - ? JSON.stringify({ data: [{ id: "nvidia/nemotron-3-ultra-550b-a55b" }] }) - : "", - })); - } else if (failure === "final running check") { - mocks.areContainersRunning.mockReturnValue(false); - } else { - mocks.commitLegacyMigration.mockResolvedValue({ - ok: false, - reason: "new dual-Station transaction changed before commit", - }); - } + configureFailure(); const profile = detectVllmProfile({ platform: "station", type: "nvidia" }); await expect( @@ -651,7 +667,7 @@ describe("dual DGX Station vLLM install orchestration", () => { expect(mocks.rollbackLegacyMigration).toHaveBeenCalledWith(plan(), LEGACY_MIGRATION); expect(mocks.cleanup).not.toHaveBeenCalled(); - expect(mocks.commitLegacyMigration.mock.calls.length > 0).toBe(failure === "commit validation"); + expect(mocks.commitLegacyMigration).toHaveBeenCalledTimes(expectedCommitCalls); }); it("rolls back a new pair when unauthenticated model inventory is exposed", async () => { diff --git a/src/lib/inference/vllm-ownership.test.ts b/src/lib/inference/vllm-ownership.test.ts index 41e92c6902..1a18e912c8 100644 --- a/src/lib/inference/vllm-ownership.test.ts +++ b/src/lib/inference/vllm-ownership.test.ts @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { beforeEach, describe, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; const mocks = vi.hoisted(() => ({ dockerCapture: vi.fn(), @@ -40,6 +40,7 @@ function vllmContainerRow( } beforeEach(() => vi.clearAllMocks()); +afterEach(() => vi.unstubAllEnvs()); describe("managed vLLM ownership", () => { it("recognizes only the exact running container with the managed label", () => { @@ -85,10 +86,8 @@ describe("managed vLLM ownership", () => { }); it("checks the canonical local daemon before an ambient remote Docker host", () => { - const previousDockerHost = process.env.DOCKER_HOST; - const previousDockerContext = process.env.DOCKER_CONTEXT; - process.env.DOCKER_HOST = "ssh://builder.example.test"; - delete process.env.DOCKER_CONTEXT; + vi.stubEnv("DOCKER_HOST", "ssh://builder.example.test"); + vi.stubEnv("DOCKER_CONTEXT", undefined); mocks.dockerCapture.mockImplementation( (_args: readonly string[], options?: { env?: NodeJS.ProcessEnv }) => options?.env?.DOCKER_CONTEXT === "default" @@ -101,19 +100,12 @@ describe("managed vLLM ownership", () => { : "", ); - try { - expect(isNemoClawManagedVllmRunning()).toBe(true); - expect(mocks.dockerCapture).toHaveBeenCalledTimes(1); - expect(mocks.dockerCapture.mock.calls[0]?.[1]?.env).toMatchObject({ - DOCKER_CONTEXT: "default", - }); - expect(mocks.dockerCapture.mock.calls[0]?.[1]?.env).not.toHaveProperty("DOCKER_HOST"); - } finally { - if (previousDockerHost === undefined) delete process.env.DOCKER_HOST; - else process.env.DOCKER_HOST = previousDockerHost; - if (previousDockerContext === undefined) delete process.env.DOCKER_CONTEXT; - else process.env.DOCKER_CONTEXT = previousDockerContext; - } + expect(isNemoClawManagedVllmRunning()).toBe(true); + expect(mocks.dockerCapture).toHaveBeenCalledTimes(1); + expect(mocks.dockerCapture.mock.calls[0]?.[1]?.env).toMatchObject({ + DOCKER_CONTEXT: "default", + }); + expect(mocks.dockerCapture.mock.calls[0]?.[1]?.env).not.toHaveProperty("DOCKER_HOST"); }); it.each([ diff --git a/src/lib/inference/vllm-station-cluster-lifecycle.test-support.ts b/src/lib/inference/vllm-station-cluster-lifecycle.test-support.ts new file mode 100644 index 0000000000..3be60c2218 --- /dev/null +++ b/src/lib/inference/vllm-station-cluster-lifecycle.test-support.ts @@ -0,0 +1,384 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { AsyncLocalStorage } from "node:async_hooks"; +import { vi } from "vitest"; +import { DUAL_STATION_VLLM_RUNTIME, type DualStationVllmPlan } from "./vllm-station-cluster"; +import { + DUAL_STATION_VLLM_API_KEY_FINGERPRINT_LABEL, + DUAL_STATION_VLLM_CLUSTER_LABEL, + DUAL_STATION_VLLM_ENDPOINT_LABEL, + DUAL_STATION_VLLM_GPU_LABEL, + DUAL_STATION_VLLM_GPU_SMOKE_LABEL, + DUAL_STATION_VLLM_HEAD_CONTAINER_NAME, + DUAL_STATION_VLLM_LAUNCH_CONTRACT_LABEL, + DUAL_STATION_VLLM_LAUNCH_SCHEMA_LABEL, + DUAL_STATION_VLLM_MANAGED_LABEL, + DUAL_STATION_VLLM_ROLE_LABEL, + DUAL_STATION_VLLM_TRANSACTION_LABEL, + DUAL_STATION_VLLM_WORKER_CONTAINER_NAME, + type DualStationDockerOptions, + type DualStationLegacyMigration, + type DualStationVllmLifecycleDeps, + type StartDualStationVllmResult, +} from "./vllm-station-cluster-lifecycle"; +import type { DualStationSshBinding } from "./vllm-station-ssh-binding"; + +export type LifecycleFakeContainer = { + id: string; + name: string; + state: string; + image: string; + labels: Record; +}; + +export type LifecycleHarnessOptions = { + failRole?: "head" | "worker"; + invalidIdRole?: "head" | "worker"; + failSmokeTarget?: "local" | "peer"; + failSmokeCleanupTarget?: "local" | "peer"; + missingImageTarget?: "local" | "peer"; + smokeGpuOutput?: Partial>; + lateCreateRole?: "head" | "worker"; + failedRoleForeignTransaction?: "head" | "worker"; + failFinalInspectionRole?: "head" | "worker"; + failLegacyBackupRemoval?: boolean; +}; + +type LifecycleHarnessFixture = { + apiKey: string; + fakeContainer: ( + role: "head" | "worker", + overrides?: Partial, + ) => LifecycleFakeContainer; + headSmokeId: string; + legacyHeadId: string; + plan: () => DualStationVllmPlan; + workerSmokeId: string; +}; + +export function dualStationDockerValues(args: readonly string[], flag: string): string[] { + return args.flatMap((arg, index) => + arg === flag && index < args.length - 1 ? [args[index + 1]] : [], + ); +} + +export function requireLegacyMigration( + result: StartDualStationVllmResult, +): DualStationLegacyMigration { + if (!result.ok || !result.legacyMigration) { + throw new Error("expected legacy migration handle"); + } + return result.legacyMigration; +} + +function raise(message: string): never { + throw new Error(message); +} + +function row(container: LifecycleFakeContainer): string { + return [ + container.id, + container.name, + container.state, + container.image, + container.labels[DUAL_STATION_VLLM_MANAGED_LABEL] ?? "", + container.labels[DUAL_STATION_VLLM_ROLE_LABEL] ?? "", + container.labels[DUAL_STATION_VLLM_ENDPOINT_LABEL] ?? "", + container.labels[DUAL_STATION_VLLM_CLUSTER_LABEL] ?? "", + container.labels[DUAL_STATION_VLLM_GPU_LABEL] ?? "", + container.labels[DUAL_STATION_VLLM_LAUNCH_SCHEMA_LABEL] ?? "", + container.labels[DUAL_STATION_VLLM_LAUNCH_CONTRACT_LABEL] ?? "", + container.labels[DUAL_STATION_VLLM_API_KEY_FINGERPRINT_LABEL] ?? "", + container.labels[DUAL_STATION_VLLM_TRANSACTION_LABEL] ?? "", + ].join("\t"); +} + +export function createDualStationLifecycleHarness( + fixture: LifecycleHarnessFixture, + options: LifecycleHarnessOptions = {}, +) { + const containers = new Map(); + const operations: Array<{ + kind: "capture" | "rename" | "rm" | "run" | "start" | "stop"; + target: string; + value: string; + }> = []; + const captureOptions: Array = []; + const rmOptions: Array = []; + const runCalls: Array<{ + args: readonly string[]; + options: DualStationDockerOptions | undefined; + }> = []; + const buildRemoteDockerEnv = vi.fn((binding: DualStationSshBinding) => ({ + TARGET: "peer", + DOCKER_HOST: `ssh://${binding.sshUser}@${binding.resolvedHost}`, + VLLM_API_KEY: "ambient-must-be-stripped", + })); + let nonceCounter = 0; + let transactionCounter = 0; + let lifecycleLockActive = 0; + let maxLifecycleLockActive = 0; + let lifecycleLockTail = Promise.resolve(); + const lifecycleLockContext = new AsyncLocalStorage(); + let lateContainer: { targetName: string; container: LifecycleFakeContainer } | null = null; + const launchedRoles = new Set<"head" | "worker">(); + const managedInspectionCounts = { head: 0, worker: 0 }; + let finalInspectionFailureInjected = false; + + async function acquireLifecycleLock(operation: () => Promise | T): Promise { + const previous = lifecycleLockTail; + let release: () => void = () => undefined; + lifecycleLockTail = new Promise((resolve) => { + release = resolve; + }); + await previous; + lifecycleLockActive += 1; + maxLifecycleLockActive = Math.max(maxLifecycleLockActive, lifecycleLockActive); + try { + return await lifecycleLockContext.run(true, operation); + } finally { + lifecycleLockActive -= 1; + release(); + } + } + + function target(optionsArg?: DualStationDockerOptions): string { + return String(optionsArg?.env?.TARGET ?? "unknown"); + } + + function key(targetName: string, name: string): string { + return `${targetName}:${name}`; + } + + function exactContainerById( + targetName: string, + containerId: string, + ): { + containerKey: string; + entries: LifecycleFakeContainer[]; + container: LifecycleFakeContainer; + } | null { + for (const [containerKey, entries] of containers.entries()) { + if (!containerKey.startsWith(`${targetName}:`)) continue; + const container = entries.find((entry) => entry.id === containerId); + if (container) return { containerKey, entries, container }; + } + return null; + } + + const deps: DualStationVllmLifecycleDeps = { + buildLocalDockerEnv: () => ({ + TARGET: "local", + VLLM_API_KEY: "ambient-must-be-stripped", + }), + buildRemoteDockerEnv, + createProbeNonce: () => { + nonceCounter += 1; + return nonceCounter.toString(16).padStart(32, "0"); + }, + createTransactionId: () => { + transactionCounter += 1; + return transactionCounter.toString(16).padStart(32, "0"); + }, + effectiveControllerUid: () => fixture.plan().local.uid, + readControllerUid: () => fixture.plan().local.uid, + loadApiKey: () => fixture.apiKey, + localInterfaceAddresses: () => [fixture.plan().masterAddress], + waitBeforeReconcile: async () => { + const pending = lateContainer; + lateContainer = null; + return pending + ? void containers.set(key(pending.targetName, pending.container.name), [pending.container]) + : undefined; + }, + withLifecycleLock: async (operation: () => Promise | T) => + lifecycleLockContext.getStore() ? await operation() : await acquireLifecycleLock(operation), + dockerCapture: (args, optionsArg) => { + captureOptions.push(optionsArg); + const targetName = target(optionsArg); + if (args[0] === "container" && args[1] === "rename") { + const containerId = args[2]; + const newName = args[3]; + operations.push({ + kind: "rename", + target: targetName, + value: `${containerId}:${newName}`, + }); + const located = exactContainerById(targetName, containerId); + if (!located || (containers.get(key(targetName, newName)) ?? []).length > 0) { + return raise("rename failed"); + } + containers.set( + located.containerKey, + located.entries.filter((entry) => entry.id !== containerId), + ); + located.container.name = newName; + containers.set(key(targetName, newName), [located.container]); + return ""; + } + if (args[0] === "container" && (args[1] === "start" || args[1] === "stop")) { + const action = args[1]; + const containerId = args.at(-1) ?? ""; + operations.push({ kind: action, target: targetName, value: containerId }); + const located = exactContainerById(targetName, containerId); + if (!located) return raise(`${action} failed`); + located.container.state = action === "start" ? "running" : "exited"; + return containerId; + } + switch (args[0]) { + case "image": { + operations.push({ kind: "capture", target: targetName, value: `image:${args.at(-1)}` }); + return options.missingImageTarget === targetName + ? raise("missing image") + : `sha256:${"f".repeat(64)}\n`; + } + case "wait": + operations.push({ kind: "capture", target: targetName, value: `wait:${args[1]}` }); + return "0\n"; + case "logs": { + operations.push({ kind: "capture", target: targetName, value: `logs:${args[1]}` }); + const defaultUuid = + targetName === "local" ? fixture.plan().local.gpu.uuid : fixture.plan().peer.gpu.uuid; + return `${options.smokeGpuOutput?.[targetName as "local" | "peer"] ?? defaultUuid}\n`; + } + default: + break; + } + const filter = dualStationDockerValues(args, "--filter")[0] ?? ""; + const name = filter.replace(/^name=\^\//, "").replace(/\$$/, ""); + operations.push({ kind: "capture", target: targetName, value: name }); + const isSmokeInspection = + dualStationDockerValues(args, "--format")[0]?.includes(DUAL_STATION_VLLM_GPU_SMOKE_LABEL) ?? + false; + let inspected = containers.get(key(targetName, name)) ?? []; + const inspectedRole = + name === DUAL_STATION_VLLM_HEAD_CONTAINER_NAME + ? "head" + : name === DUAL_STATION_VLLM_WORKER_CONTAINER_NAME + ? "worker" + : null; + if (!isSmokeInspection && inspectedRole && launchedRoles.has(inspectedRole)) { + managedInspectionCounts[inspectedRole] += 1; + if ( + options.failFinalInspectionRole === inspectedRole && + managedInspectionCounts[inspectedRole] === 2 && + !finalInspectionFailureInjected + ) { + finalInspectionFailureInjected = true; + inspected = inspected.map((container) => ({ ...container, state: "exited" })); + } + } + return inspected + .map((container) => + isSmokeInspection + ? [ + container.id, + container.name, + container.image, + container.labels[DUAL_STATION_VLLM_GPU_SMOKE_LABEL] ?? "", + container.labels[DUAL_STATION_VLLM_ROLE_LABEL] ?? "", + ].join("\t") + : row(container), + ) + .join("\n"); + }, + dockerRunDetached: (args, optionsArg) => { + const targetName = target(optionsArg); + const name = dualStationDockerValues(args, "--name")[0]; + runCalls.push({ args: [...args], options: optionsArg }); + operations.push({ kind: "run", target: targetName, value: name }); + const labels = Object.fromEntries( + dualStationDockerValues(args, "--label").map((label) => { + const separator = label.indexOf("="); + return [label.slice(0, separator), label.slice(separator + 1)]; + }), + ); + switch (name.startsWith("nemoclaw-vllm-gpu-smoke-")) { + case true: + return options.failSmokeTarget === targetName + ? { status: 1, stdout: "", stderr: "smoke failed" } + : (() => { + const smokeContainer: LifecycleFakeContainer = { + id: targetName === "local" ? fixture.headSmokeId : fixture.workerSmokeId, + name, + state: "exited", + image: DUAL_STATION_VLLM_RUNTIME.image, + labels, + }; + containers.set(key(targetName, name), [smokeContainer]); + return { status: 0, stdout: `${smokeContainer.id}\n` }; + })(); + default: + break; + } + const role = name === DUAL_STATION_VLLM_HEAD_CONTAINER_NAME ? "head" : "worker"; + launchedRoles.add(role); + const imageIndex = args.indexOf("/bin/bash") + 1; + const container = fixture.fakeContainer(role, { + image: args[imageIndex], + labels, + }); + return options.failedRoleForeignTransaction === role + ? (() => { + container.labels[DUAL_STATION_VLLM_TRANSACTION_LABEL] = "f".repeat(32); + containers.set(key(targetName, name), [container]); + return { status: 1, stdout: "", stderr: "ambiguous failed create" }; + })() + : options.lateCreateRole === role + ? (() => { + lateContainer = { targetName, container }; + return { status: null, stdout: "", stderr: "timed out" }; + })() + : options.failRole === role + ? { status: 1, stdout: "", stderr: "failed" } + : (() => { + containers.set(key(targetName, name), [container]); + return { + status: 0, + stdout: + options.invalidIdRole === role ? "not-a-container-id\n" : `${container.id}\n`, + }; + })(); + }, + dockerForceRm: (containerId, optionsArg) => { + rmOptions.push(optionsArg); + const targetName = target(optionsArg); + operations.push({ kind: "rm", target: targetName, value: containerId }); + const shouldFail = + (options.failSmokeCleanupTarget === targetName && + (containerId === fixture.workerSmokeId || containerId === fixture.headSmokeId)) || + (options.failLegacyBackupRemoval && containerId === fixture.legacyHeadId); + const match = [...containers.entries()].find( + ([containerKey, entries]) => + containerKey.startsWith(`${targetName}:`) && + entries.some((entry) => entry.id === containerId), + ); + return shouldFail || !match + ? { status: 1 } + : (() => { + const [containerKey, entries] = match; + const remaining = entries.filter((entry) => entry.id !== containerId); + containers.set(containerKey, remaining); + return { status: 0 }; + })(); + }, + }; + + function seed(targetName: "local" | "peer", container: LifecycleFakeContainer): void { + const containerKey = key(targetName, container.name); + containers.set(containerKey, [...(containers.get(containerKey) ?? []), container]); + } + + return { + buildRemoteDockerEnv, + captureOptions, + containers, + deps, + getMaxLifecycleLockActive: () => maxLifecycleLockActive, + operations, + rmOptions, + runCalls, + seed, + }; +} diff --git a/src/lib/inference/vllm-station-cluster-lifecycle.test.ts b/src/lib/inference/vllm-station-cluster-lifecycle.test.ts index 3401ee32ab..6bb89e56f2 100644 --- a/src/lib/inference/vllm-station-cluster-lifecycle.test.ts +++ b/src/lib/inference/vllm-station-cluster-lifecycle.test.ts @@ -1,7 +1,6 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { AsyncLocalStorage } from "node:async_hooks"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; @@ -25,7 +24,6 @@ import { DUAL_STATION_VLLM_ROLE_LABEL, DUAL_STATION_VLLM_TRANSACTION_LABEL, DUAL_STATION_VLLM_WORKER_CONTAINER_NAME, - type DualStationDockerOptions, type DualStationVllmLifecycleDeps, dualStationVllmApiKeyFingerprint, dualStationVllmClusterId, @@ -37,8 +35,14 @@ import { startDualStationManagedVllm, withDualStationManagedVllmLifecycle, } from "./vllm-station-cluster-lifecycle"; +import { + createDualStationLifecycleHarness, + dualStationDockerValues as dockerValues, + type LifecycleFakeContainer as FakeContainer, + type LifecycleHarnessOptions as HarnessOptions, + requireLegacyMigration, +} from "./vllm-station-cluster-lifecycle.test-support"; import { withDualStationVllmLifecycleLock } from "./vllm-station-lifecycle-lock"; -import type { DualStationSshBinding } from "./vllm-station-ssh-binding"; import { createDualStationSshBindingFixture, type DualStationSshBindingFixture, @@ -136,42 +140,6 @@ function fixturePlan(): DualStationVllmPlan { }; } -type FakeContainer = { - id: string; - name: string; - state: string; - image: string; - labels: Record; -}; - -function dockerValues(args: readonly string[], flag: string): string[] { - return args.flatMap((arg, index) => - arg === flag && index < args.length - 1 ? [args[index + 1]] : [], - ); -} - -function raise(message: string): never { - throw new Error(message); -} - -function row(container: FakeContainer): string { - return [ - container.id, - container.name, - container.state, - container.image, - container.labels[DUAL_STATION_VLLM_MANAGED_LABEL] ?? "", - container.labels[DUAL_STATION_VLLM_ROLE_LABEL] ?? "", - container.labels[DUAL_STATION_VLLM_ENDPOINT_LABEL] ?? "", - container.labels[DUAL_STATION_VLLM_CLUSTER_LABEL] ?? "", - container.labels[DUAL_STATION_VLLM_GPU_LABEL] ?? "", - container.labels[DUAL_STATION_VLLM_LAUNCH_SCHEMA_LABEL] ?? "", - container.labels[DUAL_STATION_VLLM_LAUNCH_CONTRACT_LABEL] ?? "", - container.labels[DUAL_STATION_VLLM_API_KEY_FINGERPRINT_LABEL] ?? "", - container.labels[DUAL_STATION_VLLM_TRANSACTION_LABEL] ?? "", - ].join("\t"); -} - function fakeContainer( role: "head" | "worker", overrides: Partial = {}, @@ -201,298 +169,18 @@ function fakeContainer( }; } -type HarnessOptions = { - failRole?: "head" | "worker"; - invalidIdRole?: "head" | "worker"; - failSmokeTarget?: "local" | "peer"; - failSmokeCleanupTarget?: "local" | "peer"; - missingImageTarget?: "local" | "peer"; - smokeGpuOutput?: Partial>; - lateCreateRole?: "head" | "worker"; - failedRoleForeignTransaction?: "head" | "worker"; - failFinalInspectionRole?: "head" | "worker"; - failLegacyBackupRemoval?: boolean; -}; - function harness(options: HarnessOptions = {}) { - const containers = new Map(); - const operations: Array<{ - kind: "capture" | "rename" | "rm" | "run" | "start" | "stop"; - target: string; - value: string; - }> = []; - const captureOptions: Array = []; - const rmOptions: Array = []; - const runCalls: Array<{ - args: readonly string[]; - options: DualStationDockerOptions | undefined; - }> = []; - const buildRemoteDockerEnv = vi.fn((binding: DualStationSshBinding) => ({ - TARGET: "peer", - DOCKER_HOST: `ssh://${binding.sshUser}@${binding.resolvedHost}`, - VLLM_API_KEY: "ambient-must-be-stripped", - })); - let nonceCounter = 0; - let transactionCounter = 0; - let lifecycleLockActive = 0; - let maxLifecycleLockActive = 0; - let lifecycleLockTail = Promise.resolve(); - const lifecycleLockContext = new AsyncLocalStorage(); - let lateContainer: { targetName: string; container: FakeContainer } | null = null; - const launchedRoles = new Set<"head" | "worker">(); - const managedInspectionCounts = { head: 0, worker: 0 }; - let finalInspectionFailureInjected = false; - - async function acquireLifecycleLock(operation: () => Promise | T): Promise { - const previous = lifecycleLockTail; - let release: () => void = () => undefined; - lifecycleLockTail = new Promise((resolve) => { - release = resolve; - }); - await previous; - lifecycleLockActive += 1; - maxLifecycleLockActive = Math.max(maxLifecycleLockActive, lifecycleLockActive); - try { - return await lifecycleLockContext.run(true, operation); - } finally { - lifecycleLockActive -= 1; - release(); - } - } - - function target(optionsArg?: DualStationDockerOptions): string { - return String(optionsArg?.env?.TARGET ?? "unknown"); - } - - function key(targetName: string, name: string): string { - return `${targetName}:${name}`; - } - - function exactContainerById( - targetName: string, - containerId: string, - ): { containerKey: string; entries: FakeContainer[]; container: FakeContainer } | null { - for (const [containerKey, entries] of containers.entries()) { - if (!containerKey.startsWith(`${targetName}:`)) continue; - const container = entries.find((entry) => entry.id === containerId); - if (container) return { containerKey, entries, container }; - } - return null; - } - - const deps: DualStationVllmLifecycleDeps = { - buildLocalDockerEnv: () => ({ - TARGET: "local", - VLLM_API_KEY: "ambient-must-be-stripped", - }), - buildRemoteDockerEnv, - createProbeNonce: () => { - nonceCounter += 1; - return nonceCounter.toString(16).padStart(32, "0"); - }, - createTransactionId: () => { - transactionCounter += 1; - return transactionCounter.toString(16).padStart(32, "0"); - }, - effectiveControllerUid: () => fixturePlan().local.uid, - readControllerUid: () => fixturePlan().local.uid, - loadApiKey: () => API_KEY, - localInterfaceAddresses: () => [fixturePlan().masterAddress], - waitBeforeReconcile: async () => { - const pending = lateContainer; - lateContainer = null; - return pending - ? void containers.set(key(pending.targetName, pending.container.name), [pending.container]) - : undefined; - }, - withLifecycleLock: async (operation: () => Promise | T) => - lifecycleLockContext.getStore() ? await operation() : await acquireLifecycleLock(operation), - dockerCapture: (args, optionsArg) => { - captureOptions.push(optionsArg); - const targetName = target(optionsArg); - if (args[0] === "container" && args[1] === "rename") { - const containerId = args[2]; - const newName = args[3]; - operations.push({ - kind: "rename", - target: targetName, - value: `${containerId}:${newName}`, - }); - const located = exactContainerById(targetName, containerId); - if (!located || (containers.get(key(targetName, newName)) ?? []).length > 0) { - return raise("rename failed"); - } - containers.set( - located.containerKey, - located.entries.filter((entry) => entry.id !== containerId), - ); - located.container.name = newName; - containers.set(key(targetName, newName), [located.container]); - return ""; - } - if (args[0] === "container" && (args[1] === "start" || args[1] === "stop")) { - const action = args[1]; - const containerId = args.at(-1) ?? ""; - operations.push({ kind: action, target: targetName, value: containerId }); - const located = exactContainerById(targetName, containerId); - if (!located) return raise(`${action} failed`); - located.container.state = action === "start" ? "running" : "exited"; - return containerId; - } - switch (args[0]) { - case "image": { - operations.push({ kind: "capture", target: targetName, value: `image:${args.at(-1)}` }); - return options.missingImageTarget === targetName - ? raise("missing image") - : `sha256:${"f".repeat(64)}\n`; - } - case "wait": - operations.push({ kind: "capture", target: targetName, value: `wait:${args[1]}` }); - return "0\n"; - case "logs": { - operations.push({ kind: "capture", target: targetName, value: `logs:${args[1]}` }); - const defaultUuid = - targetName === "local" ? fixturePlan().local.gpu.uuid : fixturePlan().peer.gpu.uuid; - return `${options.smokeGpuOutput?.[targetName as "local" | "peer"] ?? defaultUuid}\n`; - } - default: - break; - } - const filter = dockerValues(args, "--filter")[0] ?? ""; - const name = filter.replace(/^name=\^\//, "").replace(/\$$/, ""); - operations.push({ kind: "capture", target: targetName, value: name }); - const isSmokeInspection = - dockerValues(args, "--format")[0]?.includes(DUAL_STATION_VLLM_GPU_SMOKE_LABEL) ?? false; - let inspected = containers.get(key(targetName, name)) ?? []; - const inspectedRole = - name === DUAL_STATION_VLLM_HEAD_CONTAINER_NAME - ? "head" - : name === DUAL_STATION_VLLM_WORKER_CONTAINER_NAME - ? "worker" - : null; - if (!isSmokeInspection && inspectedRole && launchedRoles.has(inspectedRole)) { - managedInspectionCounts[inspectedRole] += 1; - if ( - options.failFinalInspectionRole === inspectedRole && - managedInspectionCounts[inspectedRole] === 2 && - !finalInspectionFailureInjected - ) { - finalInspectionFailureInjected = true; - inspected = inspected.map((container) => ({ ...container, state: "exited" })); - } - } - return inspected - .map((container) => - isSmokeInspection - ? [ - container.id, - container.name, - container.image, - container.labels[DUAL_STATION_VLLM_GPU_SMOKE_LABEL] ?? "", - container.labels[DUAL_STATION_VLLM_ROLE_LABEL] ?? "", - ].join("\t") - : row(container), - ) - .join("\n"); + return createDualStationLifecycleHarness( + { + apiKey: API_KEY, + fakeContainer, + headSmokeId: HEAD_SMOKE_ID, + legacyHeadId: LEGACY_HEAD_ID, + plan: fixturePlan, + workerSmokeId: WORKER_SMOKE_ID, }, - dockerRunDetached: (args, optionsArg) => { - const targetName = target(optionsArg); - const name = dockerValues(args, "--name")[0]; - runCalls.push({ args: [...args], options: optionsArg }); - operations.push({ kind: "run", target: targetName, value: name }); - const labels = Object.fromEntries( - dockerValues(args, "--label").map((label) => { - const separator = label.indexOf("="); - return [label.slice(0, separator), label.slice(separator + 1)]; - }), - ); - switch (name.startsWith("nemoclaw-vllm-gpu-smoke-")) { - case true: - return options.failSmokeTarget === targetName - ? { status: 1, stdout: "", stderr: "smoke failed" } - : (() => { - const smokeContainer: FakeContainer = { - id: targetName === "local" ? HEAD_SMOKE_ID : WORKER_SMOKE_ID, - name, - state: "exited", - image: DUAL_STATION_VLLM_RUNTIME.image, - labels, - }; - containers.set(key(targetName, name), [smokeContainer]); - return { status: 0, stdout: `${smokeContainer.id}\n` }; - })(); - default: - break; - } - const role = name === DUAL_STATION_VLLM_HEAD_CONTAINER_NAME ? "head" : "worker"; - launchedRoles.add(role); - const imageIndex = args.indexOf("/bin/bash") + 1; - const container = fakeContainer(role, { - image: args[imageIndex], - labels, - }); - return options.failedRoleForeignTransaction === role - ? (() => { - container.labels[DUAL_STATION_VLLM_TRANSACTION_LABEL] = "f".repeat(32); - containers.set(key(targetName, name), [container]); - return { status: 1, stdout: "", stderr: "ambiguous failed create" }; - })() - : options.lateCreateRole === role - ? (() => { - lateContainer = { targetName, container }; - return { status: null, stdout: "", stderr: "timed out" }; - })() - : options.failRole === role - ? { status: 1, stdout: "", stderr: "failed" } - : (() => { - containers.set(key(targetName, name), [container]); - return { - status: 0, - stdout: - options.invalidIdRole === role ? "not-a-container-id\n" : `${container.id}\n`, - }; - })(); - }, - dockerForceRm: (containerId, optionsArg) => { - rmOptions.push(optionsArg); - const targetName = target(optionsArg); - operations.push({ kind: "rm", target: targetName, value: containerId }); - const shouldFail = - (options.failSmokeCleanupTarget === targetName && - (containerId === WORKER_SMOKE_ID || containerId === HEAD_SMOKE_ID)) || - (options.failLegacyBackupRemoval && containerId === LEGACY_HEAD_ID); - const match = [...containers.entries()].find( - ([containerKey, entries]) => - containerKey.startsWith(`${targetName}:`) && - entries.some((entry) => entry.id === containerId), - ); - return shouldFail || !match - ? { status: 1 } - : (() => { - const [containerKey, entries] = match; - const remaining = entries.filter((entry) => entry.id !== containerId); - containers.set(containerKey, remaining); - return { status: 0 }; - })(); - }, - }; - - function seed(targetName: "local" | "peer", container: FakeContainer): void { - const containerKey = key(targetName, container.name); - containers.set(containerKey, [...(containers.get(containerKey) ?? []), container]); - } - - return { - buildRemoteDockerEnv, - captureOptions, - containers, - deps, - getMaxLifecycleLockActive: () => maxLifecycleLockActive, - operations, - rmOptions, - runCalls, - seed, - }; + options, + ); } type LifecycleHarness = ReturnType; @@ -1079,13 +767,12 @@ describe("dual-Station managed vLLM lifecycle", () => { reusedExisting: false, legacyMigration: expect.objectContaining({ legacyContainerId: LEGACY_HEAD_ID }), }); - if (!started.ok || !started.legacyMigration) - throw new Error("expected legacy migration handle"); + const legacyMigration = requireLegacyMigration(started); expect( fake.operations.some(({ kind, value }) => kind === "rm" && value === LEGACY_HEAD_ID), ).toBe(false); await expect( - commitDualStationLegacyMigration(fixturePlan(), started.legacyMigration, fake.deps), + commitDualStationLegacyMigration(fixturePlan(), legacyMigration, fake.deps), ).resolves.toEqual({ ok: true, cleanupWarnings: [] }); const cutoverOrder = fake.operations .filter( @@ -1110,11 +797,10 @@ describe("dual-Station managed vLLM lifecycle", () => { const fake = harness(); seedLegacyHead(fake); const started = await startDualStationManagedVllm(fixturePlan(), START_CONFIG, fake.deps); - if (!started.ok || !started.legacyMigration) - throw new Error("expected legacy migration handle"); + const legacyMigration = requireLegacyMigration(started); await expect( - rollbackDualStationLegacyMigration(fixturePlan(), started.legacyMigration, fake.deps), + rollbackDualStationLegacyMigration(fixturePlan(), legacyMigration, fake.deps), ).resolves.toEqual({ ok: true }); expectRestoredLegacyHead(fake); }); @@ -1171,10 +857,9 @@ describe("dual-Station managed vLLM lifecycle", () => { const fake = harness({ failLegacyBackupRemoval: true }); seedLegacyHead(fake); const started = await startDualStationManagedVllm(fixturePlan(), START_CONFIG, fake.deps); - if (!started.ok || !started.legacyMigration) - throw new Error("expected legacy migration handle"); + const legacyMigration = requireLegacyMigration(started); await expect( - commitDualStationLegacyMigration(fixturePlan(), started.legacyMigration, fake.deps), + commitDualStationLegacyMigration(fixturePlan(), legacyMigration, fake.deps), ).resolves.toMatchObject({ ok: true, cleanupWarnings: [expect.stringContaining("legacy backup")], diff --git a/src/lib/inference/vllm-station-lifecycle-lock.test.ts b/src/lib/inference/vllm-station-lifecycle-lock.test.ts index da3ccc6f8d..93ad9aa0d6 100644 --- a/src/lib/inference/vllm-station-lifecycle-lock.test.ts +++ b/src/lib/inference/vllm-station-lifecycle-lock.test.ts @@ -56,13 +56,25 @@ describe("dual-Station controller UID binding", () => { }); it.each([ - ["group-writable parent", controllerUidStat("directory", { mode: 0o40775 }), null, "1001\n"], - ["non-traversable parent", controllerUidStat("directory", { mode: 0o40700 }), null, "1001\n"], - ["non-root-owned file", controllerUidStat("directory"), { uid: 1001 }, "1001\n"], - ["wrong file mode", controllerUidStat("directory"), { mode: 0o100664 }, "1001\n"], - ["root UID content", controllerUidStat("directory"), null, "0\n"], - ["multiple UID lines", controllerUidStat("directory"), { size: 10 }, "1001\n1002\n"], - ])("rejects an unsafe controller binding: %s", (_case, directory, fileOverride, contents) => { + [ + "group-writable parent", + controllerUidStat("directory", { mode: 0o40775 }), + null, + "1001\n", + [], + ], + [ + "non-traversable parent", + controllerUidStat("directory", { mode: 0o40700 }), + null, + "1001\n", + [], + ], + ["non-root-owned file", controllerUidStat("directory"), { uid: 1001 }, "1001\n", [[19]]], + ["wrong file mode", controllerUidStat("directory"), { mode: 0o100664 }, "1001\n", [[19]]], + ["root UID content", controllerUidStat("directory"), null, "0\n", [[19]]], + ["multiple UID lines", controllerUidStat("directory"), { size: 10 }, "1001\n1002\n", [[19]]], + ])("rejects an unsafe controller binding: %s", (_case, directory, fileOverride, contents, expectedCloseCalls) => { const close = vi.fn(); expect(() => readDualStationControllerUid({ @@ -73,7 +85,7 @@ describe("dual-Station controller UID binding", () => { close, }), ).toThrow(/Dual-Station controller/u); - if ((directory.mode & 0o7777) === 0o755) expect(close).toHaveBeenCalledWith(19); + expect(close.mock.calls).toEqual(expectedCloseCalls); }); it("refuses a direct lock call from an account other than the prepared controller", () => { diff --git a/src/lib/inference/vllm-station-model-staging.test-support.ts b/src/lib/inference/vllm-station-model-staging.test-support.ts new file mode 100644 index 0000000000..f56f615b99 --- /dev/null +++ b/src/lib/inference/vllm-station-model-staging.test-support.ts @@ -0,0 +1,204 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import path from "node:path"; + +import { vi } from "vitest"; + +import { resolveDualStationSimulationFixturePython } from "../../../scripts/simulate-dual-station.mts"; + +import type { + ModelStagingCommandOptions, + ModelStagingCommandResult, +} from "./vllm-station-model-staging"; + +type ModelStagingFixtureCommand = ( + file: string, + args: readonly string[], + options: ModelStagingCommandOptions, +) => Promise; + +function result(stdout = "", status = 0): ModelStagingCommandResult { + return { status, stdout, stderr: "" }; +} + +function runPython( + args: readonly string[], + options: ModelStagingCommandOptions, +): ModelStagingCommandResult { + const completed = spawnSync(resolveDualStationSimulationFixturePython(), [...args], { + encoding: "utf8", + env: options.env, + input: options.input, + timeout: options.timeoutMs, + }); + return { + status: completed.status, + stdout: completed.stdout ?? "", + stderr: completed.stderr ?? "", + error: completed.error?.message, + timedOut: (completed.error as NodeJS.ErrnoException | undefined)?.code === "ETIMEDOUT", + }; +} + +function unexpectedCommand(file: string): never { + throw new Error(`unexpected command: ${file}`); +} + +/** + * Branching in these runners models external command behavior; assertions stay in the test file. + */ +export function createPostAuditMutationRunner(snapshot: string) { + const state = { + configMode: 0, + snapshotMode: 0, + transferSource: "", + transferredConfig: "", + }; + let pythonCall = 0; + let sshCall = 0; + const runCommand = vi.fn(async (file, args, options) => { + if (file === "python3") { + pythonCall += 1; + const audit = runPython(args, options); + if (pythonCall === 2 && audit.status === 0) { + fs.writeFileSync(path.join(snapshot, "config.json"), "changed-after-audit"); + } + return audit; + } + if (file === "ssh") { + sshCall += 1; + return result(sshCall <= 2 ? '{"state":"transfer"}' : '{"state":"ready"}'); + } + if (file === "rsync") { + state.transferSource = String(args.at(-2)).replace(/\/$/, ""); + state.transferredConfig = fs.readFileSync( + path.join(state.transferSource, "config.json"), + "utf8", + ); + state.snapshotMode = fs.statSync(state.transferSource).mode & 0o777; + state.configMode = fs.statSync(path.join(state.transferSource, "config.json")).mode & 0o777; + return result(); + } + return unexpectedCommand(file); + }); + return { runCommand, state }; +} + +export function createBetweenAuditMutationRunner(snapshot: string) { + const state = { materializedSnapshot: "" }; + let pythonCall = 0; + const runCommand = vi.fn(async (file, args, options) => { + if (file === "python3") { + pythonCall += 1; + if (args[3] !== undefined) state.materializedSnapshot = String(args[3]); + const audit = runPython(args, options); + if (pythonCall === 1 && audit.status === 0) { + fs.writeFileSync(path.join(snapshot, "config.json"), "changed-between-audits"); + } + return audit; + } + if (file === "ssh") { + return result(pythonCall === 1 ? '{"state":"transfer"}' : '{"state":"cleaned"}'); + } + return unexpectedCommand(file); + }); + return { runCommand, state }; +} + +export function createPythonOnlyRunner() { + return vi.fn(async (file, args, options) => { + if (file !== "python3") return unexpectedCommand(file); + return runPython(args, options); + }); +} + +export function createManifestPeerPythonRunner(options: { + localManifest: string; + peerHome: string; + peerInputPrefix?: string; +}) { + return vi.fn(async (file, _args, commandOptions) => { + if (file === "python3") return result(options.localManifest); + if (file === "ssh") { + return runPython(["-"], { + ...commandOptions, + env: { ...commandOptions.env, HOME: options.peerHome }, + input: `${options.peerInputPrefix ?? ""}${commandOptions.input ?? ""}`, + }); + } + return unexpectedCommand(file); + }); +} + +export function createPeerIntegrityRunner(options: { localManifest: string; peerHome: string }) { + const state = { stagingPath: "" }; + const runCommand = vi.fn(async (file, args, commandOptions) => { + if (file === "python3") return result(options.localManifest); + if (file === "ssh") { + return runPython(["-"], { + ...commandOptions, + env: { ...commandOptions.env, HOME: options.peerHome }, + }); + } + if (file === "rsync") { + const destination = String(args.at(-1)); + state.stagingPath = destination.slice(destination.indexOf(":") + 1).replace(/\/$/, ""); + fs.mkdirSync(state.stagingPath, { mode: 0o700, recursive: true }); + fs.writeFileSync(path.join(state.stagingPath, "config.json"), "xx"); + return result(); + } + return unexpectedCommand(file); + }); + return { runCommand, state }; +} + +export function createAtomicIdentityReplacementRunner(peerHome: string) { + const state = { materializedSnapshot: "" }; + let sshCall = 0; + const runCommand = vi.fn(async (file, args, options) => { + if (file === "python3") { + if (args[3] !== undefined) state.materializedSnapshot = String(args[3]); + return runPython(args, options); + } + if (file === "ssh") { + sshCall += 1; + const ampleCapacity = `import shutil +class _NemoClawDiskUsage: + free = 1 << 50 +shutil.disk_usage = lambda _path: _NemoClawDiskUsage() +`; + const replaceInstalledIdentity = + sshCall === 3 + ? `import os +_nemoclaw_original_rename = os.rename +def _nemoclaw_replace_after_rename(source, destination): + _nemoclaw_original_rename(source, destination) + _nemoclaw_original_rename(destination, str(destination) + ".nemoclaw-test-original") + os.mkdir(destination, 0o700) +os.rename = _nemoclaw_replace_after_rename +` + : ""; + return runPython(["-"], { + ...options, + env: { ...options.env, HOME: peerHome }, + input: `${ampleCapacity}${replaceInstalledIdentity}${options.input ?? ""}`, + }); + } + if (file === "rsync") { + const source = String(args.at(-2)).replace(/\/$/, ""); + const destination = String(args.at(-1)); + const stagingPath = destination.slice(destination.indexOf(":") + 1).replace(/\/$/, ""); + for (const entry of fs.readdirSync(source)) { + fs.cpSync(path.join(source, entry), path.join(stagingPath, entry), { + recursive: true, + }); + } + return result(); + } + return unexpectedCommand(file); + }); + return { runCommand, state }; +} diff --git a/src/lib/inference/vllm-station-model-staging.test.ts b/src/lib/inference/vllm-station-model-staging.test.ts index b9511d5cef..22e0aad728 100644 --- a/src/lib/inference/vllm-station-model-staging.test.ts +++ b/src/lib/inference/vllm-station-model-staging.test.ts @@ -1,7 +1,6 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { spawnSync } from "node:child_process"; import { createHash } from "node:crypto"; import fs from "node:fs"; import os from "node:os"; @@ -9,14 +8,19 @@ import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { resolveDualStationSimulationFixturePython } from "../../../scripts/simulate-dual-station.mts"; - import { DUAL_STATION_VLLM_RUNTIME, type DualStationVllmPlan } from "./vllm-station-cluster"; import { - type ModelStagingCommandOptions, type ModelStagingCommandResult, stageDualStationModelSnapshot, } from "./vllm-station-model-staging"; +import { + createAtomicIdentityReplacementRunner, + createBetweenAuditMutationRunner, + createManifestPeerPythonRunner, + createPeerIntegrityRunner, + createPostAuditMutationRunner, + createPythonOnlyRunner, +} from "./vllm-station-model-staging.test-support"; import { createDualStationSshBindingFixture, type DualStationSshBindingFixture, @@ -156,25 +160,6 @@ function sufficientStatfs() { return vi.fn().mockResolvedValue({ bavail: 1024n * 1024n * 1024n, bsize: 4096n }); } -function runPython( - args: readonly string[], - options: ModelStagingCommandOptions, -): ModelStagingCommandResult { - const completed = spawnSync(resolveDualStationSimulationFixturePython(), [...args], { - encoding: "utf8", - env: options.env, - input: options.input, - timeout: options.timeoutMs, - }); - return { - status: completed.status, - stdout: completed.stdout ?? "", - stderr: completed.stderr ?? "", - error: completed.error?.message, - timedOut: (completed.error as NodeJS.ErrnoException | undefined)?.code === "ETIMEDOUT", - }; -} - function createLocalSnapshotFixture(): { root: string; home: string; snapshot: string } { const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-model-snapshot-")); const home = path.join(root, "home"); @@ -298,52 +283,19 @@ describe("dual-Station pinned model staging", () => { const fixture = createLocalSnapshotFixture(); const fixturePlan = plan(); fixturePlan.local.home = fixture.home; - let pythonCall = 0; - let sshCall = 0; - let transferSource = ""; - let transferredConfig = ""; - let snapshotMode = 0; - let configMode = 0; - const runCommand = vi.fn( - async ( - file: string, - args: readonly string[], - options: ModelStagingCommandOptions, - ): Promise => { - if (file === "python3") { - pythonCall += 1; - const audit = runPython(args, options); - if (pythonCall === 2 && audit.status === 0) { - fs.writeFileSync(path.join(fixture.snapshot, "config.json"), "changed-after-audit"); - } - return audit; - } - if (file === "ssh") { - sshCall += 1; - return result(sshCall <= 2 ? '{"state":"transfer"}' : '{"state":"ready"}'); - } - if (file === "rsync") { - transferSource = String(args.at(-2)).replace(/\/$/, ""); - transferredConfig = fs.readFileSync(path.join(transferSource, "config.json"), "utf8"); - snapshotMode = fs.statSync(transferSource).mode & 0o777; - configMode = fs.statSync(path.join(transferSource, "config.json")).mode & 0o777; - return result(); - } - throw new Error(`unexpected command: ${file}`); - }, - ); + const { runCommand, state } = createPostAuditMutationRunner(fixture.snapshot); const statfs = sufficientStatfs(); try { await expect( stageDualStationModelSnapshot(fixturePlan, { runCommand, statfs }), ).resolves.toEqual({ ok: true, transferred: true }); - expect(transferredConfig).toBe("{}"); - expect(snapshotMode).toBe(0o500); - expect(configMode).toBe(0o400); - expect(transferSource).not.toBe(fixture.snapshot); - expect(path.dirname(path.dirname(transferSource))).toBe(modelRootForHome(fixture.home)); - expect(fs.existsSync(transferSource)).toBe(false); + expect(state.transferredConfig).toBe("{}"); + expect(state.snapshotMode).toBe(0o500); + expect(state.configMode).toBe(0o400); + expect(state.transferSource).not.toBe(fixture.snapshot); + expect(path.dirname(path.dirname(state.transferSource))).toBe(modelRootForHome(fixture.home)); + expect(fs.existsSync(state.transferSource)).toBe(false); } finally { fs.rmSync(fixture.root, { force: true, recursive: true }); } @@ -353,29 +305,7 @@ describe("dual-Station pinned model staging", () => { const fixture = createLocalSnapshotFixture(); const fixturePlan = plan(); fixturePlan.local.home = fixture.home; - let pythonCall = 0; - let materializedSnapshot = ""; - const runCommand = vi.fn( - async ( - file: string, - args: readonly string[], - options: ModelStagingCommandOptions, - ): Promise => { - if (file === "python3") { - pythonCall += 1; - if (args[3] !== undefined) materializedSnapshot = String(args[3]); - const audit = runPython(args, options); - if (pythonCall === 1 && audit.status === 0) { - fs.writeFileSync(path.join(fixture.snapshot, "config.json"), "changed-between-audits"); - } - return audit; - } - if (file === "ssh") { - return result(pythonCall === 1 ? '{"state":"transfer"}' : '{"state":"cleaned"}'); - } - throw new Error(`unexpected command: ${file}`); - }, - ); + const { runCommand, state } = createBetweenAuditMutationRunner(fixture.snapshot); try { await expect( @@ -393,7 +323,7 @@ describe("dual-Station pinned model staging", () => { "python3", "ssh", ]); - expect(fs.existsSync(materializedSnapshot)).toBe(false); + expect(fs.existsSync(state.materializedSnapshot)).toBe(false); expect(runCommand).not.toHaveBeenCalledWith("rsync", expect.anything(), expect.anything()); } finally { fs.rmSync(fixture.root, { force: true, recursive: true }); @@ -409,16 +339,7 @@ describe("dual-Station pinned model staging", () => { fs.writeFileSync(outside, "{}"); fs.unlinkSync(tokenizer); fs.symlinkSync(outside, tokenizer); - const runCommand = vi.fn( - async ( - file: string, - args: readonly string[], - options: ModelStagingCommandOptions, - ): Promise => { - if (file !== "python3") throw new Error(`unexpected command: ${file}`); - return runPython(args, options); - }, - ); + const runCommand = createPythonOnlyRunner(); try { await expect( @@ -474,22 +395,10 @@ describe("dual-Station pinned model staging", () => { fs.mkdirSync(peerStaging, { mode: 0o700 }); fs.writeFileSync(path.join(peerStaging, "config.json"), "{"); const statfs = sufficientStatfs(); - const runCommand = vi.fn( - async ( - file: string, - _args: readonly string[], - options: ModelStagingCommandOptions, - ): Promise => { - if (file === "python3") return result(manifest()); - if (file === "ssh") { - return runPython(["-"], { - ...options, - env: { ...options.env, HOME: peerHome }, - }); - } - throw new Error(`unexpected command: ${file}`); - }, - ); + const runCommand = createManifestPeerPythonRunner({ + localManifest: manifest(), + peerHome, + }); try { await expect( @@ -534,28 +443,16 @@ describe("dual-Station pinned model staging", () => { const fixturePlan = plan(); fixturePlan.peer.home = peerHome; const statfs = sufficientStatfs(); - const runCommand = vi.fn( - async ( - file: string, - _args: readonly string[], - options: ModelStagingCommandOptions, - ): Promise => { - if (file === "python3") return result(manifest()); - if (file === "ssh") { - const noCapacity = `import shutil + const noCapacity = `import shutil class _NemoClawDiskUsage: free = 0 shutil.disk_usage = lambda _path: _NemoClawDiskUsage() `; - return runPython(["-"], { - ...options, - env: { ...options.env, HOME: peerHome }, - input: `${noCapacity}${options.input ?? ""}`, - }); - } - throw new Error(`unexpected command: ${file}`); - }, - ); + const runCommand = createManifestPeerPythonRunner({ + localManifest: manifest(), + peerHome, + peerInputPrefix: noCapacity, + }); try { await expect( @@ -667,30 +564,10 @@ shutil.disk_usage = lambda _path: _NemoClawDiskUsage() fs.mkdirSync(peerHome, { mode: 0o700 }); const fixturePlan = plan(); fixturePlan.peer.home = peerHome; - let stagingPath = ""; - const runCommand = vi.fn( - async ( - file: string, - args: readonly string[], - options: ModelStagingCommandOptions, - ): Promise => { - if (file === "python3") return result(manifest()); - if (file === "ssh") { - return runPython(["-"], { - ...options, - env: { ...options.env, HOME: peerHome }, - }); - } - if (file === "rsync") { - const destination = String(args.at(-1)); - stagingPath = destination.slice(destination.indexOf(":") + 1).replace(/\/$/, ""); - fs.mkdirSync(stagingPath, { mode: 0o700, recursive: true }); - fs.writeFileSync(path.join(stagingPath, "config.json"), "xx"); - return result(); - } - throw new Error(`unexpected command: ${file}`); - }, - ); + const { runCommand, state } = createPeerIntegrityRunner({ + localManifest: manifest(), + peerHome, + }); const statfs = sufficientStatfs(); try { @@ -710,7 +587,7 @@ shutil.disk_usage = lambda _path: _NemoClawDiskUsage() "ssh", "ssh", ]); - expect(fs.existsSync(stagingPath)).toBe(false); + expect(fs.existsSync(state.stagingPath)).toBe(false); } finally { fs.rmSync(remoteRoot, { force: true, recursive: true }); } @@ -724,56 +601,7 @@ shutil.disk_usage = lambda _path: _NemoClawDiskUsage() const fixturePlan = plan(); fixturePlan.local.home = fixture.home; fixturePlan.peer.home = peerHome; - let sshCall = 0; - let materializedSnapshot = ""; - const runCommand = vi.fn( - async ( - file: string, - args: readonly string[], - options: ModelStagingCommandOptions, - ): Promise => { - if (file === "python3") { - if (args[3] !== undefined) materializedSnapshot = String(args[3]); - return runPython(args, options); - } - if (file === "ssh") { - sshCall += 1; - const ampleCapacity = `import shutil -class _NemoClawDiskUsage: - free = 1 << 50 -shutil.disk_usage = lambda _path: _NemoClawDiskUsage() -`; - const replaceInstalledIdentity = - sshCall === 3 - ? `import os -_nemoclaw_original_rename = os.rename -def _nemoclaw_replace_after_rename(source, destination): - _nemoclaw_original_rename(source, destination) - _nemoclaw_original_rename(destination, str(destination) + ".nemoclaw-test-original") - os.mkdir(destination, 0o700) -os.rename = _nemoclaw_replace_after_rename -` - : ""; - return runPython(["-"], { - ...options, - env: { ...options.env, HOME: peerHome }, - input: `${ampleCapacity}${replaceInstalledIdentity}${options.input ?? ""}`, - }); - } - if (file === "rsync") { - const source = String(args.at(-2)).replace(/\/$/, ""); - const destination = String(args.at(-1)); - const stagingPath = destination.slice(destination.indexOf(":") + 1).replace(/\/$/, ""); - for (const entry of fs.readdirSync(source)) { - fs.cpSync(path.join(source, entry), path.join(stagingPath, entry), { - recursive: true, - }); - } - return result(); - } - throw new Error(`unexpected command: ${file}`); - }, - ); + const { runCommand, state } = createAtomicIdentityReplacementRunner(peerHome); try { await expect( @@ -795,7 +623,7 @@ os.rename = _nemoclaw_replace_after_rename "ssh", "ssh", ]); - expect(fs.existsSync(materializedSnapshot)).toBe(false); + expect(fs.existsSync(state.materializedSnapshot)).toBe(false); expect(fs.existsSync(snapshotForHome(peerHome))).toBe(true); expect(fs.existsSync(`${snapshotForHome(peerHome)}.nemoclaw-test-original`)).toBe(true); } finally { diff --git a/src/lib/inference/vllm.test.ts b/src/lib/inference/vllm.test.ts index 6bd8a7289a..042e8d4c6f 100644 --- a/src/lib/inference/vllm.test.ts +++ b/src/lib/inference/vllm.test.ts @@ -152,13 +152,18 @@ function mockSuccessfulVllmInstall( ["container", () => (ownershipQueue.shift() ?? (() => ""))()], ["ps", () => `${containerName}\n`], ]); + const dockerCaptureByContextAndCommand = new Map string>([ + ["default:container", () => ""], + ]); mocks.dockerCapture.mockImplementation( - (args: readonly string[], options?: { env?: NodeJS.ProcessEnv }) => { - if (args[0] === "container" && options?.env?.DOCKER_CONTEXT === "default") { - return ""; - } - return (dockerCaptureByCommand.get(args[0] ?? "") ?? (() => ""))(); - }, + (args: readonly string[], options?: { env?: NodeJS.ProcessEnv }) => + ( + dockerCaptureByContextAndCommand.get( + `${options?.env?.DOCKER_CONTEXT ?? "ambient"}:${args[0] ?? ""}`, + ) ?? + dockerCaptureByCommand.get(args[0] ?? "") ?? + (() => "") + )(), ); } @@ -1325,13 +1330,22 @@ describe("installVllm model resolution", () => { delete process.env.DOCKER_CONTEXT; const profile = detectVllmProfile({ platform: "spark", type: "nvidia" })!; mockSuccessfulVllmInstall(profile.containerName); - mocks.dockerCapture.mockImplementation( - (args: readonly string[], options?: { env?: NodeJS.ProcessEnv }) => { - if (args[0] === "container" && options?.env?.DOCKER_CONTEXT === "default") { + const dockerCaptureByContextAndCommand = new Map string>([ + [ + "default:container", + () => { throw new Error("canonical Docker unavailable"); - } - return vllmContainerRow(profile.containerName); - }, + }, + ], + ["ambient:container", () => vllmContainerRow(profile.containerName)], + ]); + mocks.dockerCapture.mockImplementation( + (args: readonly string[], options?: { env?: NodeJS.ProcessEnv }) => + ( + dockerCaptureByContextAndCommand.get( + `${options?.env?.DOCKER_CONTEXT ?? "ambient"}:${args[0] ?? ""}`, + ) ?? (() => vllmContainerRow(profile.containerName)) + )(), ); const result = await installVllm(profile, { From 2b94049cce59333b513ef210d135e723d0f562c6 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 16 Jul 2026 21:25:45 -0700 Subject: [PATCH 29/74] test(vllm): keep fixture support out of build Signed-off-by: Aaron Erickson --- .../vllm-station-cluster-lifecycle.test.ts | 14 +++++++------- .../inference/vllm-station-model-staging.test.ts | 13 ++++++------- .../vllm-station-cluster-lifecycle-test-support.ts | 9 ++++++--- .../vllm-station-model-staging-test-support.ts | 4 ++-- 4 files changed, 21 insertions(+), 19 deletions(-) rename src/lib/inference/vllm-station-cluster-lifecycle.test-support.ts => test/support/vllm-station-cluster-lifecycle-test-support.ts (98%) rename src/lib/inference/vllm-station-model-staging.test-support.ts => test/support/vllm-station-model-staging-test-support.ts (98%) diff --git a/src/lib/inference/vllm-station-cluster-lifecycle.test.ts b/src/lib/inference/vllm-station-cluster-lifecycle.test.ts index 6bb89e56f2..0b6b03fea6 100644 --- a/src/lib/inference/vllm-station-cluster-lifecycle.test.ts +++ b/src/lib/inference/vllm-station-cluster-lifecycle.test.ts @@ -5,6 +5,13 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { + createDualStationLifecycleHarness, + dualStationDockerValues as dockerValues, + type LifecycleFakeContainer as FakeContainer, + type LifecycleHarnessOptions as HarnessOptions, + requireLegacyMigration, +} from "../../../test/support/vllm-station-cluster-lifecycle-test-support"; import { DUAL_STATION_VLLM_RUNTIME, type DualStationVllmPlan } from "./vllm-station-cluster"; import { areDualStationManagedVllmContainersRunning, @@ -35,13 +42,6 @@ import { startDualStationManagedVllm, withDualStationManagedVllmLifecycle, } from "./vllm-station-cluster-lifecycle"; -import { - createDualStationLifecycleHarness, - dualStationDockerValues as dockerValues, - type LifecycleFakeContainer as FakeContainer, - type LifecycleHarnessOptions as HarnessOptions, - requireLegacyMigration, -} from "./vllm-station-cluster-lifecycle.test-support"; import { withDualStationVllmLifecycleLock } from "./vllm-station-lifecycle-lock"; import { createDualStationSshBindingFixture, diff --git a/src/lib/inference/vllm-station-model-staging.test.ts b/src/lib/inference/vllm-station-model-staging.test.ts index 22e0aad728..d1b13333a9 100644 --- a/src/lib/inference/vllm-station-model-staging.test.ts +++ b/src/lib/inference/vllm-station-model-staging.test.ts @@ -7,12 +7,6 @@ import os from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; - -import { DUAL_STATION_VLLM_RUNTIME, type DualStationVllmPlan } from "./vllm-station-cluster"; -import { - type ModelStagingCommandResult, - stageDualStationModelSnapshot, -} from "./vllm-station-model-staging"; import { createAtomicIdentityReplacementRunner, createBetweenAuditMutationRunner, @@ -20,7 +14,12 @@ import { createPeerIntegrityRunner, createPostAuditMutationRunner, createPythonOnlyRunner, -} from "./vllm-station-model-staging.test-support"; +} from "../../../test/support/vllm-station-model-staging-test-support"; +import { DUAL_STATION_VLLM_RUNTIME, type DualStationVllmPlan } from "./vllm-station-cluster"; +import { + type ModelStagingCommandResult, + stageDualStationModelSnapshot, +} from "./vllm-station-model-staging"; import { createDualStationSshBindingFixture, type DualStationSshBindingFixture, diff --git a/src/lib/inference/vllm-station-cluster-lifecycle.test-support.ts b/test/support/vllm-station-cluster-lifecycle-test-support.ts similarity index 98% rename from src/lib/inference/vllm-station-cluster-lifecycle.test-support.ts rename to test/support/vllm-station-cluster-lifecycle-test-support.ts index 3be60c2218..ad30ad30eb 100644 --- a/src/lib/inference/vllm-station-cluster-lifecycle.test-support.ts +++ b/test/support/vllm-station-cluster-lifecycle-test-support.ts @@ -3,7 +3,10 @@ import { AsyncLocalStorage } from "node:async_hooks"; import { vi } from "vitest"; -import { DUAL_STATION_VLLM_RUNTIME, type DualStationVllmPlan } from "./vllm-station-cluster"; +import { + DUAL_STATION_VLLM_RUNTIME, + type DualStationVllmPlan, +} from "../../src/lib/inference/vllm-station-cluster"; import { DUAL_STATION_VLLM_API_KEY_FINGERPRINT_LABEL, DUAL_STATION_VLLM_CLUSTER_LABEL, @@ -21,8 +24,8 @@ import { type DualStationLegacyMigration, type DualStationVllmLifecycleDeps, type StartDualStationVllmResult, -} from "./vllm-station-cluster-lifecycle"; -import type { DualStationSshBinding } from "./vllm-station-ssh-binding"; +} from "../../src/lib/inference/vllm-station-cluster-lifecycle"; +import type { DualStationSshBinding } from "../../src/lib/inference/vllm-station-ssh-binding"; export type LifecycleFakeContainer = { id: string; diff --git a/src/lib/inference/vllm-station-model-staging.test-support.ts b/test/support/vllm-station-model-staging-test-support.ts similarity index 98% rename from src/lib/inference/vllm-station-model-staging.test-support.ts rename to test/support/vllm-station-model-staging-test-support.ts index f56f615b99..90849f7b55 100644 --- a/src/lib/inference/vllm-station-model-staging.test-support.ts +++ b/test/support/vllm-station-model-staging-test-support.ts @@ -7,12 +7,12 @@ import path from "node:path"; import { vi } from "vitest"; -import { resolveDualStationSimulationFixturePython } from "../../../scripts/simulate-dual-station.mts"; +import { resolveDualStationSimulationFixturePython } from "../../scripts/simulate-dual-station.mts"; import type { ModelStagingCommandOptions, ModelStagingCommandResult, -} from "./vllm-station-model-staging"; +} from "../../src/lib/inference/vllm-station-model-staging"; type ModelStagingFixtureCommand = ( file: string, From 30411b8dc8e9eb8710edd85993a898367e7f2a64 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 16 Jul 2026 21:35:49 -0700 Subject: [PATCH 30/74] test(vllm): secure Station lock fixture Signed-off-by: Aaron Erickson --- .../vllm-station-lifecycle-lock.test.ts | 36 ++++++++++--------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/src/lib/inference/vllm-station-lifecycle-lock.test.ts b/src/lib/inference/vllm-station-lifecycle-lock.test.ts index 93ad9aa0d6..f654e12cef 100644 --- a/src/lib/inference/vllm-station-lifecycle-lock.test.ts +++ b/src/lib/inference/vllm-station-lifecycle-lock.test.ts @@ -89,24 +89,28 @@ describe("dual-Station controller UID binding", () => { }); it("refuses a direct lock call from an account other than the prepared controller", () => { - const stateDir = path.join(os.tmpdir(), `nemoclaw-station-refused-lock-${String(process.pid)}`); - fs.rmSync(stateDir, { recursive: true, force: true }); + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-station-refused-lock-")); + const stateDir = path.join(root, "state"); const effectiveUid = process.getuid?.() ?? 0; const operation = vi.fn(); - expect(() => - withDualStationVllmLifecycleLock( - operation, - { stateDir, pollIntervalMs: 5, timeoutMs: 250, corruptLockGraceMs: 5 }, - { - readControllerUid: () => (effectiveUid > 0 ? effectiveUid + 1 : 1), - effectiveControllerUid: () => effectiveUid, - }, - ), - ).toThrow( - /requires a non-root effective controller UID|does not match prepared controller UID/u, - ); - expect(operation).not.toHaveBeenCalled(); - expect(fs.existsSync(stateDir)).toBe(false); + try { + expect(() => + withDualStationVllmLifecycleLock( + operation, + { stateDir, pollIntervalMs: 5, timeoutMs: 250, corruptLockGraceMs: 5 }, + { + readControllerUid: () => (effectiveUid > 0 ? effectiveUid + 1 : 1), + effectiveControllerUid: () => effectiveUid, + }, + ), + ).toThrow( + /requires a non-root effective controller UID|does not match prepared controller UID/u, + ); + expect(operation).not.toHaveBeenCalled(); + expect(fs.existsSync(stateDir)).toBe(false); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } }); }); From 6c3fc062dc77ce96cc49574a2db0382dfb8a52fe Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 16 Jul 2026 21:48:04 -0700 Subject: [PATCH 31/74] fix(vllm): harden Station preflight validation Signed-off-by: Aaron Erickson --- .../vllm-station-model-staging.test.ts | 37 +++++++++++++++++++ .../inference/vllm-station-model-staging.ts | 15 ++++++-- src/lib/inference/vllm.test.ts | 22 ++++++----- ...install-station-controller-binding.test.ts | 13 +++++-- 4 files changed, 70 insertions(+), 17 deletions(-) diff --git a/src/lib/inference/vllm-station-model-staging.test.ts b/src/lib/inference/vllm-station-model-staging.test.ts index d1b13333a9..3e2383fea0 100644 --- a/src/lib/inference/vllm-station-model-staging.test.ts +++ b/src/lib/inference/vllm-station-model-staging.test.ts @@ -473,6 +473,43 @@ shutil.disk_usage = lambda _path: _NemoClawDiskUsage() } }); + it("does not credit a full-sized corrupt partial file toward remote capacity", async () => { + const remoteRoot = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-model-peer-corrupt-")); + const peerHome = path.join(remoteRoot, "peer-home"); + fs.mkdirSync(peerHome, { mode: 0o700 }); + const fixturePlan = plan(); + fixturePlan.peer.home = peerHome; + const peerStaging = peerStagingForPlan(fixturePlan); + fs.mkdirSync(peerStaging, { mode: 0o700, recursive: true }); + fs.writeFileSync(path.join(peerStaging, "config.json"), "xx"); + const statfs = sufficientStatfs(); + const headroomOnly = `import shutil +class _NemoClawDiskUsage: + free = 5 * 1024 * 1024 * 1024 +shutil.disk_usage = lambda _path: _NemoClawDiskUsage() +`; + const runCommand = createManifestPeerPythonRunner({ + localManifest: manifest(), + peerHome, + peerInputPrefix: headroomOnly, + }); + + try { + await expect( + stageDualStationModelSnapshot(fixturePlan, { runCommand, statfs }), + ).resolves.toEqual({ + ok: false, + reason: + "peer snapshot preflight failed: peer model cache does not have enough free space for the pinned snapshot", + }); + expect(runCommand.mock.calls.map((call) => call[0])).toEqual(["python3", "ssh"]); + expect(statfs).not.toHaveBeenCalled(); + expect(fs.readFileSync(path.join(peerStaging, "config.json"), "utf8")).toBe("xx"); + } finally { + fs.rmSync(remoteRoot, { force: true, recursive: true }); + } + }); + it("gives concurrent reversed and different local heads disjoint retry-safe staging paths", async () => { const original = plan(); const reversed = plan(); diff --git a/src/lib/inference/vllm-station-model-staging.ts b/src/lib/inference/vllm-station-model-staging.ts index 9a20c17d1d..158d69e52d 100644 --- a/src/lib/inference/vllm-station-model-staging.ts +++ b/src/lib/inference/vllm-station-model-staging.ts @@ -549,7 +549,7 @@ def verify_parent_chain(): def verify_partial(root): if root.is_symlink() or not root.is_dir(): fail("peer staging path is unsafe") - expected_files = {item["path"]: item["size"] for item in EXPECTED["files"]} + expected_files = {item["path"]: item for item in EXPECTED["files"]} expected_directories = set(EXPECTED["directories"]) entry_count = 0 reusable_bytes = 0 @@ -569,14 +569,21 @@ def verify_partial(root): relative = relative_name(root, candidate) mode = candidate.lstat().st_mode size = candidate.stat().st_size + expected = expected_files.get(relative) if ( stat.S_ISLNK(mode) or not stat.S_ISREG(mode) - or relative not in expected_files - or size > expected_files[relative] + or expected is None + or size > expected["size"] ): fail("peer staging path contains an unsafe entry") - reusable_bytes += size + if size == expected["size"]: + digest = hashlib.sha256() + with candidate.open("rb") as handle: + for chunk in iter(lambda: handle.read(8 * 1024 * 1024), b""): + digest.update(chunk) + if digest.hexdigest() == expected["sha256"]: + reusable_bytes += size entry_count += 1 if entry_count > 8192: fail("peer staging path exceeds the safety bound") diff --git a/src/lib/inference/vllm.test.ts b/src/lib/inference/vllm.test.ts index 042e8d4c6f..5785c7c0b0 100644 --- a/src/lib/inference/vllm.test.ts +++ b/src/lib/inference/vllm.test.ts @@ -129,7 +129,7 @@ function vllmContainerRow( function mockSuccessfulVllmInstall( containerName: string, - ownershipResponses: readonly (() => string)[] = [() => "", () => ""], + ownershipResponses: readonly (() => string)[] = [() => "", () => "", () => "", () => ""], ): void { const runCaptureByCommand: Record = { curl: '{"data":[]}', @@ -153,7 +153,7 @@ function mockSuccessfulVllmInstall( ["ps", () => `${containerName}\n`], ]); const dockerCaptureByContextAndCommand = new Map string>([ - ["default:container", () => ""], + ["default:container", () => (ownershipQueue.shift() ?? (() => ""))()], ]); mocks.dockerCapture.mockImplementation( (args: readonly string[], options?: { env?: NodeJS.ProcessEnv }) => @@ -1273,7 +1273,8 @@ describe("installVllm model resolution", () => { it("replaces only an existing managed container by its inspected ID", async () => { const profile = detectVllmProfile({ platform: "spark", type: "nvidia" })!; const managed = vllmContainerRow(profile.containerName); - mockSuccessfulVllmInstall(profile.containerName, [() => managed, () => managed]); + const managedResponses = Array.from({ length: 4 }, () => () => managed); + mockSuccessfulVllmInstall(profile.containerName, managedResponses); mocks.dockerImageInspectFormat.mockReturnValue("sha256:cached-image"); const result = await installVllm(profile, { @@ -1405,9 +1406,8 @@ describe("installVllm model resolution", () => { "false", ])("preserves a same-name container with managed label %j before downloads", async (label) => { const profile = detectVllmProfile({ platform: "spark", type: "nvidia" })!; - mockSuccessfulVllmInstall(profile.containerName, [ - () => vllmContainerRow(profile.containerName, { label }), - ]); + const foreign = vllmContainerRow(profile.containerName, { label }); + mockSuccessfulVllmInstall(profile.containerName, [() => foreign, () => foreign]); const result = await installVllm(profile, { hasImage: true, @@ -1452,10 +1452,12 @@ describe("installVllm model resolution", () => { it("rechecks ownership after downloads and preserves a replacement container", async () => { const profile = detectVllmProfile({ platform: "spark", type: "nvidia" })!; - mockSuccessfulVllmInstall(profile.containerName, [ - () => vllmContainerRow(profile.containerName), - () => vllmContainerRow(profile.containerName, { label: "" }), - ]); + const managed = vllmContainerRow(profile.containerName); + const foreign = vllmContainerRow(profile.containerName, { label: "" }); + mockSuccessfulVllmInstall( + profile.containerName, + [managed, managed, foreign, foreign].map((row) => () => row), + ); mocks.dockerImageInspectFormat.mockReturnValue("sha256:cached-image"); const result = await installVllm(profile, { diff --git a/test/install-station-controller-binding.test.ts b/test/install-station-controller-binding.test.ts index 1b82073b16..a1ab7fb982 100644 --- a/test/install-station-controller-binding.test.ts +++ b/test/install-station-controller-binding.test.ts @@ -109,13 +109,20 @@ config_dir="$HOME/etc/nemoclaw" binding_file="$config_dir/dual-station-controller-uid" mkdir -p "$HOME/etc" controller_uid=1001 +binding_owned=0 preparation_controller_uid() { printf '%s\\n' "$controller_uid"; } ensure_root_directory_safe() { mkdir -p "$1"; } assert_root_directory_safe() { [[ -d "$1" ]]; } -root_regular_file_is_safe() { [[ -f "$1" ]]; } +root_regular_file_is_safe() { ((binding_owned == 1)) && [[ -f "$1" ]]; } root_directory_is_safe_unprivileged() { [[ -d "$1" ]]; } -root_regular_file_is_safe_unprivileged() { [[ -f "$1" ]]; } -sudo() { if [[ "$1" == "chown" ]]; then return 0; fi; "$@"; } +root_regular_file_is_safe_unprivileged() { ((binding_owned == 1)) && [[ -f "$1" ]]; } +sudo() { + if [[ "$1" == "chown" && "$2" == "root:root" && "$3" == "\${config_dir}/.dual-station-controller-uid."* ]]; then + binding_owned=1 + return 0 + fi + "$@" +} ensure_dual_station_controller_uid_binding "$config_dir" "$binding_file" ensure_dual_station_controller_uid_binding "$config_dir" "$binding_file" controller_uid=1002 From 8f210a084b0a497d1a9487826c31069ae8e4037c Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 16 Jul 2026 22:00:30 -0700 Subject: [PATCH 32/74] test(vllm): fail closed on Station fixtures Signed-off-by: Aaron Erickson --- .../vllm-station-model-staging.test.ts | 28 +++++++++++---- src/lib/inference/vllm.test.ts | 18 +++------- test/support/vllm-ownership-test-support.ts | 30 ++++++++++++++++ test/vllm-ownership-test-support.test.ts | 34 +++++++++++++++++++ 4 files changed, 91 insertions(+), 19 deletions(-) create mode 100644 test/support/vllm-ownership-test-support.ts create mode 100644 test/vllm-ownership-test-support.test.ts diff --git a/src/lib/inference/vllm-station-model-staging.test.ts b/src/lib/inference/vllm-station-model-staging.test.ts index 3e2383fea0..98f44310f0 100644 --- a/src/lib/inference/vllm-station-model-staging.test.ts +++ b/src/lib/inference/vllm-station-model-staging.test.ts @@ -481,30 +481,46 @@ shutil.disk_usage = lambda _path: _NemoClawDiskUsage() fixturePlan.peer.home = peerHome; const peerStaging = peerStagingForPlan(fixturePlan); fs.mkdirSync(peerStaging, { mode: 0o700, recursive: true }); - fs.writeFileSync(path.join(peerStaging, "config.json"), "xx"); - const statfs = sufficientStatfs(); + const corruptSnapshotBytes = 64 * 1024 * 1024; + const corruptPartial = path.join(peerStaging, "config.json"); + fs.writeFileSync(corruptPartial, ""); + fs.truncateSync(corruptPartial, corruptSnapshotBytes); + const capacityManifest = JSON.stringify({ + schemaVersion: 1, + files: [ + { + path: "config.json", + size: corruptSnapshotBytes, + sha256: createHash("sha256").update("expected snapshot contents").digest("hex"), + }, + ], + directories: [], + totalBytes: corruptSnapshotBytes, + }); const headroomOnly = `import shutil class _NemoClawDiskUsage: free = 5 * 1024 * 1024 * 1024 shutil.disk_usage = lambda _path: _NemoClawDiskUsage() `; const runCommand = createManifestPeerPythonRunner({ - localManifest: manifest(), + localManifest: capacityManifest, peerHome, peerInputPrefix: headroomOnly, }); try { await expect( - stageDualStationModelSnapshot(fixturePlan, { runCommand, statfs }), + stageDualStationModelSnapshot(fixturePlan, { + runCommand, + statfs: sufficientStatfs(), + }), ).resolves.toEqual({ ok: false, reason: "peer snapshot preflight failed: peer model cache does not have enough free space for the pinned snapshot", }); expect(runCommand.mock.calls.map((call) => call[0])).toEqual(["python3", "ssh"]); - expect(statfs).not.toHaveBeenCalled(); - expect(fs.readFileSync(path.join(peerStaging, "config.json"), "utf8")).toBe("xx"); + expect(fs.statSync(corruptPartial).size).toBe(corruptSnapshotBytes); } finally { fs.rmSync(remoteRoot, { force: true, recursive: true }); } diff --git a/src/lib/inference/vllm.test.ts b/src/lib/inference/vllm.test.ts index 5785c7c0b0..45a9576901 100644 --- a/src/lib/inference/vllm.test.ts +++ b/src/lib/inference/vllm.test.ts @@ -6,6 +6,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { createStrictVllmOwnershipCapture } from "../../../test/support/vllm-ownership-test-support"; const mocks = vi.hoisted(() => ({ dockerCapture: vi.fn(), @@ -147,23 +148,14 @@ function mockSuccessfulVllmInstall( }); mocks.dockerSpawn.mockReturnValue(mockDockerSpawnSuccess()); mocks.dockerRunDetached.mockReturnValue({ status: 0, stdout: "", stderr: "", error: null }); - const ownershipQueue = [...ownershipResponses]; const dockerCaptureByCommand = new Map string>([ - ["container", () => (ownershipQueue.shift() ?? (() => ""))()], ["ps", () => `${containerName}\n`], ]); - const dockerCaptureByContextAndCommand = new Map string>([ - ["default:container", () => (ownershipQueue.shift() ?? (() => ""))()], - ]); + const ambientContext = process.env.DOCKER_CONTEXT ?? "ambient"; mocks.dockerCapture.mockImplementation( - (args: readonly string[], options?: { env?: NodeJS.ProcessEnv }) => - ( - dockerCaptureByContextAndCommand.get( - `${options?.env?.DOCKER_CONTEXT ?? "ambient"}:${args[0] ?? ""}`, - ) ?? - dockerCaptureByCommand.get(args[0] ?? "") ?? - (() => "") - )(), + createStrictVllmOwnershipCapture(ownershipResponses, ambientContext, (command) => + (dockerCaptureByCommand.get(command) ?? (() => ""))(), + ), ); } diff --git a/test/support/vllm-ownership-test-support.ts b/test/support/vllm-ownership-test-support.ts new file mode 100644 index 0000000000..a648f29810 --- /dev/null +++ b/test/support/vllm-ownership-test-support.ts @@ -0,0 +1,30 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +interface DockerCaptureTestOptions { + readonly env?: NodeJS.ProcessEnv; +} + +type OwnershipResponse = () => string; + +/** Route every ownership inspection explicitly and fail closed on fixture drift. */ +export function createStrictVllmOwnershipCapture( + ownershipResponses: readonly OwnershipResponse[], + ambientContext: string, + fallback: (command: string) => string, +): (args: readonly string[], options?: DockerCaptureTestOptions) => string { + const queue = [...ownershipResponses]; + const allowedContexts = new Set(["default", ambientContext]); + return (args, options) => { + const command = args[0] ?? ""; + if (command !== "container") return fallback(command); + + const context = options?.env?.DOCKER_CONTEXT ?? "ambient"; + if (!allowedContexts.has(context)) { + throw new Error(`Unexpected vLLM ownership inspection context: ${context}`); + } + const response = queue.shift(); + if (!response) throw new Error("Unexpected extra vLLM ownership inspection"); + return response(); + }; +} diff --git a/test/vllm-ownership-test-support.test.ts b/test/vllm-ownership-test-support.test.ts new file mode 100644 index 0000000000..02d8e102c2 --- /dev/null +++ b/test/vllm-ownership-test-support.test.ts @@ -0,0 +1,34 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; +import { createStrictVllmOwnershipCapture } from "./support/vllm-ownership-test-support"; + +describe("vLLM ownership test capture", () => { + it("consumes explicit canonical and ambient ownership responses", () => { + const capture = createStrictVllmOwnershipCapture( + [() => "canonical", () => "ambient"], + "builder", + (command) => `fallback:${command}`, + ); + + expect(capture(["container"], { env: { DOCKER_CONTEXT: "default" } })).toBe("canonical"); + expect(capture(["container"], { env: { DOCKER_CONTEXT: "builder" } })).toBe("ambient"); + expect(capture(["ps"])).toBe("fallback:ps"); + }); + + it("preserves explicit absence but rejects ownership response exhaustion", () => { + const capture = createStrictVllmOwnershipCapture([() => ""], "ambient", () => ""); + + expect(capture(["container"], { env: { DOCKER_CONTEXT: "default" } })).toBe(""); + expect(() => capture(["container"])).toThrow("Unexpected extra vLLM ownership inspection"); + }); + + it("rejects ownership inspection through an unregistered Docker context", () => { + const capture = createStrictVllmOwnershipCapture([() => ""], "ambient", () => ""); + + expect(() => capture(["container"], { env: { DOCKER_CONTEXT: "surprise" } })).toThrow( + "Unexpected vLLM ownership inspection context: surprise", + ); + }); +}); From 8fa029ece474c7438d24615770a8d1fb168d1367 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 16 Jul 2026 22:13:48 -0700 Subject: [PATCH 33/74] fix(vllm): reject conflicting Station peer model Signed-off-by: Aaron Erickson --- src/lib/inference/vllm-dual-station.test.ts | 26 +++++++++++++---- src/lib/inference/vllm.ts | 32 +++++++++++++++------ 2 files changed, 45 insertions(+), 13 deletions(-) diff --git a/src/lib/inference/vllm-dual-station.test.ts b/src/lib/inference/vllm-dual-station.test.ts index e3600fb90a..247be5d995 100644 --- a/src/lib/inference/vllm-dual-station.test.ts +++ b/src/lib/inference/vllm-dual-station.test.ts @@ -443,18 +443,34 @@ describe("dual DGX Station vLLM install orchestration", () => { expect(promptFn).not.toHaveBeenCalledWith(expect.stringContaining("Choose model")); }); - it("keeps an explicit model override ahead of peer-driven automatic selection", async () => { - process.env.NEMOCLAW_VLLM_MODEL = "nemotron-3-nano-4b"; - mocks.runCapture.mockReturnValue(""); + it("rejects a configured peer combined with an explicit non-Ultra model", async () => { + process.env.NEMOCLAW_VLLM_MODEL = "deepseek-r1-distill-70b"; + delete process.env.HF_TOKEN; + delete process.env.HUGGING_FACE_HUB_TOKEN; + const beforeInstall = vi.fn(); + const promptFn = vi.fn(); const profile = detectVllmProfile({ platform: "station", type: "nvidia" }); await expect( - installVllm(profile!, { hasImage: true, nonInteractive: true, promptFn: vi.fn() }), + installVllm(profile!, { + hasImage: true, + nonInteractive: true, + promptFn, + beforeInstall, + }), ).resolves.toEqual({ ok: false }); + expect(promptFn).not.toHaveBeenCalled(); + expect(beforeInstall).not.toHaveBeenCalled(); expect(mocks.probeCapability).not.toHaveBeenCalled(); expect(mocks.stageModelSnapshot).not.toHaveBeenCalled(); - expect(errorSpy).toHaveBeenCalledWith(" vLLM install failed: docker not found on PATH"); + expect(mocks.runCapture).not.toHaveBeenCalled(); + expect(mocks.dockerPullWithProgressWatchdog).not.toHaveBeenCalled(); + expect(mocks.dockerSpawn).not.toHaveBeenCalled(); + expect(errorSpy).toHaveBeenCalledWith( + " vLLM install failed: NEMOCLAW_DGX_STATION_PEER requires the DGX Station dual-serving model. " + + "Unset NEMOCLAW_VLLM_MODEL or select nemotron-3-ultra-550b-a55b; the explicit model override remains authoritative.", + ); }); it("stages a missing peer snapshot after download and requires a ready re-probe", async () => { diff --git a/src/lib/inference/vllm.ts b/src/lib/inference/vllm.ts index 37ae43cf03..50d59c7915 100644 --- a/src/lib/inference/vllm.ts +++ b/src/lib/inference/vllm.ts @@ -1153,12 +1153,35 @@ export async function installVllm( let peerModelSnapshot: "ready" | "staging-required" | null = null; const explicitModel = String(process.env.NEMOCLAW_VLLM_MODEL ?? "").trim(); const configuredPeer = String(process.env[NEMOCLAW_DGX_STATION_PEER_ENV] ?? "").trim(); + const ultra = + profile.platform === "station" && configuredPeer + ? VLLM_MODELS.find((candidate) => candidate.envValue === "nemotron-3-ultra-550b-a55b") + : undefined; + + if (profile.platform === "station" && configuredPeer) { + if (!ultra) { + console.error(" vLLM install failed: Nemotron Ultra is missing from the model registry"); + return { ok: false }; + } + const normalizedExplicitModel = explicitModel.toLowerCase(); + if ( + normalizedExplicitModel && + normalizedExplicitModel !== ultra.envValue.toLowerCase() && + normalizedExplicitModel !== ultra.id.toLowerCase() + ) { + console.error( + ` vLLM install failed: ${NEMOCLAW_DGX_STATION_PEER_ENV} requires the DGX Station dual-serving model. ` + + "Unset NEMOCLAW_VLLM_MODEL or select nemotron-3-ultra-550b-a55b; the explicit model override remains authoritative.", + ); + return { ok: false }; + } + } // Model selection lives in `resolveVllmInstallModel` so this entry point // stays focused on the docker side effects. Gated-model access is checked // there before any docker work happens. let resolved: Awaited>; - if (profile.platform === "station" && configuredPeer && !explicitModel) { + if (profile.platform === "station" && configuredPeer && !explicitModel && ultra) { const capability = probeDualStationVllmCapability(); if (capability.kind !== "ready") { const reason = @@ -1168,13 +1191,6 @@ export async function installVllm( console.error(` Dual DGX Station setup unavailable: ${reason}`); return { ok: false }; } - const ultra = VLLM_MODELS.find( - (candidate) => candidate.envValue === "nemotron-3-ultra-550b-a55b", - ); - if (!ultra) { - console.error(" vLLM install failed: Nemotron Ultra is missing from the model registry"); - return { ok: false }; - } resolved = await resolveVllmInstallModel( { ...profile, defaultModel: ultra }, { From 7bc49df8ab4fa3831f19982d9aeaab8866abc3ac Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 16 Jul 2026 22:38:24 -0700 Subject: [PATCH 34/74] docs(vllm): correct dual Station network boundary Signed-off-by: Aaron Erickson --- docs/inference/set-up-vllm.mdx | 13 ++++++++--- src/lib/inference/vllm-models.ts | 5 ++-- .../vllm-station-cluster-lifecycle.test.ts | 9 ++++++++ test/inference-options-docs.test.ts | 23 +++++++++++++++++++ 4 files changed, 45 insertions(+), 5 deletions(-) diff --git a/docs/inference/set-up-vllm.mdx b/docs/inference/set-up-vllm.mdx index 6f7fdefefe..8e869f06c6 100644 --- a/docs/inference/set-up-vllm.mdx +++ b/docs/inference/set-up-vllm.mdx @@ -19,10 +19,11 @@ The vLLM `/v1/responses` endpoint does not run the configured tool-call parser, Local vLLM does not require authentication by default. -NemoClaw needs port `8000` on host loopback for validation and on the OpenShell Docker bridge for sandbox traffic. +Existing-server and single-host managed-vLLM paths need port `8000` on host loopback for validation and on the OpenShell Docker bridge for sandbox traffic. Use a host firewall with default-deny inbound rules. Allow TCP port `8000` only from the OpenShell Docker subnet to its gateway address, keep loopback access, and deny the port on every other interface. Do not expose the port to your LAN or the internet. +The qualified dual-DGX Station path instead binds its bearer-protected `/v1` API to one selected private rail address and uses host networking on both runtime containers; follow the additional isolation guidance in the DGX Station section below. ## Use an Existing Server @@ -162,8 +163,14 @@ Configure the physical rails and SSH host-key and authentication trust before in NemoClaw does not modify rail configuration, enroll SSH trust, or reboot either host automatically. If either host requires a reboot after pinned prerequisite changes, the installer stops with status `10` and resumes only after the manual reboot with owner-only state tied to the printed exact revision and reciprocal identity. The registered Ultra recipe tracks the [official DGX Station deployment guide](https://github.com/NVIDIA-NeMo/Nemotron/blob/287ae845639d2ce998998cb8fd1f70a3fa943c0b/usage-cookbook/Nemotron-3-Ultra/StationDeploymentGuide/README.md) and configures the pinned model revision, CPU offload, `16 GB` of shared memory, memory/stack ulimits, MTP speculative decoding, and the Nemotron reasoning and tool-call parsers. -NemoClaw intentionally keeps its existing bridge-networked managed-inference topology instead of importing the playbook's host-network setting. -The container publishes port `8000` through Docker, so apply the firewall guidance at the top of this page. +The single-Station fallback keeps NemoClaw's bridge-networked managed-inference topology and publishes port `8000` through Docker. +The qualified dual-Station runtime intentionally uses Docker host networking on both containers so NCCL and RDMA can bind the validated direct-attach rails. +The head binds only the selected rank-0 address on a qualified private `/30` rail and requires the generated bearer key for `/v1`; `/health` remains unauthenticated for readiness. +The worker is headless and exposes no API. +Neither dual-Station container publishes a Docker port, so Docker bridge isolation and port-mapping rules do not protect this path. +As compensating controls, both containers run as the probed non-root UID and GID with a read-only root filesystem, all Linux capabilities dropped, `no-new-privileges`, only the selected GPU UUID and exact `uverbs` devices, and a read-only model cache; the worker does not receive the serving key. +Treat both Stations and their direct rails as one trusted runtime boundary. +Allow port `8000` from the OpenShell Docker subnet only to the selected rank-0 rail address, deny it on management and LAN interfaces, and restrict distributed-serving traffic to the reciprocal addresses on the two qualified private rails. DGX Station and its managed-vLLM paths remain Deferred until NemoClaw completes end-to-end validation on physical DGX Station hardware. diff --git a/src/lib/inference/vllm-models.ts b/src/lib/inference/vllm-models.ts index a735e45054..1afbc82f93 100644 --- a/src/lib/inference/vllm-models.ts +++ b/src/lib/inference/vllm-models.ts @@ -258,8 +258,9 @@ export const VLLM_MODELS: readonly VllmModelDef[] = [ imageDownloadSizeBytes: NEMOTRON_ULTRA_STATION_IMAGE.arm64.downloadSizeBytes, modelDownloadSizeBytes: 352_381_245_521, loadTimeoutSec: 3600, - // Keep NemoClaw's bridge-networked local-inference boundary instead of - // importing the playbook's host-network setting. + // The single-host runtime keeps NemoClaw's bridge-networked local- + // inference boundary. The qualified dual-Station lifecycle intentionally + // builds a separate host-networked launch contract for NCCL/RDMA. dockerRunArgs: ["--shm-size", "16g", "--ulimit", "memlock=-1", "--ulimit", "stack=67108864"], }, // The digest-pinned vLLM image already contains the serving package, and diff --git a/src/lib/inference/vllm-station-cluster-lifecycle.test.ts b/src/lib/inference/vllm-station-cluster-lifecycle.test.ts index 0b6b03fea6..34875da5fa 100644 --- a/src/lib/inference/vllm-station-cluster-lifecycle.test.ts +++ b/src/lib/inference/vllm-station-cluster-lifecycle.test.ts @@ -237,6 +237,15 @@ describe("dual-Station managed vLLM run argv", () => { expect(args).toEqual( expect.arrayContaining(["--network", "host", "--shm-size", "16g", "--read-only"]), ); + expect( + args.some( + (arg) => + arg === "-p" || + arg === "--publish" || + arg.startsWith("-p=") || + arg.startsWith("--publish="), + ), + ).toBe(false); expect(dockerValues(args, "--workdir")).toEqual(["/home/vllm"]); expect(dockerValues(args, "--tmpfs")).toEqual([ "/tmp:rw,nosuid,nodev,size=17179869184", diff --git a/test/inference-options-docs.test.ts b/test/inference-options-docs.test.ts index d73dd42f57..679bd2d117 100644 --- a/test/inference-options-docs.test.ts +++ b/test/inference-options-docs.test.ts @@ -288,6 +288,29 @@ describe("inference setup navigation", () => { expect(markdown).toContain("only from the OpenShell Docker subnet to its gateway address"); }); + it("documents the dual-Station host-network trust boundary", () => { + const markdown = fs.readFileSync(vllmSetupPath, "utf8"); + + expect(markdown).toContain( + "Existing-server and single-host managed-vLLM paths need port `8000`", + ); + expect(markdown).toContain( + "qualified dual-Station runtime intentionally uses Docker host networking", + ); + expect(markdown).toContain("Neither dual-Station container publishes a Docker port"); + expect(markdown).toContain("all Linux capabilities dropped"); + expect(markdown).toContain("only the selected GPU UUID and exact `uverbs` devices"); + expect(markdown).toContain("worker does not receive the serving key"); + expect(markdown).toContain("`/health` remains unauthenticated for readiness"); + expect(markdown).toContain("deny it on management and LAN interfaces"); + expect(markdown).not.toContain( + "keeps its existing bridge-networked managed-inference topology instead of importing the playbook's host-network setting", + ); + expect(markdown).not.toContain( + "NemoClaw needs port `8000` on host loopback for validation and on the OpenShell Docker bridge", + ); + }); + it("keeps managed image tags, digests, and compressed sizes in sync with source", () => { const markdown = fs.readFileSync(vllmSetupPath, "utf8"); const entries = [ From fb27f56afddc1185093f5612e4139af0e928efb1 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 16 Jul 2026 22:41:23 -0700 Subject: [PATCH 35/74] test(vllm): pin dual Station port isolation Signed-off-by: Aaron Erickson --- src/lib/inference/vllm-station-cluster-lifecycle.test.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/lib/inference/vllm-station-cluster-lifecycle.test.ts b/src/lib/inference/vllm-station-cluster-lifecycle.test.ts index 34875da5fa..000735dfbf 100644 --- a/src/lib/inference/vllm-station-cluster-lifecycle.test.ts +++ b/src/lib/inference/vllm-station-cluster-lifecycle.test.ts @@ -241,9 +241,12 @@ describe("dual-Station managed vLLM run argv", () => { args.some( (arg) => arg === "-p" || + arg === "-P" || arg === "--publish" || + arg === "--publish-all" || arg.startsWith("-p=") || - arg.startsWith("--publish="), + arg.startsWith("--publish=") || + arg.startsWith("--publish-all="), ), ).toBe(false); expect(dockerValues(args, "--workdir")).toEqual(["/home/vllm"]); From 74130039cb974d013816bcb47662c5b2c45776c5 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 16 Jul 2026 22:44:15 -0700 Subject: [PATCH 36/74] test(vllm): cover attached Docker publish flags Signed-off-by: Aaron Erickson --- src/lib/inference/vllm-station-cluster-lifecycle.test.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/lib/inference/vllm-station-cluster-lifecycle.test.ts b/src/lib/inference/vllm-station-cluster-lifecycle.test.ts index 000735dfbf..ad3ab4f93d 100644 --- a/src/lib/inference/vllm-station-cluster-lifecycle.test.ts +++ b/src/lib/inference/vllm-station-cluster-lifecycle.test.ts @@ -240,11 +240,10 @@ describe("dual-Station managed vLLM run argv", () => { expect( args.some( (arg) => - arg === "-p" || - arg === "-P" || + arg.startsWith("-p") || + arg.startsWith("-P") || arg === "--publish" || arg === "--publish-all" || - arg.startsWith("-p=") || arg.startsWith("--publish=") || arg.startsWith("--publish-all="), ), From be74387a4b7975b12e822802524f06d01f5a80f1 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Fri, 17 Jul 2026 00:32:01 -0700 Subject: [PATCH 37/74] fix(installer): recognize P3830 DGX Station identity Share Station firmware matching across TypeScript probes. Accept bounded P3830 identifiers in both shell entry gates. Make controller UID binding defaults explicit. Signed-off-by: Aaron Erickson --- scripts/install.sh | 27 ++++++++++++++----- scripts/lib/dgx-station-peer.mts | 6 ++--- scripts/prepare-dgx-station-host.sh | 14 ++++++---- src/lib/inference/dgx-station-identity.ts | 11 ++++++++ src/lib/inference/nim.ts | 9 ++----- src/lib/inference/vllm-station-cluster.ts | 19 +++++++------ test/install-express-prompt.test.ts | 16 ++++++++--- test/install-station-host-preparation.test.ts | 5 +++- test/install-station-pair-preparation.test.ts | 11 ++++++++ 9 files changed, 80 insertions(+), 38 deletions(-) create mode 100644 src/lib/inference/dgx-station-identity.ts diff --git a/scripts/install.sh b/scripts/install.sh index d3b0e3ed2d..01ef115866 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -2852,6 +2852,13 @@ is_wsl_host() { # and Windows WSL from the host environment. Echoes "DGX Spark", # "DGX Station", "Windows WSL", or empty. Used to gate the express install # prompt; only platforms with a known sensible default are offered. +is_station_gb300_product() { + local product=${1:-} + [[ "$product" =~ (^|[^[:alnum:]])[Pp]3830([^[:alnum:]]|$) ]] \ + || [[ "$product" == *[Ss][Tt][Aa][Tt][Ii][Oo][Nn]* && + "$product" == *[Gg][Bb]300* ]] +} + detect_express_platform() { local model="" if is_wsl_host; then @@ -2865,14 +2872,20 @@ detect_express_platform() { model="$(tr -d '\0' /dev/null || true)" fi case "$model" in - *DGX*Spark*) printf "DGX Spark" ;; - *Station*GB300*) - if [ -e /etc/dgx-release ] || [ -L /etc/dgx-release ]; then - printf "Unsupported DGX Station OS" - else - printf "DGX Station" - fi + *DGX*Spark*) + printf "DGX Spark" + return ;; + esac + if is_station_gb300_product "$model"; then + if [ -e /etc/dgx-release ] || [ -L /etc/dgx-release ]; then + printf "Unsupported DGX Station OS" + else + printf "DGX Station" + fi + return + fi + case "$model" in *DGX*Station*) printf "Unsupported DGX Station generation" ;; *) ;; esac diff --git a/scripts/lib/dgx-station-peer.mts b/scripts/lib/dgx-station-peer.mts index 186cf3e924..e2470c585b 100644 --- a/scripts/lib/dgx-station-peer.mts +++ b/scripts/lib/dgx-station-peer.mts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import net from "node:net"; +import { isDgxStationGb300Product } from "../../src/lib/inference/dgx-station-identity.ts"; import { stationKnownHostsDigest } from "../../src/lib/inference/vllm-station-ssh-binding.ts"; export const DUAL_STATION_RESUME_SCHEMA_VERSION = 1; @@ -355,10 +356,7 @@ function selectedGb300(host: StationDiscoveryHost, label: string): StationDiscov function assertStationIdentity(host: StationDiscoveryHost, label: string): void { if ( - !( - /DGX[_\s-]+Station/i.test(host.productName) || - (/Station/i.test(host.productName) && /GB300/i.test(host.productName)) - ) || + !isDgxStationGb300Product(host.productName) || !/^(?:aarch64|arm64)$/i.test(host.architecture) ) { throw new Error(`${label} is not a verified arm64 DGX Station GB300`); diff --git a/scripts/prepare-dgx-station-host.sh b/scripts/prepare-dgx-station-host.sh index 777e741cde..c89edf9e22 100755 --- a/scripts/prepare-dgx-station-host.sh +++ b/scripts/prepare-dgx-station-host.sh @@ -114,9 +114,11 @@ is_valid_mode() { esac } -is_station_product() { +is_station_gb300_product() { local product=${1:-} - [[ "$product" == *"Station"* && "$product" == *"GB300"* ]] + [[ "$product" =~ (^|[^[:alnum:]])[Pp]3830([^[:alnum:]]|$) ]] \ + || [[ "$product" == *[Ss][Tt][Aa][Tt][Ii][Oo][Nn]* && + "$product" == *[Gg][Bb]300* ]] } is_preparation_critical_unit() { @@ -291,7 +293,7 @@ check_platform() { || fatal "DGX OS/BaseOS is outside this recipe's validated boundary; use the generic Ubuntu 24.04 ARM64 image" product="$( /ConnectX[- ]?8|\bCX-?8\b/i.test(rail.pciName)); if (cx8Rails.length !== 2) return null; @@ -1281,10 +1274,16 @@ function buildStaticPlan( local: StationHostProbe, peer: StationHostProbe, ): StaticPlan | PlanFailure { - if (!isDgxStationProduct(local.productName) || !/^(?:aarch64|arm64)$/i.test(local.architecture)) { + if ( + !isDgxStationGb300Product(local.productName) || + !/^(?:aarch64|arm64)$/i.test(local.architecture) + ) { return unavailable("local-not-station", "local host is not a verified arm64 DGX Station"); } - if (!isDgxStationProduct(peer.productName) || !/^(?:aarch64|arm64)$/i.test(peer.architecture)) { + if ( + !isDgxStationGb300Product(peer.productName) || + !/^(?:aarch64|arm64)$/i.test(peer.architecture) + ) { return unavailable("peer-not-station", "configured peer is not a verified arm64 DGX Station"); } diff --git a/test/install-express-prompt.test.ts b/test/install-express-prompt.test.ts index b5c0f81262..0994aff588 100644 --- a/test/install-express-prompt.test.ts +++ b/test/install-express-prompt.test.ts @@ -637,15 +637,23 @@ detect_express_platform expect(result.stdout).toBe("Windows WSL"); }); - it("recognizes Station GB300 OEM firmware as DGX Station", () => { - const result = detectExpressPlatformForProductName("Dell Pro Max with Station GB300"); + it.each([ + "P3830", + "NVIDIA P3830 Rev A", + "Dell Pro Max with Station GB300", + ])("recognizes supported Station GB300 firmware as DGX Station: %s", (productName) => { + const result = detectExpressPlatformForProductName(productName); expect(result.status, `${result.stdout}${result.stderr}`).toBe(0); expect(result.stdout).toBe("DGX Station"); }); - it("requires both Station and GB300 for the OEM firmware match", () => { - for (const productName of ["Dell Pro Max with Station GB200", "Dell Pro Max with GB300"]) { + it("rejects partial and unsupported Station product identifiers", () => { + for (const productName of [ + "Acme XP3830 Workstation", + "Dell Pro Max with Station GB200", + "Dell Pro Max with GB300", + ]) { const result = detectExpressPlatformForProductName(productName); expect(result.status, `${result.stdout}${result.stderr}`).toBe(0); diff --git a/test/install-station-host-preparation.test.ts b/test/install-station-host-preparation.test.ts index eaf7f9cc58..c4dba2661b 100644 --- a/test/install-station-host-preparation.test.ts +++ b/test/install-station-host-preparation.test.ts @@ -102,13 +102,16 @@ package_state 'docker-ce=5:29.6.1-1~ubuntu.24.04~noble' }); it.each([ + ["P3830", true], + ["NVIDIA P3830 Rev A", true], ["Dell Pro Max with Station GB300", true], ["NVIDIA DGX Station GB300", true], + ["Acme XP3830 Workstation", false], ["NVIDIA DGX Station A100", false], ["Dell Pro Max with Station GB200", false], ["Dell Pro Max with GB300", false], ])("accepts only Station GB300 DMI: %s", (product, accepted) => { - const { result } = runSourced(STATION_PREPARE, `is_station_product "$PRODUCT"`, { + const { result } = runSourced(STATION_PREPARE, `is_station_gb300_product "$PRODUCT"`, { PRODUCT: product, }); diff --git a/test/install-station-pair-preparation.test.ts b/test/install-station-pair-preparation.test.ts index d2695afa07..7e39f3b1de 100644 --- a/test/install-station-pair-preparation.test.ts +++ b/test/install-station-pair-preparation.test.ts @@ -290,6 +290,17 @@ describe("deterministic dual-DGX Station peer discovery", () => { expect(deriveDiscoveryCandidates(stationHost("local"))).toEqual(["10.10.0.2", "10.10.0.6"]); }); + it.each([ + "DGX-Station", + "P3830", + "NVIDIA Station GB300", + ])("accepts an existing Station firmware product identifier: %s", (productName) => { + const host = stationHost("local"); + host.productName = productName; + + expect(deriveDiscoveryCandidates(host)).toEqual(["10.10.0.2", "10.10.0.6"]); + }); + it("rejects extra rails, duplicate identities, and non-jumbo links", () => { const extraRail = stationHost("local"); extraRail.rails.push({ From 61afaf94d1d8a9e018b75306a1765cfc7e538c28 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Mon, 20 Jul 2026 14:50:20 -0700 Subject: [PATCH 38/74] docs: describe automatic dual Station selection --- docs/resources/prompt-assets/dgx-station.md | 13 ++++++++----- test/starter-prompt-docs.test.ts | 8 ++++++-- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/docs/resources/prompt-assets/dgx-station.md b/docs/resources/prompt-assets/dgx-station.md index ce873ebbd2..e67487221b 100644 --- a/docs/resources/prompt-assets/dgx-station.md +++ b/docs/resources/prompt-assets/dgx-station.md @@ -10,9 +10,9 @@ Use these instructions only after hardware detection confirms DGX Station. Use the selected maintained release's official installer as the authority for Station qualification, host preparation, model selection, consent, and reboot or login resume. Do not run the Station preparation helper separately or reproduce Express by pre-setting provider and model environment variables. -The installer provides these Station Express model choices: +The installer provides these Station Express choices: -1. The ordinary installer defaults to `nemotron-3-ultra-550b-a55b`, served as `nvidia/nemotron-3-ultra-550b-a55b`. +1. The ordinary installer checks for one already-trusted peer at the deterministic counterpart on each of two configured private `/30` ConnectX-8 rails. A qualified pair selects `nemotron-3-ultra-550b-a55b`, served as `nvidia/nemotron-3-ultra-550b-a55b`; otherwise it retains the single-Station `deepseek-v4-flash` default, served as `deepseek-ai/DeepSeek-V4-Flash`. 2. The explicit `--station-deepseek` flag selects `deepseek-v4-flash`, served as `deepseek-ai/DeepSeek-V4-Flash`. Both choices use the same Station detection, host-preparation, consent, suggested-policy, default-sandbox, and exact-revision resume flow. @@ -22,20 +22,22 @@ Before asking for consent, explain all of these boundaries: - On generic Ubuntu, Station Express may install or change the pinned NVIDIA open driver, Docker with Buildx, NVIDIA Container Toolkit, and the reviewed factory `dkms` transition. On qualified factory images, the installer follows its bounded validation and repair path instead of replacing the factory stack. - Official Station preparation may add the trusted local account to the `docker` group, which grants root-equivalent control and is suitable only for a trusted single-user development host. - Official Station preparation may require an operator-controlled reboot and resumes only with the exact accepted NemoClaw revision. +- NemoClaw does not configure the two private rails, scan the network, enroll SSH trust, or reboot either Station. The operator owns physical isolation, firewalling, SSH trust, and manual reboots. +- The dual-Station runtime uses unauthenticated coordination traffic, including TCP port `29501`. Both Stations and every host that can reach either rail must be mutually trusted; a shared `/24` is not equivalent to the required direct private `/30` rails. - Nemotron Ultra Express discloses an approximately `352 GB` model download. DeepSeek Express downloads its pinned vLLM container and model data. Both require enough space on the model-cache filesystem and Docker storage. - DGX Station remains an evaluation path with deferred end-to-end validation on physical hardware, so startup may still fail after readiness checks. -Ask: "Which DGX Station Express model would you like?" +Ask: "Which DGX Station Express option would you like?" Choices: -1. Nemotron 3 Ultra 550B, the ordinary installer default. +1. Automatic pair selection: use Nemotron 3 Ultra 550B only if a trusted two-Station pair qualifies; otherwise use single-Station DeepSeek V4 Flash. 2. DeepSeek V4 Flash, the explicit `--station-deepseek` override. 3. Neither, let me choose the runtime and model normally. If a Station Express model is selected: - Set `NEMOCLAW_AGENT` to the agent already selected in the starter prompt. -- For Nemotron Ultra, run the ordinary installer without `--station-deepseek`. +- For automatic pair selection, run the ordinary installer without `--station-deepseek`. Do not supply a peer unless the user already selected an exact, pretrusted peer; an explicit peer must qualify or setup stops rather than falling back. - For DeepSeek, pass `--station-deepseek` and no other model-selection override. - Do not set `NEMOCLAW_PROVIDER`, `NEMOCLAW_VLLM_MODEL`, `NEMOCLAW_MODEL`, `NEMOCLAW_NON_INTERACTIVE`, `NEMOCLAW_YES`, `NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE`, or `NEMOCLAW_NO_EXPRESS`. - Leave `NEMOCLAW_SANDBOX_NAME`, `NEMOCLAW_POLICY_TIER`, web-search settings, and messaging settings unset so the installer applies its Express defaults. @@ -44,6 +46,7 @@ If a Station Express model is selected: - Let the installer present its third-party-software notice and complete Express summary. Keep each official confirmation visible, wait for the user's response, and do not pre-answer or suppress it. - Do not pass `--force-station-install` unless the installer rejects release metadata on genuine Station GB300 hardware and the user separately chooses the documented temporary override. - Follow the installer's printed exact-revision command after a required reboot or login transition. +- Do not describe Ultra as selected until the installer reports that reciprocal Station, GPU, rail, MAC, route, neighbor, and jumbo-frame checks qualified the pair. If Station Express is declined, continue with the normal provider selection. Offer existing vLLM when a ready server is detected, managed vLLM, supported local Ollama, and every hosted or compatible provider supported by the selected agent. diff --git a/test/starter-prompt-docs.test.ts b/test/starter-prompt-docs.test.ts index 73e9d3d2de..8fdebb38d2 100644 --- a/test/starter-prompt-docs.test.ts +++ b/test/starter-prompt-docs.test.ts @@ -758,7 +758,7 @@ describe("starter prompt docs CTA", () => { expect(stationSource).toContain("`nvidia/nemotron-3-ultra-550b-a55b`"); expect(stationSource).toContain("`deepseek-v4-flash`"); expect(stationSource).toContain("`deepseek-ai/DeepSeek-V4-Flash`"); - expect(stationSource).toContain("Nemotron 3 Ultra 550B, the ordinary installer default"); + expect(stationSource).toContain("Automatic pair selection"); expect(stationSource).toContain( "DeepSeek V4 Flash, the explicit `--station-deepseek` override", ); @@ -768,8 +768,12 @@ describe("starter prompt docs CTA", () => { expect(stationSource).toContain( "Do not run `scripts/prepare-dgx-station-host.sh --check`, `--verify`, or `--apply` separately", ); - expect(stationSource).toContain("For Nemotron Ultra, run the ordinary installer without"); + expect(stationSource).toContain( + "For automatic pair selection, run the ordinary installer without", + ); expect(stationSource).toContain("For DeepSeek, pass `--station-deepseek`"); + expect(stationSource).toContain("TCP port `29501`"); + expect(stationSource).toContain("shared `/24`"); for (const environmentName of [ "NEMOCLAW_PROVIDER", "NEMOCLAW_VLLM_MODEL", From c1ab8c9227003f4e85b41c05dc2a2c42380b1eb5 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Mon, 20 Jul 2026 14:51:51 -0700 Subject: [PATCH 39/74] docs: repin Station prompt asset --- docs/resources/starter-prompt.md | 6 +++--- test/starter-prompt-docs.test.ts | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/resources/starter-prompt.md b/docs/resources/starter-prompt.md index 945b3882e0..8e9107e1ab 100644 --- a/docs/resources/starter-prompt.md +++ b/docs/resources/starter-prompt.md @@ -68,9 +68,9 @@ Use `NEMOCLAW_AGENT=langchain-deepagents-code` or `nemo-deepagents onboard` for After the readiness check, load exactly one matching instruction asset before provider selection: -- Confirmed DGX Spark: [DGX Spark Express instructions](https://raw.githubusercontent.com/NVIDIA/NemoClaw/f814db4f7708ecf9ab054fcee449f11c95076dfd/docs/resources/prompt-assets/dgx-spark.md). -- Confirmed DGX Station: [DGX Station installation instructions](https://raw.githubusercontent.com/NVIDIA/NemoClaw/f814db4f7708ecf9ab054fcee449f11c95076dfd/docs/resources/prompt-assets/dgx-station.md). -- Officially detected Windows WSL: [Windows WSL Express instructions](https://raw.githubusercontent.com/NVIDIA/NemoClaw/f814db4f7708ecf9ab054fcee449f11c95076dfd/docs/resources/prompt-assets/windows-wsl.md). +- Confirmed DGX Spark: [DGX Spark Express instructions](https://raw.githubusercontent.com/NVIDIA/NemoClaw/61afaf94d1d8a9e018b75306a1765cfc7e538c28/docs/resources/prompt-assets/dgx-spark.md). +- Confirmed DGX Station: [DGX Station installation instructions](https://raw.githubusercontent.com/NVIDIA/NemoClaw/61afaf94d1d8a9e018b75306a1765cfc7e538c28/docs/resources/prompt-assets/dgx-station.md). +- Officially detected Windows WSL: [Windows WSL Express instructions](https://raw.githubusercontent.com/NVIDIA/NemoClaw/61afaf94d1d8a9e018b75306a1765cfc7e538c28/docs/resources/prompt-assets/windows-wsl.md). Read the matching raw Markdown file completely and follow it before continuing. Do not load a platform asset for any other computer. diff --git a/test/starter-prompt-docs.test.ts b/test/starter-prompt-docs.test.ts index 8fdebb38d2..f6e3f46beb 100644 --- a/test/starter-prompt-docs.test.ts +++ b/test/starter-prompt-docs.test.ts @@ -30,7 +30,7 @@ const repoRoot = path.resolve(__dirname, ".."); const starterPromptMarkdownSource = path.join(repoRoot, "docs", "resources", "starter-prompt.md"); // CI resolves this exact Git commit and byte-compares its prompt-asset blobs with // the local files. The digests independently assert those same immutable bytes. -const promptAssetRevision = "f814db4f7708ecf9ab054fcee449f11c95076dfd"; +const promptAssetRevision = "61afaf94d1d8a9e018b75306a1765cfc7e538c28"; type PromptAsset = { path: string; @@ -53,7 +53,7 @@ const promptAssets = { ), dgxStation: definePromptAsset( "docs/resources/prompt-assets/dgx-station.md", - "783a3f5973e1471178c78ead1952b904f3888dc603f5e079ad31d82cd2037f9a", // gitleaks:allow -- pinned prompt-asset SHA-256 + "0a72ba41863e387c91e28893a8bccb488fede271c211296463c8eff2faddddbd", // gitleaks:allow -- pinned prompt-asset SHA-256 ), windowsWsl: definePromptAsset( "docs/resources/prompt-assets/windows-wsl.md", @@ -889,7 +889,7 @@ describe("starter prompt docs CTA", () => { ); const stationConfirmationIndex = stationSource.indexOf("Choices:"); const stationDefaultIndex = stationSource.indexOf( - "For Nemotron Ultra, run the ordinary installer without", + "For automatic pair selection, run the ordinary installer without", ); const stationOverrideIndex = stationSource.indexOf("For DeepSeek, pass `--station-deepseek`"); expect(stationDisclosureIndex).toBeGreaterThan(-1); From 62c1eb407091cf227df166777374d707197ffb86 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Mon, 20 Jul 2026 14:56:45 -0700 Subject: [PATCH 40/74] docs: refresh Station prompt asset pin --- docs/resources/starter-prompt.md | 6 +++--- test/starter-prompt-docs.test.ts | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/resources/starter-prompt.md b/docs/resources/starter-prompt.md index 6311fbb9f2..b3678825db 100644 --- a/docs/resources/starter-prompt.md +++ b/docs/resources/starter-prompt.md @@ -68,9 +68,9 @@ Use `NEMOCLAW_AGENT=langchain-deepagents-code` or `nemo-deepagents onboard` for After the readiness check, load exactly one matching instruction asset before provider selection: -- Confirmed DGX Spark: [DGX Spark Express instructions](https://raw.githubusercontent.com/NVIDIA/NemoClaw/c718a78c5794574a98fdd885d94466c3b6794153/docs/resources/prompt-assets/dgx-spark.md). -- Confirmed DGX Station: [DGX Station installation instructions](https://raw.githubusercontent.com/NVIDIA/NemoClaw/c718a78c5794574a98fdd885d94466c3b6794153/docs/resources/prompt-assets/dgx-station.md). -- Officially detected Windows WSL: [Windows WSL Express instructions](https://raw.githubusercontent.com/NVIDIA/NemoClaw/c718a78c5794574a98fdd885d94466c3b6794153/docs/resources/prompt-assets/windows-wsl.md). +- Confirmed DGX Spark: [DGX Spark Express instructions](https://raw.githubusercontent.com/NVIDIA/NemoClaw/1dffcfec19036f907563103db61f294dd7bbd49d/docs/resources/prompt-assets/dgx-spark.md). +- Confirmed DGX Station: [DGX Station installation instructions](https://raw.githubusercontent.com/NVIDIA/NemoClaw/1dffcfec19036f907563103db61f294dd7bbd49d/docs/resources/prompt-assets/dgx-station.md). +- Officially detected Windows WSL: [Windows WSL Express instructions](https://raw.githubusercontent.com/NVIDIA/NemoClaw/1dffcfec19036f907563103db61f294dd7bbd49d/docs/resources/prompt-assets/windows-wsl.md). Read the matching raw Markdown file completely and follow it before continuing. Do not load a platform asset for any other computer. diff --git a/test/starter-prompt-docs.test.ts b/test/starter-prompt-docs.test.ts index 0e0a84ce96..7eb44c3600 100644 --- a/test/starter-prompt-docs.test.ts +++ b/test/starter-prompt-docs.test.ts @@ -30,7 +30,7 @@ const repoRoot = path.resolve(__dirname, ".."); const starterPromptMarkdownSource = path.join(repoRoot, "docs", "resources", "starter-prompt.md"); // CI resolves this Git commit and byte-compares its prompt-asset blobs with // the local files. The digests independently assert those same immutable bytes. -const promptAssetRevision = "c718a78c5794574a98fdd885d94466c3b6794153"; +const promptAssetRevision = "1dffcfec19036f907563103db61f294dd7bbd49d"; type PromptAsset = { path: string; @@ -53,7 +53,7 @@ const promptAssets = { ), dgxStation: definePromptAsset( "docs/resources/prompt-assets/dgx-station.md", - "f0c61cef93da203cecda2424eb1fc5680d56ffd679a518bfc98d26b2e82be381", // gitleaks:allow -- pinned prompt-asset SHA-256 + "49452c76f7f9eed493bcf8ef893b964218005eac6e68cc2a63fd5910f271d85c", // gitleaks:allow -- pinned prompt-asset SHA-256 ), windowsWsl: definePromptAsset( "docs/resources/prompt-assets/windows-wsl.md", From e664b19aa92db74ea719e9d06678b76e86a44f26 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Mon, 20 Jul 2026 15:04:15 -0700 Subject: [PATCH 41/74] test: keep ownership fixture linear --- src/lib/inference/vllm.test.ts | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/lib/inference/vllm.test.ts b/src/lib/inference/vllm.test.ts index dc82200025..d0a820ff8a 100644 --- a/src/lib/inference/vllm.test.ts +++ b/src/lib/inference/vllm.test.ts @@ -153,19 +153,19 @@ function mockSuccessfulVllmInstall( mocks.dockerSpawn.mockReturnValue(mockDockerSpawnSuccess()); mocks.dockerRunDetached.mockReturnValue({ status: 0, stdout: "", stderr: "", error: null }); const ownershipQueue = [...ownershipResponses]; - let canonicalInspection = true; + let ownershipCallIndex = 0; + const ownershipHandlers = [ + (): string => "", + (): string => + ( + ownershipQueue.shift() ?? + (() => { + throw new Error("Unexpected extra ambient vLLM ownership inspection"); + }) + )(), + ]; const dockerCaptureByCommand = new Map string>([ - [ - "container", - () => { - if (canonicalInspection) { - canonicalInspection = false; - return ""; - } - canonicalInspection = true; - return (ownershipQueue.shift() ?? (() => ""))(); - }, - ], + ["container", () => ownershipHandlers[ownershipCallIndex++ % ownershipHandlers.length]()], ["ps", () => `${containerName}\n`], ]); mocks.dockerCapture.mockImplementation((args: readonly string[]) => From f05cec29b9a9b88ea8c4cfb6960f44b1662374df Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Mon, 20 Jul 2026 15:08:36 -0700 Subject: [PATCH 42/74] fix: scope controller binding dependency --- scripts/prepare-dgx-station-host.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/prepare-dgx-station-host.sh b/scripts/prepare-dgx-station-host.sh index a67468a3d7..6f464b9b0b 100755 --- a/scripts/prepare-dgx-station-host.sh +++ b/scripts/prepare-dgx-station-host.sh @@ -2460,7 +2460,6 @@ run_check() { } run_apply() { - require_command cmp require_command sudo acquire_sudo common_preflight From c90f795849f303e591d2f9498e220e2be0dd0370 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Mon, 20 Jul 2026 15:18:15 -0700 Subject: [PATCH 43/74] test: provide Ultra token in storage fixtures --- src/lib/inference/vllm-install-storage.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/inference/vllm-install-storage.test.ts b/src/lib/inference/vllm-install-storage.test.ts index 54e9fa0a62..2499371607 100644 --- a/src/lib/inference/vllm-install-storage.test.ts +++ b/src/lib/inference/vllm-install-storage.test.ts @@ -152,7 +152,7 @@ describe("managed vLLM install storage", () => { stdoutWrite = vi.spyOn(process.stdout, "write").mockImplementation(() => true); delete process.env.NEMOCLAW_VLLM_MODEL; delete process.env.NEMOCLAW_VLLM_EXTRA_ARGS_JSON; - delete process.env.HF_TOKEN; + process.env.HF_TOKEN = "hf_test"; delete process.env.HUGGING_FACE_HUB_TOKEN; }); From 43105d281eba49502dd286eba25aa476c0ef3ed1 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 21 Jul 2026 10:57:16 -0700 Subject: [PATCH 44/74] feat(vllm): align dual Station runtime with playbook Signed-off-by: Aaron Erickson --- docs/get-started/dgx-station-preparation.mdx | 5 +- docs/get-started/quickstart.mdx | 2 +- docs/inference/set-up-vllm.mdx | 7 +- docs/resources/prompt-assets/dgx-station.md | 4 +- scripts/install.sh | 9 +- ...llm-dual-station-simulator.test-support.ts | 10 +- .../vllm-dual-station-simulator.test.ts | 4 +- src/lib/inference/vllm-dual-station.test.ts | 6 +- src/lib/inference/vllm-models.test.ts | 74 +++++------ src/lib/inference/vllm-models.ts | 121 +++++++++++++----- .../vllm-station-cluster-lifecycle.test.ts | 16 ++- .../vllm-station-cluster-lifecycle.ts | 14 +- .../inference/vllm-station-cluster.test.ts | 8 +- src/lib/inference/vllm-station-cluster.ts | 9 +- .../vllm-station-model-staging.test.ts | 1 + .../inference/vllm-station-model-staging.ts | 1 + src/lib/inference/vllm.ts | 9 ++ test/install-station-pair-preparation.test.ts | 8 +- 18 files changed, 194 insertions(+), 114 deletions(-) diff --git a/docs/get-started/dgx-station-preparation.mdx b/docs/get-started/dgx-station-preparation.mdx index 7d502c2cac..7101cee19d 100644 --- a/docs/get-started/dgx-station-preparation.mdx +++ b/docs/get-started/dgx-station-preparation.mdx @@ -131,6 +131,9 @@ For the complete support status and direct GPU policy boundaries, see [Platform ## Prepare a Two-Station Pair +Complete NVIDIA's [two-Station CX8 fabric playbook](https://build.nvidia.com/station/connect-two-stations/instructions) before running NemoClaw pair preparation. +The published [dual-Station Nemotron Ultra playbook](https://build.nvidia.com/station/nemoclaw/dual-nodes) is the runtime recipe tracked by this managed path. + When no peer or model is selected explicitly, Station Express checks for one trusted reciprocal peer at the deterministic counterpart address on each of two configured private `/30` ConnectX-8 rails. It automatically selects the pinned `nemotron-3-ultra-550b-a55b` recipe only after both Stations pass reciprocal identity, GPU, route, neighbor, and jumbo-frame checks. If no trusted pair qualifies, Express retains the single-Station `deepseek-v4-flash` default. @@ -148,7 +151,7 @@ If either Station requires a reboot, the installer stops with status `10` and na Owner-only resume state binds the exact NemoClaw revision, preparation helper, SSH host key, GPU identities, and reciprocal rails; the installer never reboots either host automatically. -The dual-Station runtime uses unauthenticated PyTorch and vLLM coordination traffic, including TCP port `29501`. +The dual-Station runtime uses unauthenticated Ray, NCCL, and vLLM coordination traffic, including the Ray head on TCP port `6379` and Ray's worker traffic. Treat both Stations and every host that can reach either private rail as mutually trusted. The operator owns physical rail isolation, host firewalling, SSH trust, and reboot control. Do not use the pair path on a shared or routed network without separately reviewed isolation evidence. diff --git a/docs/get-started/quickstart.mdx b/docs/get-started/quickstart.mdx index 1bde0e3ed7..2b4adb7455 100644 --- a/docs/get-started/quickstart.mdx +++ b/docs/get-started/quickstart.mdx @@ -121,7 +121,7 @@ Use these details when your first-run path needs more control. DGX Spark, qualifying Station GB300 hosts, and Windows WSL offer an interactive express-install path that chooses a managed local inference option for the platform. Station accepts the generic Ubuntu 24.04 ARM64 image and stock DGX OS `7.2.0`, `7.4.0`, or `7.5.0` when a safe, root-owned `/etc/dgx-release` marker identifies `DGX Server for GALAXY-GB300`. - On a qualifying Station with no explicit model or peer, accepting the prompt checks for one already-trusted peer at the deterministic counterpart of two configured private `/30` ConnectX-8 rails. A qualified pair selects the pinned `nemotron-3-ultra-550b-a55b` managed-vLLM recipe; otherwise Express retains the single-Station `deepseek-v4-flash` default. + On a qualifying Station with no explicit model or peer, accepting the prompt checks for one already-trusted peer at the deterministic counterpart of two configured private `/30` ConnectX-8 rails. A qualified pair selects the pinned `nemotron-3-ultra-550b-a55b` managed-vLLM recipe aligned with NVIDIA's [dual-Station playbook](https://build.nvidia.com/station/nemoclaw/dual-nodes); otherwise Express retains the single-Station `deepseek-v4-flash` default. [Prepare DGX Station to Install NemoClaw](additional-setup/dgx-station-preparation) defines Station qualification, host preparation, trusted-pair and network boundaries, repair limits, and reboot handoff. By default, unknown versions, unsafe release markers, NVIDIA BaseOS images, and other Station generations stop before host preparation; set `NEMOCLAW_PROVIDER` or `NEMOCLAW_NO_EXPRESS=1` to bypass Station host automation. For an explicit temporary override on genuine Station GB300 hardware with unrecognized release metadata, follow the `--force-station-install` safeguards in the Station preparation guide. diff --git a/docs/inference/set-up-vllm.mdx b/docs/inference/set-up-vllm.mdx index 89e3486eca..04e217b39a 100644 --- a/docs/inference/set-up-vllm.mdx +++ b/docs/inference/set-up-vllm.mdx @@ -88,6 +88,7 @@ Managed profiles and model-specific recipes use immutable image digests: - DGX Spark and DGX Station models without a model-specific runtime use the `linux/arm64` digest `sha256:9204569b17ee4c0eff75194b8e6e458479c8aee18953b5ab9cf359fcdac659e2` with a compressed layer size of `9.60 GB` under `nvcr.io/nvidia/vllm:26.05.post1-py3`. - The DGX Station Nemotron 3 Ultra express recipe uses the multi-platform index digest `sha256:0fec7ec5f3e6bc168e54899935fb0557da908a4832a1dbc88e2debcf2f889416` under `vllm/vllm-openai:v0.22.0`; on DGX Station, that index selects a `linux/arm64` manifest with a compressed layer size of `10.67 GB`. It also pins Hugging Face revision `183968f87ae4cedce3039313cac1fd43d112c578` for the approximately `352.38 GB` model download. +- A qualified two-Station Ultra pair instead uses the ARM64 manifest digest `sha256:2cc49b81319f7a66a33dd8bd63a7bfddae079122b33ce51989b6828a1f038c37` under `vllm/vllm-openai:v0.25.1-aarch64`, with `10.24 GB` of compressed layers. - Generic Linux `arm64` hosts use `sha256:447995cbb57e6c7cf792cab95e9852e5f62b5fb6d2f39e030fa4eda9a54eadb4` with a compressed layer size of `9.28 GB` under `nvcr.io/nvidia/vllm:26.03.post1-py3`. - Generic Linux `amd64` hosts use `sha256:7be6c2f676c36059a494fe17254e69ae5c677535ba6191044e5fc8e42a91c773` with a compressed layer size of `8.93 GB` under `nvcr.io/nvidia/vllm:26.03.post1-py3`. @@ -168,15 +169,17 @@ Without terminal access, the installer stops before it installs Docker or build Configure the physical rails and SSH host-key and authentication trust before installation. NemoClaw does not modify rail configuration, enroll SSH trust, or reboot either host automatically. If either host requires a reboot after pinned prerequisite changes, the installer stops with status `10` and resumes only after the manual reboot with owner-only state tied to the printed exact revision and reciprocal identity. -The registered Ultra recipe tracks the [official DGX Station deployment guide](https://github.com/NVIDIA-NeMo/Nemotron/blob/287ae845639d2ce998998cb8fd1f70a3fa943c0b/usage-cookbook/Nemotron-3-Ultra/StationDeploymentGuide/README.md) and configures the pinned model revision, CPU offload, `16 GB` of shared memory, memory/stack ulimits, MTP speculative decoding, and the Nemotron reasoning and tool-call parsers. +The registered two-Station Ultra recipe tracks the [latest NVIDIA dual-Station playbook](https://build.nvidia.com/station/nemoclaw/dual-nodes). +It pins vLLM `0.25.1` and Ray `2.56.0`, uses one tensor-parallel rank per Station with pipeline parallelism across the pair, serves the `nemotron-ultra` alias with a `262144`-token model limit, and retains the Nemotron reasoning and tool-call parsers. The single-Station fallback keeps NemoClaw's bridge-networked managed-inference topology and publishes port `8000` through Docker. The qualified dual-Station runtime intentionally uses Docker host networking on both containers so NCCL and RDMA can bind the validated direct-attach rails. The head binds only the selected rank-0 address on a qualified private `/30` rail and requires the generated bearer key for `/v1`; `/health` remains unauthenticated for readiness. -The worker is headless and exposes no API. +The worker joins the Ray cluster and exposes no vLLM API. Neither dual-Station container publishes a Docker port, so Docker bridge isolation and port-mapping rules do not protect this path. As compensating controls, both containers run as the probed non-root UID and GID with a read-only root filesystem, all Linux capabilities dropped, `no-new-privileges`, only the selected GPU UUID and exact `uverbs` devices, and a read-only model cache; the worker does not receive the serving key. Treat both Stations and their direct rails as one trusted runtime boundary. Allow port `8000` from the OpenShell Docker subnet only to the selected rank-0 rail address, deny it on management and LAN interfaces, and restrict distributed-serving traffic to the reciprocal addresses on the two qualified private rails. +That distributed traffic includes the Ray head on TCP `6379` and Ray worker ports; do not expose either rail to an untrusted or routed network. DGX Station remains Deferred. diff --git a/docs/resources/prompt-assets/dgx-station.md b/docs/resources/prompt-assets/dgx-station.md index 37dd4e64a8..09d2ca173c 100644 --- a/docs/resources/prompt-assets/dgx-station.md +++ b/docs/resources/prompt-assets/dgx-station.md @@ -12,7 +12,7 @@ Do not run the Station preparation helper separately or reproduce Express by pre The installer provides these Station Express choices: -1. The ordinary installer checks for one already-trusted peer at the deterministic counterpart on each of two configured private `/30` ConnectX-8 rails. A qualified pair selects `nemotron-3-ultra-550b-a55b`, served as `nvidia/nemotron-3-ultra-550b-a55b`; otherwise it retains the single-Station `deepseek-v4-flash` default, served as `deepseek-ai/DeepSeek-V4-Flash`. +1. The ordinary installer checks for one already-trusted peer at the deterministic counterpart on each of two configured private `/30` ConnectX-8 rails. A qualified pair selects `nemotron-3-ultra-550b-a55b`, served as `nemotron-ultra` through the vLLM 0.25.1 and Ray 2.56.0 dual-Station recipe; otherwise it retains the single-Station `deepseek-v4-flash` default, served as `deepseek-ai/DeepSeek-V4-Flash`. 2. The explicit `--station-deepseek` flag selects `deepseek-v4-flash`, served as `deepseek-ai/DeepSeek-V4-Flash`. Both choices use the same Station detection, host-preparation, consent, suggested-policy, default-sandbox, and revision resume flow. @@ -23,7 +23,7 @@ Before asking for consent, explain all of these boundaries: - Official Station preparation may add the trusted local account to the `docker` group, which grants root-equivalent control and is suitable only for a trusted single-user development host. - Official Station preparation may require an operator-controlled reboot and resumes only with the accepted NemoClaw revision. - NemoClaw does not configure the two private rails, scan the network, enroll SSH trust, or reboot either Station. The operator owns physical isolation, firewalling, SSH trust, and manual reboots. -- The dual-Station runtime uses unauthenticated coordination traffic, including TCP port `29501`. Both Stations and every host that can reach either rail must be mutually trusted; a shared `/24` is not equivalent to the required direct private `/30` rails. +- The dual-Station runtime uses unauthenticated Ray, NCCL, and vLLM coordination traffic, including the Ray head on TCP port `6379` and Ray worker traffic. Both Stations and every host that can reach either rail must be mutually trusted; a shared `/24` is not equivalent to the required direct private `/30` rails. - Nemotron Ultra Express discloses an approximately `352 GB` model download. DeepSeek Express downloads its pinned vLLM container and model data. Both require enough space on the model-cache filesystem and Docker storage. - DGX Station remains an evaluation path with deferred end-to-end validation on physical hardware, so startup may still fail after readiness checks. diff --git a/scripts/install.sh b/scripts/install.sh index e4132cd5ce..00427c4fca 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -3081,6 +3081,7 @@ validate_express_platform_boundary() { STATION_ULTRA_VLLM_MODEL="nemotron-3-ultra-550b-a55b" STATION_ULTRA_SERVED_MODEL="nvidia/nemotron-3-ultra-550b-a55b" +STATION_ULTRA_DUAL_SERVED_MODEL="nemotron-ultra" STATION_ULTRA_LEGACY_VLLM_IMAGE="vllm/vllm-openai@sha256:0fec7ec5f3e6bc168e54899935fb0557da908a4832a1dbc88e2debcf2f889416" STATION_DEEPSEEK_VLLM_MODEL="deepseek-v4-flash" STATION_DEEPSEEK_SERVED_MODEL="deepseek-ai/DeepSeek-V4-Flash" @@ -3230,7 +3231,7 @@ configure_station_express_model() { "$STATION_ULTRA_VLLM_MODEL" | "nvidia/nvidia-nemotron-3-ultra-550b-a55b-nvfp4") NEMOCLAW_MODEL="$STATION_ULTRA_SERVED_MODEL" ;; - "$STATION_ULTRA_SERVED_MODEL") + "$STATION_ULTRA_SERVED_MODEL" | "$STATION_ULTRA_DUAL_SERVED_MODEL") # The served alias is useful in route output but is not a Hugging Face # repository ID. Normalize it to the registered model slug before the # existing managed-vLLM selector consumes it. @@ -3254,7 +3255,7 @@ station_dual_model_requested() { local selected_model selected_model="$(normalize_station_vllm_model "${NEMOCLAW_VLLM_MODEL:-}")" case "$selected_model" in - "" | "$STATION_ULTRA_VLLM_MODEL" | "$STATION_ULTRA_SERVED_MODEL" | "nvidia/nvidia-nemotron-3-ultra-550b-a55b-nvfp4") + "" | "$STATION_ULTRA_VLLM_MODEL" | "$STATION_ULTRA_SERVED_MODEL" | "$STATION_ULTRA_DUAL_SERVED_MODEL" | "nvidia/nvidia-nemotron-3-ultra-550b-a55b-nvfp4") return 0 ;; *) @@ -4049,13 +4050,13 @@ ensure_station_express_pair() { NEMOCLAW_DGX_STATION_PEER="$peer_target" NEMOCLAW_DGX_STATION_SSH_BINDING="$ssh_binding" export NEMOCLAW_DGX_STATION_PEER NEMOCLAW_DGX_STATION_SSH_BINDING + NEMOCLAW_MODEL="$STATION_ULTRA_DUAL_SERVED_MODEL" + export NEMOCLAW_MODEL if [ "${_STATION_EXPRESS_MODEL_WAS_EXPLICIT:-0}" = "0" ]; then # Leave the vLLM selector unset. The trusted peer is the existing # installVllm signal that performs the authoritative full capability # probe and selects Nemotron Ultra without a second prompt. unset NEMOCLAW_VLLM_MODEL - NEMOCLAW_MODEL="$STATION_ULTRA_SERVED_MODEL" - export NEMOCLAW_MODEL fi ok "Trusted reciprocal dual-DGX Station pair is ready (${peer_target})" ;; diff --git a/src/lib/inference/vllm-dual-station-simulator.test-support.ts b/src/lib/inference/vllm-dual-station-simulator.test-support.ts index 500bd9ce5e..d81d405c00 100644 --- a/src/lib/inference/vllm-dual-station-simulator.test-support.ts +++ b/src/lib/inference/vllm-dual-station-simulator.test-support.ts @@ -383,11 +383,11 @@ export function createDualStationLifecycleSimulator() { throw new Error("both simulated roles must be running before registration"); } if ( - !worker.command.includes("--node-rank 1") || - !worker.command.includes("--headless") || - !worker.command.includes("--master-addr 192.168.240.1") || - !head.command.includes("--node-rank 0") || - head.command.includes("--headless") || + !worker.command.includes("ray start --address=192.168.240.1:6379") || + !worker.command.includes("--node-ip-address=192.168.240.2") || + worker.command.includes("vllm serve") || + !head.command.includes("ray start --head --node-ip-address=192.168.240.1 --port=6379") || + !head.command.includes("--distributed-executor-backend ray") || worker.labels[DUAL_STATION_VLLM_TRANSACTION_LABEL] !== head.labels[DUAL_STATION_VLLM_TRANSACTION_LABEL] ) { diff --git a/src/lib/inference/vllm-dual-station-simulator.test.ts b/src/lib/inference/vllm-dual-station-simulator.test.ts index dd8e9adf0f..e9e5c6702d 100644 --- a/src/lib/inference/vllm-dual-station-simulator.test.ts +++ b/src/lib/inference/vllm-dual-station-simulator.test.ts @@ -97,7 +97,7 @@ describe("connected dual-Station planner-to-lifecycle simulator", () => { role === "head" ? "http://192.168.240.1:8000" : "headless", [DUAL_STATION_VLLM_CLUSTER_LABEL]: dualStationVllmClusterId(plan), [DUAL_STATION_VLLM_GPU_LABEL]: role === "head" ? LOCAL_GPU : PEER_GPU, - [DUAL_STATION_VLLM_LAUNCH_SCHEMA_LABEL]: "1", + [DUAL_STATION_VLLM_LAUNCH_SCHEMA_LABEL]: "2", [DUAL_STATION_VLLM_LAUNCH_CONTRACT_LABEL]: dualStationVllmLaunchContract(plan, role), [DUAL_STATION_VLLM_API_KEY_FINGERPRINT_LABEL]: expectedFingerprint, [DUAL_STATION_VLLM_TRANSACTION_LABEL]: "1".padStart(32, "0"), @@ -158,7 +158,7 @@ describe("connected dual-Station planner-to-lifecycle simulator", () => { role === "head" ? "http://192.168.240.1:8000" : "headless", [DUAL_STATION_VLLM_CLUSTER_LABEL]: dualStationVllmClusterId(plan), [DUAL_STATION_VLLM_GPU_LABEL]: role === "head" ? LOCAL_GPU : PEER_GPU, - [DUAL_STATION_VLLM_LAUNCH_SCHEMA_LABEL]: "1", + [DUAL_STATION_VLLM_LAUNCH_SCHEMA_LABEL]: "2", [DUAL_STATION_VLLM_LAUNCH_CONTRACT_LABEL]: dualStationVllmLaunchContract(plan, role), [DUAL_STATION_VLLM_API_KEY_FINGERPRINT_LABEL]: expectedFingerprint, [DUAL_STATION_VLLM_TRANSACTION_LABEL]: transactionId, diff --git a/src/lib/inference/vllm-dual-station.test.ts b/src/lib/inference/vllm-dual-station.test.ts index 32a61697c2..07f222642e 100644 --- a/src/lib/inference/vllm-dual-station.test.ts +++ b/src/lib/inference/vllm-dual-station.test.ts @@ -267,7 +267,7 @@ beforeEach(() => { ok: true, httpStatus: 200, message: "ok", - body: JSON.stringify({ data: [{ id: "nvidia/nemotron-3-ultra-550b-a55b" }] }), + body: JSON.stringify({ data: [{ id: "nemotron-ultra" }] }), } : { ok: false, httpStatus: 401, message: "unauthorized", body: "" }; }); @@ -642,7 +642,7 @@ describe("dual DGX Station vLLM install orchestration", () => { httpStatus: 200, message: "ok", body: args.at(-1)?.endsWith("/v1/models") - ? JSON.stringify({ data: [{ id: "nvidia/nemotron-3-ultra-550b-a55b" }] }) + ? JSON.stringify({ data: [{ id: "nemotron-ultra" }] }) : "", })); }, @@ -692,7 +692,7 @@ describe("dual DGX Station vLLM install orchestration", () => { httpStatus: 200, message: "ok", body: args.at(-1)?.endsWith("/v1/models") - ? JSON.stringify({ data: [{ id: "nvidia/nemotron-3-ultra-550b-a55b" }] }) + ? JSON.stringify({ data: [{ id: "nemotron-ultra" }] }) : "", })); const profile = detectVllmProfile({ platform: "station", type: "nvidia" }); diff --git a/src/lib/inference/vllm-models.test.ts b/src/lib/inference/vllm-models.test.ts index a2f6eccbed..90b50d69af 100644 --- a/src/lib/inference/vllm-models.test.ts +++ b/src/lib/inference/vllm-models.test.ts @@ -122,65 +122,51 @@ describe("vllm model registry", () => { ); }); - it("builds the pinned two-Station Nemotron Ultra vLLM v0.22 head command", () => { + it("builds the pinned two-Station Nemotron Ultra vLLM v0.25.1 Ray head command", () => { const cmd = buildNemotronUltraDistributedServeCommand({ nodeRank: 0, masterAddr: "192.168.240.1", - masterPort: 29501, + masterPort: 6379, }); - expect(cmd).toBe( - [ - "export VLLM_WEIGHT_OFFLOADING_DISABLE_PIN_MEMORY=1", - "&& export VLLM_NVFP4_GEMM_BACKEND=flashinfer-trtllm", - "&& export VLLM_FLOAT32_MATMUL_PRECISION=high", - "&& vllm serve nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4", - "--tensor-parallel-size 2", - "--pipeline-parallel-size 1", - "--data-parallel-size 1", - "--port 8000", - "--trust-remote-code", - "--nnodes 2", - "--node-rank 0", - "--master-addr 192.168.240.1", - "--master-port 29501", - "--max-model-len 32768", - "--revision 183968f87ae4cedce3039313cac1fd43d112c578", - "--served-model-name nvidia/nemotron-3-ultra-550b-a55b", - "--host 192.168.240.1", - "--cpu-offload-gb 150", - "--cpu-offload-params experts", - "--max-num-seqs 16", - "--gpu-memory-utilization 0.85", - "--reasoning-parser nemotron_v3", - "--enable-auto-tool-choice", - "--tool-call-parser qwen3_coder", - `--default-chat-template-kwargs '{"enable_thinking":true,"force_nonempty_content":true}'`, - ].join(" "), + expect(cmd).toContain('python3 -m pip install --user --no-cache-dir "ray==2.56.0"'); + expect(cmd).toContain( + "ray start --head --node-ip-address=192.168.240.1 --port=6379 --num-gpus=1", ); - expect(cmd).not.toContain("--headless"); + expect(cmd).toContain("vllm serve nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4"); + expect(cmd).toContain("--tensor-parallel-size 1"); + expect(cmd).toContain("--pipeline-parallel-size 2"); + expect(cmd).toContain("--distributed-executor-backend ray"); + expect(cmd).toContain("--kv-cache-dtype fp8"); + expect(cmd).toContain("--max-model-len 262144"); + expect(cmd).toContain("--distributed-timeout-seconds 7200"); + expect(cmd).toContain("--served-model-name nemotron-ultra"); + expect(cmd).toContain("--host 192.168.240.1"); + expect(cmd).toContain("--max-num-seqs 256"); + expect(cmd).toContain("--gpu-memory-utilization 0.9"); expect(cmd).not.toContain("--kernel_config"); expect(cmd).not.toContain("--speculative-config"); + expect(cmd).not.toContain("--cpu-offload"); }); - it("adds headless mode only to the rank-1 Nemotron Ultra command", () => { + it("builds a Ray worker command without starting a second API server", () => { const head = buildNemotronUltraDistributedServeCommand({ nodeRank: 0, masterAddr: "192.168.240.1", - masterPort: 29501, + masterPort: 6379, }); const worker = buildNemotronUltraDistributedServeCommand({ nodeRank: 1, masterAddr: "192.168.240.1", - masterPort: 29501, + masterPort: 6379, + nodeAddr: "192.168.240.2", }); - expect(worker).toBe( - head - .replace("--node-rank 0", "--node-rank 1") - .replace("--master-port 29501", "--master-port 29501 --headless") - .replace("--host 192.168.240.1", "--host 127.0.0.1"), + expect(worker).toContain( + "ray start --address=192.168.240.1:6379 --node-ip-address=192.168.240.2 --num-gpus=1 --block", ); + expect(worker).not.toContain("vllm serve"); + expect(head).toContain("vllm serve"); }); it("rejects invalid two-Station Nemotron Ultra rendezvous options", () => { @@ -188,7 +174,7 @@ describe("vllm model registry", () => { buildNemotronUltraDistributedServeCommand({ nodeRank: 0, masterAddr: "station a", - masterPort: 29501, + masterPort: 6379, }), ).toThrow(/masterAddr/); expect(() => @@ -198,6 +184,14 @@ describe("vllm model registry", () => { masterPort: 70000, }), ).toThrow(/masterPort/); + expect(() => + buildNemotronUltraDistributedServeCommand({ + nodeRank: 1, + masterAddr: "192.168.240.1", + masterPort: 6379, + nodeAddr: "worker.example.com", + }), + ).toThrow(/nodeAddr/); }); it("rejects an unknown NEMOCLAW_VLLM_MODEL with a helpful message", () => { diff --git a/src/lib/inference/vllm-models.ts b/src/lib/inference/vllm-models.ts index d43de3648c..3a48fd0f80 100644 --- a/src/lib/inference/vllm-models.ts +++ b/src/lib/inference/vllm-models.ts @@ -52,6 +52,15 @@ export const NEMOTRON_ULTRA_STATION_IMAGE = { }, } as const; +/** Runtime pinned from the published dual-DGX-Station playbook. */ +export const NEMOTRON_ULTRA_DUAL_STATION_IMAGE = { + tag: "vllm/vllm-openai:v0.25.1-aarch64", + arm64: { + ref: "vllm/vllm-openai@sha256:2cc49b81319f7a66a33dd8bd63a7bfddae079122b33ce51989b6828a1f038c37", + downloadSizeBytes: 10_238_912_364, + }, +} as const; + export interface VllmModelDef { /** Hugging Face model id (also passed to `vllm serve`). */ id: string; @@ -513,18 +522,20 @@ function rewriteVllmArgs( } export interface NemotronUltraDistributedServeOptions { - /** vLLM multiprocessing rank: 0 serves the API and 1 runs headless. */ + /** Ray role: rank 0 owns the API and rank 1 is the worker. */ nodeRank: 0 | 1; - /** Routable rank-0 address used by both nodes for the distributed rendezvous. */ + /** Routable head address used by both nodes for the Ray control plane. */ masterAddr: string; - /** Routable rank-0 port used by both nodes for the distributed rendezvous. */ + /** Routable Ray head port. */ masterPort: number; + /** Routable address of the node running this command. */ + nodeAddr?: string; } /** - * Build one side of the validated two-Station Nemotron Ultra vLLM v0.22 - * direct-tensor-parallel launch. Existing callers keep the single-node - * registry command unless they opt into this rank/address/port API. + * Build one side of the published two-Station Nemotron Ultra vLLM v0.25.1 + * Ray pipeline-parallel launch. Existing callers keep the single-node + * registry command unless they opt into this role/address/port API. */ export function buildNemotronUltraDistributedServeCommand( options: NemotronUltraDistributedServeOptions, @@ -543,6 +554,13 @@ export function buildNemotronUltraDistributedServeCommand( ) { throw new Error("Nemotron Ultra distributed masterPort must be an integer from 1 to 65535."); } + const nodeAddr = (options.nodeAddr ?? masterAddr).trim(); + if (net.isIP(nodeAddr) !== 4) { + throw new Error("Nemotron Ultra distributed nodeAddr must be a canonical IPv4 address."); + } + if (options.nodeRank === 0 && nodeAddr !== masterAddr) { + throw new Error("Nemotron Ultra Ray head nodeAddr must match masterAddr."); + } const model = VLLM_MODELS.find( (candidate) => candidate.envValue === "nemotron-3-ultra-550b-a55b", @@ -553,48 +571,87 @@ export function buildNemotronUltraDistributedServeCommand( ); } - const sharedArgs = rewriteVllmArgs(SHARED_VLLM_ARGS, { "--tensor-parallel-size": "2" }); + const sharedArgs = rewriteVllmArgs(SHARED_VLLM_ARGS, { + "--tensor-parallel-size": "1", + "--pipeline-parallel-size": "2", + }); const modelArgs = rewriteVllmArgs( model.modelArgs, { // Rank 0 binds only to the selected direct-attach RoCE address. This // keeps the API off the management network while still giving the // OpenShell route a host-reachable endpoint. The lifecycle also enables - // vLLM bearer authentication. The headless worker exposes no API. - "--host": options.nodeRank === 0 ? masterAddr : "127.0.0.1", - "--max-num-seqs": "16", - "--gpu-memory-utilization": "0.85", + // vLLM bearer authentication. The Ray worker exposes no API. + "--host": masterAddr, + "--max-num-seqs": "256", + "--gpu-memory-utilization": "0.9", }, - new Set(["--kernel_config", "--speculative-config"]), + new Set([ + "--cpu-offload-gb", + "--cpu-offload-params", + "--kernel_config", + "--speculative-config", + "--default-chat-template-kwargs", + ]), ); - const serveEnv = { - ...model.serveEnv, - VLLM_FLOAT32_MATMUL_PRECISION: "high", - }; - const envPrefix = `${Object.entries(serveEnv) - .map(([key, value]) => `export ${key}=${value}`) - .join(" && ")} && `; const args = [ ...sharedArgs, - "--nnodes", - "2", - "--node-rank", - String(options.nodeRank), - "--master-addr", - shellQuote(masterAddr), - "--master-port", - String(options.masterPort), - ...(options.nodeRank === 1 ? ["--headless"] : []), + "--distributed-executor-backend", + "ray", + "--kv-cache-dtype", + "fp8", "--max-model-len", - "32768", + "262144", + "--distributed-timeout-seconds", + "7200", + "--enable-prefix-caching", "--revision", model.revision, "--served-model-name", - model.servedModelId, + "nemotron-ultra", ...modelArgs, ]; - - return `${envPrefix}vllm serve ${model.id} ${args.join(" ")}`; + const bootstrap = [ + "set -euo pipefail", + 'export PATH="$HOME/.local/bin:$PATH"', + 'python3 -m pip install --user --no-cache-dir "ray==2.56.0"', + ]; + if (options.nodeRank === 1) { + return [ + ...bootstrap, + "python3 - <<'PY'", + "import socket", + "import time", + `address = (${JSON.stringify(masterAddr)}, ${String(options.masterPort)})`, + "deadline = time.time() + 3600", + "while True:", + " try:", + " with socket.create_connection(address, timeout=5):", + " break", + " except OSError:", + " if time.time() >= deadline:", + ' raise TimeoutError("Ray head did not become reachable within 3600 seconds")', + " time.sleep(5)", + "PY", + `exec ray start --address=${shellQuote(`${masterAddr}:${String(options.masterPort)}`)} --node-ip-address=${shellQuote(nodeAddr)} --num-gpus=1 --block`, + ].join("\n"); + } + return [ + ...bootstrap, + `ray start --head --node-ip-address=${shellQuote(masterAddr)} --port=${String(options.masterPort)} --num-gpus=1`, + "python3 - <<'PY'", + "import time", + "import ray", + 'ray.init(address="auto")', + "deadline = time.time() + 3600", + 'while ray.cluster_resources().get("GPU", 0) < 2:', + " if time.time() >= deadline:", + ' raise TimeoutError("peer DGX Station GPU did not join Ray within 3600 seconds")', + " time.sleep(5)", + "print(ray.cluster_resources())", + "PY", + `exec vllm serve ${model.id} ${args.join(" ")}`, + ].join("\n"); } /** diff --git a/src/lib/inference/vllm-station-cluster-lifecycle.test.ts b/src/lib/inference/vllm-station-cluster-lifecycle.test.ts index ad3ab4f93d..57cdb6f6cc 100644 --- a/src/lib/inference/vllm-station-cluster-lifecycle.test.ts +++ b/src/lib/inference/vllm-station-cluster-lifecycle.test.ts @@ -160,7 +160,7 @@ function fakeContainer( [DUAL_STATION_VLLM_CLUSTER_LABEL]: dualStationVllmClusterId(fixturePlan()), [DUAL_STATION_VLLM_GPU_LABEL]: role === "head" ? fixturePlan().local.gpu.uuid : fixturePlan().peer.gpu.uuid, - [DUAL_STATION_VLLM_LAUNCH_SCHEMA_LABEL]: "1", + [DUAL_STATION_VLLM_LAUNCH_SCHEMA_LABEL]: "2", [DUAL_STATION_VLLM_LAUNCH_CONTRACT_LABEL]: dualStationVllmLaunchContract(fixturePlan(), role), [DUAL_STATION_VLLM_API_KEY_FINGERPRINT_LABEL]: API_KEY_FINGERPRINT, [DUAL_STATION_VLLM_TRANSACTION_LABEL]: TRANSACTION_ID, @@ -190,6 +190,8 @@ function seedLegacyHead(fake: LifecycleHarness): void { "local", fakeContainer("head", { id: LEGACY_HEAD_ID, + image: + "vllm/vllm-openai@sha256:0fec7ec5f3e6bc168e54899935fb0557da908a4832a1dbc88e2debcf2f889416", labels: { [DUAL_STATION_VLLM_MANAGED_LABEL]: "true" }, }), ); @@ -301,7 +303,7 @@ describe("dual-Station managed vLLM run argv", () => { expect(args).toContain(plan.runtime.image); expect(dockerValues(args, "--label")).toEqual( expect.arrayContaining([ - `${DUAL_STATION_VLLM_LAUNCH_SCHEMA_LABEL}=1`, + `${DUAL_STATION_VLLM_LAUNCH_SCHEMA_LABEL}=2`, `${DUAL_STATION_VLLM_LAUNCH_CONTRACT_LABEL}=${dualStationVllmLaunchContract(plan, role)}`, `${DUAL_STATION_VLLM_API_KEY_FINGERPRINT_LABEL}=${API_KEY_FINGERPRINT}`, `${DUAL_STATION_VLLM_TRANSACTION_LABEL}=${TRANSACTION_ID}`, @@ -324,9 +326,13 @@ describe("dual-Station managed vLLM run argv", () => { ]); expect(args.filter((arg) => arg === "-lc")).toHaveLength(1); expect(imageIndex).toBe(args.length - 3); - expect(command).toContain(`--node-rank ${role === "head" ? "0" : "1"}`); - expect(command).toContain(role === "head" ? "--host 192.168.240.1" : "--host 127.0.0.1"); - expect(command.includes("--headless")).toBe(role === "worker"); + expect(command).toContain('python3 -m pip install --user --no-cache-dir "ray==2.56.0"'); + expect(command).toContain( + role === "head" + ? "ray start --head --node-ip-address=192.168.240.1 --port=6379 --num-gpus=1" + : "ray start --address=192.168.240.1:6379 --node-ip-address=192.168.240.2 --num-gpus=1 --block", + ); + expect(command.includes("vllm serve")).toBe(role === "head"); }); it.each(["head", "worker"] as const)("builds a bounded, no-pull %s GPU smoke command", (role) => { diff --git a/src/lib/inference/vllm-station-cluster-lifecycle.ts b/src/lib/inference/vllm-station-cluster-lifecycle.ts index 70cfedcafe..49474be2bd 100644 --- a/src/lib/inference/vllm-station-cluster-lifecycle.ts +++ b/src/lib/inference/vllm-station-cluster-lifecycle.ts @@ -34,10 +34,10 @@ export const DUAL_STATION_VLLM_API_KEY_FINGERPRINT_LABEL = "com.nvidia.nemoclaw.vllm-api-key-fingerprint"; export const DUAL_STATION_VLLM_TRANSACTION_LABEL = "com.nvidia.nemoclaw.vllm-transaction"; export const DUAL_STATION_VLLM_GPU_SMOKE_LABEL = "com.nvidia.nemoclaw.gpu-smoke"; -export const DUAL_STATION_VLLM_MASTER_PORT = 29501; +export const DUAL_STATION_VLLM_MASTER_PORT = 6379; const HEAD_API_PORT = 8000; -// The pinned vLLM 0.22 image ships a non-root-ready /home/vllm. Each Station +// The pinned Station vLLM images ship a non-root-ready /home/vllm. Each Station // mounts an owner-only tmpfs there and runs as the probed model-cache owner. const VLLM_RUNTIME_HOME = "/home/vllm"; const HF_CACHE_CONTAINER_DIR = `${VLLM_RUNTIME_HOME}/.cache/huggingface`; @@ -58,7 +58,7 @@ const SAFE_GPU_UUID_PATTERN = /^GPU-[A-Za-z0-9-]{8,123}$/; const GPU_SMOKE_NONCE_PATTERN = /^[a-f0-9]{32}$/; const TRANSACTION_ID_PATTERN = /^[a-f0-9]{32}$/; const GPU_SMOKE_CONTAINER_PREFIX = "nemoclaw-vllm-gpu-smoke"; -const DUAL_STATION_VLLM_LAUNCH_SCHEMA = "1"; +const DUAL_STATION_VLLM_LAUNCH_SCHEMA = "2"; const VLLM_FINGERPRINT_CONTEXT = "nemoclaw-dual-station-vllm-api-key\0"; // Compatibility bridge for schema-less single-Station Ultra containers from // the v0.0.86 rollback window before dual launch schema 1. @@ -234,6 +234,7 @@ function assertSafePlan(plan: DualStationVllmPlan): void { plan.runtime.modelRevision !== DUAL_STATION_VLLM_RUNTIME.modelRevision || plan.runtime.servedModelId !== DUAL_STATION_VLLM_RUNTIME.servedModelId || plan.runtime.tensorParallelSize !== DUAL_STATION_VLLM_RUNTIME.tensorParallelSize || + plan.runtime.pipelineParallelSize !== DUAL_STATION_VLLM_RUNTIME.pipelineParallelSize || plan.runtime.nodeCount !== DUAL_STATION_VLLM_RUNTIME.nodeCount ) { throw new Error("Dual-Station vLLM requires the exact pinned runtime contract."); @@ -388,6 +389,7 @@ function buildDualStationVllmBaseRunArgs( const clusterId = clusterIdForPlan(plan); const args = [ "--pull=never", + "--init", "--restart", "unless-stopped", "--network", @@ -441,8 +443,12 @@ function buildDualStationVllmBaseRunArgs( appendEnv(args, "HF_HUB_OFFLINE", "1"); appendEnv(args, "TRANSFORMERS_OFFLINE", "1"); appendEnv(args, "VLLM_HOST_IP", endpoints[0].address); + appendEnv(args, "VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS", role === "head" ? "7200" : "3600"); + appendEnv(args, "VLLM_ALLOW_LONG_MAX_MODEL_LEN", "1"); appendEnv(args, "NCCL_IB_HCA", endpoints.map((item) => item.rdmaDevice).join(",")); appendEnv(args, "NCCL_IB_DISABLE", "0"); + appendEnv(args, "NCCL_IB_ADDR_FAMILY", "AF_INET"); + appendEnv(args, "NCCL_IB_ROCE_VERSION_NUM", "2"); appendEnv(args, "NCCL_IB_GID_INDEX", String(plan.roceGidIndex)); appendEnv(args, "NCCL_IB_TC", "106"); appendEnv(args, "NCCL_IB_QPS_PER_CONNECTION", "4"); @@ -474,6 +480,7 @@ function buildDualStationVllmBaseRunArgs( nodeRank: role === "head" ? 0 : 1, masterAddr: plan.masterAddress, masterPort: DUAL_STATION_VLLM_MASTER_PORT, + nodeAddr: endpoints[0].address, }), ); return args; @@ -704,7 +711,6 @@ function inspectManagedContainer( if ( spec.role === "head" && name === spec.name && - image === spec.image && image === LEGACY_SINGLE_STATION_MIGRATION_IMAGE && (state === "running" || (options.allowStoppedLegacy && state === "exited")) && managed === "true" && diff --git a/src/lib/inference/vllm-station-cluster.test.ts b/src/lib/inference/vllm-station-cluster.test.ts index fa929b94db..f0eef48228 100644 --- a/src/lib/inference/vllm-station-cluster.test.ts +++ b/src/lib/inference/vllm-station-cluster.test.ts @@ -379,7 +379,7 @@ describe("probeDualStationVllmCapability", () => { ); }); - it("returns a deterministic two-rail TP2 plan and permits one auxiliary non-GB300 GPU", () => { + it("returns a deterministic two-rail Ray PP2 plan and permits one auxiliary non-GB300 GPU", () => { const deps = fixtureDeps(); const result = runWith(deps); @@ -394,9 +394,11 @@ describe("probeDualStationVllmCapability", () => { }, runtime: { image: - "vllm/vllm-openai@sha256:0fec7ec5f3e6bc168e54899935fb0557da908a4832a1dbc88e2debcf2f889416", + "vllm/vllm-openai@sha256:2cc49b81319f7a66a33dd8bd63a7bfddae079122b33ce51989b6828a1f038c37", modelRevision: "183968f87ae4cedce3039313cac1fd43d112c578", - tensorParallelSize: 2, + servedModelId: "nemotron-ultra", + tensorParallelSize: 1, + pipelineParallelSize: 2, nodeCount: 2, }, local: { diff --git a/src/lib/inference/vllm-station-cluster.ts b/src/lib/inference/vllm-station-cluster.ts index b8042ef2e7..a13711e608 100644 --- a/src/lib/inference/vllm-station-cluster.ts +++ b/src/lib/inference/vllm-station-cluster.ts @@ -8,7 +8,7 @@ import path from "node:path"; import { buildSubprocessEnv } from "../subprocess-env"; import { isDgxStationGb300Product } from "./dgx-station-identity"; import { buildVllmSshTransportEnv } from "./vllm-docker-env"; -import { NEMOTRON_ULTRA_STATION_IMAGE, VLLM_MODELS } from "./vllm-models"; +import { NEMOTRON_ULTRA_DUAL_STATION_IMAGE, VLLM_MODELS } from "./vllm-models"; import { type DualStationSshBinding, dualStationPinnedSshArgs, @@ -44,11 +44,12 @@ if (!ultraModel?.revision) { } export const DUAL_STATION_VLLM_RUNTIME = Object.freeze({ - image: NEMOTRON_ULTRA_STATION_IMAGE.arm64.ref, + image: NEMOTRON_ULTRA_DUAL_STATION_IMAGE.arm64.ref, modelId: ultraModel.id, modelRevision: ultraModel.revision, - servedModelId: ultraModel.servedModelId ?? ultraModel.id, - tensorParallelSize: 2 as const, + servedModelId: "nemotron-ultra", + tensorParallelSize: 1 as const, + pipelineParallelSize: 2 as const, nodeCount: 2 as const, }); diff --git a/src/lib/inference/vllm-station-model-staging.test.ts b/src/lib/inference/vllm-station-model-staging.test.ts index b85d21bb44..5b80725a02 100644 --- a/src/lib/inference/vllm-station-model-staging.test.ts +++ b/src/lib/inference/vllm-station-model-staging.test.ts @@ -144,6 +144,7 @@ function peerStagingForPlan(value: DualStationVllmPlan): string { DUAL_STATION_VLLM_RUNTIME.modelRevision, DUAL_STATION_VLLM_RUNTIME.servedModelId, String(DUAL_STATION_VLLM_RUNTIME.tensorParallelSize), + String(DUAL_STATION_VLLM_RUNTIME.pipelineParallelSize), String(DUAL_STATION_VLLM_RUNTIME.nodeCount), value.local.gpu.uuid, value.peer.gpu.uuid, diff --git a/src/lib/inference/vllm-station-model-staging.ts b/src/lib/inference/vllm-station-model-staging.ts index 158d69e52d..1992e95263 100644 --- a/src/lib/inference/vllm-station-model-staging.ts +++ b/src/lib/inference/vllm-station-model-staging.ts @@ -170,6 +170,7 @@ function stagingTransactionId(plan: DualStationVllmPlan): string { DUAL_STATION_VLLM_RUNTIME.modelRevision, DUAL_STATION_VLLM_RUNTIME.servedModelId, String(DUAL_STATION_VLLM_RUNTIME.tensorParallelSize), + String(DUAL_STATION_VLLM_RUNTIME.pipelineParallelSize), String(DUAL_STATION_VLLM_RUNTIME.nodeCount), plan.local.gpu.uuid, plan.peer.gpu.uuid, diff --git a/src/lib/inference/vllm.ts b/src/lib/inference/vllm.ts index 3fa935de14..a9eb313a49 100644 --- a/src/lib/inference/vllm.ts +++ b/src/lib/inference/vllm.ts @@ -37,6 +37,7 @@ import { } from "./vllm-docker-env"; import { buildVllmServeCommand, + NEMOTRON_ULTRA_DUAL_STATION_IMAGE, NEMOTRON_ULTRA_STATION_IMAGE, parseVllmExtraServeArgs, VLLM_EXTRA_ARGS_ENV, @@ -1451,6 +1452,14 @@ async function runVllmInstall( } } if (dualStationPlan) { + servedModelId = dualStationPlan.runtime.servedModelId; + runtimeProfile = { + ...runtimeProfile, + image: dualStationPlan.runtime.image, + imageDownloadSizeBytes: NEMOTRON_ULTRA_DUAL_STATION_IMAGE.arm64.downloadSizeBytes, + imageUnpackedSizeBytes: undefined, + loadTimeoutSec: 7200, + }; if (VLLM_PORT !== 8000) { console.error( " Dual DGX Station setup requires the default vLLM port 8000; unset NEMOCLAW_VLLM_PORT and retry.", diff --git a/test/install-station-pair-preparation.test.ts b/test/install-station-pair-preparation.test.ts index fbe75ee021..15ddc771bf 100644 --- a/test/install-station-pair-preparation.test.ts +++ b/test/install-station-pair-preparation.test.ts @@ -1215,9 +1215,7 @@ printf 'RESULT peer=%s model=%s selector=%s binding=%s\n' "\${NEMOCLAW_DGX_STATI ); try { expect(result.status, output).toBe(0); - expect(output).toContain( - "RESULT peer=10.10.0.2 model=nvidia/nemotron-3-ultra-550b-a55b selector=", - ); + expect(output).toContain("RESULT peer=10.10.0.2 model=nemotron-ultra selector="); const expectedToken = JSON.parse(coordinatorResult("ready")).sshBinding; expect(output).toContain(`binding=${expectedToken}`); const args = fs.readFileSync(argsFile, "utf8"); @@ -1263,9 +1261,7 @@ printf 'RESULT peer=%s model=%s binding=%s\n' "$NEMOCLAW_DGX_STATION_PEER" "$NEM ); try { expect(result.status, output).toBe(0); - expect(output).toContain( - `RESULT peer=${explicitPeer} model=nvidia/nemotron-3-ultra-550b-a55b`, - ); + expect(output).toContain(`RESULT peer=${explicitPeer} model=nemotron-ultra`); expect(output).toContain(`binding=${JSON.parse(coordinatorResult("ready")).sshBinding}`); const args = fs.readFileSync(argsFile, "utf8"); expect(args).toContain(`--explicit-peer ${explicitPeer}`); From 65bc93c76050abacbcffdfdce95b5b7475b5bf30 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 21 Jul 2026 15:31:10 -0700 Subject: [PATCH 45/74] test(docs): refresh merged Station prompt pin Signed-off-by: Aaron Erickson --- docs/resources/starter-prompt.md | 6 +++--- test/starter-prompt-docs.test.ts | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/resources/starter-prompt.md b/docs/resources/starter-prompt.md index 9965d363c2..6473e5ff62 100644 --- a/docs/resources/starter-prompt.md +++ b/docs/resources/starter-prompt.md @@ -79,9 +79,9 @@ Set `NEMOCLAW_AGENT=langchain-deepagents-code` for Deep Agents, or use `nemo-dee After the readiness check, load exactly one matching instruction asset before provider selection: -- Confirmed DGX Spark: [DGX Spark Express instructions](https://raw.githubusercontent.com/NVIDIA/NemoClaw/ebf29b024c7a24e69ab8386b68e426d31a4c6821/docs/resources/prompt-assets/dgx-spark.md). -- Confirmed DGX Station: [DGX Station installation instructions](https://raw.githubusercontent.com/NVIDIA/NemoClaw/ebf29b024c7a24e69ab8386b68e426d31a4c6821/docs/resources/prompt-assets/dgx-station.md). -- Officially detected Windows WSL: [Windows WSL Express instructions](https://raw.githubusercontent.com/NVIDIA/NemoClaw/ebf29b024c7a24e69ab8386b68e426d31a4c6821/docs/resources/prompt-assets/windows-wsl.md). +- Confirmed DGX Spark: [DGX Spark Express instructions](https://raw.githubusercontent.com/NVIDIA/NemoClaw/962f80ced8918e823ab5355f5056d729ce9232c1/docs/resources/prompt-assets/dgx-spark.md). +- Confirmed DGX Station: [DGX Station installation instructions](https://raw.githubusercontent.com/NVIDIA/NemoClaw/962f80ced8918e823ab5355f5056d729ce9232c1/docs/resources/prompt-assets/dgx-station.md). +- Officially detected Windows WSL: [Windows WSL Express instructions](https://raw.githubusercontent.com/NVIDIA/NemoClaw/962f80ced8918e823ab5355f5056d729ce9232c1/docs/resources/prompt-assets/windows-wsl.md). Read the matching raw Markdown file completely and follow it before continuing. Do not load a platform asset for any other computer. diff --git a/test/starter-prompt-docs.test.ts b/test/starter-prompt-docs.test.ts index 832e4fa841..a8e1739c75 100644 --- a/test/starter-prompt-docs.test.ts +++ b/test/starter-prompt-docs.test.ts @@ -30,7 +30,7 @@ const repoRoot = path.resolve(__dirname, ".."); const starterPromptMarkdownSource = path.join(repoRoot, "docs", "resources", "starter-prompt.md"); // CI resolves this Git commit and byte-compares its prompt-asset blobs with // the local files. The digests independently assert those same immutable bytes. -const promptAssetRevision = "ebf29b024c7a24e69ab8386b68e426d31a4c6821"; +const promptAssetRevision = "962f80ced8918e823ab5355f5056d729ce9232c1"; type PromptAsset = { path: string; @@ -53,7 +53,7 @@ const promptAssets = { ), dgxStation: definePromptAsset( "docs/resources/prompt-assets/dgx-station.md", - "9f506ece27dcda3cf85735d7c6a80846a7727696b825cf8ca161334ac6925c1f", // gitleaks:allow -- pinned prompt-asset SHA-256 + "eed210be783c39080b38c54d4aa158ea825333c3d92b8dcc2244dae8a633554a", // gitleaks:allow -- pinned prompt-asset SHA-256 ), windowsWsl: definePromptAsset( "docs/resources/prompt-assets/windows-wsl.md", @@ -817,7 +817,7 @@ describe("starter prompt docs CTA", () => { expect(sparkSource).toContain("nvidia/Qwen3.6-35B-A3B-NVFP4"); expect(sparkSource).toContain("Leave `NEMOCLAW_VLLM_MODEL` and `NEMOCLAW_MODEL` unset"); expect(stationSource).toContain("`nemotron-3-ultra-550b-a55b`"); - expect(stationSource).toContain("`nvidia/nemotron-3-ultra-550b-a55b`"); + expect(stationSource).toContain("`nemotron-ultra`"); expect(stationSource).toContain("`deepseek-v4-flash`"); expect(stationSource).toContain("`deepseek-ai/DeepSeek-V4-Flash`"); expect(stationSource).toContain("Automatic pair selection"); @@ -841,7 +841,7 @@ describe("starter prompt docs CTA", () => { "For automatic pair selection, run the ordinary installer without", ); expect(stationSource).toContain("For DeepSeek, pass `--station-deepseek`"); - expect(stationSource).toContain("TCP port `29501`"); + expect(stationSource).toContain("TCP port `6379`"); expect(stationSource).toContain("shared `/24`"); for (const environmentName of [ "NEMOCLAW_PROVIDER", From 5cd75356bff720fc14ffbaf751c64c1e598da58d Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 21 Jul 2026 15:39:32 -0700 Subject: [PATCH 46/74] test(vllm): align Station checks with Ray runtime Signed-off-by: Aaron Erickson --- docs/inference/set-up-vllm.mdx | 1 + src/lib/inference/local-vllm-auth.test.ts | 2 +- test/install-station-container-coexistence.test.ts | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/inference/set-up-vllm.mdx b/docs/inference/set-up-vllm.mdx index 0c91a14f6f..5eda3cd9b7 100644 --- a/docs/inference/set-up-vllm.mdx +++ b/docs/inference/set-up-vllm.mdx @@ -166,6 +166,7 @@ Setting `NEMOCLAW_VLLM_MODEL` remains authoritative; setting `NEMOCLAW_DGX_STATI Pass `--station-deepseek` to select the existing `deepseek-v4-flash` recipe explicitly while retaining the same one-confirmation express flow. The flag requires an interactive terminal; in a `curl | bash` pipeline, `/dev/tty` must be available. Without terminal access, the installer stops before it installs Docker or build dependencies instead of silently continuing with another configuration. +The registered single-Station Ultra recipe tracks the [official DGX Station deployment guide](https://github.com/NVIDIA-NeMo/Nemotron/blob/287ae845639d2ce998998cb8fd1f70a3fa943c0b/usage-cookbook/Nemotron-3-Ultra/StationDeploymentGuide/README.md) and configures the pinned model revision, CPU offload, `16 GB` of shared memory, memory/stack ulimits, MTP speculative decoding, and the Nemotron reasoning and tool-call parsers. Configure the physical rails and SSH host-key and authentication trust before installation. NemoClaw does not modify rail configuration, enroll SSH trust, or reboot either host automatically. If either host requires a reboot after pinned prerequisite changes, the installer stops with status `10` and resumes only after the manual reboot with owner-only state tied to the printed exact revision and reciprocal identity. diff --git a/src/lib/inference/local-vllm-auth.test.ts b/src/lib/inference/local-vllm-auth.test.ts index a8b5fdcb3a..4c8390143a 100644 --- a/src/lib/inference/local-vllm-auth.test.ts +++ b/src/lib/inference/local-vllm-auth.test.ts @@ -62,7 +62,7 @@ function productionManagedBaseUrlResolver( BASE_URL, "b".repeat(64), "GPU-12345678", - "1", + "2", "c".repeat(64), apiKeyFingerprint, "d".repeat(32), diff --git a/test/install-station-container-coexistence.test.ts b/test/install-station-container-coexistence.test.ts index ab10619fe6..0fabb3b069 100644 --- a/test/install-station-container-coexistence.test.ts +++ b/test/install-station-container-coexistence.test.ts @@ -110,6 +110,7 @@ check_package_managers_idle() { :; } check_dpkg_database_health() { :; } check_failed_units() { :; } check_agent_and_inference_conflicts() { :; } +verify_dual_station_controller_uid_binding() { :; } driver_loaded_exact() { return 0; } package_is_ready() { return 0; } verify_gpu() { :; } From 09d579172e36ce15fdc9d70666b0a4a815e7371c Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 21 Jul 2026 15:50:40 -0700 Subject: [PATCH 47/74] test(vllm): remove dual Station simulator Signed-off-by: Aaron Erickson --- CONTRIBUTING.md | 34 -- package.json | 1 - scripts/simulate-dual-station.mts | 270 ----------- ...llm-dual-station-simulator-command.test.ts | 143 ------ ...llm-dual-station-simulator.test-support.ts | 435 ------------------ .../vllm-dual-station-simulator.test.ts | 182 -------- .../inference/vllm-station-cluster.test.ts | 36 +- .../vllm-station-ssh-binding.test-support.ts | 10 - ...vllm-station-model-staging-test-support.ts | 20 +- 9 files changed, 41 insertions(+), 1090 deletions(-) delete mode 100644 scripts/simulate-dual-station.mts delete mode 100644 src/lib/inference/vllm-dual-station-simulator-command.test.ts delete mode 100644 src/lib/inference/vllm-dual-station-simulator.test-support.ts delete mode 100644 src/lib/inference/vllm-dual-station-simulator.test.ts diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 347bb14734..2635111ef5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -307,7 +307,6 @@ These are the primary npm scripts for day-to-day development: | `npm run test:watch` | Watch the CLI, plugin, and E2E-support projects and rerun affected tests | | `npm run test:shuffle` | Shuffle test order in the focused source projects without collecting coverage | | `npm run test:diagnose:leaks` | Report async-resource leaks and diagnose a Vitest process that hangs during shutdown | -| `npm run test:dual-station-sim` | Run the guarded, fixture-backed dual-DGX Station planner and lifecycle simulator | | `npm run test:integration` | Clean-build the CLI and run root integration and installer tests | | `npm run test:package` | Clean-build CLI/plugin artifacts and run compiled-package contracts | | `npm run test:live-e2e` | Opt into live E2E scenarios (mutates real external state) | @@ -329,39 +328,6 @@ npx vitest run --project e2e-support This project is fast and does not run live targets. Live E2E remains opt-in through `npm run test:live-e2e` or the applicable GitHub Actions workflow. -### Dual-Station Simulation - -Run the dual-Station simulator when changing DGX Station topology qualification, strict peer SSH -binding, model staging, distributed launch arguments, lifecycle ownership, rollback, or managed-vLLM -source orchestration: - -```bash -npm run test:dual-station-sim -``` - -The command selects only the repository's audited non-live source suites, gives the Vitest child a -private temporary home, cache, and temp directory, passes a minimal allowlist of local process -variables, forces live projects off, and prepends fail-fast shims for Docker, SSH, networking, GPU, -Python, and download commands. Its planner fixtures -model two GB300 systems, two 400 Gbit/s MTU-9000 CX-8 rails, reciprocal private `/30` links, direct -routes and neighbors, jumbo frames, and a common RoCEv2 GID. Its in-memory Docker adapters exercise -the exact production launch contracts, worker-first ordering, ownership, transaction, rollback, -reconciliation, secret placement, and cleanup logic. A connected contract test passes the real -planner output into those lifecycle functions and models registration-gated readiness, bearer auth, -worker loss, and recovery in memory. - -The simulator does not execute or cover `install.sh` pair-preparation integration; its audited suites -begin at the managed-vLLM source contracts. - -The selected suites use in-memory adapters rather than a real Docker daemon, SSH target, network -interface, GPU, image pull, or model download. They may execute local fixture-only shell syntax -checks and therefore require a POSIX development host. The runner is bounded to two minutes. This -guarding is defense in depth around a narrowly reviewed suite list, not an OS network sandbox. The -simulated service is not a real vLLM server. This simulator does not prove NCCL, RoCE, GPUDirect -RDMA, physical link behavior, or real TP=2 inference. -Use an explicitly authorized live two-Station smoke for those claims; do not add production -`SKIP_GPU`, `SKIP_RDMA`, or assumed-ready switches to make this simulator pass. - ### Test Declarative Behavior Do not read a shipped YAML, JSON, manifest, workflow, or E2E runtime file only to assert its keys, diff --git a/package.json b/package.json index 7d25111e3e..a0b9c0068b 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,6 @@ "test:watch": "vitest watch --project cli --project plugin --project e2e-support", "test:shuffle": "vitest run --project cli --project plugin --project e2e-support --sequence.shuffle.tests --coverage=false", "test:diagnose:leaks": "vitest run --project cli --project plugin --project e2e-support --detectAsyncLeaks --coverage=false --reporter=default --reporter=hanging-process", - "test:dual-station-sim": "tsx scripts/simulate-dual-station.mts", "test:integration": "npm run clean:cli && npm run build:cli && vitest run --project integration --project installer-integration", "test:package": "npm run clean:cli && npm --prefix nemoclaw run clean && npm run build:cli && npm --prefix nemoclaw run build && vitest run --project package-contract", "test:coverage:cli": "npm run clean:cli && npm run build:cli && tsx scripts/check-dist-sourcemaps.mts dist && vitest run --project cli --project integration --coverage --coverage.reporter=text-summary --coverage.reporter=json-summary --coverage.reportsDirectory=coverage/cli --coverage.include=\"bin/**/*.js\" --coverage.include=\"src/**/*.ts\" --coverage.exclude=\"test/**/*.js\" --coverage.exclude=\"test/**/*.ts\" && tsx scripts/check-coverage-ratchet.mts coverage/cli/coverage-summary.json ci/coverage-threshold-cli.json \"CLI coverage\"", diff --git a/scripts/simulate-dual-station.mts b/scripts/simulate-dual-station.mts deleted file mode 100644 index 407fc9036b..0000000000 --- a/scripts/simulate-dual-station.mts +++ /dev/null @@ -1,270 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { spawnSync } from "node:child_process"; -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; -import process from "node:process"; -import { fileURLToPath } from "node:url"; - -export const DUAL_STATION_SIMULATION_SUITES = Object.freeze([ - "src/lib/inference/vllm-models.test.ts", - "src/lib/inference/vllm-station-cluster.test.ts", - "src/lib/inference/vllm-station-cluster-lifecycle.test.ts", - "src/lib/inference/vllm-dual-station.test.ts", - "src/lib/inference/vllm-station-model-staging.test.ts", - "src/lib/inference/vllm-station-ssh-binding.test.ts", - "src/lib/inference/vllm-dual-station-simulator.test.ts", - "src/lib/inference/vllm-dual-station-simulator-command.test.ts", -]); - -export const DUAL_STATION_SIMULATION_POISON_EXECUTABLES = Object.freeze([ - "curl", - "docker", - "ibstat", - "ibv_devinfo", - "ip", - "mpirun", - "nccl-tests", - "nvidia-smi", - "ping", - "python", - "python3", - "rdma", - "rsync", - "scp", - "sftp", - "ssh", - "wget", -]); - -export const DUAL_STATION_SIMULATION_TIMEOUT_MS = 120_000; -export const DUAL_STATION_SIMULATION_FIXTURE_PYTHON_ENV = - "NEMOCLAW_TEST_DUAL_STATION_FIXTURE_PYTHON"; - -const INHERITED_ENV_KEYS = Object.freeze([ - "CI", - "COLORTERM", - "FORCE_COLOR", - "LANG", - "LC_ALL", - "LC_CTYPE", - "NO_COLOR", - "PATH", - "TERM", -]); - -export interface DualStationSimulationInvocation { - command: string; - args: string[]; - env: NodeJS.ProcessEnv; -} - -interface SimulationPoisonBin { - directory: string; - cacheDirectory: string; - homeDirectory: string; - tempDirectory: string; - cleanup(): void; -} - -export function createSimulationPoisonBin(): SimulationPoisonBin { - const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-dual-station-sim-")); - fs.chmodSync(directory, 0o700); - const cacheDirectory = path.join(directory, "cache"); - const homeDirectory = path.join(directory, "home"); - const tempDirectory = path.join(directory, "tmp"); - for (const child of [cacheDirectory, homeDirectory, tempDirectory]) { - fs.mkdirSync(child, { mode: 0o700 }); - } - for (const executable of DUAL_STATION_SIMULATION_POISON_EXECUTABLES) { - const unixFile = path.join(directory, executable); - fs.writeFileSync( - unixFile, - `#!/bin/sh\necho "dual-Station simulator blocked external command: ${executable}" >&2\nexit 97\n`, - { mode: 0o700 }, - ); - fs.chmodSync(unixFile, 0o700); - fs.writeFileSync( - `${unixFile}.cmd`, - `@echo off\r\necho dual-Station simulator blocked external command: ${executable} 1>&2\r\nexit /b 97\r\n`, - { mode: 0o600 }, - ); - } - return { - directory, - cacheDirectory, - homeDirectory, - tempDirectory, - cleanup: () => fs.rmSync(directory, { recursive: true, force: true }), - }; -} - -export function dualStationSimulationEnvironment( - source: NodeJS.ProcessEnv = process.env, -): NodeJS.ProcessEnv { - const env: NodeJS.ProcessEnv = {}; - for (const key of INHERITED_ENV_KEYS) { - if (source[key] !== undefined) env[key] = source[key]; - } - env.NEMOCLAW_RUN_BRANCH_VALIDATION_E2E = "0"; - env.NEMOCLAW_RUN_LIVE_E2E = "0"; - return env; -} - -function canonicalExecutable(candidate: string): string | null { - try { - const canonical = fs.realpathSync(candidate); - if (!path.isAbsolute(canonical) || path.normalize(canonical) !== canonical) return null; - const metadata = fs.statSync(canonical); - if (!metadata.isFile()) return null; - fs.accessSync(canonical, fs.constants.X_OK); - return canonical; - } catch { - return null; - } -} - -/** - * Capture one absolute Python interpreter for behavioral fixtures before PATH - * is poisoned. The simulator passes only this canonical path to its test - * worker; production probes and every PATH-based Python invocation remain - * guarded by the exit-97 shims. - */ -export function resolveDualStationSimulationFixturePython( - sourceEnv: NodeJS.ProcessEnv = process.env, -): string { - const captured = sourceEnv[DUAL_STATION_SIMULATION_FIXTURE_PYTHON_ENV]; - if (captured !== undefined) { - if (!path.isAbsolute(captured) || path.normalize(captured) !== captured) { - throw new Error("The dual-Station simulator fixture Python path must be absolute"); - } - const canonical = canonicalExecutable(captured); - if (!canonical || canonical !== captured) { - throw new Error("The dual-Station simulator fixture Python path is not canonical executable"); - } - return canonical; - } - - for (const directory of (sourceEnv.PATH ?? "").split(path.delimiter)) { - if (!directory || !path.isAbsolute(directory) || path.normalize(directory) !== directory) { - continue; - } - const canonical = canonicalExecutable(path.join(directory, "python3")); - if (canonical) return canonical; - } - throw new Error( - "The dual-Station simulator requires python3 for its local embedded-script fixtures", - ); -} - -export function buildDualStationSimulationInvocation( - repositoryRoot: string, - sourceEnv: NodeJS.ProcessEnv = process.env, -): DualStationSimulationInvocation { - const vitestEntry = path.join(repositoryRoot, "node_modules", "vitest", "vitest.mjs"); - if (!fs.existsSync(vitestEntry)) { - throw new Error(`Repository-local Vitest entry point is missing: ${vitestEntry}`); - } - return { - command: process.execPath, - args: [ - vitestEntry, - "run", - "--project", - "cli", - ...DUAL_STATION_SIMULATION_SUITES, - "--reporter=dot", - ], - env: dualStationSimulationEnvironment(sourceEnv), - }; -} - -export function assertDualStationSimulationPlatform(platform = process.platform): void { - if (platform === "win32") { - throw new Error( - "The dual-Station simulator requires a POSIX host because its SSH-binding fixtures validate POSIX modes and shell syntax.", - ); - } -} - -function repositoryRoot(): string { - return path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); -} - -function describeSimulation(): void { - process.stdout.write( - `${JSON.stringify( - { - mode: "simulation-only", - localProcesses: [ - "repository-local Vitest worker", - "fixture-only shell syntax checks", - "captured absolute Python interpreter for embedded-script fixtures", - ], - liveTargets: [], - externalCommandShims: DUAL_STATION_SIMULATION_POISON_EXECUTABLES, - suites: DUAL_STATION_SIMULATION_SUITES, - }, - null, - 2, - )}\n`, - ); -} - -export function main(argv: readonly string[] = process.argv.slice(2)): number { - if (argv.length === 1 && argv[0] === "--describe") { - describeSimulation(); - return 0; - } - if (argv.length > 0) { - throw new Error(`Unknown dual-Station simulator argument: ${argv.join(" ")}`); - } - - assertDualStationSimulationPlatform(); - const invocation = buildDualStationSimulationInvocation(repositoryRoot()); - const fixturePython = resolveDualStationSimulationFixturePython(invocation.env); - const poisonBin = createSimulationPoisonBin(); - invocation.env.PATH = [poisonBin.directory, invocation.env.PATH] - .filter((entry): entry is string => Boolean(entry)) - .join(path.delimiter); - invocation.env.HOME = poisonBin.homeDirectory; - invocation.env.XDG_CACHE_HOME = poisonBin.cacheDirectory; - invocation.env.TEMP = poisonBin.tempDirectory; - invocation.env.TMP = poisonBin.tempDirectory; - invocation.env.TMPDIR = poisonBin.tempDirectory; - invocation.env[DUAL_STATION_SIMULATION_FIXTURE_PYTHON_ENV] = fixturePython; - delete invocation.env.XDG_RUNTIME_DIR; - process.stdout.write( - "Running the guarded, fixture-backed dual-Station simulator. " + - "The audited suites use in-memory Docker adapters and no live Station target.\n", - ); - let result: ReturnType; - try { - result = spawnSync(invocation.command, invocation.args, { - cwd: repositoryRoot(), - env: invocation.env, - stdio: "inherit", - timeout: DUAL_STATION_SIMULATION_TIMEOUT_MS, - }); - } finally { - poisonBin.cleanup(); - } - if (result.error) throw result.error; - if (result.signal) { - process.stderr.write(`Dual-Station simulator terminated by signal ${result.signal}.\n`); - return 1; - } - return result.status ?? 1; -} - -const invokedFile = process.argv[1] ? path.resolve(process.argv[1]) : ""; -if (invokedFile === fileURLToPath(import.meta.url)) { - try { - process.exitCode = main(); - } catch (error) { - process.stderr.write(`${(error as Error).message}\n`); - process.exitCode = 1; - } -} diff --git a/src/lib/inference/vllm-dual-station-simulator-command.test.ts b/src/lib/inference/vllm-dual-station-simulator-command.test.ts deleted file mode 100644 index 1da2697a94..0000000000 --- a/src/lib/inference/vllm-dual-station-simulator-command.test.ts +++ /dev/null @@ -1,143 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { spawnSync } from "node:child_process"; -import fs from "node:fs"; -import path from "node:path"; - -import { describe, expect, it } from "vitest"; - -import { - assertDualStationSimulationPlatform, - buildDualStationSimulationInvocation, - createSimulationPoisonBin, - DUAL_STATION_SIMULATION_FIXTURE_PYTHON_ENV, - DUAL_STATION_SIMULATION_POISON_EXECUTABLES, - DUAL_STATION_SIMULATION_SUITES, - dualStationSimulationEnvironment, - main, - resolveDualStationSimulationFixturePython, -} from "../../../scripts/simulate-dual-station.mts"; - -describe("dual-Station simulator command", () => { - it("selects only the audited non-live source simulation suites", () => { - expect(DUAL_STATION_SIMULATION_SUITES).toHaveLength(8); - expect(DUAL_STATION_SIMULATION_SUITES).toContain( - "src/lib/inference/vllm-dual-station-simulator.test.ts", - ); - expect( - DUAL_STATION_SIMULATION_SUITES.every( - (suite) => - suite.startsWith("src/lib/inference/") && - suite.endsWith(".test.ts") && - !suite.includes("/e2e/") && - !suite.includes("/live/"), - ), - ).toBe(true); - }); - - it("poisons commands that could reach a live host or external service", () => { - expect(DUAL_STATION_SIMULATION_POISON_EXECUTABLES).toEqual( - expect.arrayContaining(["curl", "docker", "nvidia-smi", "ping", "python3", "ssh"]), - ); - - const poisonBin = createSimulationPoisonBin(); - try { - expect(fs.readFileSync(path.join(poisonBin.directory, "docker"), "utf8")).toContain( - "exit 97", - ); - expect(fs.statSync(poisonBin.homeDirectory).mode & 0o777).toBe(0o700); - expect(fs.statSync(poisonBin.cacheDirectory).mode & 0o777).toBe(0o700); - expect(fs.statSync(poisonBin.tempDirectory).mode & 0o777).toBe(0o700); - } finally { - poisonBin.cleanup(); - } - expect(fs.existsSync(poisonBin.directory)).toBe(false); - }); - - it("uses one captured absolute Python only for embedded fixtures", () => { - const fixturePython = resolveDualStationSimulationFixturePython(); - const poisonBin = createSimulationPoisonBin(); - const env = dualStationSimulationEnvironment(process.env); - env.PATH = [poisonBin.directory, env.PATH] - .filter((entry): entry is string => Boolean(entry)) - .join(path.delimiter); - env.HOME = poisonBin.homeDirectory; - env.TMPDIR = poisonBin.tempDirectory; - env[DUAL_STATION_SIMULATION_FIXTURE_PYTHON_ENV] = fixturePython; - - try { - expect(path.isAbsolute(fixturePython)).toBe(true); - expect(resolveDualStationSimulationFixturePython(env)).toBe(fixturePython); - - const guarded = spawnSync("python3", ["-c", "print('must-not-run')"], { - encoding: "utf8", - env, - }); - expect(guarded.status, guarded.stderr).toBe(97); - expect(guarded.stderr).toContain("simulator blocked external command: python3"); - - const fixture = spawnSync(fixturePython, ["-c", "print('fixture-ok')"], { - encoding: "utf8", - env, - }); - expect(fixture.status, fixture.stderr).toBe(0); - expect(fixture.stdout.trim()).toBe("fixture-ok"); - } finally { - poisonBin.cleanup(); - } - }); - - it("inherits only local process basics and forces live projects off", () => { - const env = dualStationSimulationEnvironment({ - PATH: "/fixture/bin", - HOME: "/real/home", - XDG_CACHE_HOME: "/real/cache", - XDG_RUNTIME_DIR: "/real/runtime", - ACME_SECRET: "should-not-leak", - DOCKER_HOST: "ssh://should-not-run", - GIT_ASKPASS: "/should/not/run", - GITHUB_TOKEN: "should-not-leak", - HF_TOKEN: "should-not-leak", - NEMOCLAW_DGX_STATION_PEER: "should-not-run", - NEMOCLAW_DGX_STATION_SSH_BINDING: "should-not-run", - [DUAL_STATION_SIMULATION_FIXTURE_PYTHON_ENV]: "/should/not/inherit/python3", - NEMOCLAW_RUN_BRANCH_VALIDATION_E2E: "1", - NEMOCLAW_RUN_LIVE_E2E: "1", - UNRELATED_AMBIENT_VALUE: "should-not-inherit", - }); - - expect(env).toEqual({ - PATH: "/fixture/bin", - NEMOCLAW_RUN_BRANCH_VALIDATION_E2E: "0", - NEMOCLAW_RUN_LIVE_E2E: "0", - }); - }); - - it("pins the repository-local Vitest CLI and the non-live project", () => { - const root = path.resolve(import.meta.dirname, "../../.."); - const invocation = buildDualStationSimulationInvocation(root, {}); - - expect(invocation.command).toBe(process.execPath); - expect(invocation.args).toEqual([ - path.join(root, "node_modules", "vitest", "vitest.mjs"), - "run", - "--project", - "cli", - ...DUAL_STATION_SIMULATION_SUITES, - "--reporter=dot", - ]); - }); - - it("fails clearly on native Windows instead of weakening POSIX fixture checks", () => { - expect(() => assertDualStationSimulationPlatform("win32")).toThrow("requires a POSIX host"); - expect(() => assertDualStationSimulationPlatform("darwin")).not.toThrow(); - expect(() => assertDualStationSimulationPlatform("linux")).not.toThrow(); - }); - - it("rejects arguments instead of forwarding them to another test project", () => { - expect(() => main(["--project", "e2e-live"])).toThrow( - "Unknown dual-Station simulator argument", - ); - }); -}); diff --git a/src/lib/inference/vllm-dual-station-simulator.test-support.ts b/src/lib/inference/vllm-dual-station-simulator.test-support.ts deleted file mode 100644 index d81d405c00..0000000000 --- a/src/lib/inference/vllm-dual-station-simulator.test-support.ts +++ /dev/null @@ -1,435 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { - DUAL_STATION_VLLM_RUNTIME, - NEMOCLAW_DGX_STATION_PEER_ENV, - probeDualStationVllmCapability, - type StationClusterProbeDeps, - type StationHostProbe, - type StationProbeCommandResult, - type StationRailConnectivityRequest, -} from "./vllm-station-cluster"; -import { - DUAL_STATION_VLLM_API_KEY_FINGERPRINT_LABEL, - DUAL_STATION_VLLM_CLUSTER_LABEL, - DUAL_STATION_VLLM_ENDPOINT_LABEL, - DUAL_STATION_VLLM_GPU_LABEL, - DUAL_STATION_VLLM_GPU_SMOKE_LABEL, - DUAL_STATION_VLLM_HEAD_CONTAINER_NAME, - DUAL_STATION_VLLM_LAUNCH_CONTRACT_LABEL, - DUAL_STATION_VLLM_LAUNCH_SCHEMA_LABEL, - DUAL_STATION_VLLM_MANAGED_LABEL, - DUAL_STATION_VLLM_ROLE_LABEL, - DUAL_STATION_VLLM_TRANSACTION_LABEL, - DUAL_STATION_VLLM_WORKER_CONTAINER_NAME, - type DualStationDockerOptions, - type DualStationVllmLifecycleDeps, -} from "./vllm-station-cluster-lifecycle"; -import { NEMOCLAW_DGX_STATION_SSH_BINDING_ENV } from "./vllm-station-ssh-binding"; -import { - createDualStationSshBindingFixture, - type DualStationSshBindingFixture, -} from "./vllm-station-ssh-binding.test-support"; - -export const DUAL_STATION_SIMULATOR_API_KEY = "a".repeat(64); -export const DUAL_STATION_SIMULATOR_LOCAL_GPU = "GPU-aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee"; -export const DUAL_STATION_SIMULATOR_PEER_GPU = "GPU-11111111-2222-3333-4444-555555555555"; - -const PEER_TARGET = "nvidia@station-b"; -const BINDING_TOKEN = "simulated-qualified-binding"; -const activeSshFixtures = new Set(); - -function snapshotPath(home: string): string { - return [ - home, - ".cache/huggingface/hub", - `models--${DUAL_STATION_VLLM_RUNTIME.modelId.replace("/", "--")}`, - "snapshots", - DUAL_STATION_VLLM_RUNTIME.modelRevision, - ].join("/"); -} - -function stationRail( - rdmaDevice: string, - netdev: string, - pciAddress: string, - macAddress: string, - address: string, -) { - return { - rdmaDevice, - port: 1, - netdev, - macAddress, - uverbsDevice: `/dev/infiniband/uverbs${rdmaDevice.endsWith("0") ? "0" : "1"}`, - pciAddress, - pciName: `${pciAddress} Ethernet controller: NVIDIA ConnectX-8 SuperNIC`, - state: "4: ACTIVE", - linkLayer: "Ethernet", - speedMbps: 400_000, - mtu: 9000, - ipv4Addresses: [{ address, prefixLength: 30 }], - roceV2Ipv4Gids: [{ index: 3, address }], - }; -} - -function stationHost(side: "local" | "peer"): StationHostProbe { - const local = side === "local"; - const home = local ? "/home/local" : "/home/peer"; - return { - schemaVersion: 1, - hostname: local ? "station-a" : "station-b", - productName: "NVIDIA DGX Station GB300", - architecture: "aarch64", - home, - uid: local ? 1000 : 1001, - gid: local ? 1000 : 1001, - gpus: [ - { - index: 0, - name: "NVIDIA GB300 Grace Blackwell Superchip", - uuid: local ? DUAL_STATION_SIMULATOR_LOCAL_GPU : DUAL_STATION_SIMULATOR_PEER_GPU, - }, - ], - docker: { reachable: true, nvidiaRuntime: true }, - rsyncAvailable: true, - nvidiaPeermemLoaded: true, - rails: local - ? [ - stationRail("mlx5_0", "cx8a0", "0001:03:00.0", "02:00:00:aa:00:00", "192.168.240.1"), - stationRail("mlx5_1", "cx8a1", "0001:03:00.1", "02:00:00:aa:00:01", "192.168.240.5"), - ] - : [ - // Inventory is intentionally reversed; the production planner must match by subnet. - stationRail("mlx5_1", "cx8b1", "0002:03:00.1", "02:00:00:bb:00:01", "192.168.240.6"), - stationRail("mlx5_0", "cx8b0", "0002:03:00.0", "02:00:00:bb:00:00", "192.168.240.2"), - ], - modelSnapshot: { - modelId: DUAL_STATION_VLLM_RUNTIME.modelId, - revision: DUAL_STATION_VLLM_RUNTIME.modelRevision, - path: snapshotPath(home), - directoryExists: true, - complete: true, - shardCount: 113, - reason: "", - }, - }; -} - -function command(stdout: unknown): StationProbeCommandResult { - return { - status: 0, - stdout: typeof stdout === "string" ? stdout : JSON.stringify(stdout), - }; -} - -function strictSshConfig(fixture: DualStationSshBindingFixture): string { - const binding = fixture.binding; - return [ - `hostname ${binding.resolvedHost}`, - `user ${binding.sshUser}`, - `port ${String(binding.port)}`, - `hostkeyalias ${binding.lookupHost}`, - `userknownhostsfile ${binding.knownHostsFile}`, - "globalknownhostsfile /dev/null", - "batchmode yes", - "stricthostkeychecking true", - "permitlocalcommand no", - "forwardagent no", - "forwardx11 no", - "forwardx11trusted no", - "tunnel false", - "updatehostkeys no", - "controlmaster false", - "controlpersist no", - "sendenv LANG", - "sendenv LC_*", - ].join("\n"); -} - -function connectivity(requests: readonly StationRailConnectivityRequest[]) { - return command({ - schemaVersion: 1, - checks: requests.map((request) => ({ - ...request, - routeDevice: request.netdev, - routeSource: request.sourceAddress, - routeGateway: null, - routeScope: "link", - peerMac: request.expectedPeerMac, - peerNeighborState: "REACHABLE", - jumboPing: true, - })), - }); -} - -function probeDeps(fixture: DualStationSshBindingFixture): StationClusterProbeDeps { - return { - loadPeerSshBinding: (token, expectedPeerTarget) => { - if (token !== BINDING_TOKEN) throw new Error("unexpected simulated SSH binding token"); - if (expectedPeerTarget !== PEER_TARGET) throw new Error("unexpected simulated peer target"); - return fixture.binding; - }, - probePeerSshConfig: () => command(strictSshConfig(fixture)), - probeLocalHost: () => command(stationHost("local")), - probePeerHost: () => command(stationHost("peer")), - probeLocalConnectivity: connectivity, - probePeerConnectivity: (_binding, requests) => connectivity(requests), - }; -} - -export function createDualStationSimulationPlan() { - const fixture = createDualStationSshBindingFixture(PEER_TARGET); - activeSshFixtures.add(fixture); - const capability = probeDualStationVllmCapability({ - env: { - [NEMOCLAW_DGX_STATION_PEER_ENV]: PEER_TARGET, - [NEMOCLAW_DGX_STATION_SSH_BINDING_ENV]: BINDING_TOKEN, - }, - deps: probeDeps(fixture), - }); - if (capability.kind !== "ready") throw new Error("synthetic topology did not qualify"); - return capability.plan; -} - -export function cleanupDualStationSimulationFixtures(): void { - for (const fixture of activeSshFixtures) fixture.cleanup(); - activeSshFixtures.clear(); -} - -type SimulatedContainer = { - id: string; - target: "local" | "peer"; - name: string; - state: "running" | "exited"; - image: string; - labels: Record; - visibleGpu: string | null; - command: string; - environment: NodeJS.ProcessEnv; -}; - -export type DualStationSimulatorMutation = { - kind: "run" | "rm"; - target: "local" | "peer"; - nameOrId: string; - args?: readonly string[]; - options?: DualStationDockerOptions; -}; - -function dockerValues(args: readonly string[], flag: string): string[] { - return args.flatMap((arg, index) => - arg === flag && index < args.length - 1 ? [args[index + 1]] : [], - ); -} - -export function createDualStationLifecycleSimulator() { - const containers = new Map(); - const mutations: DualStationSimulatorMutation[] = []; - const healthDuringManagedLaunch: number[] = []; - let idCounter = 0; - let nonceCounter = 0; - let transactionCounter = 0; - let registeredWorkerId: string | null = null; - - const targetFor = (options?: DualStationDockerOptions): "local" | "peer" => - options?.env?.SIMULATED_DOCKER_TARGET === "peer" ? "peer" : "local"; - const keyFor = (target: "local" | "peer", name: string) => `${target}:${name}`; - const nextId = () => (++idCounter).toString(16).padStart(64, "0"); - const getById = (id: string) => [...containers.values()].find((item) => item.id === id); - const managedPair = () => ({ - head: containers.get(keyFor("local", DUAL_STATION_VLLM_HEAD_CONTAINER_NAME)), - worker: containers.get(keyFor("peer", DUAL_STATION_VLLM_WORKER_CONTAINER_NAME)), - }); - const managedReady = () => { - const { head, worker } = managedPair(); - return ( - head?.state === "running" && worker?.state === "running" && registeredWorkerId === worker.id - ); - }; - - const deps = { - buildLocalDockerEnv: () => ({ - SIMULATED_DOCKER_TARGET: "local", - VLLM_API_KEY: "ambient-secret-must-be-stripped", - }), - buildRemoteDockerEnv: () => ({ - SIMULATED_DOCKER_TARGET: "peer", - VLLM_API_KEY: "ambient-secret-must-be-stripped", - }), - createProbeNonce: () => (++nonceCounter).toString(16).padStart(32, "0"), - createTransactionId: () => (++transactionCounter).toString(16).padStart(32, "0"), - effectiveControllerUid: () => stationHost("local").uid, - readControllerUid: () => stationHost("local").uid, - waitBeforeReconcile: async () => undefined, - withLifecycleLock: async (operation: () => Promise | T): Promise => await operation(), - loadApiKey: () => DUAL_STATION_SIMULATOR_API_KEY, - localInterfaceAddresses: () => ["192.168.240.1"], - dockerCapture: (args: readonly string[], options?: DualStationDockerOptions): string => { - const target = targetFor(options); - if (args[0] === "image") return `sha256:${"f".repeat(64)}\n`; - if (args[0] === "wait") return getById(args[1]) ? "0\n" : "1\n"; - if (args[0] === "logs") return `${getById(args[1])?.visibleGpu ?? ""}\n`; - - const nameFilter = dockerValues(args, "--filter")[0] ?? ""; - const name = nameFilter.replace(/^name=\^\//u, "").replace(/\$$/u, ""); - const item = containers.get(keyFor(target, name)); - if (!item) return ""; - const format = dockerValues(args, "--format")[0] ?? ""; - if (format.includes(DUAL_STATION_VLLM_GPU_SMOKE_LABEL)) { - return [ - item.id, - item.name, - item.image, - item.labels[DUAL_STATION_VLLM_GPU_SMOKE_LABEL] ?? "", - item.labels[DUAL_STATION_VLLM_ROLE_LABEL] ?? "", - ].join("\t"); - } - return [ - item.id, - item.name, - item.state, - item.image, - item.labels[DUAL_STATION_VLLM_MANAGED_LABEL] ?? "", - item.labels[DUAL_STATION_VLLM_ROLE_LABEL] ?? "", - item.labels[DUAL_STATION_VLLM_ENDPOINT_LABEL] ?? "", - item.labels[DUAL_STATION_VLLM_CLUSTER_LABEL] ?? "", - item.labels[DUAL_STATION_VLLM_GPU_LABEL] ?? "", - item.labels[DUAL_STATION_VLLM_LAUNCH_SCHEMA_LABEL] ?? "", - item.labels[DUAL_STATION_VLLM_LAUNCH_CONTRACT_LABEL] ?? "", - item.labels[DUAL_STATION_VLLM_API_KEY_FINGERPRINT_LABEL] ?? "", - item.labels[DUAL_STATION_VLLM_TRANSACTION_LABEL] ?? "", - ].join("\t"); - }, - dockerRunDetached: (args: readonly string[], options?: DualStationDockerOptions) => { - const target = targetFor(options); - const name = dockerValues(args, "--name")[0]; - const labels = Object.fromEntries( - dockerValues(args, "--label").map((entry) => { - const separator = entry.indexOf("="); - return [entry.slice(0, separator), entry.slice(separator + 1)]; - }), - ); - const visibleGpu = (dockerValues(args, "--gpus")[0] ?? "").replace(/^device=/u, "") || null; - const item: SimulatedContainer = { - id: nextId(), - target, - name, - state: name.startsWith("nemoclaw-vllm-gpu-smoke-") ? "exited" : "running", - image: DUAL_STATION_VLLM_RUNTIME.image, - labels, - visibleGpu, - command: args.at(-1) ?? "", - environment: { ...options?.env }, - }; - containers.set(keyFor(target, name), item); - mutations.push({ - kind: "run", - target, - nameOrId: name, - args: [...args], - options, - }); - if ( - name === DUAL_STATION_VLLM_WORKER_CONTAINER_NAME || - name === DUAL_STATION_VLLM_HEAD_CONTAINER_NAME - ) { - healthDuringManagedLaunch.push(managedReady() ? 200 : 503); - } - return { status: 0, stdout: `${item.id}\n` }; - }, - dockerForceRm: (containerId: string, options?: DualStationDockerOptions) => { - const target = targetFor(options); - const item = getById(containerId); - if (!item || item.target !== target) return { status: 1 }; - if (registeredWorkerId === item.id) registeredWorkerId = null; - containers.delete(keyFor(item.target, item.name)); - mutations.push({ kind: "rm", target, nameOrId: containerId, options }); - return { status: 0 }; - }, - } satisfies DualStationVllmLifecycleDeps; - - function serviceRequest(path: string, authorization?: string) { - // This is an in-memory API contract simulator, not inference or a network/RDMA test. - if (path === "/health") return { status: managedReady() ? 200 : 503, body: null }; - const { head } = managedPair(); - if ( - !head?.environment.VLLM_API_KEY || - authorization !== `Bearer ${head.environment.VLLM_API_KEY}` - ) - return { status: 401, body: { error: "unauthorized" } }; - if (!managedReady()) return { status: 503, body: { error: "worker unavailable" } }; - if (path === "/v1/models") { - return { - status: 200, - body: { data: [{ id: DUAL_STATION_VLLM_RUNTIME.servedModelId }] }, - }; - } - if (path === "/v1/chat/completions") { - return { - status: 200, - body: { - choices: [{ message: { role: "assistant", content: "SIMULATED_OK" } }], - }, - }; - } - return { status: 404, body: null }; - } - - function registerWorker(): void { - const { head, worker } = managedPair(); - if (head?.state !== "running" || worker?.state !== "running") { - throw new Error("both simulated roles must be running before registration"); - } - if ( - !worker.command.includes("ray start --address=192.168.240.1:6379") || - !worker.command.includes("--node-ip-address=192.168.240.2") || - worker.command.includes("vllm serve") || - !head.command.includes("ray start --head --node-ip-address=192.168.240.1 --port=6379") || - !head.command.includes("--distributed-executor-backend ray") || - worker.labels[DUAL_STATION_VLLM_TRANSACTION_LABEL] !== - head.labels[DUAL_STATION_VLLM_TRANSACTION_LABEL] - ) { - throw new Error("simulated worker launch contract cannot register with the head"); - } - registeredWorkerId = worker.id; - } - - function loseWorker(): void { - const worker = containers.get(keyFor("peer", DUAL_STATION_VLLM_WORKER_CONTAINER_NAME)); - if (!worker) throw new Error("simulated worker was not running"); - worker.state = "exited"; - registeredWorkerId = null; - } - - return { - containers, - deps, - healthDuringManagedLaunch, - loseWorker, - mutations, - registerWorker, - serviceRequest, - }; -} - -export function dualStationManagedRuns(mutations: readonly DualStationSimulatorMutation[]) { - return mutations.filter( - (mutation) => - mutation.kind === "run" && - (mutation.nameOrId === DUAL_STATION_VLLM_WORKER_CONTAINER_NAME || - mutation.nameOrId === DUAL_STATION_VLLM_HEAD_CONTAINER_NAME), - ); -} - -export function dualStationSimulatorLabels( - mutation: DualStationSimulatorMutation, -): Record { - return Object.fromEntries( - dockerValues(mutation.args ?? [], "--label").map((entry) => { - const separator = entry.indexOf("="); - return [entry.slice(0, separator), entry.slice(separator + 1)]; - }), - ); -} diff --git a/src/lib/inference/vllm-dual-station-simulator.test.ts b/src/lib/inference/vllm-dual-station-simulator.test.ts deleted file mode 100644 index e9e5c6702d..0000000000 --- a/src/lib/inference/vllm-dual-station-simulator.test.ts +++ /dev/null @@ -1,182 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { afterEach, describe, expect, it } from "vitest"; - -import { - DUAL_STATION_SIMULATOR_API_KEY as API_KEY, - cleanupDualStationSimulationFixtures, - createDualStationLifecycleSimulator, - createDualStationSimulationPlan, - dualStationManagedRuns, - dualStationSimulatorLabels, - DUAL_STATION_SIMULATOR_LOCAL_GPU as LOCAL_GPU, - DUAL_STATION_SIMULATOR_PEER_GPU as PEER_GPU, -} from "./vllm-dual-station-simulator.test-support"; -import { DUAL_STATION_VLLM_RUNTIME } from "./vllm-station-cluster"; -import { - areDualStationManagedVllmContainersRunning, - cleanupDualStationManagedVllm, - DUAL_STATION_VLLM_API_KEY_FINGERPRINT_LABEL, - DUAL_STATION_VLLM_CLUSTER_LABEL, - DUAL_STATION_VLLM_ENDPOINT_LABEL, - DUAL_STATION_VLLM_GPU_LABEL, - DUAL_STATION_VLLM_HEAD_CONTAINER_NAME, - DUAL_STATION_VLLM_LAUNCH_CONTRACT_LABEL, - DUAL_STATION_VLLM_LAUNCH_SCHEMA_LABEL, - DUAL_STATION_VLLM_MANAGED_LABEL, - DUAL_STATION_VLLM_ROLE_LABEL, - DUAL_STATION_VLLM_TRANSACTION_LABEL, - DUAL_STATION_VLLM_WORKER_CONTAINER_NAME, - type DualStationVllmRole, - dualStationVllmApiKeyFingerprint, - dualStationVllmClusterId, - dualStationVllmLaunchContract, - preflightDualStationGpuRuntime, - preflightDualStationManagedVllm, - startDualStationManagedVllm, -} from "./vllm-station-cluster-lifecycle"; - -afterEach(cleanupDualStationSimulationFixtures); - -describe("connected dual-Station planner-to-lifecycle simulator", () => { - it("passes a synthetic plan through launch, simulated registration, loss, recovery, and cleanup", async () => { - const plan = createDualStationSimulationPlan(); - expect(plan).toMatchObject({ - masterAddress: "192.168.240.1", - roceGidIndex: 3, - runtime: DUAL_STATION_VLLM_RUNTIME, - rails: [ - { local: { netdev: "cx8a0" }, peer: { netdev: "cx8b0" } }, - { local: { netdev: "cx8a1" }, peer: { netdev: "cx8b1" } }, - ], - }); - - const sim = createDualStationLifecycleSimulator(); - expect(sim.serviceRequest("/health")).toEqual({ status: 503, body: null }); - expect(preflightDualStationManagedVllm(plan, sim.deps)).toEqual({ - ok: true, - }); - expect(await preflightDualStationGpuRuntime(plan, sim.deps)).toEqual({ - ok: true, - }); - expect(areDualStationManagedVllmContainersRunning(plan, sim.deps)).toBe(false); - - const firstStart = await startDualStationManagedVllm(plan, { apiKey: API_KEY }, sim.deps); - expect(firstStart).toMatchObject({ ok: true, reusedExisting: false }); - expect(areDualStationManagedVllmContainersRunning(plan, sim.deps)).toBe(true); - expect(sim.healthDuringManagedLaunch.slice(0, 2)).toEqual([503, 503]); - expect(sim.serviceRequest("/health")).toEqual({ status: 503, body: null }); - sim.registerWorker(); - expect(sim.serviceRequest("/health")).toEqual({ status: 200, body: null }); - expect(sim.serviceRequest("/v1/models").status).toBe(401); - expect(sim.serviceRequest("/v1/models", `Bearer ${API_KEY}`)).toEqual({ - status: 200, - body: { data: [{ id: DUAL_STATION_VLLM_RUNTIME.servedModelId }] }, - }); - expect(sim.serviceRequest("/v1/chat/completions", `Bearer ${API_KEY}`)).toEqual({ - status: 200, - body: { - choices: [{ message: { role: "assistant", content: "SIMULATED_OK" } }], - }, - }); - - const firstRuns = dualStationManagedRuns(sim.mutations); - expect(firstRuns.map(({ target, nameOrId }) => [target, nameOrId])).toEqual([ - ["peer", DUAL_STATION_VLLM_WORKER_CONTAINER_NAME], - ["local", DUAL_STATION_VLLM_HEAD_CONTAINER_NAME], - ]); - const expectedFingerprint = dualStationVllmApiKeyFingerprint(API_KEY); - for (const [index, run] of firstRuns.entries()) { - const role: DualStationVllmRole = index === 0 ? "worker" : "head"; - const labels = dualStationSimulatorLabels(run); - expect(labels).toEqual({ - [DUAL_STATION_VLLM_MANAGED_LABEL]: "true", - [DUAL_STATION_VLLM_ROLE_LABEL]: role, - [DUAL_STATION_VLLM_ENDPOINT_LABEL]: - role === "head" ? "http://192.168.240.1:8000" : "headless", - [DUAL_STATION_VLLM_CLUSTER_LABEL]: dualStationVllmClusterId(plan), - [DUAL_STATION_VLLM_GPU_LABEL]: role === "head" ? LOCAL_GPU : PEER_GPU, - [DUAL_STATION_VLLM_LAUNCH_SCHEMA_LABEL]: "2", - [DUAL_STATION_VLLM_LAUNCH_CONTRACT_LABEL]: dualStationVllmLaunchContract(plan, role), - [DUAL_STATION_VLLM_API_KEY_FINGERPRINT_LABEL]: expectedFingerprint, - [DUAL_STATION_VLLM_TRANSACTION_LABEL]: "1".padStart(32, "0"), - }); - expect(JSON.stringify(run.args)).not.toContain(API_KEY); - expect(JSON.stringify(labels)).not.toContain(API_KEY); - expect(run.options?.env?.VLLM_API_KEY).toBe(role === "head" ? API_KEY : undefined); - } - for (const mutation of sim.mutations.filter((item) => !firstRuns.includes(item))) { - expect(JSON.stringify(mutation.args ?? [])).not.toContain(API_KEY); - expect(mutation.options?.env?.VLLM_API_KEY).toBeUndefined(); - } - - sim.loseWorker(); - expect(areDualStationManagedVllmContainersRunning(plan, sim.deps)).toBe(false); - expect(sim.serviceRequest("/health")).toEqual({ status: 503, body: null }); - expect(sim.serviceRequest("/v1/chat/completions", `Bearer ${API_KEY}`).status).toBe(503); - - const recovered = await startDualStationManagedVllm(plan, { apiKey: API_KEY }, sim.deps); - expect(recovered).toMatchObject({ ok: true, reusedExisting: false }); - expect(areDualStationManagedVllmContainersRunning(plan, sim.deps)).toBe(true); - expect(sim.serviceRequest("/health")).toEqual({ status: 503, body: null }); - expect(sim.serviceRequest("/v1/chat/completions", `Bearer ${API_KEY}`).status).toBe(503); - sim.registerWorker(); - expect(sim.serviceRequest("/health")).toEqual({ status: 200, body: null }); - expect(sim.serviceRequest("/v1/chat/completions", `Bearer ${API_KEY}`).status).toBe(200); - - const allRuns = dualStationManagedRuns(sim.mutations); - expect(allRuns.map(({ target, nameOrId }) => [target, nameOrId])).toEqual([ - ["peer", DUAL_STATION_VLLM_WORKER_CONTAINER_NAME], - ["local", DUAL_STATION_VLLM_HEAD_CONTAINER_NAME], - ["peer", DUAL_STATION_VLLM_WORKER_CONTAINER_NAME], - ["local", DUAL_STATION_VLLM_HEAD_CONTAINER_NAME], - ]); - expect( - allRuns - .slice(2) - .map((run) => dualStationSimulatorLabels(run)[DUAL_STATION_VLLM_TRANSACTION_LABEL]), - ).toEqual(["2".padStart(32, "0"), "2".padStart(32, "0")]); - expect( - allRuns.map( - (run) => dualStationSimulatorLabels(run)[DUAL_STATION_VLLM_LAUNCH_CONTRACT_LABEL], - ), - ).toEqual([ - dualStationVllmLaunchContract(plan, "worker"), - dualStationVllmLaunchContract(plan, "head"), - dualStationVllmLaunchContract(plan, "worker"), - dualStationVllmLaunchContract(plan, "head"), - ]); - expect(sim.healthDuringManagedLaunch).toEqual([503, 503, 503, 503]); - for (const [index, run] of allRuns.entries()) { - const role: DualStationVllmRole = index % 2 === 0 ? "worker" : "head"; - const transactionId = index < 2 ? "1".padStart(32, "0") : "2".padStart(32, "0"); - expect(dualStationSimulatorLabels(run)).toEqual({ - [DUAL_STATION_VLLM_MANAGED_LABEL]: "true", - [DUAL_STATION_VLLM_ROLE_LABEL]: role, - [DUAL_STATION_VLLM_ENDPOINT_LABEL]: - role === "head" ? "http://192.168.240.1:8000" : "headless", - [DUAL_STATION_VLLM_CLUSTER_LABEL]: dualStationVllmClusterId(plan), - [DUAL_STATION_VLLM_GPU_LABEL]: role === "head" ? LOCAL_GPU : PEER_GPU, - [DUAL_STATION_VLLM_LAUNCH_SCHEMA_LABEL]: "2", - [DUAL_STATION_VLLM_LAUNCH_CONTRACT_LABEL]: dualStationVllmLaunchContract(plan, role), - [DUAL_STATION_VLLM_API_KEY_FINGERPRINT_LABEL]: expectedFingerprint, - [DUAL_STATION_VLLM_TRANSACTION_LABEL]: transactionId, - }); - } - for (const mutation of sim.mutations) { - expect(JSON.stringify(mutation.args ?? [])).not.toContain(API_KEY); - expect(JSON.stringify(dualStationSimulatorLabels(mutation))).not.toContain(API_KEY); - const isManagedHeadRun = - mutation.kind === "run" && mutation.nameOrId === DUAL_STATION_VLLM_HEAD_CONTAINER_NAME; - expect(mutation.options?.env?.VLLM_API_KEY).toBe(isManagedHeadRun ? API_KEY : undefined); - } - - const cleanup = await cleanupDualStationManagedVllm(plan, sim.deps); - expect(cleanup).toMatchObject({ ok: true }); - expect(cleanup.ok && cleanup.removedContainerIds).toHaveLength(2); - expect(areDualStationManagedVllmContainersRunning(plan, sim.deps)).toBe(false); - expect(sim.serviceRequest("/health")).toEqual({ status: 503, body: null }); - expect(sim.containers.size).toBe(0); - }); -}); diff --git a/src/lib/inference/vllm-station-cluster.test.ts b/src/lib/inference/vllm-station-cluster.test.ts index f0eef48228..32f0cc1683 100644 --- a/src/lib/inference/vllm-station-cluster.test.ts +++ b/src/lib/inference/vllm-station-cluster.test.ts @@ -1,7 +1,6 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import assert from "node:assert/strict"; import { type SpawnSyncOptionsWithStringEncoding, spawnSync } from "node:child_process"; import fs from "node:fs"; import os from "node:os"; @@ -9,8 +8,6 @@ import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { resolveDualStationSimulationFixturePython } from "../../../scripts/simulate-dual-station.mts"; - import { buildRemoteVllmDockerEnv } from "./vllm-docker-env"; import { createStationClusterProbeDeps, @@ -32,11 +29,27 @@ import { import { createDualStationSshBindingFixture, type DualStationSshBindingFixture, - retargetDualStationSshBindingFixture, } from "./vllm-station-ssh-binding.test-support"; const LOCAL_HOME = "/home/local"; const PEER_HOME = "/home/nvidia"; + +function resolveFixturePython(): string { + for (const directory of (process.env.PATH ?? "").split(path.delimiter)) { + if (!directory || !path.isAbsolute(directory) || path.normalize(directory) !== directory) { + continue; + } + try { + const candidate = fs.realpathSync(path.join(directory, "python3")); + fs.accessSync(candidate, fs.constants.X_OK); + if (fs.statSync(candidate).isFile()) return candidate; + } catch { + // Keep searching PATH for an executable fixture interpreter. + } + } + throw new Error("python3 is required for the Station host-probe fixtures"); +} + function strictDockerSshConfig(binding: DualStationSshBinding): string { return [ `hostname ${binding.resolvedHost}`, @@ -233,11 +246,10 @@ function fixtureDeps( } function runWith(deps: StationClusterProbeDeps, target = "nvidia@station-b") { - sshFixture = retargetDualStationSshBindingFixture( - sshFixture, - target, - validatePeerTarget(target).ok, - ); + if (validatePeerTarget(target).ok && sshFixture.binding.peerTarget !== target) { + sshFixture.cleanup(); + sshFixture = createDualStationSshBindingFixture(target); + } return probeDualStationVllmCapability({ env: { [NEMOCLAW_DGX_STATION_PEER_ENV]: target, @@ -373,7 +385,7 @@ describe("probeDualStationVllmCapability", () => { peerModelSnapshot: "ready", plan: { peerSshBinding: { peerTarget: target } }, }); - assert(result.kind === "ready", "expected ready fixture"); + if (result.kind !== "ready") throw new Error("expected ready fixture"); expect(buildRemoteVllmDockerEnv(result.plan.peerSshBinding, {}).DOCKER_HOST).toBe( `ssh://${result.plan.peerSshBinding.sshUser}@${result.plan.peerSshBinding.resolvedHost}`, ); @@ -872,7 +884,7 @@ describe("probe command boundary", () => { }, ); createStationClusterProbeDeps(recordingSpawn).probeLocalHost(); - const python = resolveDualStationSimulationFixturePython(); + const python = resolveFixturePython(); try { const executed = spawnSync(python, ["-"], { @@ -913,7 +925,7 @@ describe("probe command boundary", () => { }, ); createStationClusterProbeDeps(recordingSpawn).probeLocalHost(); - const python = resolveDualStationSimulationFixturePython(); + const python = resolveFixturePython(); const fixturePrelude = String.raw` import pathlib import stat as fixture_stat diff --git a/src/lib/inference/vllm-station-ssh-binding.test-support.ts b/src/lib/inference/vllm-station-ssh-binding.test-support.ts index b431fc83f2..4213d50dcb 100644 --- a/src/lib/inference/vllm-station-ssh-binding.test-support.ts +++ b/src/lib/inference/vllm-station-ssh-binding.test-support.ts @@ -53,13 +53,3 @@ export function createDualStationSshBindingFixture( cleanup: () => fs.rmSync(root, { recursive: true, force: true }), }; } - -export function retargetDualStationSshBindingFixture( - fixture: DualStationSshBindingFixture, - peerTarget: string, - enabled = true, -): DualStationSshBindingFixture { - if (!enabled || fixture.binding.peerTarget === peerTarget) return fixture; - fixture.cleanup(); - return createDualStationSshBindingFixture(peerTarget); -} diff --git a/test/support/vllm-station-model-staging-test-support.ts b/test/support/vllm-station-model-staging-test-support.ts index 3cf767330b..f4cbae11be 100644 --- a/test/support/vllm-station-model-staging-test-support.ts +++ b/test/support/vllm-station-model-staging-test-support.ts @@ -7,8 +7,6 @@ import path from "node:path"; import { vi } from "vitest"; -import { resolveDualStationSimulationFixturePython } from "../../scripts/simulate-dual-station.mts"; - import type { ModelStagingCommandOptions, ModelStagingCommandResult, @@ -24,11 +22,27 @@ function result(stdout = "", status = 0): ModelStagingCommandResult { return { status, stdout, stderr: "" }; } +function resolveFixturePython(): string { + for (const directory of (process.env.PATH ?? "").split(path.delimiter)) { + if (!directory || !path.isAbsolute(directory) || path.normalize(directory) !== directory) { + continue; + } + try { + const candidate = fs.realpathSync(path.join(directory, "python3")); + fs.accessSync(candidate, fs.constants.X_OK); + if (fs.statSync(candidate).isFile()) return candidate; + } catch { + // Keep searching PATH for an executable fixture interpreter. + } + } + throw new Error("python3 is required for the Station model-staging fixtures"); +} + function runPython( args: readonly string[], options: ModelStagingCommandOptions, ): ModelStagingCommandResult { - const completed = spawnSync(resolveDualStationSimulationFixturePython(), [...args], { + const completed = spawnSync(resolveFixturePython(), [...args], { encoding: "utf8", env: options.env, input: options.input, From 829855b11311ccd554e1b465e6d7f6f1ee27c975 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 21 Jul 2026 16:07:26 -0700 Subject: [PATCH 48/74] test(vllm): satisfy fixture growth guard Signed-off-by: Aaron Erickson --- .../inference/vllm-station-cluster.test.ts | 38 +++++++------------ .../vllm-station-fixture.test-support.ts | 21 ++++++++++ .../vllm-station-ssh-binding.test-support.ts | 10 +++++ ...vllm-station-model-staging-test-support.ts | 20 ++-------- 4 files changed, 47 insertions(+), 42 deletions(-) create mode 100644 src/lib/inference/vllm-station-fixture.test-support.ts diff --git a/src/lib/inference/vllm-station-cluster.test.ts b/src/lib/inference/vllm-station-cluster.test.ts index 32f0cc1683..6d79ab45db 100644 --- a/src/lib/inference/vllm-station-cluster.test.ts +++ b/src/lib/inference/vllm-station-cluster.test.ts @@ -28,28 +28,14 @@ import { } from "./vllm-station-ssh-binding"; import { createDualStationSshBindingFixture, + retargetDualStationSshBindingFixture, type DualStationSshBindingFixture, } from "./vllm-station-ssh-binding.test-support"; +import { resolveStationFixturePython } from "./vllm-station-fixture.test-support"; const LOCAL_HOME = "/home/local"; const PEER_HOME = "/home/nvidia"; -function resolveFixturePython(): string { - for (const directory of (process.env.PATH ?? "").split(path.delimiter)) { - if (!directory || !path.isAbsolute(directory) || path.normalize(directory) !== directory) { - continue; - } - try { - const candidate = fs.realpathSync(path.join(directory, "python3")); - fs.accessSync(candidate, fs.constants.X_OK); - if (fs.statSync(candidate).isFile()) return candidate; - } catch { - // Keep searching PATH for an executable fixture interpreter. - } - } - throw new Error("python3 is required for the Station host-probe fixtures"); -} - function strictDockerSshConfig(binding: DualStationSshBinding): string { return [ `hostname ${binding.resolvedHost}`, @@ -246,10 +232,11 @@ function fixtureDeps( } function runWith(deps: StationClusterProbeDeps, target = "nvidia@station-b") { - if (validatePeerTarget(target).ok && sshFixture.binding.peerTarget !== target) { - sshFixture.cleanup(); - sshFixture = createDualStationSshBindingFixture(target); - } + sshFixture = retargetDualStationSshBindingFixture( + sshFixture, + target, + validatePeerTarget(target).ok, + ); return probeDualStationVllmCapability({ env: { [NEMOCLAW_DGX_STATION_PEER_ENV]: target, @@ -385,9 +372,10 @@ describe("probeDualStationVllmCapability", () => { peerModelSnapshot: "ready", plan: { peerSshBinding: { peerTarget: target } }, }); - if (result.kind !== "ready") throw new Error("expected ready fixture"); - expect(buildRemoteVllmDockerEnv(result.plan.peerSshBinding, {}).DOCKER_HOST).toBe( - `ssh://${result.plan.peerSshBinding.sshUser}@${result.plan.peerSshBinding.resolvedHost}`, + expect(result.kind).toBe("ready"); + const ready = result as Extract; + expect(buildRemoteVllmDockerEnv(ready.plan.peerSshBinding, {}).DOCKER_HOST).toBe( + `ssh://${ready.plan.peerSshBinding.sshUser}@${ready.plan.peerSshBinding.resolvedHost}`, ); }); @@ -884,7 +872,7 @@ describe("probe command boundary", () => { }, ); createStationClusterProbeDeps(recordingSpawn).probeLocalHost(); - const python = resolveFixturePython(); + const python = resolveStationFixturePython(); try { const executed = spawnSync(python, ["-"], { @@ -925,7 +913,7 @@ describe("probe command boundary", () => { }, ); createStationClusterProbeDeps(recordingSpawn).probeLocalHost(); - const python = resolveFixturePython(); + const python = resolveStationFixturePython(); const fixturePrelude = String.raw` import pathlib import stat as fixture_stat diff --git a/src/lib/inference/vllm-station-fixture.test-support.ts b/src/lib/inference/vllm-station-fixture.test-support.ts new file mode 100644 index 0000000000..d3d335efa1 --- /dev/null +++ b/src/lib/inference/vllm-station-fixture.test-support.ts @@ -0,0 +1,21 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import path from "node:path"; + +export function resolveStationFixturePython(): string { + for (const directory of (process.env.PATH ?? "").split(path.delimiter)) { + if (!directory || !path.isAbsolute(directory) || path.normalize(directory) !== directory) { + continue; + } + try { + const candidate = fs.realpathSync(path.join(directory, "python3")); + fs.accessSync(candidate, fs.constants.X_OK); + if (fs.statSync(candidate).isFile()) return candidate; + } catch { + // Keep searching PATH for an executable fixture interpreter. + } + } + throw new Error("python3 is required for the Station fixtures"); +} diff --git a/src/lib/inference/vllm-station-ssh-binding.test-support.ts b/src/lib/inference/vllm-station-ssh-binding.test-support.ts index 4213d50dcb..b7cd16ce35 100644 --- a/src/lib/inference/vllm-station-ssh-binding.test-support.ts +++ b/src/lib/inference/vllm-station-ssh-binding.test-support.ts @@ -53,3 +53,13 @@ export function createDualStationSshBindingFixture( cleanup: () => fs.rmSync(root, { recursive: true, force: true }), }; } + +export function retargetDualStationSshBindingFixture( + fixture: DualStationSshBindingFixture, + peerTarget: string, + validTarget: boolean, +): DualStationSshBindingFixture { + if (!validTarget || fixture.binding.peerTarget === peerTarget) return fixture; + fixture.cleanup(); + return createDualStationSshBindingFixture(peerTarget); +} diff --git a/test/support/vllm-station-model-staging-test-support.ts b/test/support/vllm-station-model-staging-test-support.ts index f4cbae11be..ee9f21cf7b 100644 --- a/test/support/vllm-station-model-staging-test-support.ts +++ b/test/support/vllm-station-model-staging-test-support.ts @@ -7,6 +7,8 @@ import path from "node:path"; import { vi } from "vitest"; +import { resolveStationFixturePython } from "../../src/lib/inference/vllm-station-fixture.test-support"; + import type { ModelStagingCommandOptions, ModelStagingCommandResult, @@ -22,27 +24,11 @@ function result(stdout = "", status = 0): ModelStagingCommandResult { return { status, stdout, stderr: "" }; } -function resolveFixturePython(): string { - for (const directory of (process.env.PATH ?? "").split(path.delimiter)) { - if (!directory || !path.isAbsolute(directory) || path.normalize(directory) !== directory) { - continue; - } - try { - const candidate = fs.realpathSync(path.join(directory, "python3")); - fs.accessSync(candidate, fs.constants.X_OK); - if (fs.statSync(candidate).isFile()) return candidate; - } catch { - // Keep searching PATH for an executable fixture interpreter. - } - } - throw new Error("python3 is required for the Station model-staging fixtures"); -} - function runPython( args: readonly string[], options: ModelStagingCommandOptions, ): ModelStagingCommandResult { - const completed = spawnSync(resolveFixturePython(), [...args], { + const completed = spawnSync(resolveStationFixturePython(), [...args], { encoding: "utf8", env: options.env, input: options.input, From 39138da67e7b98440def2bcd27d7711d00548277 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 21 Jul 2026 16:17:03 -0700 Subject: [PATCH 49/74] fix(station): tighten preparation prerequisites Signed-off-by: Aaron Erickson --- scripts/prepare-dgx-station-host.sh | 1 + src/lib/onboard/setup-nim-flow.test.ts | 2 +- src/lib/onboard/setup-nim-flow.ts | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/prepare-dgx-station-host.sh b/scripts/prepare-dgx-station-host.sh index 6f464b9b0b..086feaac60 100755 --- a/scripts/prepare-dgx-station-host.sh +++ b/scripts/prepare-dgx-station-host.sh @@ -2486,6 +2486,7 @@ run_apply() { require_command apt-cache require_command apt-get + require_command cmp require_command curl require_command dpkg require_command gpg diff --git a/src/lib/onboard/setup-nim-flow.test.ts b/src/lib/onboard/setup-nim-flow.test.ts index 3e7bd9d664..97edbbbd9d 100644 --- a/src/lib/onboard/setup-nim-flow.test.ts +++ b/src/lib/onboard/setup-nim-flow.test.ts @@ -900,7 +900,7 @@ describe("createSetupNim", () => { }), ); - await expect(setupNim(null)).rejects.toThrow("vLLM is already running on the managed endpoint"); + await expect(setupNim(null)).rejects.toThrow("vLLM is already running on this host"); expect(error).toHaveBeenCalledWith(expect.stringContaining("Select Local vLLM")); expect(error).toHaveBeenCalledWith(expect.stringContaining("stop the existing server")); diff --git a/src/lib/onboard/setup-nim-flow.ts b/src/lib/onboard/setup-nim-flow.ts index 23ffa257fe..5bd16fb46f 100644 --- a/src/lib/onboard/setup-nim-flow.ts +++ b/src/lib/onboard/setup-nim-flow.ts @@ -567,7 +567,7 @@ export function createSetupNim( } if (vllmRunning) { const message = - "vLLM is already running on the managed endpoint. " + + "vLLM is already running on this host. " + "Select Local vLLM, or stop the existing server before selecting the managed install path."; deps.error(` ${message}`); if (deps.isNonInteractive()) { From 85371981fc9c5dac7d42ba93587a707141104712 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 21 Jul 2026 16:35:32 -0700 Subject: [PATCH 50/74] fix(installer): retain incomplete Station pair state Signed-off-by: Aaron Erickson --- scripts/checks/vitest-project-overlap.mts | 1 + scripts/install.sh | 12 +++-- test/install-station-resume-cleanup.test.ts | 59 +++++++++++++++++++++ test/test-boundary-guards.test.ts | 1 + vitest.config.ts | 2 + 5 files changed, 71 insertions(+), 4 deletions(-) create mode 100644 test/install-station-resume-cleanup.test.ts diff --git a/scripts/checks/vitest-project-overlap.mts b/scripts/checks/vitest-project-overlap.mts index 50415d515c..b612b0586d 100644 --- a/scripts/checks/vitest-project-overlap.mts +++ b/scripts/checks/vitest-project-overlap.mts @@ -48,6 +48,7 @@ const INSTALLER_INTEGRATION_TESTS = new Set([ "test/install-preflight.test.ts", "test/install-station-controller-binding.test.ts", "test/install-station-pair-preparation.test.ts", + "test/install-station-resume-cleanup.test.ts", "test/install-station-dgx-os.test.ts", "test/install-station-docker-repository.test.ts", "test/install-station-host-preparation.test.ts", diff --git a/scripts/install.sh b/scripts/install.sh index f1ae3e26bc..62c271bd2d 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -4486,10 +4486,14 @@ main() { fi finalize_install - if [[ "${_SELECTED_EXPRESS_PLATFORM:-}" == "DGX Station" ]]; then - clear_station_dual_pair_resume - clear_station_express_resume - fi + clear_station_resume_after_completed_onboarding +} + +clear_station_resume_after_completed_onboarding() { + [[ "${_SELECTED_EXPRESS_PLATFORM:-}" == "DGX Station" ]] || return 0 + [[ "${ONBOARD_RAN:-false}" == true ]] || return 0 + clear_station_dual_pair_resume + clear_station_express_resume } # Print the completion summary, then propagate a fatal/non-zero result when the diff --git a/test/install-station-resume-cleanup.test.ts b/test/install-station-resume-cleanup.test.ts new file mode 100644 index 0000000000..4cf42a4cb8 --- /dev/null +++ b/test/install-station-resume-cleanup.test.ts @@ -0,0 +1,59 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { INSTALLER_PAYLOAD, TEST_SYSTEM_PATH } from "./helpers/installer-sourced-env"; + +describe("DGX Station installer resume cleanup", () => { + it("preserves pair and SSH-binding state when interactive host preflight skips onboarding", () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-station-resume-cleanup-")); + const result = spawnSync( + "bash", + [ + "--noprofile", + "--norc", + "-c", + ` +source "$INSTALLER_UNDER_TEST" >/dev/null +pair_state="$HOME/.nemoclaw/station-dual-pair-resume.json" +binding_state="\${pair_state}.ssh-binding" +mkdir -p "$binding_state" +printf '{}\n' >"$pair_state" +printf 'binding\n' >"$binding_state/token" +printf 'resume\n' >"$HOME/.nemoclaw/station-express-resume" +_SELECTED_EXPRESS_PLATFORM='DGX Station' +ONBOARD_RAN=false +clear_station_resume_after_completed_onboarding +printf 'PAIR=%s BINDING=%s EXPRESS=%s\n' \ + "$([ -f "$pair_state" ] && printf present)" \ + "$([ -f "$binding_state/token" ] && printf present)" \ + "$([ -f "$HOME/.nemoclaw/station-express-resume" ] && printf present)" +`, + ], + { + cwd: path.resolve(import.meta.dirname, ".."), + encoding: "utf8", + env: { + ...process.env, + HOME: home, + INSTALLER_UNDER_TEST: INSTALLER_PAYLOAD, + PATH: TEST_SYSTEM_PATH, + }, + }, + ); + + try { + const output = `${result.stdout}${result.stderr}`; + expect(result.status, output).toBe(0); + expect(output).toContain("PAIR=present BINDING=present EXPRESS=present"); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + }); +}); diff --git a/test/test-boundary-guards.test.ts b/test/test-boundary-guards.test.ts index b1f63e87c5..e76bd8a953 100644 --- a/test/test-boundary-guards.test.ts +++ b/test/test-boundary-guards.test.ts @@ -755,6 +755,7 @@ describe("Vitest project membership boundary", () => { ["test/install-preflight.test.ts", "installer-integration"], ["test/install-station-controller-binding.test.ts", "installer-integration"], ["test/install-station-pair-preparation.test.ts", "installer-integration"], + ["test/install-station-resume-cleanup.test.ts", "installer-integration"], ["test/install-station-dgx-os.test.ts", "installer-integration"], ["test/install-station-docker-repository.test.ts", "installer-integration"], ["test/install-station-host-preparation.test.ts", "installer-integration"], diff --git a/vitest.config.ts b/vitest.config.ts index 7366c3592a..4b5b1b27ed 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -126,6 +126,7 @@ export default defineConfig({ "test/install-preflight-docker-bootstrap.test.ts", "test/install-station-controller-binding.test.ts", "test/install-station-pair-preparation.test.ts", + "test/install-station-resume-cleanup.test.ts", "test/install-station-dgx-os.test.ts", "test/install-station-docker-repository.test.ts", "test/install-station-host-preparation.test.ts", @@ -151,6 +152,7 @@ export default defineConfig({ "test/install-preflight-docker-bootstrap.test.ts", "test/install-station-controller-binding.test.ts", "test/install-station-pair-preparation.test.ts", + "test/install-station-resume-cleanup.test.ts", "test/install-station-dgx-os.test.ts", "test/install-station-docker-repository.test.ts", "test/install-station-host-preparation.test.ts", From 131129a176f79eaa3b62f11f5719bb68d7a647e6 Mon Sep 17 00:00:00 2001 From: Senthil Ravichandran Date: Sat, 25 Jul 2026 15:24:01 -0700 Subject: [PATCH 51/74] fix(vllm): harden managed Station recovery Signed-off-by: Senthil Ravichandran --- docs/get-started/dgx-station-preparation.mdx | 4 +++- docs/inference/set-up-vllm.mdx | 3 ++- docs/reference/commands.mdx | 2 +- docs/resources/prompt-assets/dgx-station.md | 4 ++-- src/lib/inference/local.test.ts | 11 +++++++++++ src/lib/inference/local.ts | 15 ++++++++++++--- src/lib/inference/vllm-api-key.test.ts | 20 +++++++++++++++++++- src/lib/inference/vllm-api-key.ts | 5 ++++- src/lib/onboard/provider-host-state.test.ts | 12 ++++++++++++ src/lib/onboard/provider-host-state.ts | 11 ++++++++--- src/lib/onboard/setup-nim-vllm.test.ts | 17 +++++++++++++++++ src/lib/onboard/setup-nim-vllm.ts | 2 +- 12 files changed, 92 insertions(+), 14 deletions(-) diff --git a/docs/get-started/dgx-station-preparation.mdx b/docs/get-started/dgx-station-preparation.mdx index 54d0b26a05..76c57aa172 100644 --- a/docs/get-started/dgx-station-preparation.mdx +++ b/docs/get-started/dgx-station-preparation.mdx @@ -181,7 +181,9 @@ NemoClaw checks only the two deterministic `/30` counterpart addresses; it does Station preparation binds the preparing non-root account's UID in root-owned `/etc/nemoclaw/dual-station-controller-uid`. Rebinding requires an administrator to remove that file before preparation is rerun as the replacement account. If either Station requires a reboot, the installer stops with status `10` and names the host to reboot manually. -Owner-only resume state binds the exact NemoClaw revision, preparation helper, SSH host key, GPU identities, and reciprocal rails; the installer never reboots either host automatically. +The initial local-host receipt preserves the accepted NemoClaw revision and Express selections. +After reciprocal peer qualification begins, owner-only pair state additionally binds the preparation helper, SSH host key, GPU identities, and reciprocal rails. +The installer never reboots either host automatically. The dual-Station runtime uses unauthenticated Ray, NCCL, and vLLM coordination traffic, including the Ray head on TCP port `6379` and Ray's worker traffic. diff --git a/docs/inference/set-up-vllm.mdx b/docs/inference/set-up-vllm.mdx index 88fc71cda7..1a0be80ba8 100644 --- a/docs/inference/set-up-vllm.mdx +++ b/docs/inference/set-up-vllm.mdx @@ -211,7 +211,8 @@ Without terminal access, the installer stops before it installs Docker or build The registered single-Station Ultra recipe tracks the [official DGX Station deployment guide](https://github.com/NVIDIA-NeMo/Nemotron/blob/287ae845639d2ce998998cb8fd1f70a3fa943c0b/usage-cookbook/Nemotron-3-Ultra/StationDeploymentGuide/README.md) and configures the pinned model revision, CPU offload, `16 GB` of shared memory, memory/stack ulimits, MTP speculative decoding, and the Nemotron reasoning and tool-call parsers. Configure the physical rails and SSH host-key and authentication trust before installation. NemoClaw does not modify rail configuration, enroll SSH trust, or reboot either host automatically. -If either host requires a reboot after pinned prerequisite changes, the installer stops with status `10` and resumes only after the manual reboot with owner-only state tied to the printed exact revision and reciprocal identity. +If the local host requires a reboot during initial preparation, the installer stops with status `10`; its owner-only receipt preserves the printed exact revision and Express selections. +After reciprocal peer qualification begins, the pair state additionally binds SSH, GPU, and rail identity and names either host that must be rebooted manually. The registered two-Station Ultra recipe tracks the [latest NVIDIA dual-Station playbook](https://build.nvidia.com/station/nemoclaw/dual-nodes). It pins vLLM `0.25.1` and Ray `2.56.0`, uses one tensor-parallel rank per Station with pipeline parallelism across the pair, serves the `nemotron-ultra` alias with a `262144`-token model limit, and retains the Nemotron reasoning and tool-call parsers. The single-Station fallback keeps NemoClaw's bridge-networked managed-inference topology and publishes port `8000` through Docker. diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 62fd0fd255..2305734653 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -3490,7 +3490,7 @@ Set them before running `$$nemoclaw onboard`. | `SANDBOX_NAME` | sandbox name | Compatibility spelling used after `NEMOCLAW_SANDBOX_NAME` and `NEMOCLAW_SANDBOX`. | | `NEMOCLAW_INSTALL_REF` | git ref | For internal installer commands: the git ref to install from. A nonempty value takes precedence over `NEMOCLAW_INSTALL_TAG`. Overridden by the `--install-ref` flag. | | `NEMOCLAW_INSTALL_TAG` | release tag | For internal installer commands: the release tag to install when `NEMOCLAW_INSTALL_REF` is unset or empty. Defaults to the admin-promoted `lkg` tag when unset. Overridden by the `--install-tag` flag. | -| `NEMOCLAW_VLLM_MODEL` | registry slug or Hugging Face model id | Selects the model the managed-vLLM install path serves and remains authoritative during DGX Station installer setup. Recognised slugs: `qwen3.6-27b`, `qwen3.6-35b-a3b-nvfp4`, `nemotron-3-nano-4b`, `deepseek-v4-flash`, `nemotron-3-ultra-550b-a55b`, `deepseek-r1-distill-70b`. Unset uses the per-platform profile default, except that DGX Station installer setup automatically selects `nemotron-3-ultra-550b-a55b` after one pretrusted reciprocal pair qualifies. With both model and peer unset, no qualifying pair leaves the Station `deepseek-v4-flash` default in place. Gated models (e.g. `deepseek-r1-distill-70b`) require `HF_TOKEN` or `HUGGING_FACE_HUB_TOKEN`. | +| `NEMOCLAW_VLLM_MODEL` | registry slug or Hugging Face model id | Selects the model the managed-vLLM install path serves and remains authoritative during DGX Station installer setup. Recognised slugs: `qwen3.6-27b`, `qwen3.6-35b-a3b-nvfp4`, `nemotron-3-nano-4b`, `deepseek-v4-flash`, `nemotron-3-ultra-550b-a55b`, `deepseek-r1-distill-70b`. Station Express selects `nemotron-3-ultra-550b-a55b`; a qualified reciprocal pair uses the distributed topology, while no qualifying pair retains the single-Station Ultra topology. Outside Station Express, unset uses the per-platform profile default. Gated models (e.g. `deepseek-r1-distill-70b`) require `HF_TOKEN` or `HUGGING_FACE_HUB_TOKEN`. | | `NEMOCLAW_DGX_STATION_PEER` | SSH host or `user@host` | Selects one exact, already-trusted DGX Station peer for Nemotron 3 Ultra pair qualification. The peer must match the reciprocal private `/30` rail and hardware checks; an explicit peer failure stops setup instead of falling back. NemoClaw does not enroll SSH trust or accept a port or SSH option in this value. When unset, DGX Station installer discovery checks only the two deterministic `/30` counterpart addresses. A peer cannot be combined with an explicit non-Ultra model; conflicting explicit selections fail before pair preparation. | | `NEMOCLAW_DGX_STATION_SSH_BINDING` | opaque installer-managed token | Carries the qualified peer endpoint and host-key binding from DGX Station pair preparation into the current managed-vLLM install. The installer creates and clears this token; operators should not set or persist it. Missing, changed, or mismatched binding state fails before peer SSH or Docker work. | | `NEMOCLAW_VLLM_EXTRA_ARGS_JSON` | JSON array of non-blank strings | Appends advanced operator-owned tokens to the managed `vllm serve` command after NemoClaw's registry defaults. Example: `["--max-num-seqs","2"]`. Malformed JSON, non-string tokens, or blank tokens fail before Docker work starts. | diff --git a/docs/resources/prompt-assets/dgx-station.md b/docs/resources/prompt-assets/dgx-station.md index 1534cd1100..c910cb5fd9 100644 --- a/docs/resources/prompt-assets/dgx-station.md +++ b/docs/resources/prompt-assets/dgx-station.md @@ -12,7 +12,7 @@ Do not run the Station preparation helper separately or reproduce Express by pre The installer provides these Station Express choices: -1. The ordinary installer checks for one already-trusted peer at the deterministic counterpart on each of two configured private `/30` ConnectX-8 rails. A qualified pair selects `nemotron-3-ultra-550b-a55b`, served as `nemotron-ultra` through the vLLM 0.25.1 and Ray 2.56.0 dual-Station recipe; otherwise it retains the single-Station `deepseek-v4-flash` default, served as `deepseek-ai/DeepSeek-V4-Flash`. +1. The ordinary installer selects `nemotron-3-ultra-550b-a55b`, served as `nemotron-ultra`, and checks for one already-trusted peer at the deterministic counterpart on each of two configured private `/30` ConnectX-8 rails. A qualified pair uses the vLLM 0.25.1 and Ray 2.56.0 dual-Station recipe; otherwise it retains the single-Station Ultra recipe. 2. The explicit `--station-deepseek` flag selects `deepseek-v4-flash`, served as `deepseek-ai/DeepSeek-V4-Flash`. Both choices use the same Station detection, host-preparation, consent, suggested-policy, default-sandbox, and revision resume flow. @@ -31,7 +31,7 @@ Before asking for consent, explain all of these boundaries: Ask: "Which DGX Station Express option would you like?" Choices: -1. Automatic pair selection: use Nemotron 3 Ultra 550B only if a trusted two-Station pair qualifies; otherwise use single-Station DeepSeek V4 Flash. +1. Automatic pair selection: use Nemotron 3 Ultra 550B with a qualified trusted pair when available; otherwise use the single-Station Ultra recipe. 2. DeepSeek V4 Flash, the explicit `--station-deepseek` override. 3. Neither, let me choose the runtime and model normally. diff --git a/src/lib/inference/local.test.ts b/src/lib/inference/local.test.ts index 8cb4b6e471..80fd021069 100644 --- a/src/lib/inference/local.test.ts +++ b/src/lib/inference/local.test.ts @@ -31,6 +31,7 @@ import { getOllamaModelOptions, getOllamaProbeCommand, getOllamaWarmupCommand, + isLocalProviderProbeOutputHealthy, isOllamaRunnerCrash, LOCAL_INFERENCE_SANDBOX_HOST_URL_ENV, parseOllamaList, @@ -175,6 +176,16 @@ describe("local inference helpers", () => { ]); }); + it("requires HTTP 200 for managed health output and rejects curl connection status 000", () => { + expect(isLocalProviderProbeOutputHealthy("http://10.40.0.1:8000/health", "200")).toBe(true); + expect(isLocalProviderProbeOutputHealthy("http://10.40.0.1:8000/health", "204")).toBe(false); + expect(isLocalProviderProbeOutputHealthy("http://10.40.0.1:8000/health", "000")).toBe(false); + expect(isLocalProviderProbeOutputHealthy("http://127.0.0.1:8000/v1/models", "000")).toBe(false); + expect( + isLocalProviderProbeOutputHealthy("http://127.0.0.1:8000/v1/models", '{"data":[]}'), + ).toBe(true); + }); + it("validates a reachable local provider", () => { let callCount = 0; const mockCapture = () => { diff --git a/src/lib/inference/local.ts b/src/lib/inference/local.ts index a6428dc285..c9c29d970b 100644 --- a/src/lib/inference/local.ts +++ b/src/lib/inference/local.ts @@ -556,6 +556,12 @@ export function getLocalProviderAvailabilityEndpoint(provider: string): string | return getLocalProviderHealthEndpoint(provider); } +export function isLocalProviderProbeOutputHealthy(endpoint: string, output: string): boolean { + const normalized = output.trim(); + if (!normalized || normalized === "000") return false; + return endpoint.endsWith("/health") ? normalized === "200" : true; +} + export function getLocalProviderHealthCheck(provider: string): string[] | null { const endpoint = getLocalProviderAvailabilityEndpoint(provider); if (provider === "vllm-local" && endpoint?.endsWith("/health")) { @@ -593,7 +599,10 @@ export function isLocalProviderHostHealthy( const command = getLocalProviderHealthCheck(provider); if (!command) return false; const capture = runCaptureImpl ?? runCapture; - return Boolean(capture(command, { ignoreError: true })); + return isLocalProviderProbeOutputHealthy( + command.at(-1) ?? "", + capture(command, { ignoreError: true }), + ); } export function getLocalProviderLabel(provider: string): string | null { @@ -954,7 +963,7 @@ export function validateLocalProvider( } const output = capture(command, { ignoreError: true }); - if (!output) { + if (!isLocalProviderProbeOutputHealthy(command.at(-1) ?? "", output)) { switch (provider) { case "vllm-local": return { @@ -979,7 +988,7 @@ export function validateLocalProvider( // Retry container reachability check with backoff for (let attempt = 1; attempt <= CONTAINER_CHECK_MAX_ATTEMPTS; attempt++) { const containerOutput = capture(containerCommand, { ignoreError: true }); - if (containerOutput) { + if (isLocalProviderProbeOutputHealthy(containerCommand.at(-1) ?? "", containerOutput)) { return { ok: true }; } if (attempt < CONTAINER_CHECK_MAX_ATTEMPTS) { diff --git a/src/lib/inference/vllm-api-key.test.ts b/src/lib/inference/vllm-api-key.test.ts index 6b550c63d3..ac64880640 100644 --- a/src/lib/inference/vllm-api-key.test.ts +++ b/src/lib/inference/vllm-api-key.test.ts @@ -5,7 +5,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { afterEach, describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import { dualStationVllmApiKeyPath, @@ -22,6 +22,8 @@ function temporaryDirectory(): string { } afterEach(() => { + vi.doUnmock("node:fs"); + vi.resetModules(); for (const directory of temporaryDirectories.splice(0)) { fs.rmSync(directory, { force: true, recursive: true }); } @@ -70,4 +72,20 @@ describe("dual-Station vLLM API key persistence", () => { expect(() => loadDualStationVllmApiKey({ stateDir })).toThrow("symbolic link"); expect(() => ensureDualStationVllmApiKey({ stateDir })).toThrow("symbolic link"); }); + + it("fails closed when secure no-follow opens are unavailable", async () => { + vi.resetModules(); + vi.doMock("node:fs", async (importOriginal) => { + const actual = await importOriginal(); + const constants = { ...actual.constants, O_NOFOLLOW: undefined }; + return { ...actual, constants, default: { ...actual.default, constants } }; + }); + const unavailable = await import("./vllm-api-key"); + + expect(() => + unavailable.ensureDualStationVllmApiKey({ + stateDir: path.join(temporaryDirectory(), "state"), + }), + ).toThrow("Secure no-follow file opens are unavailable"); + }); }); diff --git a/src/lib/inference/vllm-api-key.ts b/src/lib/inference/vllm-api-key.ts index f8f44fa3ff..0eda887c52 100644 --- a/src/lib/inference/vllm-api-key.ts +++ b/src/lib/inference/vllm-api-key.ts @@ -96,7 +96,10 @@ export function ensureDualStationVllmApiKey(options: DualStationVllmApiKeyOption } const filePath = dualStationVllmApiKeyPath(stateDir); - const noFollow = fs.constants.O_NOFOLLOW ?? 0; + const noFollow = fs.constants.O_NOFOLLOW; + if (typeof noFollow !== "number") { + throw new Error("Secure no-follow file opens are unavailable on this platform"); + } let fd: number | undefined; try { fd = fs.openSync( diff --git a/src/lib/onboard/provider-host-state.test.ts b/src/lib/onboard/provider-host-state.test.ts index 794179a35f..82c995869a 100644 --- a/src/lib/onboard/provider-host-state.test.ts +++ b/src/lib/onboard/provider-host-state.test.ts @@ -148,6 +148,18 @@ describe("detectInferenceProviderHostState", () => { ); }); + it("does not treat curl connection status 000 as a running vLLM", () => { + const state = detectWithDeps( + buildDeps({ + runCapture: vi.fn((command) => + command.join(" ").includes("127.0.0.1:8000/v1/models") ? "000" : "", + ), + }), + ); + + expect(state.vllmRunning).toBe(false); + }); + it("detects a reachable Windows-host Ollama beside WSL-local Ollama and warns outside mirrored networking", () => { const logs: string[] = []; const deps = buildDeps({ diff --git a/src/lib/onboard/provider-host-state.ts b/src/lib/onboard/provider-host-state.ts index 64b274dd67..4a41d5a8f8 100644 --- a/src/lib/onboard/provider-host-state.ts +++ b/src/lib/onboard/provider-host-state.ts @@ -6,6 +6,7 @@ import { OLLAMA_PORT } from "../core/ports"; import { findReachableOllamaHost, getLocalProviderAvailabilityEndpoint, + isLocalProviderProbeOutputHealthy, OLLAMA_HOST_DOCKER_INTERNAL, } from "../inference/local"; import type { NvidiaPlatform } from "../inference/nim"; @@ -121,9 +122,13 @@ function probeVllmRunning(runCapture: RunCapture): boolean { const writeOut = endpoint.endsWith("/health") ? ["--noproxy", "*", "--write-out", "%{http_code}"] : []; - return !!runCapture(["curl", "-sf", ...LOCAL_PROVIDER_PROBE_CURL_ARGS, ...writeOut, endpoint], { - ignoreError: true, - }); + const output = runCapture( + ["curl", "-sf", ...LOCAL_PROVIDER_PROBE_CURL_ARGS, ...writeOut, endpoint], + { + ignoreError: true, + }, + ); + return isLocalProviderProbeOutputHealthy(endpoint, output); } function probeWindowsOllamaReachable(input: { diff --git a/src/lib/onboard/setup-nim-vllm.test.ts b/src/lib/onboard/setup-nim-vllm.test.ts index c2c825deb4..bc7682d0ea 100644 --- a/src/lib/onboard/setup-nim-vllm.test.ts +++ b/src/lib/onboard/setup-nim-vllm.test.ts @@ -176,6 +176,23 @@ describe("setupNim vLLM route containment", () => { ); }); + it("treats an unexpected undefined managed binding as absent", async () => { + const runCapture = vi.fn(() => JSON.stringify({ data: [{ id: "served/model" }] })); + const queryVllmModels = vi.fn(() => ""); + const handler = createSetupNimVllmHandler( + deps({ + runCapture, + getManagedVllmProviderBinding: () => undefined as never, + queryVllmModels, + }), + ); + + await expect(handler(state(null))).resolves.toBe("selected"); + expect(runCapture).toHaveBeenCalled(); + expect(queryVllmModels).not.toHaveBeenCalled(); + expect(console.log).toHaveBeenCalledWith(" ✓ Using existing vLLM on localhost:8000"); + }); + it("fails closed before endpoint probes when managed auth state is unsafe", async () => { const queryVllmModels = vi.fn(() => ""); const validateOpenAiLikeSelection = vi.fn(async () => ({ ok: true })); diff --git a/src/lib/onboard/setup-nim-vllm.ts b/src/lib/onboard/setup-nim-vllm.ts index b593446899..a996156c76 100644 --- a/src/lib/onboard/setup-nim-vllm.ts +++ b/src/lib/onboard/setup-nim-vllm.ts @@ -247,7 +247,7 @@ export function createSetupNimVllmHandler( } const apiKey = managedBinding?.apiKey ?? null; - const managedDualEndpoint = managedBinding !== null; + const managedDualEndpoint = managedBinding != null; console.log( managedDualEndpoint ? " ✓ Using managed dual-Station vLLM endpoint" From fff4087865221213a76e4f47a3a87fb7c5f272ad Mon Sep 17 00:00:00 2001 From: Senthil Ravichandran Date: Sat, 25 Jul 2026 15:27:20 -0700 Subject: [PATCH 52/74] docs(station): repin corrected prompt asset Signed-off-by: Senthil Ravichandran --- docs/resources/starter-prompt.md | 6 +++--- src/lib/inference/vllm-api-key.test.ts | 3 ++- test/starter-prompt-docs.test.ts | 4 ++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/docs/resources/starter-prompt.md b/docs/resources/starter-prompt.md index 6473e5ff62..786546ae72 100644 --- a/docs/resources/starter-prompt.md +++ b/docs/resources/starter-prompt.md @@ -79,9 +79,9 @@ Set `NEMOCLAW_AGENT=langchain-deepagents-code` for Deep Agents, or use `nemo-dee After the readiness check, load exactly one matching instruction asset before provider selection: -- Confirmed DGX Spark: [DGX Spark Express instructions](https://raw.githubusercontent.com/NVIDIA/NemoClaw/962f80ced8918e823ab5355f5056d729ce9232c1/docs/resources/prompt-assets/dgx-spark.md). -- Confirmed DGX Station: [DGX Station installation instructions](https://raw.githubusercontent.com/NVIDIA/NemoClaw/962f80ced8918e823ab5355f5056d729ce9232c1/docs/resources/prompt-assets/dgx-station.md). -- Officially detected Windows WSL: [Windows WSL Express instructions](https://raw.githubusercontent.com/NVIDIA/NemoClaw/962f80ced8918e823ab5355f5056d729ce9232c1/docs/resources/prompt-assets/windows-wsl.md). +- Confirmed DGX Spark: [DGX Spark Express instructions](https://raw.githubusercontent.com/NVIDIA/NemoClaw/131129a176f79eaa3b62f11f5719bb68d7a647e6/docs/resources/prompt-assets/dgx-spark.md). +- Confirmed DGX Station: [DGX Station installation instructions](https://raw.githubusercontent.com/NVIDIA/NemoClaw/131129a176f79eaa3b62f11f5719bb68d7a647e6/docs/resources/prompt-assets/dgx-station.md). +- Officially detected Windows WSL: [Windows WSL Express instructions](https://raw.githubusercontent.com/NVIDIA/NemoClaw/131129a176f79eaa3b62f11f5719bb68d7a647e6/docs/resources/prompt-assets/windows-wsl.md). Read the matching raw Markdown file completely and follow it before continuing. Do not load a platform asset for any other computer. diff --git a/src/lib/inference/vllm-api-key.test.ts b/src/lib/inference/vllm-api-key.test.ts index ac64880640..f302d4fef0 100644 --- a/src/lib/inference/vllm-api-key.test.ts +++ b/src/lib/inference/vllm-api-key.test.ts @@ -78,7 +78,8 @@ describe("dual-Station vLLM API key persistence", () => { vi.doMock("node:fs", async (importOriginal) => { const actual = await importOriginal(); const constants = { ...actual.constants, O_NOFOLLOW: undefined }; - return { ...actual, constants, default: { ...actual.default, constants } }; + const actualDefault = (actual as { default?: typeof fs }).default ?? actual; + return { ...actual, constants, default: { ...actualDefault, constants } }; }); const unavailable = await import("./vllm-api-key"); diff --git a/test/starter-prompt-docs.test.ts b/test/starter-prompt-docs.test.ts index a8e1739c75..a4c7857101 100644 --- a/test/starter-prompt-docs.test.ts +++ b/test/starter-prompt-docs.test.ts @@ -30,7 +30,7 @@ const repoRoot = path.resolve(__dirname, ".."); const starterPromptMarkdownSource = path.join(repoRoot, "docs", "resources", "starter-prompt.md"); // CI resolves this Git commit and byte-compares its prompt-asset blobs with // the local files. The digests independently assert those same immutable bytes. -const promptAssetRevision = "962f80ced8918e823ab5355f5056d729ce9232c1"; +const promptAssetRevision = "131129a176f79eaa3b62f11f5719bb68d7a647e6"; type PromptAsset = { path: string; @@ -53,7 +53,7 @@ const promptAssets = { ), dgxStation: definePromptAsset( "docs/resources/prompt-assets/dgx-station.md", - "eed210be783c39080b38c54d4aa158ea825333c3d92b8dcc2244dae8a633554a", // gitleaks:allow -- pinned prompt-asset SHA-256 + "609462094721020de6d9968fb8e9fe1e8347e60fdbace1611583bd44faae1875", // gitleaks:allow -- pinned prompt-asset SHA-256 ), windowsWsl: definePromptAsset( "docs/resources/prompt-assets/windows-wsl.md", From bf46e62f901825f19e570c17f8c870a0eae04fbc Mon Sep 17 00:00:00 2001 From: Senthil Ravichandran Date: Sat, 25 Jul 2026 15:31:56 -0700 Subject: [PATCH 53/74] docs(station): clarify Ultra topology selection Signed-off-by: Senthil Ravichandran --- docs/resources/prompt-assets/dgx-station.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/resources/prompt-assets/dgx-station.md b/docs/resources/prompt-assets/dgx-station.md index c910cb5fd9..3e16fd7066 100644 --- a/docs/resources/prompt-assets/dgx-station.md +++ b/docs/resources/prompt-assets/dgx-station.md @@ -12,7 +12,8 @@ Do not run the Station preparation helper separately or reproduce Express by pre The installer provides these Station Express choices: -1. The ordinary installer selects `nemotron-3-ultra-550b-a55b`, served as `nemotron-ultra`, and checks for one already-trusted peer at the deterministic counterpart on each of two configured private `/30` ConnectX-8 rails. A qualified pair uses the vLLM 0.25.1 and Ray 2.56.0 dual-Station recipe; otherwise it retains the single-Station Ultra recipe. +1. The ordinary installer selects `nemotron-3-ultra-550b-a55b` and checks for one already-trusted peer at the deterministic counterpart on each of two configured private `/30` ConnectX-8 rails. + A qualified pair uses the vLLM 0.25.1 and Ray 2.56.0 dual-Station recipe served as `nemotron-ultra`; otherwise it retains the single-Station Ultra recipe served as `nvidia/nemotron-3-ultra-550b-a55b`. 2. The explicit `--station-deepseek` flag selects `deepseek-v4-flash`, served as `deepseek-ai/DeepSeek-V4-Flash`. Both choices use the same Station detection, host-preparation, consent, suggested-policy, default-sandbox, and revision resume flow. @@ -47,7 +48,7 @@ If a Station Express model is selected: - Let the installer present its third-party-software notice and complete Express summary. Keep each official confirmation visible, wait for the user's response, and do not pre-answer or suppress it. - Do not pass `--force-station-install` unless the installer rejects release metadata on genuine Station GB300 hardware and the user separately chooses the documented temporary override. - Follow the command that the installer prints after a required reboot or login transition. -- Do not describe Ultra as selected until the installer reports that reciprocal Station, GPU, rail, MAC, route, neighbor, and jumbo-frame checks qualified the pair. +- Describe Ultra as the ordinary Express selection. Describe the distributed two-Station topology as selected only after the installer reports that reciprocal Station, GPU, rail, MAC, route, neighbor, and jumbo-frame checks qualified the pair. If Station Express is declined, continue with the normal provider selection. Offer existing vLLM when a ready server is detected, managed vLLM, supported local Ollama, and every hosted or compatible provider supported by the selected agent. From 976112b76032dbb017bd117f08ac4e7de930f190 Mon Sep 17 00:00:00 2001 From: Senthil Ravichandran Date: Sat, 25 Jul 2026 15:35:27 -0700 Subject: [PATCH 54/74] docs(station): repin Ultra topology prompt Signed-off-by: Senthil Ravichandran --- docs/resources/starter-prompt.md | 6 +++--- test/starter-prompt-docs.test.ts | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/resources/starter-prompt.md b/docs/resources/starter-prompt.md index 786546ae72..257641dd68 100644 --- a/docs/resources/starter-prompt.md +++ b/docs/resources/starter-prompt.md @@ -79,9 +79,9 @@ Set `NEMOCLAW_AGENT=langchain-deepagents-code` for Deep Agents, or use `nemo-dee After the readiness check, load exactly one matching instruction asset before provider selection: -- Confirmed DGX Spark: [DGX Spark Express instructions](https://raw.githubusercontent.com/NVIDIA/NemoClaw/131129a176f79eaa3b62f11f5719bb68d7a647e6/docs/resources/prompt-assets/dgx-spark.md). -- Confirmed DGX Station: [DGX Station installation instructions](https://raw.githubusercontent.com/NVIDIA/NemoClaw/131129a176f79eaa3b62f11f5719bb68d7a647e6/docs/resources/prompt-assets/dgx-station.md). -- Officially detected Windows WSL: [Windows WSL Express instructions](https://raw.githubusercontent.com/NVIDIA/NemoClaw/131129a176f79eaa3b62f11f5719bb68d7a647e6/docs/resources/prompt-assets/windows-wsl.md). +- Confirmed DGX Spark: [DGX Spark Express instructions](https://raw.githubusercontent.com/NVIDIA/NemoClaw/bf46e62f901825f19e570c17f8c870a0eae04fbc/docs/resources/prompt-assets/dgx-spark.md). +- Confirmed DGX Station: [DGX Station installation instructions](https://raw.githubusercontent.com/NVIDIA/NemoClaw/bf46e62f901825f19e570c17f8c870a0eae04fbc/docs/resources/prompt-assets/dgx-station.md). +- Officially detected Windows WSL: [Windows WSL Express instructions](https://raw.githubusercontent.com/NVIDIA/NemoClaw/bf46e62f901825f19e570c17f8c870a0eae04fbc/docs/resources/prompt-assets/windows-wsl.md). Read the matching raw Markdown file completely and follow it before continuing. Do not load a platform asset for any other computer. diff --git a/test/starter-prompt-docs.test.ts b/test/starter-prompt-docs.test.ts index a4c7857101..b42b1ff5b6 100644 --- a/test/starter-prompt-docs.test.ts +++ b/test/starter-prompt-docs.test.ts @@ -30,7 +30,7 @@ const repoRoot = path.resolve(__dirname, ".."); const starterPromptMarkdownSource = path.join(repoRoot, "docs", "resources", "starter-prompt.md"); // CI resolves this Git commit and byte-compares its prompt-asset blobs with // the local files. The digests independently assert those same immutable bytes. -const promptAssetRevision = "131129a176f79eaa3b62f11f5719bb68d7a647e6"; +const promptAssetRevision = "bf46e62f901825f19e570c17f8c870a0eae04fbc"; type PromptAsset = { path: string; @@ -53,7 +53,7 @@ const promptAssets = { ), dgxStation: definePromptAsset( "docs/resources/prompt-assets/dgx-station.md", - "609462094721020de6d9968fb8e9fe1e8347e60fdbace1611583bd44faae1875", // gitleaks:allow -- pinned prompt-asset SHA-256 + "9b620ffe898847718fc25e230039c0bf07dd574e26a27479c45297eb17d2cfa4", // gitleaks:allow -- pinned prompt-asset SHA-256 ), windowsWsl: definePromptAsset( "docs/resources/prompt-assets/windows-wsl.md", From 60ecde1121aff9360ddc6d0b09a68e260d28acfc Mon Sep 17 00:00:00 2001 From: Senthil Ravichandran Date: Sat, 25 Jul 2026 20:16:29 -0700 Subject: [PATCH 55/74] fix(vllm): complete fresh dual Station preparation Signed-off-by: Senthil Ravichandran --- scripts/lib/dgx-station-peer.mts | 17 ++++--- scripts/prepare-dgx-station-host.sh | 17 ++++--- ...tall-station-container-coexistence.test.ts | 48 +++++++++++++++++++ ...install-station-controller-binding.test.ts | 21 ++++++++ test/install-station-pair-preparation.test.ts | 16 +++---- 5 files changed, 99 insertions(+), 20 deletions(-) diff --git a/scripts/lib/dgx-station-peer.mts b/scripts/lib/dgx-station-peer.mts index e2470c585b..74575705ea 100644 --- a/scripts/lib/dgx-station-peer.mts +++ b/scripts/lib/dgx-station-peer.mts @@ -7,6 +7,7 @@ import { stationKnownHostsDigest } from "../../src/lib/inference/vllm-station-ss export const DUAL_STATION_RESUME_SCHEMA_VERSION = 1; export const STATION_PREP_REBOOT_REQUIRED_EXIT = 10; +export const STATION_PREP_LOGIN_REQUIRED_EXIT = 11; const DIRECT_RAIL_PREFIX_LENGTH = 30; const SAFE_TARGET_PATTERN = @@ -843,11 +844,9 @@ export function prepareDualStationPair( ...plan.identity, }; deps.writeResumeState(state); - if (options.reuseExistingManagedPair || options.migrateLegacySingleStationHead) { - deps.log("Binding the local Station controller account without disrupting managed inference"); - if (deps.runLocalHelper("--bind-controller") !== 0) { - throw new Error("Local DGX Station controller UID binding failed"); - } + deps.log("Binding the local Station controller account to the qualified pair"); + if (deps.runLocalHelper("--bind-controller") !== 0) { + throw new Error("Local DGX Station controller UID binding failed"); } if (options.reuseExistingManagedPair) { deps.log("Binding the reciprocal peer controller account without disrupting managed inference"); @@ -880,7 +879,13 @@ export function prepareDualStationPair( }; } if (applyStatus !== 0) { - throw new Error("Peer DGX Station host preparation failed; refusing single-Station fallback"); + if (applyStatus !== STATION_PREP_LOGIN_REQUIRED_EXIT) { + throw new Error("Peer DGX Station host preparation failed; refusing single-Station fallback"); + } + deps.log("Peer Docker access requires a new login; reopening SSH before verification"); + } + if (deps.runRemoteHelper(binding, "--bind-controller") !== 0) { + throw new Error("Peer DGX Station controller UID binding failed"); } if (deps.runRemoteHelper(binding, "--verify") !== 0) { throw new Error("Peer DGX Station verification failed; refusing single-Station fallback"); diff --git a/scripts/prepare-dgx-station-host.sh b/scripts/prepare-dgx-station-host.sh index 04488aacc0..f3e7eafe0c 100755 --- a/scripts/prepare-dgx-station-host.sh +++ b/scripts/prepare-dgx-station-host.sh @@ -1152,17 +1152,25 @@ host_docker_sudo() { } query_host_docker() { - local output + local output allow_sudo=0 DOCKER_QUERY_OUTPUT="" command -v docker >/dev/null 2>&1 || return 2 if output="$(host_docker "$@" 2>/dev/null)"; then DOCKER_QUERY_OUTPUT="$output" return 0 fi - if [[ "$MODE" == "--apply" ]] && output="$(host_docker_sudo "$@" 2>/dev/null)"; then + if [[ "$MODE" == "--apply" || + ("$MODE" == "--check" && "${NEMOCLAW_STATION_PREP_SUDO_NONINTERACTIVE:-0}" == "1") ]]; then + allow_sudo=1 + fi + if ((allow_sudo == 1)) && output="$(host_docker_sudo "$@" 2>/dev/null)"; then DOCKER_QUERY_OUTPUT="$output" if ((DOCKER_QUERY_USES_SUDO == 0)); then - info "docker_access=sudo_until_group_membership_is_active" + if [[ "$MODE" == "--check" ]]; then + info "docker_access=sudo_for_noninteractive_read_only_check" + else + info "docker_access=sudo_until_group_membership_is_active" + fi DOCKER_QUERY_USES_SUDO=1 fi return 0 @@ -2465,7 +2473,6 @@ verify_apply_state() { nvidia-ctk cdi list | grep -Fxq 'nvidia.com/gpu=all' || fatal "CDI verification failed" sudo docker image inspect "$ACCEPTANCE_IMAGE" >/dev/null 2>&1 || fatal "Digest-pinned acceptance image is missing" verify_docker_container_baseline - verify_dual_station_controller_uid_binding info "STATION_HOST_READY" } @@ -2538,7 +2545,6 @@ verify_host() { run_cdi_test_user || fatal "CDI verification did not expose the qualified GB300: ${GPU_ROWS_ERROR}" run_gpus_test_user || fatal "Docker --gpus verification did not expose the qualified GB300: ${GPU_ROWS_ERROR}" verify_docker_container_baseline - verify_dual_station_controller_uid_binding info "docker=$(docker version --format '{{.Server.Version}}') expected_docker=${DOCKER_VERSION} toolkit=$(nvidia-ctk --version | head -n1) expected_toolkit=${TOOLKIT_VERSION}" info "STATION_HOST_READY" } @@ -2653,7 +2659,6 @@ run_verify() { verify_cdi_refresh_lifecycle fi verify_dgx_os_runtime_user - verify_dual_station_controller_uid_binding return 0 fi warn_retained_package_versions diff --git a/test/install-station-container-coexistence.test.ts b/test/install-station-container-coexistence.test.ts index 0c2437af0c..abcd07a669 100644 --- a/test/install-station-container-coexistence.test.ts +++ b/test/install-station-container-coexistence.test.ts @@ -58,6 +58,54 @@ capture_docker_container_baseline expect(output).toContain("docker_container_baseline_total=0 running=0"); }); + it("uses noninteractive sudo only for read-only peer checks before Docker group access", () => { + const { result, output } = runStationPreparation( + ` +MODE='--check' +ps() { printf '%s %s bash bash prepare-dgx-station-host.sh --check\n' "$$" "$PPID"; } +ss() { :; } +docker() { return 1; } +sudo() { + if [[ "$1" == "-n" ]]; then shift; fi + case "$*" in + 'docker ps -aq --no-trunc'|'docker ps -q --no-trunc') return 0 ;; + *) return 1 ;; + esac +} +systemctl() { return 0; } +capture_docker_container_baseline +`, + { + NEMOCLAW_STATION_PREP_SUDO_NONINTERACTIVE: "1", + PATH: `${path.dirname(process.execPath)}:${TEST_SYSTEM_PATH}`, + }, + ); + + expect(result.status, output).toBe(0); + expect(output).toContain("docker_access=sudo_for_noninteractive_read_only_check"); + expect(output).toContain("docker_container_baseline_total=0 running=0"); + }); + + it("does not hide missing Docker group access during verify", () => { + const { result, output } = runStationPreparation( + ` +MODE='--verify' +docker() { return 1; } +sudo() { return 0; } +systemctl() { return 0; } +capture_docker_container_baseline +`, + { + NEMOCLAW_STATION_PREP_SUDO_NONINTERACTIVE: "1", + PATH: `${path.dirname(process.execPath)}:${TEST_SYSTEM_PATH}`, + }, + ); + + expect(result.status, output).not.toBe(0); + expect(output).toContain("Docker is active but inaccessible to this login"); + expect(output).not.toContain("docker_access=sudo_"); + }); + it("fails closed when Docker is installed but its container state cannot be queried", () => { const { result, output } = runStationPreparation( ` diff --git a/test/install-station-controller-binding.test.ts b/test/install-station-controller-binding.test.ts index 86c7923c79..cf8ad53141 100644 --- a/test/install-station-controller-binding.test.ts +++ b/test/install-station-controller-binding.test.ts @@ -83,6 +83,27 @@ main --non-interactive --yes-i-accept-third-party-software } describe("DGX Station controller UID binding", () => { + it("keeps ordinary Station verification independent of dual-pair binding", () => { + const { home, result, output } = runSourced(` +require_command() { :; } +common_preflight() { STATION_HOST_PROFILE=ai-developer-tools; } +station_uses_factory_runtime() { return 0; } +verify_dgx_os_runtime_user() { printf 'FACTORY_RUNTIME_VERIFIED\\n'; } +verify_dual_station_controller_uid_binding() { + printf 'UNEXPECTED_PAIR_BINDING_CHECK\\n' + return 1 +} +run_verify +`); + try { + expect(result.status, output).toBe(0); + expect(output).toContain("FACTORY_RUNTIME_VERIFIED"); + expect(output).not.toContain("UNEXPECTED_PAIR_BINDING_CHECK"); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + }); + it("runs binding-only preparation without workload inspection and retains sudo acquisition", () => { const { home, result, output } = runSourced(` require_command() { :; } diff --git a/test/install-station-pair-preparation.test.ts b/test/install-station-pair-preparation.test.ts index 2ecebcb941..e782b65ae8 100644 --- a/test/install-station-pair-preparation.test.ts +++ b/test/install-station-pair-preparation.test.ts @@ -340,9 +340,8 @@ describe("deterministic dual-DGX Station peer discovery", () => { it("accepts one pretrusted reciprocal peer and runs preparation in order", () => { const harness = new PreparationHarness(); trustFirstRail(harness); - + harness.remoteHelperStatus.set("--apply", 11); const result = prepareDualStationPair(preparationOptions(), harness.deps); - expect(result.kind).toBe("ready"); expect(result.kind === "ready" && result.peerTarget).toBe("10.10.0.2"); expect(result.kind === "ready" && result.binding).toEqual(sshBinding()); @@ -353,20 +352,23 @@ describe("deterministic dual-DGX Station peer discovery", () => { expect(harness.calls.indexOf("state:write:remote-preparation")).toBeLessThan( harness.calls.indexOf("remote:10.10.0.2:--check"), ); + expect(harness.calls).toContain("local:--bind-controller"); expect(harness.calls.filter((call) => call.startsWith("remote:"))).toEqual([ "remote:10.10.0.2:--check", "remote:10.10.0.2:--apply", + "remote:10.10.0.2:--bind-controller", "remote:10.10.0.2:--verify", ]); + expect(harness.calls).toContain( + "log:Peer Docker access requires a new login; reopening SSH before verification", + ); }); - it("accepts two rail aliases only when their exact SSH identity is coherent", () => { const coherent = new PreparationHarness(); coherent.trusted.set("10.10.0.2", sshBinding("10.10.0.2")); coherent.trusted.set("10.10.0.6", sshBinding("10.10.0.6")); expect(prepareDualStationPair(preparationOptions(), coherent.deps).kind).toBe("ready"); expect(coherent.calls.filter((call) => call.startsWith("probe:peer:"))).toHaveLength(1); - const ambiguous = new PreparationHarness(); ambiguous.trusted.set("10.10.0.2", sshBinding("10.10.0.2")); ambiguous.trusted.set("10.10.0.6", sshBinding("10.10.0.6", "AAAAC3NzaChangedKey")); @@ -571,7 +573,6 @@ describe("dual-DGX Station reboot resume and reuse", () => { const first = new PreparationHarness(); trustFirstRail(first); first.remoteHelperStatus.set("--apply", 10); - const interrupted = prepareDualStationPair(preparationOptions(), first.deps); expect(interrupted.kind).toBe("reboot-required"); expect(interrupted.kind === "reboot-required" && interrupted.binding).toEqual(sshBinding()); @@ -579,7 +580,6 @@ describe("dual-DGX Station reboot resume and reuse", () => { expect(first.resume?.helperSha256).toBe(HELPER_SHA256); expect(first.calls.some((call) => call.endsWith(":--verify"))).toBe(true); expect(first.calls).not.toContain("remote:10.10.0.2:--verify"); - const resumed = new PreparationHarness(); resumed.resume = structuredClone(first.resume); trustFirstRail(resumed); @@ -588,10 +588,10 @@ describe("dual-DGX Station reboot resume and reuse", () => { expect(resumed.calls.filter((call) => call.startsWith("remote:"))).toEqual([ "remote:10.10.0.2:--check", "remote:10.10.0.2:--apply", + "remote:10.10.0.2:--bind-controller", "remote:10.10.0.2:--verify", ]); }); - it("rejects revision, helper, host-key, GPU, and rail substitution on resume", () => { const scenarios: Array<{ name: string; @@ -685,7 +685,6 @@ describe("dual-DGX Station reboot resume and reuse", () => { it("binds only the active local controller before preparing a peer for legacy migration", () => { const harness = new PreparationHarness(); trustFirstRail(harness); - expect( prepareDualStationPair( { ...preparationOptions(), migrateLegacySingleStationHead: true }, @@ -698,6 +697,7 @@ describe("dual-DGX Station reboot resume and reuse", () => { expect(harness.calls.filter((call) => call.startsWith("remote:"))).toEqual([ "remote:10.10.0.2:--check", "remote:10.10.0.2:--apply", + "remote:10.10.0.2:--bind-controller", "remote:10.10.0.2:--verify", ]); }); From 4af77a4a3b922b07ee43283132d560db86bbfd54 Mon Sep 17 00:00:00 2001 From: Senthil Ravichandran Date: Sat, 25 Jul 2026 22:29:02 -0700 Subject: [PATCH 56/74] fix(station): preserve dual-node resume validation Signed-off-by: Senthil Ravichandran --- scripts/install.sh | 25 ++++-- scripts/prepare-dual-dgx-station.mts | 24 +++--- test/install-gateway-state-root.test.ts | 17 ++++ ...pare-dual-dgx-station-resume-state.test.ts | 80 +++++++++++++++++++ 4 files changed, 129 insertions(+), 17 deletions(-) diff --git a/scripts/install.sh b/scripts/install.sh index cd2d0bd547..6e73907539 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -259,18 +259,30 @@ resolve_nemoclaw_gateway_port() { printf "%s" "$port" } +nemoclaw_state_root() { + local home="${HOME%/}" + [ -n "$home" ] || home="/" + if [ "$home" = "/" ]; then + printf "/.nemoclaw" + else + printf "%s/.nemoclaw" "$home" + fi +} + nemoclaw_state_dir() { - local port + local port root port="$(resolve_nemoclaw_gateway_port)" || return 1 + root="$(nemoclaw_state_root)" || return 1 if [ "$port" -eq 8080 ]; then - printf "%s/.nemoclaw" "$HOME" + printf "%s" "$root" else - printf "%s/.nemoclaw/gateways/%s" "$HOME" "$port" + printf "%s/gateways/%s" "$root" "$port" fi } assert_nemoclaw_state_path_safe() { - local target="$1" root="${HOME}/.nemoclaw" current relative component + local target="$1" root current relative component + root="$(nemoclaw_state_root)" || return 1 case "$target" in "$root" | "$root"/*) ;; *) error "Refusing NemoClaw state path outside ${root}: ${target}" ;; @@ -296,7 +308,7 @@ assert_nemoclaw_state_path_safe() { ensure_nemoclaw_state_dir() { local state_dir root gateways_dir state_dir="$(nemoclaw_state_dir)" || return 1 - root="${HOME}/.nemoclaw" + root="$(nemoclaw_state_root)" || return 1 gateways_dir="${root}/gateways" assert_nemoclaw_state_path_safe "$state_dir" (umask 077 && mkdir -p "$state_dir") || error "Could not create NemoClaw state directory: ${state_dir}" @@ -3667,7 +3679,8 @@ assert_station_express_resume_file_safe() { } assert_station_express_resume_directory_safe() { - local state_dir=$1 root="${HOME}/.nemoclaw" current relative component mode + local state_dir=$1 root current relative component mode + root="$(nemoclaw_state_root)" || return 1 assert_nemoclaw_state_path_safe "$state_dir" current="$root" relative="${state_dir#"$root"}" diff --git a/scripts/prepare-dual-dgx-station.mts b/scripts/prepare-dual-dgx-station.mts index b9d53e23c1..8a8580001a 100755 --- a/scripts/prepare-dual-dgx-station.mts +++ b/scripts/prepare-dual-dgx-station.mts @@ -212,7 +212,7 @@ print(json.dumps({ }, separators=(",", ":"))) `; -const CONNECTIVITY_PROBE = String.raw` +export const CONNECTIVITY_PROBE = String.raw` import ipaddress import json import subprocess @@ -251,23 +251,25 @@ for offset in (0, 3): routes = json.loads(output) route = routes[0] if isinstance(routes, list) and routes else {} route_device = route.get("dev", "") if isinstance(route, dict) else "" - route_source = route.get("prefsrc", route.get("src", "")) if isinstance(route, dict) else "" + route_source = route.get("prefsrc", route.get("src", route.get("from", ""))) if isinstance(route, dict) else "" route_gateway = route.get("gateway") if isinstance(route, dict) else None except json.JSONDecodeError: pass network = str(ipaddress.ip_network(source + "/30", strict=False)) - link_rc, link_output = run(["ip", "-j", "route", "show", "exact", network, "dev", netdev]) + link_rc, link_output = run(["ip", "-j", "route", "show", "exact", network]) if link_rc == 0: try: link_routes = json.loads(link_output) - link_route = link_routes[0] if isinstance(link_routes, list) and link_routes else {} - if ( - isinstance(link_route, dict) - and link_route.get("dst") == network - and link_route.get("dev") == netdev - and link_route.get("gateway") is None - ): - route_scope = link_route.get("scope", "") + matching_routes = [ + route for route in link_routes if ( + isinstance(route, dict) + and route.get("dst") == network + and route.get("dev") == netdev + and route.get("gateway") is None + ) + ] if isinstance(link_routes, list) else [] + if len(matching_routes) == 1: + route_scope = matching_routes[0].get("scope", "") except json.JSONDecodeError: pass ping_rc, _ = run([ diff --git a/test/install-gateway-state-root.test.ts b/test/install-gateway-state-root.test.ts index 5ad91c9488..a40ffe51df 100644 --- a/test/install-gateway-state-root.test.ts +++ b/test/install-gateway-state-root.test.ts @@ -154,6 +154,23 @@ printf 'state=%s\n' "$(nemoclaw_state_dir)"`, } }); + it("normalizes a trailing slash in HOME before selecting its state root", () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-installer-home-slash-")); + try { + const result = runInstallerFunctions( + `${home}/`, + `NEMOCLAW_GATEWAY_PORT=8080 +printf 'state=%s\n' "$(nemoclaw_state_dir)"`, + ); + + expect(result.status, result.output).toBe(0); + expect(result.output).toContain(`state=${home}/.nemoclaw`); + expect(result.output).not.toContain(`${home}//.nemoclaw`); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + }); + it("rejects an overlong digit-only gateway port before selecting its state root (#7203)", () => { const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-installer-overlong-port-")); try { diff --git a/test/prepare-dual-dgx-station-resume-state.test.ts b/test/prepare-dual-dgx-station-resume-state.test.ts index 5858cd7b26..a9726f8e4d 100644 --- a/test/prepare-dual-dgx-station-resume-state.test.ts +++ b/test/prepare-dual-dgx-station-resume-state.test.ts @@ -1,15 +1,18 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { spawnSync } from "node:child_process"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { expect, it, vi } from "vitest"; import type { DualStationResumeState } from "../scripts/lib/dgx-station-peer.mts"; import { + CONNECTIVITY_PROBE, clearDualStationResumeState, writeDualStationResumeState, } from "../scripts/prepare-dual-dgx-station.mts"; +import { TEST_SYSTEM_PATH } from "./helpers/installer-sourced-env"; function readyState(): DualStationResumeState { return { @@ -47,6 +50,83 @@ function captureThrown(operation: () => void): unknown { return null; } +it("accepts direct routes from iproute2 JSON that omits redundant filtered fields", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-pair-iproute2-")); + const bin = path.join(root, "bin"); + fs.mkdirSync(bin, { mode: 0o700 }); + fs.writeFileSync( + path.join(bin, "ip"), + `#!/usr/bin/env bash +case "$*" in + "-j route get 10.10.0.2 from 10.10.0.1 oif rail0") + printf '%s\\n' '[{"dst":"10.10.0.2","from":"10.10.0.1","dev":"rail0","flags":[]}]' + ;; + "-j route get 10.10.0.6 from 10.10.0.5 oif rail1") + printf '%s\\n' '[{"dst":"10.10.0.6","from":"10.10.0.5","dev":"rail1","flags":[]}]' + ;; + "-j route show exact 10.10.0.0/30") + printf '%s\\n' '[{"dst":"10.10.0.0/30","dev":"rail0","scope":"link","prefsrc":"10.10.0.1"}]' + ;; + "-j route show exact 10.10.0.4/30") + printf '%s\\n' '[{"dst":"10.10.0.4/30","dev":"rail1","scope":"link","prefsrc":"10.10.0.5"}]' + ;; + "-j neighbor show to 10.10.0.2 dev rail0") + printf '%s\\n' '[{"dst":"10.10.0.2","dev":"rail0","lladdr":"02:00:00:00:00:02","state":["STALE"]}]' + ;; + "-j neighbor show to 10.10.0.6 dev rail1") + printf '%s\\n' '[{"dst":"10.10.0.6","dev":"rail1","lladdr":"02:00:00:00:00:06","state":["REACHABLE"]}]' + ;; + *) + exit 97 + ;; +esac +`, + { mode: 0o700 }, + ); + fs.writeFileSync(path.join(bin, "ping"), "#!/usr/bin/env bash\nexit 0\n", { + mode: 0o700, + }); + + try { + const result = spawnSync( + "python3", + ["-", "rail0", "10.10.0.1", "10.10.0.2", "rail1", "10.10.0.5", "10.10.0.6"], + { + encoding: "utf8", + env: { ...process.env, PATH: `${bin}:${TEST_SYSTEM_PATH}` }, + input: CONNECTIVITY_PROBE, + }, + ); + + expect(result.status, `${result.stdout}${result.stderr}`).toBe(0); + expect(JSON.parse(result.stdout)).toEqual({ + schemaVersion: 1, + checks: [ + expect.objectContaining({ + netdev: "rail0", + routeDevice: "rail0", + routeSource: "10.10.0.1", + routeScope: "link", + peerMac: "02:00:00:00:00:02", + peerNeighborState: "STALE", + jumboPing: true, + }), + expect.objectContaining({ + netdev: "rail1", + routeDevice: "rail1", + routeSource: "10.10.0.5", + routeScope: "link", + peerMac: "02:00:00:00:00:06", + peerNeighborState: "REACHABLE", + jumboPing: true, + }), + ], + }); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } +}); + it("preserves the primary resume-state write error when temporary cleanup also fails", () => { const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-pair-state-cleanup-")); fs.chmodSync(directory, 0o700); From a8f35150cee86dd6147935d03de409c92bba4449 Mon Sep 17 00:00:00 2001 From: Senthil Ravichandran Date: Sat, 25 Jul 2026 22:43:42 -0700 Subject: [PATCH 57/74] fix(station): normalize local vllm resume path Signed-off-by: Senthil Ravichandran --- scripts/lib/station-vllm-conflict.sh | 4 +++- .../install-station-vllm-continuation.test.ts | 19 +++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/scripts/lib/station-vllm-conflict.sh b/scripts/lib/station-vllm-conflict.sh index 82696b564f..0c91178d30 100644 --- a/scripts/lib/station-vllm-conflict.sh +++ b/scripts/lib/station-vllm-conflict.sh @@ -9,7 +9,9 @@ _STATION_LOCAL_VLLM_SELECTED="" station_local_vllm_resume_file() { - printf '%s/.nemoclaw/station-local-vllm-resume' "$HOME" + local state_root + state_root="$(nemoclaw_state_root)" || return 1 + printf '%s/station-local-vllm-resume' "$state_root" } assert_station_local_vllm_resume_file_safe() { diff --git a/test/install-station-vllm-continuation.test.ts b/test/install-station-vllm-continuation.test.ts index df0918f19c..87a0a9c202 100644 --- a/test/install-station-vllm-continuation.test.ts +++ b/test/install-station-vllm-continuation.test.ts @@ -47,6 +47,25 @@ function runStationPreparationSourced(body: string) { } describe("installer Station Local vLLM continuation", () => { + it("normalizes a trailing slash in HOME before selecting continuation state", () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-vllm-resume-home-slash-")); + try { + const { result, output } = runInstallerSourced( + ` +load_station_vllm_conflict_helpers +printf 'resume=%s\\n' "$(station_local_vllm_resume_file)" +`, + `${home}/`, + ); + + expect(result.status, output).toBe(0); + expect(output).toContain(`resume=${home}/.nemoclaw/station-local-vllm-resume`); + expect(output).not.toContain(`${home}//.nemoclaw`); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + }); + it("ignores diagnostic processes that mention vLLM", () => { const { result, output } = runInstallerSourced(` load_station_vllm_conflict_helpers From 546e67b9ea0277b755095dfc14151cb0d5a73dfb Mon Sep 17 00:00:00 2001 From: Senthil Ravichandran Date: Sat, 25 Jul 2026 22:51:57 -0700 Subject: [PATCH 58/74] fix(station): validate unfiltered peer neighbors Signed-off-by: Senthil Ravichandran --- scripts/prepare-dual-dgx-station.mts | 16 +++++++++++----- ...prepare-dual-dgx-station-resume-state.test.ts | 4 ++-- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/scripts/prepare-dual-dgx-station.mts b/scripts/prepare-dual-dgx-station.mts index 8a8580001a..4d5f888b54 100755 --- a/scripts/prepare-dual-dgx-station.mts +++ b/scripts/prepare-dual-dgx-station.mts @@ -275,14 +275,20 @@ for offset in (0, 3): ping_rc, _ = run([ "ping", "-4", "-M", "do", "-s", "8972", "-c", "1", "-W", "2", "-I", source, peer, ]) - neighbor_rc, neighbor_output = run(["ip", "-j", "neighbor", "show", "to", peer, "dev", netdev]) + neighbor_rc, neighbor_output = run(["ip", "-j", "neighbor", "show", "to", peer]) if neighbor_rc == 0: try: neighbors = json.loads(neighbor_output) - neighbor = neighbors[0] if isinstance(neighbors, list) and neighbors else {} - if isinstance(neighbor, dict) and neighbor.get("dst") == peer and neighbor.get("dev") == netdev: - peer_mac = str(neighbor.get("lladdr", "")).lower() - raw_state = neighbor.get("state", "") + matching_neighbors = [ + neighbor for neighbor in neighbors if ( + isinstance(neighbor, dict) + and neighbor.get("dst") == peer + and neighbor.get("dev") == netdev + ) + ] if isinstance(neighbors, list) else [] + if len(matching_neighbors) == 1: + peer_mac = str(matching_neighbors[0].get("lladdr", "")).lower() + raw_state = matching_neighbors[0].get("state", "") peer_neighbor_state = ",".join(raw_state) if isinstance(raw_state, list) else str(raw_state) except json.JSONDecodeError: pass diff --git a/test/prepare-dual-dgx-station-resume-state.test.ts b/test/prepare-dual-dgx-station-resume-state.test.ts index a9726f8e4d..bd4c5a9e69 100644 --- a/test/prepare-dual-dgx-station-resume-state.test.ts +++ b/test/prepare-dual-dgx-station-resume-state.test.ts @@ -70,10 +70,10 @@ case "$*" in "-j route show exact 10.10.0.4/30") printf '%s\\n' '[{"dst":"10.10.0.4/30","dev":"rail1","scope":"link","prefsrc":"10.10.0.5"}]' ;; - "-j neighbor show to 10.10.0.2 dev rail0") + "-j neighbor show to 10.10.0.2") printf '%s\\n' '[{"dst":"10.10.0.2","dev":"rail0","lladdr":"02:00:00:00:00:02","state":["STALE"]}]' ;; - "-j neighbor show to 10.10.0.6 dev rail1") + "-j neighbor show to 10.10.0.6") printf '%s\\n' '[{"dst":"10.10.0.6","dev":"rail1","lladdr":"02:00:00:00:00:06","state":["REACHABLE"]}]' ;; *) From 950cb3a5966778772ffd2ea9d60cbf68a5921111 Mon Sep 17 00:00:00 2001 From: Senthil Ravichandran Date: Mon, 27 Jul 2026 10:30:35 -0700 Subject: [PATCH 59/74] fix(vllm): keep Nemotron Ultra public --- src/lib/inference/vllm-dual-station.test.ts | 17 ++++++++--------- src/lib/inference/vllm-models.test.ts | 8 +++----- src/lib/inference/vllm-models.ts | 2 +- 3 files changed, 12 insertions(+), 15 deletions(-) diff --git a/src/lib/inference/vllm-dual-station.test.ts b/src/lib/inference/vllm-dual-station.test.ts index 07f222642e..fa042efe81 100644 --- a/src/lib/inference/vllm-dual-station.test.ts +++ b/src/lib/inference/vllm-dual-station.test.ts @@ -404,7 +404,7 @@ describe("dual DGX Station vLLM install orchestration", () => { expect(mocks.probeCapability).toHaveBeenCalledTimes(2); }); - it("fails through the normal gated-model resolver before side effects without an HF token", async () => { + it("installs the public Nemotron Ultra model without an HF token", async () => { delete process.env.NEMOCLAW_VLLM_MODEL; delete process.env.HF_TOKEN; delete process.env.HUGGING_FACE_HUB_TOKEN; @@ -418,15 +418,14 @@ describe("dual DGX Station vLLM install orchestration", () => { promptFn: vi.fn(), beforeInstall, }), - ).resolves.toEqual({ ok: false }); + ).resolves.toEqual({ ok: true }); - expect(mocks.probeCapability).toHaveBeenCalledTimes(1); - expect(beforeInstall).not.toHaveBeenCalled(); - expect(mocks.preflightOwnership).not.toHaveBeenCalled(); - expect(mocks.dockerPullWithProgressWatchdog).not.toHaveBeenCalled(); - expect(mocks.dockerSpawn).not.toHaveBeenCalled(); - expect(mocks.stageModelSnapshot).not.toHaveBeenCalled(); - expect(errorSpy).toHaveBeenCalledWith(expect.stringContaining("gated on Hugging Face")); + expect(mocks.probeCapability).toHaveBeenCalledTimes(2); + expect(beforeInstall).toHaveBeenCalledWith("nemotron-ultra"); + expect(mocks.preflightOwnership).toHaveBeenCalled(); + expect(mocks.dockerSpawn).toHaveBeenCalled(); + expect(mocks.stageModelSnapshot).toHaveBeenCalled(); + expect(errorSpy).not.toHaveBeenCalledWith(expect.stringContaining("gated on Hugging Face")); }); it("skips only the model picker for an interactive qualified-peer install", async () => { diff --git a/src/lib/inference/vllm-models.test.ts b/src/lib/inference/vllm-models.test.ts index 90b50d69af..bb928a9859 100644 --- a/src/lib/inference/vllm-models.test.ts +++ b/src/lib/inference/vllm-models.test.ts @@ -225,12 +225,10 @@ describe("vllm model registry", () => { ); }); - it("keeps the pinned Nemotron Ultra recipe behind Hugging Face access", () => { + it("keeps the public Nemotron Ultra recipe usable without a Hugging Face token", () => { const ultra = VLLM_MODELS.find((m) => m.envValue === "nemotron-3-ultra-550b-a55b"); - expect(ultra?.gated).toBe(true); - expect(() => assertGatedModelAccess(ultra!, {} as NodeJS.ProcessEnv)).toThrow( - /gated on Hugging Face/, - ); + expect(ultra?.gated).toBe(false); + expect(() => assertGatedModelAccess(ultra!, {} as NodeJS.ProcessEnv)).not.toThrow(); }); it("never rejects a non-gated model regardless of token state", () => { diff --git a/src/lib/inference/vllm-models.ts b/src/lib/inference/vllm-models.ts index 4c237db533..6591e80569 100644 --- a/src/lib/inference/vllm-models.ts +++ b/src/lib/inference/vllm-models.ts @@ -263,7 +263,7 @@ export const VLLM_MODELS: readonly VllmModelDef[] = [ "--default-chat-template-kwargs", `'{"enable_thinking":true,"force_nonempty_content":true}'`, ], - gated: true, + gated: false, platforms: ["station"], serveEnv: { VLLM_WEIGHT_OFFLOADING_DISABLE_PIN_MEMORY: "1", From c8998d6f700ea33c0c6f40648cd6c20376bf9e01 Mon Sep 17 00:00:00 2001 From: Senthil Ravichandran Date: Mon, 27 Jul 2026 10:51:37 -0700 Subject: [PATCH 60/74] fix(onboard): preserve dual Station model intent Signed-off-by: Senthil Ravichandran --- .../onboard/station-express-resume.test.ts | 31 +++++++++++++++++++ src/lib/onboard/station-express-resume.ts | 23 ++++++++++++-- 2 files changed, 52 insertions(+), 2 deletions(-) diff --git a/src/lib/onboard/station-express-resume.test.ts b/src/lib/onboard/station-express-resume.test.ts index 81b0255067..5247a87064 100644 --- a/src/lib/onboard/station-express-resume.test.ts +++ b/src/lib/onboard/station-express-resume.test.ts @@ -49,6 +49,15 @@ function expressEnv(): NodeJS.ProcessEnv { }; } +function dualExpressEnv(): NodeJS.ProcessEnv { + return { + ...expressEnv(), + NEMOCLAW_MODEL: "nemotron-ultra", + NEMOCLAW_DGX_STATION_PEER: "192.168.240.2", + NEMOCLAW_DGX_STATION_SSH_BINDING: "sha256:qualified-pair-binding", + }; +} + function receiptText(generation = receiptGeneration, model = "nemotron-3-ultra-550b-a55b"): string { return `revision=${receiptRevision}\nmodel=${model}\ngeneration=${generation}\n`; } @@ -101,6 +110,28 @@ describe("DGX Station Express resume (#7048)", () => { }); }); + it("captures the qualified dual-Station served alias in the sealed intent", () => { + expect(getStationExpressResumeIntent(dualExpressEnv(), "my-assistant")).toEqual({ + ok: true, + intent: { + ...ultraIntent, + servedModel: "nemotron-ultra", + checkpointModel: "nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4", + }, + }); + }); + + it("rejects the dual-Station served alias without both qualified pair signals", () => { + for (const missing of ["NEMOCLAW_DGX_STATION_PEER", "NEMOCLAW_DGX_STATION_SSH_BINDING"]) { + const env = dualExpressEnv(); + delete env[missing]; + expect(getStationExpressResumeIntent(env, "my-assistant")).toEqual({ + ok: false, + message: "DGX Station Express has a conflicting NEMOCLAW_MODEL value.", + }); + } + }); + it("carries the installer receipt generation in the persisted intent", () => { const env = expressEnv(); env[STATION_EXPRESS_RECEIPT_GENERATION_ENV] = receiptGeneration; diff --git a/src/lib/onboard/station-express-resume.ts b/src/lib/onboard/station-express-resume.ts index 2a888a8626..85a2198a28 100644 --- a/src/lib/onboard/station-express-resume.ts +++ b/src/lib/onboard/station-express-resume.ts @@ -99,6 +99,8 @@ const BOUND_RECEIPT_INTENT_KEYS = const SPARK_INTENT_KEYS = "kind,sandboxName,version"; const SPARK_INTENT_KEYS_WITH_MODEL = "kind,model,sandboxName,version"; const SPARK_EXPRESS_PROVIDER = "install-vllm"; +const STATION_ULTRA_ENV_VALUE = "nemotron-3-ultra-550b-a55b"; +const STATION_ULTRA_DUAL_SERVED_MODEL = "nemotron-ultra"; const STATION_EXPRESS_INSTALLER_RESUME_FILE = "station-express-resume"; const STATION_EXPRESS_RETIREMENT_CLAIM_PREFIX = `${STATION_EXPRESS_INSTALLER_RESUME_FILE}.retiring-`; const STATION_EXPRESS_RETIREMENT_CLAIM_RECEIPT = "receipt"; @@ -144,6 +146,21 @@ function servedModel(model: VllmModelDef): string { return model.servedModelId ?? model.id; } +function qualifiedDualStationServedModel( + env: NodeJS.ProcessEnv, + model: VllmModelDef, +): string | null { + if ( + model.envValue !== STATION_ULTRA_ENV_VALUE || + String(env.NEMOCLAW_DGX_STATION_PEER ?? "").trim().length === 0 || + String(env.NEMOCLAW_DGX_STATION_SSH_BINDING ?? "").trim().length === 0 + ) { + return null; + } + const selected = String(env.NEMOCLAW_MODEL ?? "").trim(); + return selected === STATION_ULTRA_DUAL_SERVED_MODEL ? selected : null; +} + function identifiesCheckpoint(model: VllmModelDef, value: string): boolean { const normalized = value.toLowerCase(); return [model.envValue, model.id, model.servedModelId].some( @@ -812,7 +829,7 @@ function expectedEnvironment( if (includeProviderSelection) { expected.NEMOCLAW_PROVIDER = "install-vllm"; expected.NEMOCLAW_VLLM_MODEL = model.envValue; - expected.NEMOCLAW_MODEL = servedModel(model); + expected.NEMOCLAW_MODEL = intent.servedModel ?? servedModel(model); } return expected; } @@ -895,10 +912,12 @@ export function getStationExpressResumeIntent( message: "DGX Station Express requires a registered Station vLLM model and sandbox name.", }; } - const intent: StationExpressResumeIntent = { + const dualServedModel = qualifiedDualStationServedModel(env, model); + const intent: StationResumeIntent = { version: STATION_EXPRESS_INTENT_VERSION, model: model.envValue, sandboxName, + ...(dualServedModel ? { servedModel: dualServedModel, checkpointModel: model.id } : {}), ...(isValidStationExpressReceiptGeneration(env[STATION_EXPRESS_RECEIPT_GENERATION_ENV]) ? { receiptGeneration: env[STATION_EXPRESS_RECEIPT_GENERATION_ENV] } : {}), From 432ab5ead2f5e948a6de4153b2bfbe006369a9ef Mon Sep 17 00:00:00 2001 From: Senthil Ravichandran Date: Mon, 27 Jul 2026 11:02:55 -0700 Subject: [PATCH 61/74] fix(vllm): accept local default Docker context Signed-off-by: Senthil Ravichandran --- src/lib/inference/vllm-station-cluster.test.ts | 17 +++++++++++++++++ src/lib/inference/vllm-station-cluster.ts | 7 ++++--- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/src/lib/inference/vllm-station-cluster.test.ts b/src/lib/inference/vllm-station-cluster.test.ts index 6d79ab45db..d234f6eeaf 100644 --- a/src/lib/inference/vllm-station-cluster.test.ts +++ b/src/lib/inference/vllm-station-cluster.test.ts @@ -361,6 +361,23 @@ describe("probeDualStationVllmCapability", () => { expect(deps.calls.localHost).not.toHaveBeenCalled(); }); + it("accepts the local default Docker context selected by Station host preparation", () => { + const deps = fixtureDeps(); + + expect( + probeDualStationVllmCapability({ + env: { + [NEMOCLAW_DGX_STATION_PEER_ENV]: "nvidia@station-b", + [NEMOCLAW_DGX_STATION_SSH_BINDING_ENV]: sshFixture.token, + DOCKER_CONTEXT: "default", + }, + deps, + }), + ).toMatchObject({ kind: "ready" }); + expect(deps.calls.sshConfig).toHaveBeenCalledOnce(); + expect(deps.calls.localHost).toHaveBeenCalledOnce(); + }); + it.each([ "station-b", "nvidia@station-b", diff --git a/src/lib/inference/vllm-station-cluster.ts b/src/lib/inference/vllm-station-cluster.ts index a13711e608..9e03a1af47 100644 --- a/src/lib/inference/vllm-station-cluster.ts +++ b/src/lib/inference/vllm-station-cluster.ts @@ -1524,9 +1524,10 @@ export function probeDualStationVllmCapability( `${NEMOCLAW_DGX_STATION_SSH_BINDING_ENV} must identify the installer-qualified peer`, ); } - const localDockerOverride = DUAL_STATION_LOCAL_DOCKER_OVERRIDE_ENV_NAMES.find( - (name) => env[name] !== undefined && String(env[name]).trim() !== "", - ); + const localDockerOverride = DUAL_STATION_LOCAL_DOCKER_OVERRIDE_ENV_NAMES.find((name) => { + const value = String(env[name] ?? "").trim(); + return value !== "" && !(name === "DOCKER_CONTEXT" && value === "default"); + }); if (localDockerOverride) { return unavailable( "local-docker-unavailable", From b46549e557290b88ca2135ab3e86107939f7e454 Mon Sep 17 00:00:00 2001 From: Senthil Ravichandran Date: Mon, 27 Jul 2026 11:10:58 -0700 Subject: [PATCH 62/74] fix(vllm): accept detected local Docker socket Signed-off-by: Senthil Ravichandran --- src/lib/inference/vllm-station-cluster.test.ts | 18 ++++++++++++++---- src/lib/inference/vllm-station-cluster.ts | 6 +++++- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/src/lib/inference/vllm-station-cluster.test.ts b/src/lib/inference/vllm-station-cluster.test.ts index d234f6eeaf..e7bea9239f 100644 --- a/src/lib/inference/vllm-station-cluster.test.ts +++ b/src/lib/inference/vllm-station-cluster.test.ts @@ -344,7 +344,10 @@ describe("probeDualStationVllmCapability", () => { expect(deps.calls.localHost).not.toHaveBeenCalled(); }); - it("rejects an ambient Docker client override before mixing it with local hardware", () => { + it.each([ + ["named Docker context", { DOCKER_CONTEXT: "remote-builder" }], + ["remote Docker host", { DOCKER_HOST: "ssh://builder.example" }], + ])("rejects an ambient %s before mixing it with local hardware", (_label, dockerEnv) => { const deps = fixtureDeps(); expect( @@ -352,7 +355,7 @@ describe("probeDualStationVllmCapability", () => { env: { [NEMOCLAW_DGX_STATION_PEER_ENV]: "nvidia@station-b", [NEMOCLAW_DGX_STATION_SSH_BINDING_ENV]: sshFixture.token, - DOCKER_CONTEXT: "remote-builder", + ...dockerEnv, }, deps, }), @@ -361,7 +364,14 @@ describe("probeDualStationVllmCapability", () => { expect(deps.calls.localHost).not.toHaveBeenCalled(); }); - it("accepts the local default Docker context selected by Station host preparation", () => { + it.each([ + ["context selected by Station host preparation", { DOCKER_CONTEXT: "default" }], + ["socket detected by the NemoClaw runner", { DOCKER_HOST: "unix:///run/docker.sock" }], + [ + "socket alias detected by the NemoClaw runner", + { DOCKER_HOST: "unix:///var/run/docker.sock" }, + ], + ])("accepts the local default Docker %s", (_label, dockerEnv) => { const deps = fixtureDeps(); expect( @@ -369,7 +379,7 @@ describe("probeDualStationVllmCapability", () => { env: { [NEMOCLAW_DGX_STATION_PEER_ENV]: "nvidia@station-b", [NEMOCLAW_DGX_STATION_SSH_BINDING_ENV]: sshFixture.token, - DOCKER_CONTEXT: "default", + ...dockerEnv, }, deps, }), diff --git a/src/lib/inference/vllm-station-cluster.ts b/src/lib/inference/vllm-station-cluster.ts index 9e03a1af47..5816bc3d28 100644 --- a/src/lib/inference/vllm-station-cluster.ts +++ b/src/lib/inference/vllm-station-cluster.ts @@ -1526,7 +1526,11 @@ export function probeDualStationVllmCapability( } const localDockerOverride = DUAL_STATION_LOCAL_DOCKER_OVERRIDE_ENV_NAMES.find((name) => { const value = String(env[name] ?? "").trim(); - return value !== "" && !(name === "DOCKER_CONTEXT" && value === "default"); + const isLocalDefaultSelection = + (name === "DOCKER_CONTEXT" && value === "default") || + (name === "DOCKER_HOST" && + (value === "unix:///run/docker.sock" || value === "unix:///var/run/docker.sock")); + return value !== "" && !isLocalDefaultSelection; }); if (localDockerOverride) { return unavailable( From b7ffe7d495999e74f63a9e5b08eade31ac72ed27 Mon Sep 17 00:00:00 2001 From: Senthil Ravichandran Date: Mon, 27 Jul 2026 11:23:25 -0700 Subject: [PATCH 63/74] fix(vllm): probe Station GPU runtime directly Factory Station images can provide CDI-backed --gpus support without registering a named nvidia Docker runtime. Verify the actual no-pull GPU container contract instead. The qualified RoCE path uses CUDA DMA-BUF/Data Direct, so it does not require nvidia_peermem. Signed-off-by: Senthil Ravichandran --- .../inference/vllm-station-cluster.test.ts | 70 ++++++++++++++++--- src/lib/inference/vllm-station-cluster.ts | 34 ++++----- 2 files changed, 76 insertions(+), 28 deletions(-) diff --git a/src/lib/inference/vllm-station-cluster.test.ts b/src/lib/inference/vllm-station-cluster.test.ts index e7bea9239f..1d5237de22 100644 --- a/src/lib/inference/vllm-station-cluster.test.ts +++ b/src/lib/inference/vllm-station-cluster.test.ts @@ -35,6 +35,8 @@ import { resolveStationFixturePython } from "./vllm-station-fixture.test-support const LOCAL_HOME = "/home/local"; const PEER_HOME = "/home/nvidia"; +const STATION_ACCEPTANCE_IMAGE = + "docker.io/library/ubuntu@sha256:7f622ca8766bccb22f04242ecb6f19f770b2f08827dc4b8c707de5e78a6da7ab"; function strictDockerSshConfig(binding: DualStationSshBinding): string { return [ @@ -142,7 +144,6 @@ function hostFixture(side: "local" | "peer"): StationHostProbe { ], docker: { reachable: true, nvidiaRuntime: true }, rsyncAvailable: true, - nvidiaPeermemLoaded: true, rails: isLocal ? [ rail("mlx5_0", "cx8a0", "0001:03:00.0", "192.168.240.1"), @@ -556,13 +557,6 @@ describe("probeDualStationVllmCapability", () => { host.docker.nvidiaRuntime = false; }, }, - { - name: "missing peer nvidia_peermem", - code: "peer-fabric-unavailable", - mutate: (host: StationHostProbe) => { - host.nvidiaPeermemLoaded = false; - }, - }, { name: "slow peer rail", code: "peer-fabric-unavailable", @@ -858,6 +852,10 @@ describe("probe command boundary", () => { ); expect(args.join(" ")).not.toMatch(/keyscan|accept-new|StrictHostKeyChecking=no/); expect(options.input).toEqual(expect.stringContaining('docker", "info')); + expect(options.input).toEqual(expect.stringContaining('"docker", "run", "--rm"')); + expect(options.input).toEqual(expect.stringContaining('"--pull=never"')); + expect(options.input).toEqual(expect.stringContaining('"--gpus", "all"')); + expect(options.input).toEqual(expect.stringContaining(STATION_ACCEPTANCE_IMAGE)); expect(options.input).toEqual(expect.stringContaining("/sys/firmware/devicetree/base/model")); expect(options.timeout).toBe(20_000); expect(options.maxBuffer).toBe(1024 * 1024); @@ -921,6 +919,62 @@ describe("probe command boundary", () => { } }); + it("executes the host probe against the digest-pinned no-pull GPU runtime contract", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-station-runtime-fixture-")); + const home = path.join(root, "home"); + const bin = path.join(root, "bin"); + fs.mkdirSync(home, { mode: 0o700 }); + fs.mkdirSync(bin, { mode: 0o700 }); + + let probeScript = ""; + const recordingSpawn = vi.fn( + ( + _file: string, + _args: readonly string[], + options: SpawnSyncOptionsWithStringEncoding, + ): StationProbeCommandResult => { + probeScript = typeof options.input === "string" ? options.input : ""; + return command({}); + }, + ); + createStationClusterProbeDeps(recordingSpawn).probeLocalHost(); + const python = resolveStationFixturePython(); + const fixturePrelude = String.raw` +import subprocess + +class FixtureResult: + def __init__(self, returncode, stdout=""): + self.returncode = returncode + self.stdout = stdout + self.stderr = "" + +def fixture_run(argv, **_kwargs): + if argv and argv[0] == "nvidia-smi": + return FixtureResult(0, "0, NVIDIA GB300, GPU-11111111-2222-3333-4444-555555555555") + if argv[:2] == ["docker", "info"]: + return FixtureResult(0, "29.2.1") + if argv[:2] == ["docker", "run"]: + return FixtureResult(0, "GPU-11111111-2222-3333-4444-555555555555") + return FixtureResult(127) + +subprocess.run = fixture_run +`; + + try { + const executed = spawnSync(python, ["-"], { + encoding: "utf8", + env: { ...process.env, HOME: home, PATH: bin }, + input: `${fixturePrelude}\n${probeScript}`, + timeout: 20_000, + }); + expect(executed.status, executed.stderr).toBe(0); + const observed = JSON.parse(executed.stdout) as StationHostProbe; + expect(observed.docker).toEqual({ reachable: true, nvidiaRuntime: true }); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } + }); + it("executes the host probe and refuses a non-character uverbs device", () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-station-verbs-fixture-")); const home = path.join(root, "home"); diff --git a/src/lib/inference/vllm-station-cluster.ts b/src/lib/inference/vllm-station-cluster.ts index 5816bc3d28..580e3f66b7 100644 --- a/src/lib/inference/vllm-station-cluster.ts +++ b/src/lib/inference/vllm-station-cluster.ts @@ -25,6 +25,8 @@ const MAX_PROBE_OUTPUT_BYTES = 1024 * 1024; const PREFERRED_ROCE_GID_INDEX = 3; const EXPECTED_ULTRA_WEIGHT_SHARDS = 113; const DIRECT_RAIL_PREFIX_LENGTH = 30; +const STATION_RUNTIME_PROBE_IMAGE = + "docker.io/library/ubuntu@sha256:7f622ca8766bccb22f04242ecb6f19f770b2f08827dc4b8c707de5e78a6da7ab"; const DUAL_STATION_LOCAL_DOCKER_OVERRIDE_ENV_NAMES = [ "DOCKER_API_VERSION", "DOCKER_CERT_PATH", @@ -109,7 +111,6 @@ export interface StationHostProbe { nvidiaRuntime: boolean; }; rsyncAvailable: boolean; - nvidiaPeermemLoaded: boolean; rails: StationRailProbe[]; modelSnapshot: StationModelSnapshotProbe; } @@ -316,16 +317,22 @@ def gpu_inventory(): return result def docker_state(): - rc, output = run(["docker", "info", "--format", "{{json .Runtimes}}"]) + rc, _ = run(["docker", "info", "--format", "{{.ServerVersion}}"]) if rc != 0: return {"reachable": False, "nvidiaRuntime": False} - try: - runtimes = json.loads(output) - except (TypeError, json.JSONDecodeError): - runtimes = {} + runtime_rc, runtime_output = run([ + "docker", "run", "--rm", "--pull=never", "--network", "none", + "--read-only", "--cap-drop", "ALL", "--security-opt", "no-new-privileges", + "--pids-limit", "64", "--memory", "512m", "--cpus", "1", + "--gpus", "all", "${STATION_RUNTIME_PROBE_IMAGE}", + "nvidia-smi", "--query-gpu=uuid", "--format=csv,noheader", + ], timeout=15) return { "reachable": True, - "nvidiaRuntime": isinstance(runtimes, dict) and "nvidia" in runtimes, + "nvidiaRuntime": runtime_rc == 0 and any( + line.strip().startswith("GPU-") + for line in runtime_output.splitlines() + ), } def ipv4_addresses(netdev): @@ -531,7 +538,6 @@ def snapshot_state(): "reason": "; ".join(dict.fromkeys(reasons)), } -modules = read_text("/proc/modules").splitlines() payload = { "schemaVersion": ${String(HOST_PROBE_SCHEMA_VERSION)}, "hostname": socket.gethostname(), @@ -543,7 +549,6 @@ payload = { "gpus": gpu_inventory(), "docker": docker_state(), "rsyncAvailable": shutil.which("rsync") is not None, - "nvidiaPeermemLoaded": any(line.split(maxsplit=1)[0] == "nvidia_peermem" for line in modules if line), "rails": rail_inventory(), "modelSnapshot": snapshot_state(), } @@ -970,10 +975,6 @@ export function parseStationHostProbe(stdout: string): StationHostProbe { nvidiaRuntime: requireBoolean(docker.nvidiaRuntime, "host probe.docker.nvidiaRuntime"), }, rsyncAvailable: requireBoolean(record.rsyncAvailable, "host probe.rsyncAvailable"), - nvidiaPeermemLoaded: requireBoolean( - record.nvidiaPeermemLoaded, - "host probe.nvidiaPeermemLoaded", - ), rails: requireArray(record.rails, "host probe.rails", 32).map((entry, index) => parseRail(entry, `host probe.rails[${String(index)}]`), ), @@ -1314,13 +1315,6 @@ function buildStaticPlan( "peer Docker daemon and NVIDIA runtime could not both be verified", ); } - if (!local.nvidiaPeermemLoaded) { - return unavailable("local-fabric-unavailable", "nvidia_peermem is not loaded locally"); - } - if (!peer.nvidiaPeermemLoaded) { - return unavailable("peer-fabric-unavailable", "nvidia_peermem is not loaded on the peer"); - } - const localRails = qualifiedRails(local); if (!localRails) { return unavailable( From d9008b6a36f0ec30b5b04646225fc580618785a9 Mon Sep 17 00:00:00 2001 From: Senthil Ravichandran Date: Mon, 27 Jul 2026 11:30:40 -0700 Subject: [PATCH 64/74] fix(vllm): accept Station route JSON variants Factory Station iproute2 reports the selected source under the JSON from key and omits the redundant device field when route show is already filtered by device. Accept those forms while preserving the direct-route, scope-link, neighbor identity, and jumbo-frame checks. Signed-off-by: Senthil Ravichandran --- .../inference/vllm-station-cluster.test.ts | 106 ++++++++++++++++++ src/lib/inference/vllm-station-cluster.ts | 8 +- 2 files changed, 112 insertions(+), 2 deletions(-) diff --git a/src/lib/inference/vllm-station-cluster.test.ts b/src/lib/inference/vllm-station-cluster.test.ts index 1d5237de22..abd05d9c42 100644 --- a/src/lib/inference/vllm-station-cluster.test.ts +++ b/src/lib/inference/vllm-station-cluster.test.ts @@ -1129,6 +1129,112 @@ subprocess.run = fixture_run ); expect(options.input).toEqual(expect.stringContaining('"-I", source, peer')); }); + + it("executes the connectivity probe against the route JSON emitted on Station", () => { + const requests = [ + { + netdev: "cx8r0", + sourceAddress: "192.168.240.1", + peerAddress: "192.168.240.2", + expectedPeerMac: "ac:3a:e2:de:3a:23", + }, + { + netdev: "cx8r1", + sourceAddress: "192.168.240.5", + peerAddress: "192.168.240.6", + expectedPeerMac: "ac:3a:e2:de:3a:24", + }, + ]; + let probeScript = ""; + let probeArgs: readonly string[] = []; + const recordingSpawn = vi.fn( + ( + _file: string, + args: readonly string[], + options: SpawnSyncOptionsWithStringEncoding, + ): StationProbeCommandResult => { + probeArgs = args; + probeScript = typeof options.input === "string" ? options.input : ""; + return command({}); + }, + ); + createStationClusterProbeDeps(recordingSpawn).probeLocalConnectivity(requests); + const fixturePrelude = String.raw` +import json +import subprocess + +class FixtureResult: + def __init__(self, returncode, stdout=""): + self.returncode = returncode + self.stdout = stdout + self.stderr = "" + +def fixture_run(argv, **_kwargs): + if argv[:4] == ["ip", "-j", "route", "get"]: + peer, source, netdev = argv[4], argv[6], argv[8] + return FixtureResult(0, json.dumps([{ + "dst": peer, + "from": source, + "dev": netdev, + "flags": [], + "uid": 1000, + "cache": [], + }])) + if argv[:5] == ["ip", "-j", "route", "show", "exact"]: + network = argv[5] + return FixtureResult(0, json.dumps([{ + "dst": network, + "protocol": "kernel", + "scope": "link", + "prefsrc": "192.168.240.1", + "metric": 100, + "flags": [], + }])) + if argv and argv[0] == "ping": + return FixtureResult(0) + if argv[:3] == ["ip", "-j", "neighbor"]: + peer, netdev = argv[5], argv[7] + mac = { + "192.168.240.2": "ac:3a:e2:de:3a:23", + "192.168.240.6": "ac:3a:e2:de:3a:24", + }[peer] + return FixtureResult(0, json.dumps([{ + "dst": peer, + "lladdr": mac, + "state": ["REACHABLE"], + "dev": netdev, + }])) + return FixtureResult(127) + +subprocess.run = fixture_run +`; + const executed = spawnSync(resolveStationFixturePython(), probeArgs, { + encoding: "utf8", + input: `${fixturePrelude}\n${probeScript}`, + timeout: 20_000, + }); + + expect(executed.status, executed.stderr).toBe(0); + const observed = JSON.parse(executed.stdout) as { + checks: Array>; + }; + expect(observed.checks).toEqual([ + expect.objectContaining({ + netdev: "cx8r0", + routeDevice: "cx8r0", + routeSource: "192.168.240.1", + routeScope: "link", + jumboPing: true, + }), + expect.objectContaining({ + netdev: "cx8r1", + routeDevice: "cx8r1", + routeSource: "192.168.240.5", + routeScope: "link", + jumboPing: true, + }), + ]); + }); }); describe("parseStationHostProbe", () => { diff --git a/src/lib/inference/vllm-station-cluster.ts b/src/lib/inference/vllm-station-cluster.ts index 580e3f66b7..9e4e5e47e3 100644 --- a/src/lib/inference/vllm-station-cluster.ts +++ b/src/lib/inference/vllm-station-cluster.ts @@ -594,7 +594,11 @@ for offset in (0, 3): routes = json.loads(output) route = routes[0] if isinstance(routes, list) and routes else {} route_device = route.get("dev", "") if isinstance(route, dict) else "" - route_source = route.get("prefsrc", route.get("src", "")) if isinstance(route, dict) else "" + route_source = ( + route.get("prefsrc") or route.get("src") or route.get("from", "") + if isinstance(route, dict) + else "" + ) route_gateway = route.get("gateway") if isinstance(route, dict) else None except json.JSONDecodeError: pass @@ -607,7 +611,7 @@ for offset in (0, 3): if ( isinstance(link_route, dict) and link_route.get("dst") == network - and link_route.get("dev") == netdev + and link_route.get("dev", netdev) == netdev and link_route.get("gateway") is None ): route_scope = link_route.get("scope", "") From eada67642a35f9887906b6f8de952c827a54c51b Mon Sep 17 00:00:00 2001 From: Senthil Ravichandran Date: Mon, 27 Jul 2026 11:36:12 -0700 Subject: [PATCH 65/74] fix(vllm): accept filtered Station neighbor JSON Factory Station iproute2 omits the redundant device field when neighbor output is already filtered by device. Accept the omission while still rejecting any emitted device that does not match the qualified rail. Signed-off-by: Senthil Ravichandran --- src/lib/inference/vllm-station-cluster.test.ts | 1 - src/lib/inference/vllm-station-cluster.ts | 6 +++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/lib/inference/vllm-station-cluster.test.ts b/src/lib/inference/vllm-station-cluster.test.ts index abd05d9c42..8b0f32bbbc 100644 --- a/src/lib/inference/vllm-station-cluster.test.ts +++ b/src/lib/inference/vllm-station-cluster.test.ts @@ -1202,7 +1202,6 @@ def fixture_run(argv, **_kwargs): "dst": peer, "lladdr": mac, "state": ["REACHABLE"], - "dev": netdev, }])) return FixtureResult(127) diff --git a/src/lib/inference/vllm-station-cluster.ts b/src/lib/inference/vllm-station-cluster.ts index 9e4e5e47e3..3ce57481f2 100644 --- a/src/lib/inference/vllm-station-cluster.ts +++ b/src/lib/inference/vllm-station-cluster.ts @@ -626,7 +626,11 @@ for offset in (0, 3): try: neighbors = json.loads(neighbor_output) neighbor = neighbors[0] if isinstance(neighbors, list) and neighbors else {} - if isinstance(neighbor, dict) and neighbor.get("dst") == peer and neighbor.get("dev") == netdev: + if ( + isinstance(neighbor, dict) + and neighbor.get("dst") == peer + and neighbor.get("dev", netdev) == netdev + ): peer_mac = str(neighbor.get("lladdr", "")).lower() raw_state = neighbor.get("state", "") peer_neighbor_state = ",".join(raw_state) if isinstance(raw_state, list) else str(raw_state) From 204798b3a496ba29fc0addc733c5dae698b86954 Mon Sep 17 00:00:00 2001 From: Senthil Ravichandran Date: Mon, 27 Jul 2026 13:16:05 -0700 Subject: [PATCH 66/74] fix(vllm): allow Ray on Station runtime tmpfs Signed-off-by: Senthil Ravichandran --- .../vllm-station-cluster-lifecycle.test.ts | 4 ++-- src/lib/inference/vllm-station-cluster-lifecycle.ts | 13 +++++++++---- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/lib/inference/vllm-station-cluster-lifecycle.test.ts b/src/lib/inference/vllm-station-cluster-lifecycle.test.ts index 57cdb6f6cc..b265ac256a 100644 --- a/src/lib/inference/vllm-station-cluster-lifecycle.test.ts +++ b/src/lib/inference/vllm-station-cluster-lifecycle.test.ts @@ -253,7 +253,7 @@ describe("dual-Station managed vLLM run argv", () => { expect(dockerValues(args, "--workdir")).toEqual(["/home/vllm"]); expect(dockerValues(args, "--tmpfs")).toEqual([ "/tmp:rw,nosuid,nodev,size=17179869184", - `/home/vllm:rw,nosuid,nodev,uid=${String(expectedNode.uid)},gid=${String(expectedNode.gid)},mode=0700,size=68719476736`, + `/home/vllm:rw,nosuid,nodev,exec,uid=${String(expectedNode.uid)},gid=${String(expectedNode.gid)},mode=0700,size=68719476736`, ]); expect(dockerValues(args, "--user")).toEqual([ `${String(expectedNode.uid)}:${String(expectedNode.gid)}`, @@ -360,7 +360,7 @@ describe("dual-Station managed vLLM run argv", () => { expect(dockerValues(args, "--device")).toEqual(expectedDevices); expect(dockerValues(args, "--tmpfs")).toEqual([ "/tmp:rw,nosuid,nodev,size=17179869184", - `/home/vllm:rw,nosuid,nodev,uid=${String(expectedNode.uid)},gid=${String(expectedNode.gid)},mode=0700,size=68719476736`, + `/home/vllm:rw,nosuid,nodev,noexec,uid=${String(expectedNode.uid)},gid=${String(expectedNode.gid)},mode=0700,size=68719476736`, ]); expect(dockerValues(args, "--env")).toEqual( expect.arrayContaining([ diff --git a/src/lib/inference/vllm-station-cluster-lifecycle.ts b/src/lib/inference/vllm-station-cluster-lifecycle.ts index 49474be2bd..3f6c90a12d 100644 --- a/src/lib/inference/vllm-station-cluster-lifecycle.ts +++ b/src/lib/inference/vllm-station-cluster-lifecycle.ts @@ -367,8 +367,11 @@ function runtimeUser(node: DualStationVllmPlan["local"]): string { return `${String(node.uid)}:${String(node.gid)}`; } -function runtimeHomeTmpfs(node: DualStationVllmPlan["local"]): string { - return `${VLLM_RUNTIME_HOME}:rw,nosuid,nodev,uid=${String(node.uid)},gid=${String(node.gid)},mode=0700,size=68719476736`; +function runtimeHomeTmpfs( + node: DualStationVllmPlan["local"], + execution: "exec" | "noexec", +): string { + return `${VLLM_RUNTIME_HOME}:rw,nosuid,nodev,${execution},uid=${String(node.uid)},gid=${String(node.gid)},mode=0700,size=68719476736`; } function appendRuntimeHomeEnv(args: string[]): void { @@ -406,7 +409,9 @@ function buildDualStationVllmBaseRunArgs( "--tmpfs", "/tmp:rw,nosuid,nodev,size=17179869184", "--tmpfs", - runtimeHomeTmpfs(node), + // Ray is installed under this home and loads its native _raylet module. + // Docker tmpfs mounts default to noexec, so managed serving opts in. + runtimeHomeTmpfs(node, "exec"), "--cap-drop", "ALL", // Non-root GPU/RDMA memory registration is bounded by this explicit rlimit; @@ -569,7 +574,7 @@ export function buildDualStationGpuSmokeRunArgs( "--tmpfs", "/tmp:rw,nosuid,nodev,size=17179869184", "--tmpfs", - runtimeHomeTmpfs(node), + runtimeHomeTmpfs(node, "noexec"), "--cap-drop", "ALL", "--ulimit", From dd03ed8e45fb0e5f1e1ffa7f1fdeb65d5e6a0a94 Mon Sep 17 00:00:00 2001 From: Senthil Ravichandran Date: Mon, 27 Jul 2026 14:39:16 -0700 Subject: [PATCH 67/74] fix(vllm): keep FlashInfer cubin links writable Signed-off-by: Senthil Ravichandran --- .../inference/vllm-station-cluster-lifecycle.test.ts | 1 + src/lib/inference/vllm-station-cluster-lifecycle.ts | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/src/lib/inference/vllm-station-cluster-lifecycle.test.ts b/src/lib/inference/vllm-station-cluster-lifecycle.test.ts index b265ac256a..97d7ea907a 100644 --- a/src/lib/inference/vllm-station-cluster-lifecycle.test.ts +++ b/src/lib/inference/vllm-station-cluster-lifecycle.test.ts @@ -253,6 +253,7 @@ describe("dual-Station managed vLLM run argv", () => { expect(dockerValues(args, "--workdir")).toEqual(["/home/vllm"]); expect(dockerValues(args, "--tmpfs")).toEqual([ "/tmp:rw,nosuid,nodev,size=17179869184", + `/usr/local/lib/python3.12/dist-packages/flashinfer_cubin/cubins/flashinfer:rw,nosuid,nodev,noexec,uid=${String(expectedNode.uid)},gid=${String(expectedNode.gid)},mode=0700,size=16777216`, `/home/vllm:rw,nosuid,nodev,exec,uid=${String(expectedNode.uid)},gid=${String(expectedNode.gid)},mode=0700,size=68719476736`, ]); expect(dockerValues(args, "--user")).toEqual([ diff --git a/src/lib/inference/vllm-station-cluster-lifecycle.ts b/src/lib/inference/vllm-station-cluster-lifecycle.ts index 3f6c90a12d..8c0f9b3f6d 100644 --- a/src/lib/inference/vllm-station-cluster-lifecycle.ts +++ b/src/lib/inference/vllm-station-cluster-lifecycle.ts @@ -42,6 +42,8 @@ const HEAD_API_PORT = 8000; const VLLM_RUNTIME_HOME = "/home/vllm"; const HF_CACHE_CONTAINER_DIR = `${VLLM_RUNTIME_HOME}/.cache/huggingface`; const HF_HUB_CACHE_CONTAINER_DIR = "/model-cache"; +const FLASHINFER_CUBIN_RUNTIME_DIR = + "/usr/local/lib/python3.12/dist-packages/flashinfer_cubin/cubins/flashinfer"; const DOCKER_INSPECT_TIMEOUT_MS = 10_000; const DOCKER_MUTATION_TIMEOUT_MS = 60_000; const DOCKER_GPU_SMOKE_TIMEOUT_MS = 30_000; @@ -374,6 +376,10 @@ function runtimeHomeTmpfs( return `${VLLM_RUNTIME_HOME}:rw,nosuid,nodev,${execution},uid=${String(node.uid)},gid=${String(node.gid)},mode=0700,size=68719476736`; } +function flashinferCubinTmpfs(node: DualStationVllmPlan["local"]): string { + return `${FLASHINFER_CUBIN_RUNTIME_DIR}:rw,nosuid,nodev,noexec,uid=${String(node.uid)},gid=${String(node.gid)},mode=0700,size=16777216`; +} + function appendRuntimeHomeEnv(args: string[]): void { appendEnv(args, "HOME", VLLM_RUNTIME_HOME); appendEnv(args, "USER", "vllm"); @@ -409,6 +415,11 @@ function buildDualStationVllmBaseRunArgs( "--tmpfs", "/tmp:rw,nosuid,nodev,size=17179869184", "--tmpfs", + // FlashInfer links generated-kernel headers beneath its installed cubin + // package. Keep only that empty subtree writable; packaged cubins and the + // rest of the image remain read-only. + flashinferCubinTmpfs(node), + "--tmpfs", // Ray is installed under this home and loads its native _raylet module. // Docker tmpfs mounts default to noexec, so managed serving opts in. runtimeHomeTmpfs(node, "exec"), From a3b7d268ec05fbfdf393e3645d870e54031dce95 Mon Sep 17 00:00:00 2001 From: Senthil Ravichandran Date: Mon, 27 Jul 2026 16:14:47 -0700 Subject: [PATCH 68/74] fix(vllm): clean up dual Station runtime Signed-off-by: Senthil Ravichandran --- docs/inference/set-up-vllm.mdx | 4 + docs/manage-sandboxes/uninstall-nemoclaw.mdx | 4 + docs/reference/commands.mdx | 3 + docs/reference/host-files-and-state.mdx | 4 + .../uninstall/run-plan-dual-station.test.ts | 124 ++++++++ src/lib/actions/uninstall/run-plan.test.ts | 2 +- src/lib/actions/uninstall/run-plan.ts | 45 +++ src/lib/inference/vllm-dual-station.test.ts | 61 ++++ .../vllm-station-runtime-cleanup-entry.ts | 21 ++ .../vllm-station-runtime-receipt-path.ts | 4 + .../vllm-station-runtime-receipt.test.ts | 255 +++++++++++++++ .../inference/vllm-station-runtime-receipt.ts | 297 ++++++++++++++++++ src/lib/inference/vllm-station-ssh-binding.ts | 34 ++ src/lib/inference/vllm.ts | 18 ++ 14 files changed, 875 insertions(+), 1 deletion(-) create mode 100644 src/lib/actions/uninstall/run-plan-dual-station.test.ts create mode 100644 src/lib/inference/vllm-station-runtime-cleanup-entry.ts create mode 100644 src/lib/inference/vllm-station-runtime-receipt-path.ts create mode 100644 src/lib/inference/vllm-station-runtime-receipt.test.ts create mode 100644 src/lib/inference/vllm-station-runtime-receipt.ts diff --git a/docs/inference/set-up-vllm.mdx b/docs/inference/set-up-vllm.mdx index 1a0be80ba8..8e997ea8a4 100644 --- a/docs/inference/set-up-vllm.mdx +++ b/docs/inference/set-up-vllm.mdx @@ -213,6 +213,10 @@ Configure the physical rails and SSH host-key and authentication trust before in NemoClaw does not modify rail configuration, enroll SSH trust, or reboot either host automatically. If the local host requires a reboot during initial preparation, the installer stops with status `10`; its owner-only receipt preserves the printed exact revision and Express selections. After reciprocal peer qualification begins, the pair state additionally binds SSH, GPU, and rail identity and names either host that must be rebooted manually. +After the managed pair passes readiness and container validation, NemoClaw writes an owner-only cleanup receipt and a copied SSH binding under the selected gateway state root. +The receipt contains no serving API key. +It records the exact peer, cluster, and GPU identities needed to revalidate and remove both managed containers during full uninstall. +If NemoClaw cannot write this receipt, setup stops and rolls back a newly started pair instead of leaving a runtime that full uninstall cannot safely reach. The registered two-Station Ultra recipe tracks the [latest NVIDIA dual-Station playbook](https://build.nvidia.com/station/nemoclaw/dual-nodes). It pins vLLM `0.25.1` and Ray `2.56.0`, uses one tensor-parallel rank per Station with pipeline parallelism across the pair, serves the `nemotron-ultra` alias with a `262144`-token model limit, and retains the Nemotron reasoning and tool-call parsers. The single-Station fallback keeps NemoClaw's bridge-networked managed-inference topology and publishes port `8000` through Docker. diff --git a/docs/manage-sandboxes/uninstall-nemoclaw.mdx b/docs/manage-sandboxes/uninstall-nemoclaw.mdx index 0ff274df89..d24643b606 100644 --- a/docs/manage-sandboxes/uninstall-nemoclaw.mdx +++ b/docs/manage-sandboxes/uninstall-nemoclaw.mdx @@ -38,6 +38,10 @@ A custom-port uninstall does not stop or remove the default gateway service or i For the default gateway, the uninstall command preserves `~/.nemoclaw/rebuild-backups/`, `~/.nemoclaw/backups/`, and `~/.nemoclaw/sandboxes.json` by default. A non-default gateway uses the corresponding entries under `~/.nemoclaw/gateways//`. When uninstall confirms that no sibling gateways remain, it also removes the shared CLI, services, images, providers, configuration, models, and swap. +If the selected state root contains a managed dual-Station cleanup receipt, NemoClaw first revalidates the recorded peer, cluster, and GPU identities and removes both exact managed vLLM containers. +Only after that pair cleanup succeeds does NemoClaw start the remaining full-uninstall steps. +If validation or pair cleanup fails, uninstall exits nonzero and keeps the receipt so you can resolve the reported SSH, Docker, or peer-host error and retry. +Pair cleanup can partially complete before an error, so inspect both Stations before retrying. When sibling gateways remain, it removes only the selected gateway's resources and port-scoped state and preserves those shared host resources. If the OpenShell command is unavailable or its gateway list cannot be read, uninstall cannot confirm that the selected gateway is the last one, so it uses the same scoped path and preserves the shared resources. When the command itself is unavailable, uninstall exits nonzero before OpenShell cleanup so you can restore the command and retry. diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 75590a9fbb..a76c3ffcf9 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -3297,6 +3297,9 @@ It prints the versioned URL of the matching `uninstall.sh` so you can download, When the gateway is externally supervised, uninstall preserves its process, Docker resources, and OpenShell binaries. It still deletes the selected sandboxes and attempts to remove the modern local gateway registration. When uninstall confirms that no sibling gateways remain, it also deletes NemoClaw provider registrations. +For a managed dual-Station vLLM runtime, full uninstall revalidates the exact recorded pair and removes both managed containers before starting the remaining uninstall steps. +If that cleanup fails, uninstall exits nonzero, preserves its owner-only cleanup receipt, and tells you to resolve the reported peer error before retrying. +Pair cleanup can partially complete before an error; verify both Stations before the retry. It does not use the legacy `gateway destroy` command for that gateway. diff --git a/docs/reference/host-files-and-state.mdx b/docs/reference/host-files-and-state.mdx index abb95336a0..17d3d2b238 100644 --- a/docs/reference/host-files-and-state.mdx +++ b/docs/reference/host-files-and-state.mdx @@ -24,6 +24,8 @@ Share redacted diagnostics only. ## Files +In the table, `` is `~/.nemoclaw/` for the default gateway or `~/.nemoclaw/gateways//` for a non-default gateway. + | Path | Purpose | Safe to delete | |---|---|---| | `~/.nemoclaw/config.json` | Host-level CLI configuration and defaults created by onboarding or config commands. | Only if you want NemoClaw to forget host defaults and rebuild them on the next setup. | @@ -32,6 +34,8 @@ Share redacted diagnostics only. | `~/.nemoclaw/onboard-session.json` | Resume marker for an onboarding attempt that failed before completion. | Yes, when you intentionally want to discard the failed session and start over. Prefer `$$nemoclaw onboard --fresh` when available. | | `~/.nemoclaw/usage-notice.json` | Records the third-party software notice version in `acceptedVersion` and the acceptance time in `acceptedAt`. Install, onboarding, and rebuild flows consult this file and prompt again when its recorded version differs from the current notice or the file is absent. | Yes; deleting it makes the next applicable install, onboarding, or rebuild flow prompt for acceptance again. | | `~/.nemoclaw/ollama-proxy-token` | Local auth token used by the host-side Ollama auth proxy. | Yes, but re-run onboarding afterward so NemoClaw recreates and registers the proxy token. | +| `/dual-station-vllm-runtime.json` | Owner-only managed dual-Station cleanup receipt. It contains no serving API key and binds the peer, cluster, and GPU identities used to revalidate and remove both managed vLLM containers during full uninstall. | No while the managed pair exists. Run `$$nemoclaw uninstall`; it removes the receipt after both exact containers are removed. | +| `/dual-station-vllm-runtime.json.ssh-binding/` | Owner-only copied SSH host-key and Docker-command binding needed to reach the recorded worker during full uninstall. | No while the managed pair exists. It is removed with the cleanup receipt after pair cleanup succeeds. | `sandboxes.json` is the current registry file name. If you see `registry.json` in older tests, notes, or discussions, treat it as legacy wording for the sandbox registry unless a specific release note says otherwise. diff --git a/src/lib/actions/uninstall/run-plan-dual-station.test.ts b/src/lib/actions/uninstall/run-plan-dual-station.test.ts new file mode 100644 index 0000000000..3570e727f3 --- /dev/null +++ b/src/lib/actions/uninstall/run-plan-dual-station.test.ts @@ -0,0 +1,124 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it, vi } from "vitest"; + +import { + type RunResult, + runUninstallPlan as runUninstallPlanBase, + type UninstallRunDeps, + type UninstallRunOptions, +} from "./run-plan"; + +function ok(stdout = ""): RunResult { + return { status: 0, stdout, stderr: "" }; +} + +function runUninstallPlan(options: UninstallRunOptions, deps: UninstallRunDeps) { + return runUninstallPlanBase(options, { + resolveGatewayTeardownAuthority: ({ gatewayName, gatewayPort }) => ({ + gatewayName, + gatewayPort, + mode: "nemoclaw-managed", + source: gatewayPort === 8080 ? "packaged-service" : "standalone", + endpoint: null, + stateDir: null, + supervisor: null, + requiredCapabilities: [], + }), + ...deps, + }); +} + +function okWithKnownGatewayList(command: string, args: readonly string[]): RunResult { + return command === "openshell" && args[0] === "gateway" && args[1] === "list" + ? ok(JSON.stringify([{ name: "nemoclaw" }])) + : ok(); +} + +describe("dual-Station runtime uninstall", () => { + it("removes a managed pair before the remaining full-uninstall steps", () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-uninstall-dual-pair-")); + const stateDir = path.join(home, ".nemoclaw"); + fs.mkdirSync(stateDir, { mode: 0o700 }); + fs.writeFileSync(path.join(stateDir, "dual-station-vllm-runtime.json"), "{}\n", { + mode: 0o600, + }); + const runDualStationRuntimeCleanup = vi.fn(() => ok()); + const rmSync = vi.fn(); + const runDocker = vi.fn(() => ok()); + + try { + const result = runUninstallPlan( + { assumeYes: true, deleteModels: false, keepOpenShell: true }, + { + commandExists: () => true, + env: { HOME: home, TMPDIR: home } as NodeJS.ProcessEnv, + existsSync: () => false, + isTty: false, + log: vi.fn(), + rmSync, + run: okWithKnownGatewayList, + runDocker, + runDualStationRuntimeCleanup, + }, + ); + + expect(result.exitCode).toBe(0); + expect(runDualStationRuntimeCleanup).toHaveBeenCalledOnce(); + expect(runDocker).toHaveBeenCalled(); + expect(runDualStationRuntimeCleanup.mock.invocationCallOrder[0]).toBeLessThan( + runDocker.mock.invocationCallOrder[0], + ); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + }); + + it("does not start the remaining uninstall steps when managed pair cleanup fails", () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-uninstall-dual-fail-")); + const stateDir = path.join(home, ".nemoclaw"); + fs.mkdirSync(stateDir, { mode: 0o700 }); + fs.writeFileSync(path.join(stateDir, "dual-station-vllm-runtime.json"), "{}\n", { + mode: 0o600, + }); + const errors: string[] = []; + const rmSync = vi.fn(); + const runDocker = vi.fn(() => ok()); + + try { + const result = runUninstallPlan( + { assumeYes: true, deleteModels: false, keepOpenShell: false }, + { + commandExists: () => true, + env: { HOME: home, TMPDIR: home } as NodeJS.ProcessEnv, + error: (message) => errors.push(message), + existsSync: () => false, + isTty: false, + log: vi.fn(), + rmSync, + run: okWithKnownGatewayList, + runDocker, + runDualStationRuntimeCleanup: () => ({ + status: 1, + stdout: "", + stderr: "peer unavailable", + }), + }, + ); + + expect(result.exitCode).toBe(1); + expect(rmSync).not.toHaveBeenCalled(); + expect(runDocker).not.toHaveBeenCalled(); + expect(errors).toContain( + "Managed dual-Station cleanup did not complete. NemoClaw did not start the remaining uninstall steps. Resolve the reported cleanup error and retry uninstall.", + ); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + }); +}); diff --git a/src/lib/actions/uninstall/run-plan.test.ts b/src/lib/actions/uninstall/run-plan.test.ts index 101f03410c..1eb4a92fc2 100644 --- a/src/lib/actions/uninstall/run-plan.test.ts +++ b/src/lib/actions/uninstall/run-plan.test.ts @@ -10,9 +10,9 @@ import { describe, expect, it, vi } from "vitest"; import { buildRunPlan, type RunResult, + runUninstallPlan as runUninstallPlanBase, type UninstallRunDeps, type UninstallRunOptions, - runUninstallPlan as runUninstallPlanBase, } from "./run-plan"; function ok(stdout = ""): RunResult { diff --git a/src/lib/actions/uninstall/run-plan.ts b/src/lib/actions/uninstall/run-plan.ts index 4e6d7eb3a6..77c3676042 100644 --- a/src/lib/actions/uninstall/run-plan.ts +++ b/src/lib/actions/uninstall/run-plan.ts @@ -27,6 +27,7 @@ import { } from "../../domain/uninstall/paths"; import { buildUninstallPlan, type UninstallPlan } from "../../domain/uninstall/plan"; import { isOllamaAuthProxyCommandLine } from "../../inference/ollama/process"; +import { DUAL_STATION_VLLM_RUNTIME_RECEIPT_FILE } from "../../inference/vllm-station-runtime-receipt-path"; import { buildDockerGatewayDebEnvFile } from "../../onboard/docker-driver-gateway-env"; import { getNemoclawOpenShellGatewayUserServicePath, @@ -88,6 +89,7 @@ export interface UninstallRunDeps { rmSync?: typeof fs.rmSync; run?: (command: string, args: string[], options?: SpawnSyncOptions) => RunResult; runDocker?: (args: string[], options?: SpawnSyncOptions) => RunResult; + runDualStationRuntimeCleanup?: (options?: SpawnSyncOptions) => RunResult; } export interface UninstallRunOutcome { @@ -334,6 +336,7 @@ interface UninstallRuntime { rmSync: typeof fs.rmSync; run: (command: string, args: string[], options?: SpawnSyncOptions) => RunResult; runDocker: (args: string[], options?: SpawnSyncOptions) => RunResult; + runDualStationRuntimeCleanup: (options?: SpawnSyncOptions) => RunResult; warn: (message: string) => void; } @@ -367,6 +370,22 @@ function buildRuntime(deps: UninstallRunDeps): UninstallRuntime { rmSync: deps.rmSync ?? fs.rmSync, run: deps.run ?? defaultRun, runDocker: deps.runDocker ?? defaultRunDocker, + runDualStationRuntimeCleanup: + deps.runDualStationRuntimeCleanup ?? + ((options = {}) => + defaultRun( + process.execPath, + [ + path.resolve( + __dirname, + "..", + "..", + "inference", + "vllm-station-runtime-cleanup-entry.js", + ), + ], + options, + )), warn: deps.error ?? ((message) => console.warn(message)), }; } @@ -1183,6 +1202,29 @@ function dockerIsAvailable(runtime: UninstallRuntime): boolean { return true; } +function removeManagedDualStationRuntime( + paths: UninstallPaths, + runtime: UninstallRuntime, +): boolean { + const receiptPath = path.join(paths.nemoclawStateDir, DUAL_STATION_VLLM_RUNTIME_RECEIPT_FILE); + try { + fs.lstatSync(receiptPath); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return true; + runtime.error(`Could not inspect managed dual-Station rollback state: ${formatError(error)}`); + return false; + } + const result = runtime.runDualStationRuntimeCleanup({ + env: runtime.env, + stdio: "inherit", + }); + if (result.status === 0) return true; + runtime.error( + "Managed dual-Station cleanup did not complete. NemoClaw did not start the remaining uninstall steps. Resolve the reported cleanup error and retry uninstall.", + ); + return false; +} + function removeDockerContainers(runtime: UninstallRuntime, gatewayName?: string): void { const result = runtime.runDocker(["ps", "-a", "--format", "{{.ID}} {{.Image}} {{.Names}}"], { env: runtime.env, @@ -1508,6 +1550,9 @@ function executePlan( for (const [index, step] of plan.steps.entries()) { runtime.log(`[${index + 1}/${plan.steps.length}] ${planStepDisplayName(step.name, branding)}`); if (step.name === "Stopping services") { + if (!scopedToSelectedGateway && !removeManagedDualStationRuntime(paths, runtime)) { + return { ok: false }; + } if ( !options.keepOpenShell && !externallySupervised && diff --git a/src/lib/inference/vllm-dual-station.test.ts b/src/lib/inference/vllm-dual-station.test.ts index fa042efe81..8c1d97d65c 100644 --- a/src/lib/inference/vllm-dual-station.test.ts +++ b/src/lib/inference/vllm-dual-station.test.ts @@ -27,6 +27,7 @@ const mocks = vi.hoisted(() => ({ measureDirectorySizeBytes: vi.fn(), preflightGpuRuntime: vi.fn(), preflightOwnership: vi.fn(), + persistRuntimeReceipt: vi.fn(), probeCapability: vi.fn(), probeDockerStorage: vi.fn(), probeHostStorage: vi.fn(), @@ -90,6 +91,10 @@ vi.mock("./vllm-station-cluster-lifecycle", () => ({ withDualStationManagedVllmLifecycle: mocks.withLifecycle, })); +vi.mock("./vllm-station-runtime-receipt", () => ({ + persistDualStationVllmRuntimeReceipt: mocks.persistRuntimeReceipt, +})); + vi.mock("./vllm-api-key", () => ({ ensureDualStationVllmApiKey: mocks.ensureApiKey, loadDualStationVllmApiKey: mocks.loadApiKey, @@ -234,6 +239,7 @@ beforeEach(() => { mocks.areContainersRunning.mockReturnValue(true); mocks.cleanup.mockReturnValue({ ok: true, removedContainerIds: [] }); mocks.commitLegacyMigration.mockResolvedValue({ ok: true, cleanupWarnings: [] }); + mocks.persistRuntimeReceipt.mockImplementation(() => {}); mocks.rollbackLegacyMigration.mockResolvedValue({ ok: true }); mocks.measureDirectorySizeBytes.mockReturnValue(0n); mocks.probeDockerStorage.mockReturnValue({ @@ -314,6 +320,9 @@ describe("dual DGX Station vLLM install orchestration", () => { expect(lifecycleActive).toBe(true); return true; }); + mocks.persistRuntimeReceipt.mockImplementation(() => { + expect(lifecycleActive).toBe(true); + }); mocks.stageModelSnapshot.mockImplementation(async () => { expect(lifecycleActive).toBe(true); return { ok: true, transferred: false }; @@ -361,6 +370,7 @@ describe("dual DGX Station vLLM install orchestration", () => { mocks.dockerSpawn.mock.invocationCallOrder[0], ); expect(mocks.areContainersRunning).toHaveBeenCalledWith(clusterPlan); + expect(mocks.persistRuntimeReceipt).toHaveBeenCalledWith(clusterPlan); expect(mocks.withLifecycle).toHaveBeenCalledTimes(2); expect(lifecycleActive).toBe(false); expect(mocks.probeCapability).toHaveBeenCalledTimes(2); @@ -383,6 +393,57 @@ describe("dual DGX Station vLLM install orchestration", () => { expect(mocks.dockerImageInspectFormat.mock.calls[0][2].env.DOCKER_CONTEXT).toBe("default"); }); + it("rolls back a newly started pair when durable rollback state cannot be written", async () => { + const clusterPlan = plan(); + mocks.probeCapability.mockReturnValue({ + kind: "ready", + plan: clusterPlan, + peerModelSnapshot: "ready", + }); + mocks.persistRuntimeReceipt.mockImplementation(() => { + throw new Error("receipt write failed"); + }); + const profile = detectVllmProfile({ platform: "station", type: "nvidia" }); + + await expect( + installVllm(profile!, { hasImage: true, nonInteractive: true, promptFn: vi.fn() }), + ).resolves.toEqual({ ok: false }); + + expect(mocks.cleanup).toHaveBeenCalledWith(clusterPlan); + expect(errorSpy).toHaveBeenCalledWith( + expect.stringContaining("could not persist the dual-Station cleanup receipt"), + ); + }); + + it("removes a committed migrated pair when durable rollback state cannot be written", async () => { + const clusterPlan = plan(); + mocks.probeCapability.mockReturnValue({ + kind: "ready", + plan: clusterPlan, + peerModelSnapshot: "ready", + }); + mocks.startManaged.mockReturnValue({ + ok: true, + baseUrl: HEAD_BASE_URL, + headContainerId: HEAD_ID, + workerContainerId: WORKER_ID, + reusedExisting: false, + legacyMigration: LEGACY_MIGRATION, + }); + mocks.persistRuntimeReceipt.mockImplementation(() => { + throw new Error("receipt write failed"); + }); + const profile = detectVllmProfile({ platform: "station", type: "nvidia" }); + + await expect( + installVllm(profile!, { hasImage: true, nonInteractive: true, promptFn: vi.fn() }), + ).resolves.toEqual({ ok: false }); + + expect(mocks.commitLegacyMigration).toHaveBeenCalledWith(clusterPlan, LEGACY_MIGRATION); + expect(mocks.cleanup).toHaveBeenCalledWith(clusterPlan); + expect(mocks.rollbackLegacyMigration).not.toHaveBeenCalled(); + }); + it("selects pinned Nemotron Ultra without prompting when an explicit peer qualifies", async () => { delete process.env.NEMOCLAW_VLLM_MODEL; const promptFn = vi.fn(); diff --git a/src/lib/inference/vllm-station-runtime-cleanup-entry.ts b/src/lib/inference/vllm-station-runtime-cleanup-entry.ts new file mode 100644 index 0000000000..c0f0821b41 --- /dev/null +++ b/src/lib/inference/vllm-station-runtime-cleanup-entry.ts @@ -0,0 +1,21 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { cleanupInstalledDualStationVllmRuntime } from "./vllm-station-runtime-receipt"; + +async function main(): Promise { + const result = await cleanupInstalledDualStationVllmRuntime(); + if (result.kind === "not-installed") return; + console.log( + `Removed managed dual-Station vLLM containers: ${result.removedContainerIds.join(", ")}`, + ); +} + +main().catch((error: unknown) => { + console.error( + `Refusing uninstall before managed dual-Station cleanup: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + process.exitCode = 1; +}); diff --git a/src/lib/inference/vllm-station-runtime-receipt-path.ts b/src/lib/inference/vllm-station-runtime-receipt-path.ts new file mode 100644 index 0000000000..42e96cb474 --- /dev/null +++ b/src/lib/inference/vllm-station-runtime-receipt-path.ts @@ -0,0 +1,4 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export const DUAL_STATION_VLLM_RUNTIME_RECEIPT_FILE = "dual-station-vllm-runtime.json"; diff --git a/src/lib/inference/vllm-station-runtime-receipt.test.ts b/src/lib/inference/vllm-station-runtime-receipt.test.ts new file mode 100644 index 0000000000..88d3834f2e --- /dev/null +++ b/src/lib/inference/vllm-station-runtime-receipt.test.ts @@ -0,0 +1,255 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { DUAL_STATION_VLLM_RUNTIME, type DualStationVllmPlan } from "./vllm-station-cluster"; +import { + cleanupInstalledDualStationVllmRuntime, + dualStationVllmRuntimeReceiptPath, + persistDualStationVllmRuntimeReceipt, +} from "./vllm-station-runtime-receipt"; +import { + createDualStationSshBindingFixture, + type DualStationSshBindingFixture, +} from "./vllm-station-ssh-binding.test-support"; + +let root: string; +let stateDir: string; +let sshFixture: DualStationSshBindingFixture; + +beforeEach(() => { + root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-dual-runtime-receipt-")); + stateDir = path.join(root, ".nemoclaw"); + sshFixture = createDualStationSshBindingFixture(); +}); + +afterEach(() => { + vi.unstubAllEnvs(); + vi.resetModules(); + sshFixture.cleanup(); + fs.rmSync(root, { recursive: true, force: true }); +}); + +function plan(): DualStationVllmPlan { + return { + peerSshBinding: sshFixture.binding, + runtime: DUAL_STATION_VLLM_RUNTIME, + local: { + hostname: "station-a", + home: "/home/nvidia", + uid: 1000, + gid: 1000, + gpu: { + index: 0, + name: "NVIDIA GB300", + uuid: "GPU-aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", + }, + }, + peer: { + hostname: "station-b", + home: "/home/nvidia", + uid: 1000, + gid: 1000, + gpu: { + index: 0, + name: "NVIDIA GB300", + uuid: "GPU-99999999-8888-7777-6666-555555555555", + }, + }, + rails: [ + { + index: 0, + subnet: "192.168.240.0/30", + local: { + rdmaDevice: "mlx5_0", + netdev: "cx8r0", + macAddress: "02:00:00:00:00:01", + uverbsDevice: "/dev/infiniband/uverbs0", + pciAddress: "0001:03:00.0", + address: "192.168.240.1", + }, + peer: { + rdmaDevice: "mlx5_0", + netdev: "cx8r0", + macAddress: "02:00:00:00:00:02", + uverbsDevice: "/dev/infiniband/uverbs0", + pciAddress: "0002:03:00.0", + address: "192.168.240.2", + }, + }, + { + index: 1, + subnet: "192.168.240.4/30", + local: { + rdmaDevice: "mlx5_1", + netdev: "cx8r1", + macAddress: "02:00:00:00:00:05", + uverbsDevice: "/dev/infiniband/uverbs1", + pciAddress: "0001:03:00.1", + address: "192.168.240.5", + }, + peer: { + rdmaDevice: "mlx5_1", + netdev: "cx8r1", + macAddress: "02:00:00:00:00:06", + uverbsDevice: "/dev/infiniband/uverbs1", + pciAddress: "0002:03:00.1", + address: "192.168.240.6", + }, + }, + ], + masterAddress: "192.168.240.1", + roceGidIndex: 3, + }; +} + +describe("dual-Station vLLM runtime rollback receipt", () => { + it("uses the selected gateway state root", async () => { + vi.stubEnv("NEMOCLAW_GATEWAY_PORT", "18080"); + vi.resetModules(); + + const { dualStationVllmRuntimeReceiptPath: selectedReceiptPath } = await import( + "./vllm-station-runtime-receipt" + ); + + expect(selectedReceiptPath()).toBe( + path.join(os.homedir(), ".nemoclaw", "gateways", "18080", "dual-station-vllm-runtime.json"), + ); + }); + + it("writes a private cleanup receipt and removes both exact containers before retiring it", async () => { + const expectedPlan = plan(); + persistDualStationVllmRuntimeReceipt(expectedPlan, { stateDir }); + const receiptPath = dualStationVllmRuntimeReceiptPath(stateDir); + expect(fs.statSync(receiptPath).mode & 0o777).toBe(0o600); + expect(fs.lstatSync(`${receiptPath}.ssh-binding`).isDirectory()).toBe(true); + + const cleanupManagedVllm = vi.fn(async () => ({ + ok: true as const, + removedContainerIds: ["worker-id", "head-id"], + })); + const probeCapability = vi.fn(() => ({ + kind: "ready" as const, + plan: expectedPlan, + peerModelSnapshot: "ready" as const, + })); + + await expect( + cleanupInstalledDualStationVllmRuntime({ + stateDir, + cleanupManagedVllm, + probeCapability, + }), + ).resolves.toEqual({ + kind: "removed", + removedContainerIds: ["worker-id", "head-id"], + }); + expect(cleanupManagedVllm).toHaveBeenCalledWith(expectedPlan); + expect(fs.existsSync(receiptPath)).toBe(false); + expect(fs.existsSync(`${receiptPath}.ssh-binding`)).toBe(false); + }); + + it("preserves rollback state when the qualified pair no longer matches", async () => { + const expectedPlan = plan(); + persistDualStationVllmRuntimeReceipt(expectedPlan, { stateDir }); + const changedPlan = { + ...expectedPlan, + peer: { + ...expectedPlan.peer, + gpu: { + ...expectedPlan.peer.gpu, + uuid: "GPU-bbbbbbbb-cccc-dddd-eeee-ffffffffffff", + }, + }, + }; + const cleanupManagedVllm = vi.fn(); + + await expect( + cleanupInstalledDualStationVllmRuntime({ + stateDir, + cleanupManagedVllm, + probeCapability: () => ({ + kind: "ready", + plan: changedPlan, + peerModelSnapshot: "ready", + }), + }), + ).rejects.toThrow("runtime identity changed"); + expect(cleanupManagedVllm).not.toHaveBeenCalled(); + expect(fs.existsSync(dualStationVllmRuntimeReceiptPath(stateDir))).toBe(true); + }); + + it("refuses to overwrite rollback ownership with a different pair", () => { + const expectedPlan = plan(); + persistDualStationVllmRuntimeReceipt(expectedPlan, { stateDir }); + const receiptPath = dualStationVllmRuntimeReceiptPath(stateDir); + const original = fs.readFileSync(receiptPath, "utf8"); + const changedPlan = { + ...expectedPlan, + peer: { + ...expectedPlan.peer, + gpu: { + ...expectedPlan.peer.gpu, + uuid: "GPU-bbbbbbbb-cccc-dddd-eeee-ffffffffffff", + }, + }, + }; + + expect(() => persistDualStationVllmRuntimeReceipt(changedPlan, { stateDir })).toThrow( + "different managed dual-Station runtime receipt", + ); + expect(fs.readFileSync(receiptPath, "utf8")).toBe(original); + }); + + it("refuses a symbolic-link receipt before peer probing or cleanup", async () => { + const receiptPath = dualStationVllmRuntimeReceiptPath(stateDir); + fs.mkdirSync(stateDir, { mode: 0o700 }); + const target = path.join(root, "redirected-receipt.json"); + fs.writeFileSync(target, "{}\n", { mode: 0o600 }); + fs.symlinkSync(target, receiptPath); + const probeCapability = vi.fn(); + const cleanupManagedVllm = vi.fn(); + + await expect( + cleanupInstalledDualStationVllmRuntime({ + stateDir, + probeCapability, + cleanupManagedVllm, + }), + ).rejects.toThrow("symbolic link"); + expect(probeCapability).not.toHaveBeenCalled(); + expect(cleanupManagedVllm).not.toHaveBeenCalled(); + }); + + it("refuses a non-private receipt before peer probing or cleanup", async () => { + const expectedPlan = plan(); + persistDualStationVllmRuntimeReceipt(expectedPlan, { stateDir }); + fs.chmodSync(dualStationVllmRuntimeReceiptPath(stateDir), 0o644); + const probeCapability = vi.fn(); + const cleanupManagedVllm = vi.fn(); + + await expect( + cleanupInstalledDualStationVllmRuntime({ + stateDir, + probeCapability, + cleanupManagedVllm, + }), + ).rejects.toThrow("private regular file"); + expect(probeCapability).not.toHaveBeenCalled(); + expect(cleanupManagedVllm).not.toHaveBeenCalled(); + }); + + it("does nothing when no managed pair receipt exists", async () => { + await expect( + cleanupInstalledDualStationVllmRuntime({ + stateDir, + probeCapability: vi.fn(), + cleanupManagedVllm: vi.fn(), + }), + ).resolves.toEqual({ kind: "not-installed" }); + }); +}); diff --git a/src/lib/inference/vllm-station-runtime-receipt.ts b/src/lib/inference/vllm-station-runtime-receipt.ts new file mode 100644 index 0000000000..2d5674e822 --- /dev/null +++ b/src/lib/inference/vllm-station-runtime-receipt.ts @@ -0,0 +1,297 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { getNemoclawStateRoot } from "../state/state-root"; +import { ensureLocalAdapterStateDir } from "./local-adapter-lifecycle"; +import { buildLocalDualStationDockerEnv } from "./vllm-docker-env"; +import { + type DualStationVllmPlan, + NEMOCLAW_DGX_STATION_PEER_ENV, + probeDualStationVllmCapability, +} from "./vllm-station-cluster"; +import { + cleanupDualStationManagedVllm, + dualStationVllmClusterId, +} from "./vllm-station-cluster-lifecycle"; +import { DUAL_STATION_VLLM_RUNTIME_RECEIPT_FILE } from "./vllm-station-runtime-receipt-path"; +import { + clearDualStationSshBinding, + copyDualStationSshBinding, + encodeDualStationSshBindingHandoff, + NEMOCLAW_DGX_STATION_SSH_BINDING_ENV, +} from "./vllm-station-ssh-binding"; + +const RECEIPT_SCHEMA_VERSION = 1; +const MAX_RECEIPT_BYTES = 16 * 1024; +const SHA256_HEX_PATTERN = /^[a-f0-9]{64}$/; +const GPU_UUID_PATTERN = /^GPU-[A-Za-z0-9-]+$/; +const RECEIPT_KEYS = [ + "clusterId", + "localGpuUuid", + "peerGpuUuid", + "peerTarget", + "schemaVersion", + "sshBinding", +] as const; + +interface DualStationVllmRuntimeReceipt { + schemaVersion: 1; + peerTarget: string; + sshBinding: string; + clusterId: string; + localGpuUuid: string; + peerGpuUuid: string; +} + +export interface DualStationVllmRuntimeReceiptOptions { + stateDir?: string; + /** @internal Test seams for the external probe and lifecycle mutation. */ + probeCapability?: typeof probeDualStationVllmCapability; + cleanupManagedVllm?: typeof cleanupDualStationManagedVllm; +} + +export type DualStationVllmRuntimeCleanupResult = + | { kind: "not-installed" } + | { kind: "removed"; removedContainerIds: string[] }; + +function defaultStateDir(): string { + return getNemoclawStateRoot(os.homedir()); +} + +export function dualStationVllmRuntimeReceiptPath(stateDir = defaultStateDir()): string { + return path.join(stateDir, DUAL_STATION_VLLM_RUNTIME_RECEIPT_FILE); +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function requireString(value: unknown, label: string, pattern: RegExp, maxLength: number): string { + if ( + typeof value !== "string" || + value.length === 0 || + value.length > maxLength || + value !== value.trim() || + !pattern.test(value) + ) { + throw new Error(`${label} is invalid`); + } + return value; +} + +function parseReceipt(value: unknown): DualStationVllmRuntimeReceipt { + if (!isRecord(value) || value.schemaVersion !== RECEIPT_SCHEMA_VERSION) { + throw new Error("Dual-Station vLLM runtime receipt schema is unsupported"); + } + const keys = Object.keys(value).sort(); + const expected = [...RECEIPT_KEYS].sort(); + if (keys.length !== expected.length || keys.some((key, index) => key !== expected[index])) { + throw new Error("Dual-Station vLLM runtime receipt fields are invalid"); + } + return { + schemaVersion: 1, + peerTarget: requireString( + value.peerTarget, + "Dual-Station vLLM runtime peer", + /^(?:[A-Za-z_][A-Za-z0-9._-]*@)?[A-Za-z0-9][A-Za-z0-9.-]{0,252}$/, + 286, + ), + sshBinding: requireString( + value.sshBinding, + "Dual-Station vLLM runtime SSH binding", + /^[A-Za-z0-9_-]+$/, + 8192, + ), + clusterId: requireString( + value.clusterId, + "Dual-Station vLLM runtime cluster", + SHA256_HEX_PATTERN, + 64, + ), + localGpuUuid: requireString( + value.localGpuUuid, + "Dual-Station vLLM local GPU", + GPU_UUID_PATTERN, + 128, + ), + peerGpuUuid: requireString( + value.peerGpuUuid, + "Dual-Station vLLM peer GPU", + GPU_UUID_PATTERN, + 128, + ), + }; +} + +function assertPrivateReceipt(stat: fs.Stats, filePath: string): void { + if (!stat.isFile() || (stat.mode & 0o777) !== 0o600) { + throw new Error( + `Dual-Station vLLM runtime receipt must be a private regular file: ${filePath}`, + ); + } + if (typeof process.getuid === "function" && stat.uid !== process.getuid()) { + throw new Error( + `Dual-Station vLLM runtime receipt is not owned by the current user: ${filePath}`, + ); + } +} + +function loadReceipt( + options: DualStationVllmRuntimeReceiptOptions = {}, +): DualStationVllmRuntimeReceipt | null { + const filePath = dualStationVllmRuntimeReceiptPath(options.stateDir ?? defaultStateDir()); + const noFollow = fs.constants.O_NOFOLLOW; + if (typeof noFollow !== "number") { + throw new Error("Secure no-follow file opens are unavailable on this platform"); + } + const nonBlock = fs.constants.O_NONBLOCK ?? 0; + let fd: number | undefined; + try { + try { + fd = fs.openSync(filePath, fs.constants.O_RDONLY | noFollow | nonBlock); + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + if (code === "ENOENT") return null; + if (code === "ELOOP") { + throw new Error( + `Refusing to read dual-Station vLLM runtime receipt through a symbolic link: ${filePath}`, + ); + } + throw error; + } + const opened = fs.fstatSync(fd); + assertPrivateReceipt(opened, filePath); + if (opened.size < 2 || opened.size > MAX_RECEIPT_BYTES) { + throw new Error(`Dual-Station vLLM runtime receipt is malformed: ${filePath}`); + } + return parseReceipt(JSON.parse(fs.readFileSync(fd, "utf8"))); + } catch (error) { + if (error instanceof SyntaxError) { + throw new Error(`Dual-Station vLLM runtime receipt is malformed: ${filePath}`); + } + throw error; + } finally { + if (fd !== undefined) fs.closeSync(fd); + } +} + +function fsyncDirectory(directory: string): void { + const fd = fs.openSync(directory, fs.constants.O_RDONLY); + try { + fs.fsyncSync(fd); + } finally { + fs.closeSync(fd); + } +} + +function writeReceipt( + receipt: DualStationVllmRuntimeReceipt, + options: DualStationVllmRuntimeReceiptOptions = {}, +): void { + const stateDir = options.stateDir ?? defaultStateDir(); + ensureLocalAdapterStateDir(stateDir); + const filePath = dualStationVllmRuntimeReceiptPath(stateDir); + const temporary = `${filePath}.tmp-${process.pid}-${Date.now().toString(16)}`; + let fd: number | undefined; + try { + fd = fs.openSync( + temporary, + fs.constants.O_WRONLY | fs.constants.O_CREAT | fs.constants.O_EXCL | fs.constants.O_NOFOLLOW, + 0o600, + ); + assertPrivateReceipt(fs.fstatSync(fd), temporary); + fs.writeFileSync(fd, `${JSON.stringify(receipt)}\n`, "utf8"); + fs.fsyncSync(fd); + fs.closeSync(fd); + fd = undefined; + fs.renameSync(temporary, filePath); + fsyncDirectory(stateDir); + } finally { + if (fd !== undefined) fs.closeSync(fd); + try { + fs.unlinkSync(temporary); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + } +} + +/** Persist the exact trusted peer needed to remove a successful managed pair. */ +export function persistDualStationVllmRuntimeReceipt( + plan: DualStationVllmPlan, + options: DualStationVllmRuntimeReceiptOptions = {}, +): void { + const stateDir = options.stateDir ?? defaultStateDir(); + ensureLocalAdapterStateDir(stateDir); + const clusterId = dualStationVllmClusterId(plan); + const existing = loadReceipt({ stateDir }); + if ( + existing && + (existing.peerTarget !== plan.peerSshBinding.peerTarget || + existing.clusterId !== clusterId || + existing.localGpuUuid !== plan.local.gpu.uuid || + existing.peerGpuUuid !== plan.peer.gpu.uuid) + ) { + throw new Error("A different managed dual-Station runtime receipt already owns rollback state"); + } + const receiptPath = dualStationVllmRuntimeReceiptPath(stateDir); + const runtimeBinding = copyDualStationSshBinding(receiptPath, plan.peerSshBinding); + writeReceipt( + { + schemaVersion: 1, + peerTarget: runtimeBinding.peerTarget, + sshBinding: encodeDualStationSshBindingHandoff(runtimeBinding), + clusterId, + localGpuUuid: plan.local.gpu.uuid, + peerGpuUuid: plan.peer.gpu.uuid, + }, + { stateDir }, + ); +} + +function clearReceipt(stateDir: string): void { + const filePath = dualStationVllmRuntimeReceiptPath(stateDir); + fs.unlinkSync(filePath); + clearDualStationSshBinding(filePath); + fsyncDirectory(stateDir); +} + +/** + * Remove both exact owned containers before ordinary uninstall can retire the + * controller state required to reach the worker. + */ +export async function cleanupInstalledDualStationVllmRuntime( + options: DualStationVllmRuntimeReceiptOptions = {}, +): Promise { + const stateDir = options.stateDir ?? defaultStateDir(); + const receipt = loadReceipt({ stateDir }); + if (!receipt) return { kind: "not-installed" }; + const capability = (options.probeCapability ?? probeDualStationVllmCapability)({ + env: buildLocalDualStationDockerEnv({ + [NEMOCLAW_DGX_STATION_PEER_ENV]: receipt.peerTarget, + [NEMOCLAW_DGX_STATION_SSH_BINDING_ENV]: receipt.sshBinding, + }), + }); + if (capability.kind !== "ready") { + const reason = + capability.kind === "unavailable" ? capability.reason : "runtime peer is not configured"; + throw new Error(`Could not revalidate the managed dual-Station pair: ${reason}`); + } + if ( + dualStationVllmClusterId(capability.plan) !== receipt.clusterId || + capability.plan.local.gpu.uuid !== receipt.localGpuUuid || + capability.plan.peer.gpu.uuid !== receipt.peerGpuUuid + ) { + throw new Error("Managed dual-Station runtime identity changed; refusing pair cleanup"); + } + const cleanup = await (options.cleanupManagedVllm ?? cleanupDualStationManagedVllm)( + capability.plan, + ); + if (!cleanup.ok) throw new Error(cleanup.reason); + clearReceipt(stateDir); + return { kind: "removed", removedContainerIds: cleanup.removedContainerIds }; +} diff --git a/src/lib/inference/vllm-station-ssh-binding.ts b/src/lib/inference/vllm-station-ssh-binding.ts index ceb413ec27..bfce00f32c 100644 --- a/src/lib/inference/vllm-station-ssh-binding.ts +++ b/src/lib/inference/vllm-station-ssh-binding.ts @@ -737,6 +737,40 @@ export function encodeDualStationSshBindingHandoff(binding: DualStationSshBindin return Buffer.from(JSON.stringify(handoff), "utf8").toString("base64url"); } +/** + * Promote an installer-qualified binding into a separate owner-only lifecycle + * tree before the temporary installer resume state is retired. + */ +export function copyDualStationSshBinding( + statePath: string, + binding: DualStationSshBinding, +): DualStationSshBinding { + const canonical = validateDualStationSshBindingFiles(binding); + const knownHosts = readBoundedFile( + canonical.knownHostsFile, + 0o600, + MAX_KNOWN_HOSTS_BYTES, + "Station SSH known-hosts file", + ) + .toString("utf8") + .trimEnd() + .split("\n"); + return writeDualStationSshBinding( + statePath, + { + requestedTarget: canonical.peerTarget, + sshTarget: canonical.peerTarget, + resolvedHost: canonical.resolvedHost, + sshUser: canonical.sshUser, + port: canonical.port, + lookupHost: canonical.lookupHost, + hostKeyDigest: canonical.hostKeyDigest, + knownHostsLines: knownHosts, + }, + { dockerCliFile: canonical.dockerCliFile }, + ); +} + export function loadDualStationSshBindingHandoff( token: string, expectedPeerTarget: string, diff --git a/src/lib/inference/vllm.ts b/src/lib/inference/vllm.ts index b72835c853..1b0eb7222e 100644 --- a/src/lib/inference/vllm.ts +++ b/src/lib/inference/vllm.ts @@ -69,6 +69,7 @@ import { withDualStationManagedVllmLifecycle, } from "./vllm-station-cluster-lifecycle"; import { stageDualStationModelSnapshot } from "./vllm-station-model-staging"; +import { persistDualStationVllmRuntimeReceipt } from "./vllm-station-runtime-receipt"; import { findUnwritableModelCachePath, formatStorageBytes, @@ -1869,6 +1870,7 @@ async function runVllmInstall( return { ok: false }; } + let legacyMigrationCommitted = false; if (start.legacyMigration) { const commit = await commitDualStationLegacyMigration( dualStationPlan, @@ -1882,6 +1884,22 @@ async function runVllmInstall( for (const warning of commit.cleanupWarnings) { console.error(` vLLM cleanup warning: ${warning}`); } + legacyMigrationCommitted = true; + } + + try { + persistDualStationVllmRuntimeReceipt(dualStationPlan); + } catch (error) { + if (legacyMigrationCommitted) { + const cleanup = await cleanupDualStationManagedVllm(dualStationPlan); + if (!cleanup.ok) console.error(` vLLM rollback warning: ${cleanup.reason}`); + } else { + await rollbackStartedPair(); + } + console.error( + ` vLLM install failed: could not persist the dual-Station cleanup receipt: ${(error as Error).message}`, + ); + return { ok: false }; } console.log(` ✓ vLLM ready across two DGX Stations at ${start.baseUrl}`); From 8cfde42c2155d20af20d6b1331d178fc69b61f22 Mon Sep 17 00:00:00 2001 From: Senthil Ravichandran Date: Mon, 27 Jul 2026 16:30:09 -0700 Subject: [PATCH 69/74] fix(installer): recognize current dual Station runtime Signed-off-by: Senthil Ravichandran --- docs/inference/set-up-vllm.mdx | 1 + scripts/install.sh | 2 +- src/lib/inference/vllm-station-cluster-lifecycle.ts | 2 +- test/install-station-pair-preparation.test.ts | 3 ++- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/inference/set-up-vllm.mdx b/docs/inference/set-up-vllm.mdx index 8e997ea8a4..1307ddf2a3 100644 --- a/docs/inference/set-up-vllm.mdx +++ b/docs/inference/set-up-vllm.mdx @@ -190,6 +190,7 @@ After the Stations pass reciprocal GPU, rail, MAC, route, neighbor, and jumbo-fr No managed-vLLM runtime image or model download starts before the pair-qualification or single-Station fallback decision completes. If no trusted pair qualifies, Express retains the existing single-Station Ultra recipe. Setting `NEMOCLAW_VLLM_MODEL` remains authoritative; setting `NEMOCLAW_DGX_STATION_PEER` requests that exact already-trusted peer and fails closed if it does not qualify. +When Express finds a complete running NemoClaw-managed dual-Station head, it defers the workload-rejecting host-preparation probe only while it revalidates the reciprocal pair and managed lifecycle contract; incomplete or mismatched workloads remain blocked. To select the existing `deepseek-v4-flash` recipe while retaining the same one-confirmation express flow, run: ```bash diff --git a/scripts/install.sh b/scripts/install.sh index 6e73907539..ff2ff7b1e6 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -4154,7 +4154,7 @@ station_managed_dual_head_running() { "$running" == "true" && "$managed" == "true" && "$role" == "head" && - "$schema" == "1" && + "$schema" == "2" && "$cluster" =~ ^[a-f0-9]{64}$ && "$launch_contract" =~ ^[a-f0-9]{64}$ && "$api_fingerprint" =~ ^[a-f0-9]{64}$ && diff --git a/src/lib/inference/vllm-station-cluster-lifecycle.ts b/src/lib/inference/vllm-station-cluster-lifecycle.ts index 8c0f9b3f6d..c9f6326a07 100644 --- a/src/lib/inference/vllm-station-cluster-lifecycle.ts +++ b/src/lib/inference/vllm-station-cluster-lifecycle.ts @@ -60,7 +60,7 @@ const SAFE_GPU_UUID_PATTERN = /^GPU-[A-Za-z0-9-]{8,123}$/; const GPU_SMOKE_NONCE_PATTERN = /^[a-f0-9]{32}$/; const TRANSACTION_ID_PATTERN = /^[a-f0-9]{32}$/; const GPU_SMOKE_CONTAINER_PREFIX = "nemoclaw-vllm-gpu-smoke"; -const DUAL_STATION_VLLM_LAUNCH_SCHEMA = "2"; +export const DUAL_STATION_VLLM_LAUNCH_SCHEMA = "2"; const VLLM_FINGERPRINT_CONTEXT = "nemoclaw-dual-station-vllm-api-key\0"; // Compatibility bridge for schema-less single-Station Ultra containers from // the v0.0.86 rollback window before dual launch schema 1. diff --git a/test/install-station-pair-preparation.test.ts b/test/install-station-pair-preparation.test.ts index e782b65ae8..3853bf895f 100644 --- a/test/install-station-pair-preparation.test.ts +++ b/test/install-station-pair-preparation.test.ts @@ -27,6 +27,7 @@ import { strictStationPrepSshTransportArgs, writeDualStationResumeState, } from "../scripts/prepare-dual-dgx-station.mts"; +import { DUAL_STATION_VLLM_LAUNCH_SCHEMA } from "../src/lib/inference/vllm-station-cluster-lifecycle.ts"; import { stationKnownHostsDigest, strictStationSshTransportArgs, @@ -1445,7 +1446,7 @@ ensure_station_express_pair "true", "true", "head", - "1", + DUAL_STATION_VLLM_LAUNCH_SCHEMA, "c".repeat(64), "d".repeat(64), "e".repeat(64), From 56fe654fa42219f5adb5d0ffc686b013a5dcb281 Mon Sep 17 00:00:00 2001 From: Senthil Ravichandran Date: Mon, 27 Jul 2026 16:39:26 -0700 Subject: [PATCH 70/74] fix(installer): reconcile recovered Station runtime Signed-off-by: Senthil Ravichandran --- docs/inference/set-up-vllm.mdx | 1 + scripts/install.sh | 3 ++- test/install-preexisting-sandbox-recovery.test.ts | 9 ++++++++- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/docs/inference/set-up-vllm.mdx b/docs/inference/set-up-vllm.mdx index 1307ddf2a3..589a7f2d06 100644 --- a/docs/inference/set-up-vllm.mdx +++ b/docs/inference/set-up-vllm.mdx @@ -191,6 +191,7 @@ No managed-vLLM runtime image or model download starts before the pair-qualifica If no trusted pair qualifies, Express retains the existing single-Station Ultra recipe. Setting `NEMOCLAW_VLLM_MODEL` remains authoritative; setting `NEMOCLAW_DGX_STATION_PEER` requests that exact already-trusted peer and fails closed if it does not qualify. When Express finds a complete running NemoClaw-managed dual-Station head, it defers the workload-rejecting host-preparation probe only while it revalidates the reciprocal pair and managed lifecycle contract; incomplete or mismatched workloads remain blocked. +If the installer also recovers existing sandboxes, it still reconciles the newly accepted Station Express runtime state before completing. To select the existing `deepseek-v4-flash` recipe while retaining the same one-confirmation express flow, run: ```bash diff --git a/scripts/install.sh b/scripts/install.sh index ff2ff7b1e6..4df5f8963b 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -4882,7 +4882,8 @@ main() { if [[ "${_PREEXISTING_SANDBOX_ORPHANED:-false}" == true ]]; then # #6520: do not claim recovery when recorded sandboxes are stranded. warn "Some recorded sandboxes could not be recovered; skipping generic onboarding." - elif [[ "${_STATION_EXPRESS_RESUME_LOADED:-}" == "1" ]] \ + elif [[ "${_SELECTED_EXPRESS_PLATFORM:-}" == "DGX Station" ]] \ + || [[ "${_STATION_EXPRESS_RESUME_LOADED:-}" == "1" ]] \ || station_express_receipt_retirement_pending; then info "Existing sandboxes recovered; reconciling DGX Station Express onboarding state." run_onboard || error "Onboarding did not complete successfully." diff --git a/test/install-preexisting-sandbox-recovery.test.ts b/test/install-preexisting-sandbox-recovery.test.ts index 87cf490d92..a55f924595 100644 --- a/test/install-preexisting-sandbox-recovery.test.ts +++ b/test/install-preexisting-sandbox-recovery.test.ts @@ -29,6 +29,7 @@ function runRecoveryBeforeOnboard( options: { registryJson?: string; singleSession?: boolean; + stationExpressSelected?: boolean; stationResumeLoaded?: boolean; prepareState?: (tmp: string) => void; } = {}, @@ -49,6 +50,7 @@ function runRecoveryBeforeOnboard( path.join(payloadLibDir, "station-vllm-conflict.sh"), ); fs.mkdirSync(path.join(tmp, ".nemoclaw")); + fs.chmodSync(path.join(tmp, ".nemoclaw"), 0o700); fs.writeFileSync( path.join(tmp, ".nemoclaw", "sandboxes.json"), options.registryJson ?? '{"sandboxes":{}}', @@ -97,7 +99,9 @@ exit 0 preflight_usage_notice_prompt() { :; } ensure_docker() { :; } ensure_openshell_build_deps() { :; } - maybe_offer_express_install() { :; } + maybe_offer_express_install() { + ${options.stationExpressSelected ? '_SELECTED_EXPRESS_PLATFORM="DGX Station"' : ":"} + } sleep() { printf 'sleep=%s\n' "$*" >> "${callLog}"; } step() { :; } install_nodejs() { :; } @@ -110,6 +114,8 @@ exit 0 install_nemoclaw() { :; } verify_nemoclaw() { _CLI_PATH="${cli}"; } run_installer_host_preflight() { return 0; } + ensure_station_express_host() { :; } + ensure_station_express_pair() { :; } run_onboard() { "${cli}" onboard; } restore_onboard_forward_after_post_checks() { return 0; } print_done() { printf 'PRINT_DONE\n'; } @@ -148,6 +154,7 @@ describe("install.sh pre-existing sandbox recovery ordering (#6114)", () => { }); it.each([ + ["a selected Station Express attempt", { stationExpressSelected: true }], ["a loaded Station receipt", { stationResumeLoaded: true }], [ "a pending Station receipt retirement", From 57866d0712dc73d5a038afd36a1eaff17495a0cd Mon Sep 17 00:00:00 2001 From: Senthil Ravichandran Date: Mon, 27 Jul 2026 16:58:18 -0700 Subject: [PATCH 71/74] fix(vllm): adopt running dual Station cleanup state Signed-off-by: Senthil Ravichandran --- docs/inference/set-up-vllm.mdx | 3 +- .../run-plan-gateway-segregation.test.ts | 16 ++++++- src/lib/actions/uninstall/run-plan.ts | 8 +++- src/lib/inference/vllm-dual-station.test.ts | 33 ++++++++++++- src/lib/inference/vllm.ts | 46 +++++++++++++++++++ src/lib/onboard.ts | 7 ++- src/lib/onboard/setup-nim-vllm.test.ts | 45 ++++++++++++++++++ src/lib/onboard/setup-nim-vllm.ts | 14 ++++++ 8 files changed, 167 insertions(+), 5 deletions(-) diff --git a/docs/inference/set-up-vllm.mdx b/docs/inference/set-up-vllm.mdx index 589a7f2d06..827f5b6e71 100644 --- a/docs/inference/set-up-vllm.mdx +++ b/docs/inference/set-up-vllm.mdx @@ -216,6 +216,7 @@ NemoClaw does not modify rail configuration, enroll SSH trust, or reboot either If the local host requires a reboot during initial preparation, the installer stops with status `10`; its owner-only receipt preserves the printed exact revision and Express selections. After reciprocal peer qualification begins, the pair state additionally binds SSH, GPU, and rail identity and names either host that must be rebooted manually. After the managed pair passes readiness and container validation, NemoClaw writes an owner-only cleanup receipt and a copied SSH binding under the selected gateway state root. +A later onboarding run that reuses the same validated managed pair recreates this cleanup ownership before accepting the endpoint. The receipt contains no serving API key. It records the exact peer, cluster, and GPU identities needed to revalidate and remove both managed containers during full uninstall. If NemoClaw cannot write this receipt, setup stops and rolls back a newly started pair instead of leaving a runtime that full uninstall cannot safely reach. @@ -261,7 +262,7 @@ NEMOCLAW_PROVIDER=install-vllm \ On DGX Spark and DGX Station, `NEMOCLAW_PROVIDER=install-vllm` is sufficient for a non-interactive run. Add `NEMOCLAW_EXPERIMENTAL=1` on a generic Linux NVIDIA GPU host. Non-interactive runs use the profile default unless you set `NEMOCLAW_VLLM_MODEL`. -The commands above invoke `nemoclaw onboard` directly, so a DGX Station run with no model or peer selects the `deepseek-v4-flash` profile default. +The commands above invoke `$$nemoclaw onboard` directly, so a DGX Station run with no model or peer selects the `deepseek-v4-flash` profile default. Supplying `NEMOCLAW_PROVIDER=install-vllm` to the shell installer enters the Station host-preparation boundary while retaining that profile default. To request a non-interactive pair through this path, also set `NEMOCLAW_DGX_STATION_PEER`; the exact peer must qualify, and the installer selects the distributed Nemotron 3 Ultra recipe. diff --git a/src/lib/actions/uninstall/run-plan-gateway-segregation.test.ts b/src/lib/actions/uninstall/run-plan-gateway-segregation.test.ts index 5b517ee54d..2b3c93d9c3 100644 --- a/src/lib/actions/uninstall/run-plan-gateway-segregation.test.ts +++ b/src/lib/actions/uninstall/run-plan-gateway-segregation.test.ts @@ -12,9 +12,9 @@ import { readGatewayRegistryFile } from "../../state/gateway-registry"; import { migrateLegacyPortState } from "../../state/legacy-port-migration"; import { type RunResult, + runUninstallPlan as runUninstallPlanBase, type UninstallRunDeps, type UninstallRunOptions, - runUninstallPlan as runUninstallPlanBase, } from "./run-plan"; function ok(stdout = ""): RunResult { @@ -1044,6 +1044,11 @@ describe("uninstall gateway-port segregation (#3053)", () => { }, }), ); + const runtimeReceipt = path.join(stateDir, "dual-station-vllm-runtime.json"); + const runtimeBinding = `${runtimeReceipt}.ssh-binding`; + fs.writeFileSync(runtimeReceipt, "{}\n", { mode: 0o600 }); + fs.mkdirSync(runtimeBinding, { mode: 0o700 }); + fs.writeFileSync(path.join(runtimeBinding, "known_hosts"), "host-key\n", { mode: 0o600 }); const logs: string[] = []; const openshellCalls: string[][] = []; const result = runUninstallPlan( @@ -1067,6 +1072,8 @@ describe("uninstall gateway-port segregation (#3053)", () => { expect(openshellCalls).toContainEqual(["gateway", "select", "nemoclaw"]); expect(openshellCalls).not.toContainEqual(["sandbox", "delete", "--all"]); expect(logs.join("\n")).toContain("Sibling gateways remain"); + expect(fs.existsSync(runtimeReceipt)).toBe(true); + expect(fs.existsSync(runtimeBinding)).toBe(true); } finally { fs.rmSync(tmpHome, { recursive: true, force: true }); } @@ -1086,6 +1093,11 @@ describe("uninstall gateway-port segregation (#3053)", () => { }, }), ); + const runtimeReceipt = path.join(stateDir, "dual-station-vllm-runtime.json"); + const runtimeBinding = `${runtimeReceipt}.ssh-binding`; + fs.writeFileSync(runtimeReceipt, "{}\n", { mode: 0o600 }); + fs.mkdirSync(runtimeBinding, { mode: 0o700 }); + fs.writeFileSync(path.join(runtimeBinding, "known_hosts"), "host-key\n", { mode: 0o600 }); const openshellCalls: string[][] = []; const warnings: string[] = []; let gatewayListCalls = 0; @@ -1121,6 +1133,8 @@ describe("uninstall gateway-port segregation (#3053)", () => { expect(openshellCalls).toContainEqual(["gateway", "select", "nemoclaw"]); expect(openshellCalls).not.toContainEqual(["sandbox", "delete", "--all"]); expect(warnings.join("\n")).toContain("switching to gateway-scoped cleanup"); + expect(fs.existsSync(runtimeReceipt)).toBe(true); + expect(fs.existsSync(runtimeBinding)).toBe(true); } finally { fs.rmSync(tmpHome, { recursive: true, force: true }); } diff --git a/src/lib/actions/uninstall/run-plan.ts b/src/lib/actions/uninstall/run-plan.ts index 77c3676042..f891ecad5d 100644 --- a/src/lib/actions/uninstall/run-plan.ts +++ b/src/lib/actions/uninstall/run-plan.ts @@ -1732,7 +1732,13 @@ function executePlan( ? [GATEWAYS_SUBDIR, path.basename(paths.managedSwapMarkerPath)] : []), ...(scopedToSelectedGateway && selectedIsDefault ? ["source"] : []), - ...(scopedToSelectedGateway ? HTTPS_PIN_RUNTIME_ADAPTER_STATE_ENTRIES : []), + ...(scopedToSelectedGateway + ? [ + ...HTTPS_PIN_RUNTIME_ADAPTER_STATE_ENTRIES, + DUAL_STATION_VLLM_RUNTIME_RECEIPT_FILE, + `${DUAL_STATION_VLLM_RUNTIME_RECEIPT_FILE}.ssh-binding`, + ] + : []), ], runtime, ) diff --git a/src/lib/inference/vllm-dual-station.test.ts b/src/lib/inference/vllm-dual-station.test.ts index 8c1d97d65c..46871ba1cd 100644 --- a/src/lib/inference/vllm-dual-station.test.ts +++ b/src/lib/inference/vllm-dual-station.test.ts @@ -100,7 +100,11 @@ vi.mock("./vllm-api-key", () => ({ loadDualStationVllmApiKey: mocks.loadApiKey, })); -import { detectVllmProfile, installVllm } from "./vllm"; +import { + detectVllmProfile, + installVllm, + persistConfiguredDualStationVllmRuntimeReceipt, +} from "./vllm"; import { DUAL_STATION_VLLM_RUNTIME, type DualStationVllmPlan } from "./vllm-station-cluster"; import { createDualStationSshBindingFixture, @@ -298,6 +302,33 @@ afterEach(() => { process.env = { ...originalEnv }; }); +describe("dual DGX Station running-runtime receipt adoption", () => { + it("persists cleanup ownership for the exact configured running pair", async () => { + await expect(persistConfiguredDualStationVllmRuntimeReceipt()).resolves.toEqual({ + ok: true, + persisted: true, + }); + + expect(mocks.probeCapability).toHaveBeenCalledOnce(); + expect(mocks.preflightOwnership).toHaveBeenCalledWith(plan()); + expect(mocks.areContainersRunning).toHaveBeenCalledWith(plan()); + expect(mocks.persistRuntimeReceipt).toHaveBeenCalledWith(plan()); + }); + + it("fails closed when the configured pair no longer has exact ownership", async () => { + mocks.preflightOwnership.mockReturnValue({ + ok: false, + reason: "worker ownership changed", + }); + + await expect(persistConfiguredDualStationVllmRuntimeReceipt()).resolves.toEqual({ + ok: false, + reason: "worker ownership changed", + }); + expect(mocks.persistRuntimeReceipt).not.toHaveBeenCalled(); + }); +}); + describe("dual DGX Station vLLM install orchestration", () => { it("pulls both immutable images, preflights GPUs, authenticates, and starts worker plus head", async () => { const clusterPlan = plan(); diff --git a/src/lib/inference/vllm.ts b/src/lib/inference/vllm.ts index 1b0eb7222e..3c0be8506f 100644 --- a/src/lib/inference/vllm.ts +++ b/src/lib/inference/vllm.ts @@ -884,6 +884,52 @@ export function isNemoClawManagedVllmRunning(): boolean { return (ownership.kind === "managed" || ownership.kind === "dual-managed") && ownership.running; } +export type PersistConfiguredDualStationVllmRuntimeResult = + | { ok: true; persisted: boolean } + | { ok: false; reason: string }; + +/** + * Adopt an already-running installer-qualified pair into durable uninstall + * ownership after onboarding has authenticated and validated its endpoint. + */ +export async function persistConfiguredDualStationVllmRuntimeReceipt(): Promise { + const configuredPeer = String(process.env[NEMOCLAW_DGX_STATION_PEER_ENV] ?? "").trim(); + if (!configuredPeer) return { ok: true, persisted: false }; + + const capability = probeDualStationVllmCapability(); + if (capability.kind !== "ready") { + const reason = + capability.kind === "unavailable" + ? capability.reason + : "the configured dual-Station peer disappeared"; + return { ok: false, reason }; + } + + try { + return await withDualStationManagedVllmLifecycle(async () => { + const preflight = preflightDualStationManagedVllm(capability.plan); + if (!preflight.ok) return { ok: false, reason: preflight.reason }; + if (!areDualStationManagedVllmContainersRunning(capability.plan)) { + return { + ok: false, + reason: "the managed dual-Station containers changed before receipt persistence", + }; + } + try { + persistDualStationVllmRuntimeReceipt(capability.plan); + } catch (error) { + return { ok: false, reason: (error as Error).message }; + } + return { ok: true, persisted: true }; + }); + } catch (error) { + return { + ok: false, + reason: `dual-Station lifecycle lock failed: ${(error as Error).message}`, + }; + } +} + function startContainer( profile: VllmProfile, model: VllmModelDef, diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index a237d36ba8..e36a408c88 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -251,7 +251,11 @@ const { switchToWindowsOllamaHost, printWindowsOllamaTimeoutDiagnostics, } = require("./inference/ollama/windows"); -const { installVllm, isNemoClawManagedVllmRunning } = require("./inference/vllm"); +const { + installVllm, + isNemoClawManagedVllmRunning, + persistConfiguredDualStationVllmRuntimeReceipt, +} = require("./inference/vllm"); const inferenceConfig: typeof import("./inference/config") = require("./inference/config"); const { DEFAULT_CLOUD_MODEL, getProviderSelectionConfig, parseGatewayInference } = inferenceConfig; @@ -1089,6 +1093,7 @@ const handleVllmSelection = createSetupNimVllmHandler({ VLLM_PORT, runCapture, getLocalProviderBaseUrl, getLocalProviderValidationBaseUrl, getManagedVllmProviderBinding: localInference.getManagedDualStationVllmProviderBinding, queryVllmModels: (baseUrl, apiKey) => { const result = localInference.probeVllmModels(baseUrl, apiKey); return result.ok ? result.body : ""; }, isSafeModelId, requireValue, validateOpenAiLikeSelection, applyVllmRuntimeContextWindow: localInference.applyVllmRuntimeContextWindow, isDgxSparkHost: () => nim.detectNvidiaPlatform() === "spark", isNemoClawManagedVllmRunning, + persistConfiguredDualStationVllmRuntimeReceipt, exitProcess: (code) => process.exit(code), }); const ollamaModelSize: typeof import("./inference/ollama/model-size") = require("./inference/ollama/model-size"); diff --git a/src/lib/onboard/setup-nim-vllm.test.ts b/src/lib/onboard/setup-nim-vllm.test.ts index bc7682d0ea..acf6cf7b53 100644 --- a/src/lib/onboard/setup-nim-vllm.test.ts +++ b/src/lib/onboard/setup-nim-vllm.test.ts @@ -40,6 +40,10 @@ function deps(overrides: Partial = {}): SetupNimVllmDeps { applyVllmRuntimeContextWindow: vi.fn(), isDgxSparkHost: () => false, isNemoClawManagedVllmRunning: () => false, + persistConfiguredDualStationVllmRuntimeReceipt: async () => ({ + ok: true, + persisted: false, + }), exitProcess: (code) => { throw new Error(`exit ${code}`); }, @@ -176,6 +180,47 @@ describe("setupNim vLLM route containment", () => { ); }); + it("persists cleanup ownership after validating a managed dual endpoint", async () => { + const persistConfiguredDualStationVllmRuntimeReceipt = vi.fn(async () => ({ + ok: true, + persisted: true, + })); + const handler = createSetupNimVllmHandler( + deps({ + getManagedVllmProviderBinding: () => ({ + baseUrl: "http://10.40.0.1:8000/v1", + apiKey: "a".repeat(64), + }), + queryVllmModels: () => JSON.stringify({ data: [{ id: "served/model" }] }), + persistConfiguredDualStationVllmRuntimeReceipt, + }), + ); + + await expect(handler(state(null))).resolves.toBe("selected"); + expect(persistConfiguredDualStationVllmRuntimeReceipt).toHaveBeenCalledOnce(); + }); + + it("fails closed when managed dual cleanup ownership cannot be persisted", async () => { + const handler = createSetupNimVllmHandler( + deps({ + getManagedVllmProviderBinding: () => ({ + baseUrl: "http://10.40.0.1:8000/v1", + apiKey: "a".repeat(64), + }), + queryVllmModels: () => JSON.stringify({ data: [{ id: "served/model" }] }), + persistConfiguredDualStationVllmRuntimeReceipt: async () => ({ + ok: false, + reason: "pair identity changed", + }), + }), + ); + + await expect(handler(state(null))).rejects.toThrow("exit 1"); + expect(console.error).toHaveBeenCalledWith( + " Managed dual-Station cleanup ownership could not be persisted: pair identity changed", + ); + }); + it("treats an unexpected undefined managed binding as absent", async () => { const runCapture = vi.fn(() => JSON.stringify({ data: [{ id: "served/model" }] })); const queryVllmModels = vi.fn(() => ""); diff --git a/src/lib/onboard/setup-nim-vllm.ts b/src/lib/onboard/setup-nim-vllm.ts index a996156c76..4acc141e39 100644 --- a/src/lib/onboard/setup-nim-vllm.ts +++ b/src/lib/onboard/setup-nim-vllm.ts @@ -50,6 +50,10 @@ export interface SetupNimVllmDeps { applyVllmRuntimeContextWindow(models: VllmModels, model: string): void; isDgxSparkHost?: () => boolean; isNemoClawManagedVllmRunning?: () => boolean; + persistConfiguredDualStationVllmRuntimeReceipt(): Promise<{ + ok: boolean; + reason?: string; + }>; exitProcess(code: number): never; } @@ -353,6 +357,16 @@ export function createSetupNimVllmHandler( return "retry-selection"; } + if (managedDualEndpoint) { + const receipt = await deps.persistConfiguredDualStationVllmRuntimeReceipt(); + if (!receipt.ok) { + console.error( + ` Managed dual-Station cleanup ownership could not be persisted: ${receipt.reason ?? "unknown error"}`, + ); + deps.exitProcess(1); + } + } + if (modelIdentity) state.vllmModelIdentity = modelIdentity; deps.applyVllmRuntimeContextWindow(models, state.model); if (validation.api !== "openai-completions") { From 4615f2523d179da6d872fe33b1d226804b36c58f Mon Sep 17 00:00:00 2001 From: Senthil Ravichandran Date: Mon, 27 Jul 2026 17:08:11 -0700 Subject: [PATCH 72/74] fix(vllm): require cleanup receipt on reuse Signed-off-by: Senthil Ravichandran --- docs/manage-sandboxes/uninstall-nemoclaw.mdx | 3 ++- src/lib/inference/vllm-dual-station.test.ts | 11 +++++++++ src/lib/inference/vllm.ts | 7 +++++- src/lib/onboard/setup-nim-vllm.test.ts | 25 ++++++++++++++++++-- src/lib/onboard/setup-nim-vllm.ts | 23 +++++++++++------- 5 files changed, 57 insertions(+), 12 deletions(-) diff --git a/docs/manage-sandboxes/uninstall-nemoclaw.mdx b/docs/manage-sandboxes/uninstall-nemoclaw.mdx index d24643b606..de15155192 100644 --- a/docs/manage-sandboxes/uninstall-nemoclaw.mdx +++ b/docs/manage-sandboxes/uninstall-nemoclaw.mdx @@ -38,11 +38,12 @@ A custom-port uninstall does not stop or remove the default gateway service or i For the default gateway, the uninstall command preserves `~/.nemoclaw/rebuild-backups/`, `~/.nemoclaw/backups/`, and `~/.nemoclaw/sandboxes.json` by default. A non-default gateway uses the corresponding entries under `~/.nemoclaw/gateways//`. When uninstall confirms that no sibling gateways remain, it also removes the shared CLI, services, images, providers, configuration, models, and swap. -If the selected state root contains a managed dual-Station cleanup receipt, NemoClaw first revalidates the recorded peer, cluster, and GPU identities and removes both exact managed vLLM containers. +During full uninstall, a managed dual-Station cleanup receipt makes NemoClaw first revalidate the recorded peer, cluster, and GPU identities and remove both exact managed vLLM containers. Only after that pair cleanup succeeds does NemoClaw start the remaining full-uninstall steps. If validation or pair cleanup fails, uninstall exits nonzero and keeps the receipt so you can resolve the reported SSH, Docker, or peer-host error and retry. Pair cleanup can partially complete before an error, so inspect both Stations before retrying. When sibling gateways remain, it removes only the selected gateway's resources and port-scoped state and preserves those shared host resources. +This gateway-scoped path leaves the managed pair running and preserves its cleanup receipt and copied SSH binding, including when you pass `--destroy-user-data`. If the OpenShell command is unavailable or its gateway list cannot be read, uninstall cannot confirm that the selected gateway is the last one, so it uses the same scoped path and preserves the shared resources. When the command itself is unavailable, uninstall exits nonzero before OpenShell cleanup so you can restore the command and retry. Either way, the preserved entries above stay unless you pass `--destroy-user-data`. diff --git a/src/lib/inference/vllm-dual-station.test.ts b/src/lib/inference/vllm-dual-station.test.ts index 46871ba1cd..82f04bfff9 100644 --- a/src/lib/inference/vllm-dual-station.test.ts +++ b/src/lib/inference/vllm-dual-station.test.ts @@ -327,6 +327,17 @@ describe("dual DGX Station running-runtime receipt adoption", () => { }); expect(mocks.persistRuntimeReceipt).not.toHaveBeenCalled(); }); + + it("fails closed when the managed peer configuration is missing", async () => { + delete process.env.NEMOCLAW_DGX_STATION_PEER; + + await expect(persistConfiguredDualStationVllmRuntimeReceipt()).resolves.toEqual({ + ok: false, + reason: "the managed dual-Station peer configuration is missing", + }); + expect(mocks.probeCapability).not.toHaveBeenCalled(); + expect(mocks.persistRuntimeReceipt).not.toHaveBeenCalled(); + }); }); describe("dual DGX Station vLLM install orchestration", () => { diff --git a/src/lib/inference/vllm.ts b/src/lib/inference/vllm.ts index 3c0be8506f..4a6b74b8f2 100644 --- a/src/lib/inference/vllm.ts +++ b/src/lib/inference/vllm.ts @@ -894,7 +894,12 @@ export type PersistConfiguredDualStationVllmRuntimeResult = */ export async function persistConfiguredDualStationVllmRuntimeReceipt(): Promise { const configuredPeer = String(process.env[NEMOCLAW_DGX_STATION_PEER_ENV] ?? "").trim(); - if (!configuredPeer) return { ok: true, persisted: false }; + if (!configuredPeer) { + return { + ok: false, + reason: "the managed dual-Station peer configuration is missing", + }; + } const capability = probeDualStationVllmCapability(); if (capability.kind !== "ready") { diff --git a/src/lib/onboard/setup-nim-vllm.test.ts b/src/lib/onboard/setup-nim-vllm.test.ts index acf6cf7b53..822e0a2cf0 100644 --- a/src/lib/onboard/setup-nim-vllm.test.ts +++ b/src/lib/onboard/setup-nim-vllm.test.ts @@ -42,7 +42,7 @@ function deps(overrides: Partial = {}): SetupNimVllmDeps { isNemoClawManagedVllmRunning: () => false, persistConfiguredDualStationVllmRuntimeReceipt: async () => ({ ok: true, - persisted: false, + persisted: true, }), exitProcess: (code) => { throw new Error(`exit ${code}`); @@ -182,7 +182,7 @@ describe("setupNim vLLM route containment", () => { it("persists cleanup ownership after validating a managed dual endpoint", async () => { const persistConfiguredDualStationVllmRuntimeReceipt = vi.fn(async () => ({ - ok: true, + ok: true as const, persisted: true, })); const handler = createSetupNimVllmHandler( @@ -221,6 +221,27 @@ describe("setupNim vLLM route containment", () => { ); }); + it("fails closed when a managed endpoint is accepted without writing cleanup ownership", async () => { + const handler = createSetupNimVllmHandler( + deps({ + getManagedVllmProviderBinding: () => ({ + baseUrl: "http://10.40.0.1:8000/v1", + apiKey: "a".repeat(64), + }), + queryVllmModels: () => JSON.stringify({ data: [{ id: "served/model" }] }), + persistConfiguredDualStationVllmRuntimeReceipt: async () => ({ + ok: true, + persisted: false, + }), + }), + ); + + await expect(handler(state(null))).rejects.toThrow("exit 1"); + expect(console.error).toHaveBeenCalledWith( + " Managed dual-Station cleanup ownership could not be persisted: the managed dual-Station cleanup receipt was not written", + ); + }); + it("treats an unexpected undefined managed binding as absent", async () => { const runCapture = vi.fn(() => JSON.stringify({ data: [{ id: "served/model" }] })); const queryVllmModels = vi.fn(() => ""); diff --git a/src/lib/onboard/setup-nim-vllm.ts b/src/lib/onboard/setup-nim-vllm.ts index 4acc141e39..58fbce3c5d 100644 --- a/src/lib/onboard/setup-nim-vllm.ts +++ b/src/lib/onboard/setup-nim-vllm.ts @@ -50,10 +50,16 @@ export interface SetupNimVllmDeps { applyVllmRuntimeContextWindow(models: VllmModels, model: string): void; isDgxSparkHost?: () => boolean; isNemoClawManagedVllmRunning?: () => boolean; - persistConfiguredDualStationVllmRuntimeReceipt(): Promise<{ - ok: boolean; - reason?: string; - }>; + persistConfiguredDualStationVllmRuntimeReceipt(): Promise< + | { + ok: true; + persisted: boolean; + } + | { + ok: false; + reason: string; + } + >; exitProcess(code: number): never; } @@ -359,10 +365,11 @@ export function createSetupNimVllmHandler( if (managedDualEndpoint) { const receipt = await deps.persistConfiguredDualStationVllmRuntimeReceipt(); - if (!receipt.ok) { - console.error( - ` Managed dual-Station cleanup ownership could not be persisted: ${receipt.reason ?? "unknown error"}`, - ); + if (!receipt.ok || !receipt.persisted) { + const reason = receipt.ok + ? "the managed dual-Station cleanup receipt was not written" + : receipt.reason; + console.error(` Managed dual-Station cleanup ownership could not be persisted: ${reason}`); deps.exitProcess(1); } } From 1f0078f72f5aa76f4ac1578b014a7b717cdf5960 Mon Sep 17 00:00:00 2001 From: Senthil Ravichandran Date: Mon, 27 Jul 2026 17:18:42 -0700 Subject: [PATCH 73/74] docs(vllm): clarify scoped pair preservation Signed-off-by: Senthil Ravichandran --- docs/manage-sandboxes/uninstall-nemoclaw.mdx | 3 ++- docs/reference/host-files-and-state.mdx | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/manage-sandboxes/uninstall-nemoclaw.mdx b/docs/manage-sandboxes/uninstall-nemoclaw.mdx index de15155192..c8579ffd0c 100644 --- a/docs/manage-sandboxes/uninstall-nemoclaw.mdx +++ b/docs/manage-sandboxes/uninstall-nemoclaw.mdx @@ -46,7 +46,8 @@ When sibling gateways remain, it removes only the selected gateway's resources a This gateway-scoped path leaves the managed pair running and preserves its cleanup receipt and copied SSH binding, including when you pass `--destroy-user-data`. If the OpenShell command is unavailable or its gateway list cannot be read, uninstall cannot confirm that the selected gateway is the last one, so it uses the same scoped path and preserves the shared resources. When the command itself is unavailable, uninstall exits nonzero before OpenShell cleanup so you can restore the command and retry. -Either way, the preserved entries above stay unless you pass `--destroy-user-data`. +The preserved `rebuild-backups/`, `backups/`, and `sandboxes.json` entries stay unless you pass `--destroy-user-data`. +That flag does not override gateway-scoped preservation of the managed pair receipt or its copied SSH binding. Interactive runs prompt before they remove the preserved entries, and the default answer keeps them. For non-interactive runs using `--yes`, `NEMOCLAW_NON_INTERACTIVE=1`, or a non-TTY shell, pass `--destroy-user-data` or set `NEMOCLAW_UNINSTALL_DESTROY_USER_DATA=1` to acknowledge data loss and remove the preserved entries. diff --git a/docs/reference/host-files-and-state.mdx b/docs/reference/host-files-and-state.mdx index 17d3d2b238..6bec4512cf 100644 --- a/docs/reference/host-files-and-state.mdx +++ b/docs/reference/host-files-and-state.mdx @@ -34,8 +34,8 @@ In the table, `` is `~/.nemoclaw/` for the default gateway | `~/.nemoclaw/onboard-session.json` | Resume marker for an onboarding attempt that failed before completion. | Yes, when you intentionally want to discard the failed session and start over. Prefer `$$nemoclaw onboard --fresh` when available. | | `~/.nemoclaw/usage-notice.json` | Records the third-party software notice version in `acceptedVersion` and the acceptance time in `acceptedAt`. Install, onboarding, and rebuild flows consult this file and prompt again when its recorded version differs from the current notice or the file is absent. | Yes; deleting it makes the next applicable install, onboarding, or rebuild flow prompt for acceptance again. | | `~/.nemoclaw/ollama-proxy-token` | Local auth token used by the host-side Ollama auth proxy. | Yes, but re-run onboarding afterward so NemoClaw recreates and registers the proxy token. | -| `/dual-station-vllm-runtime.json` | Owner-only managed dual-Station cleanup receipt. It contains no serving API key and binds the peer, cluster, and GPU identities used to revalidate and remove both managed vLLM containers during full uninstall. | No while the managed pair exists. Run `$$nemoclaw uninstall`; it removes the receipt after both exact containers are removed. | -| `/dual-station-vllm-runtime.json.ssh-binding/` | Owner-only copied SSH host-key and Docker-command binding needed to reach the recorded worker during full uninstall. | No while the managed pair exists. It is removed with the cleanup receipt after pair cleanup succeeds. | +| `/dual-station-vllm-runtime.json` | Owner-only managed dual-Station cleanup receipt. It contains no serving API key and binds the peer, cluster, and GPU identities used to revalidate and remove both managed vLLM containers during full uninstall. | No while the managed pair exists. A full `$$nemoclaw uninstall` removes the receipt after both exact containers are removed; gateway-scoped uninstall preserves it. | +| `/dual-station-vllm-runtime.json.ssh-binding/` | Owner-only copied SSH host-key and Docker-command binding needed to reach the recorded worker during full uninstall. | No while the managed pair exists. Full uninstall removes it with the cleanup receipt after pair cleanup succeeds; gateway-scoped uninstall preserves it. | `sandboxes.json` is the current registry file name. If you see `registry.json` in older tests, notes, or discussions, treat it as legacy wording for the sandbox registry unless a specific release note says otherwise. From 8011ae2b37e1aa5b452e613ad06980a583dbe54f Mon Sep 17 00:00:00 2001 From: Senthil Ravichandran Date: Mon, 27 Jul 2026 17:26:38 -0700 Subject: [PATCH 74/74] test(installer): align dual Station merged-base fixtures Signed-off-by: Senthil Ravichandran --- src/lib/onboard.ts | 11 +++-------- test/install-station-controller-binding.test.ts | 2 +- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index e36a408c88..7b00aa1ad8 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -251,11 +251,7 @@ const { switchToWindowsOllamaHost, printWindowsOllamaTimeoutDiagnostics, } = require("./inference/ollama/windows"); -const { - installVllm, - isNemoClawManagedVllmRunning, - persistConfiguredDualStationVllmRuntimeReceipt, -} = require("./inference/vllm"); +const vllmInference = require("./inference/vllm"); const inferenceConfig: typeof import("./inference/config") = require("./inference/config"); const { DEFAULT_CLOUD_MODEL, getProviderSelectionConfig, parseGatewayInference } = inferenceConfig; @@ -1092,8 +1088,7 @@ const { const handleVllmSelection = createSetupNimVllmHandler({ VLLM_PORT, runCapture, getLocalProviderBaseUrl, getLocalProviderValidationBaseUrl, getManagedVllmProviderBinding: localInference.getManagedDualStationVllmProviderBinding, queryVllmModels: (baseUrl, apiKey) => { const result = localInference.probeVllmModels(baseUrl, apiKey); return result.ok ? result.body : ""; }, isSafeModelId, requireValue, validateOpenAiLikeSelection, - applyVllmRuntimeContextWindow: localInference.applyVllmRuntimeContextWindow, isDgxSparkHost: () => nim.detectNvidiaPlatform() === "spark", isNemoClawManagedVllmRunning, - persistConfiguredDualStationVllmRuntimeReceipt, + applyVllmRuntimeContextWindow: localInference.applyVllmRuntimeContextWindow, isDgxSparkHost: () => nim.detectNvidiaPlatform() === "spark", isNemoClawManagedVllmRunning: vllmInference.isNemoClawManagedVllmRunning, persistConfiguredDualStationVllmRuntimeReceipt: vllmInference.persistConfiguredDualStationVllmRuntimeReceipt, exitProcess: (code) => process.exit(code), }); const ollamaModelSize: typeof import("./inference/ollama/model-size") = require("./inference/ollama/model-size"); @@ -3586,7 +3581,7 @@ function getSetupNimDeps(): SetupNimDeps { handleRunningOllamaSelection, handleWindowsHostOllamaSelection, handleInstallOllamaSelection, - installVllm, + installVllm: vllmInference.installVllm, handleVllmSelection, handleRoutedSelection, coerceAgentInferenceApi: inferenceConfig.coerceAgentInferenceApi, diff --git a/test/install-station-controller-binding.test.ts b/test/install-station-controller-binding.test.ts index cf8ad53141..991d3c85ce 100644 --- a/test/install-station-controller-binding.test.ts +++ b/test/install-station-controller-binding.test.ts @@ -330,7 +330,7 @@ printf 'MIGRATE=%s REUSE=%s\\n' "$_STATION_EXPRESS_MIGRATING_LEGACY_HEAD" "$_STA "true", "true", "head", - "1", + "2", "c".repeat(64), "d".repeat(64), "e".repeat(64),