diff --git a/configs/config_openshell.yml b/configs/config_openshell.yml index 788e38efe..84bc659d8 100644 --- a/configs/config_openshell.yml +++ b/configs/config_openshell.yml @@ -1,7 +1,5 @@ -# EXPERIMENTAL: AI-Q deep research over a local, single-operator OpenShell sandbox. -# Run ./scripts/setup_openshell.sh first to provision the gateway and named sandbox. -# Jobs attach to that shared sandbox; per-job directories prevent filename collisions -# but are not a security boundary. Do not run mutually untrusted jobs concurrently. +# EXPERIMENTAL: one policy-bound OpenShell sandbox per deep-research job. +# Canonical operator guide: docs/source/deployment/openshell.md general: use_uvloop: true @@ -52,16 +50,6 @@ llms: chat_template_kwargs: enable_thinking: true - gpt_oss_llm: - _type: nim - model_name: openai/gpt-oss-120b - base_url: https://integrate.api.nvidia.com/v1 - temperature: 1.0 - top_p: 1.0 - max_tokens: 256000 - api_key: ${NVIDIA_API_KEY} - max_retries: 10 - summary_llm: _type: nim model_name: nvidia/nemotron-mini-4b-instruct @@ -139,13 +127,31 @@ functions: deep_research_sandbox: _type: deep_research_sandbox provider: openshell - sandbox_name: ${AIQ_OPENSHELL_SANDBOX_NAME:-aiq-openshell-demo} + openshell_image: ${AIQ_OPENSHELL_IMAGE:-aiq-openshell-demo:latest} policy: ${AIQ_OPENSHELL_POLICY_FILE:-configs/openshell/generated/aiq-openshell-policy.yaml} workdir: /sandbox - network: blocked + # The OpenShell policy is authoritative. This declaration is an upper bound: + # startup fails if the policy grants a hostname not listed here. + network: allowlist + network_allow: + - api.github.com + - github.com + - integrate.api.nvidia.com + - api.tavily.com + - google.serper.dev + - pypi.org + - files.pythonhosted.org + - huggingface.co + - cdn-lfs.huggingface.co + - export.arxiv.org + - api.semanticscholar.org + - registry.npmjs.org timeout: 1200 idle_timeout: 1800 - delete_on_exit: false + policy_load_timeout_seconds: 30 + delete_on_exit: true + attest: true + require_hard_landlock: ${AIQ_OPENSHELL_REQUIRE_HARD_LANDLOCK:-true} artifact_capture: enabled: true max_file_bytes: 50000000 @@ -154,11 +160,11 @@ functions: deep_research_agent: _type: deep_research_agent enable_citation_verification: true - orchestrator_llm: gpt_oss_llm + orchestrator_llm: nemotron_super_llm source_router_llm: nemotron_super_llm researcher_llm: nemotron_super_llm - planner_llm: gpt_oss_llm - writer_llm: gpt_oss_llm + planner_llm: nemotron_super_llm + writer_llm: nemotron_super_llm exclude_tools: - web_search_tool verbose: true diff --git a/configs/openshell/README.md b/configs/openshell/README.md new file mode 100644 index 000000000..aea09a248 --- /dev/null +++ b/configs/openshell/README.md @@ -0,0 +1,15 @@ +# OpenShell Policies + +`aiq-research-policy.yaml` is the checked-in, production-oriented policy sample +used by schema and regression tests. It is not the runtime default. + +Run `scripts/openshell/setup_openshell.sh` to generate the canonical runtime policy at: + +```text +configs/openshell/generated/aiq-openshell-policy.yaml +``` + +The generated file reflects the selected network preset and Landlock mode and is +the default consumed by `configs/config_openshell.yml`, the strict gateway probe, +and the live acceptance suite. Do not commit environment-specific generated +policies or credentials. diff --git a/configs/openshell/aiq-research-policy.yaml b/configs/openshell/aiq-research-policy.yaml index 8bb9775fa..bafc5a998 100644 --- a/configs/openshell/aiq-research-policy.yaml +++ b/configs/openshell/aiq-research-policy.yaml @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # -# Generated by scripts/setup_openshell.sh. +# Generated by scripts/openshell/setup_openshell.sh. version: 1 @@ -11,9 +11,8 @@ filesystem_policy: - /usr - /lib - /etc - - /app - /var/log - - /proc/self + - /proc - /dev/urandom read_write: - /sandbox @@ -21,11 +20,12 @@ filesystem_policy: - /tmp - /dev/null -# best_effort lets the sandbox start on hosts without Landlock (e.g. Docker Desktop on -# macOS), but there filesystem confinement is silently dropped. Acceptable only for the -# local single-operator demo; production must use `hard_requirement` (fail closed). +# Fail closed when the host cannot enforce Landlock filesystem confinement. Local demo +# environments that intentionally accept weaker isolation must generate a separate policy +# with `scripts/openshell/setup_openshell.sh --landlock-compatibility best_effort` and configure +# `require_hard_landlock: false` explicitly. landlock: - compatibility: best_effort + compatibility: hard_requirement process: run_as_user: sandbox diff --git a/deploy/openshell/Dockerfile.aiq-demo b/deploy/openshell/Dockerfile.aiq-demo index 06ff906ce..1176f0ecc 100644 --- a/deploy/openshell/Dockerfile.aiq-demo +++ b/deploy/openshell/Dockerfile.aiq-demo @@ -7,7 +7,7 @@ ENV DEBIAN_FRONTEND=noninteractive # Optional OpenShell sandbox log verbosity. Defaults to `warn` (OpenShell's stock # sandbox level, so default behavior is unchanged). Rebuild with -# `--build-arg OPENSHELL_SANDBOX_LOG_LEVEL=debug` (or `setup_openshell.sh +# `--build-arg OPENSHELL_SANDBOX_LOG_LEVEL=debug` (or `scripts/openshell/setup_openshell.sh # --sandbox-log-level debug`) to surface process/relay detail (`OCSF PROC:` etc.) # in the container logs (`/var/log/openshell.*.log`, `openshell logs `). ARG OPENSHELL_SANDBOX_LOG_LEVEL=warn diff --git a/docs/source/architecture/agents/deep-researcher.md b/docs/source/architecture/agents/deep-researcher.md index 5b6ee8e4d..77adcecab 100644 --- a/docs/source/architecture/agents/deep-researcher.md +++ b/docs/source/architecture/agents/deep-researcher.md @@ -11,7 +11,7 @@ an orchestrator using the [`deepagents`](https://docs.langchain.com/oss/python/d **Location:** `src/aiq_agent/agents/deep_researcher/agent.py` -For optional DeepAgents sandbox execution (Modal) and operational notes, see +For optional DeepAgents sandbox execution (Modal or OpenShell) and operational notes, see [Deep Research Sandbox](./sandbox.md). ## Purpose diff --git a/docs/source/architecture/agents/sandbox.md b/docs/source/architecture/agents/sandbox.md index 6e33128e5..87227e66f 100644 --- a/docs/source/architecture/agents/sandbox.md +++ b/docs/source/architecture/agents/sandbox.md @@ -6,9 +6,12 @@ SPDX-License-Identifier: Apache-2.0 # Deep Research Sandbox Notes Deep research can optionally run DeepAgents `execute` calls through a sandbox provider -(Modal, OpenShell, or any registered provider). Modal creates a sandbox per job. The -experimental OpenShell path attaches to a pre-created named sandbox shared by its jobs; -job-scoped directories prevent filename collisions but are not a security boundary. +(Modal, OpenShell, or any registered provider). Modal and OpenShell create a fresh physical +sandbox per job. OpenShell binds the configured policy at creation and attests the authoritative +effective policy source, content, hash, and active revision before exposing the execution backend. +On the supported OpenShell `0.0.80` stack, current, active, revision, and effective-config versions +must all be positive and agree with the exact submitted policy/hash; missing capabilities or any +version disagreement fails closed. The sandbox is an internal execution detail. There are no sandbox-specific API endpoints, and job-level auth remains responsible for submit, stream, status, @@ -17,16 +20,20 @@ runtime (`.../job/{job_id}/artifacts`), which is also auth-scoped to the job. > **Developer reference:** the full architecture, provider contract, config schema, > artifact pipeline, and troubleshooting live next to the code in -> [`src/aiq_agent/agents/deep_researcher/sandbox/README.md`](../../../../src/aiq_agent/agents/deep_researcher/sandbox/README.md). +> [`src/aiq_agent/agents/deep_researcher/sandbox/README.md`](https://github.com/NVIDIA-AI-Blueprints/aiq/blob/develop/src/aiq_agent/agents/deep_researcher/sandbox/README.md). +> Operators should use the canonical [OpenShell deployment guide](../../deployment/openshell.md) +> for setup, authenticated gateway lifecycle, supported platforms, acceptance, and cleanup. ## Current Behavior -- Modal uses one sandbox per deep research job. OpenShell currently attaches jobs to - the configured shared sandbox name and is intended for local, single-operator testing. +- Modal and OpenShell use one physical sandbox per deep research job. OpenShell shared + attachment is available only through an explicit debug-only opt-in and is not job-isolated. - Synchronous sandbox-enabled runs use an internal per-agent runtime ID. - Providers are selected by config (`sandbox.provider` + `providers.`); the provider is validated against the registry and gated by its declared capabilities. - OpenShell policy is provisioned externally and is not verified when AI-Q attaches. + OpenShell policy YAML is parsed strictly against the installed SDK schema, applied in the + job's creation spec, checked against the declared network upper bound (hostless/CIDR overrides + are rejected), and attested through the gateway status/config RPCs before use. - Job IDs must satisfy each provider's object-name rules (Modal: 64 chars or fewer, alphanumeric plus dash/period/underscore). - `timeout` bounds individual execution. Other lifecycle controls are provider-dependent. @@ -46,16 +53,19 @@ and [Production Considerations](../../deployment/production.md#artifact-storage) ## Operational Notes -- High-concurrency Modal runs create one sandbox per job. OpenShell runs share the named - sandbox and must not be used concurrently for mutually untrusted jobs. Optional submit-path +- High-concurrency Modal and OpenShell runs create one sandbox per job. Optional submit-path caps (`AIQ_MAX_SANDBOXES_PER_PRINCIPAL` / `AIQ_MAX_SANDBOXES_GLOBAL`, default-off) bound - concurrency/cost but do not provide filesystem isolation. + concurrency and cost. - Custom client-supplied job IDs must not be reused for a new job. - Manifest checkpoints preserve completed artifacts after successful sandbox commands. The terminal finalizer harvests once before cleanup on success/failure; cancellation harvests only when the provider is idle and otherwise terminates immediately. - The runtime closes provider sessions on success, failure, cancellation, and timeout. - A named OpenShell sandbox persists when `delete_on_exit` is disabled. +- Per-job OpenShell mode requires `delete_on_exit: true`. A persistent shared sandbox is + possible only through the explicit debug attachment settings. +- OpenShell provisioning and authenticated gateway lifecycle have separate owners. See the + [OpenShell responsibility table](../../deployment/openshell.md#responsibility-and-lifecycle-ownership) + rather than duplicating operational commands here. ## Current Safeguards @@ -67,3 +77,4 @@ The following safeguards are in place: with MIME-from-bytes spoof rejection, SVG sanitization, and an inline-render allowlist. - Sandbox quota and concurrency controls, and artifact retention via job-expiry cleanup. - Structured lifecycle logging for sandbox create, reuse, failure, and cleanup. +- Structured `sandbox.attestation` and `sandbox.cleanup` events. diff --git a/docs/source/deployment/index.md b/docs/source/deployment/index.md index 0c938f256..83a5e22b3 100644 --- a/docs/source/deployment/index.md +++ b/docs/source/deployment/index.md @@ -39,6 +39,8 @@ All containerized deployments run the same three services: - **[Production Considerations](./production.md)** -- Guidance on managed databases, horizontal scaling, security hardening, monitoring, and resource requirements. +- **[OpenShell](./openshell.md)** -- Optional policy-bound execution for generated deep-research code, including supported platforms, authenticated gateway ownership, policy/config pairing, deterministic live acceptance, and safe cleanup. + ## Quick Start For the fastest path to a running stack: diff --git a/docs/source/deployment/openshell.md b/docs/source/deployment/openshell.md new file mode 100644 index 000000000..437710b48 --- /dev/null +++ b/docs/source/deployment/openshell.md @@ -0,0 +1,515 @@ + + +# OpenShell Deployment + +This is the canonical operator guide for running AI-Q deep-research code in +[NVIDIA OpenShell](https://docs.nvidia.com/openshell/latest/about/installation). +It owns the setup, deployment, acceptance, and troubleshooting contract. The +architecture and implementation pages describe invariants and extension points; +they intentionally link here for operational steps. + +OpenShell is the primary path for current sandbox-enabled deep-research +validation. It does not replace AI-Q's non-sandbox workflow defaults, and it +remains experimental until the Linux hard-Landlock acceptance gate passes. + +## AI-Q Environment Prerequisite + +Run all commands in this guide from the AI-Q repository root using the standard +AI-Q virtual environment: + +```bash +./scripts/setup.sh +source .venv/bin/activate +``` + +`setup.sh` installs the locked AI-Q dependencies, API frontends, data sources, +pre-commit hooks, and UI packages. A new shell must run +`source .venv/bin/activate` again before validation, tests, or E2E startup. + +### Setup Command Overview + +| Command | Purpose | +|---|---| +| `./scripts/setup.sh` | Create `.venv` and install the standard AI-Q backend, frontend, data-source, and development dependencies | +| `source .venv/bin/activate` | Select the repository Python environment in the current shell | +| `./scripts/openshell/setup_openshell.sh --policy offline` | Install the certified OpenShell SDK/adapter, generate a production hard-Landlock policy, and build the sandbox image | +| `./scripts/openshell/setup_openshell.sh --local-demo --policy offline` | Generate the explicit `best_effort` local-demo policy instead of the production policy | +| `./scripts/openshell/install_gateway.sh` | Explicitly install or repair the certified packaged gateway on Apple Silicon macOS | +| `./scripts/openshell/check_versions.py` | Report safe CLI, SDK, package, and live-gateway version diagnostics | +| `./scripts/openshell/start_openshell_gateway.sh` | Start or reuse the packaged authenticated gateway and run the disposable strict capability probe | +| `nat validate --config_file configs/config_openshell.yml` | Validate the AI-Q workflow and its OpenShell policy/config pairing before startup | +| `./scripts/start_e2e.sh --start-openshell-gateway --config_file configs/config_openshell.yml` | Re-run gateway readiness, then start the AI-Q backend and UI | + +## Purpose and Security Boundary + +OpenShell executes code generated during deep research. AI-Q orchestration, +inference, retrieval tools, credentials, checkpoints, and report state remain on +the host side. AI-Q does not copy the host environment or model credentials into +the sandbox creation specification. + +In production, every job receives a distinct physical sandbox bound to the +submitted policy. AI-Q verifies the authoritative policy source, content, hash, +and revision before exposing the execution adapter. Successful, failed, +timed-out, and cancelled jobs must delete the sandboxes they own. + +AI-Q does not reproduce OpenShell's private policy-hash algorithm. It requires +the effective-config policy and revision policy to both equal the submitted +protobuf, requires both authoritative hashes to be non-empty and equal, and +emits that OpenShell-provided hash in the sanitized attestation event. + +Attaching to an existing shared sandbox is an explicit debug escape hatch. It is +not job-isolated and is not a production mode. OpenShell is an external runtime +and authentication boundary, not merely a Python dependency: an operator must +own the gateway service, registration, credentials, version, and availability. + +## Supported Platforms + +| Path | Intended use | Landlock | Gateway owner | AI-Q status | +|---|---|---|---|---| +| Linux + Docker | Production acceptance | `hard_requirement` | systemd or external operator | Tested path after the live suite passes | +| macOS + Docker Desktop | Local demo | `best_effort` permitted explicitly | Homebrew | Demo only | +| Linux + Podman | Operator-managed evaluation | `hard_requirement` | external operator/system service | Supported upstream, not automated or certified by AI-Q acceptance | +| Remote authenticated gateway | Managed deployment | Gateway-host dependent | external operator | Accepted only after the AI-Q live suite passes | +| Windows/WSL | -- | -- | -- | Outside current AI-Q setup-script support | + +OpenShell supports Docker and rootless Podman upstream. AI-Q's provisioning and +acceptance automation certify only the exercised Docker path. Follow the +[official OpenShell installation documentation](https://docs.nvidia.com/openshell/latest/about/installation) +for gateway-host prerequisites and upstream runtime support. + +## Known Limitations + +- OpenShell integration remains experimental until the Linux/Docker live suite + passes with Landlock `hard_requirement`; macOS `best_effort` results are local + functional evidence only. +- Docker is the only AI-Q-automated runtime path. Podman is supported upstream + but not automated or production-accepted by AI-Q; Windows/WSL is unsupported. +- `best_effort` permits execution when Landlock is unavailable, so macOS/Docker + Desktop does not provide the production filesystem-confinement guarantee. +- OpenShell does not currently satisfy AI-Q's optional CPU/memory resource-limit + capability. Configuring those limits fails closed rather than silently + ignoring them. +- Shared named-sandbox attachment is debug-only and is not a tenant or job + isolation boundary. +- The authenticated gateway is an operator-owned external service. AI-Q can + validate or start a packaged local service, but E2E shutdown intentionally + does not stop the gateway. + +## Version Compatibility + +`scripts/openshell/setup_openshell.sh` accepts only the certified OpenShell +version. `latest` and other exact releases are rejected so an evaluation upgrade +cannot silently become a production stack. The strict gateway launcher then +requires the virtual-environment CLI, SDK, packaged local CLI, and live gateway +to match that version. + +The published `langchain-nvidia-openshell==0.1.0` metadata still declares +`deepagents<0.6`, while AI-Q uses DeepAgents 0.6.x. The OpenShell setup therefore +installs the optional adapter and then restores AI-Q's required DeepAgents and +OpenShell versions. This is intentionally isolated from `scripts/setup.sh`; it +may repeat package installation, and it remains an upstream metadata limitation +until a compatible adapter release is published. Do not move an unreleased +adapter into the AI-Q lockfile or hide the conflict with a dependency override. +Until that release exists, `pip check` reports the adapter's declared +DeepAgents-range conflict; this PR must not describe that metadata state as +clean even though the tested adapter surface works with AI-Q's locked runtime. + +AI-Q currently requires and defaults to OpenShell `0.0.80`. It is the first +released version that acknowledges the initial sandbox-scoped policy revision as +`LOADED` after successful policy-engine construction and exposes immutable +request-level labels/selectors through the Python SDK. Earlier releases can leave +the revision `PENDING` with `current_policy_version=0` or omit ownership labels, +so they fail AI-Q's strict readiness checks. + +The certified release is necessary but not sufficient: runtime security decisions +remain capability-based. On the supported `0.0.80` stack, the current, active, +revision, and effective-config versions must all be positive and agree. Any +missing capability or version/content/hash/source mismatch fails closed. + +## Responsibility and Lifecycle Ownership + +| Component | Owns | Must not do | +|---|---|---| +| `scripts/openshell/setup_openshell.sh` | SDK/adapter install, policy generation, image build | Start, stop, register, select, probe, or kill gateways | +| `scripts/openshell/install_gateway.sh` | Explicit installation or repair of the official packaged local macOS gateway | Install Linux/remote gateways, create custom taps, weaken TLS, or launch raw binaries | +| Homebrew/systemd/external operator | Long-running gateway service and credentials | Delegate process ownership to AI-Q setup | +| `scripts/openshell/start_openshell_gateway.sh` | Validate registration/auth, optionally start a packaged service, select the gateway, and run the strict disposable capability probe | Install or upgrade gateways, launch raw binaries, stop externally managed services, or persist credentials | +| AI-Q runtime | Per-job create, readiness, attestation, execution, and terminal deletion | Reuse a shared sandbox without explicit debug opt-in | +| Live pytest fixtures | Acceptance-test resources and verified teardown | Leave resources for manual cleanup | + +Provisioning and long-running service lifecycle are deliberately separate. E2E +shutdown never stops a Homebrew-, systemd-, or operator-managed gateway. + +## Policy and AI-Q Config Pairing + +Both policy layers are enforced: + +- The OpenShell policy is authoritative at the gateway. +- `network` and `network_allow` in the AI-Q config are an upper bound on that + policy. They never grant additional access. +- `network: blocked` permits no policy endpoint. +- `network: allowlist` requires every endpoint to have a non-empty normalized + host, and every host must appear in `network_allow`. +- Hostless endpoints and `allowed_ips` or CIDR exceptions are rejected because + the public AI-Q policy does not model those exceptions. +- Production requires both `landlock.compatibility: hard_requirement` in the + policy and `require_hard_landlock: true` in the AI-Q config. +- A local demo using `best_effort` requires both the policy value + `best_effort` and `AIQ_OPENSHELL_REQUIRE_HARD_LANDLOCK=false` when validating + or running the standard OpenShell config. +- Custom policies must explicitly include OpenShell's proxy filesystem baseline, + including read-only `/proc`. Otherwise the supervisor creates an enriched + revision whose content and hash correctly fail AI-Q's exact attestation. The + generated policy from `scripts/openshell/setup_openshell.sh` already includes this baseline. + +Any mismatch fails closed before the execution adapter is available. Keep +environment-specific generated policies out of commits, and never put +credentials in policy or workflow configuration files. + +## Environment Contract + +The gateway launcher, AI-Q runtime, and live suite use these non-secret settings: + +| Variable | Default | Purpose | +|---|---|---| +| `AIQ_OPENSHELL_LIVE_TESTS` | unset | Must equal `1` to enable live tests | +| `AIQ_OPENSHELL_GATEWAY_NAME` | active gateway | Registered gateway name | +| `AIQ_OPENSHELL_POLICY_FILE` | `configs/openshell/generated/aiq-openshell-policy.yaml` | Policy submitted and attested | +| `AIQ_OPENSHELL_IMAGE` | `aiq-openshell-demo:latest` | Prebuilt sandbox image | +| `AIQ_OPENSHELL_EXPECTED_GATEWAY_VERSION` | installed SDK version | Optional exact live-test override | +| `AIQ_OPENSHELL_REQUIRE_HARD_LANDLOCK` | `true` | Set `false` only for an explicit local `best_effort` demo | +| `AIQ_OPENSHELL_LIVE_ALLOW_BEST_EFFORT` | unset | Explicit non-production macOS/demo opt-in | + +## Linux Production Acceptance + +First install and register an authenticated packaged gateway, or arrange an +externally operated gateway, using the official OpenShell documentation. The +registration must use HTTPS and mTLS, OIDC, or trusted edge authentication. +Do not launch a raw `openshell-gateway` process. + +From the AI-Q repository root, provision the pinned SDK, hard policy, and image: + +```bash +./scripts/openshell/setup_openshell.sh \ + --openshell-version 0.0.80 \ + --policy offline \ + --landlock-compatibility hard_requirement +``` + +Select the authenticated registration and prove version, policy, selector, +execution, and deletion capabilities. Omit `--reuse-existing` only when the gateway is a local packaged +service that the launcher may start through systemd. + +```bash +./scripts/openshell/start_openshell_gateway.sh \ + --gateway-name openshell \ + --image-name aiq-openshell-demo:latest \ + --policy-file configs/openshell/generated/aiq-openshell-policy.yaml +``` + +The readiness probe proves that the submitted policy is effective, but it does +not independently require hard Landlock. Production acceptance therefore uses +the `hard_requirement` policy generated above and the matching +`require_hard_landlock: true` AI-Q config. + +Export the same image and policy for the AI-Q process: + +```bash +export AIQ_OPENSHELL_GATEWAY_NAME=openshell +export AIQ_OPENSHELL_IMAGE=aiq-openshell-demo:latest +export AIQ_OPENSHELL_POLICY_FILE="$PWD/configs/openshell/generated/aiq-openshell-policy.yaml" +export AIQ_OPENSHELL_EXPECTED_GATEWAY_VERSION=0.0.80 +``` + +Validate and start AI-Q with the production pairing in +`configs/config_openshell.yml`: + +```bash +source .venv/bin/activate +nat validate --config_file configs/config_openshell.yml +./scripts/start_e2e.sh --config_file configs/config_openshell.yml +``` + +Because the gateway was already verified above, `--start-openshell-gateway` is +not needed here. It is an optional convenience that reruns the same strict probe +before E2E startup. + +In a separate shell with the same exported settings, run the required acceptance +suite: + +```bash +AIQ_OPENSHELL_LIVE_TESTS=1 \ +AIQ_OPENSHELL_GATEWAY_NAME=openshell \ +AIQ_OPENSHELL_POLICY_FILE=configs/openshell/generated/aiq-openshell-policy.yaml \ +AIQ_OPENSHELL_IMAGE=aiq-openshell-demo:latest \ +AIQ_OPENSHELL_EXPECTED_GATEWAY_VERSION=0.0.80 \ +.venv/bin/python -m pytest -m integration -vv \ + tests/aiq_agent/agents/deep_researcher/sandbox/test_openshell_live.py +``` + +Only this Linux, hard-Landlock run can be recorded as production acceptance. + +## macOS Local Demo + +Use Docker Desktop and the official `nvidia/openshell` Homebrew service. First +provision AI-Q's optional Python components, policy, and image. This step also +prints safe component diagnostics when the packaged gateway is missing or stale. + +macOS ships Bash 3.2. Install Bash 5 when the setup script reports unsupported +Bash behavior: + +```bash +brew install bash +/opt/homebrew/bin/bash ./scripts/openshell/setup_openshell.sh \ + --openshell-version 0.0.80 \ + --local-demo \ + --policy offline +``` + +If setup reports `packaged_gateway_missing` or +`component_version_mismatch`, inspect the explicit operation and then run it as +the logged-in user: + +```bash +./scripts/openshell/install_gateway.sh --dry-run +./scripts/openshell/install_gateway.sh +``` + +For Colima, persist the driver configuration in OpenShell's service environment +instead of the caller's transient launchd environment: + +```bash +./scripts/openshell/install_gateway.sh --colima +# Or select a specific local socket: +./scripts/openshell/install_gateway.sh \ + --docker-host "unix://$HOME/.colima/default/docker.sock" +``` + +The wrapper downloads the installer from the certified OpenShell tag to a +temporary file, verifies its checked-in SHA-256, and invokes the official +installer with `OPENSHELL_VERSION=v0.0.80`. It refuses root, non-Apple-Silicon +hosts, and ambiguous OpenShell installations. It never pipes a download into a +shell, creates an AI-Q tap, launches a raw gateway, disables TLS, or stores +credentials. + +OpenShell's release installer stages its formula in a local `nvidia/openshell` +tap created with Homebrew's `--no-git` mode. Because that tap has no Git remote, +`brew upgrade openshell` cannot fetch a newly released formula; it can leave the +packaged gateway on `0.0.72` while AI-Q's virtual-environment CLI/SDK is `0.0.80`. +Rerun the explicit pinned installer wrapper when directed. Do not copy a formula +manually, create a custom AI-Q tap, use `launchctl setenv`, or keep multiple +OpenShell service identities. + +You can inspect the safe version report directly: + +```bash +.venv/bin/python scripts/openshell/check_versions.py +.venv/bin/python scripts/openshell/check_versions.py --json +``` + +Then validate the standard config, validate the gateway, and start AI-Q with the +explicit local-demo environment override: + +```bash +source .venv/bin/activate +AIQ_OPENSHELL_REQUIRE_HARD_LANDLOCK=false \ + nat validate --config_file configs/config_openshell.yml + +./scripts/openshell/start_openshell_gateway.sh \ + --gateway-name openshell \ + --image-name aiq-openshell-demo:latest \ + --policy-file configs/openshell/generated/aiq-openshell-policy.yaml + +AIQ_OPENSHELL_POLICY_FILE=configs/openshell/generated/aiq-openshell-policy.yaml \ +AIQ_OPENSHELL_IMAGE=aiq-openshell-demo:latest \ +AIQ_OPENSHELL_EXPECTED_GATEWAY_VERSION=0.0.80 \ +AIQ_OPENSHELL_REQUIRE_HARD_LANDLOCK=false \ +./scripts/start_e2e.sh --config_file configs/config_openshell.yml \ + 2>&1 | tee e2e-openshell-0.0.80.log +``` + +Run the same mechanics through the convenience wrapper with the explicit demo +opt-in: + +```bash +.venv/bin/python scripts/openshell/smoke_openshell_isolation.py \ + --gateway openshell \ + --policy configs/openshell/generated/aiq-openshell-policy.yaml \ + --image aiq-openshell-demo:latest \ + --expected-gateway-version 0.0.80 \ + --allow-best-effort-landlock +``` + +A passing macOS run is useful local evidence, but it does not satisfy Linux +production acceptance. + +## Existing Remote Gateway + +The remote gateway must already be registered over HTTPS with mTLS, OIDC, or +trusted edge authentication. The launcher validates the registration and refuses +to substitute a local gateway if the remote service is unavailable: + +```bash +./scripts/openshell/start_openshell_gateway.sh \ + --gateway-name enterprise \ + --reuse-existing \ + --image-name aiq-openshell-demo:latest \ + --policy-file configs/openshell/generated/aiq-openshell-policy.yaml +``` + +The disposable strict capability probe is mandatory. After it passes, +export `AIQ_OPENSHELL_GATEWAY_NAME=enterprise` and run the live suite. Never fall +back to a plaintext registration, insecure TLS, or a local raw gateway. + +## Shared Debug Attachment + +Create a named shared sandbox only when debugging requires it: + +```bash +./scripts/openshell/start_openshell_gateway.sh \ + --gateway-name openshell \ + --create-shared-debug-sandbox \ + --sandbox-name aiq-openshell-demo +``` + +Attachment also requires `allow_shared_sandbox: true` and an explicit +`existing_sandbox_name` in a local AI-Q config. When a policy is supplied, AI-Q +requires strict effective-policy attestation and rejects `attest: false`. Without +a policy, the attachment still requires READY/loaded-version checks and emits +`assurance=reduced`. The attaching job never owns or deletes the shared sandbox; +the operator or test fixture that created it remains responsible. + +## Expected Runtime Behavior + +The human-readable contract is: + +- One running deep-research job creates one physical OpenShell sandbox. +- Two concurrent jobs create two distinct sandbox names and physical IDs. +- Active jobs are discoverable with `--selector aiq=deep-research`, with a + distinct `aiq-job-id` gateway label for each job. +- Attestation succeeds before the execution adapter is exposed. +- Success, command failure, timeout, and cancellation delete owned sandboxes. +- Cancelling one job does not delete or replace another job's sandbox. +- `sandbox.attestation` reports sanitized status, policy version, hash, source, + assurance, and reason code. +- `sandbox.cleanup` reports `started`, `succeeded`, or `failed`, with stable + `reason_codes` on failure. +- Credentials, policy contents, SDK response bodies, and exception messages are + not emitted in lifecycle events or failure logs. + +The final job state is separate from physical cleanup: verify the cleanup event +and absence from the gateway rather than assuming a terminal job status deleted +the resource. + +## Artifact Capture + +`configs/config_openshell.yml` enables durable sandbox artifact capture. Successful +`execute` calls checkpoint declared artifacts, and terminal finalization performs +one idempotent scan before sandbox cleanup. Metadata is stored in the job database; +bytes use the configured SQL or S3-compatible artifact blob provider. Clients +receive `artifact.update` metadata with a `content_url`, never raw bytes in SSE. + +For validation, storage configuration, event payloads, and report rendering, see +the developer [artifact runtime](https://github.com/NVIDIA-AI-Blueprints/aiq/blob/develop/src/aiq_agent/agents/deep_researcher/sandbox/README.md#artifact-runtime) +and [production artifact storage](./production.md#artifact-storage) guides. + +## Acceptance Tests + +The canonical acceptance entry point is pytest: + +```bash +AIQ_OPENSHELL_LIVE_TESTS=1 \ +AIQ_OPENSHELL_GATEWAY_NAME=openshell \ +AIQ_OPENSHELL_POLICY_FILE=configs/openshell/generated/aiq-openshell-policy.yaml \ +AIQ_OPENSHELL_IMAGE=aiq-openshell-demo:latest \ +AIQ_OPENSHELL_EXPECTED_GATEWAY_VERSION=0.0.80 \ +.venv/bin/python -m pytest -m integration -vv \ + tests/aiq_agent/agents/deep_researcher/sandbox/test_openshell_live.py +``` + +The suite contains three independently reported tests: + +- `test_live_per_job_isolation_attestation_and_cancellation` proves distinct + sandboxes, authoritative source/content/hash/revision attestation, isolated + cancellation, selector membership, continued execution, and terminal deletion. +- `test_live_failure_cleanup_and_log_redaction` proves cleanup after a deterministic + failed command and verifies that a credential-shaped exception canary reaches + neither logs nor events. +- `test_live_shared_policy_mismatch_is_rejected` proves that a structurally + different claimed policy cannot attach successfully, while the directly owned + shared sandbox remains usable until fixture teardown. + +Every fixture registers resources immediately, tears them down in reverse order, +and verifies deletion through the gateway. A teardown failure fails the test even +when the test body also failed. Without `AIQ_OPENSHELL_LIVE_TESTS=1`, all three +tests are collected and skipped before optional OpenShell imports or gateway +connections. + +`scripts/openshell/smoke_openshell_isolation.py` is a convenience wrapper only. It maps its +arguments to the environment contract, enables the live gate, and returns pytest's +exit code unchanged. Pytest owns every assertion and cleanup fixture. + +Record the non-secret gateway version, policy path, image tag, platform, and +Landlock mode with acceptance results. Do not record registrations, environment +values, policy contents, response bodies, or credentials. + +## Inspection and Troubleshooting + +Inspect only registered resources and sanitized AI-Q lifecycle events: + +```bash +.venv/bin/openshell status +.venv/bin/openshell gateway list -o json +.venv/bin/openshell sandbox list +.venv/bin/openshell sandbox list --selector aiq=deep-research -o json +``` + +The selected gateway registration must be HTTPS and report `mtls`, `oidc`, or +trusted edge authentication. During a job, the selector must show one owned +sandbox per active deep-research job. After termination, each owned name must be +absent from direct and selector listings. Use the sandbox name from sanitized `sandbox.attestation` or +`sandbox.cleanup` events; do not expose full SDK payloads to logs. + +| Failure | Safe action | +|---|---| +| Generated policy or image is missing | Run `scripts/openshell/setup_openshell.sh` and reuse the exact paths it prints. | +| `packaged_gateway_missing` on Apple Silicon macOS | Run `./scripts/openshell/install_gateway.sh --dry-run`, then explicitly approve the installer. | +| `component_version_mismatch` on local macOS | Run `.venv/bin/python scripts/openshell/check_versions.py`, then use the exact local installer remediation it prints. | +| `ambiguous_gateway_installation` | Remove the obsolete OpenShell formula/service identity through Homebrew before retrying. Do not let AI-Q guess which service to replace. | +| `remote_gateway_version_mismatch` | Coordinate an upgrade with the registered gateway owner. AI-Q never replaces a remote service with a local one. | +| `gateway_unavailable` | Start or verify the packaged local service with `scripts/openshell/start_openshell_gateway.sh`, or contact the remote gateway owner. Do not reinstall a matching stack merely because the service is stopped. | +| CLI, SDK, and gateway versions differ | Do not start AI-Q or create a probe sandbox. Align all reported components to the certified version and rerun the launcher. | +| `request_labels_unsupported` | The installed Python SDK cannot persist gateway labels required for AI-Q ownership and selectors; install a supported release. | +| `policy_status_inconsistent` | The effective policy matches but its revision never became `LOADED`. This is an OpenShell lifecycle failure, not a Landlock-mode mismatch. Do not disable attestation. | +| `policy_content_mismatch` | Regenerate the policy with `scripts/openshell/setup_openshell.sh`, or add the required OpenShell proxy filesystem baseline (including read-only `/proc`) to a custom policy. Do not weaken exact attestation. | +| `selector_mismatch` | The probe was not discoverable through gateway metadata. Do not rely on Docker/template labels as a substitute. | +| Registration is plaintext or unauthenticated | Register an HTTPS gateway with mTLS, OIDC, or trusted edge authentication. Do not bypass the launcher check. | +| Docker daemon is unavailable | Start the operator-owned Docker service and rerun provisioning/probe. | +| Podman is selected | Follow upstream OpenShell guidance; do not report the path as AI-Q production-accepted. | +| Landlock policy/config mismatch | Pair `hard_requirement` with the default config, or set `AIQ_OPENSHELL_REQUIRE_HARD_LANDLOCK=false` only with an explicit `best_effort` demo policy. | +| Policy is broader than `network_allow` | Remove the endpoint or add its exact normalized hostname to the declared upper bound. Do not add CIDR exceptions. | +| Sandbox never becomes Ready | Inspect the owning gateway/runtime service, image availability, and sanitized sandbox status; do not dump SDK bodies. | +| Probe or job deletion cannot be verified | Treat acceptance as failed, identify the exact sandbox, and retry explicit deletion through the registered gateway. | +| macOS reports Bash 3.2 incompatibility | Install Bash 5 and invoke the setup script with its absolute path. | + +For a named sandbox that this operator owns, use explicit cleanup and verify its +absence: + +```bash +.venv/bin/openshell sandbox delete +.venv/bin/openshell sandbox list +``` + +Manage a packaged gateway only through its owner: + +```bash +brew services restart nvidia/openshell/openshell +systemctl --user restart openshell-gateway +``` + +Never use broad `pkill`, launch the raw gateway binary, enable insecure TLS, or +perform destructive cleanup without first identifying the owned resource. Do not +stop an externally managed gateway from AI-Q shutdown logic. diff --git a/docs/source/deployment/production.md b/docs/source/deployment/production.md index 52d2efb53..d7a673c88 100644 --- a/docs/source/deployment/production.md +++ b/docs/source/deployment/production.md @@ -113,6 +113,15 @@ The compose stack mounts `configs/` as read-only (`:ro`), preventing the applica Store API keys in `deploy/.env` and ensure the file is not committed to version control (it is listed in `.gitignore`). Never embed keys in configuration files or Dockerfiles. +### Sandbox Runtime Ownership + +Treat optional sandbox runtimes as separate execution and authentication boundaries. +Production OpenShell requires an explicitly owned authenticated gateway, a distinct +policy-bound sandbox per job, verified terminal cleanup, and hard Landlock enforcement. +Follow the [Linux production acceptance](./openshell.md#linux-production-acceptance) +and [policy/config pairing](./openshell.md#policy-and-ai-q-config-pairing) contracts; +do not infer production readiness from a macOS best-effort demo. + ## Monitoring ### Liveness and Readiness Endpoints diff --git a/docs/source/index.md b/docs/source/index.md index 6efe71720..e4953ff80 100644 --- a/docs/source/index.md +++ b/docs/source/index.md @@ -101,6 +101,7 @@ Authentication <./deployment/authentication.md> Async Job Content Encryption <./deployment/content-encryption.md> Observability <./deployment/observability.md> Production <./deployment/production.md> +OpenShell <./deployment/openshell.md> Kubernetes <./deployment/kubernetes.md> Amazon OpenSearch Serverless <./deployment/aws-opensearch-serverless.md> ``` diff --git a/docs/source/integration/rest-api.md b/docs/source/integration/rest-api.md index 670421e0f..cea841aed 100644 --- a/docs/source/integration/rest-api.md +++ b/docs/source/integration/rest-api.md @@ -318,6 +318,12 @@ Events streamed during job execution. Refer to the [Data Flow](../architecture/d | `tool.start` / `tool.end` | Tool invocation lifecycle. Includes tool name, input, and output | | `artifact.update` | Structured updates: todos, files, citations (`citation_source`, `citation_use`), output content | +Sandbox-generated files also use `artifact.update` with `data.type: "file"`. +Their metadata includes `artifact_id`, `job_id`, `file_path`, authenticated +`content_url`, `kind`, validated `mime_type`, `size_bytes`, `sha256`, `title`, +`caption`, and `inline`. File bytes are never included in SSE; fetch them through +the authenticated `content_url`. + ## Agent Registration Agents are registered by type so the async job runner can load them dynamically. Registration happens at import time (typically in a NeMo Agent Toolkit plugin module): diff --git a/docs/source/resources/troubleshooting.md b/docs/source/resources/troubleshooting.md index 0477fe842..46379e700 100644 --- a/docs/source/resources/troubleshooting.md +++ b/docs/source/resources/troubleshooting.md @@ -37,6 +37,7 @@ Common issues and solutions for the AI-Q blueprint. | Clarifier keeps asking questions | Too many clarification turns | Reduce `max_turns`, or set `enable_clarifier: false` in the workflow to disable clarification | | SSE stream disconnects | Network timeout | Client auto-reconnects using `last_event_id`; refer to [Data Flow](../architecture/data-flow.md) | | Job status stuck on RUNNING | Dask worker crashed | Check Dask logs; the ghost job reaper will eventually mark it FAILURE | +| OpenShell setup, attestation, readiness, or deletion fails | Gateway, version, policy/config, image, or service-owner mismatch | Follow the canonical [OpenShell inspection and troubleshooting guide](../deployment/openshell.md#inspection-and-troubleshooting) | ## Nemotron Super — Build Endpoint Availability diff --git a/frontends/aiq_api/src/aiq_api/jobs/runner.py b/frontends/aiq_api/src/aiq_api/jobs/runner.py index 932a6c937..f8091e950 100644 --- a/frontends/aiq_api/src/aiq_api/jobs/runner.py +++ b/frontends/aiq_api/src/aiq_api/jobs/runner.py @@ -885,19 +885,12 @@ async def run_agent_job( finally: # Ensure terminal-path events are not left in the batch buffer. - if event_store is not None and hasattr(event_store, "flush"): - try: - event_store.flush() - except Exception as exc: - logger.warning( - "Final event flush failed for job %s exception=%s", - job_id, - exc.__class__.__name__, - ) + await _flush_event_store(event_store, job_id=job_id) if cancellation_monitor: cancellation_monitor.stop() # Idempotent fallback for failures before a terminal branch finalized the runtime. await asyncio.to_thread(_teardown_sandbox, sandbox_runtime, job_id=job_id, interrupted=interrupted) + await _flush_event_store(event_store, job_id=job_id) # Clean up job-scoped auth token if _auth_token_reset is not None: from ._auth_context import job_auth_token @@ -921,13 +914,7 @@ def _store_terminal_event_best_effort(event_store, event: dict) -> None: def _harvest_sandbox_artifacts(sandbox_runtime: Any | None, *, job_id: str, interrupted: bool) -> None: - """Persist captured artifacts on a terminal path without releasing the sandbox. - - Callable before the terminal job status so artifact metadata is durable, yet it never invokes - the provider's unbounded ``close()``/``terminate()``. Resource release stays in - ``_teardown_sandbox`` (run from ``finally``) so a hanging SDK cleanup cannot strand a finished - job in ``RUNNING`` with a stream that never terminates. The harvest is idempotent. - """ + """Persist captured artifacts on a terminal path without releasing the sandbox.""" if sandbox_runtime is None: return finalize_artifacts = getattr(sandbox_runtime, "finalize_artifacts", None) @@ -942,16 +929,35 @@ def _harvest_sandbox_artifacts(sandbox_runtime: Any | None, *, job_id: str, inte ) +async def _flush_event_store(event_store: Any | None, *, job_id: str) -> None: + """Flush terminal events off-loop without replacing the job result.""" + if event_store is None or not hasattr(event_store, "flush"): + return + try: + await asyncio.to_thread(event_store.flush) + except Exception as exc: # noqa: BLE001 - terminal observability must not replace the job result + logger.warning("Event store flush failed for job %s (%s)", job_id, type(exc).__name__) + + def _teardown_sandbox(sandbox_runtime: Any | None, *, job_id: str, interrupted: bool) -> None: """Harvest artifacts and release sandbox resources on a terminal path. - Interrupted jobs (cancel/timeout) call ``terminate()`` so a still-running ``execute`` is - forcibly preempted; normal paths call ``close()`` gracefully. Both are idempotent. This runs - off the event loop (``asyncio.to_thread``) so the SDK session close cannot block the worker. + Prefers ``finalize(interrupted=...)`` when available. On the legacy fallback, + interrupted/cancelled jobs call ``terminate()`` so a still-running ``execute`` is forcibly + preempted; normal failure and success paths call ``close()`` gracefully. Both are idempotent. + This runs off the event loop (``asyncio.to_thread``) so SDK cleanup cannot block the worker. """ if sandbox_runtime is None: return _harvest_sandbox_artifacts(sandbox_runtime, job_id=job_id, interrupted=interrupted) + finalize = getattr(sandbox_runtime, "finalize", None) + if callable(finalize): + try: + if not finalize(interrupted=interrupted): + logger.warning("Sandbox cleanup reported failure for job %s", job_id) + except Exception as exc: # noqa: BLE001 - cleanup must never replace the job result + logger.warning("Sandbox cleanup failed for job %s (%s)", job_id, type(exc).__name__) + return teardown = getattr(sandbox_runtime, "terminate", None) if interrupted else None if teardown is None: teardown = getattr(sandbox_runtime, "close", None) @@ -963,7 +969,7 @@ def _teardown_sandbox(sandbox_runtime: Any | None, *, job_id: str, interrupted: # Secret-safe: log only the exception type. A provider cleanup error can carry a # credential or internal hostname, which must never reach the logs (matches the # finalize_artifacts handler above). - logger.warning("Sandbox cleanup failed for job %s exception=%s", job_id, exc.__class__.__name__) + logger.warning("Sandbox cleanup failed for job %s (%s)", job_id, type(exc).__name__) def _create_agent_instance( diff --git a/pyproject.toml b/pyproject.toml index c92aedd44..79aed9faa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -82,6 +82,11 @@ viz = [ "pygraphviz>=1.11", ] +[tool.aiq.openshell] +release-tag = "v0.0.80" +adapter-version = "0.1.0" +installer-sha256 = "c15d6cb8090e1c7c8d79a320b5bcbdaf1c15c2363942d81e84b56e03b836249e" # pragma: allowlist secret + # ----------------------------------------------------------------------------- # Workspace members are installed via: uv sync # Individual packages: uv pip install -e ./frontends/cli diff --git a/scripts/README.md b/scripts/README.md index 12a7dbbef..c62402812 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -49,63 +49,39 @@ Starts the agent in CLI mode with browser-based authentication. | `--verbose` or `-v` | Enable verbose logging | | `--config_file ` | Use a custom configuration file | -### `setup_openshell.sh` - OpenShell Sandbox Setup - -Sets up the experimental, local single-operator NVIDIA OpenShell path for AI-Q. Run this once before using -`configs/config_openshell.yml` with `start_cli.sh` or `start_e2e.sh`. It installs -the `openshell` SDK and the `langchain-nvidia-openshell` adapter, starts/verifies -the local OpenShell gateway, builds the sandbox image, generates a network policy, -and creates the named sandbox `aiq-openshell-demo`. Inference is unaffected (it -stays host-side, routed to NVIDIA Build); only generated code runs in the -network-blocked sandbox. - -The generated configuration attaches all jobs to one named sandbox. Per-job directories -avoid filename collisions but do not isolate mutually untrusted jobs, and AI-Q does not -verify the provisioned policy when attaching. Do not treat this setup as a multi-tenant -security boundary. +### `openshell/` - OpenShell Sandbox Utilities + +`openshell/setup_openshell.sh` installs the certified SDK/adapter, generates a policy, and builds +the reusable image. `openshell/install_gateway.sh` is the explicit Apple Silicon macOS +entry point for installing the official packaged gateway. `openshell/start_openshell_gateway.sh` +validates an authenticated registered +gateway and performs a disposable version/policy/selector/execution/cleanup probe. AI-Q +then owns one attested physical sandbox per job. OpenShell `0.0.80` is the supported +version because it contains the required policy-revision and request-label fixes. +The quick start below is the Linux production pairing; activate the repository virtual +environment first. macOS requires the explicit local-demo pairing in the canonical guide. ```bash -./scripts/setup_openshell.sh --policy offline +source .venv/bin/activate +./scripts/openshell/setup_openshell.sh --openshell-version 0.0.80 --policy offline +./scripts/openshell/start_openshell_gateway.sh --gateway-name openshell ./scripts/start_e2e.sh --config_file configs/config_openshell.yml -# or direct serve: -dotenv -f deploy/.env run .venv/bin/nat serve --config_file configs/config_openshell.yml --host 0.0.0.0 --port 8000 -``` - -Useful version examples: - -```bash -./scripts/setup_openshell.sh --openshell-version 0.0.72 -./scripts/setup_openshell.sh --openshell-version latest -./scripts/setup_openshell.sh --list-openshell-versions -``` - -In the interactive version prompt, pressing Enter selects `0.0.72`. - -The setup installs the `openshell` SDK plus the official `langchain-nvidia-openshell` -adapter (`OpenShellSandbox`), published on PyPI. The script installs it from PyPI by -default; set `LANGCHAIN_NVIDIA_REPO` or pass `--langchain-nvidia` to use another -`uv pip install` spec or a local checkout. - -Useful policy examples: - -```bash -./scripts/setup_openshell.sh --policy offline -./scripts/setup_openshell.sh --policy research -./scripts/setup_openshell.sh --policy python-packages -./scripts/setup_openshell.sh --policy custom --allow github,pypi,nvidia,tavily ``` -Verify and clean up: +For a macOS local demo: ```bash -.venv/bin/openshell status -.venv/bin/openshell sandbox list # expect: aiq-openshell-demo ... Ready -.venv/bin/openshell sandbox delete aiq-openshell-demo -# Inspect, then stop only the gateway you started (avoid killing other sessions): -pgrep -fl openshell-gateway # find the PID(s) -kill # stop the specific process +/opt/homebrew/bin/bash ./scripts/openshell/setup_openshell.sh --local-demo --policy offline +./scripts/openshell/install_gateway.sh --dry-run +./scripts/openshell/install_gateway.sh +AIQ_OPENSHELL_REQUIRE_HARD_LANDLOCK=false \ + ./scripts/start_e2e.sh --start-openshell-gateway --config_file configs/config_openshell.yml ``` +For supported platforms, production versus local-demo policy pairing, remote gateways, +live pytest acceptance, and troubleshooting, use the canonical +[OpenShell deployment guide](../docs/source/deployment/openshell.md). This README is only +the script-discovery surface. ### `start_server_in_debug_mode.sh` - Server Mode @@ -113,7 +89,7 @@ Starts the NAT FastAPI server for deep research with async job support. ```bash ./scripts/start_server_in_debug_mode.sh -./scripts/start_server_in_debug_mode.sh--port 8080 +./scripts/start_server_in_debug_mode.sh --port 8080 ./scripts/start_server_in_debug_mode.sh --config_file configs/config_web_frag.yml ``` @@ -166,12 +142,27 @@ Starts the AI-Q API backend for use by Agent Skills such as `aiq-research`. This ### `start_e2e.sh` - End-to-End Mode Starts both backend and frontend for full WebSocket support and HITL workflows. +Complete the standard setup and activate the virtual environment first: + +```bash +./scripts/setup.sh +source .venv/bin/activate +``` ```bash ./scripts/start_e2e.sh ./scripts/start_e2e.sh --config_file configs/config_openshell.yml +./scripts/start_e2e.sh --start-openshell-gateway --config_file configs/config_openshell.yml --port 8080 ``` +**Options:** + +| Option | Description | +|--------|-------------| +| `--config_file ` | Use a custom configuration file | +| `--port ` | Backend port and frontend backend URL (default: 8000) | +| `--start-openshell-gateway` | Start/reuse the packaged authenticated gateway and run its strict capability probe before E2E | + **Services:** | Service | URL | @@ -187,7 +178,7 @@ Starts both backend and frontend for full WebSocket support and HITL workflows. | `configs/config_web_frag.yml` | Server/E2E mode with Foundational RAG | | `configs/config_web_default_llamaindex.yml` | Server/E2E mode with LlamaIndex | | `configs/config_skills.yml` | Deep research with DeepAgents skills + Modal sandbox | -| `configs/config_openshell.yml` | Experimental local single-operator OpenShell sandbox + artifact capture (run `setup_openshell.sh` first) | +| `configs/config_openshell.yml` | Experimental per-job OpenShell sandbox + artifact capture (run `scripts/openshell/setup_openshell.sh` first) | ## Development Workflow diff --git a/scripts/openshell/check_openshell_readiness.py b/scripts/openshell/check_openshell_readiness.py new file mode 100644 index 000000000..5e2869c61 --- /dev/null +++ b/scripts/openshell/check_openshell_readiness.py @@ -0,0 +1,306 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Prove OpenShell version, policy, selector, execution, and cleanup capabilities.""" + +from __future__ import annotations + +import argparse +import importlib.metadata +import re +import subprocess +import sys +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Any +from uuid import uuid4 + +_REPO_ROOT = Path(__file__).resolve().parents[2] +_SRC_ROOT = _REPO_ROOT / "src" +if str(_SRC_ROOT) not in sys.path: + sys.path.insert(0, str(_SRC_ROOT)) + +_VERSION_PATTERN = re.compile( + r"(?P0|[1-9]\d*)\." + r"(?P0|[1-9]\d*)\." + r"(?P0|[1-9]\d*)" + r"(?:(?:-dev\.|\.dev)(?P\d+)(?:\+g(?P[0-9a-fA-F]+))?)?" +) + + +class ReadinessError(RuntimeError): + """A sanitized readiness failure safe to print for operators.""" + + def __init__(self, reason_code: str) -> None: + super().__init__(reason_code) + self.reason_code = reason_code + + +@dataclass(frozen=True) +class ReadinessConfig: + """Non-secret inputs for one disposable readiness probe.""" + + gateway: str | None + image: str + policy: Path + openshell_bin: Path + ready_timeout_seconds: float + policy_load_timeout_seconds: float + + +def _version_from_cli(binary: Path) -> str: + try: + result = subprocess.run( + [str(binary), "--version"], + check=False, + capture_output=True, + text=True, + timeout=10, + ) + except (OSError, subprocess.SubprocessError) as exc: + raise ReadinessError("version_check_failed") from exc + match = _VERSION_PATTERN.search(result.stdout) + if result.returncode != 0 or match is None: + raise ReadinessError("version_check_failed") + return match.group(0) + + +def _version_identity(value: str) -> tuple[int, int, int, int | None, str | None] | None: + """Normalize equivalent Python and Cargo development-version spellings.""" + match = _VERSION_PATTERN.fullmatch(value) + if match is None: + return None + dev = match.group("dev") + sha = match.group("sha") + return ( + int(match.group("major")), + int(match.group("minor")), + int(match.group("patch")), + int(dev) if dev is not None else None, + sha.lower() if sha is not None else None, + ) + + +def _is_not_found(exc: BaseException, grpc: Any) -> bool: + return isinstance(exc, grpc.Call) and exc.code() == grpc.StatusCode.NOT_FOUND + + +def _verify_absent(client: Any, name: str, selector: str, grpc: Any) -> None: + try: + client.get(name) + except grpc.RpcError as exc: + if not _is_not_found(exc, grpc): + raise ReadinessError("cleanup_failed") from exc + else: + raise ReadinessError("cleanup_failed") + try: + selected = client.list(label_selector=selector) + except Exception as exc: # noqa: BLE001 - never expose SDK response details + raise ReadinessError("cleanup_failed") from exc + if any(getattr(item, "name", None) == name for item in selected): + raise ReadinessError("cleanup_failed") + + +def _verify_policy( + *, + client: Any, + sandbox: Any, + expected_policy: Any, + timeout_seconds: float, + openshell_pb2: Any, + sandbox_pb2: Any, +) -> None: + stub = getattr(client, "_stub", None) + if stub is None or not hasattr(stub, "GetSandboxPolicyStatus") or not hasattr(stub, "GetSandboxConfig"): + raise ReadinessError("sdk_capability_missing") + status_request = openshell_pb2.GetSandboxPolicyStatusRequest(name=sandbox.name, version=0) + config_request = sandbox_pb2.GetSandboxConfigRequest(sandbox_id=sandbox.id or sandbox.name) + deadline = time.monotonic() + timeout_seconds + pending_effective = False + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise ReadinessError("policy_status_inconsistent" if pending_effective else "attestation_timeout") + try: + refreshed = client.get(sandbox.name) + status = stub.GetSandboxPolicyStatus(status_request, timeout=min(5.0, remaining)) + config = stub.GetSandboxConfig(config_request, timeout=min(5.0, remaining)) + except Exception as exc: # noqa: BLE001 - never expose SDK response details + raise ReadinessError("rpc_failed") from exc + if getattr(refreshed, "phase", None) != openshell_pb2.SANDBOX_PHASE_READY: + raise ReadinessError("not_ready") + revision = getattr(status, "revision", None) + revision_status = getattr(revision, "status", openshell_pb2.POLICY_STATUS_UNSPECIFIED) + revision_version = getattr(revision, "version", 0) + config_version = getattr(config, "version", 0) + if getattr(revision, "load_error", "") or revision_status == openshell_pb2.POLICY_STATUS_FAILED: + raise ReadinessError("policy_status_failed") + if revision_status != openshell_pb2.POLICY_STATUS_LOADED: + pending_effective = ( + revision_status == openshell_pb2.POLICY_STATUS_PENDING + and revision_version > 0 + and revision_version == config_version + and getattr(config, "policy_source", sandbox_pb2.POLICY_SOURCE_UNSPECIFIED) + == sandbox_pb2.POLICY_SOURCE_SANDBOX + and getattr(config, "policy", None) == expected_policy + and getattr(revision, "policy", None) == expected_policy + and bool(getattr(config, "policy_hash", "")) + and getattr(config, "policy_hash", "") == getattr(revision, "policy_hash", "") + ) + time.sleep(min(0.5, remaining)) + continue + if not revision_version or revision_version != config_version: + raise ReadinessError("version_mismatch") + for reported in ( + getattr(status, "active_version", 0), + getattr(refreshed, "current_policy_version", 0), + ): + if not isinstance(reported, int) or reported <= 0 or reported != revision_version: + raise ReadinessError("version_mismatch") + if getattr(config, "policy_source", sandbox_pb2.POLICY_SOURCE_UNSPECIFIED) != sandbox_pb2.POLICY_SOURCE_SANDBOX: + raise ReadinessError("policy_source_mismatch") + if getattr(config, "policy", None) != expected_policy or getattr(revision, "policy", None) != expected_policy: + raise ReadinessError("policy_content_mismatch") + config_hash = getattr(config, "policy_hash", "") + revision_hash = getattr(revision, "policy_hash", "") + if not config_hash or not revision_hash: + raise ReadinessError("policy_hash_missing") + if config_hash != revision_hash: + raise ReadinessError("policy_hash_mismatch") + return + + +def run_check(config: ReadinessConfig) -> tuple[str, str]: + """Run one strict probe and return the validated SDK and gateway version.""" + try: + import grpc + from openshell._proto import openshell_pb2 + from openshell._proto import sandbox_pb2 + from openshell.sandbox import SandboxClient + except ImportError as exc: + raise ReadinessError("sdk_unavailable") from exc + + from aiq_agent.agents.deep_researcher.sandbox.providers.openshell import _accepts_keyword + from aiq_agent.agents.deep_researcher.sandbox.providers.openshell import _build_sandbox_spec + from aiq_agent.agents.deep_researcher.sandbox.providers.openshell import _parse_policy_proto + from aiq_agent.agents.deep_researcher.sandbox.providers.openshell import _read_policy_data + + try: + sdk_version = importlib.metadata.version("openshell") + except importlib.metadata.PackageNotFoundError as exc: + raise ReadinessError("sdk_unavailable") from exc + cli_version = _version_from_cli(config.openshell_bin) + labels = {"aiq": "readiness-probe"} + selector = "aiq=readiness-probe" + policy_data = _read_policy_data(str(config.policy), require_hard_landlock=False) + expected_policy = _parse_policy_proto(policy_data, policy_path=str(config.policy)) + spec = _build_sandbox_spec( + policy=expected_policy, + image=config.image, + job_id=f"readiness-{uuid4().hex[:10]}", + labels=labels, + ) + + with SandboxClient.from_active_cluster(cluster=config.gateway) as client: + try: + gateway_version = client.health().version + except Exception as exc: # noqa: BLE001 - never expose SDK response details + raise ReadinessError("gateway_unavailable") from exc + version_identities = { + _version_identity(cli_version), + _version_identity(sdk_version), + _version_identity(gateway_version), + } + if None in version_identities or len(version_identities) != 1: + raise ReadinessError("version_mismatch") + if ( + not _accepts_keyword(client.create, "name") + or not _accepts_keyword(client.create, "labels") + or not _accepts_keyword(client.list, "label_selector") + ): + raise ReadinessError("request_labels_unsupported") + + sandbox_name = f"aiq-readiness-{uuid4().hex[:12]}" + cleanup_name = sandbox_name + primary_error: ReadinessError | None = None + try: + sandbox = client.create(spec=spec, name=sandbox_name, labels=labels) + cleanup_name = getattr(sandbox, "name", None) or sandbox_name + if sandbox.name != sandbox_name: + raise ReadinessError("probe_failed") + sandbox = client.wait_ready(sandbox_name, timeout_seconds=config.ready_timeout_seconds) + selected = client.list(label_selector=selector) + matches = [item for item in selected if getattr(item, "name", None) == sandbox_name] + if len(selected) != 1 or len(matches) != 1 or dict(getattr(matches[0], "labels", {})) != labels: + raise ReadinessError("selector_mismatch") + _verify_policy( + client=client, + sandbox=sandbox, + expected_policy=expected_policy, + timeout_seconds=config.policy_load_timeout_seconds, + openshell_pb2=openshell_pb2, + sandbox_pb2=sandbox_pb2, + ) + result = client.exec(sandbox.id or sandbox.name, ["sh", "-c", "printf %s aiq-openshell-ready"]) + if getattr(result, "exit_code", 1) != 0 or getattr(result, "stdout", "") != "aiq-openshell-ready": + raise ReadinessError("execution_failed") + except ReadinessError as exc: + primary_error = exc + except Exception as exc: # noqa: BLE001 - never expose SDK response details + primary_error = ReadinessError("probe_failed") + primary_error.__cause__ = exc + finally: + try: + try: + client.get(cleanup_name) + except grpc.RpcError as exc: + if not _is_not_found(exc, grpc): + raise + else: + if not client.delete(cleanup_name): + raise ReadinessError("cleanup_failed") + client.wait_deleted(cleanup_name) + _verify_absent(client, cleanup_name, selector, grpc) + except Exception as exc: # noqa: BLE001 - cleanup failure overrides probe failure + raise ReadinessError("cleanup_failed") from exc + if primary_error is not None: + raise primary_error + return sdk_version, gateway_version + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--gateway-name", default=None) + parser.add_argument("--image-name", default="aiq-openshell-demo:latest") + parser.add_argument("--policy-file", type=Path, required=True) + parser.add_argument("--openshell-bin", type=Path, required=True) + parser.add_argument("--ready-timeout-seconds", type=float, default=120.0) + parser.add_argument("--policy-load-timeout-seconds", type=float, default=30.0) + return parser + + +def main(argv: list[str] | None = None) -> int: + args = _parser().parse_args(argv) + config = ReadinessConfig( + gateway=args.gateway_name, + image=args.image_name, + policy=args.policy_file, + openshell_bin=args.openshell_bin, + ready_timeout_seconds=args.ready_timeout_seconds, + policy_load_timeout_seconds=args.policy_load_timeout_seconds, + ) + try: + sdk_version, gateway_version = run_check(config) + except ReadinessError as exc: + print(f"OpenShell readiness check failed: {exc.reason_code}", file=sys.stderr) + return 1 + except Exception: # noqa: BLE001 - unexpected details may contain gateway or SDK data + print("OpenShell readiness check failed: unexpected_error", file=sys.stderr) + return 1 + print(f"OpenShell strict readiness succeeded: sdk={sdk_version} gateway={gateway_version}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/openshell/check_versions.py b/scripts/openshell/check_versions.py new file mode 100644 index 000000000..7377f6230 --- /dev/null +++ b/scripts/openshell/check_versions.py @@ -0,0 +1,234 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Inspect the allowlisted OpenShell components without exposing configuration data.""" + +from __future__ import annotations + +import argparse +import importlib.metadata +import json +import platform +import re +import shutil +import subprocess +import sys +from dataclasses import asdict +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from version_contract import load_contract + +_REPO_ROOT = Path(__file__).resolve().parents[2] +_VERSION_PATTERN = re.compile(r"\d+\.\d+\.\d+(?:(?:-dev\.|\.dev)\d+(?:\+g[0-9a-fA-F]+)?)?") +_LOCAL_REMEDIATION = "./scripts/openshell/install_gateway.sh" +_OFFICIAL_FORMULA = "nvidia/openshell/openshell" + + +@dataclass(frozen=True) +class ComponentReport: + """Safe, automation-friendly OpenShell component report.""" + + certified_version: str + sdk_version: str | None + virtualenv_cli_version: str | None + homebrew_formula: str | None + homebrew_formula_version: str | None + packaged_cli_version: str | None + live_gateway_version: str | None + gateway_type: str | None + reason_code: str | None + remediation: str | None + + +def _run(command: list[str]) -> str | None: + try: + result = subprocess.run(command, check=False, capture_output=True, text=True, timeout=15) + except (OSError, subprocess.SubprocessError): + return None + return result.stdout.strip() if result.returncode == 0 else None + + +def _extract_version(value: str | None) -> str | None: + match = _VERSION_PATTERN.search(value or "") + return match.group(0) if match is not None else None + + +def _cli_version(binary: Path) -> str | None: + if not binary.is_file(): + return None + return _extract_version(_run([str(binary), "--version"])) + + +def _gateway_type(binary: Path, gateway_name: str) -> str | None: + if not binary.is_file(): + return None + payload = _run([str(binary), "gateway", "list", "-o", "json"]) + if payload is None: + return None + try: + registrations = json.loads(payload) + except json.JSONDecodeError: + return None + if not isinstance(registrations, list): + return None + for registration in registrations: + if isinstance(registration, dict) and registration.get("name") == gateway_name: + value = registration.get("type") + return value.lower() if isinstance(value, str) else None + return None + + +def _homebrew_components() -> tuple[list[str], str | None, str | None, str | None]: + brew = shutil.which("brew") + if brew is None: + return [], None, None, None + formula_output = _run([brew, "list", "--formula", "--full-name"]) or "" + formulas = sorted( + {line.strip() for line in formula_output.splitlines() if line.strip().split("/")[-1] == "openshell"} + ) + service_output = _run([brew, "services", "list", "--json"]) + if service_output: + try: + services = json.loads(service_output) + except json.JSONDecodeError: + services = [] + if isinstance(services, list): + service_formulas = [ + entry["name"] + for entry in services + if isinstance(entry, dict) + and isinstance(entry.get("name"), str) + and entry["name"].split("/")[-1] == "openshell" + ] + if formulas: + service_formulas = [name for name in service_formulas if "/" in name and name not in formulas] + formulas.extend(service_formulas) + formulas = sorted(set(formulas)) + if len(formulas) != 1: + return formulas, None, None, None + formula = formulas[0] + formula_version = _extract_version(_run([brew, "list", "--versions", formula])) + prefix = _run([brew, "--prefix", formula]) + packaged_cli = _cli_version(Path(prefix) / "bin" / "openshell") if prefix else None + return formulas, formula, formula_version, packaged_cli + + +def _live_gateway_version(gateway_name: str) -> str | None: + try: + from openshell.sandbox import SandboxClient + + with SandboxClient.from_active_cluster(cluster=gateway_name) as client: + return _extract_version(str(client.health().version)) + except Exception: # noqa: BLE001 - diagnostics intentionally discard SDK and gateway details + return None + + +def inspect_components( + *, + gateway_name: str, + include_live: bool = True, + system: str | None = None, +) -> ComponentReport: + """Inspect versions and return one sanitized reason and remediation.""" + contract = load_contract() + certified = contract.version + try: + sdk_version = _extract_version(importlib.metadata.version("openshell")) + except importlib.metadata.PackageNotFoundError: + sdk_version = None + cli_path = _REPO_ROOT / ".venv" / "bin" / "openshell" + cli_version = _cli_version(cli_path) + gateway_type = _gateway_type(cli_path, gateway_name) + current_system = system or platform.system() + + formulas: list[str] = [] + formula = formula_version = packaged_cli_version = None + if current_system == "Darwin" and gateway_type != "remote": + formulas, formula, formula_version, packaged_cli_version = _homebrew_components() + gateway_version = _live_gateway_version(gateway_name) if include_live else None + + reason = None + remediation = None + local_macos = current_system == "Darwin" and gateway_type != "remote" + if local_macos and (len(formulas) > 1 or (formula is not None and formula != _OFFICIAL_FORMULA)): + reason = "ambiguous_gateway_installation" + elif local_macos and formula is None: + reason = "packaged_gateway_missing" + elif sdk_version != certified or cli_version != certified: + reason = "component_version_mismatch" + remediation = ( + "Run ./scripts/openshell/setup_openshell.sh " + f"--openshell-version {certified} to repair the AI-Q environment." + ) + elif local_macos and (formula_version != certified or packaged_cli_version != certified): + reason = "component_version_mismatch" + elif include_live and gateway_version is None: + reason = "gateway_unavailable" + if gateway_type == "remote": + remediation = f"Restore registered gateway '{gateway_name}' through its external operator." + else: + remediation = "Run ./scripts/openshell/start_openshell_gateway.sh to start or verify the packaged service." + elif include_live and gateway_version != certified: + reason = "remote_gateway_version_mismatch" if gateway_type == "remote" else "component_version_mismatch" + + if reason is not None and remediation is None: + if local_macos: + remediation = _LOCAL_REMEDIATION + else: + remediation = f"Upgrade registered gateway '{gateway_name}' and its CLI/SDK to OpenShell {certified}." + + return ComponentReport( + certified_version=certified, + sdk_version=sdk_version, + virtualenv_cli_version=cli_version, + homebrew_formula=formula, + homebrew_formula_version=formula_version, + packaged_cli_version=packaged_cli_version, + live_gateway_version=gateway_version, + gateway_type=gateway_type, + reason_code=reason, + remediation=remediation, + ) + + +def _print_human(report: ComponentReport) -> None: + values: list[tuple[str, Any]] = [ + ("Certified version", report.certified_version), + ("AI-Q SDK version", report.sdk_version), + ("AI-Q virtual-environment CLI version", report.virtualenv_cli_version), + ("Homebrew formula", report.homebrew_formula), + ("Homebrew formula version", report.homebrew_formula_version), + ("Packaged CLI version", report.packaged_cli_version), + ("Live gateway version", report.live_gateway_version), + ] + for label, value in values: + print(f"{label}: {value or 'not detected'}") + if report.reason_code: + print(f"Reason: {report.reason_code}", file=sys.stderr) + print(f"Remediation: {report.remediation}", file=sys.stderr) + else: + print("OpenShell component versions match the certified stack.") + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--gateway-name", default="openshell") + parser.add_argument("--skip-live", action="store_true") + parser.add_argument("--json", action="store_true") + return parser + + +def main(argv: list[str] | None = None) -> int: + args = _parser().parse_args(argv) + report = inspect_components(gateway_name=args.gateway_name, include_live=not args.skip_live) + if args.json: + print(json.dumps(asdict(report), sort_keys=True)) + else: + _print_human(report) + return 1 if report.reason_code else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/openshell/install_gateway.sh b/scripts/openshell/install_gateway.sh new file mode 100755 index 000000000..fc29f0cad --- /dev/null +++ b/scripts/openshell/install_gateway.sh @@ -0,0 +1,201 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Explicitly install the certified packaged OpenShell gateway for local Apple Silicon demos. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +PYTHON_BIN="${PYTHON_BIN:-$REPO_ROOT/.venv/bin/python}" +DRY_RUN=false +ASSUME_YES=false +USE_COLIMA=false +DOCKER_HOST_VALUE="" +GATEWAY_NAME="${AIQ_OPENSHELL_GATEWAY_NAME:-openshell}" + +usage() { + cat <<'EOF' +Usage: scripts/openshell/install_gateway.sh [options] + +Installs or reinstalls AI-Q's certified OpenShell release through OpenShell's +official tagged installer. This is local-demo tooling for Apple Silicon macOS; +Linux and remote gateways remain operator-owned. + +Options: + --dry-run Print the planned release and operations only. + --yes Skip the interactive confirmation. + --colima Persist the Docker driver and default Colima socket. + --docker-host unix://PATH Persist a specific local Docker socket (implies --colima). + --gateway-name NAME Registered local gateway name (default: openshell). + -h, --help Show this help. + +Canonical operator guide: docs/source/deployment/openshell.md +EOF +} + +fail() { + echo "ERROR: $*" >&2 + exit 1 +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --dry-run) + DRY_RUN=true + shift + ;; + --yes) + ASSUME_YES=true + shift + ;; + --colima) + USE_COLIMA=true + shift + ;; + --docker-host) + [[ $# -ge 2 ]] || fail "--docker-host requires a unix:// path" + DOCKER_HOST_VALUE="$2" + USE_COLIMA=true + shift 2 + ;; + --gateway-name) + [[ $# -ge 2 ]] || fail "--gateway-name requires a name" + GATEWAY_NAME="$2" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + usage >&2 + fail "Unknown option: $1" + ;; + esac +done + +[[ "$(id -u)" -ne 0 ]] || fail "Run this installer as the logged-in macOS user, not root" +[[ "$(uname -s)" == "Darwin" && "$(uname -m)" == "arm64" ]] \ + || fail "The AI-Q gateway installer supports Apple Silicon macOS local gateways only" +command -v brew >/dev/null 2>&1 || fail "Homebrew is required" +[[ -x "$PYTHON_BIN" ]] || fail "AI-Q Python was not found; run ./scripts/setup.sh first" + +RELEASE_TAG="$($PYTHON_BIN "$SCRIPT_DIR/version_contract.py" --field release-tag)" +EXPECTED_SHA256="$($PYTHON_BIN "$SCRIPT_DIR/version_contract.py" --field installer-sha256)" +INSTALLER_URL="https://raw.githubusercontent.com/NVIDIA/OpenShell/$RELEASE_TAG/install.sh" + +if [[ -n "$DOCKER_HOST_VALUE" && ! "$DOCKER_HOST_VALUE" =~ ^unix://[^[:space:]]+$ ]]; then + fail "--docker-host must be a unix:// path" +fi +if [[ "$USE_COLIMA" == "true" && -z "$DOCKER_HOST_VALUE" ]]; then + DOCKER_HOST_VALUE="unix://$HOME/.colima/default/docker.sock" +fi + +installed_formulas="$(brew list --formula --full-name 2>/dev/null \ + | awk -F/ '$NF == "openshell" {print}' \ + | sort -u || true)" +formula_count="$(printf '%s\n' "$installed_formulas" | awk 'NF {count++} END {print count+0}')" +if [[ "$formula_count" -gt 1 ]]; then + fail "ambiguous_gateway_installation: multiple OpenShell formulas are installed" +fi +if [[ "$formula_count" -eq 1 ]]; then + installed_formula="$(printf '%s\n' "$installed_formulas" | awk 'NF {print; exit}')" + if [[ "$installed_formula" != "nvidia/openshell/openshell" ]]; then + fail "ambiguous_gateway_installation: remove the non-official OpenShell formula before continuing" + fi +fi +services_json="$(brew services list --json 2>/dev/null || printf '[]')" +service_names="$(BREW_SERVICES_JSON="$services_json" "$PYTHON_BIN" - <<'PY' +import json +import os + +try: + services = json.loads(os.environ.get("BREW_SERVICES_JSON", "[]")) +except json.JSONDecodeError: + services = [] +names = { + item["name"] + for item in services + if isinstance(item, dict) + and isinstance(item.get("name"), str) + and item["name"].split("/")[-1] == "openshell" +} +print("\n".join(sorted(names))) +PY +)" +service_count="$(printf '%s\n' "$service_names" | awk 'NF {count++} END {print count+0}')" +if [[ "$service_count" -gt 1 ]]; then + fail "ambiguous_gateway_installation: multiple OpenShell services are registered" +fi + +cat <&2; exit 1; } +[[ "$OPENSHELL_ADAPTER_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] \ + || { echo "ERROR: invalid OpenShell adapter contract" >&2; exit 1; } +DEFAULT_OPENSHELL_VERSION="${OPENSHELL_RELEASE_TAG#v}" OPENSHELL_VERSION="${AIQ_OPENSHELL_VERSION:-}" -OPENSHELL_VERSION_USER_SUPPLIED=false -if [[ -n "$OPENSHELL_VERSION" ]]; then - OPENSHELL_VERSION_USER_SUPPLIED=true -fi -OPENSHELL_LATEST_VERSION="" -OPENSHELL_AVAILABLE_VERSIONS="" -PYTHON_BIN="" # Official OpenShell deepagents adapter: the `langchain-nvidia-openshell` partner # package, now published on PyPI. Override with LANGCHAIN_NVIDIA_REPO to use a git # spec or a local checkout (e.g. to test an unreleased adapter build). -DEFAULT_LANGCHAIN_NVIDIA_INSTALL_SPEC="langchain-nvidia-openshell==0.1.0" +DEFAULT_LANGCHAIN_NVIDIA_INSTALL_SPEC="langchain-nvidia-openshell==$OPENSHELL_ADAPTER_VERSION" LANGCHAIN_NVIDIA_REPO="${LANGCHAIN_NVIDIA_REPO:-$DEFAULT_LANGCHAIN_NVIDIA_INSTALL_SPEC}" -SANDBOX_NAME="${AIQ_OPENSHELL_SANDBOX_NAME:-aiq-openshell-demo}" IMAGE_NAME="${AIQ_OPENSHELL_IMAGE:-aiq-openshell-demo:latest}" # Sandbox log verbosity baked into the image (RUST_LOG). Default `warn` is OpenShell's # stock sandbox level; set to `debug` to surface in-container process/relay detail. @@ -48,16 +50,12 @@ SANDBOX_LOG_LEVEL="${AIQ_OPENSHELL_SANDBOX_LOG_LEVEL:-warn}" POLICY_PRESET="${AIQ_OPENSHELL_POLICY:-}" POLICY_ALLOWLIST="${AIQ_OPENSHELL_POLICY_ALLOWLIST:-${AIQ_OPENSHELL_POLICY_SERVICES:-}}" POLICY_FILE="${AIQ_OPENSHELL_POLICY_FILE:-$REPO_ROOT/configs/openshell/generated/aiq-openshell-policy.yaml}" -GATEWAY_NAME="${AIQ_OPENSHELL_GATEWAY_NAME:-aiq-local}" -GATEWAY_PORT="${AIQ_OPENSHELL_GATEWAY_PORT:-8080}" +LANDLOCK_COMPATIBILITY="${AIQ_OPENSHELL_LANDLOCK_COMPATIBILITY:-hard_requirement}" +LANDLOCK_COMPATIBILITY_CLI=false +LOCAL_DEMO=false DOCKER_BIN="${DOCKER_BIN:-}" OPENSHELL_BIN="${OPENSHELL_BIN:-}" -OPENSHELL_GATEWAY_LAUNCH_BIN="${OPENSHELL_GATEWAY_LAUNCH_BIN:-}" -GATEWAY_ENDPOINT="" -GATEWAY_DISABLE_TLS=false -RESTART_GATEWAY=true BUILD_IMAGE=true -CREATE_SANDBOX=true LIST_OPENSHELL_VERSIONS=false SUPPORTED_SERVICES="github,pypi,nvidia,tavily,serper,huggingface,arxiv,semantic-scholar,npm" @@ -65,23 +63,24 @@ SUPPORTED_POLICIES="offline,research,python-packages,ai-dev,custom" usage() { cat <= 0.0.72. + --list-openshell-versions Print the certified OpenShell version. -h, --help Show this help. Examples: - scripts/setup_openshell.sh - scripts/setup_openshell.sh --policy python-packages - scripts/setup_openshell.sh --policy custom --allow github,pypi,nvidia,tavily - scripts/setup_openshell.sh --openshell-version latest --policy offline + scripts/openshell/setup_openshell.sh + scripts/openshell/setup_openshell.sh --policy python-packages + scripts/openshell/setup_openshell.sh --policy custom --allow github,pypi,nvidia,tavily + scripts/openshell/setup_openshell.sh --openshell-version 0.0.80 --policy offline + scripts/openshell/setup_openshell.sh --local-demo --policy offline EOF } @@ -130,7 +127,6 @@ while [[ $# -gt 0 ]]; do case "$1" in --openshell-version) OPENSHELL_VERSION="$2" - OPENSHELL_VERSION_USER_SUPPLIED=true shift 2 ;; --policy) @@ -149,10 +145,15 @@ while [[ $# -gt 0 ]]; do POLICY_FILE="$2" shift 2 ;; - --sandbox-name) - SANDBOX_NAME="$2" + --landlock-compatibility) + LANDLOCK_COMPATIBILITY="$2" + LANDLOCK_COMPATIBILITY_CLI=true shift 2 ;; + --local-demo) + LOCAL_DEMO=true + shift + ;; --image-name) IMAGE_NAME="$2" shift 2 @@ -165,33 +166,16 @@ while [[ $# -gt 0 ]]; do LANGCHAIN_NVIDIA_REPO="$2" shift 2 ;; - --gateway-name) - GATEWAY_NAME="$2" - shift 2 - ;; - --gateway-port) - GATEWAY_PORT="$2" - shift 2 - ;; --docker-bin) DOCKER_BIN="$2" shift 2 ;; - --gateway-bin) - OPENSHELL_GATEWAY_LAUNCH_BIN="$2" - shift 2 - ;; - --no-restart-gateway) - RESTART_GATEWAY=false - shift - ;; --skip-build) BUILD_IMAGE=false shift ;; - --skip-sandbox) - CREATE_SANDBOX=false - shift + --create-shared-debug-sandbox|--sandbox-name|--gateway-name|--gateway-port|--gateway-bin|--no-restart-gateway|--skip-sandbox) + fail "Gateway and debug-sandbox lifecycle moved to scripts/openshell/start_openshell_gateway.sh" ;; --list-policies) echo "$SUPPORTED_POLICIES" | tr ',' '\n' @@ -216,6 +200,13 @@ while [[ $# -gt 0 ]]; do esac done +if [[ "$LOCAL_DEMO" == "true" ]]; then + if [[ "$LANDLOCK_COMPATIBILITY_CLI" == "true" && "$LANDLOCK_COMPATIBILITY" != "best_effort" ]]; then + fail "--local-demo conflicts with --landlock-compatibility $LANDLOCK_COMPATIBILITY" + fi + LANDLOCK_COMPATIBILITY="best_effort" +fi + detect_os() { case "$(uname -s)" in Darwin) @@ -246,132 +237,12 @@ EOF fi } -resolve_python() { - if [[ -n "$PYTHON_BIN" && -x "$PYTHON_BIN" ]]; then - return - fi - PYTHON_BIN="$(uv python find 2>/dev/null || true)" - if [[ -z "$PYTHON_BIN" || ! -x "$PYTHON_BIN" ]]; then - PYTHON_BIN="$(command -v python3 || command -v python || true)" - fi - if [[ -z "$PYTHON_BIN" || ! -x "$PYTHON_BIN" ]]; then - fail "Python was not found. Install Python 3.11+ or run ./scripts/setup.sh first." - fi -} - -fetch_openshell_versions() { - resolve_python - log "Checking available OpenShell versions" - local output - if ! output="$("$PYTHON_BIN" - "$MIN_OPENSHELL_VERSION" <<'PY' -import json -import re -import sys -import urllib.request - -min_version = sys.argv[1] -version_re = re.compile(r"^\d+\.\d+\.\d+$") - -def parse(version: str) -> tuple[int, int, int]: - return tuple(int(part) for part in version.split(".")) - -try: - with urllib.request.urlopen("https://pypi.org/pypi/openshell/json", timeout=15) as response: - payload = json.load(response) -except Exception as exc: - raise SystemExit(f"failed to fetch OpenShell versions from PyPI: {exc}") - -minimum = parse(min_version) -versions = [] -for version, files in payload.get("releases", {}).items(): - if not version_re.match(version): - continue - if not files: - continue - parsed = parse(version) - if parsed >= minimum: - versions.append((parsed, version)) - -if not versions: - raise SystemExit(f"no OpenShell releases found at or above {min_version}") - -versions.sort() -print(versions[-1][1]) -print(",".join(version for _, version in versions)) -PY -)"; then - fail "$output" - fi - - OPENSHELL_LATEST_VERSION="$(printf '%s\n' "$output" | sed -n '1p')" - OPENSHELL_AVAILABLE_VERSIONS="$(printf '%s\n' "$output" | sed -n '2p')" - - if [[ -z "$OPENSHELL_LATEST_VERSION" || -z "$OPENSHELL_AVAILABLE_VERSIONS" ]]; then - fail "Could not determine available OpenShell versions from PyPI." - fi - echo "OpenShell version range: $MIN_OPENSHELL_VERSION through $OPENSHELL_LATEST_VERSION" -} - -version_is_available() { - case ",$OPENSHELL_AVAILABLE_VERSIONS," in - *",$1,"*) - return 0 - ;; - *) - return 1 - ;; - esac -} - -version_prompt_text() { - echo "OpenShell version [press Enter for ${DEFAULT_OPENSHELL_VERSION}; type latest for ${OPENSHELL_LATEST_VERSION}]: " -} - -choose_openshell_version_interactively() { - local candidate - while true; do - read -r -p "$(version_prompt_text)" candidate - candidate="${candidate:-$DEFAULT_OPENSHELL_VERSION}" - if [[ "$candidate" == "latest" ]]; then - candidate="$OPENSHELL_LATEST_VERSION" - fi - if version_is_available "$candidate"; then - OPENSHELL_VERSION="$candidate" - return - fi - echo "OpenShell version '$candidate' was not found between $MIN_OPENSHELL_VERSION and $OPENSHELL_LATEST_VERSION." - echo "Try an exact released version, '$DEFAULT_OPENSHELL_VERSION', or 'latest'." - done -} - resolve_openshell_version() { - fetch_openshell_versions - - if [[ -n "$OPENSHELL_VERSION" ]]; then - if [[ "$OPENSHELL_VERSION" == "latest" ]]; then - OPENSHELL_VERSION="$OPENSHELL_LATEST_VERSION" - fi - if version_is_available "$OPENSHELL_VERSION"; then - echo "OpenShell version selected: $OPENSHELL_VERSION" - return - fi - - if [[ -t 0 && "$OPENSHELL_VERSION_USER_SUPPLIED" == "true" ]]; then - echo "OpenShell version '$OPENSHELL_VERSION' was not found between $MIN_OPENSHELL_VERSION and $OPENSHELL_LATEST_VERSION." - choose_openshell_version_interactively - echo "OpenShell version selected: $OPENSHELL_VERSION" - return - fi - fail "OpenShell version '$OPENSHELL_VERSION' was not found between $MIN_OPENSHELL_VERSION and $OPENSHELL_LATEST_VERSION." - fi - - if [[ -t 0 ]]; then - choose_openshell_version_interactively - else + if [[ -z "$OPENSHELL_VERSION" ]]; then OPENSHELL_VERSION="$DEFAULT_OPENSHELL_VERSION" - if ! version_is_available "$OPENSHELL_VERSION"; then - fail "Default OpenShell version '$OPENSHELL_VERSION' was not found on PyPI." - fi + fi + if [[ "$OPENSHELL_VERSION" == "latest" || "$OPENSHELL_VERSION" != "$DEFAULT_OPENSHELL_VERSION" ]]; then + fail "OpenShell '$OPENSHELL_VERSION' is not certified for this AI-Q release. Use exact version $DEFAULT_OPENSHELL_VERSION; certify upgrades in a separate development change." fi echo "OpenShell version selected: $OPENSHELL_VERSION" } @@ -407,31 +278,40 @@ install_openshell_python() { # NOTE: expand editable_args only when non-empty; macOS bash 3.2 errors on # "${arr[@]}" for an empty array under `set -u`. local adapter_install_args=() - if [[ ${#editable_args[@]} -eq 0 ]]; then - adapter_install_args=(--reinstall-package langchain-nvidia-openshell) - else + if [[ ${#editable_args[@]} -gt 0 ]]; then adapter_install_args=("${editable_args[@]}") fi - if ! uv pip install "${adapter_install_args[@]}" "$adapter_install_spec"; then + local adapter_install_failed=false + if [[ ${#adapter_install_args[@]} -eq 0 ]]; then + uv pip install "$adapter_install_spec" || adapter_install_failed=true + else + uv pip install "${adapter_install_args[@]}" "$adapter_install_spec" || adapter_install_failed=true + fi + if [[ "$adapter_install_failed" == "true" ]]; then cat <=0.6.5). The adapter's - # code only uses the stable deepagents BaseSandbox/protocol surface (the same imports - # AI-Q's own sandbox package uses on 0.6.x), so reasserting the floor AI-Q needs is - # safe. This is the OpenShell setup script, so keeping AI-Q runnable is the goal. - log "Reasserting deepagents>=0.6.5 (AI-Q runtime floor) after adapter install" - uv pip install "deepagents>=0.6.5" + # Adapter 0.1.0 still declares deepagents<0.6 and can otherwise downgrade AI-Q's + # locked DeepAgents 0.6.x runtime. Restore the complete AI-Q lock while retaining + # optional packages that are intentionally absent from the base project metadata. + # This keeps repeated setup deterministic without pretending the upstream adapter + # metadata is compatible; `pip check` remains a documented upstream limitation. + log "Restoring locked AI-Q dependencies while retaining optional OpenShell packages" + uv sync --dev --inexact + + # Adapter dependency resolution must not silently change the operator-selected + # OpenShell SDK/CLI version. Reapply the exact pin after every dependent package. + log "Reasserting exact OpenShell version after adapter install: openshell==$OPENSHELL_VERSION" + uv pip install "openshell==$OPENSHELL_VERSION" local installed installed="$("$VENV_DIR/bin/python" - <<'PY' @@ -439,21 +319,14 @@ import openshell print(getattr(openshell, "__version__", "unknown")) PY )" - # The adapter pins openshell>=0.0.68 and may upgrade the package above the requested - # version during its own install; only a version BELOW the requested floor is an error - # (an exact-match check would spuriously fail on that allowed adapter-driven upgrade). + # CLI, SDK, and gateway compatibility is validated again by the strict readiness + # checker. Provisioning still guarantees that the selected local SDK is exact. if ! "$VENV_DIR/bin/python" - "$OPENSHELL_VERSION" "$installed" <<'PY' import sys - - -def parts(v): - return tuple(int(p) for p in v.split(".")[:3] if p.isdigit()) - - -sys.exit(0 if parts(sys.argv[2]) >= parts(sys.argv[1]) else 1) +sys.exit(0 if sys.argv[2] == sys.argv[1] else 1) PY then - fail "Installed openshell $installed is older than the requested floor $OPENSHELL_VERSION" + fail "Installed openshell $installed does not match the requested version $OPENSHELL_VERSION" fi "$VENV_DIR/bin/python" - <<'PY' import langchain_nvidia_openshell # noqa: F401 @@ -482,6 +355,13 @@ resolve_openshell_cli() { resolve_docker() { log "Resolving Docker CLI" + # Docker Desktop stores both the CLI and credential helper here. Prepend the + # directory before resolving `docker` so public-image pulls do not follow a + # stale /usr/local symlink and then fail to locate docker-credential-desktop. + local docker_desktop_bin="/Applications/Docker.app/Contents/Resources/bin" + if [[ "$OS_NAME" == "macos" && -x "$docker_desktop_bin/docker" ]]; then + export PATH="$docker_desktop_bin:$PATH" + fi local candidates=( "$DOCKER_BIN" "$(command -v docker || true)" @@ -514,7 +394,7 @@ Install or link Docker CLI, then rerun: If Docker is installed but not on PATH, rerun with: - scripts/setup_openshell.sh --docker-bin /path/to/docker + scripts/openshell/setup_openshell.sh --docker-bin /path/to/docker EOF else cat <<'EOF' @@ -527,7 +407,7 @@ Install Docker for your Linux distribution, then rerun. For example: If Docker is installed but not on PATH, rerun with: - scripts/setup_openshell.sh --docker-bin /path/to/docker + scripts/openshell/setup_openshell.sh --docker-bin /path/to/docker EOF fi exit 1 @@ -566,7 +446,7 @@ DOCKER_HOST is set to: Start that Docker daemon or update DOCKER_HOST, then rerun: - scripts/setup_openshell.sh + scripts/openshell/setup_openshell.sh EOF exit 1 fi @@ -590,7 +470,7 @@ Docker Desktop appears to be installed at: Start Docker Desktop, wait until it reports that Docker is running, then rerun: - scripts/setup_openshell.sh + scripts/openshell/setup_openshell.sh If you use a remote Docker daemon, set DOCKER_HOST before rerunning. EOF @@ -601,7 +481,7 @@ Docker CLI was found, but the Docker daemon is not reachable. No Colima socket was selected and Docker Desktop was not found in /Applications or ~/Applications. Install and start Docker Desktop for macOS, then rerun: - scripts/setup_openshell.sh + scripts/openshell/setup_openshell.sh If you use Colima, start it first. For example: @@ -630,132 +510,6 @@ EOF exit 1 } -resolve_gateway_launcher() { - if [[ -n "$OPENSHELL_GATEWAY_LAUNCH_BIN" && -x "$OPENSHELL_GATEWAY_LAUNCH_BIN" ]]; then - echo "OpenShell gateway launcher: $OPENSHELL_GATEWAY_LAUNCH_BIN" - return - fi - - local candidates=( - "/opt/homebrew/opt/openshell/libexec/openshell-gateway-homebrew-service" - "$(command -v openshell-gateway || true)" - "/opt/homebrew/bin/openshell-gateway" - "/usr/local/bin/openshell-gateway" - "/usr/bin/openshell-gateway" - "$HOME/.local/bin/openshell-gateway" - "$VENV_DIR/bin/openshell-gateway" - ) - local candidate - for candidate in "${candidates[@]}"; do - if [[ -n "$candidate" && -x "$candidate" ]]; then - OPENSHELL_GATEWAY_LAUNCH_BIN="$candidate" - echo "OpenShell gateway launcher: $OPENSHELL_GATEWAY_LAUNCH_BIN" - return - fi - done - - if [[ "$OS_NAME" == "macos" ]]; then - # The nvidia/openshell Homebrew tap (github.com/nvidia/homebrew-openshell) is not - # published, so `brew install nvidia/openshell/openshell` 404s. Use OpenShell's - # official installer instead, which sets up the gateway (local brew service + mTLS). - log "Installing the OpenShell gateway via the official installer (NVIDIA/OpenShell)" - curl -LsSf https://raw.githubusercontent.com/NVIDIA/OpenShell/main/install.sh | sh - local installed_candidate - for installed_candidate in \ - "/opt/homebrew/opt/openshell/libexec/openshell-gateway-homebrew-service" \ - "$(command -v openshell-gateway || true)" \ - "/opt/homebrew/bin/openshell-gateway"; do - if [[ -n "$installed_candidate" && -x "$installed_candidate" ]]; then - OPENSHELL_GATEWAY_LAUNCH_BIN="$installed_candidate" - echo "OpenShell gateway launcher: $OPENSHELL_GATEWAY_LAUNCH_BIN" - return - fi - done - fi - - cat </dev/null 2>&1 || true - sleep 2 - rm -f /tmp/aiq-openshell-gateway.log - if [[ "$GATEWAY_DISABLE_TLS" == "true" ]]; then - nohup env OPENSHELL_SERVER_PORT="$GATEWAY_PORT" \ - OPENSHELL_DRIVERS=docker \ - DOCKER_HOST="${DOCKER_HOST:-}" \ - "$OPENSHELL_GATEWAY_LAUNCH_BIN" --disable-tls >/tmp/aiq-openshell-gateway.log 2>&1 & - else - nohup env OPENSHELL_SERVER_PORT="$GATEWAY_PORT" \ - OPENSHELL_DRIVERS=docker \ - DOCKER_HOST="${DOCKER_HOST:-}" \ - "$OPENSHELL_GATEWAY_LAUNCH_BIN" >/tmp/aiq-openshell-gateway.log 2>&1 & - fi - "$OPENSHELL_BIN" gateway remove "$GATEWAY_NAME" >/dev/null 2>&1 || true - if [[ "$GATEWAY_DISABLE_TLS" == "true" ]]; then - "$OPENSHELL_BIN" gateway add --name "$GATEWAY_NAME" "$GATEWAY_ENDPOINT" --local - else - "$OPENSHELL_BIN" gateway add --name "$GATEWAY_NAME" "$GATEWAY_ENDPOINT" --local --gateway-insecure - fi - "$OPENSHELL_BIN" gateway select "$GATEWAY_NAME" - fi - - local attempt - for attempt in $(seq 1 60); do - if "$OPENSHELL_BIN" status; then - return - fi - sleep 1 - done - - echo "OpenShell gateway log:" - LC_ALL=C tr -d '\000' /dev/null | sed -n '1,220p' || true - fail "OpenShell gateway did not become reachable" -} - print_policy_menu() { cat <<'EOF' @@ -864,6 +618,13 @@ resolve_policy() { if [[ "$POLICY_ALLOWLIST" == *offline* && "$POLICY_ALLOWLIST" != "offline" ]]; then fail "Use either offline or a service allowlist, not both: $POLICY_ALLOWLIST" fi + case "$LANDLOCK_COMPATIBILITY" in + hard_requirement|best_effort) + ;; + *) + fail "Unsupported Landlock compatibility '$LANDLOCK_COMPATIBILITY'; use hard_requirement or best_effort" + ;; + esac } validate_service() { @@ -877,23 +638,24 @@ validate_service() { } emit_policy_header() { - cat >"$POLICY_FILE" <<'EOF' + cat >"$POLICY_FILE" </dev/null 2>&1 || true - local create_log="/tmp/${SANDBOX_NAME}-openshell-create.log" - local policy_label="${POLICY_PRESET//,/_}" - rm -f "$create_log" - "$OPENSHELL_BIN" sandbox create \ - --name "$SANDBOX_NAME" \ - --from "$IMAGE_NAME" \ - --policy "$POLICY_FILE" \ - --label aiq=openshell \ - --label aiq-policy="$policy_label" \ - --no-tty \ - -- sleep infinity >"$create_log" 2>&1 & - - local attempt - for attempt in $(seq 1 120); do - if "$OPENSHELL_BIN" sandbox list | grep -F "$SANDBOX_NAME" | grep -F "Ready" >/dev/null 2>&1; then - "$OPENSHELL_BIN" sandbox list | grep -F "$SANDBOX_NAME" || true - return - fi - sleep 1 - done - - echo "Sandbox create log:" - sed -n '1,220p' "$create_log" || true - fail "Timed out waiting for sandbox '$SANDBOX_NAME' to become Ready" } print_next_steps() { + local runtime_config="configs/config_openshell.yml" + local runtime_env="" + local landlock_note="Production defaults require hard Landlock; no runtime override is needed." + if [[ "$LANDLOCK_COMPATIBILITY" == "best_effort" ]]; then + runtime_env="AIQ_OPENSHELL_REQUIRE_HARD_LANDLOCK=false " + landlock_note="This is a local best_effort policy. Prefix validation, CLI, and E2E commands with +AIQ_OPENSHELL_REQUIRE_HARD_LANDLOCK=false." + fi cat < argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--gateway", + default=os.getenv("AIQ_OPENSHELL_GATEWAY_NAME"), + help="Registered OpenShell gateway name; default uses the active gateway", + ) + parser.add_argument( + "--policy", + default=os.getenv( + "AIQ_OPENSHELL_POLICY_FILE", + "configs/openshell/generated/aiq-openshell-policy.yaml", + ), + help="Policy submitted and attested by the live suite", + ) + parser.add_argument( + "--image", + default=os.getenv("AIQ_OPENSHELL_IMAGE", "aiq-openshell-demo:latest"), + help="Prebuilt OpenShell sandbox image", + ) + parser.add_argument( + "--expected-gateway-version", + default=os.getenv("AIQ_OPENSHELL_EXPECTED_GATEWAY_VERSION"), + help="Optional exact gateway version; default requires it to match the installed SDK", + ) + parser.add_argument( + "--allow-best-effort-landlock", + action="store_true", + help="Permit a local-demo best_effort policy; never use this for production acceptance", + ) + return parser.parse_args(argv) + + +def _environment(args: argparse.Namespace, source: Mapping[str, str] | None = None) -> dict[str, str]: + env = dict(os.environ if source is None else source) + env["AIQ_OPENSHELL_LIVE_TESTS"] = "1" + if args.gateway: + env["AIQ_OPENSHELL_GATEWAY_NAME"] = args.gateway + else: + env.pop("AIQ_OPENSHELL_GATEWAY_NAME", None) + env["AIQ_OPENSHELL_POLICY_FILE"] = args.policy + env["AIQ_OPENSHELL_IMAGE"] = args.image + if args.expected_gateway_version: + env["AIQ_OPENSHELL_EXPECTED_GATEWAY_VERSION"] = args.expected_gateway_version + else: + env.pop("AIQ_OPENSHELL_EXPECTED_GATEWAY_VERSION", None) + if args.allow_best_effort_landlock: + env["AIQ_OPENSHELL_LIVE_ALLOW_BEST_EFFORT"] = "1" + return env + + +def _command() -> list[str]: + return [sys.executable, "-m", "pytest", "-m", "integration", "-vv", str(_LIVE_TEST)] + + +def main(argv: list[str] | None = None) -> int: + args = _args(argv) + result = subprocess.run( # noqa: S603 - fixed argv, no shell, operator-selected interpreter + _command(), + cwd=_REPO_ROOT, + env=_environment(args), + check=False, + ) + return result.returncode + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/openshell/start_openshell_gateway.sh b/scripts/openshell/start_openshell_gateway.sh new file mode 100755 index 000000000..ee331e4b6 --- /dev/null +++ b/scripts/openshell/start_openshell_gateway.sh @@ -0,0 +1,271 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Start or reuse an authenticated OpenShell gateway and prove strict AI-Q capabilities. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +VENV_DIR="$REPO_ROOT/.venv" + +GATEWAY_NAME="${AIQ_OPENSHELL_GATEWAY_NAME:-openshell}" +IMAGE_NAME="${AIQ_OPENSHELL_IMAGE:-aiq-openshell-demo:latest}" +POLICY_FILE="${AIQ_OPENSHELL_POLICY_FILE:-$REPO_ROOT/configs/openshell/generated/aiq-openshell-policy.yaml}" +SANDBOX_NAME="${AIQ_OPENSHELL_SANDBOX_NAME:-aiq-openshell-demo}" +OPENSHELL_BIN="${OPENSHELL_BIN:-}" +PYTHON_BIN="${PYTHON_BIN:-}" +START_SERVICE=true +CREATE_SHARED_DEBUG_SANDBOX=false +STATUS_ATTEMPTS="${AIQ_OPENSHELL_STATUS_ATTEMPTS:-60}" +POLL_DELAY="${AIQ_OPENSHELL_POLL_DELAY:-1}" +READY_TIMEOUT_SECONDS="${AIQ_OPENSHELL_READY_TIMEOUT_SECONDS:-120}" +POLICY_LOAD_TIMEOUT_SECONDS="${AIQ_OPENSHELL_POLICY_LOAD_TIMEOUT_SECONDS:-30}" +READINESS_CHECKER="$SCRIPT_DIR/check_openshell_readiness.py" +VERSION_INSPECTOR="$SCRIPT_DIR/check_versions.py" + +usage() { + cat <<'EOF' +Usage: scripts/openshell/start_openshell_gateway.sh [options] + +Starts or reuses the official packaged OpenShell gateway service, validates the +selected registration is authenticated, and performs a mandatory disposable +policy/selector/execution/cleanup probe. The script never launches the raw gateway binary. +Long-running service and credential ownership remains with Homebrew, systemd, or +the external operator. + +Canonical operator guide: docs/source/deployment/openshell.md + +Options: + --gateway-name NAME Registered gateway name (default: openshell). + --image-name NAME Image used for the readiness probe. + --policy-file PATH Policy used for the readiness probe. + --reuse-existing, --no-start Do not start a local packaged service. + --create-shared-debug-sandbox Create/reuse a persistent named debug sandbox. + --sandbox-name NAME Shared debug sandbox name. + -h, --help Show this help. +EOF +} + +fail() { + echo "ERROR: $*" >&2 + exit 1 +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --gateway-name) + GATEWAY_NAME="$2" + shift 2 + ;; + --image-name) + IMAGE_NAME="$2" + shift 2 + ;; + --policy-file) + POLICY_FILE="$2" + shift 2 + ;; + --reuse-existing|--no-start) + START_SERVICE=false + shift + ;; + --create-shared-debug-sandbox) + CREATE_SHARED_DEBUG_SANDBOX=true + shift + ;; + --sandbox-name) + SANDBOX_NAME="$2" + shift 2 + ;; + --gateway-bin) + fail "Raw openshell-gateway launch is unsupported; use the packaged service or --reuse-existing" + ;; + -h|--help) + usage + exit 0 + ;; + *) + usage + fail "Unknown option: $1" + ;; + esac +done + +resolve_dependencies() { + if [[ -n "${OPENSHELL_GATEWAY_LAUNCH_BIN:-}" ]]; then + fail "OPENSHELL_GATEWAY_LAUNCH_BIN is unsupported; raw gateway processes are not an authenticated lifecycle" + fi + if [[ -n "${OPENSHELL_GATEWAY_ENDPOINT:-}" ]]; then + fail "Direct OPENSHELL_GATEWAY_ENDPOINT bypasses registration authentication checks" + fi + case "${OPENSHELL_GATEWAY_INSECURE:-}" in + 1|true|TRUE|yes|YES|on|ON) + fail "OPENSHELL_GATEWAY_INSECURE is forbidden for AI-Q OpenShell lifecycle checks" + ;; + esac + if [[ ! -f "$POLICY_FILE" ]]; then + fail "Policy file not found: $POLICY_FILE; run scripts/openshell/setup_openshell.sh first" + fi + + if [[ -z "$OPENSHELL_BIN" ]]; then + if [[ -x "$VENV_DIR/bin/openshell" ]]; then + OPENSHELL_BIN="$VENV_DIR/bin/openshell" + else + OPENSHELL_BIN="$(command -v openshell || true)" + fi + fi + if [[ -z "$OPENSHELL_BIN" || ! -x "$OPENSHELL_BIN" ]]; then + fail "OpenShell CLI not found; run scripts/openshell/setup_openshell.sh first" + fi + + if [[ -z "$PYTHON_BIN" ]]; then + if [[ -x "$VENV_DIR/bin/python" ]]; then + PYTHON_BIN="$VENV_DIR/bin/python" + else + PYTHON_BIN="$(command -v python3 || true)" + fi + fi + if [[ -z "$PYTHON_BIN" || ! -x "$PYTHON_BIN" ]]; then + fail "Python 3 is required to validate structured gateway metadata" + fi + if [[ ! -f "$READINESS_CHECKER" ]]; then + fail "OpenShell readiness checker not found" + fi + if [[ ! -f "$VERSION_INSPECTOR" ]]; then + fail "OpenShell version inspector not found" + fi +} + +validate_gateway_registration() { + local gateway_json + gateway_json="$("$OPENSHELL_BIN" gateway list -o json)" + GATEWAY_TYPE="$(GATEWAY_JSON="$gateway_json" "$PYTHON_BIN" - "$GATEWAY_NAME" <<'PY' +import json +import os +import sys + +name = sys.argv[1] +try: + payload = json.loads(os.environ["GATEWAY_JSON"]) +except (KeyError, json.JSONDecodeError): + raise SystemExit("gateway metadata is not valid JSON") +if not isinstance(payload, list): + raise SystemExit("gateway metadata must be a JSON list") +record = next((item for item in payload if isinstance(item, dict) and item.get("name") == name), None) +if record is None: + raise SystemExit(f"registered gateway not found: {name}") +endpoint = record.get("endpoint") +auth = record.get("auth") +if not isinstance(endpoint, str) or not endpoint.lower().startswith("https://"): + raise SystemExit("gateway registration must use HTTPS") +if auth not in {"mtls", "oidc", "cloudflare_jwt"}: + raise SystemExit("gateway registration must use mTLS, OIDC, or edge authentication") +print(record.get("type", "remote")) +PY +)" || fail "Gateway '$GATEWAY_NAME' is missing or not authenticated" + "$OPENSHELL_BIN" gateway select "$GATEWAY_NAME" >/dev/null +} + +start_packaged_service() { + if [[ "$GATEWAY_TYPE" != "local" ]]; then + fail "Remote gateway '$GATEWAY_NAME' is unreachable; this script will not start a local replacement" + fi + case "$(uname -s)" in + Darwin) + command -v brew >/dev/null 2>&1 || fail "Homebrew is required to start the packaged OpenShell service" + brew services start nvidia/openshell/openshell >/dev/null + ;; + Linux) + command -v systemctl >/dev/null 2>&1 || fail "systemctl is required to start the packaged OpenShell service" + systemctl --user start openshell-gateway + ;; + *) + fail "Unsupported operating system for packaged gateway startup" + ;; + esac +} + +wait_for_gateway() { + local attempt + if "$OPENSHELL_BIN" status >/dev/null 2>&1; then + return + fi + if [[ "$START_SERVICE" != "true" ]]; then + fail "Gateway '$GATEWAY_NAME' is not reachable and --reuse-existing forbids service startup" + fi + start_packaged_service + for attempt in $(seq 1 "$STATUS_ATTEMPTS"); do + if "$OPENSHELL_BIN" status >/dev/null 2>&1; then + return + fi + sleep "$POLL_DELAY" + done + fail "Authenticated gateway '$GATEWAY_NAME' did not become ready" +} + +check_component_versions() { + if [[ "${1:-}" == "skip-live" ]]; then + "$PYTHON_BIN" "$VERSION_INSPECTOR" --gateway-name "$GATEWAY_NAME" --skip-live \ + || fail "OpenShell component version check failed" + else + "$PYTHON_BIN" "$VERSION_INSPECTOR" --gateway-name "$GATEWAY_NAME" \ + || fail "OpenShell component version check failed" + fi +} + +sandbox_is_listed() { + "$OPENSHELL_BIN" sandbox list | grep -F "$1" >/dev/null 2>&1 +} + +sandbox_is_ready() { + "$OPENSHELL_BIN" sandbox list | grep -F "$1" | grep -F "Ready" >/dev/null 2>&1 +} + +run_strict_readiness_check() { + if ! "$PYTHON_BIN" "$READINESS_CHECKER" \ + --gateway-name "$GATEWAY_NAME" \ + --image-name "$IMAGE_NAME" \ + --policy-file "$POLICY_FILE" \ + --openshell-bin "$OPENSHELL_BIN" \ + --ready-timeout-seconds "$READY_TIMEOUT_SECONDS" \ + --policy-load-timeout-seconds "$POLICY_LOAD_TIMEOUT_SECONDS"; then + fail "OpenShell strict readiness check failed" + fi +} + +create_shared_debug_sandbox() { + if [[ "$CREATE_SHARED_DEBUG_SANDBOX" != "true" ]]; then + return + fi + if sandbox_is_ready "$SANDBOX_NAME"; then + echo "Reusing existing shared debug sandbox: $SANDBOX_NAME" + return + fi + if sandbox_is_listed "$SANDBOX_NAME"; then + fail "Shared debug sandbox exists but is not Ready: $SANDBOX_NAME" + fi + "$OPENSHELL_BIN" sandbox create \ + --name "$SANDBOX_NAME" \ + --from "$IMAGE_NAME" \ + --policy "$POLICY_FILE" \ + --label aiq=shared-debug \ + --no-auto-providers \ + --no-tty \ + -- true >/dev/null + sandbox_is_ready "$SANDBOX_NAME" || fail "Shared debug sandbox did not become Ready" + echo "Created shared debug sandbox: $SANDBOX_NAME" +} + +main() { + resolve_dependencies + validate_gateway_registration + check_component_versions skip-live + wait_for_gateway + check_component_versions + run_strict_readiness_check + create_shared_debug_sandbox +} + +main diff --git a/scripts/openshell/version_contract.py b/scripts/openshell/version_contract.py new file mode 100644 index 000000000..2dd24068e --- /dev/null +++ b/scripts/openshell/version_contract.py @@ -0,0 +1,86 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Read and validate AI-Q's certified OpenShell release contract.""" + +from __future__ import annotations + +import argparse +import json +import re +import tomllib +from dataclasses import asdict +from dataclasses import dataclass +from pathlib import Path + +_REPO_ROOT = Path(__file__).resolve().parents[2] +_EXACT_DEPENDENCY = re.compile(r"^(?P[A-Za-z0-9_.-]+)==(?P[^;\s]+)") + + +@dataclass(frozen=True) +class OpenShellContract: + """Allowlisted release metadata used by setup and diagnostics.""" + + version: str + release_tag: str + installer_sha256: str + adapter_version: str | None + + +def load_contract(pyproject: Path | None = None) -> OpenShellContract: + """Load the release contract and cross-check a published optional extra when present.""" + path = pyproject or (_REPO_ROOT / "pyproject.toml") + data = tomllib.loads(path.read_text(encoding="utf-8")) + table = data.get("tool", {}).get("aiq", {}).get("openshell", {}) + release_tag = table.get("release-tag") + adapter_version = table.get("adapter-version") + checksum = table.get("installer-sha256") + if not isinstance(release_tag, str) or re.fullmatch(r"v\d+\.\d+\.\d+", release_tag) is None: + raise ValueError("invalid_openshell_release_tag") + if not isinstance(checksum, str) or re.fullmatch(r"[0-9a-f]{64}", checksum) is None: + raise ValueError("invalid_openshell_installer_checksum") + if not isinstance(adapter_version, str) or re.fullmatch(r"\d+\.\d+\.\d+", adapter_version) is None: + raise ValueError("invalid_openshell_adapter_version") + + version = release_tag.removeprefix("v") + extra = data.get("project", {}).get("optional-dependencies", {}).get("openshell", []) + exact = {} + for requirement in extra: + if isinstance(requirement, str) and (match := _EXACT_DEPENDENCY.match(requirement)) is not None: + exact[match.group("name").lower()] = match.group("version") + openshell_version = exact.get("openshell") + if openshell_version is not None and openshell_version != version: + raise ValueError("openshell_extra_version_mismatch") + extra_adapter_version = exact.get("langchain-nvidia-openshell") + if extra_adapter_version is not None and extra_adapter_version != adapter_version: + raise ValueError("openshell_adapter_extra_version_mismatch") + + return OpenShellContract( + version=version, + release_tag=release_tag, + installer_sha256=checksum, + adapter_version=adapter_version, + ) + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--field", choices=("version", "release-tag", "installer-sha256", "adapter-version")) + parser.add_argument("--json", action="store_true") + return parser + + +def main(argv: list[str] | None = None) -> int: + args = _parser().parse_args(argv) + contract = load_contract() + if args.json: + print(json.dumps(asdict(contract), sort_keys=True)) + elif args.field: + print(getattr(contract, args.field.replace("-", "_")) or "") + else: + print(contract.version) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/start_e2e.sh b/scripts/start_e2e.sh index a08074b60..f2d5812b3 100755 --- a/scripts/start_e2e.sh +++ b/scripts/start_e2e.sh @@ -19,10 +19,16 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(dirname "$SCRIPT_DIR")" UI_DIR="$PROJECT_ROOT/frontends/ui" +VENV_DIR="$PROJECT_ROOT/.venv" +PYTHON_BIN="$VENV_DIR/bin/python" +NAT_BIN="$VENV_DIR/bin/nat" # Default config file CONFIG_FILE="configs/config_web_default_llamaindex.yml" PORT=8000 +START_OPENSHELL_GATEWAY=false +BACKEND_PID="" +FRONTEND_PID="" # Parse command line arguments while [[ $# -gt 0 ]]; do @@ -35,16 +41,22 @@ while [[ $# -gt 0 ]]; do PORT="$2" shift 2 ;; + --start-openshell-gateway) + START_OPENSHELL_GATEWAY=true + shift + ;; --help|-h) echo "Usage: $0 [OPTIONS]" echo "" echo "Options:" echo " --config_file Path to config file (default: configs/config_web_default_llamaindex.yml)" echo " --port PORT Backend server port (default: 8000)" + echo " --start-openshell-gateway Start/verify an authenticated gateway and run its strict capability probe" echo " --help, -h Show this help message" echo "" - echo "Example:" + echo "Examples:" echo " $0 --config_file configs/config_web_default_llamaindex.yml --port 8000" + echo " $0 --start-openshell-gateway --config_file configs/config_openshell.yml --port 8000" exit 0 ;; *) @@ -66,11 +78,11 @@ fi cleanup() { echo "" echo "Shutting down services..." - if [ ! -z "$BACKEND_PID" ]; then - kill $BACKEND_PID 2>/dev/null || true + if [[ -n "${BACKEND_PID:-}" ]]; then + kill "$BACKEND_PID" 2>/dev/null || true fi - if [ ! -z "$FRONTEND_PID" ]; then - kill $FRONTEND_PID 2>/dev/null || true + if [[ -n "${FRONTEND_PID:-}" ]]; then + kill "$FRONTEND_PID" 2>/dev/null || true fi exit 0 } @@ -107,12 +119,32 @@ check_env() { check_dependencies() { echo "Checking Python dependencies..." - if ! python -c "import nat" 2>/dev/null; then + if [ ! -x "$PYTHON_BIN" ]; then + echo "AI-Q virtual environment not found at $VENV_DIR" + echo "Run ./scripts/setup.sh or ./scripts/openshell/setup_openshell.sh first." + exit 1 + fi + + if ! "$PYTHON_BIN" -c "import nat" 2>/dev/null; then echo "NAT not installed. Installing dependencies..." - pip install -e . + "$PYTHON_BIN" -m pip install -e . + fi + if [ ! -x "$NAT_BIN" ]; then + echo "NAT CLI not found at $NAT_BIN" + echo "Run ./scripts/setup.sh or ./scripts/openshell/setup_openshell.sh first." + exit 1 fi - echo "Python dependencies installed" + echo "Python dependencies installed ($PYTHON_BIN)" +} + +check_openshell_component_versions() { + if ! grep -Eq '^[[:space:]]*provider:[[:space:]]*openshell([[:space:]]|$)' "$PROJECT_ROOT/$CONFIG_FILE"; then + return + fi + echo "Checking the certified OpenShell component stack..." + "$PYTHON_BIN" "$PROJECT_ROOT/scripts/openshell/check_versions.py" \ + --gateway-name "${AIQ_OPENSHELL_GATEWAY_NAME:-openshell}" } check_ui_dependencies() { @@ -154,11 +186,19 @@ start_backend() { echo "Config: $CONFIG_FILE" echo "" - nat serve --config_file "$CONFIG_FILE" --host 0.0.0.0 --port "$PORT" & + "$NAT_BIN" serve --config_file "$CONFIG_FILE" --host 0.0.0.0 --port "$PORT" & BACKEND_PID=$! echo "Backend PID: $BACKEND_PID" } +start_openshell_gateway() { + if [[ "$START_OPENSHELL_GATEWAY" != "true" ]]; then + return + fi + echo "Starting/verifying authenticated OpenShell gateway..." + "$PROJECT_ROOT/scripts/openshell/start_openshell_gateway.sh" +} + wait_for_backend() { echo "Waiting for backend to be ready..." local max_attempts=150 @@ -210,6 +250,12 @@ main() { check_dependencies echo "" + start_openshell_gateway + echo "" + + check_openshell_component_versions + echo "" + if check_ui_dependencies; then HAS_UI=true else diff --git a/src/aiq_agent/agents/deep_researcher/agent.py b/src/aiq_agent/agents/deep_researcher/agent.py index 811995065..30d5294eb 100644 --- a/src/aiq_agent/agents/deep_researcher/agent.py +++ b/src/aiq_agent/agents/deep_researcher/agent.py @@ -154,6 +154,10 @@ def __init__( self.orchestrator_middleware = self.middleware_set.orchestrator self.middleware = self.researcher_middleware + def finalize(self, *, interrupted: bool) -> bool: + """Release this request's sandbox runtime exactly once.""" + return self.deepagents_runtime.finalize(interrupted=interrupted) + def _load_prompts(self) -> dict[str, str]: """Load all prompts for subagents.""" prompts = {} diff --git a/src/aiq_agent/agents/deep_researcher/custom_middleware.py b/src/aiq_agent/agents/deep_researcher/custom_middleware.py index 944ef39f5..4409e011a 100644 --- a/src/aiq_agent/agents/deep_researcher/custom_middleware.py +++ b/src/aiq_agent/agents/deep_researcher/custom_middleware.py @@ -18,11 +18,15 @@ import asyncio import json import logging +import re from pathlib import Path +from pathlib import PurePosixPath from langchain.agents.middleware import AgentMiddleware +from langchain.agents.middleware import hook_config from langchain.agents.middleware.types import ModelResponse from langchain_core.messages import AIMessage +from langchain_core.messages import HumanMessage from langchain_core.messages import SystemMessage from langchain_core.messages import ToolMessage @@ -43,6 +47,9 @@ # route-local key. The guard reads raw state, so it must accept both forms or it # blocks the orchestrator forever on sandboxed runs. _SOURCE_ROUTING_STATE_KEYS = (_SOURCE_ROUTING_PATH, "/source_routing.json") +_UNRESOLVED_SANDBOX_PATH_PATTERN = re.compile( + r"<\s*sandbox_(?:artifact_dir|workdir)\s*>|\{\{\s*sandbox_(?:artifact_dir|workdir)\s*\}\}" +) class SourceRoutingGuardMiddleware(AgentMiddleware): @@ -157,6 +164,130 @@ async def awrap_tool_call(self, request, handler): return await handler(request.override(tool_call=modified)) +class FilesystemToolCallGuardMiddleware(AgentMiddleware): + """Normalize safe filesystem aliases and reject unresolved sandbox path templates.""" + + async def awrap_tool_call(self, request, handler): + """Repair ``read_file(path=...)`` and fail before executing placeholder paths.""" + tool_call = request.tool_call + if not isinstance(tool_call, dict): + return await handler(request) + args = tool_call.get("args") + if not isinstance(args, dict): + return await handler(request) + + if tool_call.get("name") == "read_file" and isinstance(args.get("path"), str): + normalized_args = {key: value for key, value in args.items() if key != "path"} + normalized_args.setdefault("file_path", args["path"]) + modified = {**tool_call, "args": normalized_args} + return await handler(request.override(tool_call=modified)) + + if tool_call.get("name") == "execute" and isinstance(args.get("command"), str): + command = args["command"] + unresolved = _UNRESOLVED_SANDBOX_PATH_PATTERN.search(command) + if unresolved is not None: + return ToolMessage( + content=( + f"Command not executed: unresolved sandbox path placeholder {unresolved.group(0)}. " + "Use the exact sandbox_workdir or sandbox_artifact_dir path from your instructions." + ), + tool_call_id=tool_call.get("id", "filesystem-tool-call-guard"), + name="execute", + status="error", + ) + + return await handler(request) + + +class RequiredOutputFileMiddleware(AgentMiddleware): + """Verify a model's file-backed completion marker before ending its run. + + A model can claim that it wrote a file without making the filesystem tool call. + Keep recovery local to that agent: request one corrective model turn, then fail + with a stable reason code instead of restarting the surrounding workflow. + """ + + def __init__( + self, + *, + paths: tuple[str, ...] = ("/shared/output.md", "/output.md"), + completion_marker: str = "Wrote /shared/output.md", + max_retries: int = 1, + reason_code: str = "writer_output_missing", + ) -> None: + """Configure the accepted state paths and bounded corrective turns.""" + if not paths: + raise ValueError("paths must not be empty") + if max_retries < 0: + raise ValueError("max_retries must be non-negative") + self.paths = paths + self.completion_marker = completion_marker + self.max_retries = max_retries + self.reason_code = reason_code + self._retry_message = ( + "The required final output file is missing or empty. Do not repeat research or regenerate artifacts. " + f"Call write_file with file_path={paths[0]} and the complete final Markdown, confirm the tool " + f"succeeds, and only then return `{completion_marker}`." + ) + + @staticmethod + def _files_from_state(state: object) -> object: + return state.get("files", {}) if isinstance(state, dict) else getattr(state, "files", {}) + + @staticmethod + def _entry_has_content(entry: object) -> bool: + if isinstance(entry, dict): + entry = entry.get("content") + if isinstance(entry, bytes): + return bool(entry.strip()) + if isinstance(entry, str): + return bool(entry.strip()) + if isinstance(entry, list): + return any(isinstance(line, str) and line.strip() for line in entry) + return False + + def _required_output_exists(self, state: object) -> bool: + files = self._files_from_state(state) + return isinstance(files, dict) and any(self._entry_has_content(files.get(path)) for path in self.paths) + + def _retry_count(self, messages: list[object]) -> int: + return sum(isinstance(message, HumanMessage) and message.content == self._retry_message for message in messages) + + def _check_after_model(self, state: object) -> dict[str, object] | None: + messages = state.get("messages", []) if isinstance(state, dict) else getattr(state, "messages", []) + if not isinstance(messages, list) or not messages: + return None + last_message = messages[-1] + if not isinstance(last_message, AIMessage) or last_message.tool_calls: + return None + if last_message.text.strip() != self.completion_marker: + return None + if self._required_output_exists(state): + return None + + retry_count = self._retry_count(messages) + if retry_count >= self.max_retries: + raise RuntimeError(self.reason_code) + + logger.warning( + "Agent reported file-backed completion before the required output existed; requesting corrective turn" + ) + return { + "messages": [HumanMessage(content=self._retry_message)], + "jump_to": "model", + } + + @hook_config(can_jump_to=["model"]) + def after_model(self, state, runtime): + """Verify synchronous writer completion and request one local repair when needed.""" + return self._check_after_model(state) + + @hook_config(can_jump_to=["model"]) + async def aafter_model(self, state, runtime): + """Verify asynchronous writer completion and request one local repair when needed.""" + return self._check_after_model(state) + + # Common hallucinated tool name mappings _TOOL_NAME_ALIASES: dict[str, str] = { "open_file": "read_file", @@ -563,11 +694,35 @@ async def awrap_tool_call(self, request, handler): result_status = result.get("status") if isinstance(result, dict) else getattr(result, "status", None) if tool_name == "execute" and result_status != "error": try: - await asyncio.to_thread(self.artifact_manager.harvest_after_execute) + captured = await asyncio.to_thread(self.artifact_manager.harvest_after_execute) except Exception as exc: # noqa: BLE001 - artifact capture must not fail the agent logger.warning("Artifact checkpoint harvest failed (%s)", type(exc).__name__) + else: + result = self._append_checkpoint_result(result, captured) return result + @staticmethod + def _append_checkpoint_result(result: object, captured: object) -> object: + """Tell the model the exact safe filenames captured from a valid manifest.""" + if not isinstance(result, ToolMessage) or not isinstance(result.content, str): + return result + if not isinstance(captured, (list, tuple)) or not captured: + return result + + lines = ["Artifact checkpoint captured these exact filenames:"] + for artifact in captured[:10]: + filename = PurePosixPath(str(getattr(artifact, "filename", ""))).name + if not filename: + continue + if bool(getattr(artifact, "inline", False)): + lines.append(f"- {filename} (inline): embed as ![caption](artifact://{filename})") + else: + lines.append(f"- {filename} (downloadable; not marked inline)") + if len(lines) == 1: + return result + content = result.content.rstrip() + "\n\n" + "\n".join(lines) + return result.model_copy(update={"content": content}) + class PlanPersistenceMiddleware(AgentMiddleware): """Persists the planner's structured ResearchPlan to the shared filesystem. diff --git a/src/aiq_agent/agents/deep_researcher/deepagents_runtime.py b/src/aiq_agent/agents/deep_researcher/deepagents_runtime.py index 793946414..1c62db9b6 100644 --- a/src/aiq_agent/agents/deep_researcher/deepagents_runtime.py +++ b/src/aiq_agent/agents/deep_researcher/deepagents_runtime.py @@ -33,10 +33,12 @@ from pydantic import ConfigDict from pydantic import Field from pydantic import field_validator +from pydantic import model_validator from nat.data_models.function import FunctionBaseConfig from .sandbox.config import ArtifactCaptureConfig +from .sandbox.logging_utils import log_sandbox_failure logger = logging.getLogger(__name__) @@ -86,19 +88,52 @@ class DeepResearchSandboxConfig(FunctionBaseConfig, name="deep_research_sandbox" default=(), description="Python packages to install into the Modal sandbox image.", ) - # OpenShell-specific (used when provider == "openshell"). The named sandbox is created - # out-of-band by scripts/setup_openshell.sh and attached to by name. - sandbox_name: str | None = Field(default=None, description="Existing named OpenShell sandbox to attach to.") + # OpenShell-specific (used when provider == "openshell"). The secure default creates + # a new policy-bound physical sandbox for each AI-Q job. + openshell_image: str = Field( + default="aiq-openshell-demo:latest", + description="Container image used for per-job OpenShell sandboxes.", + ) + existing_sandbox_name: str | None = Field( + default=None, + description="Debug-only existing OpenShell sandbox; requires allow_shared_sandbox=true.", + ) + sandbox_name: str | None = Field( + default=None, + description="Deprecated alias for existing_sandbox_name; requires allow_shared_sandbox=true.", + ) + allow_shared_sandbox: bool = Field( + default=False, + description="Allow debug attachment to a shared OpenShell sandbox; not job-isolated.", + ) gateway: str | None = Field( default=None, description="OpenShell gateway endpoint/name (null uses the locally selected gateway).", ) - policy: str | None = Field(default=None, description="OpenShell policy file path (requires a named sandbox).") + policy: str | None = Field(default=None, description="OpenShell policy YAML applied at per-job creation.") ready_timeout_seconds: float = Field( default=300.0, + gt=0, + allow_inf_nan=False, description="Seconds to wait for the OpenShell sandbox to become ready.", ) - delete_on_exit: bool = Field(default=False, description="Delete the OpenShell sandbox when the session closes.") + policy_load_timeout_seconds: float = Field( + default=30.0, + gt=0, + allow_inf_nan=False, + description="Seconds to wait for the authoritative OpenShell policy revision to become loaded.", + ) + delete_on_exit: bool = Field(default=True, description="Delete the OpenShell sandbox when the session closes.") + attest: bool = Field(default=True, description="Fail closed unless the OpenShell policy revision is loaded.") + expected_policy_version: int | None = Field( + default=None, + ge=1, + description="Optional exact OpenShell policy revision pin.", + ) + require_hard_landlock: bool = Field( + default=True, + description="Require landlock.compatibility=hard_requirement in the configured policy.", + ) shell: tuple[str, ...] = Field( default=("bash", "-c"), description="Shell argv prefix passed to the langchain-nvidia-openshell adapter.", @@ -107,19 +142,41 @@ class DeepResearchSandboxConfig(FunctionBaseConfig, name="deep_research_sandbox" workdir: str | None = Field(default=None, description="Working directory inside the sandbox") timeout: int = Field(default=1200, description="Maximum sandbox lifetime in seconds") idle_timeout: int = Field(default=1800, description="Sandbox idle timeout in seconds") - network: Literal["blocked", "enabled"] = Field( + network: Literal["blocked", "allowlist", "open", "enabled"] = Field( default="blocked", - description="Outbound network policy for the sandbox.", + description="Outbound network policy; 'enabled' is a deprecated alias for 'open'.", + ) + network_allow: tuple[str, ...] = Field( + default=(), + description="Hostnames allowed when network='allowlist'.", ) artifact_capture: ArtifactCaptureConfig = Field( default_factory=ArtifactCaptureConfig, description="Durable harvesting of generated artifacts (charts/CSVs). Disabled by default.", ) + @model_validator(mode="after") + def _validate_openshell_and_network(self) -> DeepResearchSandboxConfig: + if self.network == "allowlist" and not self.network_allow: + raise ValueError("network='allowlist' requires a non-empty network_allow list") + if self.provider == "openshell": + if ( + self.existing_sandbox_name is not None + and self.sandbox_name is not None + and self.existing_sandbox_name != self.sandbox_name + ): + raise ValueError("existing_sandbox_name and sandbox_name must match when both are set") + shared_name = self.existing_sandbox_name or self.sandbox_name + if shared_name and not self.allow_shared_sandbox: + raise ValueError("existing_sandbox_name/sandbox_name requires allow_shared_sandbox=true") + if not shared_name and not self.attest: + raise ValueError("Per-job OpenShell creation requires attest=true") + return self + @property - def block_network(self) -> bool: - """Return the block_network flag for this public network setting.""" - return self.network == "blocked" + def network_mode(self) -> Literal["blocked", "allowlist", "open"]: + """Return the normalized provider-neutral network mode.""" + return "open" if self.network == "enabled" else self.network class DeepAgentsRuntime: @@ -148,6 +205,10 @@ def __init__( self._job_id = str(job_id) if job_id is not None else str(uuid4()) self._backend: Any | None = None self._sandbox_provider: Any | None = None + self._event_emit = artifact_emit + self._finalize_lock = threading.Lock() + self._finalized = False + self._finalize_result: bool | None = None self.artifact_manager: Any | None = None self._artifact_finalize_lock = threading.Lock() self._artifact_finalize_attempted = False @@ -163,6 +224,8 @@ def __init__( # runtime can expose its job-scoped workdir/artifact_dir and own its lifecycle. if sandbox is not None: self._sandbox_provider = _create_sandbox_backend(sandbox, self._job_id) + if hasattr(self._sandbox_provider, "set_event_emitter"): + self._sandbox_provider.set_event_emitter(artifact_emit) self.artifact_manager = _maybe_build_artifact_manager( provider=self._sandbox_provider, job_id=self._job_id, @@ -231,7 +294,14 @@ def final_harvest(self) -> None: try: manager.final_harvest() except Exception as exc: # noqa: BLE001 - harvest is best-effort on the terminal path - logger.warning("Final artifact harvest failed for job %s (%s)", self._job_id, type(exc).__name__) + log_sandbox_failure( + logger, + operation="final_artifact_harvest", + reason_code="artifact_harvest_failed", + exc=exc, + provider=getattr(self._sandbox_provider, "provider_name", None), + sandbox=self._job_id, + ) def close(self) -> None: """Release the sandbox provider on a normal terminal job path (idempotent).""" @@ -272,6 +342,62 @@ def finalize_artifacts(self, *, interrupted: bool) -> bool: self.final_harvest() return True + def finalize(self, *, interrupted: bool) -> bool: + """Release the sandbox once and emit a truthful, sanitized cleanup outcome.""" + with self._finalize_lock: + if self._finalized: + assert self._finalize_result is not None + return self._finalize_result + + # No provider means there is no sandbox lifecycle to report; skip the + # cleanup events so non-sandbox jobs never emit an empty sandbox.cleanup. + if self._sandbox_provider is None: + self._finalize_result = True + self._finalized = True + return True + + self._emit_cleanup("started", interrupted=interrupted) + try: + if interrupted: + self.terminate() + else: + self.close() + except Exception as exc: # noqa: BLE001 - terminal cleanup cannot replace the job result + logger.warning("Sandbox cleanup failed for job %s (%s)", self._job_id, type(exc).__name__) + succeeded = False + else: + succeeded = bool(getattr(self._sandbox_provider, "cleanup_succeeded", True)) + + self._emit_cleanup("succeeded" if succeeded else "failed", interrupted=interrupted) + self._finalize_result = succeeded + self._finalized = True + return succeeded + + def _emit_cleanup(self, status: str, *, interrupted: bool) -> None: + """Emit cleanup metadata without policy contents, environment, or error text.""" + if self._event_emit is None: + return + try: + self._event_emit( + { + "type": "sandbox.cleanup", + "data": { + "status": status, + "interrupted": interrupted, + "provider": getattr(self._sandbox_provider, "provider_name", None), + "sandbox": getattr(self._sandbox_provider, "physical_sandbox_name", None) + or getattr(self._sandbox_provider, "sandbox_name", None), + "reason_codes": ( + list(getattr(self._sandbox_provider, "cleanup_failure_reason_codes", ())) + if status == "failed" + else [] + ), + }, + } + ) + except Exception as exc: # noqa: BLE001 - observability must not break terminal cleanup + logger.warning("Sandbox cleanup event emission failed for job %s (%s)", self._job_id, type(exc).__name__) + def prepare_state_files(self, files: dict[str, Any]) -> dict[str, Any]: """Normalize seeded virtual filesystem files for the configured backend.""" return _normalize_state_files(files, strip_shared_route=self._sandbox is not None) @@ -447,10 +573,17 @@ def _create_sandbox_backend(config: DeepResearchSandboxConfig, job_id: str) -> A providers = { "openshell": { "gateway": config.gateway, + "existing_sandbox_name": config.existing_sandbox_name, "sandbox_name": config.sandbox_name, + "allow_shared_sandbox": config.allow_shared_sandbox, "policy": config.policy, + "image": config.openshell_image, "ready_timeout_seconds": config.ready_timeout_seconds, + "policy_load_timeout_seconds": config.policy_load_timeout_seconds, "delete_on_exit": config.delete_on_exit, + "attest": config.attest, + "expected_policy_version": config.expected_policy_version, + "require_hard_landlock": config.require_hard_landlock, "shell": list(config.shell), } } @@ -463,7 +596,7 @@ def _create_sandbox_backend(config: DeepResearchSandboxConfig, job_id: str) -> A "workdir": workdir, "timeout": config.timeout, "idle_timeout": config.idle_timeout, - "network": {"mode": "blocked" if config.block_network else "open"}, + "network": {"mode": config.network_mode, "allow": config.network_allow}, "artifact_capture": config.artifact_capture.model_dump(), "providers": providers, } diff --git a/src/aiq_agent/agents/deep_researcher/factory.py b/src/aiq_agent/agents/deep_researcher/factory.py index 1556f7478..b5246b1ce 100644 --- a/src/aiq_agent/agents/deep_researcher/factory.py +++ b/src/aiq_agent/agents/deep_researcher/factory.py @@ -44,7 +44,9 @@ from .custom_middleware import ArtifactHarvestMiddleware from .custom_middleware import EmptyContentFixMiddleware from .custom_middleware import ExecuteTimeoutClampMiddleware +from .custom_middleware import FilesystemToolCallGuardMiddleware from .custom_middleware import PlanPersistenceMiddleware +from .custom_middleware import RequiredOutputFileMiddleware from .custom_middleware import SourceRegistryMiddleware from .custom_middleware import SourceRoutingGuardMiddleware from .custom_middleware import TodoSuppressionMiddleware @@ -469,6 +471,7 @@ def build_deep_research_subagents(context: DeepResearchGraphContext) -> list[dic middleware=[ *context.middleware_set.writer, TodoSuppressionMiddleware(), + RequiredOutputFileMiddleware(), ], prompt_values={"parent_report_context_available": context.parent_report_context_available}, skills=context.skill_sources(WRITER_AGENT), @@ -497,7 +500,10 @@ def build_deep_research_graph( # Agent-supplied execute timeouts are unreliable (LLMs pass milliseconds or arbitrarily # large values); clamp them to the configured sandbox lifetime so a single execute never # exceeds the provider's hard cap and silently fails every code run. - cross_cutting_middleware = runtime_visibility_middleware(runtime) + cross_cutting_middleware = [ + FilesystemToolCallGuardMiddleware(), + *runtime_visibility_middleware(runtime), + ] execute_ceiling = runtime.execute_timeout_seconds if execute_ceiling: cross_cutting_middleware = [ diff --git a/src/aiq_agent/agents/deep_researcher/prompts/writer.j2 b/src/aiq_agent/agents/deep_researcher/prompts/writer.j2 index 223eed25f..a7ed8ad2d 100644 --- a/src/aiq_agent/agents/deep_researcher/prompts/writer.j2 +++ b/src/aiq_agent/agents/deep_researcher/prompts/writer.j2 @@ -77,11 +77,11 @@ When synthesizing: {% if execution_enabled %}## Figures -When `answer_strategy.required_components` calls for a chart, GENERATE it yourself with the chart skill and embed it once, where it is discussed, with `![](artifact://)`. Read the relevant chart `SKILL.md` first, then write the plotting script under `{{ sandbox_workdir }}` and `execute` it so the script writes the image plus its `manifest.json` under `{{ sandbox_artifact_dir }}` (the only directory the runtime harvests). Use those per-job paths verbatim. +When `answer_strategy.required_components` calls for a chart, GENERATE it yourself with the chart skill and embed it once, where it is discussed, with `![](artifact://)`. Read the relevant chart `SKILL.md` first, then write the plotting script to `{{ sandbox_workdir }}/make_chart.py`. The script must accept its artifact directory as its first argument; run it exactly as `python3 {{ sandbox_workdir }}/make_chart.py {{ sandbox_artifact_dir }}` so it writes the image plus `manifest.json` under the only directory the runtime harvests. Never put a literal `` or `` placeholder into code or a command. After execution, use the exact filename confirmed by the artifact checkpoint in the report reference. Each `execute` runs in a fresh shell, so `cd` does NOT persist between calls and a standalone `cd` accomplishes nothing — never navigate with `cd`; put absolute paths in every command (or chain in one line as `cd && `). -NEVER paste the plotting code, a sandbox file path (e.g. `{{ sandbox_artifact_dir }}/chart.png`), or base64 image data into the report as prose - the `artifact://` token is the ONLY way a figure renders. A "plotting code" or "chart plan" section is not a substitute for the actual figure. Precede each embed with a one-sentence description of what it shows. +NEVER paste the plotting code, a sandbox file path (e.g. `{{ sandbox_artifact_dir }}/chart.png`), or base64 image data into the report as prose - the `artifact://` token is the ONLY way a figure renders. Do not call `read_file` on a generated PNG just to verify it; that injects the image's base64 bytes into model context. Verify the manifest and use the artifact-checkpoint response instead. A "plotting code" or "chart plan" section is not a substitute for the actual figure. Precede each embed with a one-sentence description of what it shows. A figure is earned: generate or embed one only when its data is source-anchored and reasonably complete. If a quantitative series is mostly undisclosed or mixes metric definitions, present the table (which shows the gaps) and state the limitation in one sentence instead of a misleading chart. diff --git a/src/aiq_agent/agents/deep_researcher/register.py b/src/aiq_agent/agents/deep_researcher/register.py index 4134b797f..14e382464 100644 --- a/src/aiq_agent/agents/deep_researcher/register.py +++ b/src/aiq_agent/agents/deep_researcher/register.py @@ -15,6 +15,7 @@ """NAT register function for deep research agent.""" +import asyncio import logging from typing import TypeVar @@ -236,10 +237,12 @@ async def deep_research_agent(config: DeepResearchAgentConfig, builder: Builder) async def _run(state: DeepResearchAgentState) -> DeepResearchAgentState: """Run deep research with a list of messages or payload.""" + active_agent = agent + owns_active_agent = False + interrupted = False try: data_sources = state.data_sources selected_tools = filter_tools_by_sources(tools, data_sources) - active_agent = agent if sandbox_config is not None or (data_sources is not None and selected_tools != tools): # Scope the Modal sandbox to the async job_id when one is in # NAT context (set by aiq_api/jobs/runner.py). Falls back to a @@ -266,6 +269,7 @@ async def _run(state: DeepResearchAgentState) -> DeepResearchAgentState: max_concurrent_source_tool_calls=config.max_concurrent_source_tool_calls, max_source_tool_batch_size=config.max_source_tool_batch_size, ) + owns_active_agent = True if all_mapped_tools_filtered_out(tools, selected_tools, data_sources): logger.warning("Deep research received data_sources with no matching tools") @@ -291,9 +295,15 @@ async def _run(state: DeepResearchAgentState) -> DeepResearchAgentState: result = await active_agent.run(state) return result + except asyncio.CancelledError: + interrupted = True + raise except Exception: logger.exception("Error in deep research execution") raise + finally: + if owns_active_agent: + await asyncio.to_thread(active_agent.finalize, interrupted=interrupted) yield FunctionInfo.from_fn(_run, description="Deep research agent for comprehensive multi-phase research.") diff --git a/src/aiq_agent/agents/deep_researcher/sandbox/README.md b/src/aiq_agent/agents/deep_researcher/sandbox/README.md index a83e01a3e..579d865a5 100644 --- a/src/aiq_agent/agents/deep_researcher/sandbox/README.md +++ b/src/aiq_agent/agents/deep_researcher/sandbox/README.md @@ -37,15 +37,25 @@ creates these on session start (`_prepare_workspace`, an idempotent `mkdir -p`) runtime injects them into prompts/skills as `sandbox_workdir`/`sandbox_artifact_dir`. This prevents accidental filename collisions and keeps harvesting scoped to the current job. -Modal creates a fresh sandbox for each job. The experimental OpenShell configuration instead -attaches jobs to one pre-created named sandbox because the SDK cannot apply the configured -policy when creating an anonymous sandbox. Per-job directories inside that sandbox are not -an access-control boundary: executed code can access sibling job directories allowed by the -shared policy. Use OpenShell only for local, single-operator testing, and do not run mutually -untrusted jobs concurrently. Physical per-job OpenShell isolation and attach-time policy -verification are follow-up work. The default policy also sets `landlock.compatibility: -best_effort`, so on hosts without Landlock (e.g. Docker Desktop on macOS) filesystem -confinement is silently dropped; production must use `hard_requirement` to fail closed. +Modal and OpenShell both create a fresh physical sandbox for each job. OpenShell parses the +configured policy with the installed SDK schema, applies it in the creation `SandboxSpec`, +waits for `READY`, and verifies the authoritative effective policy source, protobuf content, +OpenShell hash, and active revision before exposing the backend. A positive status version +alone is never treated as attestation. An optional `expected_policy_version` provides an exact +revision pin. Any creation or attestation failure closes the owning SDK context so the partially +created sandbox is deleted. + +Zero generic `current_policy_version` or `active_version` values are treated as unreported only +when the LOADED revision plus effective config agree on the same positive version, source, +protobuf content, and deterministic hash. Every positive reported version remains subject to +exact agreement. An effective policy that remains Pending beyond `policy_load_timeout_seconds` +fails with `policy_status_inconsistent`; AI-Q never treats it as successful attestation. + +OpenShell shared attachment remains available only as an explicit debug escape hatch: +`existing_sandbox_name` plus `allow_shared_sandbox: true`. It is not a job isolation boundary +and must not be used for mutually untrusted jobs. Production policy validation also requires +`landlock.compatibility: hard_requirement`; a local demo may explicitly set both the policy +and `require_hard_landlock: false` to accept `best_effort`. The agent only ever sees a `read_file`/`write_file`/`edit_file`/`execute` tool surface plus `/shared/` for durable text. Binary artifacts are harvested host-side via @@ -60,7 +70,7 @@ plus `/shared/` for durable text. Binary artifacts are harvested host-side via | `config.py` | `SandboxConfig`: common fields + nested `providers.` + `artifact_capture` + `lifecycle_scope`; legacy flat-config shim; provider validated against the registry. | | `capabilities.py` | `SandboxCapabilities` + `verify_capabilities` (fail-closed: refuse to run if a required guarantee like `block_network` is unsupported). | | `providers/modal.py` | Modal provider (cloud). Create-fresh semantics (no silent attach-by-name). | -| `providers/openshell.py` | OpenShell provider (enterprise/on-prem). Lazy, ad-hoc deps; policy requires a named sandbox. | +| `providers/openshell.py` | OpenShell provider (enterprise/on-prem). Lazy, ad-hoc deps; per-job policy creation, readiness/revision attestation, and confined transfer. | | `artifacts/models.py` | `Artifact` record (id, mime, sha256, size, provenance, status). Metadata only. | | `artifacts/manifest.py` | `manifest.json` schema + parser. | | `artifacts/store.py` | `SqlArtifactStore` coordinates SQL metadata with the configured byte provider. | @@ -125,8 +135,8 @@ sandbox: provider: openshell # registry key workdir: /sandbox # injected into prompts + skills network: # normalized, provider-neutral egress policy - mode: blocked # blocked | allowlist | open (legacy `block_network: true` => blocked) - # allow: [pypi.org] # required for mode: allowlist; needs supports_network_allowlist + mode: allowlist # blocked | allowlist | open (legacy `block_network: true` => blocked) + allow: [api.github.com, github.com] # policy grants must be a subset of this list timeout: 1200 idle_timeout: 1800 resources: # optional CPU/memory caps; omit for no limit @@ -143,8 +153,14 @@ sandbox: python_packages: [matplotlib, numpy, pandas, pillow, tabulate] openshell: gateway: null # null = locally selected gateway - sandbox_name: aiq-openshell-demo + image: aiq-openshell-demo:latest policy: configs/openshell/generated/aiq-openshell-policy.yaml + delete_on_exit: true + attest: true + policy_load_timeout_seconds: 30 + cleanup_timeout_seconds: 30 + # expected_policy_version: 1 + require_hard_landlock: true ``` The legacy flat shape (top-level `app_name`/`image`/`python_packages`) still loads and @@ -207,42 +223,31 @@ shared helper, `MarkdownRenderer/artifact-url.ts`, builds the content path): Requires `modal` + `langchain-modal` (in `pyproject`) and `modal setup`. See `docs/source/examples/skills-sandbox/index.md`. -### OpenShell (experimental, local single-operator) +### OpenShell (experimental) -> OpenShell jobs currently attach to one pre-created named sandbox. Its policy is applied by -> the setup command and is not verified when AI-Q attaches. Job directories prevent accidental -> collisions but do not isolate mutually untrusted jobs. Do not use this path as a multi-tenant -> security boundary. +Each job creates a new policy-bound OpenShell sandbox and deletes it on terminal cleanup. +AI-Q refuses startup if the YAML does not match the installed SDK schema, the policy grants a +host or hostless/CIDR override outside the declared public network contract, production Landlock +mode is not fail-closed, or the gateway cannot prove the submitted policy is effective. The +creation spec deliberately has no copied host environment or credential providers. +Owned sandboxes carry `aiq=deep-research` and a normalized `aiq-job-id` in both OpenShell +gateway metadata and runtime template metadata so operators can use label selectors reliably. Two ad-hoc deps (never in `pyproject`): the `openshell` SDK and the official `langchain-nvidia-openshell` adapter (`OpenShellSandbox`), the OpenShell partner package in [`langchain-ai/langchain-nvidia`](https://github.com/langchain-ai/langchain-nvidia/pull/303). -The adapter is published on PyPI as `langchain-nvidia-openshell` — `./scripts/setup_openshell.sh` -installs it for you (override the source with `LANGCHAIN_NVIDIA_REPO` to use a git spec or local -checkout). To install it into your `.venv` manually: +They remain lazy so selecting another provider does not install or import OpenShell. +The provider config supports per-job policy creation and an explicit shared-debug attachment; +policy-configured shared attachment is strictly attested, while policy-free attachment emits +`assurance=reduced`. -```bash -uv pip install 'langchain-nvidia-openshell==0.1.0' -``` - -One-command setup: - -```bash -./scripts/setup_openshell.sh --policy offline -./scripts/start_e2e.sh --config_file configs/config_openshell.yml -``` - -The setup script prints the environment variables needed by any later shell that starts -AI-Q. If you start the backend in a different terminal/session, export the printed values -before running `start_e2e.sh` (or put them in your local env file): - -```bash -export AIQ_OPENSHELL_SANDBOX_NAME="aiq-openshell-demo" -export AIQ_OPENSHELL_POLICY_FILE="$PWD/configs/openshell/generated/aiq-openshell-policy.yaml" -``` +Use the canonical [OpenShell deployment guide](../../../../../docs/source/deployment/openshell.md) +for installation, platform support, authenticated gateway ownership, policy/config pairing, +startup, live acceptance, and troubleshooting. Operator commands are intentionally not +duplicated in this implementation reference. Inference is routed host-side (e.g. NVIDIA Build or an internal inference hub set in the -config); the network-blocked sandbox never sees the key. +config); sandbox policy egress never requires or receives the inference key. **File-transfer gotcha:** the provider overrides file transfer with an env-free shim that passes the path via `argv`. OpenShell 0.0.57-0.0.67 strip @@ -302,9 +307,10 @@ Custom endpoints use path-style bucket addressing for MinIO compatibility. - In-container OpenShell log verbosity (opt-in): `agent.execute()` calls and their output are already logged on the AI-Q side (the `execute` tool-call events). To also see what runs inside the OpenShell container, rebuild the sandbox image with a higher `RUST_LOG`: - `./scripts/setup_openshell.sh --sandbox-log-level debug` (or `--build-arg + `./scripts/openshell/setup_openshell.sh --sandbox-log-level debug` (or `--build-arg OPENSHELL_SANDBOX_LOG_LEVEL=debug`). Default `warn` keeps OpenShell's stock behavior. - Read the container logs with `openshell logs `, the OpenShell TUI, or inside + Read the generated sandbox name from AI-Q's attestation/cleanup events, then use + `openshell logs `, the OpenShell TUI, or inside the sandbox at `/var/log/openshell.*.log` (e.g. `grep "OCSF PROC:"` for process activity). ## Testing @@ -313,8 +319,10 @@ Custom endpoints use path-style bucket addressing for MinIO compatibility. pytest tests/aiq_agent/agents/deep_researcher/sandbox/ -q ``` -All provider/artifact tests run without a live Modal/OpenShell gateway (OpenShell -compliance auto-skips when the SDK is absent). +Core provider/artifact tests use fake SDK objects and run without a live Modal/OpenShell +gateway. The exact checked-policy/protobuf schema assertion is optional when the SDK is absent. +The opt-in gateway acceptance suite and its environment contract are documented in the +[OpenShell deployment guide](../../../../../docs/source/deployment/openshell.md#acceptance-tests). ## Troubleshooting @@ -322,12 +330,8 @@ compliance auto-skips when the SDK is absent). the workspace plugin packages aren't installed. Install them (don't re-run `setup.sh`, which recreates `.venv`): `uv pip install -e ./frontends/aiq_api -e ./sources/tavily_web_search -e "./sources/knowledge_layer[llamaindex,foundational_rag]" -e ./sources/exa_web_search -e ./sources/google_scholar_paper_search` -- **`langchain-nvidia-openshell was not found in the package registry`**: the adapter is - the OpenShell partner package in `langchain-ai/langchain-nvidia` (PR #303), not yet on - PyPI. The setup script installs it from a git spec by default; override with - `LANGCHAIN_NVIDIA_REPO=` or `--langchain-nvidia /path/to/checkout`. -- **`unbound variable` in `setup_openshell.sh` on macOS**: the system bash is 3.2; run under - bash 5 (`brew install bash` then `/opt/homebrew/bin/bash ./scripts/setup_openshell.sh ...`). +- **OpenShell installation, gateway, policy, readiness, or cleanup failures**: follow the + canonical [OpenShell troubleshooting contract](../../../../../docs/source/deployment/openshell.md#inspection-and-troubleshooting). - **`network.mode` rejected at startup**: the selected provider doesn't declare the matching capability (`supports_network_policy` for `blocked`, `supports_network_allowlist` for `allowlist`). Choose a capable provider or relax `network.mode` (e.g. to `open`). diff --git a/src/aiq_agent/agents/deep_researcher/sandbox/artifacts/manager.py b/src/aiq_agent/agents/deep_researcher/sandbox/artifacts/manager.py index 66dbd1291..ff0b59317 100644 --- a/src/aiq_agent/agents/deep_researcher/sandbox/artifacts/manager.py +++ b/src/aiq_agent/agents/deep_researcher/sandbox/artifacts/manager.py @@ -36,6 +36,7 @@ from typing import Any from ..config import ArtifactCaptureConfig +from ..logging_utils import log_sandbox_failure from .manifest import ManifestEntry from .manifest import parse_manifest from .models import Artifact @@ -218,8 +219,14 @@ def resolve_report_references(self, markdown: str, artifacts: list[Artifact] | N if artifacts is None: try: artifacts = self.store.list(self.job_id) - except Exception: # noqa: BLE001 - report resolution must not fail the job - logger.warning("Could not load artifacts to resolve report references", exc_info=True) + except Exception as exc: # noqa: BLE001 - report resolution must not fail the job + log_sandbox_failure( + logger, + operation="artifact_reference_load", + reason_code="artifact_store_read_failed", + exc=exc, + sandbox=self.job_id, + ) return markdown by_id = {a.artifact_id: a for a in artifacts} @@ -251,8 +258,14 @@ def ensure_inline_artifacts_embedded(self, markdown: str, artifacts: list[Artifa if artifacts is None: try: artifacts = self.store.list(self.job_id) - except Exception: # noqa: BLE001 - embedding must not fail the job - logger.warning("Could not load artifacts to embed inline figures", exc_info=True) + except Exception as exc: # noqa: BLE001 - embedding must not fail the job + log_sandbox_failure( + logger, + operation="artifact_inline_load", + reason_code="artifact_store_read_failed", + exc=exc, + sandbox=self.job_id, + ) return markdown orphans = [ @@ -290,8 +303,14 @@ def append_artifact_index(self, markdown: str, artifacts: list[Artifact] | None if artifacts is None: try: artifacts = self.store.list(self.job_id) - except Exception: # noqa: BLE001 - indexing must not fail the job - logger.warning("Could not load artifacts to index generated outputs", exc_info=True) + except Exception as exc: # noqa: BLE001 - indexing must not fail the job + log_sandbox_failure( + logger, + operation="artifact_index_load", + reason_code="artifact_store_read_failed", + exc=exc, + sandbox=self.job_id, + ) return markdown if not artifacts: @@ -362,8 +381,14 @@ def _scan_dir(self) -> list[ManifestEntry]: """Enumerate allowed files in the artifact dir (bounded, best-effort fallback).""" try: response = self.backend.execute(f"find {shlex.quote(self.artifact_dir)} -type f") - except Exception: # noqa: BLE001 - scan is best-effort - logger.warning("Artifact scan failed for job %s", self.job_id, exc_info=True) + except Exception as exc: # noqa: BLE001 - scan is best-effort + log_sandbox_failure( + logger, + operation="artifact_scan", + reason_code="artifact_scan_failed", + exc=exc, + sandbox=self.job_id, + ) return [] output = getattr(response, "output", "") or "" entries: list[ManifestEntry] = [] diff --git a/src/aiq_agent/agents/deep_researcher/sandbox/artifacts/store.py b/src/aiq_agent/agents/deep_researcher/sandbox/artifacts/store.py index 286763447..2c08b1139 100644 --- a/src/aiq_agent/agents/deep_researcher/sandbox/artifacts/store.py +++ b/src/aiq_agent/agents/deep_researcher/sandbox/artifacts/store.py @@ -37,6 +37,7 @@ from collections.abc import Iterator from typing import Any +from ..logging_utils import log_sandbox_failure from .blob_store import ArtifactBlobStore from .blob_store import SqlArtifactBlobStore from .models import Artifact @@ -219,12 +220,24 @@ def put(self, artifact: Artifact, data: bytes) -> Artifact: content = self._blob_store.put(prepared, data) stored = prepared.model_copy(update={"status": ArtifactStatus.AVAILABLE}) self._insert_metadata(stored, content=content) - except Exception: - logger.exception("Artifact storage failed for artifact=%s", prepared.artifact_id) + except Exception as exc: + log_sandbox_failure( + logger, + operation="artifact_store", + reason_code="artifact_storage_failed", + exc=exc, + sandbox=prepared.job_id, + ) try: self._blob_store.delete(prepared) - except Exception: - logger.warning("Failed to remove artifact bytes after storage failure: %s", prepared.storage_uri) + except Exception as cleanup_exc: + log_sandbox_failure( + logger, + operation="artifact_store_rollback", + reason_code="artifact_rollback_failed", + exc=cleanup_exc, + sandbox=prepared.job_id, + ) raise return stored @@ -368,8 +381,14 @@ def _delete_artifact(self, artifact: Artifact) -> int: try: self._blob_store.delete(artifact) - except Exception: - logger.exception("Artifact byte deletion failed; retaining metadata for %s", artifact.artifact_id) + except Exception as exc: + log_sandbox_failure( + logger, + operation="artifact_byte_delete", + reason_code="artifact_delete_failed", + exc=exc, + sandbox=artifact.job_id, + ) return 0 try: with self._engine.connect() as conn: @@ -379,9 +398,13 @@ def _delete_artifact(self, artifact: Artifact) -> int: ) conn.commit() return result.rowcount - except Exception: - logger.exception( - "Artifact metadata deletion failed; retaining metadata for retry: %s", artifact.artifact_id + except Exception as exc: + log_sandbox_failure( + logger, + operation="artifact_metadata_delete", + reason_code="artifact_metadata_delete_failed", + exc=exc, + sandbox=artifact.job_id, ) return 0 diff --git a/src/aiq_agent/agents/deep_researcher/sandbox/base.py b/src/aiq_agent/agents/deep_researcher/sandbox/base.py index 4331c2329..f02f95246 100644 --- a/src/aiq_agent/agents/deep_researcher/sandbox/base.py +++ b/src/aiq_agent/agents/deep_researcher/sandbox/base.py @@ -45,6 +45,7 @@ from .capabilities import SandboxCapabilities from .config import job_scoped_artifact_dir from .config import job_scoped_workdir +from .logging_utils import log_sandbox_failure if TYPE_CHECKING: from .config import SandboxConfig @@ -100,6 +101,10 @@ def __init__(self, config: SandboxConfig, job_id: str) -> None: # state lock, so the two can never deadlock. self._state_lock = threading.Lock() self._terminated = False + self._event_emit: Callable[[dict[str, object]], None] | None = None + self._cleanup_failed = False + self._cleanup_failure_reasons: set[str] = set() + self._cleanup_state_lock = threading.Lock() # ------------------------------------------------------------------ # # Required surface (the only things a provider must implement) @@ -151,6 +156,26 @@ def try_operation_lease(self): if acquired: self._lock.release() + def set_event_emitter(self, emit: Callable[[dict[str, object]], None] | None) -> None: + """Attach the job-scoped event sink used for sanitized lifecycle events.""" + self._event_emit = emit + + def _emit_event(self, event: dict[str, object]) -> None: + """Best-effort event delivery; observability must never break execution.""" + if self._event_emit is None: + return + try: + self._event_emit(event) + except Exception as exc: # noqa: BLE001 - event persistence is non-critical + log_sandbox_failure( + logger, + operation="event_emit", + reason_code="event_emit_failed", + exc=exc, + provider=self.provider_name, + sandbox=self.sandbox_name, + ) + def close(self) -> None: """Release the underlying sandbox session, if any (idempotent). @@ -194,16 +219,49 @@ def _prepare_workspace(self, session: BaseSandbox) -> None: command = f"mkdir -p {shlex.quote(self.workdir)} {shlex.quote(self.artifact_dir)}" try: session.execute(command, timeout=self._clamp_timeout(_WORKSPACE_PREP_TIMEOUT)) - except Exception: # noqa: BLE001 - workspace prep is best-effort; real failures resurface on first write - logger.warning("Sandbox %s workspace prep failed (%s)", self.sandbox_name, command, exc_info=True) + except Exception as exc: # noqa: BLE001 - workspace prep is best-effort; real failures resurface on first write + log_sandbox_failure( + logger, + operation="workspace_prepare", + reason_code="workspace_prepare_failed", + exc=exc, + provider=self.provider_name, + sandbox=self.sandbox_name, + ) def _safe_close(self, session: BaseSandbox | None) -> None: """Best-effort close of a session; never raises on the teardown path.""" if session is not None and hasattr(session, "close"): try: session.close() - except Exception: # noqa: BLE001 - cleanup must never raise on the terminal path - logger.warning("Sandbox %s cleanup failed", self.sandbox_name, exc_info=True) + except Exception as exc: # noqa: BLE001 - cleanup must never raise on the terminal path + self._record_cleanup_failure("session_close_failed") + log_sandbox_failure( + logger, + operation="session_close", + reason_code="session_close_failed", + exc=exc, + provider=self.provider_name, + sandbox=self.sandbox_name, + ) + + def _record_cleanup_failure(self, reason_code: str) -> None: + """Record a monotonic, thread-safe cleanup failure reason.""" + with self._cleanup_state_lock: + self._cleanup_failed = True + self._cleanup_failure_reasons.add(reason_code) + + @property + def cleanup_succeeded(self) -> bool: + """Whether every cleanup attempt, including retry teardown, completed without an error.""" + with self._cleanup_state_lock: + return not self._cleanup_failed + + @property + def cleanup_failure_reason_codes(self) -> tuple[str, ...]: + """Return stable reason codes suitable for terminal lifecycle events.""" + with self._cleanup_state_lock: + return tuple(sorted(self._cleanup_failure_reasons)) @property def id(self) -> str: diff --git a/src/aiq_agent/agents/deep_researcher/sandbox/config.py b/src/aiq_agent/agents/deep_researcher/sandbox/config.py index c6cca98cd..268b3c7da 100644 --- a/src/aiq_agent/agents/deep_researcher/sandbox/config.py +++ b/src/aiq_agent/agents/deep_researcher/sandbox/config.py @@ -24,6 +24,7 @@ from __future__ import annotations +import math from typing import Any from typing import Literal @@ -90,22 +91,99 @@ class OpenShellProviderConfig(BaseModel): """OpenShell-specific sandbox settings (enterprise/on-prem example provider).""" gateway: str | None = Field(default=None, description="OpenShell gateway/cluster endpoint or name") + existing_sandbox_name: str | None = Field( + default=None, + description="Debug-only existing OpenShell sandbox to attach to; requires allow_shared_sandbox=true.", + ) sandbox_name: str | None = Field( default=None, - description="Existing named OpenShell sandbox to attach to (required when a policy file is used).", + description="Deprecated alias for existing_sandbox_name; requires allow_shared_sandbox=true.", + ) + allow_shared_sandbox: bool = Field( + default=False, + description="Allow debug attachment to a shared sandbox; this does not provide per-job isolation.", ) policy: str | None = Field( default=None, - description="OpenShell policy file path. Requires a pre-created named sandbox (sandbox_name).", + description="OpenShell policy YAML applied when the per-job sandbox is created.", + ) + image: str = Field(default="aiq-openshell-demo:latest", description="OpenShell image identifier") + ready_timeout_seconds: float = Field( + default=300.0, + gt=0, + allow_inf_nan=False, + description="Seconds to wait for the sandbox to become ready", + ) + policy_load_timeout_seconds: float = Field( + default=30.0, + gt=0, + description="Seconds to wait for the authoritative policy revision to become loaded.", + ) + cleanup_timeout_seconds: float = Field( + default=30.0, + gt=0, + description="Seconds to wait for an in-flight OpenShell context teardown to finish.", ) - image: str = Field(default="base", description="OpenShell image identifier") - ready_timeout_seconds: float = Field(default=300.0, description="Seconds to wait for the sandbox to become ready") delete_on_exit: bool = Field(default=True, description="Delete the sandbox when its session context closes") + attest: bool = Field(default=True, description="Require READY state and a loaded policy revision before execution") + expected_policy_version: int | None = Field( + default=None, + ge=1, + description="Optional exact OpenShell policy revision pin.", + ) + require_hard_landlock: bool = Field( + default=True, + description="Reject policy files that do not require Landlock filesystem enforcement.", + ) shell: tuple[str, ...] = Field( default=("bash", "-c"), description="Shell argv prefix passed to the langchain-nvidia-openshell adapter.", ) + @field_validator("cleanup_timeout_seconds") + @classmethod + def _cleanup_timeout_must_be_finite(cls, value: float) -> float: + """Reject non-finite teardown deadlines that would defeat bounded finalization.""" + if not math.isfinite(value): + raise ValueError("cleanup_timeout_seconds must be finite") + return value + + @field_validator("policy_load_timeout_seconds") + @classmethod + def _policy_load_timeout_must_be_finite(cls, value: float) -> float: + """Reject non-finite attestation deadlines that would defeat fail-closed startup.""" + if not math.isfinite(value): + raise ValueError("policy_load_timeout_seconds must be finite") + return value + + @model_validator(mode="after") + def _validate_lifecycle_mode(self) -> OpenShellProviderConfig: + """Keep shared attachment explicit and per-job creation fail-closed.""" + if self.sandbox_name and self.existing_sandbox_name and self.sandbox_name != self.existing_sandbox_name: + raise ValueError("Set only one of sandbox_name or existing_sandbox_name.") + shared_name = self.existing_sandbox_name or self.sandbox_name + if shared_name and not self.allow_shared_sandbox: + raise ValueError( + "Attaching to an existing OpenShell sandbox is debug-only and requires allow_shared_sandbox=true." + ) + if shared_name and self.policy and not self.attest: + raise ValueError("A policy-configured shared OpenShell sandbox requires attest=true.") + if not shared_name: + if not self.delete_on_exit: + raise ValueError("Per-job OpenShell sandboxes require delete_on_exit=true.") + if not self.attest: + raise ValueError("Per-job OpenShell sandboxes require attest=true.") + if self.expected_policy_version is not None and not self.attest: + raise ValueError("expected_policy_version requires attest=true.") + if not self.shell or any(not part.strip() for part in self.shell): + raise ValueError("OpenShell shell must contain at least one non-empty argv element.") + return self + + @property + def shared_sandbox_name(self) -> str | None: + """Return the explicitly configured debug attachment target, if any.""" + return self.existing_sandbox_name or self.sandbox_name + class SandboxProvidersConfig(BaseModel): """Per-provider configuration blocks. Add a provider by adding an optional field here.""" diff --git a/src/aiq_agent/agents/deep_researcher/sandbox/logging_utils.py b/src/aiq_agent/agents/deep_researcher/sandbox/logging_utils.py new file mode 100644 index 000000000..2f83f7920 --- /dev/null +++ b/src/aiq_agent/agents/deep_researcher/sandbox/logging_utils.py @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Secret-safe logging helpers for sandbox and artifact failure boundaries.""" + +from __future__ import annotations + +import logging + + +def log_sandbox_failure( + log: logging.Logger, + *, + operation: str, + reason_code: str, + exc: BaseException, + provider: str | None = None, + sandbox: str | None = None, +) -> None: + """Log only allowlisted failure metadata, never exception text or traceback.""" + log.warning( + "Sandbox failure: operation=%s reason=%s provider=%s sandbox=%s exception=%s", + operation, + reason_code, + provider, + sandbox, + type(exc).__name__, + ) diff --git a/src/aiq_agent/agents/deep_researcher/sandbox/providers/openshell.py b/src/aiq_agent/agents/deep_researcher/sandbox/providers/openshell.py index 4d870402f..5452cd8c5 100644 --- a/src/aiq_agent/agents/deep_researcher/sandbox/providers/openshell.py +++ b/src/aiq_agent/agents/deep_researcher/sandbox/providers/openshell.py @@ -19,28 +19,33 @@ the OpenShell gateway. The deepagents ``BaseSandbox`` adapter is the official ``langchain-nvidia-openshell`` partner package (``OpenShellSandbox``), the same adapter AI-Q PR #274 integrates. Both the ``openshell`` SDK and the adapter are -intentionally NOT declared in ``pyproject``; they are optional, ad-hoc -dependencies imported lazily, so this provider is never force-installed. - -Until ``langchain-ai/langchain-nvidia`` PR #303 publishes the adapter to PyPI, -install it from a git spec (see ``scripts/setup_openshell.sh`` / -``LANGCHAIN_NVIDIA_REPO``). +intentionally NOT declared in ``pyproject``; they are optional dependencies +imported lazily, so this provider is never force-installed. The canonical +operator workflow lives in ``docs/source/deployment/openshell.md``. """ from __future__ import annotations import base64 +import inspect import logging import os import re +import time +from dataclasses import dataclass +from pathlib import Path +from threading import Event from typing import TYPE_CHECKING +from typing import Any from deepagents.backends.protocol import FileDownloadResponse from deepagents.backends.protocol import FileUploadResponse from deepagents.backends.sandbox import BaseSandbox from ..base import SandboxProvider +from ..base import SandboxTerminatedError from ..capabilities import SandboxCapabilities +from ..logging_utils import log_sandbox_failure from ..registry import register_sandbox_provider if TYPE_CHECKING: @@ -103,9 +108,9 @@ def _classify_fs_error(text: str) -> str: _OPENSHELL_IMPORT_HINT = ( - "The OpenShell sandbox provider requires the `openshell>=0.0.72,<0.1` SDK and the " - "`langchain-nvidia-openshell` adapter (published on PyPI). They are optional, ad-hoc " - "dependencies. Install them with `./scripts/setup_openshell.sh` (which installs " + "The OpenShell sandbox provider requires the `openshell>=0.0.80,<0.1` SDK and the " + "`langchain-nvidia-openshell` adapter (published on PyPI). They are optional, separately installed " + "dependencies. Install them with `./scripts/openshell/setup_openshell.sh` (which installs " "`langchain-nvidia-openshell` from PyPI; override the source via `LANGCHAIN_NVIDIA_REPO`), " "and configure an OpenShell gateway before enabling this provider." ) @@ -119,20 +124,201 @@ def _normalize_openshell_name(job_id: str, prefix: str = "aiq-deep-research") -> return (normalized[:63].rstrip("-")) or prefix +def _sandbox_labels(job_id: str) -> dict[str, str]: + """Return stable gateway and runtime ownership labels for one AI-Q job.""" + return { + "aiq": "deep-research", + "aiq-job-id": _normalize_openshell_name(job_id, prefix=""), + } + + +def _accepts_keyword(callable_obj: Any, keyword: str) -> bool: + """Return whether a public SDK callable accepts one named keyword.""" + try: + parameters = inspect.signature(callable_obj).parameters.values() + except (TypeError, ValueError): + return False + return any(parameter.name == keyword or parameter.kind is inspect.Parameter.VAR_KEYWORD for parameter in parameters) + + def _is_openshell_not_found_error(exc: Exception) -> bool: """Best-effort classification of OpenShell stale-sandbox errors.""" text = str(exc).lower() return "not found" in text and ("sandbox" in text or exc.__class__.__module__.startswith("openshell")) -class OpenShellSandboxProvider(SandboxProvider): - """OpenShell backend that attaches to a configured sandbox. +def _read_policy_data(policy_path: str, *, require_hard_landlock: bool) -> dict[str, Any]: + """Read and normalize OpenShell policy YAML without importing the optional SDK.""" + try: + import yaml + except ImportError as exc: + raise ImportError(_OPENSHELL_IMPORT_HINT) from exc + + path = Path(policy_path).expanduser() + try: + raw = yaml.safe_load(path.read_text(encoding="utf-8")) + except OSError as exc: + raise ValueError(f"Could not read OpenShell policy file: {path}") from exc + except yaml.YAMLError as exc: + raise ValueError(f"Invalid OpenShell policy YAML: {path}") from exc + + if not isinstance(raw, dict): + raise ValueError(f"OpenShell policy must be a YAML mapping: {path}") + if raw.get("version") != 1: + raise ValueError("OpenShell policy version must be exactly 1") + + landlock = raw.get("landlock") + compatibility = landlock.get("compatibility") if isinstance(landlock, dict) else None + if require_hard_landlock and compatibility != "hard_requirement": + raise ValueError( + "OpenShell production policy requires landlock.compatibility=hard_requirement; " + "set require_hard_landlock=false only for an explicit local demo." + ) + + filesystem = raw.get("filesystem_policy", raw.get("filesystem")) + if not isinstance(filesystem, dict) or not any(filesystem.get(key) for key in ("read_only", "read_write")): + raise ValueError("OpenShell production policy requires non-empty filesystem read_only or read_write rules") + + process = raw.get("process") + if not isinstance(process, dict): + raise ValueError("OpenShell production policy requires a process policy") + for field in ("run_as_user", "run_as_group"): + identity = process.get(field) + if not isinstance(identity, str) or not identity.strip(): + raise ValueError(f"OpenShell production policy requires process.{field} to be a non-empty string") + if identity.strip().lower() in {"0", "root"}: + raise ValueError(f"OpenShell production policy requires a non-root process.{field}") + + network_policies = raw.get("network_policies") or {} + if not isinstance(network_policies, dict): + raise ValueError("OpenShell network_policies must be a mapping") + for policy_name, network_policy in network_policies.items(): + if not isinstance(network_policy, dict): + raise ValueError(f"OpenShell network policy {policy_name!r} must be a mapping") + endpoints = network_policy.get("endpoints") or [] + if not isinstance(endpoints, list): + raise ValueError(f"OpenShell network policy {policy_name!r} endpoints must be a list") + for endpoint in endpoints: + if not isinstance(endpoint, dict): + raise ValueError(f"OpenShell network policy {policy_name!r} contains an invalid endpoint") + if endpoint.get("enforcement") != "enforce" or endpoint.get("access") != "read-only": + raise ValueError( + f"OpenShell network policy {policy_name!r} endpoints must use " + "enforcement=enforce and access=read-only" + ) + + # OpenShell's YAML schema calls this field filesystem_policy while the + # Python proto calls it filesystem. Keep this compatibility translation in one + # place and reject every other unknown field through ParseDict below. + policy_data = dict(raw) + if "filesystem_policy" in policy_data: + if "filesystem" in policy_data: + raise ValueError("OpenShell policy cannot contain both filesystem_policy and filesystem") + policy_data["filesystem"] = policy_data.pop("filesystem_policy") + return policy_data + + +def _parse_policy_proto(policy_data: dict[str, Any], *, policy_path: str) -> Any: + """Parse one validated policy snapshot into the SDK proto with strict field validation.""" + try: + from google.protobuf.json_format import ParseDict + from openshell._proto import sandbox_pb2 + except ImportError as exc: + raise ImportError(_OPENSHELL_IMPORT_HINT) from exc + + try: + return ParseDict(policy_data, sandbox_pb2.SandboxPolicy(), ignore_unknown_fields=False) + except Exception as exc: # noqa: BLE001 - protobuf raises several parse exception types + raise ValueError(f"OpenShell policy does not match the installed SDK schema: {policy_path}") from exc + + +def _policy_network_hosts(policy_data: dict[str, Any]) -> set[str]: + """Return every hostname authorized by an OpenShell network policy.""" + policies = policy_data.get("network_policies") or {} + if not isinstance(policies, dict): + return set() + hosts: set[str] = set() + for policy in policies.values(): + if not isinstance(policy, dict): + continue + endpoints = policy.get("endpoints") or [] + if not isinstance(endpoints, list): + continue + for endpoint in endpoints: + if isinstance(endpoint, dict) and isinstance(endpoint.get("host"), str): + host = endpoint["host"].strip().lower().rstrip(".") + if host: + hosts.add(host) + return hosts + + +def _policy_network_endpoints(policy_data: dict[str, Any]) -> list[dict[str, Any]]: + """Return validated endpoint mappings from every OpenShell network policy.""" + policies = policy_data.get("network_policies") or {} + if not isinstance(policies, dict): + return [] + endpoints: list[dict[str, Any]] = [] + for policy in policies.values(): + if not isinstance(policy, dict): + continue + entries = policy.get("endpoints") or [] + if isinstance(entries, list): + endpoints.extend(endpoint for endpoint in entries if isinstance(endpoint, dict)) + return endpoints + + +def _validate_policy_network(policy_data: dict[str, Any], *, mode: str, allow: tuple[str, ...]) -> None: + """Fail closed when the policy grants more egress than the public config declares.""" + endpoints = _policy_network_endpoints(policy_data) + policy_hosts = _policy_network_hosts(policy_data) + if mode == "blocked" and endpoints: + raise ValueError("OpenShell policy grants network endpoints while sandbox.network is 'blocked'") + if mode == "allowlist": + for endpoint in endpoints: + host = endpoint.get("host") + if not isinstance(host, str) or not host.strip().rstrip("."): + raise ValueError("OpenShell allowlist endpoints require a non-empty host") + allowed_ips = endpoint.get("allowed_ips") or [] + if allowed_ips: + raise ValueError("OpenShell allowed_ips/CIDR endpoints require an explicit public CIDR contract") + configured_hosts = {host.strip().lower().rstrip(".") for host in allow} + unexpected = policy_hosts - configured_hosts + if unexpected: + raise ValueError(f"OpenShell policy grants hosts outside sandbox.network.allow: {sorted(unexpected)}") + + +@dataclass(frozen=True) +class _AttestationResult: + """Secret-free proof returned only after authoritative policy checks pass.""" + + phase: object + policy_version: int + policy_hash: str | None + policy_source: object | None + assurance: str + + +def _build_sandbox_spec( + *, + policy: Any, + image: str, + job_id: str, + labels: dict[str, str] | None = None, +) -> Any: + """Build a secret-free per-job OpenShell spec using the installed SDK schema.""" + try: + from openshell._proto import openshell_pb2 + except ImportError as exc: + raise ImportError(_OPENSHELL_IMPORT_HINT) from exc + + template = openshell_pb2.SandboxTemplate(image=image, labels=labels or _sandbox_labels(job_id)) + # Deliberately omit environment and providers. Research/model credentials stay + # on the host unless a future explicit credential-provider feature is configured. + return openshell_pb2.SandboxSpec(template=template, policy=policy) - OpenShell enforces filesystem/process/network policy at the gateway, so this - provider declares those capabilities. The SDK cannot apply or verify a policy - file while attaching: ``policy`` requires a pre-created named sandbox whose - policy is managed externally. - """ + +class OpenShellSandboxProvider(SandboxProvider): + """OpenShell backend that creates and attests a policy-bound sandbox per job.""" provider_name = "openshell" @@ -140,6 +326,9 @@ def __init__(self, config: SandboxConfig, job_id: str) -> None: """Initialize the provider, requiring the OpenShell SDK and adapter to import.""" super().__init__(config, job_id) self._os_context: object | None = None + self._os_context_entering = False + self._os_context_exit_requested = False + self._os_context_cleanup_complete: Event | None = None try: import langchain_nvidia_openshell # noqa: F401 import openshell # noqa: F401 @@ -161,6 +350,7 @@ def capabilities(self) -> SandboxCapabilities: supports_process_policy=True, supports_artifact_download=True, supports_cleanup=True, + supports_terminate=True, ) def is_recoverable_error(self, exc: Exception) -> bool: @@ -181,7 +371,7 @@ def download_files(self, paths: list[str]) -> list[FileDownloadResponse]: def _upload_files_envfree(self, files: list[tuple[str, bytes]]) -> list[FileUploadResponse]: """Upload files via argv + stdin so no ``OPENSHELL_*`` env is required.""" - sandbox = self._os_context + sandbox = self._active_os_context() responses: list[FileUploadResponse] = [] for path, content in files: if not path.startswith("/"): @@ -199,7 +389,7 @@ def _upload_files_envfree(self, files: list[tuple[str, bytes]]) -> list[FileUplo def _download_files_envfree(self, paths: list[str]) -> list[FileDownloadResponse]: """Download files via an argv bootstrap that enforces size/symlink limits in-sandbox.""" - sandbox = self._os_context + sandbox = self._active_os_context() # Cap passed to the bootstrap so oversized files are refused before transfer. max_bytes = self.config.artifact_capture.max_file_bytes responses: list[FileDownloadResponse] = [] @@ -228,11 +418,7 @@ def _download_files_envfree(self, paths: list[str]) -> list[FileDownloadResponse return responses def _create_session(self) -> BaseSandbox: - """Create/attach the OpenShell sandbox and wrap it in the official adapter. - - A configured ``policy`` requires a pre-created named sandbox, because the - SDK cannot apply policy files to anonymous sandboxes. - """ + """Create and attest a per-job OpenShell sandbox, or explicitly attach for debug.""" try: import openshell from langchain_nvidia_openshell import OpenShellSandbox @@ -241,36 +427,369 @@ def _create_session(self) -> BaseSandbox: cfg = self.config oscfg = cfg.providers.openshell - if oscfg.policy and not oscfg.sandbox_name: - raise ValueError( - "OpenShell `policy` requires `sandbox_name`. The SDK cannot apply a policy file to an " - "anonymous sandbox. Create the named sandbox with `openshell sandbox create --policy ` " - "first, then set providers.openshell.sandbox_name." - ) + + if not oscfg.shell or any(not part.strip() for part in oscfg.shell): + raise ValueError("OpenShell shell must contain at least one non-empty argv element") # Release any prior context (covers the recoverable-error reset path). self._exit_context() sandbox_kwargs: dict[str, object] = { "cluster": oscfg.gateway, - "delete_on_exit": oscfg.delete_on_exit, "ready_timeout_seconds": oscfg.ready_timeout_seconds, } - if oscfg.sandbox_name: - sandbox_kwargs["sandbox"] = oscfg.sandbox_name + shared_name = oscfg.shared_sandbox_name + expected_policy: Any | None = None + if oscfg.policy: + policy_data = _read_policy_data(oscfg.policy, require_hard_landlock=oscfg.require_hard_landlock) + _validate_policy_network( + policy_data, + mode=cfg.network.mode, + allow=cfg.network.allow, + ) + expected_policy = _parse_policy_proto(policy_data, policy_path=oscfg.policy) + + if shared_name is not None: + logger.warning( + "OpenShell shared-sandbox debug attachment enabled: sandbox=%s job=%s; physical job isolation is off", + shared_name, + self.job_id, + ) + # Attachment does not transfer ownership: never delete a shared sandbox + # that this job did not create. + sandbox_kwargs.update(sandbox=shared_name, delete_on_exit=False) + else: + if expected_policy is None: + raise ValueError("Per-job OpenShell creation requires a policy file") + labels = _sandbox_labels(self.job_id) + if not _accepts_keyword(openshell.Sandbox, "labels"): + self._fail_attestation( + phase=None, + policy_version=0, + assurance="strict", + reason_code="request_labels_unsupported", + ) + sandbox_kwargs.update( + spec=_build_sandbox_spec( + policy=expected_policy, + image=oscfg.image, + job_id=self.job_id, + labels=labels, + ), + labels=labels, + delete_on_exit=oscfg.delete_on_exit, + ) os_sandbox = openshell.Sandbox(**sandbox_kwargs) - os_sandbox.__enter__() - self._os_context = os_sandbox - backend = OpenShellSandbox(sandbox=os_sandbox, timeout=cfg.timeout, shell=oscfg.shell) - logger.info( - "OpenShell sandbox READY: id=%s gateway=%s sandbox_name=%s policy=%s", - backend.id, - oscfg.gateway, - oscfg.sandbox_name, - oscfg.policy, + self._enter_context(os_sandbox) + backend: BaseSandbox | None = None + try: + self._ensure_context_active(os_sandbox) + self.physical_sandbox_name = getattr(os_sandbox.sandbox, "name", None) + attestation = self._attest( + os_sandbox, + expected_policy=expected_policy, + require_sandbox_source=shared_name is None, + ) + self._ensure_context_active(os_sandbox) + backend = OpenShellSandbox(sandbox=os_sandbox, timeout=cfg.timeout, shell=oscfg.shell) + self._ensure_context_active(os_sandbox) + sandbox_ref = os_sandbox.sandbox + logger.info( + "OpenShell sandbox attested: id=%s name=%s policy_version=%s shared=%s", + backend.id, + getattr(sandbox_ref, "name", None), + attestation.policy_version, + shared_name is not None, + ) + self._emit_attestation(attestation=attestation, status="succeeded") + return backend + except BaseException: + self._safe_close(backend) + self._exit_context() + raise + + def _attest( + self, + os_sandbox: Any, + *, + expected_policy: Any | None, + require_sandbox_source: bool, + ) -> _AttestationResult: + """Fail closed unless authoritative RPCs prove the effective policy.""" + try: + from openshell._proto import openshell_pb2 + except ImportError as exc: + raise ImportError(_OPENSHELL_IMPORT_HINT) from exc + + sandbox_ref = os_sandbox.sandbox + phase = getattr(sandbox_ref, "phase", None) + policy_version = getattr(sandbox_ref, "current_policy_version", 0) + if phase != openshell_pb2.SANDBOX_PHASE_READY: + self._fail_attestation( + phase=phase, + policy_version=policy_version, + assurance="strict" if expected_policy is not None else "reduced", + reason_code="not_ready", + ) + + oscfg = self.config.providers.openshell + assurance = "strict" if expected_policy is not None else "reduced" + if not oscfg.attest: + return _AttestationResult( + phase=phase, + policy_version=policy_version if isinstance(policy_version, int) else 0, + policy_hash=None, + policy_source=None, + assurance=assurance, + ) + return self._wait_for_authoritative_policy( + os_sandbox, + sandbox_ref, + expected_policy=expected_policy, + require_sandbox_source=require_sandbox_source, + assurance=assurance, + ) + + def _wait_for_authoritative_policy( + self, + os_sandbox: Any, + sandbox_ref: Any, + *, + expected_policy: Any | None, + require_sandbox_source: bool, + assurance: str, + ) -> _AttestationResult: + """Verify status, effective policy, provenance, version, and hash via authoritative RPCs.""" + from openshell._proto import openshell_pb2 + from openshell._proto import sandbox_pb2 + + oscfg = self.config.providers.openshell + phase = getattr(sandbox_ref, "phase", None) + initial_policy_version = getattr(sandbox_ref, "current_policy_version", 0) + client = getattr(os_sandbox, "_client", None) + stub = getattr(client, "_stub", None) + if ( + client is None + or stub is None + or not hasattr(stub, "GetSandboxPolicyStatus") + or not hasattr(stub, "GetSandboxConfig") + ): + self._fail_attestation( + phase=phase, + policy_version=initial_policy_version, + assurance=assurance, + reason_code="rpc_unavailable", + ) + + status_request = openshell_pb2.GetSandboxPolicyStatusRequest(name=sandbox_ref.name, version=0) + sandbox_id = getattr(sandbox_ref, "id", None) or sandbox_ref.name + config_request = sandbox_pb2.GetSandboxConfigRequest(sandbox_id=sandbox_id) + deadline = time.monotonic() + oscfg.policy_load_timeout_seconds + pending_effective_policy = False + last_revision_version = initial_policy_version + while True: + self._ensure_context_active(os_sandbox) + remaining = deadline - time.monotonic() + if remaining <= 0: + self._fail_attestation( + phase=phase, + policy_version=last_revision_version, + assurance=assurance, + reason_code="policy_status_inconsistent" if pending_effective_policy else "attestation_timeout", + ) + + try: + refreshed = client.get(sandbox_ref.name) + phase = getattr(refreshed, "phase", None) + current_policy_version = getattr(refreshed, "current_policy_version", 0) + status_response = stub.GetSandboxPolicyStatus(status_request, timeout=min(5.0, remaining)) + config_response = stub.GetSandboxConfig(config_request, timeout=min(5.0, remaining)) + except Exception: # noqa: BLE001 - SDK errors must not escape with credential-bearing details + self._fail_attestation( + phase=phase, + policy_version=initial_policy_version, + assurance=assurance, + reason_code="rpc_failed", + ) + + if phase != openshell_pb2.SANDBOX_PHASE_READY: + self._fail_attestation( + phase=phase, + policy_version=current_policy_version, + assurance=assurance, + reason_code="not_ready", + ) + + revision = getattr(status_response, "revision", None) + revision_status = getattr(revision, "status", openshell_pb2.POLICY_STATUS_UNSPECIFIED) + revision_version = getattr(revision, "version", 0) + last_revision_version = revision_version + config_version = getattr(config_response, "version", 0) + if getattr(revision, "load_error", ""): + self._fail_attestation( + phase=phase, + policy_version=revision_version, + assurance=assurance, + reason_code="policy_load_error", + ) + if revision_status == openshell_pb2.POLICY_STATUS_FAILED: + self._fail_attestation( + phase=phase, + policy_version=revision_version, + assurance=assurance, + reason_code="policy_status_failed", + ) + if revision_status != openshell_pb2.POLICY_STATUS_LOADED: + config_policy = getattr(config_response, "policy", None) + revision_policy = getattr(revision, "policy", None) + policy_source = getattr(config_response, "policy_source", sandbox_pb2.POLICY_SOURCE_UNSPECIFIED) + policy_hash = getattr(config_response, "policy_hash", "") + revision_hash = getattr(revision, "policy_hash", "") + pending_effective_policy = ( + revision_status == openshell_pb2.POLICY_STATUS_PENDING + and revision_version > 0 + and revision_version == config_version + and ( + expected_policy is None + or ( + config_policy == expected_policy + and revision_policy == expected_policy + and bool(policy_hash) + and policy_hash == revision_hash + and policy_source != sandbox_pb2.POLICY_SOURCE_UNSPECIFIED + and (not require_sandbox_source or policy_source == sandbox_pb2.POLICY_SOURCE_SANDBOX) + ) + ) + ) + time.sleep(min(0.5, remaining)) + continue + + active_version = getattr(status_response, "active_version", 0) + if not revision_version or revision_version != config_version: + self._fail_attestation( + phase=phase, + policy_version=revision_version, + assurance=assurance, + reason_code="version_mismatch", + ) + effective_version = revision_version + for reported_version in (active_version, current_policy_version): + if ( + not isinstance(reported_version, int) + or reported_version <= 0 + or reported_version != effective_version + ): + self._fail_attestation( + phase=phase, + policy_version=revision_version, + assurance=assurance, + reason_code="version_mismatch", + ) + if oscfg.expected_policy_version is not None and effective_version != oscfg.expected_policy_version: + self._fail_attestation( + phase=phase, + policy_version=effective_version, + assurance=assurance, + reason_code="expected_version_mismatch", + ) + + policy_source = getattr(config_response, "policy_source", sandbox_pb2.POLICY_SOURCE_UNSPECIFIED) + policy_hash = getattr(config_response, "policy_hash", "") + revision_hash = getattr(revision, "policy_hash", "") + if expected_policy is not None: + if require_sandbox_source and policy_source != sandbox_pb2.POLICY_SOURCE_SANDBOX: + self._fail_attestation( + phase=phase, + policy_version=effective_version, + assurance=assurance, + reason_code="policy_source_mismatch", + ) + if policy_source == sandbox_pb2.POLICY_SOURCE_UNSPECIFIED: + self._fail_attestation( + phase=phase, + policy_version=effective_version, + assurance=assurance, + reason_code="policy_source_mismatch", + ) + config_policy = getattr(config_response, "policy", None) + revision_policy = getattr(revision, "policy", None) + if config_policy != expected_policy or revision_policy != expected_policy: + self._fail_attestation( + phase=phase, + policy_version=effective_version, + assurance=assurance, + reason_code="policy_content_mismatch", + ) + if not policy_hash or not revision_hash: + self._fail_attestation( + phase=phase, + policy_version=effective_version, + assurance=assurance, + reason_code="policy_hash_missing", + ) + if policy_hash != revision_hash: + self._fail_attestation( + phase=phase, + policy_version=effective_version, + assurance=assurance, + reason_code="policy_hash_mismatch", + ) + + return _AttestationResult( + phase=phase, + policy_version=effective_version, + policy_hash=policy_hash or revision_hash or None, + policy_source=policy_source, + assurance=assurance, + ) + + def _fail_attestation( + self, + *, + phase: object, + policy_version: object, + assurance: str, + reason_code: str, + ) -> None: + """Emit one classified failure and raise without SDK or policy details.""" + self._emit_attestation( + attestation=_AttestationResult( + phase=phase, + policy_version=policy_version if isinstance(policy_version, int) else 0, + policy_hash=None, + policy_source=None, + assurance=assurance, + ), + status="failed", + reason_code=reason_code, + ) + raise RuntimeError(f"OpenShell sandbox attestation failed: {reason_code}") + + def _emit_attestation( + self, + *, + attestation: _AttestationResult, + status: str, + reason_code: str | None = None, + ) -> None: + """Emit a secret-free OpenShell attestation outcome.""" + self._emit_event( + { + "type": "sandbox.attestation", + "data": { + "provider": self.provider_name, + "sandbox": getattr(self, "physical_sandbox_name", None) or self.sandbox_name, + "phase": attestation.phase, + "policy_version": attestation.policy_version, + "policy_hash": attestation.policy_hash, + "policy_source": attestation.policy_source, + "assurance": attestation.assurance, + "reason_code": reason_code, + "status": status, + }, + } ) - return backend def close(self) -> None: """Terminate the session and exit the OpenShell context manager.""" @@ -282,15 +801,117 @@ def _terminate_session(self, session: BaseSandbox | None) -> None: super()._terminate_session(session) self._exit_context() + def _active_os_context(self) -> Any: + """Return the current SDK context without racing an out-of-band teardown.""" + with self._state_lock: + ctx = self._os_context + if ctx is None: + raise SandboxTerminatedError(f"OpenShell sandbox {self.sandbox_name} has no active context") + return ctx + + def _enter_context(self, ctx: object) -> None: + """Enter an SDK context while allowing terminate() to request deferred cleanup.""" + cleanup_complete = Event() + with self._state_lock: + if self._terminated: + raise SandboxTerminatedError(f"Sandbox {self.sandbox_name} has been terminated") + if self._os_context is not None: + raise RuntimeError(f"OpenShell sandbox {self.sandbox_name} context creation is already in progress") + if self._os_context_cleanup_complete is not None and not self._os_context_cleanup_complete.is_set(): + raise RuntimeError(f"OpenShell sandbox {self.sandbox_name} context cleanup is already in progress") + self._os_context = ctx + self._os_context_entering = True + self._os_context_exit_requested = False + self._os_context_cleanup_complete = cleanup_complete + + try: + ctx.__enter__() # type: ignore[attr-defined] + except BaseException: + self._finish_context_entry(ctx, cleanup_complete, entered=False) + raise + + if not self._finish_context_entry(ctx, cleanup_complete, entered=True): + raise SandboxTerminatedError(f"Sandbox {self.sandbox_name} was closed during creation") + + def _finish_context_entry(self, ctx: object, cleanup_complete: Event, *, entered: bool) -> bool: + """Publish an entered context, or honor a pending teardown exactly once.""" + with self._state_lock: + owns_context = self._os_context is ctx + if owns_context: + self._os_context_entering = False + release_context = not entered or self._os_context_exit_requested or self._terminated + if release_context: + self._os_context = None + self._os_context_exit_requested = False + else: + release_context = entered + + if release_context: + self._close_os_context(ctx, cleanup_complete) + return entered and owns_context and not release_context + + def _ensure_context_active(self, ctx: object) -> None: + """Abort session publication when teardown won a creation race.""" + with self._state_lock: + active = self._os_context is ctx and not self._os_context_exit_requested and not self._terminated + if not active: + raise SandboxTerminatedError(f"Sandbox {self.sandbox_name} was closed during creation") + def _exit_context(self) -> None: - """Exit the OpenShell context once, swallowing cleanup errors on the terminal path.""" - ctx = self._os_context - self._os_context = None - if ctx is not None and hasattr(ctx, "__exit__"): - try: + """Exit once and boundedly await creator-owned deletion during ``__enter__``.""" + close_context: object | None = None + with self._state_lock: + ctx = self._os_context + cleanup_complete = self._os_context_cleanup_complete + if ctx is None: + wait_for_cleanup = cleanup_complete is not None and not cleanup_complete.is_set() + elif self._os_context_entering: + self._os_context_exit_requested = True + if cleanup_complete is None: + cleanup_complete = Event() + self._os_context_cleanup_complete = cleanup_complete + wait_for_cleanup = True + else: + if cleanup_complete is None: + cleanup_complete = Event() + self._os_context_cleanup_complete = cleanup_complete + self._os_context = None + self._os_context_exit_requested = False + close_context = ctx + wait_for_cleanup = False + + if close_context is not None: + self._close_os_context(close_context, cleanup_complete) + return + if wait_for_cleanup and cleanup_complete is not None: + if not cleanup_complete.wait(timeout=self.config.providers.openshell.cleanup_timeout_seconds): + self._record_cleanup_failure("cleanup_timeout") + logger.warning( + "OpenShell cleanup timed out: provider=%s sandbox=%s reason=cleanup_timeout", + self.provider_name, + self.sandbox_name, + ) + + def _close_os_context(self, ctx: object, cleanup_complete: Event) -> None: + """Drive one detached SDK context exit without replacing the job result.""" + try: + if hasattr(ctx, "__exit__"): ctx.__exit__(None, None, None) - except Exception: # noqa: BLE001 - cleanup must never raise on the terminal path - logger.warning("OpenShell sandbox %s context cleanup failed", self.sandbox_name, exc_info=True) + except Exception as exc: # noqa: BLE001 - cleanup must never raise on the terminal path + self._record_cleanup_failure("context_exit_failed") + log_sandbox_failure( + logger, + operation="context_exit", + reason_code="context_exit_failed", + exc=exc, + provider=self.provider_name, + sandbox=self.sandbox_name, + ) + finally: + cleanup_complete.set() + with self._state_lock: + if self._os_context_cleanup_complete is cleanup_complete: + self._os_context_cleanup_complete = None register_sandbox_provider("openshell", OpenShellSandboxProvider) diff --git a/src/aiq_agent/agents/deep_researcher/sandbox/registry.py b/src/aiq_agent/agents/deep_researcher/sandbox/registry.py index bfd107631..ba500da04 100644 --- a/src/aiq_agent/agents/deep_researcher/sandbox/registry.py +++ b/src/aiq_agent/agents/deep_researcher/sandbox/registry.py @@ -27,6 +27,7 @@ from typing import TYPE_CHECKING from .capabilities import verify_capabilities +from .logging_utils import log_sandbox_failure if TYPE_CHECKING: from .base import SandboxProvider @@ -80,8 +81,14 @@ def _load_entry_point_providers() -> None: for entry_point in discovered: try: provider_cls = entry_point.load() - except Exception: # noqa: BLE001 - a bad plugin must not break built-in resolution - logger.warning("Failed to load sandbox provider entry point '%s'", entry_point.name, exc_info=True) + except Exception as exc: # noqa: BLE001 - a bad plugin must not break built-in resolution + log_sandbox_failure( + logger, + operation="provider_entry_point_load", + reason_code="provider_load_failed", + exc=exc, + provider=entry_point.name, + ) continue register_sandbox_provider(entry_point.name, provider_cls) logger.info("Registered sandbox provider '%s' from entry point", entry_point.name) diff --git a/src/aiq_agent/agents/deep_researcher/skills/research/chart-generation/SKILL.md b/src/aiq_agent/agents/deep_researcher/skills/research/chart-generation/SKILL.md index 67aafdb68..76847076f 100644 --- a/src/aiq_agent/agents/deep_researcher/skills/research/chart-generation/SKILL.md +++ b/src/aiq_agent/agents/deep_researcher/skills/research/chart-generation/SKILL.md @@ -55,8 +55,12 @@ outfit. A polished chart of wrong or sparse numbers misleads more than it inform 1. Assemble the normalized rows (prefer explicit records embedded in the script). If the inputs live in `/shared/...`, `read_file` them first and embed the values; sandbox code cannot open `/shared/...`. -2. Use `write_file` to create the chart script under the `sandbox_workdir` from your - instructions (e.g. `/make_chart.py`), then `execute` that exact path. +2. Use `write_file` to create the chart script under the exact `sandbox_workdir` from your + instructions, then `execute` it with the exact `sandbox_artifact_dir` as its first argument. + For example, when your instructions provide `/sandbox/JOB/` and + `/sandbox/JOB/aiq-artifacts`, run + `python3 /sandbox/JOB/make_chart.py /sandbox/JOB/aiq-artifacts`. Never execute a literal + `` or `` token. `sandbox_workdir` is already per-job, so scripts there cannot collide with another job's leftovers. Only ever execute a script you wrote this session. The script must: - import pandas and matplotlib (use the non-interactive `Agg` backend), @@ -88,40 +92,28 @@ Each figure must appear where it is discussed, not buried in a file list: Write a `manifest.json` in your `sandbox_artifact_dir` so the runtime captures the chart with metadata. Manifest `path` values must be absolute and inside your `sandbox_artifact_dir` -(the per-job path from your instructions, e.g. `/sandbox//aiq-artifacts`): - -```json -{ - "version": 1, - "artifacts": [ - { - "path": "/revenue_chart.png", - "kind": "image", - "title": "2024 Semiconductor Revenue Comparison", - "caption": "Revenue normalized to USD billions.", - "inline": true, - "source_files": ["/shared/semiconductor_revenue_normalized.csv"] - } - ] -} -``` +(the per-job path from your instructions). Construct every manifest path from the runtime +argument as shown below; do not hand-copy an angle-bracket placeholder into JSON. Set +`inline: true` only for a raster image intended to appear in the report. ## Example Script ```python import json -import os +import sys +from pathlib import Path import matplotlib matplotlib.use("Agg") import matplotlib.pyplot as plt import pandas as pd -# ARTIFACT_DIR MUST be the exact sandbox_artifact_dir from your instructions - a per-job -# path such as /sandbox//aiq-artifacts. Copy that value here verbatim. Do NOT use a -# bare /sandbox/aiq-artifacts: the runtime only harvests files under sandbox_artifact_dir. -ARTIFACT_DIR = "" # replace with the path from your instructions -os.makedirs(ARTIFACT_DIR, exist_ok=True) +if len(sys.argv) != 2: + raise SystemExit("usage: make_chart.py ABSOLUTE_SANDBOX_ARTIFACT_DIR") +ARTIFACT_DIR = Path(sys.argv[1]) +if not ARTIFACT_DIR.is_absolute(): + raise SystemExit("artifact directory must be an absolute path") +ARTIFACT_DIR.mkdir(parents=True, exist_ok=True) rows = [ {"company": "ExampleCo", "revenue_usd_billions": 12.4, "source": "https://example.com/filing"}, @@ -135,8 +127,8 @@ ax.set_ylabel("Revenue (USD billions)") ax.set_title("2024 Revenue Comparison") fig.tight_layout() -png_path = f"{ARTIFACT_DIR}/revenue_chart.png" -csv_path = f"{ARTIFACT_DIR}/revenue_chart.csv" +png_path = ARTIFACT_DIR / "revenue_chart.png" +csv_path = ARTIFACT_DIR / "revenue_chart.csv" fig.savefig(png_path, dpi=150) df.to_csv(csv_path, index=False) @@ -144,7 +136,7 @@ manifest = { "version": 1, "artifacts": [ { - "path": png_path, + "path": str(png_path), "kind": "image", "title": "2024 Revenue Comparison", "caption": "Revenue normalized to USD billions.", @@ -153,16 +145,24 @@ manifest = { } ], } -with open(f"{ARTIFACT_DIR}/manifest.json", "w") as handle: +with (ARTIFACT_DIR / "manifest.json").open("w", encoding="utf-8") as handle: json.dump(manifest, handle) print(f"wrote {png_path}") ``` +Run the script with the two exact per-job paths given in your instructions. The second argument +must be the real absolute artifact directory, not an angle-bracket placeholder. Treat the +artifact-checkpoint response after `execute` as authoritative: reference the exact confirmed +filename in the report and do not invent or rename it later. + ## Notes and Limitations - Use the `Agg` backend; the sandbox has no display. - Keep charts legible: labeled axes, a title, and a legend when multiple series are shown. +- Do not call `read_file` on the generated PNG merely to verify it; binary reads return base64 + and waste model context. Inspect `manifest.json` with `read_file(file_path=...)` when needed, + then rely on the artifact-checkpoint response to confirm the accepted filename and inline state. - If matplotlib or pandas is unavailable, report that the sandbox image needs them rather than fabricating a chart. - Reference charts only by `artifact://`; the runtime assigns the durable id and diff --git a/tests/aiq_agent/agents/deep_researcher/sandbox/test_artifacts.py b/tests/aiq_agent/agents/deep_researcher/sandbox/test_artifacts.py index ce33bb176..ad2693a8a 100644 --- a/tests/aiq_agent/agents/deep_researcher/sandbox/test_artifacts.py +++ b/tests/aiq_agent/agents/deep_researcher/sandbox/test_artifacts.py @@ -19,6 +19,7 @@ from __future__ import annotations import json +import logging from hashlib import sha256 from io import BytesIO from types import SimpleNamespace @@ -118,6 +119,21 @@ def test_relative_path_has_no_leading_slash(self) -> None: class TestHarvest: + def test_store_and_scan_failures_do_not_log_exception_text(self, caplog: pytest.LogCaptureFixture) -> None: + store = SimpleNamespace(list=lambda _job_id: (_ for _ in ()).throw(RuntimeError("credential=do-not-log"))) + manager, _ = _make_manager(store, {}) + + with caplog.at_level(logging.WARNING): + assert manager.resolve_report_references("report") == "report" + manager.backend.execute = lambda *_args, **_kwargs: (_ for _ in ()).throw( # type: ignore[method-assign] + RuntimeError("token=do-not-log") + ) + assert manager._scan_dir() == [] + + assert "RuntimeError" in caplog.text + assert "credential=do-not-log" not in caplog.text + assert "token=do-not-log" not in caplog.text + def test_captures_manifest_artifact(self, tmp_path: Any) -> None: store = SqlArtifactStore(f"sqlite:///{tmp_path}/jobs.db") png_path = f"{_ARTIFACT_DIR}/chart.png" @@ -407,17 +423,20 @@ def test_s3_bytes_leave_sql_content_null(self, tmp_path: Any) -> None: ).scalar_one() assert content is None - def test_failed_upload_removes_metadata(self, tmp_path: Any) -> None: + def test_failed_upload_removes_metadata(self, tmp_path: Any, caplog: pytest.LogCaptureFixture) -> None: client = _FakeS3Client() client.fail_put = True store = self._store(tmp_path, client) artifact = self._artifact() - with pytest.raises(RuntimeError, match="upload failed"): - store.put(artifact, _PNG) + with caplog.at_level(logging.WARNING): + with pytest.raises(RuntimeError, match="upload failed"): + store.put(artifact, _PNG) assert store.get(artifact.job_id, artifact.artifact_id) is None assert b"".join(store.open_bytes(artifact.job_id, artifact.artifact_id)) == b"" + assert "RuntimeError" in caplog.text + assert "upload failed" not in caplog.text def test_failed_delete_retains_metadata_for_retry(self, tmp_path: Any) -> None: client = _FakeS3Client() diff --git a/tests/aiq_agent/agents/deep_researcher/sandbox/test_openshell_live.py b/tests/aiq_agent/agents/deep_researcher/sandbox/test_openshell_live.py new file mode 100644 index 000000000..515ea3930 --- /dev/null +++ b/tests/aiq_agent/agents/deep_researcher/sandbox/test_openshell_live.py @@ -0,0 +1,460 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Opt-in live acceptance tests for OpenShell isolation and lifecycle behavior.""" + +from __future__ import annotations + +import copy +import importlib.metadata +import json +import logging +import os +import platform +import shlex +from collections.abc import Callable +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +from pathlib import Path +from typing import Any +from uuid import uuid4 + +import pytest + +_REPO_ROOT = Path(__file__).resolve().parents[5] +_LIVE_ENABLED = os.getenv("AIQ_OPENSHELL_LIVE_TESTS") == "1" + +pytestmark = [ + pytest.mark.integration, + pytest.mark.skipif( + not _LIVE_ENABLED, + reason="Set AIQ_OPENSHELL_LIVE_TESTS=1 to run live OpenShell integration tests.", + ), +] + + +def _env_enabled(name: str) -> bool: + return os.getenv(name, "").strip().lower() in {"1", "true", "yes", "on"} + + +@dataclass(frozen=True) +class LiveConfig: + """Non-secret live-test settings.""" + + gateway: str | None + policy_path: Path + image: str + expected_gateway_version: str | None + allow_best_effort: bool + + +@dataclass(frozen=True) +class LiveRuntime: + """Lazy imports and validated policy state used by opted-in tests.""" + + config: LiveConfig + sandbox_client: Any + grpc: Any + openshell_pb2: Any + sandbox_pb2: Any + expected_policy: Any + policy_data: dict[str, Any] + network: dict[str, object] + build_sandbox_spec: Callable[..., Any] + + def client(self) -> Any: + return self.sandbox_client.from_active_cluster(cluster=self.config.gateway) + + +@pytest.fixture(scope="session") +def live_config() -> LiveConfig: + policy_value = os.getenv( + "AIQ_OPENSHELL_POLICY_FILE", + "configs/openshell/generated/aiq-openshell-policy.yaml", + ) + policy_path = Path(policy_value).expanduser() + if not policy_path.is_absolute(): + policy_path = _REPO_ROOT / policy_path + return LiveConfig( + gateway=os.getenv("AIQ_OPENSHELL_GATEWAY_NAME") or None, + policy_path=policy_path.resolve(), + image=os.getenv("AIQ_OPENSHELL_IMAGE", "aiq-openshell-demo:latest"), + expected_gateway_version=os.getenv("AIQ_OPENSHELL_EXPECTED_GATEWAY_VERSION") or None, + allow_best_effort=_env_enabled("AIQ_OPENSHELL_LIVE_ALLOW_BEST_EFFORT"), + ) + + +@pytest.fixture(scope="session") +def live_runtime(live_config: LiveConfig) -> LiveRuntime: + """Import optional SDK modules only after pytest has evaluated the live opt-in.""" + if not live_config.allow_best_effort and platform.system() != "Linux": + pytest.fail( + "Production OpenShell acceptance requires Linux; use the explicit best-effort demo opt-in elsewhere." + ) + + try: + import grpc + import langchain_nvidia_openshell # noqa: F401 + import openshell # noqa: F401 + from openshell._proto import openshell_pb2 + from openshell._proto import sandbox_pb2 + from openshell.sandbox import SandboxClient + except ImportError: + pytest.fail("Opted-in OpenShell live tests require the SDK and DeepAgents adapter.") + + from aiq_agent.agents.deep_researcher.sandbox.providers.openshell import _build_sandbox_spec + from aiq_agent.agents.deep_researcher.sandbox.providers.openshell import _parse_policy_proto + from aiq_agent.agents.deep_researcher.sandbox.providers.openshell import _policy_network_hosts + from aiq_agent.agents.deep_researcher.sandbox.providers.openshell import _read_policy_data + + policy_data = _read_policy_data( + str(live_config.policy_path), + require_hard_landlock=not live_config.allow_best_effort, + ) + expected_policy = _parse_policy_proto(policy_data, policy_path=str(live_config.policy_path)) + hosts = tuple(sorted(_policy_network_hosts(policy_data))) + network: dict[str, object] = {"mode": "allowlist", "allow": hosts} if hosts else {"mode": "blocked"} + + return LiveRuntime( + config=live_config, + sandbox_client=SandboxClient, + grpc=grpc, + openshell_pb2=openshell_pb2, + sandbox_pb2=sandbox_pb2, + expected_policy=expected_policy, + policy_data=policy_data, + network=network, + build_sandbox_spec=_build_sandbox_spec, + ) + + +def _is_not_found(runtime: LiveRuntime, exc: BaseException) -> bool: + return isinstance(exc, runtime.grpc.Call) and exc.code() == runtime.grpc.StatusCode.NOT_FOUND + + +def _assert_deleted(runtime: LiveRuntime, client: Any, name: str) -> None: + try: + client.get(name) + except runtime.grpc.RpcError as exc: + if _is_not_found(runtime, exc): + return + raise + raise AssertionError(f"OpenShell cleanup was not verified for sandbox {name}") + + +@dataclass +class _TrackedResource: + kind: str + value: Any + owns_remote: bool = True + + +class ResourceTracker: + """Own every test-created resource and verify teardown through the gateway.""" + + def __init__(self, runtime: LiveRuntime) -> None: + self.runtime = runtime + self.resources: list[_TrackedResource] = [] + + def provider(self, provider: Any, *, owns_remote: bool) -> Any: + self.resources.append(_TrackedResource("provider", provider, owns_remote)) + return provider + + def direct_sandbox(self, name: str) -> str: + self.resources.append(_TrackedResource("direct", name)) + return name + + def cleanup(self) -> None: + failures: list[str] = [] + for resource in reversed(self.resources): + if resource.kind == "provider": + provider = resource.value + try: + provider.terminate() + except Exception as exc: # noqa: BLE001 - teardown must continue across every owned resource + failures.append(f"provider_terminate:{type(exc).__name__}") + name = getattr(provider, "physical_sandbox_name", None) + if resource.owns_remote and name: + try: + with self.runtime.client() as client: + _assert_deleted(self.runtime, client, name) + except Exception as exc: # noqa: BLE001 - report sanitized teardown metadata after all attempts + failures.append(f"provider_delete_verify:{type(exc).__name__}") + continue + + name = resource.value + try: + with self.runtime.client() as client: + try: + client.get(name) + except self.runtime.grpc.RpcError as exc: + if not _is_not_found(self.runtime, exc): + raise + else: + if not client.delete(name): + raise RuntimeError("delete_not_acknowledged") + client.wait_deleted(name) + _assert_deleted(self.runtime, client, name) + except Exception as exc: # noqa: BLE001 - report sanitized teardown metadata after all attempts + failures.append(f"direct_delete_verify:{type(exc).__name__}") + + if failures: + pytest.fail("OpenShell fixture teardown failed: " + ", ".join(failures)) + + +@pytest.fixture +def resources(live_runtime: LiveRuntime) -> ResourceTracker: + tracker = ResourceTracker(live_runtime) + yield tracker + tracker.cleanup() + + +@pytest.fixture(scope="session", autouse=True) +def require_gateway_version(live_runtime: LiveRuntime) -> str: + with live_runtime.client() as client: + version = client.health().version + expected = live_runtime.config.expected_gateway_version or importlib.metadata.version("openshell") + assert version == expected, "The OpenShell gateway and Python SDK versions must match exactly." + return version + + +@pytest.fixture +def provider_factory( + live_runtime: LiveRuntime, + resources: ResourceTracker, +) -> Callable[..., Any]: + from aiq_agent.agents.deep_researcher.sandbox import SandboxConfig + from aiq_agent.agents.deep_researcher.sandbox import create_sandbox_backend + + def create( + job_id: str, + emitter: Callable[[dict[str, object]], None], + *, + policy_path: Path | None = None, + shared_name: str | None = None, + ) -> Any: + openshell_config: dict[str, object] = { + "gateway": live_runtime.config.gateway, + "policy": str(policy_path or live_runtime.config.policy_path), + "image": live_runtime.config.image, + "require_hard_landlock": not live_runtime.config.allow_best_effort, + } + if shared_name is not None: + openshell_config.update(existing_sandbox_name=shared_name, allow_shared_sandbox=True) + config = SandboxConfig( + provider="openshell", + workdir="/sandbox", + network=live_runtime.network, + providers={"openshell": openshell_config}, + ) + provider = create_sandbox_backend(config, job_id) + resources.provider(provider, owns_remote=shared_name is None) + provider.set_event_emitter(emitter) + return provider + + return create + + +@pytest.fixture +def direct_sandbox_factory( + live_runtime: LiveRuntime, + resources: ResourceTracker, +) -> Callable[[str], str]: + """Create a directly owned sandbox and register it before readiness polling.""" + + def create(job_id: str) -> str: + with live_runtime.client() as client: + labels = {"aiq": "shared-debug-test"} + sandbox_ref = client.create( + spec=live_runtime.build_sandbox_spec( + policy=live_runtime.expected_policy, + image=live_runtime.config.image, + job_id=job_id, + labels=labels, + ), + labels=labels, + ) + name = resources.direct_sandbox(sandbox_ref.name) + client.wait_ready(name) + return name + + return create + + +def _attestation_success(events: list[dict[str, object]], runtime: LiveRuntime, authoritative_hash: str) -> bool: + for event in events: + if event.get("type") != "sandbox.attestation": + continue + data = event.get("data") + if not isinstance(data, dict) or data.get("status") != "succeeded": + continue + return ( + isinstance(data.get("policy_version"), int) + and data["policy_version"] > 0 + and data.get("policy_hash") == authoritative_hash + and data.get("policy_source") == runtime.sandbox_pb2.POLICY_SOURCE_SANDBOX + and data.get("assurance") == "strict" + and data.get("reason_code") is None + ) + return False + + +def _assert_authoritative_policy(runtime: LiveRuntime, client: Any, name: str) -> tuple[Any, int, str]: + sandbox = client.get(name) + stub = client._stub + status = stub.GetSandboxPolicyStatus( + runtime.openshell_pb2.GetSandboxPolicyStatusRequest(name=name, version=0), + timeout=30, + ) + config = stub.GetSandboxConfig( + runtime.sandbox_pb2.GetSandboxConfigRequest(sandbox_id=sandbox.id or name), + timeout=30, + ) + revision = status.revision + assert revision.status == runtime.openshell_pb2.POLICY_STATUS_LOADED + assert not revision.load_error + assert revision.version > 0 + assert config.version == revision.version + assert status.active_version == revision.version + assert sandbox.current_policy_version == revision.version + assert config.policy_source == runtime.sandbox_pb2.POLICY_SOURCE_SANDBOX + assert config.policy == runtime.expected_policy + assert revision.policy == runtime.expected_policy + assert config.policy_hash + assert revision.policy_hash == config.policy_hash + return sandbox, revision.version, config.policy_hash + + +def test_live_per_job_isolation_attestation_and_cancellation( + live_runtime: LiveRuntime, + provider_factory: Callable[..., Any], +) -> None: + suffix = uuid4().hex[:10] + events: tuple[list[dict[str, object]], list[dict[str, object]]] = ([], []) + providers = ( + provider_factory(f"aiq-isolation-a-{suffix}", events[0].append), + provider_factory(f"aiq-isolation-b-{suffix}", events[1].append), + ) + markers = (f"owner-a-{suffix}", f"owner-b-{suffix}") + + def initialize(index: int) -> str: + provider = providers[index] + marker_file = f"{provider.workdir}/owner.txt" + command = ( + f"printf %s {shlex.quote(markers[index])} > {shlex.quote(marker_file)}; cat {shlex.quote(marker_file)}" + ) + result = provider.execute(command, timeout=30) + assert result.exit_code == 0 + assert result.output.strip() == markers[index] + return provider.physical_sandbox_name + + with ThreadPoolExecutor(max_workers=2) as pool: + names = tuple(pool.map(initialize, range(2))) + assert names[0] != names[1] + + with live_runtime.client() as client: + selected = client.list(label_selector="aiq=deep-research") + selected_by_name = {item.name: item for item in selected} + assert set(names) <= selected_by_name.keys() + for index, name in enumerate(names): + assert dict(selected_by_name[name].labels) == { + "aiq": "deep-research", + "aiq-job-id": f"aiq-isolation-{'a' if index == 0 else 'b'}-{suffix}", + } + first, first_revision, first_hash = _assert_authoritative_policy(live_runtime, client, names[0]) + second, second_revision, second_hash = _assert_authoritative_policy(live_runtime, client, names[1]) + assert first.id != second.id + assert first_revision > 0 and second_revision > 0 + assert _attestation_success(events[0], live_runtime, first_hash) + assert _attestation_success(events[1], live_runtime, second_hash) + + providers[0].terminate() + _assert_deleted(live_runtime, client, names[0]) + selected_names = {item.name for item in client.list(label_selector="aiq=deep-research")} + assert names[0] not in selected_names + assert names[1] in selected_names + assert client.get(names[1]).id == second.id + result = providers[1].execute("printf %s still-alive", timeout=30) + assert result.exit_code == 0 + assert result.output.strip() == "still-alive" + + providers[1].close() + _assert_deleted(live_runtime, client, names[1]) + assert names[1] not in {item.name for item in client.list(label_selector="aiq=deep-research")} + + +def test_live_failure_cleanup_and_log_redaction( + live_runtime: LiveRuntime, + provider_factory: Callable[..., Any], + caplog: pytest.LogCaptureFixture, +) -> None: + suffix = uuid4().hex[:10] + secret_canary = f"credential=aiq-live-{suffix}" + events: list[dict[str, object]] = [] + + def fail_event_delivery(event: dict[str, object]) -> None: + events.append(event) + raise RuntimeError(secret_canary) + + caplog.set_level(logging.DEBUG) + provider = provider_factory(f"aiq-isolation-failure-{suffix}", fail_event_delivery) + result = provider.execute("exit 23", timeout=30) + name = provider.physical_sandbox_name + assert result.exit_code == 23 + provider.close() + + with live_runtime.client() as client: + _assert_deleted(live_runtime, client, name) + assert secret_canary not in caplog.text + assert secret_canary not in json.dumps(events, default=str) + + +def test_live_shared_policy_mismatch_is_rejected( + live_runtime: LiveRuntime, + provider_factory: Callable[..., Any], + direct_sandbox_factory: Callable[[str], str], + tmp_path: Path, +) -> None: + import yaml + + suffix = uuid4().hex[:10] + mismatch_data = copy.deepcopy(live_runtime.policy_data) + filesystem = mismatch_data.get("filesystem") + assert isinstance(filesystem, dict) + read_only = list(filesystem.get("read_only") or []) + read_only.append("/__aiq_attestation_mismatch__") + filesystem["read_only"] = read_only + mismatch_path = tmp_path / "mismatched-policy.yaml" + mismatch_path.write_text(yaml.safe_dump(mismatch_data, sort_keys=False), encoding="utf-8") + + shared_name = direct_sandbox_factory(f"aiq-shared-mismatch-{suffix}") + + mismatch_events: list[dict[str, object]] = [] + provider = provider_factory( + f"aiq-mismatched-attach-{suffix}", + mismatch_events.append, + policy_path=mismatch_path, + shared_name=shared_name, + ) + with pytest.raises(RuntimeError, match="policy_content_mismatch"): + provider.execute("true", timeout=30) + + assert not any( + event.get("type") == "sandbox.attestation" + and isinstance(event.get("data"), dict) + and event["data"].get("status") == "succeeded" + for event in mismatch_events + ) + import openshell + + with live_runtime.client() as client: + shared = client.get(shared_name) + assert shared.name == shared_name + with openshell.Sandbox( + cluster=live_runtime.config.gateway, + sandbox=shared_name, + delete_on_exit=False, + ) as shared_sandbox: + result = shared_sandbox.exec(["bash", "-c", "printf %s still-usable"], timeout_seconds=30) + assert result.exit_code == 0 + assert result.stdout.strip() == "still-usable" diff --git a/tests/aiq_agent/agents/deep_researcher/sandbox/test_openshell_provider.py b/tests/aiq_agent/agents/deep_researcher/sandbox/test_openshell_provider.py index d8a1fef7f..febe4d3e5 100644 --- a/tests/aiq_agent/agents/deep_researcher/sandbox/test_openshell_provider.py +++ b/tests/aiq_agent/agents/deep_researcher/sandbox/test_openshell_provider.py @@ -13,29 +13,34 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""OpenShell provider tests: the env-free upload/download shim. - -OpenShell 0.0.57's exec does not propagate ``env`` to the child process, which -breaks the official adapter's env-var file-transfer bootstraps. Our provider -overrides ``upload_files``/``download_files`` to pass the path via argv (and data -via stdin). These tests assert that contract with a fake sandbox, so they require -only the optional SDK + adapter to be importable. -""" +"""OpenShell provider tests: creation/attestation plus confined file transfer.""" from __future__ import annotations import base64 +import importlib.util +import sys +from contextlib import contextmanager from dataclasses import dataclass +from pathlib import Path +from threading import Event +from threading import Thread +from types import ModuleType +from types import SimpleNamespace from typing import Any from unittest.mock import MagicMock +from unittest.mock import patch import pytest -pytest.importorskip("openshell") -pytest.importorskip("langchain_nvidia_openshell") - -from aiq_agent.agents.deep_researcher.sandbox.config import SandboxConfig # noqa: E402 -from aiq_agent.agents.deep_researcher.sandbox.providers.openshell import OpenShellSandboxProvider # noqa: E402 +from aiq_agent.agents.deep_researcher.sandbox.base import SandboxProvider +from aiq_agent.agents.deep_researcher.sandbox.base import SandboxTerminatedError +from aiq_agent.agents.deep_researcher.sandbox.config import SandboxConfig +from aiq_agent.agents.deep_researcher.sandbox.providers.openshell import OpenShellSandboxProvider +from aiq_agent.agents.deep_researcher.sandbox.providers.openshell import _build_sandbox_spec +from aiq_agent.agents.deep_researcher.sandbox.providers.openshell import _parse_policy_proto +from aiq_agent.agents.deep_researcher.sandbox.providers.openshell import _read_policy_data +from aiq_agent.agents.deep_researcher.sandbox.providers.openshell import _validate_policy_network @dataclass @@ -45,6 +50,40 @@ class _ExecResult: stderr: str = "" +@dataclass(frozen=True) +class _FakeProtoPart: + """Small deterministic protobuf stand-in used without the optional SDK.""" + + payload: bytes + + def SerializeToString(self, *, deterministic: bool = False) -> bytes: # noqa: N802 - protobuf API + del deterministic + return self.payload + + +class _FakePolicy: + """Structural stand-in for the OpenShell SandboxPolicy protobuf.""" + + __hash__ = None + + def __init__(self, *, version: int = 1, network_policies: dict[str, _FakeProtoPart] | None = None) -> None: + self.version = version + self.filesystem = _FakeProtoPart(b"filesystem") + self.landlock = _FakeProtoPart(b"landlock") + self.process = _FakeProtoPart(b"process") + self.network_policies = network_policies or {} + + def HasField(self, field: str) -> bool: # noqa: N802 - protobuf API + return getattr(self, field, None) is not None + + def __eq__(self, other: object) -> bool: + return isinstance(other, _FakePolicy) and vars(self) == vars(other) + + +_FAKE_POLICY = _FakePolicy() +_FAKE_POLICY_HASH = "authoritative-openshell-hash" + + class _FakeOpenShellSandbox: """Records exec calls and returns scripted results.""" @@ -63,18 +102,969 @@ def __exit__(self, *_args: object) -> None: self.exit_calls += 1 -def _provider() -> OpenShellSandboxProvider: +def _provider(**openshell_config: Any) -> OpenShellSandboxProvider: cfg = SandboxConfig( provider="openshell", network={"mode": "blocked"}, - providers={"openshell": {"sandbox_name": "demo", "delete_on_exit": False}}, + providers={"openshell": openshell_config}, ) - provider = OpenShellSandboxProvider(cfg, "job-1") + # Construct through the provider-neutral base so these unit tests run even when + # the optional OpenShell SDK/adapter are absent from normal CI. + provider = object.__new__(OpenShellSandboxProvider) + SandboxProvider.__init__(provider, cfg, "job-1") + provider._os_context = None + provider._os_context_entering = False + provider._os_context_exit_requested = False + provider._os_context_cleanup_complete = None # Avoid real session creation: a non-None _session short-circuits _session_or_create. provider._session = MagicMock() # type: ignore[assignment] return provider +class _FakeCreatedContext(_FakeOpenShellSandbox): + """Entered SDK context with a public sandbox reference for attestation.""" + + def __init__( + self, + *, + phase: int = 2, + policy_version: int = 1, + name: str = "generated", + policy: _FakePolicy = _FAKE_POLICY, + policy_source: int = 1, + ) -> None: + super().__init__() + self.enter_calls = 0 + self.sandbox = SimpleNamespace( + id=f"{name}-id", + phase=phase, + current_policy_version=policy_version, + name=name, + ) + revision_version = policy_version if policy_version > 0 else 1 + revision = SimpleNamespace( + version=revision_version, + status=2, + load_error="", + policy_hash=_FAKE_POLICY_HASH, + policy=policy, + ) + status = SimpleNamespace(revision=revision, active_version=revision_version) + config = SimpleNamespace( + version=revision_version, + policy_hash=_FAKE_POLICY_HASH, + policy=policy, + policy_source=policy_source, + ) + self._client = SimpleNamespace( + get=MagicMock(return_value=self.sandbox), + health=MagicMock(return_value=SimpleNamespace(version="0.0.80")), + _stub=SimpleNamespace( + GetSandboxPolicyStatus=MagicMock(return_value=status), + GetSandboxConfig=MagicMock(return_value=config), + ), + ) + + def __enter__(self): + self.enter_calls += 1 + return self + + +class _FakeAdapter: + """Minimal langchain-nvidia-openshell adapter used by provider lifecycle tests.""" + + def __init__(self, *, sandbox: _FakeCreatedContext, timeout: int, shell: tuple[str, ...]) -> None: + self.id = sandbox.id + self.sandbox = sandbox + self.timeout = timeout + self.shell = shell + + +@contextmanager +def _fake_optional_modules(context_factory, adapter_cls: type = _FakeAdapter): + openshell_module = ModuleType("openshell") + openshell_module.Sandbox = context_factory # type: ignore[attr-defined] + adapter_module = ModuleType("langchain_nvidia_openshell") + adapter_module.OpenShellSandbox = adapter_cls # type: ignore[attr-defined] + proto_module = ModuleType("openshell._proto") + proto_module.openshell_pb2 = SimpleNamespace( # type: ignore[attr-defined] + SANDBOX_PHASE_READY=2, + SANDBOX_PHASE_ERROR=3, + POLICY_STATUS_UNSPECIFIED=0, + POLICY_STATUS_PENDING=1, + POLICY_STATUS_LOADED=2, + POLICY_STATUS_FAILED=3, + GetSandboxPolicyStatusRequest=lambda **kwargs: SimpleNamespace(**kwargs), + ) + proto_module.sandbox_pb2 = SimpleNamespace( # type: ignore[attr-defined] + POLICY_SOURCE_UNSPECIFIED=0, + POLICY_SOURCE_SANDBOX=1, + POLICY_SOURCE_GLOBAL=2, + GetSandboxConfigRequest=lambda **kwargs: SimpleNamespace(**kwargs), + ) + with patch.dict( + sys.modules, + { + "openshell": openshell_module, + "openshell._proto": proto_module, + "langchain_nvidia_openshell": adapter_module, + }, + ): + yield + + +def _write_policy(path: Path, *, compatibility: str = "hard_requirement", extra: str = "") -> None: + path.write_text( + "\n".join( + [ + "version: 1", + "filesystem_policy:", + " include_workdir: true", + " read_write: [/sandbox]", + "landlock:", + f" compatibility: {compatibility}", + "process:", + " run_as_user: sandbox", + " run_as_group: sandbox", + "network_policies: {}", + extra, + ] + ), + encoding="utf-8", + ) + + +def test_policy_reader_normalizes_filesystem_alias_and_requires_hard_landlock(tmp_path: Path) -> None: + policy_path = tmp_path / "policy.yaml" + _write_policy(policy_path) + + policy = _read_policy_data(str(policy_path), require_hard_landlock=True) + + assert "filesystem" in policy + assert "filesystem_policy" not in policy + assert policy["landlock"]["compatibility"] == "hard_requirement" + + +def test_policy_reader_rejects_best_effort_in_production(tmp_path: Path) -> None: + policy_path = tmp_path / "policy.yaml" + _write_policy(policy_path, compatibility="best_effort") + + with pytest.raises(ValueError, match="hard_requirement"): + _read_policy_data(str(policy_path), require_hard_landlock=True) + + +def test_policy_reader_rejects_missing_file_and_wrong_version(tmp_path: Path) -> None: + with pytest.raises(ValueError, match="Could not read"): + _read_policy_data(str(tmp_path / "missing.yaml"), require_hard_landlock=True) + + policy_path = tmp_path / "policy.yaml" + policy_path.write_text("version: 2\n", encoding="utf-8") + with pytest.raises(ValueError, match="exactly 1"): + _read_policy_data(str(policy_path), require_hard_landlock=True) + + +@pytest.mark.parametrize( + ("policy", "message"), + [ + ( + "version: 1\nlandlock: {compatibility: hard_requirement}\n" + "process: {run_as_user: sandbox, run_as_group: sandbox}\n", + "filesystem", + ), + ( + "version: 1\nfilesystem_policy: {read_write: [/sandbox]}\n" + "landlock: {compatibility: hard_requirement}\n" + "process: {run_as_user: root, run_as_group: sandbox}\n", + "non-root", + ), + ( + "version: 1\nfilesystem_policy: {read_write: [/sandbox]}\n" + "landlock: {compatibility: hard_requirement}\n" + "process: {run_as_user: 1000, run_as_group: sandbox}\n", + "non-empty string", + ), + ( + "version: 1\nfilesystem_policy: {read_write: [/sandbox]}\n" + "landlock: {compatibility: hard_requirement}\n" + "process: {run_as_user: sandbox, run_as_group: sandbox}\n" + "network_policies: {bad: {endpoints: [{host: example.com, enforcement: audit, access: read-write}]}}\n", + "enforcement=enforce", + ), + ], +) +def test_policy_reader_enforces_production_floor(tmp_path: Path, policy: str, message: str) -> None: + policy_path = tmp_path / "policy.yaml" + policy_path.write_text(policy, encoding="utf-8") + + with pytest.raises(ValueError, match=message): + _read_policy_data(str(policy_path), require_hard_landlock=True) + + +def test_checked_policy_matches_installed_sdk_schema() -> None: + if importlib.util.find_spec("openshell") is None: + pytest.skip("optional OpenShell SDK is not installed") + + policy_path = "configs/openshell/aiq-research-policy.yaml" + policy_data = _read_policy_data(policy_path, require_hard_landlock=True) + policy = _parse_policy_proto(policy_data, policy_path=policy_path) + + assert policy.version == 1 + + spec = _build_sandbox_spec(policy=policy, image="aiq:test", job_id="job-1") + assert spec.template.image == "aiq:test" + assert dict(spec.template.labels) == {"aiq": "deep-research", "aiq-job-id": "job-1"} + assert not spec.template.environment + assert not spec.environment + assert not spec.providers + + +def test_sandbox_spec_normalizes_long_job_id_label() -> None: + if importlib.util.find_spec("openshell") is None: + pytest.skip("optional OpenShell SDK is not installed") + from openshell._proto import sandbox_pb2 + + spec = _build_sandbox_spec( + policy=sandbox_pb2.SandboxPolicy(version=1), + image="aiq:test", + job_id="JOB_WITH_UNSAFE_CHARS_" * 8, + ) + job_label = spec.template.labels["aiq-job-id"] + assert len(job_label) <= 63 + assert job_label == job_label.lower() + assert "_" not in job_label + + +def test_policy_network_must_not_exceed_declared_allowlist() -> None: + policy = {"network_policies": {"github": {"endpoints": [{"host": "api.github.com"}, {"host": "github.com"}]}}} + + _validate_policy_network( + policy, + mode="allowlist", + allow=("api.github.com", "github.com"), + ) + with pytest.raises(ValueError, match="github.com"): + _validate_policy_network(policy, mode="allowlist", allow=("api.github.com",)) + with pytest.raises(ValueError, match="network endpoints"): + _validate_policy_network(policy, mode="blocked", allow=()) + + +@pytest.mark.parametrize("host", [None, "", " ", "."]) +def test_allowlist_rejects_hostless_endpoints(host: str | None) -> None: + endpoint: dict[str, object] = {"allowed_ips": ["0.0.0.0/0"]} + if host is not None: + endpoint["host"] = host + policy = {"network_policies": {"cidr": {"endpoints": [endpoint]}}} + + with pytest.raises(ValueError, match="non-empty host"): + _validate_policy_network(policy, mode="allowlist", allow=("api.github.com",)) + with pytest.raises(ValueError, match="network endpoints"): + _validate_policy_network(policy, mode="blocked", allow=()) + + +@pytest.mark.parametrize("cidr", ["0.0.0.0/0", "10.0.0.0/8", "::/0", "2001:db8::/32"]) +def test_allowlist_rejects_allowed_ip_overrides(cidr: str) -> None: + policy = { + "network_policies": { + "mixed": {"endpoints": [{"host": "api.github.com", "allowed_ips": [cidr]}]}, + } + } + + with pytest.raises(ValueError, match="CIDR"): + _validate_policy_network(policy, mode="allowlist", allow=("api.github.com",)) + _validate_policy_network(policy, mode="open", allow=()) + + +def test_per_job_session_uses_policy_spec_and_attests_before_return( + tmp_path: Path, + caplog: pytest.LogCaptureFixture, +) -> None: + policy_path = tmp_path / "policy.yaml" + _write_policy(policy_path) + created: list[tuple[_FakeCreatedContext, dict[str, Any]]] = [] + + def context_factory(**kwargs: Any) -> _FakeCreatedContext: + context = _FakeCreatedContext() + created.append((context, kwargs)) + return context + + with ( + _fake_optional_modules(context_factory), + patch( + "aiq_agent.agents.deep_researcher.sandbox.providers.openshell._parse_policy_proto", + return_value=_FAKE_POLICY, + ), + patch( + "aiq_agent.agents.deep_researcher.sandbox.providers.openshell._build_sandbox_spec", + return_value="job-spec", + ), + ): + provider = OpenShellSandboxProvider( + SandboxConfig( + provider="openshell", + providers={ + "openshell": { + "policy": str(policy_path), + "image": "aiq:test", + "gateway": "sensitive-gateway-name", + } + }, + ), + "job-123", + ) + with caplog.at_level("INFO"): + backend = provider._create_session() + + context, kwargs = created[0] + assert backend.id == context.id + assert context.enter_calls == 1 + assert kwargs["spec"] == "job-spec" + assert kwargs["labels"] == {"aiq": "deep-research", "aiq-job-id": "job-123"} + assert kwargs["delete_on_exit"] is True + assert "sandbox" not in kwargs + assert provider.physical_sandbox_name == "generated" + assert "sensitive-gateway-name" not in caplog.text + context._client._stub.GetSandboxPolicyStatus.assert_called_once() # type: ignore[attr-defined] + context._client._stub.GetSandboxConfig.assert_called_once() # type: ignore[attr-defined] + + +def test_per_job_session_fails_before_creation_when_request_labels_are_unsupported(tmp_path: Path) -> None: + policy_path = tmp_path / "policy.yaml" + _write_policy(policy_path) + + def context_factory( + *, + cluster: str | None, + ready_timeout_seconds: float, + spec: object, + delete_on_exit: bool, + ) -> _FakeCreatedContext: + del cluster, ready_timeout_seconds, spec, delete_on_exit + raise AssertionError("unsupported SDK must fail before creating a sandbox") + + with ( + _fake_optional_modules(context_factory), + patch( + "aiq_agent.agents.deep_researcher.sandbox.providers.openshell._parse_policy_proto", + return_value=_FAKE_POLICY, + ), + ): + provider = OpenShellSandboxProvider( + SandboxConfig(provider="openshell", providers={"openshell": {"policy": str(policy_path)}}), + "job-123", + ) + with pytest.raises(RuntimeError, match="request_labels_unsupported"): + provider._create_session() + + assert provider._os_context is None + + +def test_two_jobs_create_distinct_specs_without_named_attachment(tmp_path: Path) -> None: + policy_path = tmp_path / "policy.yaml" + _write_policy(policy_path) + created: list[dict[str, Any]] = [] + spec_jobs: list[str] = [] + + def context_factory(**kwargs: Any) -> _FakeCreatedContext: + created.append(kwargs) + return _FakeCreatedContext(name=f"generated-{len(created)}") + + def build_spec(*, policy: Any, image: str, job_id: str, labels: dict[str, str]) -> str: + del policy, image + assert labels == {"aiq": "deep-research", "aiq-job-id": job_id} + spec_jobs.append(job_id) + return f"spec-{job_id}" + + with ( + _fake_optional_modules(context_factory), + patch( + "aiq_agent.agents.deep_researcher.sandbox.providers.openshell._parse_policy_proto", + return_value=_FAKE_POLICY, + ), + patch( + "aiq_agent.agents.deep_researcher.sandbox.providers.openshell._build_sandbox_spec", + side_effect=build_spec, + ), + ): + for job_id in ("job-a", "job-b"): + provider = OpenShellSandboxProvider( + SandboxConfig( + provider="openshell", + providers={"openshell": {"policy": str(policy_path), "image": "aiq:test"}}, + ), + job_id, + ) + provider._create_session() + + assert spec_jobs == ["job-a", "job-b"] + assert [kwargs["spec"] for kwargs in created] == ["spec-job-a", "spec-job-b"] + assert all("sandbox" not in kwargs for kwargs in created) + + +@pytest.mark.parametrize( + ("phase", "policy_version", "expected", "message"), + [ + (3, 1, None, "not_ready"), + (2, 2, 1, "expected_version_mismatch"), + ], +) +def test_attestation_failure_deletes_before_raising( + tmp_path: Path, + phase: int, + policy_version: int, + expected: int | None, + message: str, +) -> None: + policy_path = tmp_path / "policy.yaml" + _write_policy(policy_path) + context = _FakeCreatedContext(phase=phase, policy_version=policy_version) + + with ( + _fake_optional_modules(lambda **_kwargs: context), + patch( + "aiq_agent.agents.deep_researcher.sandbox.providers.openshell._parse_policy_proto", + return_value=_FAKE_POLICY, + ), + patch( + "aiq_agent.agents.deep_researcher.sandbox.providers.openshell._build_sandbox_spec", + return_value="job-spec", + ), + ): + provider = OpenShellSandboxProvider( + SandboxConfig( + provider="openshell", + providers={ + "openshell": { + "policy": str(policy_path), + "expected_policy_version": expected, + } + }, + ), + "job-123", + ) + with pytest.raises(RuntimeError, match=message): + provider._create_session() + + assert context.exit_calls == 1 + assert provider._os_context is None + + +def test_attestation_reads_authoritative_policy_before_success(tmp_path: Path) -> None: + policy_path = tmp_path / "policy.yaml" + _write_policy(policy_path) + context = _FakeCreatedContext(policy_version=1) + policy_status = context._client._stub.GetSandboxPolicyStatus # type: ignore[attr-defined] + sandbox_config = context._client._stub.GetSandboxConfig # type: ignore[attr-defined] + events: list[dict[str, object]] = [] + + with ( + _fake_optional_modules(lambda **_kwargs: context), + patch( + "aiq_agent.agents.deep_researcher.sandbox.providers.openshell._parse_policy_proto", + return_value=_FAKE_POLICY, + ), + patch( + "aiq_agent.agents.deep_researcher.sandbox.providers.openshell._build_sandbox_spec", + return_value="job-spec", + ), + ): + provider = OpenShellSandboxProvider( + SandboxConfig(provider="openshell", providers={"openshell": {"policy": str(policy_path)}}), + "job-123", + ) + provider.set_event_emitter(events.append) + + provider._create_session() + + request = policy_status.call_args.args[0] + assert request.name == context.sandbox.name + assert request.version == 0 + config_request = sandbox_config.call_args.args[0] + assert config_request.sandbox_id == context.sandbox.id + succeeded = [event for event in events if event["data"]["status"] == "succeeded"] # type: ignore[index] + assert len(succeeded) == 1 + assert succeeded[0]["data"]["policy_version"] == 1 # type: ignore[index] + assert succeeded[0]["data"]["policy_hash"] == _FAKE_POLICY_HASH # type: ignore[index] + assert succeeded[0]["data"]["policy_source"] == 1 # type: ignore[index] + assert succeeded[0]["data"]["assurance"] == "strict" # type: ignore[index] + + +def test_attestation_polls_pending_revision_until_loaded() -> None: + context = _FakeCreatedContext(policy_version=1) + status_rpc = context._client._stub.GetSandboxPolicyStatus # type: ignore[attr-defined] + loaded = status_rpc.return_value + pending = SimpleNamespace( + revision=SimpleNamespace(**{**vars(loaded.revision), "status": 1}), + active_version=0, + ) + status_rpc.side_effect = [pending, loaded] + + result = _run_attestation(context, policy_load_timeout_seconds=1.0) + + assert result.policy_version == 1 + assert status_rpc.call_count == 2 + assert context._client._stub.GetSandboxConfig.call_count == 2 # type: ignore[attr-defined] + + +def test_attestation_accepts_zero_initial_version_after_authoritative_transition() -> None: + context = _FakeCreatedContext(policy_version=0) + refreshed = SimpleNamespace(**vars(context.sandbox)) + refreshed.current_policy_version = 1 + context._client.get.return_value = refreshed # type: ignore[attr-defined] + status_rpc = context._client._stub.GetSandboxPolicyStatus # type: ignore[attr-defined] + loaded = status_rpc.return_value + pending = SimpleNamespace( + revision=SimpleNamespace(**{**vars(loaded.revision), "status": 1}), + active_version=0, + ) + status_rpc.side_effect = [pending, loaded] + + result = _run_attestation(context, policy_load_timeout_seconds=1.0) + + assert result.policy_version == 1 + assert status_rpc.call_count == 2 + assert context._client.get.call_count == 2 # type: ignore[attr-defined] + + +def test_attestation_classifies_effective_policy_that_remains_pending() -> None: + context = _FakeCreatedContext(policy_version=0) + status = context._client._stub.GetSandboxPolicyStatus.return_value # type: ignore[attr-defined] + status.revision.status = 1 + status.active_version = 0 + + with pytest.raises(RuntimeError, match="policy_status_inconsistent"): + _run_attestation(context, policy_load_timeout_seconds=0.001) + + +def _run_attestation( + context: _FakeCreatedContext, + *, + expected_policy: _FakePolicy | None = _FAKE_POLICY, + require_sandbox_source: bool = True, + **openshell_config: Any, +): + with _fake_optional_modules(lambda **_kwargs: context): + provider = _provider(**openshell_config) + provider._os_context = context + return provider._attest( + context, + expected_policy=expected_policy, + require_sandbox_source=require_sandbox_source, + ) + + +def test_attestation_fails_closed_when_authoritative_rpc_is_unavailable() -> None: + context = _FakeCreatedContext() + del context._client._stub.GetSandboxConfig # type: ignore[attr-defined] + + with pytest.raises(RuntimeError, match="rpc_unavailable"): + _run_attestation(context) + + +@pytest.mark.parametrize("field", ["active_version", "revision_version", "config_version", "current_version"]) +def test_attestation_rejects_version_disagreement(field: str) -> None: + context = _FakeCreatedContext() + status = context._client._stub.GetSandboxPolicyStatus.return_value # type: ignore[attr-defined] + config = context._client._stub.GetSandboxConfig.return_value # type: ignore[attr-defined] + if field == "active_version": + status.active_version = 2 + elif field == "revision_version": + status.revision.version = 2 + elif field == "config_version": + config.version = 2 + else: + context.sandbox.current_policy_version = 2 + + with pytest.raises(RuntimeError, match="version_mismatch"): + _run_attestation(context) + + +def test_attestation_rejects_unreported_current_and_active_versions() -> None: + context = _FakeCreatedContext(policy_version=0) + status = context._client._stub.GetSandboxPolicyStatus.return_value # type: ignore[attr-defined] + status.active_version = 0 + + with pytest.raises(RuntimeError, match="version_mismatch"): + _run_attestation(context) + + +def test_attestation_rejects_unreported_active_version_with_matching_current_version() -> None: + context = _FakeCreatedContext(policy_version=1) + status = context._client._stub.GetSandboxPolicyStatus.return_value # type: ignore[attr-defined] + status.active_version = 0 + + with pytest.raises(RuntimeError, match="version_mismatch"): + _run_attestation(context) + + +@pytest.mark.parametrize( + ("status", "load_error", "reason"), + [(3, "", "policy_status_failed"), (2, "credential=do-not-log", "policy_load_error")], +) +def test_attestation_rejects_failed_policy_revision(status: int, load_error: str, reason: str) -> None: + context = _FakeCreatedContext() + revision = context._client._stub.GetSandboxPolicyStatus.return_value.revision # type: ignore[attr-defined] + revision.status = status + revision.load_error = load_error + + with pytest.raises(RuntimeError, match=reason) as exc_info: + _run_attestation(context) + assert "credential=do-not-log" not in str(exc_info.value) + + +@pytest.mark.parametrize("surface", ["config", "revision"]) +def test_attestation_rejects_effective_policy_content_mismatch(surface: str) -> None: + context = _FakeCreatedContext() + if surface == "config": + context._client._stub.GetSandboxConfig.return_value.policy = _FakePolicy(version=2) # type: ignore[attr-defined] + else: + status = context._client._stub.GetSandboxPolicyStatus.return_value # type: ignore[attr-defined] + status.revision.policy = _FakePolicy(version=2) + + with pytest.raises(RuntimeError, match="policy_content_mismatch"): + _run_attestation(context) + + +@pytest.mark.parametrize("surface", ["config", "revision"]) +def test_attestation_rejects_policy_hash_mismatch(surface: str) -> None: + context = _FakeCreatedContext() + if surface == "config": + context._client._stub.GetSandboxConfig.return_value.policy_hash = "bad" # type: ignore[attr-defined] + else: + status = context._client._stub.GetSandboxPolicyStatus.return_value # type: ignore[attr-defined] + status.revision.policy_hash = "bad" + + with pytest.raises(RuntimeError, match="policy_hash_mismatch"): + _run_attestation(context) + + +@pytest.mark.parametrize("surface", ["config", "revision"]) +def test_attestation_rejects_missing_authoritative_policy_hash(surface: str) -> None: + context = _FakeCreatedContext() + if surface == "config": + context._client._stub.GetSandboxConfig.return_value.policy_hash = "" # type: ignore[attr-defined] + else: + status = context._client._stub.GetSandboxPolicyStatus.return_value # type: ignore[attr-defined] + status.revision.policy_hash = "" + + with pytest.raises(RuntimeError, match="policy_hash_missing"): + _run_attestation(context) + + +def test_attestation_rejects_gateway_global_policy_for_owned_sandbox() -> None: + context = _FakeCreatedContext(policy_source=2) + + with pytest.raises(RuntimeError, match="policy_source_mismatch"): + _run_attestation(context) + + +def test_policy_configured_shared_debug_accepts_matching_global_effective_policy() -> None: + context = _FakeCreatedContext(policy_source=2) + + result = _run_attestation(context, require_sandbox_source=False) + + assert result.assurance == "strict" + assert result.policy_source == 2 + + +def test_shared_debug_without_policy_reports_reduced_assurance() -> None: + context = _FakeCreatedContext(policy_source=2) + + result = _run_attestation(context, expected_policy=None, require_sandbox_source=False) + + assert result.assurance == "reduced" + assert result.policy_version == 1 + + +def test_shared_attachment_requires_explicit_debug_opt_in() -> None: + with pytest.raises(ValueError, match="allow_shared_sandbox"): + SandboxConfig(provider="openshell", providers={"openshell": {"existing_sandbox_name": "shared"}}) + + config = SandboxConfig( + provider="openshell", + providers={ + "openshell": { + "existing_sandbox_name": "shared", + "allow_shared_sandbox": True, + "delete_on_exit": False, + } + }, + ) + assert config.providers.openshell.shared_sandbox_name == "shared" + + with pytest.raises(ValueError, match="requires attest=true"): + SandboxConfig( + provider="openshell", + providers={ + "openshell": { + "existing_sandbox_name": "shared", + "allow_shared_sandbox": True, + "policy": "policy.yaml", + "attest": False, + } + }, + ) + + +def test_shared_attachment_never_deletes_unowned_sandbox() -> None: + created: list[dict[str, Any]] = [] + + def context_factory(**kwargs: Any) -> _FakeCreatedContext: + created.append(kwargs) + return _FakeCreatedContext(name="shared") + + with _fake_optional_modules(context_factory): + provider = OpenShellSandboxProvider( + SandboxConfig( + provider="openshell", + providers={ + "openshell": { + "existing_sandbox_name": "shared", + "allow_shared_sandbox": True, + "delete_on_exit": True, + } + }, + ), + "job-123", + ) + provider._create_session() + provider.close() + + assert created[0]["sandbox"] == "shared" + assert created[0]["delete_on_exit"] is False + assert "spec" not in created[0] + + +def test_per_job_mode_rejects_disabled_attestation() -> None: + with pytest.raises(ValueError, match="attest=true"): + SandboxConfig( + provider="openshell", + providers={"openshell": {"policy": "policy.yaml", "attest": False}}, + ) + + with pytest.raises(ValueError, match="delete_on_exit=true"): + SandboxConfig( + provider="openshell", + providers={"openshell": {"policy": "policy.yaml", "delete_on_exit": False}}, + ) + + +def test_adapter_construction_failure_deletes_sandbox(tmp_path: Path) -> None: + policy_path = tmp_path / "policy.yaml" + _write_policy(policy_path) + context = _FakeCreatedContext() + + class FailingAdapter: + def __init__(self, **_kwargs: Any) -> None: + raise ValueError("bad adapter configuration") + + with ( + _fake_optional_modules(lambda **_kwargs: context, FailingAdapter), + patch( + "aiq_agent.agents.deep_researcher.sandbox.providers.openshell._parse_policy_proto", + return_value=_FAKE_POLICY, + ), + patch( + "aiq_agent.agents.deep_researcher.sandbox.providers.openshell._build_sandbox_spec", + return_value="job-spec", + ), + ): + provider = OpenShellSandboxProvider( + SandboxConfig(provider="openshell", providers={"openshell": {"policy": str(policy_path)}}), + "job-123", + ) + events: list[dict[str, object]] = [] + provider.set_event_emitter(events.append) + with pytest.raises(ValueError, match="bad adapter"): + provider._create_session() + + assert context.exit_calls == 1 + assert provider._os_context is None + assert not any(event["data"]["status"] == "succeeded" for event in events) # type: ignore[index] + + +def test_attestation_success_is_emitted_after_adapter_construction(tmp_path: Path) -> None: + policy_path = tmp_path / "policy.yaml" + _write_policy(policy_path) + context = _FakeCreatedContext() + order: list[str] = [] + + class RecordingAdapter(_FakeAdapter): + def __init__(self, **kwargs: Any) -> None: + super().__init__(**kwargs) + order.append("adapter") + + with ( + _fake_optional_modules(lambda **_kwargs: context, RecordingAdapter), + patch( + "aiq_agent.agents.deep_researcher.sandbox.providers.openshell._parse_policy_proto", + return_value=_FAKE_POLICY, + ), + patch( + "aiq_agent.agents.deep_researcher.sandbox.providers.openshell._build_sandbox_spec", + return_value="job-spec", + ), + ): + provider = OpenShellSandboxProvider( + SandboxConfig(provider="openshell", providers={"openshell": {"policy": str(policy_path)}}), + "job-123", + ) + provider.set_event_emitter(lambda event: order.append(event["data"]["status"])) # type: ignore[index] + provider._create_session() + + assert order == ["adapter", "succeeded"] + + +def test_terminate_before_context_entry_prevents_creation(tmp_path: Path) -> None: + policy_path = tmp_path / "policy.yaml" + _write_policy(policy_path) + context = _FakeCreatedContext() + + class UnexpectedAdapter: + def __init__(self, **_kwargs: Any) -> None: + raise AssertionError("adapter must not be constructed after termination") + + with ( + _fake_optional_modules(lambda **_kwargs: context, UnexpectedAdapter), + patch( + "aiq_agent.agents.deep_researcher.sandbox.providers.openshell._parse_policy_proto", + return_value=_FAKE_POLICY, + ), + patch( + "aiq_agent.agents.deep_researcher.sandbox.providers.openshell._build_sandbox_spec", + return_value="job-spec", + ), + ): + provider = OpenShellSandboxProvider( + SandboxConfig(provider="openshell", providers={"openshell": {"policy": str(policy_path)}}), + "job-123", + ) + provider.terminate() + + with pytest.raises(SandboxTerminatedError): + provider._create_session() + + assert context.enter_calls == 0 + assert context.exit_calls == 0 + + +def test_terminate_during_context_entry_defers_one_exit(tmp_path: Path) -> None: + policy_path = tmp_path / "policy.yaml" + _write_policy(policy_path) + entry_started = Event() + allow_entry = Event() + + class BlockingContext(_FakeCreatedContext): + def __enter__(self): + self.enter_calls += 1 + entry_started.set() + if not allow_entry.wait(timeout=2): + raise AssertionError("context entry was not released") + return self + + class UnexpectedAdapter: + def __init__(self, **_kwargs: Any) -> None: + raise AssertionError("adapter must not be constructed after termination") + + context = BlockingContext() + errors: list[BaseException] = [] + + with ( + _fake_optional_modules(lambda **_kwargs: context, UnexpectedAdapter), + patch( + "aiq_agent.agents.deep_researcher.sandbox.providers.openshell._parse_policy_proto", + return_value=_FAKE_POLICY, + ), + patch( + "aiq_agent.agents.deep_researcher.sandbox.providers.openshell._build_sandbox_spec", + return_value="job-spec", + ), + ): + provider = OpenShellSandboxProvider( + SandboxConfig(provider="openshell", providers={"openshell": {"policy": str(policy_path)}}), + "job-123", + ) + + def create() -> None: + try: + provider._session_or_create() + except BaseException as exc: # noqa: BLE001 - capture the worker outcome for assertion + errors.append(exc) + + worker = Thread(target=create) + worker.start() + assert entry_started.wait(timeout=2) + + terminate_done = Event() + terminator = Thread(target=lambda: (provider.terminate(), terminate_done.set())) + terminator.start() + assert not terminate_done.wait(timeout=0.05) + assert context.exit_calls == 0 + allow_entry.set() + worker.join(timeout=2) + terminator.join(timeout=2) + + assert not worker.is_alive() + assert not terminator.is_alive() + assert terminate_done.is_set() + assert len(errors) == 1 and isinstance(errors[0], SandboxTerminatedError) + assert context.exit_calls == 1 + assert provider._session is None + + +def test_terminate_during_adapter_construction_cannot_publish_session(tmp_path: Path) -> None: + policy_path = tmp_path / "policy.yaml" + _write_policy(policy_path) + adapter_started = Event() + allow_adapter = Event() + context = _FakeCreatedContext() + errors: list[BaseException] = [] + events: list[dict[str, object]] = [] + + class BlockingAdapter(_FakeAdapter): + def __init__(self, **kwargs: Any) -> None: + adapter_started.set() + if not allow_adapter.wait(timeout=2): + raise AssertionError("adapter construction was not released") + super().__init__(**kwargs) + + with ( + _fake_optional_modules(lambda **_kwargs: context, BlockingAdapter), + patch( + "aiq_agent.agents.deep_researcher.sandbox.providers.openshell._parse_policy_proto", + return_value=_FAKE_POLICY, + ), + patch( + "aiq_agent.agents.deep_researcher.sandbox.providers.openshell._build_sandbox_spec", + return_value="job-spec", + ), + ): + provider = OpenShellSandboxProvider( + SandboxConfig(provider="openshell", providers={"openshell": {"policy": str(policy_path)}}), + "job-123", + ) + provider.set_event_emitter(events.append) + + def create() -> None: + try: + provider._session_or_create() + except BaseException as exc: # noqa: BLE001 - capture the worker outcome for assertion + errors.append(exc) + + worker = Thread(target=create) + worker.start() + assert adapter_started.wait(timeout=2) + + provider.terminate() + allow_adapter.set() + worker.join(timeout=2) + + assert not worker.is_alive() + assert len(errors) == 1 and isinstance(errors[0], SandboxTerminatedError) + assert context.exit_calls == 1 + assert provider._session is None + assert not any(event["data"]["status"] == "succeeded" for event in events) # type: ignore[index] + + def test_upload_passes_path_via_argv_and_data_via_stdin_no_env() -> None: provider = _provider() fake = _FakeOpenShellSandbox() @@ -195,3 +1185,173 @@ def test_terminate_exits_openshell_context_once() -> None: assert fake.exit_calls == 1 assert provider._os_context is None + + +def test_context_exit_failure_marks_cleanup_failed() -> None: + provider = _provider() + + class FailingContext: + def __exit__(self, *_args: object) -> None: + raise RuntimeError("gateway delete failed") + + provider._os_context = FailingContext() + + provider.close() + + assert provider.cleanup_succeeded is False + assert provider.cleanup_failure_reason_codes == ("context_exit_failed",) + assert provider._os_context is None + + +@pytest.mark.parametrize("timeout", [0, -1, float("inf"), float("nan")]) +def test_cleanup_timeout_must_be_positive_and_finite(timeout: float) -> None: + with pytest.raises(ValueError, match="cleanup_timeout_seconds"): + SandboxConfig(provider="openshell", providers={"openshell": {"cleanup_timeout_seconds": timeout}}) + + +@pytest.mark.parametrize("timeout", [0, -1, float("inf"), float("nan")]) +def test_policy_load_timeout_must_be_positive_and_finite(timeout: float) -> None: + with pytest.raises(ValueError, match="policy_load_timeout_seconds"): + SandboxConfig(provider="openshell", providers={"openshell": {"policy_load_timeout_seconds": timeout}}) + + +@pytest.mark.parametrize("timeout", [0, -1, float("inf"), float("nan")]) +def test_ready_timeout_must_be_positive_and_finite(timeout: float) -> None: + with pytest.raises(ValueError, match="ready_timeout_seconds"): + SandboxConfig(provider="openshell", providers={"openshell": {"ready_timeout_seconds": timeout}}) + + +def test_deferred_context_cleanup_timeout_is_terminal() -> None: + provider = _provider(cleanup_timeout_seconds=0.01) + provider._os_context = _FakeOpenShellSandbox() + provider._os_context_entering = True + provider._os_context_cleanup_complete = Event() + + provider._exit_context() + + assert provider.cleanup_succeeded is False + assert provider.cleanup_failure_reason_codes == ("cleanup_timeout",) + assert provider._os_context_exit_requested is True + + +def test_finalize_waits_for_deferred_exit_failure( + tmp_path: Path, + caplog: pytest.LogCaptureFixture, +) -> None: + from aiq_agent.agents.deep_researcher.deepagents_runtime import DeepAgentsRuntime + from aiq_agent.agents.deep_researcher.deepagents_runtime import DeepResearchSandboxConfig + + policy_path = tmp_path / "policy.yaml" + _write_policy(policy_path) + entry_started = Event() + allow_entry = Event() + + class FailingExitContext(_FakeCreatedContext): + def __enter__(self): + self.enter_calls += 1 + entry_started.set() + if not allow_entry.wait(timeout=2): + raise AssertionError("context entry was not released") + return self + + def __exit__(self, *_args: object) -> None: + self.exit_calls += 1 + raise RuntimeError("credential=do-not-log") + + context = FailingExitContext() + errors: list[BaseException] = [] + finalize_results: list[bool] = [] + events: list[dict[str, object]] = [] + + with ( + _fake_optional_modules(lambda **_kwargs: context), + patch( + "aiq_agent.agents.deep_researcher.sandbox.providers.openshell._parse_policy_proto", + return_value=_FAKE_POLICY, + ), + patch( + "aiq_agent.agents.deep_researcher.sandbox.providers.openshell._build_sandbox_spec", + return_value="job-spec", + ), + ): + provider = OpenShellSandboxProvider( + SandboxConfig( + provider="openshell", + providers={"openshell": {"policy": str(policy_path), "cleanup_timeout_seconds": 1}}, + ), + "job-123", + ) + with patch( + "aiq_agent.agents.deep_researcher.deepagents_runtime._create_sandbox_backend", + return_value=provider, + ): + runtime = DeepAgentsRuntime(sandbox=DeepResearchSandboxConfig(), artifact_emit=events.append) + + def create() -> None: + try: + provider._session_or_create() + except BaseException as exc: # noqa: BLE001 - capture worker outcome + errors.append(exc) + + creator = Thread(target=create) + creator.start() + assert entry_started.wait(timeout=2) + + finalizer = Thread(target=lambda: finalize_results.append(runtime.finalize(interrupted=True))) + with caplog.at_level("WARNING"): + finalizer.start() + assert finalizer.is_alive() + allow_entry.set() + creator.join(timeout=2) + finalizer.join(timeout=2) + + assert not creator.is_alive() + assert not finalizer.is_alive() + assert finalize_results == [False] + assert len(errors) == 1 and isinstance(errors[0], SandboxTerminatedError) + assert context.exit_calls == 1 + assert provider.cleanup_failure_reason_codes == ("context_exit_failed",) + assert [event["data"]["status"] for event in events] == ["started", "failed"] # type: ignore[index] + assert events[-1]["data"]["reason_codes"] == ["context_exit_failed"] # type: ignore[index] + assert "RuntimeError" in caplog.text + assert "credential=do-not-log" not in caplog.text + + +def test_retry_cleanup_failure_remains_terminal_failure() -> None: + from aiq_agent.agents.deep_researcher.deepagents_runtime import DeepAgentsRuntime + from aiq_agent.agents.deep_researcher.deepagents_runtime import DeepResearchSandboxConfig + + provider = _provider() + stale = MagicMock() + stale.close.side_effect = RuntimeError("stale sandbox close failed") + replacement = MagicMock() + provider._session = stale + provider._create_session = MagicMock(return_value=replacement) # type: ignore[method-assign] + provider._prepare_workspace = MagicMock() # type: ignore[method-assign] + events: list[dict[str, object]] = [] + + with patch( + "aiq_agent.agents.deep_researcher.deepagents_runtime._create_sandbox_backend", + return_value=provider, + ): + runtime = DeepAgentsRuntime( + sandbox=DeepResearchSandboxConfig(), + artifact_emit=events.append, + ) + + provider._reset_session() + + assert runtime.finalize(interrupted=False) is False + replacement.close.assert_called_once_with() + assert provider.cleanup_succeeded is False + assert [event["data"]["status"] for event in events] == ["started", "failed"] # type: ignore[index] + + +def test_sample_policy_grants_broad_proc_not_proc_self() -> None: + """OpenShell requires read access to /proc (not just /proc/self); guard against regression.""" + import yaml + + policy = yaml.safe_load(Path("configs/openshell/aiq-research-policy.yaml").read_text(encoding="utf-8")) + read_only = policy["filesystem_policy"]["read_only"] + assert "/proc" in read_only + assert "/proc/self" not in read_only diff --git a/tests/aiq_agent/agents/deep_researcher/sandbox/test_sandbox_runtime.py b/tests/aiq_agent/agents/deep_researcher/sandbox/test_sandbox_runtime.py index e20599b55..d41130320 100644 --- a/tests/aiq_agent/agents/deep_researcher/sandbox/test_sandbox_runtime.py +++ b/tests/aiq_agent/agents/deep_researcher/sandbox/test_sandbox_runtime.py @@ -22,6 +22,7 @@ from __future__ import annotations +import logging from typing import Any from unittest.mock import MagicMock @@ -250,24 +251,41 @@ def _fake_entry_points(*, group: str) -> list[Any]: finally: registry._SANDBOX_PROVIDERS.pop("ep-fake", None) - def test_broken_entry_point_is_skipped(self, monkeypatch: Any) -> None: + def test_broken_entry_point_is_skipped(self, monkeypatch: Any, caplog: pytest.LogCaptureFixture) -> None: from aiq_agent.agents.deep_researcher.sandbox import registry class _BrokenEntryPoint: name = "broken" def load(self) -> type[SandboxProvider]: - raise RuntimeError("boom") + raise RuntimeError("credential=do-not-log") monkeypatch.setattr(registry, "_entry_points_loaded", False) monkeypatch.setattr("importlib.metadata.entry_points", lambda *, group: [_BrokenEntryPoint()]) # Must not raise; built-in resolution stays intact. - registry._load_entry_point_providers() + with caplog.at_level(logging.WARNING): + registry._load_entry_point_providers() assert "broken" not in registry._SANDBOX_PROVIDERS assert "modal" in registered_providers() + assert "RuntimeError" in caplog.text + assert "credential=do-not-log" not in caplog.text class TestProviderLifecycle: + def test_event_emitter_failure_is_sanitized(self, caplog: pytest.LogCaptureFixture) -> None: + provider = _RegisteredFake(_fake_config(), "job-1") + + def fail(_event: dict[str, object]) -> None: + raise RuntimeError("credential=do-not-log") + + provider.set_event_emitter(fail) + with caplog.at_level(logging.WARNING): + provider._emit_event({"type": "test"}) + + assert "event_emit_failed" in caplog.text + assert "RuntimeError" in caplog.text + assert "credential=do-not-log" not in caplog.text + def test_session_created_lazily(self) -> None: session = MagicMock() provider = _ScriptedProvider(_fake_config(), "job-1", sessions=[session]) @@ -326,6 +344,19 @@ def test_close_releases_session(self) -> None: session.close.assert_called_once() assert provider._session is None + def test_session_close_failure_is_sanitized(self, caplog: pytest.LogCaptureFixture) -> None: + session = MagicMock() + session.close.side_effect = RuntimeError("credential=do-not-log") + provider = _ScriptedProvider(_fake_config(), "job-1", sessions=[session]) + provider._session = session + + with caplog.at_level(logging.WARNING): + provider.close() + + assert provider.cleanup_failure_reason_codes == ("session_close_failed",) + assert "RuntimeError" in caplog.text + assert "credential=do-not-log" not in caplog.text + def test_terminate_releases_session_and_blocks_further_ops(self) -> None: session = MagicMock() provider = _ScriptedProvider(_fake_config(), "job-1", sessions=[session]) @@ -421,11 +452,15 @@ def test_workspace_created_on_session_start(self) -> None: assert "/sandbox/job-xyz" in mkdir_cmd assert "/sandbox/job-xyz/aiq-artifacts" in mkdir_cmd - def test_workspace_prep_failure_does_not_abort_the_call(self) -> None: + def test_workspace_prep_failure_does_not_abort_the_call(self, caplog: pytest.LogCaptureFixture) -> None: # Best-effort: a mkdir failure is swallowed so it cannot break session creation; # a real filesystem problem resurfaces on the first actual write. session = MagicMock() - session.execute.side_effect = [RuntimeError("mkdir boom"), "ok"] + session.execute.side_effect = [RuntimeError("credential=do-not-log"), "ok"] provider = _WorkspaceProvider(_fake_config(workdir="/sandbox"), "job-1", session) - assert provider.execute("echo hi") == "ok" + with caplog.at_level(logging.WARNING): + assert provider.execute("echo hi") == "ok" + + assert "workspace_prepare_failed" in caplog.text + assert "credential=do-not-log" not in caplog.text diff --git a/tests/aiq_agent/agents/deep_researcher/test_agent.py b/tests/aiq_agent/agents/deep_researcher/test_agent.py index 421e3ecdf..e264a660f 100644 --- a/tests/aiq_agent/agents/deep_researcher/test_agent.py +++ b/tests/aiq_agent/agents/deep_researcher/test_agent.py @@ -294,6 +294,52 @@ def test_register_resolves_named_runtime_config_refs(self): assert resolved_skills is skills assert resolved_sandbox is sandbox + @pytest.mark.parametrize("owns_active_agent", [False, True]) + @pytest.mark.asyncio + async def test_registered_run_cancellation_finalizes_only_request_owned_agent(self, owns_active_agent): + """Cancellation is re-raised and only a request-scoped agent owns terminal cleanup.""" + from aiq_agent.agents.deep_researcher.deepagents_runtime import DeepResearchSandboxConfig + from aiq_agent.agents.deep_researcher.register import DeepResearchAgentConfig + from aiq_agent.agents.deep_researcher.register import deep_research_agent + + builder = MagicMock() + builder.get_tools = AsyncMock(return_value=[web_search_tool]) + builder.get_llm = AsyncMock(return_value=MagicMock()) + template_agent = MagicMock() + request_agent = MagicMock() + active_agent = request_agent if owns_active_agent else template_agent + active_agent.run = AsyncMock(side_effect=asyncio.CancelledError()) + agents = [template_agent, request_agent] if owns_active_agent else [template_agent] + config = DeepResearchAgentConfig( + orchestrator_llm="llm", + tools=["web_search_tool"], + verbose=False, + sandbox=DeepResearchSandboxConfig() if owns_active_agent else None, + ) + state = DeepResearchAgentState(messages=[HumanMessage(content="cancel this request")]) + + with ( + patch( + "aiq_agent.agents.deep_researcher.register.DeepResearcherAgent", + side_effect=agents, + ), + patch("aiq_agent.common.validate_tool_availability", return_value=(True, 1, [])), + ): + registration = deep_research_agent.__wrapped__(config, builder) + function_info = await anext(registration) + assert function_info.single_fn is not None + try: + with pytest.raises(asyncio.CancelledError): + await function_info.single_fn(state) + finally: + await registration.aclose() + + template_agent.finalize.assert_not_called() + if owns_active_agent: + request_agent.finalize.assert_called_once_with(interrupted=True) + else: + request_agent.finalize.assert_not_called() + def test_modal_sandbox_name_is_job_id(self): """Modal sandbox names use the resolved job ID directly.""" from aiq_agent.agents.deep_researcher.sandbox.providers.modal import _validate_modal_sandbox_name diff --git a/tests/aiq_agent/agents/deep_researcher/test_chart_artifact_contract.py b/tests/aiq_agent/agents/deep_researcher/test_chart_artifact_contract.py new file mode 100644 index 000000000..e0015a4e5 --- /dev/null +++ b/tests/aiq_agent/agents/deep_researcher/test_chart_artifact_contract.py @@ -0,0 +1,37 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Static contracts that keep chart generation aligned with artifact harvesting.""" + +from __future__ import annotations + +from pathlib import Path + +from aiq_agent.common import render_prompt_template + +_AGENT_ROOT = Path(__file__).parents[4] / "src" / "aiq_agent" / "agents" / "deep_researcher" + + +def test_chart_skill_uses_runtime_argument_instead_of_executable_placeholder() -> None: + skill = (_AGENT_ROOT / "skills" / "research" / "chart-generation" / "SKILL.md").read_text(encoding="utf-8") + + assert 'ARTIFACT_DIR = ""' not in skill + assert '"path": "' not in skill + assert "ARTIFACT_DIR = Path(sys.argv[1])" in skill + assert 'ARTIFACT_DIR / "manifest.json"' in skill + + +def test_writer_runs_chart_script_with_rendered_per_job_paths() -> None: + prompt = (_AGENT_ROOT / "prompts" / "writer.j2").read_text(encoding="utf-8") + rendered = render_prompt_template( + prompt, + current_datetime="2026-07-09", + execution_enabled=True, + parent_report_context_available=False, + sandbox_workdir="/sandbox/job-123", + sandbox_artifact_dir="/sandbox/job-123/aiq-artifacts", + user_info=None, + ) + + assert "python3 /sandbox/job-123/make_chart.py /sandbox/job-123/aiq-artifacts" in rendered + assert "Never put a literal `` or ``" in rendered diff --git a/tests/aiq_agent/agents/deep_researcher/test_custom_middleware.py b/tests/aiq_agent/agents/deep_researcher/test_custom_middleware.py index bd56260f1..93ef97bb4 100644 --- a/tests/aiq_agent/agents/deep_researcher/test_custom_middleware.py +++ b/tests/aiq_agent/agents/deep_researcher/test_custom_middleware.py @@ -21,13 +21,21 @@ from unittest.mock import MagicMock import pytest +from deepagents.backends import CompositeBackend +from deepagents.backends import StateBackend +from deepagents.middleware.filesystem import FilesystemMiddleware +from langchain.agents import create_agent +from langchain_core.language_models.fake_chat_models import FakeMessagesListChatModel from langchain_core.messages import AIMessage +from langchain_core.messages import HumanMessage from langchain_core.messages import SystemMessage from langchain_core.messages import ToolMessage from aiq_agent.agents.deep_researcher.custom_middleware import ArtifactHarvestMiddleware from aiq_agent.agents.deep_researcher.custom_middleware import ExecuteTimeoutClampMiddleware +from aiq_agent.agents.deep_researcher.custom_middleware import FilesystemToolCallGuardMiddleware from aiq_agent.agents.deep_researcher.custom_middleware import PlanPersistenceMiddleware +from aiq_agent.agents.deep_researcher.custom_middleware import RequiredOutputFileMiddleware from aiq_agent.agents.deep_researcher.custom_middleware import SourceRegistryMiddleware from aiq_agent.agents.deep_researcher.custom_middleware import SourceRoutingGuardMiddleware from aiq_agent.agents.deep_researcher.custom_middleware import TodoSuppressionMiddleware @@ -39,6 +47,13 @@ from aiq_agent.common.data_source_registry import reset_registry +class _ToolBindingFakeChatModel(FakeMessagesListChatModel): + """Scripted chat model that accepts the tools bound by ``create_agent``.""" + + def bind_tools(self, tools, *, tool_choice=None, **kwargs): + return self + + class TestSourceRoutingGuardMiddleware: """Tests for the orchestrator's required source-routing transition.""" @@ -197,6 +212,194 @@ async def test_non_execute_tool_passthrough(self): handler.assert_awaited_once_with(request) +class TestFilesystemToolCallGuardMiddleware: + """Filesystem calls are normalized and unresolved path templates fail before execution.""" + + @staticmethod + def _request(tool_name: str, args: dict) -> MagicMock: + request = MagicMock() + request.tool_call = {"name": tool_name, "args": args, "id": "tc1"} + + def _override(*, tool_call): + overridden = MagicMock() + overridden.tool_call = tool_call + return overridden + + request.override.side_effect = _override + return request + + @pytest.mark.asyncio + async def test_normalizes_read_file_path_alias(self) -> None: + middleware = FilesystemToolCallGuardMiddleware() + request = self._request("read_file", {"path": "/shared/output.md", "offset": 1}) + handler = AsyncMock(return_value=ToolMessage(content="ok", tool_call_id="tc1")) + + await middleware.awrap_tool_call(request, handler) + + forwarded = handler.await_args.args[0] + assert forwarded.tool_call["args"] == {"file_path": "/shared/output.md", "offset": 1} + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "placeholder", + [ + "", + "< sandbox_workdir >", + "{{ sandbox_workdir }}", + "{{sandbox_artifact_dir}}", + "{{ sandbox_workdir }}", + ], + ) + async def test_rejects_unresolved_execute_path_placeholder(self, placeholder: str) -> None: + middleware = FilesystemToolCallGuardMiddleware() + request = self._request("execute", {"command": f"python3 make_chart.py {placeholder}"}) + handler = AsyncMock(return_value=ToolMessage(content="ok", tool_call_id="tc1")) + + result = await middleware.awrap_tool_call(request, handler) + + handler.assert_not_awaited() + assert isinstance(result, ToolMessage) + assert result.status == "error" + assert placeholder in result.content + + @pytest.mark.asyncio + async def test_allows_concrete_execute_paths(self) -> None: + middleware = FilesystemToolCallGuardMiddleware() + request = self._request( + "execute", + {"command": "python3 /sandbox/job/make_chart.py /sandbox/job/aiq-artifacts"}, + ) + expected = ToolMessage(content="ok", tool_call_id="tc1") + handler = AsyncMock(return_value=expected) + + result = await middleware.awrap_tool_call(request, handler) + + assert result is expected + handler.assert_awaited_once_with(request) + + +class TestRequiredOutputFileMiddleware: + """The writer may only claim completion after a non-empty report exists.""" + + marker = "Wrote /shared/output.md" + + @staticmethod + def _state(*, files: dict | None = None, messages: list | None = None) -> dict: + return { + "files": files or {}, + "messages": messages or [AIMessage(content="Wrote /shared/output.md")], + } + + @pytest.mark.parametrize("path", ["/shared/output.md", "/output.md"]) + def test_accepts_non_empty_output_in_both_backend_path_forms(self, path: str) -> None: + middleware = RequiredOutputFileMiddleware() + state = self._state(files={path: {"content": "# Final report"}}) + + assert middleware.after_model(state, None) is None + + @pytest.mark.parametrize("content", ["", " ", b"\n", []]) + def test_empty_output_requests_one_local_corrective_turn(self, content: object) -> None: + middleware = RequiredOutputFileMiddleware() + state = self._state(files={"/output.md": {"content": content}}) + + update = middleware.after_model(state, None) + + assert update is not None + assert update["jump_to"] == "model" + correction = update["messages"][0] + assert isinstance(correction, HumanMessage) + assert "Call write_file" in str(correction.content) + assert "Do not repeat research" in str(correction.content) + + def test_does_not_interrupt_intermediate_tool_call(self) -> None: + middleware = RequiredOutputFileMiddleware() + state = self._state( + messages=[ + AIMessage( + content=self.marker, + tool_calls=[{"name": "write_file", "args": {}, "id": "tc1"}], + ) + ] + ) + + assert middleware.after_model(state, None) is None + + @pytest.mark.asyncio + async def test_async_retry_accepts_repaired_route_local_output(self) -> None: + middleware = RequiredOutputFileMiddleware() + first = middleware.after_model(self._state(), None) + correction = first["messages"][0] + repaired = self._state( + files={"/output.md": {"content": "# Final report"}}, + messages=[AIMessage(content=self.marker), correction, AIMessage(content=self.marker)], + ) + + assert await middleware.aafter_model(repaired, None) is None + + def test_repeated_false_completion_fails_with_stable_reason_code(self) -> None: + middleware = RequiredOutputFileMiddleware() + first = middleware.after_model(self._state(), None) + correction = first["messages"][0] + still_missing = self._state( + messages=[AIMessage(content=self.marker), correction, AIMessage(content=self.marker)] + ) + + with pytest.raises(RuntimeError, match="^writer_output_missing$"): + middleware.after_model(still_missing, None) + + @pytest.mark.asyncio + @pytest.mark.parametrize("shared_route", [False, True]) + async def test_graph_retry_stays_local_and_writes_required_output(self, shared_route: bool) -> None: + """The jump performs one corrective model turn and then follows the normal tool loop.""" + model = _ToolBindingFakeChatModel( + responses=[ + AIMessage(content=self.marker), + AIMessage( + content="", + tool_calls=[ + { + "name": "write_file", + "args": {"file_path": "/shared/output.md", "content": "# Final report"}, + "id": "tc1", + } + ], + ), + AIMessage(content=self.marker), + ] + ) + backend = ( + CompositeBackend(default=StateBackend(), routes={"/shared/": StateBackend()}) if shared_route else None + ) + graph = create_agent( + model, + tools=[], + middleware=[FilesystemMiddleware(backend=backend), RequiredOutputFileMiddleware()], + ) + + result = await graph.ainvoke({"messages": [HumanMessage(content="Write the report")]}) + + expected_path = "/output.md" if shared_route else "/shared/output.md" + assert result["files"][expected_path]["content"] == "# Final report" + assert [message.content for message in result["messages"]].count(self.marker) == 2 + + @pytest.mark.asyncio + async def test_graph_stops_after_bounded_false_completion_retry(self) -> None: + model = _ToolBindingFakeChatModel( + responses=[ + AIMessage(content=self.marker), + AIMessage(content=self.marker), + ] + ) + graph = create_agent( + model, + tools=[], + middleware=[FilesystemMiddleware(), RequiredOutputFileMiddleware()], + ) + + with pytest.raises(RuntimeError, match="^writer_output_missing"): + await graph.ainvoke({"messages": [HumanMessage(content="Write the report")]}) + + class TestToolNameSanitizationMiddleware: """Tests for ToolNameSanitizationMiddleware.""" @@ -775,6 +978,23 @@ async def test_checkpoint_failure_logs_only_exception_type(self, caplog: pytest. assert "RuntimeError" in caplog.text assert "credential=do-not-log" not in caplog.text + @pytest.mark.asyncio + async def test_checkpoint_returns_exact_inline_filename_to_model(self) -> None: + manager = MagicMock() + manager.harvest_after_execute.return_value = [ + SimpleNamespace(filename="capex_by_quarter.png", inline=True), + SimpleNamespace(filename="capex_by_quarter.csv", inline=False), + ] + middleware = ArtifactHarvestMiddleware(manager) + request = MagicMock() + request.tool_call = {"name": "execute"} + handler = AsyncMock(return_value=ToolMessage(content="command succeeded", tool_call_id="tc1")) + + result = await middleware.awrap_tool_call(request, handler) + + assert "artifact://capex_by_quarter.png" in result.content + assert "capex_by_quarter.csv (downloadable; not marked inline)" in result.content + class _RecordingBackend: """Minimal backend stub capturing upload_files calls (overwrite-safe).""" diff --git a/tests/aiq_agent/agents/deep_researcher/test_deepagents_runtime.py b/tests/aiq_agent/agents/deep_researcher/test_deepagents_runtime.py index 009d68523..713237ace 100644 --- a/tests/aiq_agent/agents/deep_researcher/test_deepagents_runtime.py +++ b/tests/aiq_agent/agents/deep_researcher/test_deepagents_runtime.py @@ -19,11 +19,15 @@ import logging from contextlib import nullcontext +from pathlib import Path +from threading import Event +from threading import Thread from typing import Any from unittest.mock import MagicMock from unittest.mock import patch import pytest +import yaml from deepagents.backends import CompositeBackend from deepagents.backends import FilesystemBackend from deepagents.backends import StateBackend @@ -34,12 +38,36 @@ from aiq_agent.agents.deep_researcher.deepagents_runtime import DeepAgentsRuntime from aiq_agent.agents.deep_researcher.deepagents_runtime import DeepResearchSandboxConfig from aiq_agent.agents.deep_researcher.deepagents_runtime import DeepResearchSkillsConfig +from aiq_agent.agents.deep_researcher.deepagents_runtime import _create_sandbox_backend from aiq_agent.agents.deep_researcher.deepagents_runtime import discover_skill_collections from aiq_agent.agents.deep_researcher.deepagents_runtime import resolve_skill_collections SYNTHESIS_SKILL_SOURCE = f"{BUILTIN_SKILL_SOURCE}synthesis/" +def test_openshell_workflow_only_diverges_for_sandbox_wiring() -> None: + """Keep the OpenShell workflow aligned with the standard web config.""" + + def load(path: str) -> dict[str, Any]: + text = Path(path).read_text(encoding="utf-8") + text = text.replace("${AIQ_OPENSHELL_REQUIRE_HARD_LANDLOCK:-true}", "true") + return yaml.safe_load(text) + + standard = load("configs/config_web_default_llamaindex.yml") + openshell = load("configs/config_openshell.yml") + openshell_functions = openshell["functions"].copy() + openshell_functions.pop("deep_research_skills") + openshell_functions.pop("deep_research_sandbox") + openshell_functions["deep_research_agent"] = openshell_functions["deep_research_agent"].copy() + openshell_functions["deep_research_agent"].pop("skills") + openshell_functions["deep_research_agent"].pop("sandbox") + + assert openshell["general"] == standard["general"] + assert openshell["llms"] == standard["llms"] + assert openshell_functions == standard["functions"] + assert openshell["workflow"] == standard["workflow"] + + class TestSkillCollections: """Public skill config uses collection names, not DeepAgents virtual paths.""" @@ -304,6 +332,41 @@ def find_spec(module_name: str): ): _ = DeepAgentsRuntime(sandbox=DeepResearchSandboxConfig(provider="modal")).backend + def test_public_openshell_config_maps_isolation_and_attestation(self) -> None: + public = DeepResearchSandboxConfig( + policy="policy.yaml", + openshell_image="aiq:test", + attest=True, + expected_policy_version=3, + policy_load_timeout_seconds=17, + network="allowlist", + network_allow=("api.github.com",), + ) + + with patch( + "aiq_agent.agents.deep_researcher.sandbox.create_sandbox_backend", + return_value="backend", + ) as create: + assert _create_sandbox_backend(public, "job-1") == "backend" + + resolved = create.call_args.args[0] + assert resolved.network.mode == "allowlist" + assert resolved.network.allow == ("api.github.com",) + assert resolved.providers.openshell.image == "aiq:test" + assert resolved.providers.openshell.attest is True + assert resolved.providers.openshell.expected_policy_version == 3 + assert resolved.providers.openshell.policy_load_timeout_seconds == 17 + assert resolved.providers.openshell.delete_on_exit is True + + def test_public_allowlist_requires_hosts(self) -> None: + with pytest.raises(ValueError, match="network_allow"): + DeepResearchSandboxConfig(network="allowlist") + + @pytest.mark.parametrize("timeout", [0, -1, float("inf"), float("nan")]) + def test_public_ready_timeout_must_be_positive_and_finite(self, timeout: float) -> None: + with pytest.raises(ValueError, match="ready_timeout_seconds"): + DeepResearchSandboxConfig(ready_timeout_seconds=timeout) + class TestDeepAgentsRuntimeArtifacts: """Terminal artifact harvesting is safe on normal and interrupted paths.""" @@ -354,3 +417,123 @@ def test_interrupted_finalize_harvests_only_when_provider_is_idle( assert runtime.finalize_artifacts(interrupted=True) is expected assert runtime.artifact_manager.final_harvest.call_count == int(lease_acquired) + + +class TestDeepAgentsRuntimeCleanup: + """Terminal cleanup is idempotent and reports the provider's actual outcome.""" + + def test_finalize_closes_once_and_emits_success(self) -> None: + provider = MagicMock() + provider.provider_name = "openshell" + provider.physical_sandbox_name = "sandbox-1" + provider.cleanup_succeeded = True + events: list[dict[str, object]] = [] + with patch( + "aiq_agent.agents.deep_researcher.deepagents_runtime._create_sandbox_backend", + return_value=provider, + ): + runtime = DeepAgentsRuntime( + sandbox=DeepResearchSandboxConfig(), + artifact_emit=events.append, + ) + + assert runtime.finalize(interrupted=False) is True + assert runtime.finalize(interrupted=False) is True + provider.close.assert_called_once_with() + assert provider.terminate.call_count == 0 + assert [event["data"]["status"] for event in events] == ["started", "succeeded"] # type: ignore[index] + + def test_finalize_without_provider_emits_no_cleanup_events(self) -> None: + events: list[dict[str, object]] = [] + runtime = DeepAgentsRuntime(sandbox=None, artifact_emit=events.append) + + assert runtime.finalize(interrupted=False) is True + assert runtime.finalize(interrupted=True) is True + assert events == [] + + def test_finalize_emits_failed_when_provider_observed_cleanup_error(self) -> None: + provider = MagicMock() + provider.provider_name = "openshell" + provider.sandbox_name = "logical-job" + provider.physical_sandbox_name = None + provider.cleanup_succeeded = False + events: list[dict[str, object]] = [] + with patch( + "aiq_agent.agents.deep_researcher.deepagents_runtime._create_sandbox_backend", + return_value=provider, + ): + runtime = DeepAgentsRuntime( + sandbox=DeepResearchSandboxConfig(), + artifact_emit=events.append, + ) + + assert runtime.finalize(interrupted=True) is False + provider.terminate.assert_called_once_with() + assert [event["data"]["status"] for event in events] == ["started", "failed"] # type: ignore[index] + + def test_finalize_logs_only_cleanup_exception_type(self, caplog: pytest.LogCaptureFixture) -> None: + provider = MagicMock() + provider.provider_name = "openshell" + provider.physical_sandbox_name = "sandbox-1" + provider.close.side_effect = RuntimeError("credential=do-not-log") + with patch( + "aiq_agent.agents.deep_researcher.deepagents_runtime._create_sandbox_backend", + return_value=provider, + ): + runtime = DeepAgentsRuntime(sandbox=DeepResearchSandboxConfig()) + + with caplog.at_level(logging.WARNING): + assert runtime.finalize(interrupted=False) is False + + assert "RuntimeError" in caplog.text + assert "credential=do-not-log" not in caplog.text + + def test_concurrent_finalize_waits_for_and_reuses_exact_result(self) -> None: + provider = MagicMock() + provider.provider_name = "openshell" + provider.physical_sandbox_name = "sandbox-1" + provider.cleanup_succeeded = True + cleanup_started = Event() + allow_cleanup = Event() + second_caller_started = Event() + events: list[dict[str, object]] = [] + results: list[bool] = [] + + def close() -> None: + cleanup_started.set() + if not allow_cleanup.wait(timeout=2): + raise AssertionError("cleanup was not released") + provider.cleanup_succeeded = False + + provider.close.side_effect = close + with patch( + "aiq_agent.agents.deep_researcher.deepagents_runtime._create_sandbox_backend", + return_value=provider, + ): + runtime = DeepAgentsRuntime( + sandbox=DeepResearchSandboxConfig(), + artifact_emit=events.append, + ) + + first = Thread(target=lambda: results.append(runtime.finalize(interrupted=False))) + + def finalize_again() -> None: + second_caller_started.set() + results.append(runtime.finalize(interrupted=True)) + + second = Thread(target=finalize_again) + first.start() + assert cleanup_started.wait(timeout=2) + second.start() + assert second_caller_started.wait(timeout=2) + assert results == [] + + allow_cleanup.set() + first.join(timeout=2) + second.join(timeout=2) + + assert not first.is_alive() and not second.is_alive() + assert results == [False, False] + provider.close.assert_called_once_with() + provider.terminate.assert_not_called() + assert [event["data"]["status"] for event in events] == ["started", "failed"] # type: ignore[index] diff --git a/tests/aiq_agent/agents/deep_researcher/test_factory.py b/tests/aiq_agent/agents/deep_researcher/test_factory.py index 13fe705e8..d4f14f660 100644 --- a/tests/aiq_agent/agents/deep_researcher/test_factory.py +++ b/tests/aiq_agent/agents/deep_researcher/test_factory.py @@ -24,6 +24,8 @@ from langchain_core.tools import tool from aiq_agent.agents.deep_researcher.custom_middleware import ArtifactHarvestMiddleware +from aiq_agent.agents.deep_researcher.custom_middleware import FilesystemToolCallGuardMiddleware +from aiq_agent.agents.deep_researcher.custom_middleware import RequiredOutputFileMiddleware from aiq_agent.agents.deep_researcher.custom_middleware import SourceRegistryMiddleware from aiq_agent.agents.deep_researcher.custom_middleware import SourceRoutingGuardMiddleware from aiq_agent.agents.deep_researcher.custom_middleware import TodoSuppressionMiddleware @@ -242,6 +244,7 @@ def test_subagents_route_tools_and_writer_skills(): assert any(isinstance(item, ToolVisibilityMiddleware) for item in by_name["planner-agent"]["middleware"]) assert any(isinstance(item, ToolVisibilityMiddleware) for item in by_name["writer-agent"]["middleware"]) assert any(isinstance(item, TodoSuppressionMiddleware) for item in by_name["writer-agent"]["middleware"]) + assert any(isinstance(item, RequiredOutputFileMiddleware) for item in by_name["writer-agent"]["middleware"]) def test_skill_filesystem_permissions_filter_unassigned_skill_collections(): @@ -290,6 +293,38 @@ def test_graph_uses_researcher_config_key_for_researcher_skills(): assert skills_middleware[0].sources == ["/skills/research/"] +def test_graph_wires_filesystem_tool_call_guard_cross_cutting(): + """Every agent receives the filesystem path guard, even when execution is hidden.""" + registry, tool_set, middleware_set = _tool_set_and_middleware() + runtime = DeepAgentsRuntime() + fake_graph = MagicMock() + fake_graph.with_config.return_value = fake_graph + + with ( + patch("aiq_agent.agents.deep_researcher.factory.create_deep_agent", return_value=fake_graph) as create_graph, + patch("aiq_agent.agents.deep_researcher.factory.create_agent", return_value=MagicMock()), + patch("aiq_agent.agents.deep_researcher.factory.create_summarization_middleware", return_value=MagicMock()), + ): + build_deep_research_graph( + llm_provider=_llm_provider(), + state=DeepResearchAgentState(messages=[]), + prompts=_prompts(), + tools=[web_search_tool], + runtime=runtime, + tool_set=tool_set, + middleware_set=middleware_set, + source_registry_middleware=registry, + callbacks=[], + domain_catalog_path=None, + max_research_concurrency=6, + ) + + assert any( + isinstance(middleware, FilesystemToolCallGuardMiddleware) + for middleware in create_graph.call_args.kwargs["middleware"] + ) + + def test_subagents_can_disable_source_router(): """The source-router subagent can be omitted without changing the rest of the workflow.""" provider = _llm_provider() diff --git a/tests/aiq_agent/jobs/test_runner.py b/tests/aiq_agent/jobs/test_runner.py index 080355b71..abe65bd13 100644 --- a/tests/aiq_agent/jobs/test_runner.py +++ b/tests/aiq_agent/jobs/test_runner.py @@ -2444,6 +2444,53 @@ def test_none_runtime_is_noop(self): # Must not raise when no sandbox runtime is present (non-sandbox agents). _teardown_sandbox(None, job_id="job-1", interrupted=False) + @pytest.mark.asyncio + async def test_terminal_event_flush_failure_is_nonfatal_and_sanitized(self, caplog): + from aiq_api.jobs.runner import _flush_event_store + + event_store = MagicMock() + event_store.flush.side_effect = RuntimeError("secret-bearing database detail") + + with caplog.at_level("WARNING", logger="aiq_api.jobs.runner"): + await _flush_event_store(event_store, job_id="job-1") + + event_store.flush.assert_called_once_with() + assert "Event store flush failed for job job-1 (RuntimeError)" in caplog.text + assert "secret-bearing database detail" not in caplog.text + + def test_runtime_finalizer_owns_cleanup_when_available(self): + from aiq_api.jobs.runner import _teardown_sandbox + + runtime = MagicMock(spec=["finalize", "close", "terminate"]) + _teardown_sandbox(runtime, job_id="job-1", interrupted=True) + + runtime.finalize.assert_called_once_with(interrupted=True) + runtime.close.assert_not_called() + runtime.terminate.assert_not_called() + + def test_runtime_finalizer_false_result_is_logged(self, caplog): + from aiq_api.jobs.runner import _teardown_sandbox + + runtime = MagicMock(spec=["finalize"]) + runtime.finalize.return_value = False + + with caplog.at_level("WARNING", logger="aiq_api.jobs.runner"): + _teardown_sandbox(runtime, job_id="job-1", interrupted=False) + + assert "Sandbox cleanup reported failure for job job-1" in caplog.text + + def test_runtime_finalizer_exception_is_nonfatal_and_sanitized(self, caplog): + from aiq_api.jobs.runner import _teardown_sandbox + + runtime = MagicMock(spec=["finalize"]) + runtime.finalize.side_effect = RuntimeError("credential=do-not-log") + + with caplog.at_level("WARNING", logger="aiq_api.jobs.runner"): + _teardown_sandbox(runtime, job_id="job-1", interrupted=False) + + assert "Sandbox cleanup failed for job job-1 (RuntimeError)" in caplog.text + assert "credential=do-not-log" not in caplog.text + def test_normal_path_calls_close(self): from aiq_api.jobs.runner import _teardown_sandbox @@ -2470,13 +2517,17 @@ def test_interrupted_without_terminate_falls_back_to_close(self): runtime.close.assert_called_once_with() - def test_never_raises_when_teardown_fails(self): + def test_fallback_teardown_exception_is_nonfatal_and_sanitized(self, caplog): from aiq_api.jobs.runner import _teardown_sandbox runtime = MagicMock(spec=["close", "terminate"]) - runtime.close.side_effect = RuntimeError("sdk session close failed") - # Must swallow the error; teardown is best-effort on the terminal path. - _teardown_sandbox(runtime, job_id="job-1", interrupted=False) + runtime.close.side_effect = RuntimeError("credential=do-not-log") + + with caplog.at_level("WARNING", logger="aiq_api.jobs.runner"): + _teardown_sandbox(runtime, job_id="job-1", interrupted=False) + + assert "Sandbox cleanup failed for job job-1 (RuntimeError)" in caplog.text + assert "credential=do-not-log" not in caplog.text def test_finalizes_artifacts_before_close(self): from aiq_api.jobs.runner import _teardown_sandbox diff --git a/tests/scripts/test_openshell_gateway_installer.py b/tests/scripts/test_openshell_gateway_installer.py new file mode 100644 index 000000000..9e55f6c42 --- /dev/null +++ b/tests/scripts/test_openshell_gateway_installer.py @@ -0,0 +1,268 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Behavioral tests for the explicit packaged OpenShell gateway installer.""" + +from __future__ import annotations + +import os +import stat +import subprocess +import sys +from pathlib import Path + +_REPO_ROOT = Path(__file__).resolve().parents[2] +_INSTALLER = _REPO_ROOT / "scripts" / "openshell" / "install_gateway.sh" +_CHECKSUM = "c15d6cb8090e1c7c8d79a320b5bcbdaf1c15c2363942d81e84b56e03b836249e" # pragma: allowlist secret + + +def _write_executable(path: Path, source: str) -> None: + path.write_text(source, encoding="utf-8") + path.chmod(0o755) + + +def _environment(tmp_path: Path, *, uid: int = 501, checksum: str = _CHECKSUM) -> tuple[dict[str, str], Path]: + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + log = tmp_path / "calls.log" + _write_executable( + bin_dir / "id", + f'#!/bin/bash\nif [[ "${{1:-}}" == \'-u\' ]]; then echo {uid}; else /usr/bin/id "$@"; fi\n', + ) + _write_executable( + bin_dir / "uname", + "#!/bin/bash\nif [[ \"${1:-}\" == '-s' ]]; then echo Darwin; else echo arm64; fi\n", + ) + _write_executable( + bin_dir / "brew", + """#!/bin/bash +if [[ "$*" == "list --formula --full-name" ]]; then echo nvidia/openshell/openshell; exit 0; fi +echo "brew $*" >>"$FAKE_LOG" +exit 0 +""", + ) + _write_executable( + bin_dir / "curl", + """#!/bin/bash +echo "curl $*" >>"$FAKE_LOG" +[[ "${FAKE_DOWNLOAD_FAIL:-false}" != "true" ]] || exit 22 +while [[ $# -gt 0 ]]; do + if [[ "$1" == "-o" ]]; then printf '#!/bin/sh\nexit 0\n' >"$2"; exit 0; fi + shift +done +exit 2 +""", + ) + _write_executable(bin_dir / "shasum", f"#!/bin/bash\necho '{checksum} $2'\n") + _write_executable( + bin_dir / "sh", + '#!/bin/bash\necho "installer version=${OPENSHELL_VERSION:-unset} $*" >>"$FAKE_LOG"\n', + ) + python_wrapper = bin_dir / "python" + _write_executable( + python_wrapper, + """#!/bin/bash +if [[ "${1:-}" == *version_contract.py ]]; then + case "${3:-}" in + release-tag) echo v0.0.80 ;; + installer-sha256) echo "$EXPECTED_SHA256" ;; + esac + exit 0 +fi +if [[ "${1:-}" == *check_versions.py ]]; then + echo "check-versions $*" >>"$FAKE_LOG" + exit "${FAKE_VERIFY_EXIT:-0}" +fi +exec "$REAL_PYTHON" "$@" +""", + ) + env = os.environ.copy() + env.update( + { + "PATH": f"{bin_dir}:{env['PATH']}", + "HOME": str(tmp_path / "home"), + "PYTHON_BIN": str(python_wrapper), + "REAL_PYTHON": sys.executable, + "FAKE_LOG": str(log), + "EXPECTED_SHA256": _CHECKSUM, + } + ) + return env, log + + +def _run(tmp_path: Path, *args: str, env: dict[str, str] | None = None) -> subprocess.CompletedProcess[str]: + effective_env, _ = _environment(tmp_path) if env is None else (env, tmp_path / "calls.log") + return subprocess.run( + [str(_INSTALLER), *args], + cwd=_REPO_ROOT, + env=effective_env, + text=True, + capture_output=True, + check=False, + ) + + +def test_dry_run_is_non_mutating_and_names_certified_release(tmp_path: Path) -> None: + env, log = _environment(tmp_path) + + result = _run(tmp_path, "--dry-run", env=env) + + assert result.returncode == 0, result.stderr + assert "v0.0.80" in result.stdout + assert "no files, taps, packages, or services were changed" in result.stdout + calls = log.read_text(encoding="utf-8") if log.exists() else "" + assert "curl " not in calls + assert "installer " not in calls + assert "check-versions " not in calls + + +def test_root_execution_is_refused(tmp_path: Path) -> None: + env, _ = _environment(tmp_path, uid=0) + + result = _run(tmp_path, "--yes", env=env) + + assert result.returncode != 0 + assert "not root" in result.stderr + + +def test_noninteractive_install_requires_explicit_yes(tmp_path: Path) -> None: + env, log = _environment(tmp_path) + + result = _run(tmp_path, env=env) + + assert result.returncode != 0 + assert "rerun with --yes" in result.stderr + calls = log.read_text(encoding="utf-8") if log.exists() else "" + assert "curl " not in calls + + +def test_checksum_mismatch_stops_before_official_installer(tmp_path: Path) -> None: + env, log = _environment(tmp_path, checksum="0" * 64) + config = Path(env["HOME"]) / ".config" / "openshell" / "gateway.env" + config.parent.mkdir(parents=True) + config.write_text("SAFE_SETTING=unchanged\n", encoding="utf-8") + + result = _run(tmp_path, "--yes", "--colima", env=env) + + assert result.returncode != 0 + assert "gateway_installer_checksum_mismatch" in result.stderr + calls = log.read_text(encoding="utf-8") + assert "curl " in calls + assert "installer " not in calls + assert config.read_text(encoding="utf-8") == "SAFE_SETTING=unchanged\n" + + +def test_download_failure_is_sanitized(tmp_path: Path) -> None: + env, _ = _environment(tmp_path) + env["FAKE_DOWNLOAD_FAIL"] = "true" + + result = _run(tmp_path, "--yes", env=env) + + assert result.returncode != 0 + assert "gateway_installer_download_failed" in result.stderr + + +def test_colima_configuration_preserves_unrelated_values(tmp_path: Path) -> None: + env, log = _environment(tmp_path) + config = Path(env["HOME"]) / ".config" / "openshell" / "gateway.env" + config.parent.mkdir(parents=True) + config.write_text("SAFE_SETTING=keep\nDOCKER_HOST=old\n", encoding="utf-8") + config.chmod(0o640) + + result = _run(tmp_path, "--yes", "--colima", env=env) + + assert result.returncode == 0, result.stderr + assert config.read_text(encoding="utf-8").splitlines() == [ + "SAFE_SETTING=keep", + "DOCKER_HOST=unix://" + env["HOME"] + "/.colima/default/docker.sock", + "OPENSHELL_DRIVERS=docker", + ] + assert stat.S_IMODE(config.stat().st_mode) == 0o640 + calls = log.read_text(encoding="utf-8") + assert "installer version=v0.0.80" in calls + assert "check-versions " in calls + + +def test_ambiguous_formula_installation_is_refused(tmp_path: Path) -> None: + env, log = _environment(tmp_path) + brew = Path(env["PATH"].split(":", maxsplit=1)[0]) / "brew" + _write_executable( + brew, + """#!/bin/bash +if [[ "$*" == "list --formula --full-name" ]]; then + printf '%s\n' nvidia/openshell/openshell aiq/local-openshell/openshell + exit 0 +fi +echo "brew $*" >>"$FAKE_LOG" +""", + ) + + result = _run(tmp_path, "--yes", env=env) + + assert result.returncode != 0 + assert "ambiguous_gateway_installation" in result.stderr + calls = log.read_text(encoding="utf-8") if log.exists() else "" + assert "curl " not in calls + + +def test_bare_formula_is_not_treated_as_official_from_registered_tap(tmp_path: Path) -> None: + env, log = _environment(tmp_path) + brew = Path(env["PATH"].split(":", maxsplit=1)[0]) / "brew" + _write_executable( + brew, + """#!/bin/bash +if [[ "$*" == "list --formula --full-name" ]]; then echo openshell; exit 0; fi +if [[ "$*" == "tap" ]]; then echo nvidia/openshell; exit 0; fi +echo "brew $*" >>"$FAKE_LOG" +""", + ) + + result = _run(tmp_path, "--yes", env=env) + + assert result.returncode != 0 + assert "ambiguous_gateway_installation" in result.stderr + calls = log.read_text(encoding="utf-8") if log.exists() else "" + assert "curl " not in calls + + +def test_ambiguous_service_installation_is_refused(tmp_path: Path) -> None: + env, log = _environment(tmp_path) + brew = Path(env["PATH"].split(":", maxsplit=1)[0]) / "brew" + _write_executable( + brew, + """#!/bin/bash +if [[ "$*" == "list --formula --full-name" ]]; then echo nvidia/openshell/openshell; exit 0; fi +if [[ "$*" == "services list --json" ]]; then + printf '%s\n' '[{"name":"openshell"},{"name":"aiq/local-openshell/openshell"}]' + exit 0 +fi +if [[ "$*" == "tap" ]]; then echo nvidia/openshell; exit 0; fi +echo "brew $*" >>"$FAKE_LOG" +""", + ) + + result = _run(tmp_path, "--yes", env=env) + + assert result.returncode != 0 + assert "multiple OpenShell services" in result.stderr + calls = log.read_text(encoding="utf-8") if log.exists() else "" + assert "curl " not in calls + + +def test_post_install_component_failure_is_not_reported_as_success(tmp_path: Path) -> None: + env, _ = _environment(tmp_path) + env["FAKE_VERIFY_EXIT"] = "1" + + result = _run(tmp_path, "--yes", env=env) + + assert result.returncode != 0 + assert "component verification failed" in result.stderr + + +def test_installer_source_contains_no_implicit_or_insecure_lifecycle_shortcuts() -> None: + source = _INSTALLER.read_text(encoding="utf-8") + + assert "curl |" not in source + assert "launchctl setenv" not in source + assert "tap-new aiq/" not in source + assert "openshell-gateway" not in source diff --git a/tests/scripts/test_openshell_lifecycle_scripts.py b/tests/scripts/test_openshell_lifecycle_scripts.py new file mode 100644 index 000000000..3bace0117 --- /dev/null +++ b/tests/scripts/test_openshell_lifecycle_scripts.py @@ -0,0 +1,303 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Regression tests for OpenShell provisioning and gateway lifecycle ownership.""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +_REPO_ROOT = Path(__file__).resolve().parents[2] +_GATEWAY_SCRIPT = _REPO_ROOT / "scripts" / "openshell" / "start_openshell_gateway.sh" +_SETUP_SCRIPT = _REPO_ROOT / "scripts" / "openshell" / "setup_openshell.sh" +_E2E_SCRIPT = _REPO_ROOT / "scripts" / "start_e2e.sh" + + +def _fake_openshell(tmp_path: Path) -> tuple[Path, Path, Path]: + binary = tmp_path / "openshell" + log = tmp_path / "openshell.log" + state = tmp_path / "sandbox.state" + binary.write_text( + """#!/bin/bash +set -euo pipefail +echo "$*" >>"$FAKE_LOG" +if [[ "$1 ${2:-}" == "gateway list" ]]; then + echo "$FAKE_GATEWAYS_JSON" +elif [[ "$1 ${2:-}" == "gateway select" ]]; then + exit 0 +elif [[ "$1" == "status" ]]; then + exit "${FAKE_STATUS_EXIT:-0}" +elif [[ "$1 ${2:-}" == "sandbox create" ]]; then + name="" + shift 2 + while [[ $# -gt 0 ]]; do + if [[ "$1" == "--name" ]]; then name="$2"; break; fi + shift + done + echo "$name" >"$FAKE_STATE" + [[ "${FAKE_CREATE_FAIL:-false}" != "true" ]] +elif [[ "$1 ${2:-}" == "sandbox list" ]]; then + if [[ -s "$FAKE_STATE" ]]; then + name="$(sed -n '1p' "$FAKE_STATE")" + if [[ "${FAKE_NEVER_READY:-false}" == "true" ]]; then + echo "$name Creating" + else + echo "$name Ready" + fi + fi +elif [[ "$1 ${2:-}" == "sandbox delete" ]]; then + [[ "${FAKE_DELETE_FAIL:-false}" != "true" ]] + rm -f "$FAKE_STATE" +fi +""", + encoding="utf-8", + ) + binary.chmod(0o755) + return binary, log, state + + +def _run_launcher( + tmp_path: Path, + *, + gateway: dict[str, object], + extra_env: dict[str, str] | None = None, +) -> tuple[subprocess.CompletedProcess[str], str]: + binary, log, state = _fake_openshell(tmp_path) + python_wrapper = tmp_path / "python" + python_wrapper.write_text( + """#!/bin/bash +set -euo pipefail +if [[ "${1:-}" == *check_openshell_readiness.py ]]; then + echo "readiness-checker $*" >>"$FAKE_LOG" + exit "${FAKE_READINESS_EXIT:-0}" +fi +if [[ "${1:-}" == *check_versions.py ]]; then + echo "version-inspector $*" >>"$FAKE_LOG" + exit "${FAKE_VERSION_EXIT:-0}" +fi +exec "$REAL_PYTHON" "$@" +""", + encoding="utf-8", + ) + python_wrapper.chmod(0o755) + policy = tmp_path / "policy.yaml" + policy.write_text("version: 1\n", encoding="utf-8") + env = os.environ.copy() + for key in ("OPENSHELL_GATEWAY_ENDPOINT", "OPENSHELL_GATEWAY_INSECURE", "OPENSHELL_GATEWAY_LAUNCH_BIN"): + env.pop(key, None) + env.update( + { + "OPENSHELL_BIN": str(binary), + "PYTHON_BIN": str(python_wrapper), + "REAL_PYTHON": sys.executable, + "FAKE_LOG": str(log), + "FAKE_STATE": str(state), + "FAKE_GATEWAYS_JSON": json.dumps([gateway]), + "AIQ_OPENSHELL_STATUS_ATTEMPTS": "1", + "AIQ_OPENSHELL_PROBE_ATTEMPTS": "1", + "AIQ_OPENSHELL_DELETE_ATTEMPTS": "1", + "AIQ_OPENSHELL_POLL_DELAY": "0", + } + ) + env.update(extra_env or {}) + result = subprocess.run( + [ + str(_GATEWAY_SCRIPT), + "--reuse-existing", + "--gateway-name", + "enterprise", + "--policy-file", + str(policy), + ], + cwd=_REPO_ROOT, + env=env, + text=True, + capture_output=True, + check=False, + ) + return result, log.read_text(encoding="utf-8") if log.exists() else "" + + +def test_authenticated_gateway_runs_mandatory_strict_readiness_check(tmp_path: Path) -> None: + result, calls = _run_launcher( + tmp_path, + gateway={ + "name": "enterprise", + "endpoint": "https://gateway.example.com", + "type": "remote", + "auth": "oidc", + "active": False, + }, + ) + + assert result.returncode == 0, result.stderr + assert "gateway list -o json" in calls + assert "gateway select enterprise" in calls + assert calls.count("version-inspector") == 2 + assert "readiness-checker" in calls + assert "--gateway-name enterprise" in calls + assert "gateway add" not in calls + + +def test_plaintext_gateway_is_rejected_before_probe(tmp_path: Path) -> None: + result, calls = _run_launcher( + tmp_path, + gateway={ + "name": "enterprise", + "endpoint": "http://127.0.0.1:8080", + "type": "local", + "auth": "plaintext", + "active": True, + }, + ) + + assert result.returncode != 0 + assert "not authenticated" in result.stderr + assert "sandbox create" not in calls + + +def test_raw_gateway_launcher_is_rejected_independent_of_probe_mode(tmp_path: Path) -> None: + result, calls = _run_launcher( + tmp_path, + gateway={ + "name": "enterprise", + "endpoint": "https://127.0.0.1:8080", + "type": "local", + "auth": "mtls", + "active": True, + }, + extra_env={"OPENSHELL_GATEWAY_LAUNCH_BIN": "/usr/bin/openshell-gateway"}, + ) + + assert result.returncode != 0 + assert "raw gateway" in result.stderr.lower() + assert calls == "" + + +def test_failed_strict_readiness_check_stops_launcher(tmp_path: Path) -> None: + result, calls = _run_launcher( + tmp_path, + gateway={ + "name": "enterprise", + "endpoint": "https://127.0.0.1:8080", + "type": "local", + "auth": "mtls", + "active": True, + }, + extra_env={"FAKE_READINESS_EXIT": "1"}, + ) + + assert result.returncode != 0 + assert "readiness-checker" in calls + assert "strict readiness check failed" in result.stderr.lower() + + +def test_component_mismatch_stops_before_gateway_or_readiness_probe(tmp_path: Path) -> None: + result, calls = _run_launcher( + tmp_path, + gateway={ + "name": "enterprise", + "endpoint": "https://127.0.0.1:8080", + "type": "local", + "auth": "mtls", + "active": True, + }, + extra_env={"FAKE_VERSION_EXIT": "1"}, + ) + + assert result.returncode != 0 + assert "component version check failed" in result.stderr.lower() + assert "version-inspector" in calls + assert "readiness-checker" not in calls + assert "sandbox create" not in calls + + +def test_setup_is_provisioning_only_and_migrates_old_lifecycle_flags() -> None: + source = _SETUP_SCRIPT.read_text(encoding="utf-8") + assert "pkill" not in source + assert "start_or_verify_gateway" not in source + assert '"$OPENSHELL_BIN" sandbox create' not in source + assert "--reinstall-package" not in source + assert "uv sync --dev --inexact" in source + assert 'uv pip install "deepagents' not in source + + result = subprocess.run( + [str(_SETUP_SCRIPT), "--gateway-name", "old"], + cwd=_REPO_ROOT, + text=True, + capture_output=True, + check=False, + ) + assert result.returncode != 0 + assert "start_openshell_gateway.sh" in result.stderr + + +def test_setup_policy_declares_canonical_openshell_proxy_baseline() -> None: + source = _SETUP_SCRIPT.read_text(encoding="utf-8") + policy_template = source.split("emit_policy_header()", maxsplit=1)[1].split("emit_policy_entry()", maxsplit=1)[0] + + assert " - /proc\n" in policy_template + assert "/proc/self" not in policy_template + + +@pytest.mark.parametrize("version", ["latest", "0.0.72", "0.0.81"]) +def test_setup_rejects_uncertified_openshell_versions(version: str) -> None: + result = subprocess.run( + [str(_SETUP_SCRIPT), "--openshell-version", version, "--skip-build", "--policy", "offline"], + cwd=_REPO_ROOT, + text=True, + capture_output=True, + check=False, + ) + + assert result.returncode != 0 + assert "not certified" in result.stderr + assert "0.0.80" in result.stderr + + +def test_setup_resolves_docker_desktop_cli_and_credential_helper_together() -> None: + source = _SETUP_SCRIPT.read_text(encoding="utf-8") + + assert 'docker_desktop_bin="/Applications/Docker.app/Contents/Resources/bin"' in source + assert 'export PATH="$docker_desktop_bin:$PATH"' in source + + +def test_setup_local_demo_uses_explicit_runtime_override_without_config_copy() -> None: + source = _SETUP_SCRIPT.read_text(encoding="utf-8") + + assert "--local-demo" in source + assert 'LANDLOCK_COMPATIBILITY="best_effort"' in source + assert 'runtime_env="AIQ_OPENSHELL_REQUIRE_HARD_LANDLOCK=false "' in source + assert "configs/config_openshell.local.yml" not in source + + +def test_e2e_gateway_start_is_explicit_opt_in() -> None: + source = _E2E_SCRIPT.read_text(encoding="utf-8") + result = subprocess.run( + [str(_E2E_SCRIPT), "--help"], + cwd=_REPO_ROOT, + text=True, + capture_output=True, + check=False, + ) + + assert result.returncode == 0 + assert "--start-openshell-gateway" in result.stdout + assert "START_OPENSHELL_GATEWAY=false" in source + assert 'if [[ "$START_OPENSHELL_GATEWAY" != "true" ]]' in source + assert '"$PROJECT_ROOT/scripts/openshell/start_openshell_gateway.sh"' in source + assert '"$PROJECT_ROOT/scripts/openshell/check_versions.py"' in source + assert source.index("check_openshell_component_versions\n") < source.index("start_backend\n") + assert 'PYTHON_BIN="$VENV_DIR/bin/python"' in source + assert 'NAT_BIN="$VENV_DIR/bin/nat"' in source + assert '"$NAT_BIN" serve' in source + assert 'BACKEND_PID=""' in source + assert 'FRONTEND_PID=""' in source + assert "${BACKEND_PID:-}" in source + assert "${FRONTEND_PID:-}" in source diff --git a/tests/scripts/test_openshell_readiness_checker.py b/tests/scripts/test_openshell_readiness_checker.py new file mode 100644 index 000000000..8ffda0544 --- /dev/null +++ b/tests/scripts/test_openshell_readiness_checker.py @@ -0,0 +1,440 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the strict OpenShell readiness capability checker.""" + +from __future__ import annotations + +import importlib.util +import sys +from contextlib import contextmanager +from pathlib import Path +from types import ModuleType +from types import SimpleNamespace +from typing import Any +from unittest.mock import patch + +import pytest + +_REPO_ROOT = Path(__file__).resolve().parents[2] +_CHECKER = _REPO_ROOT / "scripts" / "openshell" / "check_openshell_readiness.py" + + +@pytest.fixture(scope="module") +def checker() -> ModuleType: + spec = importlib.util.spec_from_file_location("check_openshell_readiness", _CHECKER) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +class _RpcError(Exception): + def __init__(self, code: str) -> None: + self._code = code + + def code(self) -> str: + return self._code + + +class _Grpc(ModuleType): + class StatusCode: + NOT_FOUND = "not_found" + + Call = _RpcError + RpcError = _RpcError + + +class _Stub: + def __init__(self, *, status: int = 2) -> None: + policy = SimpleNamespace(version=1) + self.status = SimpleNamespace( + active_version=1, + revision=SimpleNamespace( + version=1, + status=status, + load_error="", + policy=policy, + policy_hash="hash", + ), + ) + self.config = SimpleNamespace(version=1, policy_source=1, policy=policy, policy_hash="hash") + + def GetSandboxPolicyStatus(self, _request: object, *, timeout: float) -> object: # noqa: N802 + del timeout + return self.status + + def GetSandboxConfig(self, _request: object, *, timeout: float) -> object: # noqa: N802 + del timeout + return self.config + + +class _Client: + instance: _Client + + def __init__( + self, + *, + status: int = 2, + labels: dict[str, str] | None = None, + persist_request_labels: bool = True, + version: str = "1.2.3", + ) -> None: + self.labels = labels or {"aiq": "readiness-probe"} + self.persist_request_labels = persist_request_labels + self.version = version + self.sandbox = SimpleNamespace( + id="probe-id", + name="probe-name", + phase=2, + current_policy_version=1, + labels=self.labels, + ) + self._stub = _Stub(status=status) + self._stub.status.revision.policy = self._stub.config.policy + self.deleted = False + self.delete_calls = 0 + type(self).instance = self + + @classmethod + def from_active_cluster(cls, *, cluster: str | None) -> _Client: + del cluster + return cls.instance + + def __enter__(self) -> _Client: + return self + + def __exit__(self, *_args: object) -> None: + return None + + def health(self) -> object: + return SimpleNamespace(version=self.version) + + def create(self, *, spec: object, name: str, labels: dict[str, str]) -> object: + del spec + self.sandbox.name = name + if self.persist_request_labels: + self.sandbox.labels = labels + return self.sandbox + + def wait_ready(self, name: str, *, timeout_seconds: float) -> object: + del timeout_seconds + assert name == self.sandbox.name + return self.sandbox + + def list(self, *, label_selector: str) -> list[object]: + assert label_selector == "aiq=readiness-probe" + return [] if self.deleted else [self.sandbox] + + def get(self, name: str) -> object: + assert name == self.sandbox.name + if self.deleted: + raise _RpcError(_Grpc.StatusCode.NOT_FOUND) + return self.sandbox + + def exec(self, sandbox_id: str, command: list[str]) -> object: + assert sandbox_id == self.sandbox.id + assert command + return SimpleNamespace(exit_code=0, stdout="aiq-openshell-ready") + + def delete(self, name: str) -> bool: + assert name == self.sandbox.name + self.delete_calls += 1 + self.deleted = True + return True + + def wait_deleted(self, name: str) -> None: + assert name == self.sandbox.name + + +class _NoLabelClient(_Client): + def create(self, *, spec: object) -> object: + del spec + return self.sandbox + + def list(self) -> list[object]: + return [self.sandbox] + + +class _DeleteFailureClient(_Client): + def delete(self, name: str) -> bool: + assert name == self.sandbox.name + self.delete_calls += 1 + return False + + +class _MismatchedNameClient(_Client): + def __init__(self) -> None: + super().__init__() + self.deleted_name: str | None = None + + def create(self, *, spec: object, name: str, labels: dict[str, str]) -> object: + del spec, name + self.sandbox.name = "gateway-returned-name" + self.sandbox.labels = labels + return self.sandbox + + def delete(self, name: str) -> bool: + self.deleted_name = name + return super().delete(name) + + +@contextmanager +def _fake_runtime(client: _Client): + openshell = ModuleType("openshell") + sandbox_module = ModuleType("openshell.sandbox") + sandbox_module.SandboxClient = type(client) # type: ignore[attr-defined] + proto_module = ModuleType("openshell._proto") + proto_module.openshell_pb2 = SimpleNamespace( # type: ignore[attr-defined] + SANDBOX_PHASE_READY=2, + POLICY_STATUS_UNSPECIFIED=0, + POLICY_STATUS_PENDING=1, + POLICY_STATUS_LOADED=2, + POLICY_STATUS_FAILED=3, + GetSandboxPolicyStatusRequest=lambda **kwargs: SimpleNamespace(**kwargs), + ) + proto_module.sandbox_pb2 = SimpleNamespace( # type: ignore[attr-defined] + POLICY_SOURCE_UNSPECIFIED=0, + POLICY_SOURCE_SANDBOX=1, + GetSandboxConfigRequest=lambda **kwargs: SimpleNamespace(**kwargs), + ) + grpc = _Grpc("grpc") + with patch.dict( + sys.modules, + { + "grpc": grpc, + "openshell": openshell, + "openshell._proto": proto_module, + "openshell.sandbox": sandbox_module, + }, + ): + yield + + +def _config(checker: ModuleType, tmp_path: Path, **overrides: Any) -> object: + values = { + "gateway": "enterprise", + "image": "aiq:test", + "policy": tmp_path / "policy.yaml", + "openshell_bin": tmp_path / "openshell", + "ready_timeout_seconds": 1.0, + "policy_load_timeout_seconds": 0.01, + } + values.update(overrides) + return checker.ReadinessConfig(**values) + + +@contextmanager +def _provider_helpers(client: _Client): + prefix = "aiq_agent.agents.deep_researcher.sandbox.providers.openshell" + policy = SimpleNamespace(version=1) + client._stub.config.policy = policy + client._stub.status.revision.policy = policy + with ( + patch(f"{prefix}._read_policy_data", return_value={"version": 1}), + patch(f"{prefix}._parse_policy_proto", return_value=policy), + patch(f"{prefix}._build_sandbox_spec", return_value="spec"), + ): + yield policy + + +def test_strict_probe_verifies_and_deletes_resource(checker: ModuleType, tmp_path: Path) -> None: + client = _Client() + with ( + _provider_helpers(client), + _fake_runtime(client), + patch.object(checker.importlib.metadata, "version", return_value="1.2.3"), + patch.object(checker, "_version_from_cli", return_value="1.2.3"), + ): + versions = checker.run_check(_config(checker, tmp_path)) + + assert versions == ("1.2.3", "1.2.3") + assert client.delete_calls == 1 + assert client.deleted is True + + +def test_probe_rejects_version_mismatch_before_creation(checker: ModuleType, tmp_path: Path) -> None: + client = _Client() + with ( + _provider_helpers(client), + _fake_runtime(client), + patch.object(checker.importlib.metadata, "version", return_value="1.2.4"), + patch.object(checker, "_version_from_cli", return_value="1.2.3"), + pytest.raises(checker.ReadinessError, match="version_mismatch"), + ): + checker.run_check(_config(checker, tmp_path)) + + assert client.delete_calls == 0 + + +@pytest.mark.parametrize("field", ["active_version", "current_policy_version"]) +def test_probe_rejects_unreported_policy_versions(checker: ModuleType, tmp_path: Path, field: str) -> None: + client = _Client() + if field == "active_version": + client._stub.status.active_version = 0 + else: + client.sandbox.current_policy_version = 0 + with ( + _provider_helpers(client), + _fake_runtime(client), + patch.object(checker.importlib.metadata, "version", return_value="1.2.3"), + patch.object(checker, "_version_from_cli", return_value="1.2.3"), + pytest.raises(checker.ReadinessError, match="version_mismatch"), + ): + checker.run_check(_config(checker, tmp_path)) + + assert client.delete_calls == 1 + + +def test_probe_accepts_equivalent_python_and_cargo_development_versions( + checker: ModuleType, + tmp_path: Path, +) -> None: + client = _Client(version="0.0.79-dev.3+g616ff2f6") + with ( + _provider_helpers(client), + _fake_runtime(client), + patch.object(checker.importlib.metadata, "version", return_value="0.0.79.dev3+g616ff2f6"), + patch.object(checker, "_version_from_cli", return_value="0.0.79-dev.3+g616ff2f6"), + ): + versions = checker.run_check(_config(checker, tmp_path)) + + assert versions == ("0.0.79.dev3+g616ff2f6", "0.0.79-dev.3+g616ff2f6") + assert client.delete_calls == 1 + + +def test_probe_rejects_different_development_commit(checker: ModuleType, tmp_path: Path) -> None: + client = _Client(version="0.0.79-dev.3+g616ff2f7") + with ( + _provider_helpers(client), + _fake_runtime(client), + patch.object(checker.importlib.metadata, "version", return_value="0.0.79.dev3+g616ff2f6"), + patch.object(checker, "_version_from_cli", return_value="0.0.79-dev.3+g616ff2f6"), + pytest.raises(checker.ReadinessError, match="version_mismatch"), + ): + checker.run_check(_config(checker, tmp_path)) + + assert client.delete_calls == 0 + + +def test_probe_rejects_sdk_without_request_label_support(checker: ModuleType, tmp_path: Path) -> None: + client = _NoLabelClient() + with ( + _provider_helpers(client), + _fake_runtime(client), + patch.object(checker.importlib.metadata, "version", return_value="1.2.3"), + patch.object(checker, "_version_from_cli", return_value="1.2.3"), + pytest.raises(checker.ReadinessError, match="request_labels_unsupported"), + ): + checker.run_check(_config(checker, tmp_path)) + + assert client.delete_calls == 0 + + +def test_probe_classifies_effective_policy_that_remains_pending(checker: ModuleType, tmp_path: Path) -> None: + client = _Client(status=1) + with ( + _provider_helpers(client), + _fake_runtime(client), + patch.object(checker.importlib.metadata, "version", return_value="1.2.3"), + patch.object(checker, "_version_from_cli", return_value="1.2.3"), + pytest.raises(checker.ReadinessError, match="policy_status_inconsistent"), + ): + checker.run_check(_config(checker, tmp_path)) + + assert client.delete_calls == 1 + + +@pytest.mark.parametrize("surface", ["config", "revision"]) +def test_probe_rejects_missing_authoritative_policy_hash( + checker: ModuleType, + tmp_path: Path, + surface: str, +) -> None: + client = _Client() + target = client._stub.config if surface == "config" else client._stub.status.revision + target.policy_hash = "" + with ( + _provider_helpers(client), + _fake_runtime(client), + patch.object(checker.importlib.metadata, "version", return_value="1.2.3"), + patch.object(checker, "_version_from_cli", return_value="1.2.3"), + pytest.raises(checker.ReadinessError, match="policy_hash_missing"), + ): + checker.run_check(_config(checker, tmp_path)) + + assert client.delete_calls == 1 + + +def test_probe_rejects_unequal_authoritative_policy_hashes(checker: ModuleType, tmp_path: Path) -> None: + client = _Client() + client._stub.status.revision.policy_hash = "other-hash" + with ( + _provider_helpers(client), + _fake_runtime(client), + patch.object(checker.importlib.metadata, "version", return_value="1.2.3"), + patch.object(checker, "_version_from_cli", return_value="1.2.3"), + pytest.raises(checker.ReadinessError, match="policy_hash_mismatch"), + ): + checker.run_check(_config(checker, tmp_path)) + + assert client.delete_calls == 1 + + +def test_probe_rejects_selector_metadata_mismatch_and_cleans_up(checker: ModuleType, tmp_path: Path) -> None: + client = _Client(labels={"aiq": "wrong"}, persist_request_labels=False) + with ( + _provider_helpers(client), + _fake_runtime(client), + patch.object(checker.importlib.metadata, "version", return_value="1.2.3"), + patch.object(checker, "_version_from_cli", return_value="1.2.3"), + pytest.raises(checker.ReadinessError, match="selector_mismatch"), + ): + checker.run_check(_config(checker, tmp_path)) + + assert client.delete_calls == 1 + + +def test_probe_cleans_up_sdk_returned_name_on_name_mismatch(checker: ModuleType, tmp_path: Path) -> None: + client = _MismatchedNameClient() + with ( + _provider_helpers(client), + _fake_runtime(client), + patch.object(checker.importlib.metadata, "version", return_value="1.2.3"), + patch.object(checker, "_version_from_cli", return_value="1.2.3"), + pytest.raises(checker.ReadinessError, match="probe_failed"), + ): + checker.run_check(_config(checker, tmp_path)) + + assert client.deleted_name == "gateway-returned-name" + assert client.deleted is True + + +def test_probe_reports_cleanup_failure(checker: ModuleType, tmp_path: Path) -> None: + client = _DeleteFailureClient() + with ( + _provider_helpers(client), + _fake_runtime(client), + patch.object(checker.importlib.metadata, "version", return_value="1.2.3"), + patch.object(checker, "_version_from_cli", return_value="1.2.3"), + pytest.raises(checker.ReadinessError, match="cleanup_failed"), + ): + checker.run_check(_config(checker, tmp_path)) + + assert client.delete_calls == 1 + + +def test_main_prints_only_sanitized_reason(checker: ModuleType, capsys: pytest.CaptureFixture[str]) -> None: + with patch.object(checker, "run_check", side_effect=checker.ReadinessError("selector_mismatch")): + result = checker.main( + [ + "--policy-file", + "policy.yaml", + "--openshell-bin", + "openshell", + ] + ) + + assert result == 1 + assert capsys.readouterr().err.strip() == "OpenShell readiness check failed: selector_mismatch" diff --git a/tests/scripts/test_openshell_smoke_wrapper.py b/tests/scripts/test_openshell_smoke_wrapper.py new file mode 100644 index 000000000..e65086149 --- /dev/null +++ b/tests/scripts/test_openshell_smoke_wrapper.py @@ -0,0 +1,100 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the thin OpenShell live-acceptance wrapper.""" + +from __future__ import annotations + +import importlib.util +import subprocess +import sys +from pathlib import Path +from types import ModuleType +from types import SimpleNamespace + +import pytest + +_REPO_ROOT = Path(__file__).resolve().parents[2] +_WRAPPER = _REPO_ROOT / "scripts" / "openshell" / "smoke_openshell_isolation.py" +_LIVE_TEST = "tests/aiq_agent/agents/deep_researcher/sandbox/test_openshell_live.py" + + +@pytest.fixture +def wrapper() -> ModuleType: + spec = importlib.util.spec_from_file_location("smoke_openshell_isolation", _WRAPPER) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_wrapper_translates_arguments_and_preserves_environment( + wrapper: ModuleType, + monkeypatch: pytest.MonkeyPatch, +) -> None: + recorded: dict[str, object] = {} + + def fake_run(command: list[str], **kwargs: object) -> SimpleNamespace: + recorded.update(command=command, **kwargs) + return SimpleNamespace(returncode=0) + + monkeypatch.setenv("AIQ_EXISTING_SETTING", "preserved") + monkeypatch.setattr(subprocess, "run", fake_run) + + result = wrapper.main( + [ + "--gateway", + "enterprise", + "--policy", + "policy.yaml", + "--image", + "image:tag", + "--expected-gateway-version", + "0.0.80", + "--allow-best-effort-landlock", + ] + ) + + assert result == 0 + assert recorded["command"] == [sys.executable, "-m", "pytest", "-m", "integration", "-vv", _LIVE_TEST] + assert recorded["cwd"] == _REPO_ROOT + assert recorded["check"] is False + env = recorded["env"] + assert isinstance(env, dict) + assert env["AIQ_EXISTING_SETTING"] == "preserved" + assert env["AIQ_OPENSHELL_LIVE_TESTS"] == "1" + assert env["AIQ_OPENSHELL_GATEWAY_NAME"] == "enterprise" + assert env["AIQ_OPENSHELL_POLICY_FILE"] == "policy.yaml" + assert env["AIQ_OPENSHELL_IMAGE"] == "image:tag" + assert env["AIQ_OPENSHELL_EXPECTED_GATEWAY_VERSION"] == "0.0.80" + assert env["AIQ_OPENSHELL_LIVE_ALLOW_BEST_EFFORT"] == "1" + + +def test_wrapper_uses_generated_policy_and_single_pytest_target( + wrapper: ModuleType, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("AIQ_OPENSHELL_EXPECTED_GATEWAY_VERSION", raising=False) + monkeypatch.delenv("AIQ_OPENSHELL_POLICY_FILE", raising=False) + args = wrapper._args([]) + + assert args.policy == "configs/openshell/generated/aiq-openshell-policy.yaml" + assert args.expected_gateway_version is None + assert wrapper._command()[-1] == _LIVE_TEST + assert wrapper._command().count(_LIVE_TEST) == 1 + + +def test_wrapper_propagates_pytest_exit_code( + wrapper: ModuleType, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(subprocess, "run", lambda *args, **kwargs: SimpleNamespace(returncode=17)) + + assert wrapper.main([]) == 17 + + +def test_wrapper_does_not_import_openshell_sdk(wrapper: ModuleType) -> None: + source = _WRAPPER.read_text(encoding="utf-8") + + assert "import openshell" not in source + assert "import grpc" not in source diff --git a/tests/scripts/test_openshell_version_tools.py b/tests/scripts/test_openshell_version_tools.py new file mode 100644 index 000000000..7d69923a0 --- /dev/null +++ b/tests/scripts/test_openshell_version_tools.py @@ -0,0 +1,232 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the OpenShell release contract and safe component diagnostics.""" + +from __future__ import annotations + +import importlib.util +import json +import sys +from pathlib import Path +from types import ModuleType + +import pytest + +_REPO_ROOT = Path(__file__).resolve().parents[2] +_SCRIPT_DIR = _REPO_ROOT / "scripts" / "openshell" + + +def _load(name: str) -> ModuleType: + path = _SCRIPT_DIR / f"{name}.py" + spec = importlib.util.spec_from_file_location(name, path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + sys.path.insert(0, str(_SCRIPT_DIR)) + try: + spec.loader.exec_module(module) + finally: + sys.path.remove(str(_SCRIPT_DIR)) + return module + + +@pytest.fixture(scope="module") +def contract_module() -> ModuleType: + return _load("version_contract") + + +@pytest.fixture(scope="module") +def inspector() -> ModuleType: + _load("version_contract") + return _load("check_versions") + + +def test_release_contract_is_pinned_to_certified_installer(contract_module: ModuleType) -> None: + contract = contract_module.load_contract() + + assert contract.version == "0.0.80" + assert contract.release_tag == "v0.0.80" + assert ( + contract.installer_sha256 + == "c15d6cb8090e1c7c8d79a320b5bcbdaf1c15c2363942d81e84b56e03b836249e" # pragma: allowlist secret + ) + assert contract.adapter_version == "0.1.0" + + +def _patch_common( + inspector: ModuleType, + monkeypatch: pytest.MonkeyPatch, + *, + gateway_type: str = "local", + gateway_version: str | None = "0.0.80", +) -> None: + monkeypatch.setattr(inspector.importlib.metadata, "version", lambda _name: "0.0.80") + monkeypatch.setattr(inspector, "_cli_version", lambda _path: "0.0.80") + monkeypatch.setattr(inspector, "_gateway_type", lambda _path, _name: gateway_type) + monkeypatch.setattr(inspector, "_live_gateway_version", lambda _name: gateway_version) + + +def test_matching_local_components_are_accepted(inspector: ModuleType, monkeypatch: pytest.MonkeyPatch) -> None: + _patch_common(inspector, monkeypatch) + monkeypatch.setattr( + inspector, + "_homebrew_components", + lambda: (["nvidia/openshell/openshell"], "nvidia/openshell/openshell", "0.0.80", "0.0.80"), + ) + + report = inspector.inspect_components(gateway_name="openshell", system="Darwin") + + assert report.reason_code is None + assert report.live_gateway_version == "0.0.80" + + +def test_missing_local_package_prints_exact_installer(inspector: ModuleType, monkeypatch: pytest.MonkeyPatch) -> None: + _patch_common(inspector, monkeypatch, gateway_version=None) + monkeypatch.setattr(inspector, "_homebrew_components", lambda: ([], None, None, None)) + + report = inspector.inspect_components(gateway_name="openshell", include_live=False, system="Darwin") + + assert report.reason_code == "packaged_gateway_missing" + assert report.remediation == "./scripts/openshell/install_gateway.sh" + + +def test_matching_but_stopped_local_gateway_recommends_launcher( + inspector: ModuleType, + monkeypatch: pytest.MonkeyPatch, +) -> None: + _patch_common(inspector, monkeypatch, gateway_version=None) + monkeypatch.setattr( + inspector, + "_homebrew_components", + lambda: (["nvidia/openshell/openshell"], "nvidia/openshell/openshell", "0.0.80", "0.0.80"), + ) + + report = inspector.inspect_components(gateway_name="openshell", system="Darwin") + + assert report.reason_code == "gateway_unavailable" + assert "start_openshell_gateway.sh" in str(report.remediation) + + +@pytest.mark.parametrize( + ("formula_version", "packaged_version"), + [("0.0.72", "0.0.72"), ("0.0.80", "0.0.72")], +) +def test_stale_local_package_is_classified( + inspector: ModuleType, + monkeypatch: pytest.MonkeyPatch, + formula_version: str, + packaged_version: str, +) -> None: + _patch_common(inspector, monkeypatch) + monkeypatch.setattr( + inspector, + "_homebrew_components", + lambda: (["nvidia/openshell/openshell"], "nvidia/openshell/openshell", formula_version, packaged_version), + ) + + report = inspector.inspect_components(gateway_name="openshell", system="Darwin") + + assert report.reason_code == "component_version_mismatch" + + +def test_multiple_local_installations_fail_as_ambiguous(inspector: ModuleType, monkeypatch: pytest.MonkeyPatch) -> None: + _patch_common(inspector, monkeypatch) + monkeypatch.setattr( + inspector, + "_homebrew_components", + lambda: (["nvidia/openshell/openshell", "aiq/local-openshell/openshell"], None, None, None), + ) + + report = inspector.inspect_components(gateway_name="openshell", system="Darwin") + + assert report.reason_code == "ambiguous_gateway_installation" + + +def test_bare_formula_is_preserved_as_ambiguous(inspector: ModuleType, monkeypatch: pytest.MonkeyPatch) -> None: + _patch_common(inspector, monkeypatch) + monkeypatch.setattr(inspector.shutil, "which", lambda _name: "/opt/homebrew/bin/brew") + + def fake_run(command: list[str]) -> str | None: + args = command[1:] + if args == ["list", "--formula", "--full-name"]: + return "openshell" + if args == ["services", "list", "--json"]: + return "[]" + if args == ["list", "--versions", "openshell"]: + return "openshell 0.0.80" + if args == ["--prefix", "openshell"]: + return "/opt/homebrew/opt/openshell" + return None + + monkeypatch.setattr(inspector, "_run", fake_run) + monkeypatch.setattr(inspector, "_cli_version", lambda _path: "0.0.80") + + formulas, formula, formula_version, packaged_version = inspector._homebrew_components() + + assert formulas == ["openshell"] + assert formula == "openshell" + assert formula_version == "0.0.80" + assert packaged_version == "0.0.80" + + report = inspector.inspect_components(gateway_name="openshell", system="Darwin") + + assert report.reason_code == "ambiguous_gateway_installation" + + +def test_remote_mismatch_never_recommends_local_install(inspector: ModuleType, monkeypatch: pytest.MonkeyPatch) -> None: + _patch_common(inspector, monkeypatch, gateway_type="remote", gateway_version="0.0.72") + + report = inspector.inspect_components(gateway_name="enterprise", system="Darwin") + + assert report.reason_code == "remote_gateway_version_mismatch" + assert "install_gateway.sh" not in str(report.remediation) + assert "enterprise" in str(report.remediation) + + +def test_sdk_mismatch_recommends_aiq_environment_repair( + inspector: ModuleType, + monkeypatch: pytest.MonkeyPatch, +) -> None: + _patch_common(inspector, monkeypatch, gateway_type="remote") + monkeypatch.setattr(inspector.importlib.metadata, "version", lambda _name: "0.0.72") + + report = inspector.inspect_components(gateway_name="enterprise", system="Linux") + + assert report.reason_code == "component_version_mismatch" + assert "setup_openshell.sh --openshell-version 0.0.80" in str(report.remediation) + + +def test_json_output_contains_only_allowlisted_fields( + inspector: ModuleType, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + report = inspector.ComponentReport( + certified_version="0.0.80", + sdk_version="0.0.80", + virtualenv_cli_version="0.0.80", + homebrew_formula=None, + homebrew_formula_version=None, + packaged_cli_version=None, + live_gateway_version="0.0.80", + gateway_type="remote", + reason_code=None, + remediation=None, + ) + monkeypatch.setattr(inspector, "inspect_components", lambda **_kwargs: report) + + assert inspector.main(["--json", "--gateway-name", "enterprise"]) == 0 + payload = json.loads(capsys.readouterr().out) + assert set(payload) == { + "certified_version", + "gateway_type", + "homebrew_formula", + "homebrew_formula_version", + "live_gateway_version", + "packaged_cli_version", + "reason_code", + "remediation", + "sdk_version", + "virtualenv_cli_version", + }