diff --git a/INSTRUCTIONS.md b/INSTRUCTIONS.md index a00fec63..21a21e3d 100644 --- a/INSTRUCTIONS.md +++ b/INSTRUCTIONS.md @@ -39,7 +39,10 @@ Run from the **repo root**: uv sync ``` -`uv sync` creates a virtual environment at `.venv/`, installs all dependencies, and registers the CLI entry points (`plan-execute`, `*-mcp-server`). You can either prefix commands with `uv run` (no activation needed) or activate the venv once for your shell session: +`uv sync` creates a virtual environment at `.venv/`, installs all dependencies, +and registers the CLI entry points (`plan-execute`, `*-agent`, `*-mcp-server`). +You can either prefix commands with `uv run` (no activation needed) or activate +the venv once for your shell session: ```bash source .venv/bin/activate # macOS / Linux @@ -92,7 +95,7 @@ See [MCP Servers](#mcp-servers) for available tools and [docs/mcp-servers.md](do | `WO_DBNAME` | `workorder` | Work order database name | | `VIBRATION_DBNAME` | `vibration` | Vibration sensor database name | -**WatsonX** — plan-execute runner (when `--model-id` starts with `watsonx/`) +**WatsonX** — plan-execute runner and WatsonX-backed agent routes | Variable | Default | Description | | -------------------- | ----------------------------------- | --------------------------- | @@ -100,17 +103,26 @@ See [MCP Servers](#mcp-servers) for available tools and [docs/mcp-servers.md](do | `WATSONX_PROJECT_ID` | _(required)_ | IBM WatsonX project ID | | `WATSONX_URL` | `https://us-south.ml.cloud.ibm.com` | WatsonX endpoint (optional) | -**LiteLLM proxy** — used by every runner whenever `--model-id` carries the `litellm_proxy/` prefix (the default for claude-agent, openai-agent, deep-agent) +**LiteLLM proxy** — used by every runner whenever `--model-id` carries the `litellm_proxy/` prefix | Variable | Default | Description | | ------------------ | ------------ | -------------------------------------------------------------------- | | `LITELLM_API_KEY` | _(required)_ | LiteLLM proxy API key | | `LITELLM_BASE_URL` | _(required)_ | LiteLLM proxy base URL, e.g. `https://your-litellm-host.example.com` | -**TokenRouter** — OpenAI-compatible gateway, used whenever `--model-id` carries the `tokenrouter/` prefix (the default for direct-llm-agent) +**TokenRouter** — OpenAI-compatible gateway, used whenever `--model-id` carries the `tokenrouter/` prefix -| `TOKENROUTER_API_KEY` | _(tokenrouter/* models)_ | TokenRouter API key | -| `TOKENROUTER_BASE_URL` | _(tokenrouter/* models)_ | TokenRouter base URL, e.g. `https://api.tokenrouter.com/v1` | +| Variable | Default | Description | +| ---------------------- | ------------------------ | ------------------------------------------------------------------- | +| `TOKENROUTER_API_KEY` | _(tokenrouter/* models)_ | TokenRouter API key | +| `TOKENROUTER_BASE_URL` | _(tokenrouter/* models)_ | TokenRouter base URL, e.g. `https://api.tokenrouter.com/v1` | + +**OpenCode direct providers** — `opencode-agent` with direct provider routes + +| Variable | Default | Description | +| ------------------- | -------------------------- | ------------------------------------------------ | +| `OPENAI_API_KEY` | _(openai/* models)_ | OpenAI API key, or configure through OpenCode | +| `ANTHROPIC_API_KEY` | _(anthropic/* models)_ | Anthropic API key, or configure through OpenCode | **Stirrup code track** — `stirrup-agent` with `--code-backend docker` @@ -127,7 +139,7 @@ Six FastMCP servers cover IoT data, time-series ML, work orders, vibration diagn | Server | Tools | Categories | Backing service | | ----------- | ----- | ------------------------ | -------------------------------------- | -| `iot` | 7 | read | CouchDB (telemetry + asset registry) | +| `iot` | 7 | read | CouchDB (telemetry + asset registry) | | `utilities` | 3 | read | none | | `fmsr` | 2 | read, LLM-use | LiteLLM + `failure_modes.yaml` | | `wo` | 14 | read, write | CouchDB | @@ -154,16 +166,20 @@ query="What is the current date and time? Also list assets at site MAIN. Also ge ## Agents -Seven runners are available as CLIs registered by `uv sync`; six use MCP tools, while `direct-llm-agent` is a model-only baseline that makes a direct LiteLLM call without MCP tools, planning, retrieval, or code execution. Each is a CLI registered by `uv sync` that takes a single positional `question` argument and spawns the MCP servers as stdio subprocesses on demand. +Seven runners are available as CLIs registered by `uv sync`; six use MCP tools, +while `direct-llm-agent` is a model-only baseline that makes a direct LiteLLM +call without MCP tools, planning, retrieval, or code execution. Each takes a +single positional `question` argument and spawns the MCP servers as stdio +subprocesses on demand. | Runner | Source | Loop | Default model | | -------------- | ---------------------------- | ------------------------------------------------------------- | ----------------------------------------------------------- | | `plan-execute` | `src/agent/plan_execute/` | Custom plan → execute → summarise (no SDK) | `watsonx/meta-llama/llama-4-maverick-17b-128e-instruct-fp8` | | `claude-agent` | `src/agent/claude_agent/` | [`claude-agent-sdk`](https://github.com/anthropics/claude-agent-sdk-python) agentic loop | `litellm_proxy/aws/claude-opus-4-6` | -| `openai-agent` | `src/agent/openai_agent/` | [`openai-agents`](https://github.com/openai/openai-agents-python) SDK Runner | `litellm_proxy/azure/gpt-5.4` | +| `openai-agent` | `src/agent/openai_agent/` | [`openai-agents`](https://github.com/openai/openai-agents-python) SDK Runner | `litellm_proxy/azure/gpt-5.4` | | `deep-agent` | `src/agent/deep_agent/` | [LangChain deep-agents](https://docs.langchain.com/oss/python/deepagents/overview) (LangGraph), MCP bridged via `langchain-mcp-adapters` | `litellm_proxy/aws/claude-opus-4-6` | | `stirrup-agent` | `src/agent/stirrup_agent/` | [Stirrup](https://github.com/ArtificialAnalysis/Stirrup) agent loop (in-process), MCP via its `MCPToolProvider`; **code-capable** (writes/runs Python) | `watsonx/meta-llama/llama-4-maverick-17b-128e-instruct-fp8` | -| `opencode-agent` | `src/agent/opencode_agent/` | [OpenCode](https://opencode.ai/docs/) CLI agent loop, MCP via generated OpenCode config; web/bash/edit denied by default | `opencode/gpt-5.1-codex` | +| `opencode-agent` | `src/agent/opencode_agent/` | [OpenCode](https://opencode.ai/docs/) CLI agent loop, MCP via generated OpenCode config; MCP-only by default with optional CLI workspace mode | `opencode/gpt-5.1-codex` | | `direct-llm-agent` | `src/agent/direct_llm_agent/` | Single direct LLM call, no MCP tools, planning, retrieval, or code execution | `litellm_proxy/Azure/gpt-5-mini-2025-08-07` | - [Agents](#agents) — Stirrup specifics in [docs/stirrup-agent.md](docs/stirrup-agent.md); OpenCode specifics in [docs/opencode-agent.md](docs/opencode-agent.md) @@ -203,6 +219,7 @@ uv run direct-llm-agent "$query" | `--max-tokens N` | stirrup-agent | Max output tokens per call; keep under provider limit (default 16384) | | `--max-steps N` | opencode-agent | Max OpenCode agentic iterations (default: 30) | | `--attach URL` | opencode-agent | Attach to a running `opencode serve` instance | +| `--allow-files` / `--workspace-dir PATH` | opencode-agent | Enable local file inspection in a dedicated workspace | | `--allow-bash` / `--allow-edit` / `--allow-web` | opencode-agent | Opt into shell, file edits, or web access; all denied by default | ### Examples @@ -229,14 +246,52 @@ uv run stirrup-agent --no-code --show-trajectory \ STIRRUP_CODE_IMAGE=assetops-code \ uv run stirrup-agent --code-backend docker "$query" -# OpenCode CLI-backed agent, using TokenRouter +# OpenCode MCP-only CLI-backed agent, using TokenRouter uv run opencode-agent --model-id tokenrouter/MiniMax-M3 --show-trajectory "$query" +# OpenCode CLI workspace mode, allowing file inspection and bash in a run workspace +uv run opencode-agent \ + --model-id tokenrouter/MiniMax-M3 \ + --workspace-dir /tmp/assetopsbench-opencode/smoke \ + --allow-files \ + --allow-bash \ + --show-trajectory \ + "$query" + # Direct model-only baseline, no MCP tools uv run direct-llm-agent --model-id litellm_proxy/Azure/gpt-5-mini-2025-08-07 \ 'Return only JSON: {"test": 1}' ``` +### OpenCode scenario-suite workspace mode + +The scenario-suite runner can create one workspace per OpenCode run and pass +optional CLI capabilities through to `opencode-agent`: + +```bash +uv run python -m benchmark.scenario_suite_runner \ + --scenario-ids benchmarks/scenario_suite/scenarios.txt \ + --scenario-root /path/to/scenarios_data \ + --agent_name opencode_agent \ + --model-id tokenrouter/MiniMax-M3 \ + --opencode-workspace-root traces/opencode_workspaces \ + --opencode-allow-files \ + --opencode-allow-bash +``` + +For scenario `401`, this creates a workspace like: + +```text +traces/opencode_workspaces/opencode_agent_401 +``` + +`--opencode-allow-files`, `--opencode-allow-bash`, and +`--opencode-allow-edit` are opt-in. If any of them are enabled, +`--opencode-workspace-root` is required. + +> `--opencode-allow-bash` is not a hard OS-level sandbox. For strict filesystem +> isolation, run the benchmark inside Docker or another sandbox. + --- ## Observability @@ -325,16 +380,17 @@ uv run pytest src/ -k "integration" # only files / tests with "in ## Architecture ``` -┌──────────────────────────────────────────────────────────────┐ -│ agent/ │ -│ │ -│ PlanExecuteRunner ClaudeAgentRunner StirrupAgentRunner │ -│ OpenAIAgentRunner DeepAgentRunner │ -│ │ -└──────────────────────────┬───────────────────────────────────┘ - │ MCP protocol (stdio) - ┌─────────────────┼───────────┬──────────┬──────┬───────────┐ - ▼ ▼ ▼ ▼ ▼ ▼ - iot utilities fmsr tsfm wo vibration - (tools) (tools) (tools) (tools) (tools) (tools) -``` +┌────────────────────────────────────────────────────────────────────────┐ +│ agent/ │ +│ │ +│ PlanExecuteRunner ClaudeAgentRunner StirrupAgentRunner │ +│ OpenAIAgentRunner DeepAgentRunner OpenCodeAgentRunner │ +│ DirectLLMAgentRunner │ +│ │ +└──────────────────────────────┬─────────────────────────────────────────┘ + │ MCP protocol (stdio) + ┌─────────────────────┼───────────┬──────────┬──────┬───────────┐ + ▼ ▼ ▼ ▼ ▼ ▼ + iot utilities fmsr tsfm wo vibration + (tools) (tools) (tools) (tools) (tools) (tools) +``` \ No newline at end of file diff --git a/docs/opencode-agent.md b/docs/opencode-agent.md index 20b569b6..7d5d98f1 100644 --- a/docs/opencode-agent.md +++ b/docs/opencode-agent.md @@ -6,15 +6,21 @@ the AssetOpsBench MCP servers. It is a peer to `plan-execute`, `claude-agent`, persisted `Trajectory`, scored by the same `uv run evaluate`. OpenCode is a general CLI coding agent, but this runner is configured as a -**tools-first benchmark agent** by default. It can call the AssetOpsBench MCP -servers, read/search local files, and return an answer. Shell commands, file -edits, web access, and follow-up questions are denied unless explicitly enabled. +**tools-first benchmark agent** by default. In the default mode, it can call the +AssetOpsBench MCP servers while local file reads, shell commands, file edits, +web access, external directory access, and follow-up questions are denied unless +explicitly enabled. + +The runner also supports an optional **CLI workspace mode**. In that mode, each +benchmark run gets a dedicated workspace directory and OpenCode can be allowed +to inspect files and/or run shell commands inside that run workspace. ## Contents - [Why OpenCode](#why-opencode) - [Install](#install) - [Quick start](#quick-start) +- [Execution modes](#execution-modes) - [Model routing](#model-routing) - [Permissions and web access](#permissions-and-web-access) - [Headless server mode](#headless-server-mode) @@ -34,6 +40,8 @@ edits, web access, and follow-up questions are denied unless explicitly enabled. other tool-using runner. - **Headless automation**, via `opencode run --format json`, which makes it usable in scripts and benchmark runs. +- **Optional CLI workspace mode**, so benchmark runs can evaluate agents that + write and run local analysis code. - **Server/API mode**, via `opencode serve`, which can keep OpenCode warm across repeated runs. - **Provider flexibility**, including direct OpenCode providers and @@ -52,7 +60,7 @@ Install the project dependencies from the repo root: uv sync ``` -Confirm the AssetOpsBench entry point resolved (no model call): +Confirm the AssetOpsBench entry point resolved: ```bash uv run opencode-agent --help @@ -71,7 +79,8 @@ docs, for example: npm install -g opencode-ai ``` -Run the unit tests for the config and trajectory mapping: +Run the unit tests for config, permissions, workspace handling, and trajectory +mapping: ```bash uv run pytest src/agent/opencode_agent/tests/ -q @@ -103,6 +112,69 @@ OpenCode discovered and called the AssetOpsBench MCP tools. --- +## Execution modes + +### Default MCP-only mode + +By default, OpenCode is allowed to use AssetOpsBench MCP tools only. + +The following capabilities are denied by default: + +- local file inspection +- shell/bash commands +- file edits +- web fetch/search +- external directory access +- follow-up questions + +This is the recommended mode for normal benchmark runs where answers should come +from the configured MCP tools. + +```bash +uv run python -m benchmark.scenario_suite_runner \ + --scenario-ids benchmarks/scenario_suite/scenarios.txt \ + --scenario-root /path/to/scenarios_data \ + --agent_name opencode_agent \ + --model-id tokenrouter/MiniMax-M3 +``` + +### CLI workspace mode + +CLI workspace mode is opt-in. It gives each OpenCode run a dedicated workspace +directory and can allow file inspection and/or shell execution. + +This mode is useful when evaluating CLI-style agents that can write small Python +scripts, run local analysis, and use intermediate artifacts while still using +MCP tools for operational data. + +```bash +uv run python -m benchmark.scenario_suite_runner \ + --scenario-ids benchmarks/scenario_suite/scenarios.txt \ + --scenario-root /path/to/scenarios_data \ + --agent_name opencode_agent \ + --model-id tokenrouter/MiniMax-M3 \ + --opencode-workspace-root traces/opencode_workspaces \ + --opencode-allow-files \ + --opencode-allow-bash +``` + +For scenario `401`, this creates a workspace such as: + +```text +traces/opencode_workspaces/opencode_agent_401 +``` + +OpenCode is instructed to use the current working directory as the run workspace +when file or bash access is enabled. It should write scripts, temporary files, +intermediate data, and final artifacts there. + +> **Safety note.** `--opencode-allow-bash` is not a hard OS-level sandbox. A +> shell or Python process can still attempt to access files outside the workspace. +> For strict filesystem isolation, run the benchmark inside Docker or another +> sandboxed environment. + +--- + ## Model routing The runner's default model is `opencode/gpt-5.1-codex`. @@ -136,10 +208,11 @@ The runner configures OpenCode permissions for benchmark use: | Capability | Default | Flag to allow | | ---------- | ------- | ------------- | | AssetOpsBench MCP tools | allowed | always enabled | -| `read`, `glob`, `grep`, `lsp` | allowed | always enabled | +| `read`, `glob`, `grep`, `lsp` | denied | `--allow-files` | | shell commands | denied | `--allow-bash` | | file edits | denied | `--allow-edit` | | web fetch/search | denied | `--allow-web` | +| external directory access | denied | not exposed | | follow-up questions | denied | not exposed | Benchmark runs should not pass `--allow-web`. Without that flag, OpenCode's @@ -200,11 +273,24 @@ In addition to the [common flags](../INSTRUCTIONS.md#common-flags) (`--model-id` | `--opencode-bin PATH` | OpenCode executable path (default: `opencode`). | | `--attach URL` | Attach to a running `opencode serve` instance. | | `--timeout-s N` | Wall-clock timeout for `opencode run` (default: 900). | +| `--allow-files` | Allow file inspection tools (`read`, `glob`, `grep`, `lsp`). Disabled by default. | | `--allow-bash` | Allow shell commands. Disabled by default. | | `--allow-edit` | Allow file edits. Disabled by default. | | `--allow-web` | Allow web fetch/search. Disabled by default. | +| `--workspace-dir PATH` | Workspace directory required when files, bash, or edits are enabled. | | `--ask-permissions` | Do not auto-approve allowed permissions. Not suitable for batch runs. | +Direct CLI workspace example: + +```bash +uv run opencode-agent \ + --model-id tokenrouter/MiniMax-M3 \ + --workspace-dir /tmp/assetopsbench-opencode/smoke \ + --allow-files \ + --allow-bash \ + "Analyze the maintenance cost distribution." +``` + --- ## Benchmark-suite usage @@ -236,6 +322,31 @@ uv run python -m benchmark.scenario_suite_runner \ --no-evaluate ``` +To run with CLI workspace capabilities: + +```bash +uv run python -m benchmark.scenario_suite_runner \ + --scenario-ids benchmarks/scenario_suite/scenarios.txt \ + --scenario-root /path/to/scenarios_data \ + --agent_name opencode_agent \ + --model-id tokenrouter/MiniMax-M3 \ + --opencode-workspace-root traces/opencode_workspaces \ + --opencode-allow-files \ + --opencode-allow-bash +``` + +Scenario-suite OpenCode flags: + +| Flag | Description | +| ---- | ----------- | +| `--opencode-workspace-root PATH` | Root directory for per-run OpenCode workspaces. | +| `--opencode-allow-files` | Pass `--allow-files` to `opencode-agent`. | +| `--opencode-allow-bash` | Pass `--allow-bash` to `opencode-agent`. | +| `--opencode-allow-edit` | Pass `--allow-edit` to `opencode-agent`. | + +If any OpenCode file, bash, or edit capability is enabled, then +`--opencode-workspace-root` is required. + --- ## Validation runs @@ -251,25 +362,44 @@ opencode --version # 1. unit tests (no model call) uv run pytest src/agent/opencode_agent/tests/ -q -# 2. direct smoke test +# 2. direct MCP-only smoke test export TOKENROUTER_BASE_URL=https://api.tokenrouter.com/v1 export TOKENROUTER_API_KEY=... uv run opencode-agent --show-trajectory \ --model-id tokenrouter/MiniMax-M3 \ "What sites are available?" -# 3. persist a trajectory +# 3. direct CLI workspace smoke test +uv run opencode-agent --show-trajectory \ + --model-id tokenrouter/MiniMax-M3 \ + --workspace-dir /tmp/assetopsbench-opencode/smoke \ + --allow-files \ + --allow-bash \ + "What is the current time?" + +# 4. persist a trajectory export AGENT_TRAJECTORY_DIR=$(pwd)/traces/trajectories uv run opencode-agent --run-id opencode-smoke --scenario-id smoke \ --model-id tokenrouter/MiniMax-M3 \ "What sites are available?" -# 4. benchmark-suite dry run +# 5. benchmark-suite dry run +uv run python -m benchmark.scenario_suite_runner \ + --scenario-ids benchmarks/scenario_suite/scenarios.txt \ + --scenario-root /path/to/scenarios_data \ + --agent_name opencode_agent \ + --model-id tokenrouter/MiniMax-M3 \ + --dry-run + +# 6. benchmark-suite CLI workspace dry run uv run python -m benchmark.scenario_suite_runner \ --scenario-ids benchmarks/scenario_suite/scenarios.txt \ --scenario-root /path/to/scenarios_data \ --agent_name opencode_agent \ --model-id tokenrouter/MiniMax-M3 \ + --opencode-workspace-root traces/opencode_workspaces \ + --opencode-allow-files \ + --opencode-allow-bash \ --dry-run ``` @@ -283,9 +413,13 @@ uv run python -m benchmark.scenario_suite_runner \ | `Model not found: openai/MiniMax-M3` | Old router mapping. Use the current runner, which maps `tokenrouter/` to a custom `tokenrouter` provider instead of `openai/`. | | `TOKENROUTER_BASE_URL and TOKENROUTER_API_KEY must be set` | Set both env vars when using `tokenrouter/*`. The base URL should be OpenAI-compatible, usually ending in `/v1`. | | `LITELLM_BASE_URL and LITELLM_API_KEY must be set` | Set both env vars when using `litellm_proxy/*`. | +| `--workspace-dir is required when enabling files, edits, or bash` | Direct `opencode-agent` runs require a workspace when `--allow-files`, `--allow-bash`, or `--allow-edit` is passed. | +| `--opencode-workspace-root is required when enabling OpenCode files, bash, or edits` | Scenario-suite runs require a workspace root when any OpenCode CLI capability is enabled. | +| Workspace folders are empty | Normal if OpenCode did not need to write scripts or intermediate files. The workspace is available as scratch space, but files are created only if the model/tooling writes them. | | Run appears stuck after `Command: uv run opencode-agent ...` | Normal for long scenarios: `scenario_suite_runner` is quiet while OpenCode works. Check `traces/trajectories/scenario_suite/opencode_agent/` after completion. | | Evaluator fails even though the answer text contains the right label | The answer may be too verbose for `static_json` extraction. Prefer concise answers such as `C` or JSON like `{"answer": "C"}` for strict structured tasks. | | Web access concern | Web is denied by default. Do not pass `--allow-web` for benchmark runs. | +| Bash isolation concern | `--allow-bash` is not a hard OS sandbox. Use Docker or another sandbox for strict filesystem isolation. | | OpenCode prompts for permissions in a batch run | Do not pass `--ask-permissions`; the default auto-approves allowed tools and still denies explicitly denied tools. | --- @@ -295,7 +429,10 @@ uv run python -m benchmark.scenario_suite_runner \ - `src/agent/opencode_agent/` — `runner.py`, `cli.py`, `__init__.py`, and `tests/test_runner.py`. - `pyproject.toml` — `opencode-agent` entry point. -- `src/benchmark/scenario_suite_runner.py` — `opencode_agent` benchmark method. +- `src/benchmark/scenario_suite_runner.py` — `opencode_agent` benchmark method + and optional OpenCode workspace flags. +- `docs/opencode-agent.md` — OpenCode usage, permissions, and workspace-mode + documentation. No MCP servers are modified. The runner generates OpenCode MCP/provider config -at runtime. +at runtime. \ No newline at end of file diff --git a/src/agent/opencode_agent/cli.py b/src/agent/opencode_agent/cli.py index b1cb868b..56af77c2 100644 --- a/src/agent/opencode_agent/cli.py +++ b/src/agent/opencode_agent/cli.py @@ -9,6 +9,7 @@ from __future__ import annotations import argparse +from pathlib import Path from .._cli_common import add_common_args, print_result, run_sdk_cli @@ -88,6 +89,24 @@ def _build_parser() -> argparse.ArgumentParser: action="store_true", help="Allow OpenCode web fetch/search. Disabled by default for benchmark runs.", ) + parser.add_argument( + "--allow-files", + action="store_true", + help=( + "Allow OpenCode read/glob/grep/lsp tools inside --workspace-dir. " + "Disabled by default for tools-only benchmark runs." + ), + ) + parser.add_argument( + "--workspace-dir", + type=Path, + default=None, + metavar="PATH", + help=( + "Dedicated OpenCode run workspace. Required when enabling files, " + "edits, or bash." + ), + ) parser.add_argument( "--ask-permissions", action="store_true", @@ -109,6 +128,8 @@ async def _run(args: argparse.Namespace) -> None: allow_bash=args.allow_bash, allow_edit=args.allow_edit, allow_web=args.allow_web, + allow_files=args.allow_files, + workspace_dir=args.workspace_dir, dangerously_skip_permissions=not args.ask_permissions, ) result = await runner.run(args.question) diff --git a/src/agent/opencode_agent/runner.py b/src/agent/opencode_agent/runner.py index 44e3a9cc..4457c050 100644 --- a/src/agent/opencode_agent/runner.py +++ b/src/agent/opencode_agent/runner.py @@ -39,8 +39,12 @@ Use the configured AssetOpsBench MCP tools for operational data. Do not ask the user follow-up questions during benchmark runs; make reasonable assumptions and answer with the evidence you found. Do not edit files, run -shell commands, or browse the web unless those capabilities have been enabled -for this run. +shell commands, browse the web, or inspect local files unless those +capabilities have been enabled for this run. + +When file or bash access is enabled, use the current working directory as the +run workspace. Write any scripts, temporary files, intermediate data, and final +artifacts there. Do not read or write files outside the current workspace. """ ) @@ -79,13 +83,14 @@ def _build_permissions( allow_bash: bool = False, allow_edit: bool = False, allow_web: bool = False, + allow_files: bool = False, ) -> dict[str, Any]: """Build non-interactive permissions for benchmark-safe OpenCode runs.""" permission: dict[str, Any] = { - "read": "allow", - "glob": "allow", - "grep": "allow", - "lsp": "allow", + "read": "allow" if allow_files else "deny", + "glob": "allow" if allow_files else "deny", + "grep": "allow" if allow_files else "deny", + "lsp": "allow" if allow_files else "deny", "edit": "allow" if allow_edit else "deny", "bash": "allow" if allow_bash else "deny", "task": "deny", @@ -93,6 +98,7 @@ def _build_permissions( "question": "deny", "webfetch": "allow" if allow_web else "deny", "websearch": "allow" if allow_web else "deny", + "external_directory": "deny", "doom_loop": "deny", } for name in server_names: @@ -147,6 +153,7 @@ def _build_opencode_config( allow_bash: bool = False, allow_edit: bool = False, allow_web: bool = False, + allow_files: bool = False, ) -> tuple[dict[str, Any], dict[str, str], str]: """Return (OpenCode config, env overrides, resolved OpenCode model).""" opencode_model, provider, env = _resolve_opencode_model_and_provider(model) @@ -155,6 +162,7 @@ def _build_opencode_config( allow_bash=allow_bash, allow_edit=allow_edit, allow_web=allow_web, + allow_files=allow_files, ) config: dict[str, Any] = { "$schema": "https://opencode.ai/config.json", @@ -178,6 +186,33 @@ def _build_opencode_config( return config, env, opencode_model +def _resolve_run_dir( + *, + workspace_dir: Path | str | None = None, + allow_bash: bool = False, + allow_edit: bool = False, + allow_files: bool = False, +) -> Path: + """Return OpenCode's working directory for this run. + + The safe default is the repo root with local file/bash/edit tools denied. + Any filesystem/code capability must opt into a dedicated workspace folder. + This keeps the default comparable to tools-only MCP agents while allowing a + separate CLI/code-capable track. + """ + workspace_requested = allow_bash or allow_edit or allow_files + if workspace_requested and workspace_dir is None: + raise ValueError( + "--workspace-dir is required when enabling files, edits, or bash" + ) + if workspace_dir is None: + return _REPO_ROOT + + run_dir = Path(workspace_dir).expanduser().resolve() + run_dir.mkdir(parents=True, exist_ok=True) + return run_dir + + def _json_events(stdout: str) -> tuple[list[dict[str, Any]], list[str]]: """Parse OpenCode's JSON-lines event stream, preserving non-JSON lines.""" stripped = stdout.strip() @@ -375,6 +410,8 @@ def __init__( allow_bash: bool = False, allow_edit: bool = False, allow_web: bool = False, + allow_files: bool = False, + workspace_dir: Path | str | None = None, dangerously_skip_permissions: bool = True, ) -> None: super().__init__(llm, server_paths) @@ -385,6 +422,12 @@ def __init__( self._attach = attach self._timeout_s = timeout_s self._dangerously_skip_permissions = dangerously_skip_permissions + self._run_dir = _resolve_run_dir( + workspace_dir=workspace_dir, + allow_bash=allow_bash, + allow_edit=allow_edit, + allow_files=allow_files, + ) self._config, self._env_overrides, self._opencode_model = ( _build_opencode_config( model=model, @@ -394,6 +437,7 @@ def __init__( allow_bash=allow_bash, allow_edit=allow_edit, allow_web=allow_web, + allow_files=allow_files, ) ) @@ -416,7 +460,7 @@ async def run(self, question: str) -> AgentResult: "--agent", self._agent_name, "--dir", - str(_REPO_ROOT), + str(self._run_dir), "--title", "AssetOpsBench", ] diff --git a/src/agent/opencode_agent/tests/test_runner.py b/src/agent/opencode_agent/tests/test_runner.py index ad23feac..d1391df6 100644 --- a/src/agent/opencode_agent/tests/test_runner.py +++ b/src/agent/opencode_agent/tests/test_runner.py @@ -12,6 +12,7 @@ _build_permissions, _build_trajectory_from_events, _json_events, + _resolve_run_dir, _resolve_opencode_model_and_provider, ) @@ -36,20 +37,58 @@ def test_build_permissions_default_safe(): permission = _build_permissions(["iot", "wo"]) assert permission["iot_*"] == "allow" assert permission["wo_*"] == "allow" + assert permission["read"] == "deny" + assert permission["glob"] == "deny" + assert permission["grep"] == "deny" + assert permission["lsp"] == "deny" assert permission["bash"] == "deny" assert permission["edit"] == "deny" + assert permission["webfetch"] == "deny" assert permission["websearch"] == "deny" + assert permission["external_directory"] == "deny" assert permission["question"] == "deny" def test_build_permissions_allows_opt_in_tools(): permission = _build_permissions( - ["iot"], allow_bash=True, allow_edit=True, allow_web=True + ["iot"], + allow_bash=True, + allow_edit=True, + allow_web=True, + allow_files=True, ) + assert permission["read"] == "allow" + assert permission["glob"] == "allow" + assert permission["grep"] == "allow" + assert permission["lsp"] == "allow" assert permission["bash"] == "allow" assert permission["edit"] == "allow" assert permission["webfetch"] == "allow" assert permission["websearch"] == "allow" + assert permission["external_directory"] == "deny" + + +def test_resolve_run_dir_defaults_to_repo_root(): + run_dir = _resolve_run_dir() + assert run_dir.name == "AssetOpsBench-main" + + +def test_resolve_run_dir_requires_workspace_for_file_or_code_tools(): + import pytest + + with pytest.raises(ValueError, match="workspace-dir"): + _resolve_run_dir(allow_files=True) + with pytest.raises(ValueError, match="workspace-dir"): + _resolve_run_dir(allow_bash=True) + with pytest.raises(ValueError, match="workspace-dir"): + _resolve_run_dir(allow_edit=True) + + +def test_resolve_run_dir_creates_workspace(tmp_path): + workspace = tmp_path / "opencode-run" + run_dir = _resolve_run_dir(workspace_dir=workspace, allow_files=True) + assert run_dir == workspace.resolve() + assert run_dir.exists() def test_resolve_direct_opencode_model(): @@ -94,6 +133,7 @@ def test_build_opencode_config_includes_agent_and_mcp(): assert opencode_model == "opencode/gpt-5" assert config["agent"]["assetops"]["steps"] == 7 assert config["agent"]["assetops"]["permission"]["iot_*"] == "allow" + assert config["agent"]["assetops"]["permission"]["read"] == "deny" assert config["mcp"]["iot"]["command"] == ["uv", "run", "iot-mcp-server"] @@ -144,3 +184,16 @@ def test_runner_defaults(): assert runner._model_id == "opencode/gpt-5" assert runner._opencode_model == "opencode/gpt-5" assert runner._agent_name == "assetops" + assert runner._run_dir.name == "AssetOpsBench-main" + + +def test_runner_workspace_mode(tmp_path): + workspace = tmp_path / "run-401" + runner = OpenCodeAgentRunner( + server_paths={}, + model="opencode/gpt-5", + allow_files=True, + workspace_dir=workspace, + ) + assert runner._run_dir == workspace.resolve() + assert runner._config["agent"]["assetops"]["permission"]["read"] == "allow" diff --git a/src/benchmark/scenario_suite_runner.py b/src/benchmark/scenario_suite_runner.py index f34c2bff..2ea6782e 100644 --- a/src/benchmark/scenario_suite_runner.py +++ b/src/benchmark/scenario_suite_runner.py @@ -44,6 +44,8 @@ class MethodConfig: agent_name: str command: str model_id: str + extra_args: tuple[str, ...] = () + workspace_root: Path | None = None def load_scenario_ids(path: Path) -> list[str]: @@ -150,12 +152,20 @@ def run_agent_for_scenario( env = os.environ.copy() env["AGENT_TRAJECTORY_DIR"] = str(trajectory_dir) + extra_args = list(method.extra_args) + if method.workspace_root is not None: + workspace_dir = method.workspace_root / run_id + extra_args.extend(["--workspace-dir", str(workspace_dir)]) + if not dry_run: + workspace_dir.mkdir(parents=True, exist_ok=True) + cmd = [ "uv", "run", method.command, "--model-id", method.model_id, + *extra_args, "--scenario-id", scenario_id, "--run-id", @@ -217,6 +227,14 @@ def run_evaluation( def build_methods(args: argparse.Namespace) -> dict[str, MethodConfig]: """Build available method configs from CLI args.""" + opencode_extra_args: list[str] = [] + if args.opencode_allow_files: + opencode_extra_args.append("--allow-files") + if args.opencode_allow_bash: + opencode_extra_args.append("--allow-bash") + if args.opencode_allow_edit: + opencode_extra_args.append("--allow-edit") + return { "direct_llm": MethodConfig( agent_name="direct_llm", @@ -232,6 +250,8 @@ def build_methods(args: argparse.Namespace) -> dict[str, MethodConfig]: agent_name="opencode_agent", command="opencode-agent", model_id=args.model_id, + extra_args=tuple(opencode_extra_args), + workspace_root=args.opencode_workspace_root, ), } @@ -293,6 +313,31 @@ def _build_parser() -> argparse.ArgumentParser: default=_DEFAULT_MODEL_ID, help="Model id used by both agents.", ) + parser.add_argument( + "--opencode-workspace-root", + type=Path, + default=None, + help=( + "Root directory for per-run OpenCode workspaces. Required when " + "using --opencode-allow-files, --opencode-allow-bash, or " + "--opencode-allow-edit." + ), + ) + parser.add_argument( + "--opencode-allow-files", + action="store_true", + help="Allow opencode-agent read/glob/grep/lsp tools inside its per-run workspace.", + ) + parser.add_argument( + "--opencode-allow-bash", + action="store_true", + help="Allow opencode-agent bash in its per-run workspace.", + ) + parser.add_argument( + "--opencode-allow-edit", + action="store_true", + help="Allow opencode-agent file edits inside its per-run workspace.", + ) parser.add_argument( "--skip-existing", action="store_true", @@ -321,6 +366,17 @@ def main() -> None: parser = _build_parser() args = parser.parse_args() + opencode_workspace_required = ( + args.opencode_allow_files + or args.opencode_allow_bash + or args.opencode_allow_edit + ) + if opencode_workspace_required and args.opencode_workspace_root is None: + parser.error( + "--opencode-workspace-root is required when enabling OpenCode " + "files, bash, or edits" + ) + scenario_ids = load_scenario_ids(args.scenario_ids) methods = selected_methods( method_name=args.agent_name, diff --git a/src/benchmark/tests/test_scenario_suite_runner.py b/src/benchmark/tests/test_scenario_suite_runner.py index ae435a42..69dc36f0 100644 --- a/src/benchmark/tests/test_scenario_suite_runner.py +++ b/src/benchmark/tests/test_scenario_suite_runner.py @@ -56,8 +56,11 @@ def test_read_question_raises_when_missing(tmp_path: Path) -> None: def test_build_methods_uses_cli_defaults() -> None: args = Namespace( - direct_model_id="tokenrouter/MiniMax-M3", - stirrup_model_id="tokenrouter/MiniMax-M3", + model_id="tokenrouter/MiniMax-M3", + opencode_allow_files=False, + opencode_allow_bash=False, + opencode_allow_edit=False, + opencode_workspace_root=None, ) methods = mr.build_methods(args) @@ -66,6 +69,25 @@ def test_build_methods_uses_cli_defaults() -> None: assert methods["direct_llm"].model_id == "tokenrouter/MiniMax-M3" assert methods["stirrup_agent"].command == "stirrup-agent" assert methods["stirrup_agent"].model_id == "tokenrouter/MiniMax-M3" + assert methods["opencode_agent"].command == "opencode-agent" + assert methods["opencode_agent"].extra_args == () + assert methods["opencode_agent"].workspace_root is None + + +def test_build_methods_opencode_workspace_options(tmp_path: Path) -> None: + args = Namespace( + model_id="tokenrouter/MiniMax-M3", + opencode_allow_files=True, + opencode_allow_bash=True, + opencode_allow_edit=False, + opencode_workspace_root=tmp_path / "workspaces", + ) + + methods = mr.build_methods(args) + opencode = methods["opencode_agent"] + + assert opencode.extra_args == ("--allow-files", "--allow-bash") + assert opencode.workspace_root == tmp_path / "workspaces" def test_selected_methods_direct_llm_only() -> None: @@ -120,7 +142,7 @@ def fake_run(*args, **kwargs): monkeypatch.setattr(mr.subprocess, "run", fake_run) method = mr.MethodConfig( - name="direct_llm", + agent_name="direct_llm", command="direct-llm-agent", model_id="tokenrouter/MiniMax-M3", ) @@ -136,6 +158,53 @@ def fake_run(*args, **kwargs): assert called is False +def test_run_agent_for_scenario_adds_opencode_workspace( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + captured = {} + + def fake_run(cmd, **kwargs): + captured["cmd"] = cmd + captured["kwargs"] = kwargs + + monkeypatch.setattr(mr.subprocess, "run", fake_run) + + method = mr.MethodConfig( + agent_name="opencode_agent", + command="opencode-agent", + model_id="tokenrouter/MiniMax-M3", + extra_args=("--allow-files", "--allow-bash"), + workspace_root=tmp_path / "workspaces", + ) + + mr.run_agent_for_scenario( + method=method, + scenario_id="401", + question="Which excavator costs the most?", + trajectory_dir=tmp_path / "traj", + dry_run=False, + ) + + expected_workspace = tmp_path / "workspaces" / "opencode_agent_401" + assert expected_workspace.exists() + assert captured["cmd"] == [ + "uv", + "run", + "opencode-agent", + "--model-id", + "tokenrouter/MiniMax-M3", + "--allow-files", + "--allow-bash", + "--workspace-dir", + str(expected_workspace), + "--scenario-id", + "401", + "--run-id", + "opencode_agent_401", + "Which excavator costs the most?", + ] + + def test_run_evaluation_dry_run_does_not_call_subprocess( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -155,4 +224,4 @@ def fake_run(*args, **kwargs): dry_run=True, ) - assert called is False \ No newline at end of file + assert called is False