diff --git a/CHANGELOG.md b/CHANGELOG.md index 2bf587852..2ca61c041 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,30 @@ ## [Unreleased] +### Changed +- **BREAKING (task.md): the `environment:` frontmatter key is renamed to + `sandbox:`.** The native task-config surface now accepts only `sandbox:` + (plus `verifier.sandbox:` for the verifier's separate sandbox spec and + `verifier.sandbox_mode:` for shared/separate selection); `environment:`, + `verifier.environment:`, and `verifier.environment_mode:` no longer + validate and fail with an actionable message naming the rename. **The + one-line fix for existing task.md files is renaming the key.** + Legacy/Harbor `task.toml` imports are unaffected: the toml loader + converts `[environment]`, `[verifier.environment]`, and + `environment_mode` to the `sandbox` spellings (declaring both spellings + in one file is an error), and `bench tasks export` emits the inverse — + a stock-Harbor `[environment]`-spelled `task.toml`. All native emitters + — `model_dump_toml`, `bench tasks migrate`, task scaffolding, + skill-eval/trace/adapter task generation, rubric-review wrappers — now + write `sandbox`. Python API: the compat property + `TaskConfig.environment` is removed (use `TaskConfig.sandbox`), + `VerifierConfig.environment`/`environment_mode` became + `sandbox`/`sandbox_mode`, and the `VerifierEnvironmentMode` enum is now + `VerifierSandboxMode`. The Environment plane + (`--environment-manifest`, `benchflow.environment.manifest`, the + eval-config `environment:` docker/daytona selector) is a different + subsystem and is unchanged. + ## 0.6.7 — 2026-08-09 ### Added diff --git a/benchmarks/clawsbench/tasks/archive-amazon-shipping/task.md b/benchmarks/clawsbench/tasks/archive-amazon-shipping/task.md index 78de120c7..d8e692e44 100644 --- a/benchmarks/clawsbench/tasks/archive-amazon-shipping/task.md +++ b/benchmarks/clawsbench/tasks/archive-amazon-shipping/task.md @@ -12,7 +12,7 @@ agent: timeout_sec: 300 verifier: timeout_sec: 120 -environment: +sandbox: cpus: 1 memory_mb: 2048 allow_internet: true diff --git a/benchmarks/continuallearningbench/benchflow.py b/benchmarks/continuallearningbench/benchflow.py index 0c93a92d3..e78d22b49 100644 --- a/benchmarks/continuallearningbench/benchflow.py +++ b/benchmarks/continuallearningbench/benchflow.py @@ -193,7 +193,7 @@ def _render_task_toml(task: ContinualLearningBenchTaskInfo) -> str: [verifier] timeout_sec = 300.0 -[environment] +[sandbox] build_timeout_sec = 600 cpus = 2 memory_mb = 4096 diff --git a/benchmarks/continuallearningbench/parity_test.py b/benchmarks/continuallearningbench/parity_test.py index 873665f9f..585270fc6 100644 --- a/benchmarks/continuallearningbench/parity_test.py +++ b/benchmarks/continuallearningbench/parity_test.py @@ -80,8 +80,8 @@ def _check_structural(task_dir: Path) -> list[str]: errors.append(f"{name}: task.toml missing [agent] section") if "[verifier]" not in content: errors.append(f"{name}: task.toml missing [verifier] section") - if "[environment]" not in content: - errors.append(f"{name}: task.toml missing [environment] section") + if "[sandbox]" not in content: + errors.append(f"{name}: task.toml missing [sandbox] section") # instruction.md instruction = task_dir / "instruction.md" diff --git a/benchmarks/harvey-lab/benchflow.py b/benchmarks/harvey-lab/benchflow.py index df7bc0514..3acdbfdfb 100644 --- a/benchmarks/harvey-lab/benchflow.py +++ b/benchmarks/harvey-lab/benchflow.py @@ -89,7 +89,7 @@ def _build_task_toml( [verifier.env] ANTHROPIC_API_KEY = "${{ANTHROPIC_API_KEY}}" - [environment] + [sandbox] build_timeout_sec = 600 cpus = 1 memory_mb = 4096 diff --git a/benchmarks/hilbench/benchflow.py b/benchmarks/hilbench/benchflow.py index 5b30e5058..5a318d5f3 100644 --- a/benchmarks/hilbench/benchflow.py +++ b/benchmarks/hilbench/benchflow.py @@ -121,7 +121,7 @@ def _render_task_toml(task: HILBenchTask) -> str: [verifier] timeout_sec = {verifier_timeout} -[environment] +[sandbox] cpus = 2 memory_mb = 4096 storage_mb = 20480 diff --git a/benchmarks/mle-bench/benchflow.py b/benchmarks/mle-bench/benchflow.py index 9c47eb2a1..4db550174 100644 --- a/benchmarks/mle-bench/benchflow.py +++ b/benchmarks/mle-bench/benchflow.py @@ -415,7 +415,7 @@ def _render_task_toml(competition: MLEBenchCompetition) -> str: timeout_sec = 3600 user = "root" - [environment] + [sandbox] build_timeout_sec = 3600 cpus = 4 memory_mb = 16384 diff --git a/benchmarks/opaquetoolsbench/benchflow.py b/benchmarks/opaquetoolsbench/benchflow.py index 4909a9300..68645db51 100644 --- a/benchmarks/opaquetoolsbench/benchflow.py +++ b/benchmarks/opaquetoolsbench/benchflow.py @@ -161,7 +161,7 @@ def _render_task_toml(task: BFCLTask) -> str: [verifier] timeout_sec = {_VERIFIER_TIMEOUT} -[environment] +[sandbox] cpus = 1 memory_mb = 1024 storage_mb = 2048 diff --git a/benchmarks/programbench/benchflow.py b/benchmarks/programbench/benchflow.py index bebab30f7..11f4cadff 100644 --- a/benchmarks/programbench/benchflow.py +++ b/benchmarks/programbench/benchflow.py @@ -114,7 +114,7 @@ def _render_task_toml(task: ProgramBenchTask) -> str: [verifier] timeout_sec = {verifier_timeout} -[environment] +[sandbox] cpus = 2 memory_mb = 4096 storage_mb = 20480 diff --git a/docs/examples/task-md/generated-skill-eval/models-as-skills/optimize-quadratic-to-nlogn/task.md b/docs/examples/task-md/generated-skill-eval/models-as-skills/optimize-quadratic-to-nlogn/task.md index 6ede56805..14e4341af 100644 --- a/docs/examples/task-md/generated-skill-eval/models-as-skills/optimize-quadratic-to-nlogn/task.md +++ b/docs/examples/task-md/generated-skill-eval/models-as-skills/optimize-quadratic-to-nlogn/task.md @@ -11,7 +11,7 @@ agent: timeout_sec: 600 verifier: timeout_sec: 120 -environment: +sandbox: cpus: 1 memory_mb: 2048 allow_internet: true diff --git a/docs/examples/task-md/generated-skill-eval/models-as-skills/regex-email-parser/task.md b/docs/examples/task-md/generated-skill-eval/models-as-skills/regex-email-parser/task.md index 2f996dae4..93e54b763 100644 --- a/docs/examples/task-md/generated-skill-eval/models-as-skills/regex-email-parser/task.md +++ b/docs/examples/task-md/generated-skill-eval/models-as-skills/regex-email-parser/task.md @@ -11,7 +11,7 @@ agent: timeout_sec: 600 verifier: timeout_sec: 120 -environment: +sandbox: cpus: 1 memory_mb: 2048 allow_internet: true diff --git a/docs/examples/task-md/generated-skill-eval/models-as-skills/topo-sort-with-cycle-detection/task.md b/docs/examples/task-md/generated-skill-eval/models-as-skills/topo-sort-with-cycle-detection/task.md index 080778134..283e56bac 100644 --- a/docs/examples/task-md/generated-skill-eval/models-as-skills/topo-sort-with-cycle-detection/task.md +++ b/docs/examples/task-md/generated-skill-eval/models-as-skills/topo-sort-with-cycle-detection/task.md @@ -11,7 +11,7 @@ agent: timeout_sec: 600 verifier: timeout_sec: 120 -environment: +sandbox: cpus: 1 memory_mb: 2048 allow_internet: true diff --git a/docs/examples/task-md/harbor-parity/task.md b/docs/examples/task-md/harbor-parity/task.md index aedc5796e..3b281b5b8 100644 --- a/docs/examples/task-md/harbor-parity/task.md +++ b/docs/examples/task-md/harbor-parity/task.md @@ -26,16 +26,16 @@ verifier: JUDGE_API_KEY: ${JUDGE_API_KEY:-test} user: root network_mode: public - environment_mode: separate + sandbox_mode: separate pytest_plugins: [pytest_playwright] hardening: cleanup_conftests: false - environment: + sandbox: docker_image: ghcr.io/example/grader:latest cpus: 2 memory_mb: 1024 network_mode: no-network -environment: +sandbox: network_mode: allowlist allowed_hosts: [datasets.example.com] build_timeout_sec: 600 diff --git a/docs/examples/task-md/multi-scene/task.md b/docs/examples/task-md/multi-scene/task.md index 00171ab8d..e98ca6105 100644 --- a/docs/examples/task-md/multi-scene/task.md +++ b/docs/examples/task-md/multi-scene/task.md @@ -9,7 +9,7 @@ agent: timeout_sec: 900 verifier: timeout_sec: 180 -environment: +sandbox: cpus: 2 memory_mb: 4096 agents: diff --git a/docs/examples/task-md/nudgebench-team/task.md b/docs/examples/task-md/nudgebench-team/task.md index 89bed9e85..4bfe73ffd 100644 --- a/docs/examples/task-md/nudgebench-team/task.md +++ b/docs/examples/task-md/nudgebench-team/task.md @@ -9,7 +9,7 @@ agent: timeout_sec: 1200 verifier: timeout_sec: 240 -environment: +sandbox: cpus: 2 memory_mb: 4096 agents: diff --git a/docs/examples/task-md/real-skillsbench/3d-scan-calc/task.md b/docs/examples/task-md/real-skillsbench/3d-scan-calc/task.md index 8250628ed..b6d646b71 100644 --- a/docs/examples/task-md/real-skillsbench/3d-scan-calc/task.md +++ b/docs/examples/task-md/real-skillsbench/3d-scan-calc/task.md @@ -31,7 +31,7 @@ verifier: timeout_sec: 900 agent: timeout_sec: 900 -environment: +sandbox: build_timeout_sec: 600 cpus: 1 memory_mb: 4096 diff --git a/docs/examples/task-md/real-skillsbench/citation-check-network/README.md b/docs/examples/task-md/real-skillsbench/citation-check-network/README.md index ba93d105e..d49240e3e 100644 --- a/docs/examples/task-md/real-skillsbench/citation-check-network/README.md +++ b/docs/examples/task-md/real-skillsbench/citation-check-network/README.md @@ -10,7 +10,7 @@ declares a hardened **`network_mode: allowlist`** scoped to exactly the hosts th ## What it is -- `task.md` — same prompt / metadata as the base task, but its `environment:` +- `task.md` — same prompt / metadata as the base task, but its `sandbox:` block sets: ```yaml diff --git a/docs/examples/task-md/real-skillsbench/citation-check-network/task.md b/docs/examples/task-md/real-skillsbench/citation-check-network/task.md index 1c7a29dbe..365566870 100644 --- a/docs/examples/task-md/real-skillsbench/citation-check-network/task.md +++ b/docs/examples/task-md/real-skillsbench/citation-check-network/task.md @@ -36,7 +36,7 @@ verifier: timeout_sec: 900 agent: timeout_sec: 900 -environment: +sandbox: build_timeout_sec: 600 cpus: 1 memory_mb: 2048 diff --git a/docs/examples/task-md/real-skillsbench/citation-check/task.md b/docs/examples/task-md/real-skillsbench/citation-check/task.md index 405e53168..fdcb4476b 100644 --- a/docs/examples/task-md/real-skillsbench/citation-check/task.md +++ b/docs/examples/task-md/real-skillsbench/citation-check/task.md @@ -33,7 +33,7 @@ verifier: timeout_sec: 900 agent: timeout_sec: 900 -environment: +sandbox: build_timeout_sec: 600 cpus: 1 memory_mb: 2048 diff --git a/docs/examples/task-md/real-skillsbench/weighted-gdp-calc/task.md b/docs/examples/task-md/real-skillsbench/weighted-gdp-calc/task.md index 9c549a23b..0ebf196f0 100644 --- a/docs/examples/task-md/real-skillsbench/weighted-gdp-calc/task.md +++ b/docs/examples/task-md/real-skillsbench/weighted-gdp-calc/task.md @@ -29,7 +29,7 @@ verifier: timeout_sec: 900 agent: timeout_sec: 900 -environment: +sandbox: build_timeout_sec: 600 cpus: 1 memory_mb: 4096 diff --git a/docs/examples/task-md/user-runtime/private-facts-nudges/task.md b/docs/examples/task-md/user-runtime/private-facts-nudges/task.md index 7d24da363..08c0cd9bc 100644 --- a/docs/examples/task-md/user-runtime/private-facts-nudges/task.md +++ b/docs/examples/task-md/user-runtime/private-facts-nudges/task.md @@ -10,7 +10,7 @@ agent: timeout_sec: 600 verifier: timeout_sec: 120 -environment: +sandbox: network_mode: no-network cpus: 1 memory_mb: 2048 diff --git a/docs/task-authoring-task-md.md b/docs/task-authoring-task-md.md index d96602d2d..7a04ebd69 100644 --- a/docs/task-authoring-task-md.md +++ b/docs/task-authoring-task-md.md @@ -31,7 +31,7 @@ agent: timeout_sec: 300 # strongly recommended — unset means no wall-clock cap verifier: timeout_sec: 120 -environment: +sandbox: cpus: 1 memory_mb: 2048 --- @@ -76,7 +76,7 @@ so typos fail at parse time instead of becoming silently-ignored config: | `metadata` | Freeform mapping — difficulty, category, tags, anything descriptive | | `agent` | Agent run policy: `timeout_sec`, `user`, `network_mode`, `allowed_hosts` | | `verifier` | Verifier run policy: `timeout_sec` (default 600), `env`, `user`, `service`, … | -| `environment` | Sandbox: `docker_image`, `cpus`, `memory_mb`, `storage_mb`, `network_mode`, `env`, `workdir`, … | +| `sandbox` | Sandbox: `docker_image`, `cpus`, `memory_mb`, `storage_mb`, `network_mode`, `env`, `workdir`, … (legacy `task.toml` imports convert the Harbor `environment` table to this key; `environment:` in `task.md` is rejected with a rename hint) | | `oracle` | Oracle run policy: `env`, `timeout_sec` (import alias: `solution`) | | `source`, `artifacts`, `steps`, `multi_step_reward_strategy`, `reward` | Provenance, artifact, and reward metadata | @@ -99,7 +99,7 @@ canonical config under their short names: | Shorthand | Expands to | |---|---| | `name: hello-world` | `task.name: benchflow/hello-world` (a `/` in the value keeps your org) | -| `image: ubuntu:24.04` | `environment.docker_image: ubuntu:24.04` | +| `image: ubuntu:24.04` | `sandbox.docker_image: ubuntu:24.04` | | `verifier: verifier/` (string form) | `benchflow.verifier.path` / `.spec` / `.entrypoint` defaults | | `oracle: oracle/` (string form) | `benchflow.oracle.path` | | `profile: code-change` | Merges a named defaults bundle (see below) | diff --git a/docs/task-standard.md b/docs/task-standard.md index 152a98b47..3145f3adc 100644 --- a/docs/task-standard.md +++ b/docs/task-standard.md @@ -117,7 +117,7 @@ when unknown in native authoring mode: - `metadata` - `agent` - `verifier` -- `environment` +- `sandbox` (legacy `task.toml` import spelling: `environment`) - `oracle` (validation alias: `solution`) - `source` - `artifacts` @@ -512,7 +512,7 @@ assets as first-class objects: benchflow: provenance: images: - - field: environment.docker_image + - field: sandbox.docker_image reference: ghcr.io/org/task-image:2026-06 digest: sha256:... registry: ghcr.io @@ -767,11 +767,11 @@ Target compatibility rules: compat: source: harbor extra_paths: - - environment.modal.image + - sandbox.modal.image - steps[0].runner - verifier.reward_kit.metric extra: - environment: + sandbox: modal: image: registry.example.com/task:latest steps: @@ -949,7 +949,7 @@ semantics, and non-`main` verifier services on backends that cannot run them. It is wired into `bench tasks check --sandbox ` and the shared sandbox factory used by rollouts and `Environment.from_task()`. Unsupported parsed semantics now raise `UnsupportedTaskFeatureError` before Docker, Daytona, or -Modal construction. Safe absolute non-root `environment.workdir` values are +Modal construction. Safe absolute non-root `sandbox.workdir` values are materialized before agent and verifier setup. P2: Split native and adapter validation modes. diff --git a/src/benchflow/_utils/config_override.py b/src/benchflow/_utils/config_override.py index 4e5e2241f..476070ee0 100644 --- a/src/benchflow/_utils/config_override.py +++ b/src/benchflow/_utils/config_override.py @@ -144,8 +144,9 @@ def apply_config_override(config: Any, overlay: dict[str, Any] | None) -> Any: # Merge against the FIELD-NAME dump (``by_alias=False``) so overlays use the # canonical field names (``agent``, ``sandbox``, …); ``populate_by_name`` lets - # re-validation accept them. ``by_alias=True`` would force callers to write the - # alias (e.g. ``environment`` for ``sandbox``) — the bug this avoids. + # re-validation accept them. (``sandbox`` is now the only spelling — the + # legacy ``environment`` alias was removed in the rename — but ``oracle`` + # still serializes via alias, so the field-name dump stays load-bearing.) merged = deep_merge(config.model_dump(by_alias=False), overlay) patched = TaskConfig.model_validate(merged) logger.debug( diff --git a/src/benchflow/_utils/task_authoring/scaffolding.py b/src/benchflow/_utils/task_authoring/scaffolding.py index ebf076598..70cdd6510 100644 --- a/src/benchflow/_utils/task_authoring/scaffolding.py +++ b/src/benchflow/_utils/task_authoring/scaffolding.py @@ -333,7 +333,7 @@ def _write_legacy_task_files(task_dir: Path, name: str) -> None: [verifier] timeout_sec = 120 -[environment] +[sandbox] cpus = 1 memory_mb = 2048 """) @@ -412,7 +412,7 @@ def _write_task_md(task_dir: Path, name: str) -> None: timeout_sec: 300 verifier: timeout_sec: 120 -environment: +sandbox: cpus: 1 memory_mb: 2048 --- diff --git a/src/benchflow/acp/runtime.py b/src/benchflow/acp/runtime.py index 0de559536..0bf808155 100644 --- a/src/benchflow/acp/runtime.py +++ b/src/benchflow/acp/runtime.py @@ -549,7 +549,7 @@ async def connect_acp( dormant and the auto-approve policy ran unconditionally (#382 follow-up). ``mcp_servers`` are the task's configured MCP servers (mapped from - ``[[environment.mcp_servers]]``); they are attached to the ACP session at + ``[[sandbox.mcp_servers]]``); they are attached to the ACP session at ``session/new`` so the agent can reach them. ``None`` attaches none. Retries with exponential backoff on ConnectionError (Daytona SSH storms). diff --git a/src/benchflow/adapters/_toolathlon.py b/src/benchflow/adapters/_toolathlon.py index e33a2a653..8ada03227 100644 --- a/src/benchflow/adapters/_toolathlon.py +++ b/src/benchflow/adapters/_toolathlon.py @@ -508,7 +508,7 @@ def _toolathlon_task_toml( "metadata": metadata, "agent": {"timeout_sec": 1800.0}, "verifier": {"timeout_sec": 900.0}, - "environment": environment, + "sandbox": environment, } diff --git a/src/benchflow/adapters/harbor.py b/src/benchflow/adapters/harbor.py index cbf2b8ea6..b82951a5a 100644 --- a/src/benchflow/adapters/harbor.py +++ b/src/benchflow/adapters/harbor.py @@ -15,12 +15,14 @@ └── tests/ # test.sh — the verifier This adapter is consequently a thin *normalizer*: it loads the foreign -``task.toml`` through the native :class:`TaskConfig` validator (which already -handles Harbor's ``[environment]``-keyed sandbox section and the -``version`` -> ``schema_version`` rename), reads ``instruction.md``, and -records the build/solution/test files under their native relative paths. No -field remapping is needed — Harbor *is* the native format, which is exactly -what makes Terminal-Bench backward-compatible through this edge. +``task.toml`` through :func:`~benchflow.task.imports.import_task_config_toml`, +whose toml import boundary translates Harbor's ``[environment]``-keyed +sandbox section to the native ``sandbox`` key and handles the ``version`` -> +``schema_version`` rename before native :class:`TaskConfig` validation. It +then reads ``instruction.md`` and records the build/solution/test files under +their native relative paths. Beyond that spelling translation no field +remapping is needed — Harbor is structurally the native format, which is +exactly what makes Terminal-Bench backward-compatible through this edge. """ from __future__ import annotations diff --git a/src/benchflow/adapters/source.py b/src/benchflow/adapters/source.py index ac0200721..f23d5308e 100644 --- a/src/benchflow/adapters/source.py +++ b/src/benchflow/adapters/source.py @@ -243,7 +243,7 @@ def _materialize_mcp_atlas(ctx: _SourceContext, output_dir: Path) -> None: "MCP_ATLAS_JUDGE_MODEL": "${MCP_ATLAS_JUDGE_MODEL:-qwen/qwen-plus}", }, }, - "environment": { + "sandbox": { "cpus": 4, "memory_mb": 8192, "storage_mb": 10240, diff --git a/src/benchflow/demo_task/task.md b/src/benchflow/demo_task/task.md index f117c684e..5c59833dd 100644 --- a/src/benchflow/demo_task/task.md +++ b/src/benchflow/demo_task/task.md @@ -24,7 +24,7 @@ verifier: cleanup_conftests: true agent: timeout_sec: 120.0 -environment: +sandbox: network_mode: public build_timeout_sec: 600.0 os: linux diff --git a/src/benchflow/experimental/mcp/reviewer_server.py b/src/benchflow/experimental/mcp/reviewer_server.py index ecc024422..9dc754230 100644 --- a/src/benchflow/experimental/mcp/reviewer_server.py +++ b/src/benchflow/experimental/mcp/reviewer_server.py @@ -9,7 +9,7 @@ never has write access to /app/). Usage in task.toml: - [[environment.mcp_servers]] + [[sandbox.mcp_servers]] name = "reviewer" transport = "streamable-http" url = "http://localhost:8100/mcp" diff --git a/src/benchflow/review/wrapper.py b/src/benchflow/review/wrapper.py index 92f49ca70..f714c3b78 100644 --- a/src/benchflow/review/wrapper.py +++ b/src/benchflow/review/wrapper.py @@ -156,7 +156,7 @@ def main() -> int: timeout_sec: {verifier_timeout} agent: timeout_sec: {agent_timeout} -environment: +sandbox: docker_image: {image} workdir: /app{network_line} cpus: 1 diff --git a/src/benchflow/rollout/__init__.py b/src/benchflow/rollout/__init__.py index f532edf6a..64e674105 100644 --- a/src/benchflow/rollout/__init__.py +++ b/src/benchflow/rollout/__init__.py @@ -237,7 +237,7 @@ def _task_mcp_specs(task: Any) -> list[McpServerSpec]: - """Map the task's ``[[environment.mcp_servers]]`` entries to ACP specs. + """Map the task's ``[[sandbox.mcp_servers]]`` entries to ACP specs. This is the composition seam between the task-config layer (``MCPServerConfig``) and the ACP protocol layer (``McpServerSpec``) — kept @@ -247,7 +247,7 @@ def _task_mcp_specs(task: Any) -> list[McpServerSpec]: — reachable by the agent. Returns ``[]`` when the task declares none, preserving the historical default of attaching no MCP servers. """ - env_config = getattr(getattr(task, "config", None), "environment", None) + env_config = getattr(getattr(task, "config", None), "sandbox", None) configs = getattr(env_config, "mcp_servers", None) or [] return [ McpServerSpec( @@ -503,7 +503,7 @@ async def _run_one_environment_setup_command( async def _run_environment_setup_commands(env: Any, task: Any) -> None: """Run task-authored setup commands after sandbox start, before agent install.""" - env_config = getattr(getattr(task, "config", None), "environment", None) + env_config = getattr(getattr(task, "config", None), "sandbox", None) commands = list(getattr(env_config, "setup_commands", []) or []) if not commands: return @@ -539,7 +539,7 @@ async def _run_environment_setup_commands(env: Any, task: Any) -> None: async def _run_environment_healthcheck(env: Any, task: Any) -> None: """Gate rollout startup on the task-authored environment healthcheck.""" - env_config = getattr(getattr(task, "config", None), "environment", None) + env_config = getattr(getattr(task, "config", None), "sandbox", None) healthcheck = getattr(env_config, "healthcheck", None) if healthcheck is None: return @@ -951,7 +951,7 @@ async def setup(self) -> None: ), disallow=self._disallow_web_tools, ) - env_config = getattr(getattr(self._task, "config", None), "environment", None) + env_config = getattr(getattr(self._task, "config", None), "sandbox", None) task_skill_policy = resolve_task_skill_policy( task_path=cfg.task_path, skill_mode=cfg.recorded_skill_mode, diff --git a/src/benchflow/rollout/_setup.py b/src/benchflow/rollout/_setup.py index fd0b89ed4..33c95c7e8 100644 --- a/src/benchflow/rollout/_setup.py +++ b/src/benchflow/rollout/_setup.py @@ -51,7 +51,7 @@ def _task_disallows_internet(task: Any) -> bool: """Return True when task config requests no internet for the agent task.""" - env_config = getattr(getattr(task, "config", None), "environment", None) + env_config = getattr(getattr(task, "config", None), "sandbox", None) return getattr(env_config, "allow_internet", True) is False @@ -189,7 +189,7 @@ def _agent_process_kill_pattern(agent_launch: str) -> str | None: def _configured_task_workdir(task: Any) -> str | None: """Return the task-declared sandbox workdir, if any.""" - env_config = getattr(getattr(task, "config", None), "environment", None) + env_config = getattr(getattr(task, "config", None), "sandbox", None) value = getattr(env_config, "workdir", None) if not isinstance(value, str): return None @@ -200,7 +200,7 @@ def _configured_task_workdir(task: Any) -> str | None: def _validate_agent_workdir(workdir: str) -> None: path = PurePosixPath(workdir) if not path.is_absolute() or path == PurePosixPath("/"): - raise ValueError("environment.workdir must be an absolute non-root path") + raise ValueError("sandbox.workdir must be an absolute non-root path") async def _resolve_agent_cwd(env: Any, task: Any) -> str: @@ -231,7 +231,7 @@ async def _resolve_agent_cwd(env: Any, task: Any) -> str: if isinstance(return_code, int) and return_code != 0: stderr = (getattr(result, "stderr", "") or "").strip() raise RuntimeError( - f"failed to prepare environment.workdir {configured!r}: {stderr}" + f"failed to prepare sandbox.workdir {configured!r}: {stderr}" ) return (getattr(result, "stdout", "") or "").strip() or configured diff --git a/src/benchflow/sandbox/apple_container.py b/src/benchflow/sandbox/apple_container.py index 6b5a043c2..793e61db7 100644 --- a/src/benchflow/sandbox/apple_container.py +++ b/src/benchflow/sandbox/apple_container.py @@ -238,7 +238,7 @@ def _validate_definition(self) -> None: raise ValueError( "apple-container does not currently enforce no-network sandboxing. " "Use docker, daytona, or modal for tasks that require " - "environment.network_mode='no-network'." + "sandbox.network_mode='no-network'." ) def _image_tag(self) -> str: diff --git a/src/benchflow/sandbox/setup.py b/src/benchflow/sandbox/setup.py index 746399cea..dc7b7d358 100644 --- a/src/benchflow/sandbox/setup.py +++ b/src/benchflow/sandbox/setup.py @@ -663,13 +663,13 @@ def _create_sandbox_environment( When ``environment_manifest`` is provided, its declared controls take effect at sandbox-construction time: the manifest's runnable ``image`` - overrides ``task.config.environment.docker_image`` (so the manifest — + overrides ``task.config.sandbox.docker_image`` (so the manifest — not the task's local Dockerfile — drives image selection), and the manifest's ``task_selection`` + ``forward_env`` are resolved into a persistent env overlay so the values reach the container's entrypoint via compose and every subsequent ``sandbox.exec`` call. """ - env_config = task.config.environment + env_config = task.config.sandbox environment_dir = task_path / "environment" if not environment_dir.exists(): environment_dir = task.paths.environment_dir @@ -698,7 +698,7 @@ def _create_sandbox_environment( # Image control point — manifest's run target wins over task.toml's # docker_image so a benchmark author can pin the runtime image # from the manifest without editing every task.toml. - if env_config is task.config.environment: + if env_config is task.config.sandbox: env_config = env_config.model_copy(deep=True) env_config.docker_image = manifest_image manifest_env = resolve_manifest_runtime_env( diff --git a/src/benchflow/skill_eval/_core.py b/src/benchflow/skill_eval/_core.py index 4c006131d..9e04e99ba 100644 --- a/src/benchflow/skill_eval/_core.py +++ b/src/benchflow/skill_eval/_core.py @@ -262,24 +262,24 @@ def _build_task_toml(dataset: EvalDataset, case: EvalCase, with_skill: bool) -> if present_judge_keys: verifier["env"] = {k: f"${{{k}}}" for k in present_judge_keys} - environment_block: dict[str, Any] = { + sandbox_block: dict[str, Any] = { "cpus": 1, "memory_mb": 2048, "allow_internet": True, } if with_skill: - environment_block["skills_dir"] = dataset.skill_mount_dir + sandbox_block["skills_dir"] = dataset.skill_mount_dir # Forward per-case ``environment`` overrides (#392). The runner reads - # ``[environment.env]`` and forwards entries to the sandbox. + # ``[sandbox.env]`` and forwards entries to the sandbox. if case.environment: - environment_block["env"] = dict(case.environment) + sandbox_block["env"] = dict(case.environment) doc: dict[str, Any] = { "version": "1.0", "metadata": metadata, "agent": {"timeout_sec": dataset.timeout_sec}, "verifier": verifier, - "environment": environment_block, + "sandbox": sandbox_block, } return tomli_w.dumps(doc) diff --git a/src/benchflow/skill_policy.py b/src/benchflow/skill_policy.py index 2af91c040..d0ef57d00 100644 --- a/src/benchflow/skill_policy.py +++ b/src/benchflow/skill_policy.py @@ -143,7 +143,7 @@ def resolve_task_skill_policy( if source == SKILL_SOURCE_TASK_BUNDLED: sandbox_dir = validate_container_mount_path( declared_sandbox_skills_dir or "/skills", - "environment.skills_dir", + "sandbox.skills_dir", ) else: sandbox_dir = validate_container_mount_path("/skills") @@ -156,7 +156,7 @@ def resolve_task_skill_policy( host_dir = bundled sandbox_dir = validate_container_mount_path( declared_sandbox_skills_dir or "/skills", - "environment.skills_dir", + "sandbox.skills_dir", ) strip_bundled = False diff --git a/src/benchflow/task/__init__.py b/src/benchflow/task/__init__.py index 548dd8fd4..70543801b 100644 --- a/src/benchflow/task/__init__.py +++ b/src/benchflow/task/__init__.py @@ -26,8 +26,8 @@ TaskOS, TpuSpec, VerifierConfig, - VerifierEnvironmentMode, VerifierHardeningConfig, + VerifierSandboxMode, ) from benchflow.task.config import ( AgentConfig as TaskAgentConfig, @@ -173,7 +173,7 @@ "TpuSpec", "NetworkMode", "TaskOS", - "VerifierEnvironmentMode", + "VerifierSandboxMode", "MultiStepRewardStrategy", "PackageInfo", "Author", diff --git a/src/benchflow/task/_document_normalize.py b/src/benchflow/task/_document_normalize.py index b02df95cb..eea9fba3f 100644 --- a/src/benchflow/task/_document_normalize.py +++ b/src/benchflow/task/_document_normalize.py @@ -52,7 +52,7 @@ def normalize_task_document_frontmatter( _apply_image_shorthand( normalized, shorthand_image, - canonical_was_explicit=_has_nested(raw, ("environment", "docker_image")), + canonical_was_explicit=_has_nested(raw, ("sandbox", "docker_image")), ) _apply_path_shorthand(normalized, "verifier", verifier_path) _apply_path_shorthand(normalized, "oracle", oracle_path) @@ -150,8 +150,8 @@ def _apply_image_shorthand( return if not isinstance(value, str) or not value.strip(): raise TaskDocumentParseError("image must be a non-empty string") - environment = _ensure_mapping(normalized, "environment") - environment["docker_image"] = value.strip() + sandbox = _ensure_mapping(normalized, "sandbox") + sandbox["docker_image"] = value.strip() def _pop_path_shorthand(frontmatter: dict[str, Any], key: str) -> str | None: diff --git a/src/benchflow/task/_document_parse.py b/src/benchflow/task/_document_parse.py index db4e42c69..b3b4a8915 100644 --- a/src/benchflow/task/_document_parse.py +++ b/src/benchflow/task/_document_parse.py @@ -25,7 +25,7 @@ _mapping, normalize_task_document_frontmatter, ) -from benchflow.task.config import TaskConfig +from benchflow.task.config import TaskConfig, convert_legacy_environment_keys from benchflow.task.imports import import_task_config_toml TASK_DOCUMENT_FILENAME = "task.md" @@ -193,9 +193,11 @@ def render_task_md(frontmatter: dict[str, Any] | str, instruction: str) -> str: ``frontmatter`` may be a parsed config mapping or raw ``task.toml`` text. A legacy ``solution`` block is emitted as the native ``oracle`` block when no - ``oracle`` block is present; declaring both is rejected. Reserved section - headings embedded in ``instruction`` are escaped so they round-trip as prompt - text instead of fracturing the document into extra sections. + ``oracle`` block is present, and a legacy ``environment`` table is emitted + as the native ``sandbox`` key; declaring both spellings of either pair is + rejected. Reserved section headings embedded in ``instruction`` are escaped + so they round-trip as prompt text instead of fracturing the document into + extra sections. """ data = ( @@ -203,6 +205,10 @@ def render_task_md(frontmatter: dict[str, Any] | str, instruction: str) -> str: if isinstance(frontmatter, str) else deepcopy(frontmatter) ) + # Emitters follow the sandbox rename: legacy toml input (or a legacy + # mapping) is translated so rendered task.md frontmatter only ever + # carries the native 'sandbox' spelling. + data = convert_legacy_environment_keys(data) if "solution" in data: if "oracle" in data: raise ValueError( diff --git a/src/benchflow/task/_document_profiles.py b/src/benchflow/task/_document_profiles.py index b84fa6478..62859f40d 100644 --- a/src/benchflow/task/_document_profiles.py +++ b/src/benchflow/task/_document_profiles.py @@ -31,7 +31,7 @@ "timeout_sec": 1200, "user": "root", }, - "environment": { + "sandbox": { "docker_image": "ubuntu:24.04", "network_mode": "no-network", "cpus": 4, @@ -52,7 +52,7 @@ "metadata": { "category": "harbor-compatible", }, - "environment": { + "sandbox": { "cpus": 1, "memory_mb": 2048, }, diff --git a/src/benchflow/task/acceptance_live_validation.py b/src/benchflow/task/acceptance_live_validation.py index b2c43d4ae..fe2c1d090 100644 --- a/src/benchflow/task/acceptance_live_validation.py +++ b/src/benchflow/task/acceptance_live_validation.py @@ -93,7 +93,7 @@ def _parse_workspace( target: str | None = None if value is None: task = Task(task_dir) - target = task.config.environment.workdir or "/app" + target = task.config.sandbox.workdir or "/app" elif isinstance(value, dict): mapping = cast(dict[str, object], value) source_value = mapping.get("source", _WORKSPACE_SOURCE_CURRENT_WORKTREE) @@ -105,7 +105,7 @@ def _parse_workspace( target_value = mapping.get("target") if target_value is not None and not isinstance(target_value, str): return None, ["acceptance-live workspace.target must be a string"] - target = target_value or Task(task_dir).config.environment.workdir or "/app" + target = target_value or Task(task_dir).config.sandbox.workdir or "/app" else: return None, ["acceptance-live workspace must be a mapping when declared"] diff --git a/src/benchflow/task/config.py b/src/benchflow/task/config.py index da8f95440..ddc0b75b5 100644 --- a/src/benchflow/task/config.py +++ b/src/benchflow/task/config.py @@ -30,6 +30,115 @@ _NETWORK_HOST_LABEL_PATTERN = re.compile(r"^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$") _ENV_VAR_NAME_PATTERN = re.compile(r"^[A-Z_][A-Z0-9_]*$") +_ENVIRONMENT_RENAMED_ERROR = ( + "the 'environment' key was renamed to 'sandbox' — rename 'environment:' " + "to 'sandbox:' (legacy task.toml imports convert '[environment]' " + "automatically)" +) +_VERIFIER_ENVIRONMENT_RENAMED_ERROR = ( + "the 'verifier.environment' key was renamed to 'verifier.sandbox' — " + "rename the key (legacy task.toml imports convert " + "'[verifier.environment]' automatically)" +) +_VERIFIER_ENVIRONMENT_MODE_RENAMED_ERROR = ( + "the 'verifier.environment_mode' key was renamed to " + "'verifier.sandbox_mode' — rename the key (legacy task.toml imports " + "convert 'environment_mode' automatically)" +) + + +def _make_key_rename(old: str, new: str, reason: str): + """Build a rename closure with the key pair and error phrasing baked in. + + ``reason`` explains, in the both-declared error, why one spelling must + win (e.g. which one is the legacy Harbor spelling). + """ + + def rename(table: Any, context: str) -> None: + if not isinstance(table, dict) or old not in table: + return + if new in table: + raise ValueError( + f"task config declares both '{context}{old}' and " + f"'{context}{new}' — they are the same section " + f"({reason}); keep only '{context}{new}'" + ) + table[new] = table.pop(old) + + return rename + + +def _convert_sandbox_spelling( + data: dict[str, Any], *, old: str, new: str, old_mode: str, new_mode: str +) -> dict[str, Any]: + # True in both directions; the "keep only '{new}'" tail names the + # direction-appropriate resolution. + rename_sandbox = _make_key_rename( + old, new, "'environment' is the legacy Harbor spelling" + ) + rename_mode = _make_key_rename( + old_mode, new_mode, "'environment_mode' is the legacy Harbor spelling" + ) + + rename_sandbox(data, "") + verifier = data.get("verifier") + rename_sandbox(verifier, "verifier.") + rename_mode(verifier, "verifier.") + steps = data.get("steps") + if isinstance(steps, list): + for index in range(len(steps)): + step = steps[index] + if isinstance(step, dict): + step_verifier = step.get("verifier") + rename_sandbox(step_verifier, f"steps[{index}].verifier.") + rename_mode(step_verifier, f"steps[{index}].verifier.") + return data + + +def convert_legacy_environment_keys(data: dict[str, Any]) -> dict[str, Any]: + """Rename legacy Harbor ``environment`` keys to native ``sandbox``. + + ``task.toml`` is a foreign/legacy format (Harbor's), so its + ``environment`` spelling is translated here at the import boundary + instead of being modeled as an alias on the native schema. That keeps + exactly one spelling — ``sandbox`` — in the native surface (task.md + frontmatter, Python constructors) while Harbor and SkillsBench task.toml + corpora keep loading unchanged. Declaring both spellings in one file is + a hard error rather than a silent merge. + + Converts the top-level ``environment`` table, ``verifier.environment``, + ``verifier.environment_mode``, and their ``steps[*].verifier`` + equivalents. Mutates ``data`` in place and returns it; callers pass + freshly parsed or copied mappings. + """ + + return _convert_sandbox_spelling( + data, + old="environment", + new="sandbox", + old_mode="environment_mode", + new_mode="sandbox_mode", + ) + + +def convert_native_keys_to_legacy_environment(data: dict[str, Any]) -> dict[str, Any]: + """Rename native ``sandbox`` keys to Harbor's ``environment`` spelling. + + The exact inverse of :func:`convert_legacy_environment_keys`, for + Harbor-facing emitters (``bench tasks export``): a split layout written + for a stock Harbor consumer must spell the sandbox spec + ``[environment]`` / ``environment_mode``, which Harbor understands and + ``[sandbox]`` is invisible to. Mutates ``data`` in place and returns it. + """ + + return _convert_sandbox_spelling( + data, + old="sandbox", + new="environment", + old_mode="sandbox_mode", + new_mode="environment_mode", + ) + class TaskConfigModel(BaseModel): """Base model for task schema sections. @@ -57,8 +166,8 @@ class TaskOS(StrEnum): WINDOWS = "windows" -class VerifierEnvironmentMode(StrEnum): - """Whether the verifier runs in the agent environment or a separate one.""" +class VerifierSandboxMode(StrEnum): + """Whether the verifier runs in the agent sandbox or a separate one.""" SHARED = "shared" SEPARATE = "separate" @@ -307,16 +416,16 @@ class VerifierConfig(TaskConfigModel): default=None, description="Hostnames reachable when network_mode='allowlist'.", ) - environment_mode: VerifierEnvironmentMode | None = Field( + sandbox_mode: VerifierSandboxMode | None = Field( default=None, description=( - "Whether the verifier runs in the agent environment ('shared') " - "or a dedicated verifier environment ('separate')." + "Whether the verifier runs in the agent sandbox ('shared') " + "or a dedicated verifier sandbox ('separate')." ), ) - environment: SandboxConfig | None = Field( + sandbox: SandboxConfig | None = Field( default=None, - description="Optional separate verifier environment configuration.", + description="Optional separate verifier sandbox configuration.", ) service: str = Field( default="main", @@ -367,16 +476,23 @@ def validate_reward_range( # the reward contract in benchflow.rewards.validation. return None if value is None else validate_declared_reward_range(value) + @model_validator(mode="before") + @classmethod + def reject_renamed_environment_keys(cls, data: Any) -> Any: + if isinstance(data, dict): + if "environment" in data: + raise ValueError(_VERIFIER_ENVIRONMENT_RENAMED_ERROR) + if "environment_mode" in data: + raise ValueError(_VERIFIER_ENVIRONMENT_MODE_RENAMED_ERROR) + return data + @model_validator(mode="after") - def validate_verifier_environment(self) -> VerifierConfig: + def validate_verifier_sandbox(self) -> VerifierConfig: _validate_network_policy_fields(self.network_mode, self.allowed_hosts) - if ( - self.environment_mode == VerifierEnvironmentMode.SHARED - and self.environment is not None - ): + if self.sandbox_mode == VerifierSandboxMode.SHARED and self.sandbox is not None: raise ValueError( - "[verifier].environment_mode='shared' is incompatible with " - "[verifier.environment]" + "[verifier].sandbox_mode='shared' is incompatible with " + "[verifier.sandbox]" ) return self @@ -809,11 +925,12 @@ class StepConfig(TaskConfigModel): class TaskConfig(TaskConfigModel): - """Full task.toml configuration — the task specification ($T$). + """Full task configuration — the task specification ($T$). - Maps task.toml sections to BenchFlow's RL-aligned models. - The ``environment`` key in task.toml is loaded into ``sandbox`` - for internal use, maintaining file-level backward compatibility. + Maps task.md frontmatter / task.toml sections to BenchFlow's RL-aligned + models. The native key for the sandbox spec is ``sandbox``; the legacy + Harbor ``environment`` spelling is only accepted on the task.toml import + path, where :func:`convert_legacy_environment_keys` translates it. """ schema_version: str = "1.3" @@ -824,11 +941,7 @@ class TaskConfig(TaskConfigModel): metadata: dict[str, Any] = Field(default_factory=dict) verifier: VerifierConfig = Field(default_factory=VerifierConfig) agent: AgentConfig = Field(default_factory=AgentConfig) - # Stored as 'sandbox' internally, but loaded from 'environment' key in TOML - sandbox: SandboxConfig = Field( - default_factory=SandboxConfig, - alias="environment", - ) + sandbox: SandboxConfig = Field(default_factory=SandboxConfig) solution: SolutionConfig = Field( default_factory=SolutionConfig, serialization_alias="oracle", @@ -854,6 +967,8 @@ class TaskConfig(TaskConfigModel): @classmethod def handle_version_rename(cls, data: Any) -> Any: if isinstance(data, dict): + if "environment" in data: + raise ValueError(_ENVIRONMENT_RENAMED_ERROR) if "oracle" in data and "solution" in data: raise ValueError( "Task config cannot contain both 'oracle' and legacy " @@ -890,7 +1005,12 @@ def validate_schema_version(cls, value: str) -> str: @classmethod def model_validate_toml(cls, toml_data: str) -> TaskConfig: - toml_dict = tomllib.loads(toml_data) + # task.toml is the legacy/Harbor surface, so the loader translates + # the foreign 'environment' spelling to native 'sandbox' before + # validation. This is deliberate format translation, not an alias: + # the native schema (task.md frontmatter, direct model_validate) + # accepts only 'sandbox'. + toml_dict = convert_legacy_environment_keys(tomllib.loads(toml_data)) return cls.model_validate(toml_dict) @property @@ -917,8 +1037,3 @@ def model_dump_toml(self) -> str: if isinstance(memory, dict) and not memory: public["verifier"].pop("memory", None) return tomli_w.dumps(public) - - @property - def environment(self) -> SandboxConfig: - """Backward-compat alias: task.config.environment → task.config.sandbox.""" - return self.sandbox diff --git a/src/benchflow/task/export.py b/src/benchflow/task/export.py index 71d79ae2b..48505136c 100644 --- a/src/benchflow/task/export.py +++ b/src/benchflow/task/export.py @@ -2,6 +2,7 @@ from __future__ import annotations +import copy import json import shutil import tempfile @@ -12,6 +13,10 @@ import tomli_w +from benchflow.task.config import ( + convert_legacy_environment_keys, + convert_native_keys_to_legacy_environment, +) from benchflow.task.document import TaskDocument, render_task_md_from_legacy from benchflow.task.imports import import_task_config_toml, merge_compat_extra from benchflow.task.package import TaskPackage @@ -260,7 +265,7 @@ def export_task_to_split_layout( staged = Path(staging_root) / "export" staged.mkdir() - (staged / "task.toml").write_text(_export_task_toml(view)) + (staged / "task.toml").write_text(_export_task_toml(view, target=target)) (staged / "instruction.md").write_text(view.prompt.strip() + "\n") _copy_tree_if_exists(view.environment_dir, staged / "environment") @@ -359,12 +364,27 @@ def _load_document(view: TaskRuntimeView) -> TaskDocument | None: return TaskDocument.from_path(path) -def _export_task_toml(view: TaskRuntimeView) -> str: +def _export_task_toml( + view: TaskRuntimeView, *, target: CompatibilityTarget = "harbor" +) -> str: data = tomllib.loads(view.config.model_dump_toml()) document = _load_document(view) extra = _compat_extra(document) if extra: + # Pre-rename compat envelopes may still spell preserved keys under + # 'environment'; normalize them to native 'sandbox' spelling first so + # the merge can never place an 'environment' table NEXT TO the dumped + # 'sandbox' table (which would export a file that hard-errors on + # re-import). The deepcopy is mandatory: the converter mutates in + # place and ``_compat_extra`` returns a live sub-dict of the parsed + # TaskDocument. + extra = convert_legacy_environment_keys(copy.deepcopy(extra)) data = merge_compat_extra(data, extra) + if target == "harbor": + # Harbor speaks 'environment' / 'environment_mode'; emit the foreign + # spelling on the way out — the exact inverse of the import-side + # conversion — so a stock Harbor consumer can read the export. + data = convert_native_keys_to_legacy_environment(data) return tomli_w.dumps(data) diff --git a/src/benchflow/task/imports.py b/src/benchflow/task/imports.py index e392446ce..025563cd1 100644 --- a/src/benchflow/task/imports.py +++ b/src/benchflow/task/imports.py @@ -9,7 +9,7 @@ from pydantic import ValidationError -from benchflow.task.config import TaskConfig +from benchflow.task.config import TaskConfig, convert_legacy_environment_keys @dataclass(frozen=True) @@ -58,7 +58,10 @@ def import_task_config_toml( opt into this two-pass parse when their job is to ingest foreign tasks. """ - raw = tomllib.loads(toml_data) + # Foreign task.toml uses Harbor's 'environment' spelling for the sandbox + # spec; translate it before validation (and before recording `declared`) + # so importers and re-emitters see only the native 'sandbox' key. + raw = convert_legacy_environment_keys(tomllib.loads(toml_data)) try: config = TaskConfig.model_validate(copy.deepcopy(raw)) except ValidationError as exc: diff --git a/src/benchflow/task/runtime_capabilities.py b/src/benchflow/task/runtime_capabilities.py index a77bec38b..e624d5441 100644 --- a/src/benchflow/task/runtime_capabilities.py +++ b/src/benchflow/task/runtime_capabilities.py @@ -26,7 +26,7 @@ NetworkMode, TaskConfig, TaskOS, - VerifierEnvironmentMode, + VerifierSandboxMode, ) from benchflow.task.document import TaskDocument from benchflow.task.paths import TaskPaths, local_script_strategy_files @@ -172,8 +172,8 @@ def _append_config_issues( ) _append_network_issue( unsupported, - path="environment.network_mode", - mode=config.environment.network_mode, + path="sandbox.network_mode", + mode=config.sandbox.network_mode, sandbox=sandbox, ) _append_network_issue( @@ -183,18 +183,18 @@ def _append_config_issues( sandbox=sandbox, ) - if config.verifier.environment_mode == VerifierEnvironmentMode.SEPARATE: + if config.verifier.sandbox_mode == VerifierSandboxMode.SEPARATE: _issue( unsupported, - path="verifier.environment_mode", - reason="separate verifier environments are parsed but not executed", + path="verifier.sandbox_mode", + reason="separate verifier sandboxes are parsed but not executed", sandbox=sandbox, ) - if config.verifier.environment is not None: + if config.verifier.sandbox is not None: _issue( unsupported, - path="verifier.environment", - reason="verifier-specific environment materialization is not implemented", + path="verifier.sandbox", + reason="verifier-specific sandbox materialization is not implemented", sandbox=sandbox, ) if config.verifier.service != "main" and sandbox != "docker": @@ -205,25 +205,25 @@ def _append_config_issues( sandbox=sandbox, ) - env = config.environment + env = config.sandbox if env.os == TaskOS.WINDOWS: _issue( unsupported, - path="environment.os", + path="sandbox.os", reason="Windows task environments are parsed but not executable", sandbox=sandbox, ) if env.tpu is not None: _issue( unsupported, - path="environment.tpu", + path="sandbox.tpu", reason="TPU scheduling is parsed but not implemented", sandbox=sandbox, ) if env.gpus or env.gpu_types: _issue( unsupported, - path="environment.gpus", + path="sandbox.gpus", reason="GPU scheduling is parsed but not capability-gated", sandbox=sandbox, ) @@ -245,7 +245,7 @@ def _append_workdir_issue( if not isinstance(workdir, str) or not workdir.strip(): _issue( unsupported, - path="environment.workdir", + path="sandbox.workdir", reason="configured workdir must be a non-empty absolute path", sandbox=sandbox, ) @@ -254,7 +254,7 @@ def _append_workdir_issue( if not path.is_absolute() or path == PurePosixPath("/"): _issue( unsupported, - path="environment.workdir", + path="sandbox.workdir", reason="configured workdir must be an absolute non-root path", sandbox=sandbox, ) diff --git a/src/benchflow/traces/task_gen.py b/src/benchflow/traces/task_gen.py index 2daae92f2..02fb2d3f4 100644 --- a/src/benchflow/traces/task_gen.py +++ b/src/benchflow/traces/task_gen.py @@ -385,7 +385,7 @@ def _build_task_toml( "[verifier]", f"timeout_sec = {verifier_timeout_sec}", "", - "[environment]", + "[sandbox]", "build_timeout_sec = 600", "cpus = 1", "memory_mb = 2048", diff --git a/tests/acceptance_live_harness.py b/tests/acceptance_live_harness.py index ae60fb0b4..89b074bdf 100644 --- a/tests/acceptance_live_harness.py +++ b/tests/acceptance_live_harness.py @@ -47,7 +47,7 @@ def write_live_task(task_dir: Path, *, workdir: str | None = "/app") -> Path: "verifier": {"timeout_sec": 30}, } if workdir is not None: - frontmatter["environment"] = {"workdir": workdir} + frontmatter["sandbox"] = {"workdir": workdir} task_dir.mkdir(parents=True, exist_ok=True) (task_dir / "task.md").write_text( "---\n" diff --git a/tests/conformance/acp_smoke/task.md b/tests/conformance/acp_smoke/task.md index 663839bf9..3eb43a494 100644 --- a/tests/conformance/acp_smoke/task.md +++ b/tests/conformance/acp_smoke/task.md @@ -25,7 +25,7 @@ verifier: cleanup_conftests: true agent: timeout_sec: 600.0 -environment: +sandbox: network_mode: public build_timeout_sec: 600.0 docker_image: ubuntu:24.04 diff --git a/tests/examples/hello-world-task/task.md b/tests/examples/hello-world-task/task.md index b71cc2f64..17e8b6504 100644 --- a/tests/examples/hello-world-task/task.md +++ b/tests/examples/hello-world-task/task.md @@ -24,7 +24,7 @@ verifier: cleanup_conftests: true agent: timeout_sec: 300.0 -environment: +sandbox: network_mode: public build_timeout_sec: 600.0 os: linux diff --git a/tests/test_acp_mcp_servers.py b/tests/test_acp_mcp_servers.py index 353320a6e..f1280fbc4 100644 --- a/tests/test_acp_mcp_servers.py +++ b/tests/test_acp_mcp_servers.py @@ -91,7 +91,7 @@ def test_http_spec_carries_url_and_type() -> None: def _task_with_mcp(*configs: MCPServerConfig) -> SimpleNamespace: return SimpleNamespace( - config=SimpleNamespace(environment=SimpleNamespace(mcp_servers=list(configs))) + config=SimpleNamespace(sandbox=SimpleNamespace(mcp_servers=list(configs))) ) @@ -255,7 +255,7 @@ def test_task_mcp_specs_handles_absent_config() -> None: assert _task_mcp_specs(None) == [] assert ( _task_mcp_specs( - SimpleNamespace(config=SimpleNamespace(environment=SimpleNamespace())) + SimpleNamespace(config=SimpleNamespace(sandbox=SimpleNamespace())) ) == [] ) diff --git a/tests/test_agentcore_sandbox.py b/tests/test_agentcore_sandbox.py index 77e131a82..c9d7fcde9 100644 --- a/tests/test_agentcore_sandbox.py +++ b/tests/test_agentcore_sandbox.py @@ -521,9 +521,7 @@ def test_no_network_tasks_are_refused(self, tmp_path): from benchflow.task.config import TaskConfig from benchflow.task.runtime_capabilities import validate_task_runtime_support - config = TaskConfig.model_validate( - {"environment": {"network_mode": "no-network"}} - ) + config = TaskConfig.model_validate({"sandbox": {"network_mode": "no-network"}}) issues = validate_task_runtime_support(config, sandbox="agentcore") assert any( diff --git a/tests/test_config_override.py b/tests/test_config_override.py index f5cc9a977..878258be9 100644 --- a/tests/test_config_override.py +++ b/tests/test_config_override.py @@ -24,7 +24,7 @@ def _cfg() -> TaskConfig: "version": "1.0", "agent": {"timeout_sec": 300}, "verifier": {"timeout_sec": 120}, - "environment": {"cpus": 1, "memory_mb": 2048}, + "sandbox": {"cpus": 1, "memory_mb": 2048}, } ) @@ -138,8 +138,9 @@ def test_apply_rejects_blank_agent_prompt_prefix(): def test_apply_overrides_sandbox_by_field_name(): - # Regression: merging against by_alias=True made `sandbox` (alias - # `environment`) un-overridable via its field name. Must work now. + # Regression: merging against by_alias=True once made `sandbox` (then + # aliased to `environment`) un-overridable via its field name. `sandbox` + # is now the only spelling; overriding by field name must keep working. out = apply_config_override(_cfg(), {"sandbox": {"cpus": 8}}) assert out.sandbox.cpus == 8 diff --git a/tests/test_env_setup.py b/tests/test_env_setup.py index 1be733627..f9770a76b 100644 --- a/tests/test_env_setup.py +++ b/tests/test_env_setup.py @@ -251,7 +251,7 @@ def test_prefers_effective_task_path_environment_dir(self, tmp_path): env_config = MagicMock() task = SimpleNamespace( paths=SimpleNamespace(environment_dir=original_env_dir), - config=SimpleNamespace(environment=env_config), + config=SimpleNamespace(sandbox=env_config), ) with patch("benchflow.sandbox.docker.DockerSandbox") as docker_env: @@ -264,7 +264,7 @@ def test_modal_preflights_and_constructs_environment(self, tmp_path): rollout_paths = MagicMock() task = SimpleNamespace( paths=SimpleNamespace(environment_dir=tmp_path / "environment"), - config=SimpleNamespace(environment=env_config), + config=SimpleNamespace(sandbox=env_config), ) with patch( @@ -302,7 +302,7 @@ def test_daytona_without_sdk_fails_fast_with_install_hint(self, tmp_path): rollout_paths = MagicMock() task = SimpleNamespace( paths=SimpleNamespace(environment_dir=tmp_path / "environment"), - config=SimpleNamespace(environment=env_config), + config=SimpleNamespace(sandbox=env_config), ) with ( diff --git a/tests/test_environment_manifest_controls.py b/tests/test_environment_manifest_controls.py index cc2aa3c84..515587053 100644 --- a/tests/test_environment_manifest_controls.py +++ b/tests/test_environment_manifest_controls.py @@ -2,7 +2,7 @@ A RolloutConfig that carries an EnvironmentManifest must take effect: -1. ``image`` overrides ``task.config.environment.docker_image`` so the +1. ``image`` overrides ``task.config.sandbox.docker_image`` so the manifest — not the task's local Dockerfile — drives runtime image selection. 2. ``task_selection`` (mechanism=env_var, inject_into=entrypoint) binds @@ -12,7 +12,7 @@ environment can read declared host secrets/config. Before this fix the manifest was accepted but ignored — the sandbox was -created from ``task.config.environment`` only, the env var never made it +created from ``task.config.sandbox`` only, the env var never made it into the container, and host-side ``forward_env`` was never resolved. """ @@ -155,7 +155,7 @@ def test_manifest_image_overrides_task_docker_image(tmp_path): """Control point 1: manifest ``image`` wins over task.toml docker_image.""" task_dir = _write_task(tmp_path) task = Task(task_dir) - assert task.config.environment.docker_image == "task-toml-image:latest" + assert task.config.sandbox.docker_image == "task-toml-image:latest" captured: dict = {} @@ -176,7 +176,7 @@ def __init__(self, **kwargs): # The sandbox sees the manifest's image, not the task.toml's. assert captured["task_env_config"].docker_image == "manifest-image:latest" # And the task's config is not mutated — the override is per-rollout. - assert task.config.environment.docker_image == "task-toml-image:latest" + assert task.config.sandbox.docker_image == "task-toml-image:latest" def test_manifest_task_selection_lands_in_sandbox_persistent_env(tmp_path, monkeypatch): diff --git a/tests/test_inbound_adapters.py b/tests/test_inbound_adapters.py index dcb9a6a1f..646dc9a35 100644 --- a/tests/test_inbound_adapters.py +++ b/tests/test_inbound_adapters.py @@ -147,15 +147,15 @@ def test_unknown_extension_keys_are_preserved_not_native( assert result.compatibility is not None assert result.compatibility.config_extra == { "harbor_ext": "kept", - "environment": { + "sandbox": { "modal": {"image": "registry.example.com/task:latest"}, }, "steps": [{"runner": "harbor-step-runner"}], "verifier": {"reward_kit": {"metric": "exact_match"}}, } assert result.compatibility.config_extra_paths == ( - "environment.modal.image", "harbor_ext", + "sandbox.modal.image", "steps[0].runner", "verifier.reward_kit.metric", ) diff --git a/tests/test_internet_policy.py b/tests/test_internet_policy.py index f8218afd6..f34e2daaf 100644 --- a/tests/test_internet_policy.py +++ b/tests/test_internet_policy.py @@ -38,7 +38,7 @@ def _wire_fake_planes(trial: Rollout) -> MagicMock: def test_task_disallows_internet_from_environment_config(): task = SimpleNamespace( - config=SimpleNamespace(environment=SimpleNamespace(allow_internet=False)) + config=SimpleNamespace(sandbox=SimpleNamespace(allow_internet=False)) ) assert _task_disallows_internet(task) is True @@ -88,7 +88,7 @@ def test_create_environment_preserves_agent_network_for_llm_runs(tmp_path): original_env.model_copy.return_value = copied_env task = SimpleNamespace( paths=SimpleNamespace(environment_dir=tmp_path / "environment"), - config=SimpleNamespace(environment=original_env), + config=SimpleNamespace(sandbox=original_env), ) with patch("benchflow.sandbox.docker.DockerSandbox") as docker_env: @@ -113,7 +113,7 @@ def test_create_environment_keeps_oracle_network_policy(tmp_path): original_env.allow_internet = False task = SimpleNamespace( paths=SimpleNamespace(environment_dir=tmp_path / "environment"), - config=SimpleNamespace(environment=original_env), + config=SimpleNamespace(sandbox=original_env), ) with patch("benchflow.sandbox.docker.DockerSandbox") as docker_env: @@ -142,7 +142,7 @@ async def test_connect_as_applies_web_policy_to_role_env(tmp_path): trial._agent_cwd = "/app" trial._phase = "idle" trial._task = SimpleNamespace( - config=SimpleNamespace(environment=SimpleNamespace(allow_internet=False)) + config=SimpleNamespace(sandbox=SimpleNamespace(allow_internet=False)) ) planes = _wire_fake_planes(trial) captured = {} @@ -251,7 +251,7 @@ async def test_connect_as_applies_hard_web_policy_to_role_agent(tmp_path): def test_task_allows_internet_when_explicitly_true(): """allow_internet=True should not trigger web-tool disabling.""" task = SimpleNamespace( - config=SimpleNamespace(environment=SimpleNamespace(allow_internet=True)) + config=SimpleNamespace(sandbox=SimpleNamespace(allow_internet=True)) ) assert _task_disallows_internet(task) is False @@ -365,7 +365,7 @@ def test_create_environment_does_not_flip_when_internet_allowed(tmp_path): original_env.allow_internet = True task = SimpleNamespace( paths=SimpleNamespace(environment_dir=tmp_path / "environment"), - config=SimpleNamespace(environment=original_env), + config=SimpleNamespace(sandbox=original_env), ) with patch("benchflow.sandbox.docker.DockerSandbox") as docker_env: @@ -393,7 +393,7 @@ def test_task_toml_allow_internet_false_parsed_correctly(tmp_path): (task_dir / "instruction.md").write_text("Test instruction.") task = Task(task_dir) - assert task.config.environment.allow_internet is False + assert task.config.sandbox.allow_internet is False assert _task_disallows_internet(task) is True @@ -409,7 +409,7 @@ def test_task_toml_allow_internet_true_parsed_correctly(tmp_path): (task_dir / "instruction.md").write_text("Test instruction.") task = Task(task_dir) - assert task.config.environment.allow_internet is True + assert task.config.sandbox.allow_internet is True assert _task_disallows_internet(task) is False diff --git a/tests/test_mle_bench_adapter.py b/tests/test_mle_bench_adapter.py index dd2b0802a..beeec4ec3 100644 --- a/tests/test_mle_bench_adapter.py +++ b/tests/test_mle_bench_adapter.py @@ -328,7 +328,7 @@ def test_mle_bench_converter_keeps_private_data_out_of_environment(tmp_path: Pat config = TaskConfig.model_validate_toml((task_dir / "task.toml").read_text()) assert config.task is not None assert config.task.name == "mle-bench/fake-kaggle" - assert config.environment.allow_internet is False + assert config.sandbox.allow_internet is False dockerfile = (task_dir / "environment" / "Dockerfile").read_text() assert 'CMD ["sleep", "infinity"]' in dockerfile diff --git a/tests/test_rollout_setup_commands.py b/tests/test_rollout_setup_commands.py index c0d0e6b5b..cc6d7b027 100644 --- a/tests/test_rollout_setup_commands.py +++ b/tests/test_rollout_setup_commands.py @@ -39,7 +39,7 @@ async def exec(self, *args, **kwargs): host_lock="shared-notion-oauth", ) task = SimpleNamespace( - config=SimpleNamespace(environment=SimpleNamespace(setup_commands=[command])) + config=SimpleNamespace(sandbox=SimpleNamespace(setup_commands=[command])) ) await asyncio.gather( @@ -82,7 +82,7 @@ async def download_dir(self, source_dir, target_dir, service="main"): capture_dir_b64_env_file_var="CAPTURE_DOTENV_PATH", ) task = SimpleNamespace( - config=SimpleNamespace(environment=SimpleNamespace(setup_commands=[command])) + config=SimpleNamespace(sandbox=SimpleNamespace(setup_commands=[command])) ) await _run_environment_setup_commands(Env(), task) @@ -134,7 +134,7 @@ async def download_dir(self, source_dir, target_dir, service="main"): capture_dir_b64_env_file_var=None, ) task = SimpleNamespace( - config=SimpleNamespace(environment=SimpleNamespace(setup_commands=[command])) + config=SimpleNamespace(sandbox=SimpleNamespace(setup_commands=[command])) ) await asyncio.gather( diff --git a/tests/test_rollout_upload.py b/tests/test_rollout_upload.py index 461e9d992..67be7a659 100644 --- a/tests/test_rollout_upload.py +++ b/tests/test_rollout_upload.py @@ -111,7 +111,7 @@ async def exec_healthcheck(command: str, **kwargs): env.exec = exec_healthcheck task = SimpleNamespace( config=SimpleNamespace( - environment=SimpleNamespace( + sandbox=SimpleNamespace( healthcheck=SimpleNamespace( command="python /opt/pull_bucket.py", interval_sec=0, @@ -147,7 +147,7 @@ async def test_environment_healthcheck_fails_closed() -> None: env = FakeSetupCommandEnv(return_code=1) task = SimpleNamespace( config=SimpleNamespace( - environment=SimpleNamespace( + sandbox=SimpleNamespace( healthcheck=SimpleNamespace( command="false", interval_sec=0, @@ -170,7 +170,7 @@ async def test_environment_setup_commands_run_before_agent_install() -> None: env = FakeSetupCommandEnv() task = SimpleNamespace( config=SimpleNamespace( - environment=SimpleNamespace( + sandbox=SimpleNamespace( setup_commands=[ SimpleNamespace( command="python preprocess.py", @@ -205,7 +205,7 @@ async def test_environment_setup_commands_fail_closed_on_nonzero() -> None: env = FakeSetupCommandEnv(return_code=2) task = SimpleNamespace( config=SimpleNamespace( - environment=SimpleNamespace( + sandbox=SimpleNamespace( setup_commands=[ SimpleNamespace( command="python preprocess.py", @@ -226,13 +226,13 @@ async def test_environment_setup_commands_fail_closed_on_nonzero() -> None: @pytest.mark.asyncio async def test_resolve_agent_cwd_uses_configured_workdir() -> None: - """environment.workdir becomes the executable agent workspace.""" + """sandbox.workdir becomes the executable agent workspace.""" env = MagicMock() env.exec = AsyncMock( return_value=MagicMock(stdout="/repo\n", stderr="", return_code=0) ) task = SimpleNamespace( - config=SimpleNamespace(environment=SimpleNamespace(workdir="/repo")) + config=SimpleNamespace(sandbox=SimpleNamespace(workdir="/repo")) ) agent_cwd = await _resolve_agent_cwd(env, task) @@ -247,13 +247,13 @@ async def test_resolve_agent_cwd_uses_configured_workdir() -> None: @pytest.mark.asyncio async def test_resolve_agent_cwd_falls_back_to_container_pwd() -> None: - """Tasks without environment.workdir preserve the existing pwd discovery.""" + """Tasks without sandbox.workdir preserve the existing pwd discovery.""" env = MagicMock() env.exec = AsyncMock( return_value=MagicMock(stdout="/app\n", stderr="", return_code=0) ) task = SimpleNamespace( - config=SimpleNamespace(environment=SimpleNamespace(workdir=None)) + config=SimpleNamespace(sandbox=SimpleNamespace(workdir=None)) ) agent_cwd = await _resolve_agent_cwd(env, task) @@ -273,7 +273,7 @@ async def test_resolve_agent_cwd_avoids_root_workspace() -> None: ] ) task = SimpleNamespace( - config=SimpleNamespace(environment=SimpleNamespace(workdir=None)) + config=SimpleNamespace(sandbox=SimpleNamespace(workdir=None)) ) agent_cwd = await _resolve_agent_cwd(env, task) diff --git a/tests/test_runtime_capabilities.py b/tests/test_runtime_capabilities.py index 0b138811b..9763edda1 100644 --- a/tests/test_runtime_capabilities.py +++ b/tests/test_runtime_capabilities.py @@ -54,7 +54,7 @@ def _write_task_md( timeout_sec: 300 verifier: timeout_sec: 120 -environment: +sandbox: network_mode: no-network {extra}--- @@ -135,7 +135,7 @@ def test_validator_reports_allowlist_as_runtime_gap() -> None: "network_mode": "allowlist", "allowed_hosts": ["api.example.com"], }, - "environment": { + "sandbox": { "network_mode": "allowlist", "allowed_hosts": ["repo.example.com"], }, @@ -146,13 +146,13 @@ def test_validator_reports_allowlist_as_runtime_gap() -> None: assert [issue.path for issue in issues] == [ "agent.network_mode", - "environment.network_mode", + "sandbox.network_mode", ] def test_validator_reports_unknown_sandbox_backend() -> None: """Runtime-capability validation cannot greenlight typoed backends.""" - config = TaskConfig.model_validate({"environment": {"network_mode": "no-network"}}) + config = TaskConfig.model_validate({"sandbox": {"network_mode": "no-network"}}) issues = validate_task_runtime_support(config, sandbox="not-a-backend") @@ -167,13 +167,13 @@ def test_validator_reports_unknown_sandbox_backend() -> None: def test_validator_reports_apple_container_no_network_gap() -> None: """Guards PR #936 against silently launching no-network tasks on Apple Container.""" - config = TaskConfig.model_validate({"environment": {"network_mode": "no-network"}}) + config = TaskConfig.model_validate({"sandbox": {"network_mode": "no-network"}}) issues = validate_task_runtime_support(config, sandbox="apple-container") assert [(issue.path, issue.reason) for issue in issues] == [ ( - "environment.network_mode", + "sandbox.network_mode", "network_mode='no-network' is not enforced by apple-container", ) ] @@ -410,9 +410,9 @@ def test_sandbox_launch_rejects_unsafe_workdir_before_backend_construction( assert docker_sandbox.call_count == 0 message = str(exc_info.value) - assert "environment.workdir" in message + assert "sandbox.workdir" in message assert {feature.path for feature in exc_info.value.features} == { - "environment.workdir", + "sandbox.workdir", } @@ -433,7 +433,7 @@ def test_validator_reports_root_workdir_as_runtime_gap(tmp_path: Path) -> None: ) assert any( - issue.path == "environment.workdir" and "absolute non-root path" in issue.reason + issue.path == "sandbox.workdir" and "absolute non-root path" in issue.reason for issue in issues ) diff --git a/tests/test_skill_eval.py b/tests/test_skill_eval.py index 770c85f13..2c4b5e3d5 100644 --- a/tests/test_skill_eval.py +++ b/tests/test_skill_eval.py @@ -355,7 +355,7 @@ def test_with_skill_task_declares_neutral_skill_mount(self, skill_dir, tmp_path) task_dirs = generate_tasks(ds, output, with_skill=True) task = Task(task_dirs[0]) - assert task.config.environment.skills_dir == "/skills" + assert task.config.sandbox.skills_dir == "/skills" def test_judge_env_templates_available_host_keys_without_secrets( self, skill_dir, tmp_path, monkeypatch @@ -386,7 +386,7 @@ def test_without_skill_task_omits_skill_mount(self, skill_dir, tmp_path): task_dirs = generate_tasks(ds, output, with_skill=False) task = Task(task_dirs[0]) - assert task.config.environment.skills_dir is None + assert task.config.sandbox.skills_dir is None def test_dockerfile_with_skill_has_copy(self, skill_dir, tmp_path): ds = load_eval_dataset(skill_dir) @@ -431,8 +431,7 @@ def test_configured_skill_mount_updates_task_and_dockerfile( dockerfile = (task_dirs[0] / "environment" / "Dockerfile").read_text() assert ( - Task(task_dirs[0]).config.environment.skills_dir - == "/opt/benchflow/skill-eval" + Task(task_dirs[0]).config.sandbox.skills_dir == "/opt/benchflow/skill-eval" ) assert "COPY skills/ /opt/benchflow/skill-eval/" in dockerfile diff --git a/tests/test_skill_eval_integration.py b/tests/test_skill_eval_integration.py index 179845a4c..4ee996cf7 100644 --- a/tests/test_skill_eval_integration.py +++ b/tests/test_skill_eval_integration.py @@ -137,7 +137,7 @@ def test_generate_with_skill_tasks(self, mock_skill): assert skill_dst.exists() assert (skill_dst / "SKILL.md").exists() assert (skill_dst / "scripts" / "check_claim.py").exists() - assert Task(task_dir).config.environment.skills_dir == "/skills" + assert Task(task_dir).config.sandbox.skills_dir == "/skills" cleanup_tasks([with_dir]) assert not with_dir.exists() diff --git a/tests/test_skill_eval_sweep.py b/tests/test_skill_eval_sweep.py index 829ef65c2..a71c389f3 100644 --- a/tests/test_skill_eval_sweep.py +++ b/tests/test_skill_eval_sweep.py @@ -79,7 +79,7 @@ def test_case_environment_lands_in_task_config(self, tmp_path): task = generate_tasks(ds, tmp_path / "out", with_skill=False)[0] # The override must be exposed as sandbox env so the runner forwards it. - env_block = Task(task).config.environment.env + env_block = Task(task).config.sandbox.env assert env_block == {"CASE_ONLY": "expected-value", "ANOTHER": "x y"} def test_case_environment_lands_in_case_json(self, tmp_path): @@ -103,8 +103,8 @@ def test_no_environment_block_when_case_has_none(self, tmp_path): skill = _make_skill(tmp_path) ds = load_eval_dataset(skill) task = generate_tasks(ds, tmp_path / "out", with_skill=False)[0] - # No spurious [environment.env] section when there's nothing to forward. - assert Task(task).config.environment.env == {} + # No spurious [sandbox.env] section when there's nothing to forward. + assert Task(task).config.sandbox.env == {} # Issue #393 — TOML escape of skill_name @@ -458,4 +458,4 @@ def test_handbuilt_dataset_environment_block(self, tmp_path): ], ) task = generate_tasks(ds, tmp_path / "out", with_skill=False)[0] - assert Task(task).config.environment.env == {"FOO": "bar"} + assert Task(task).config.sandbox.env == {"FOO": "bar"} diff --git a/tests/test_skill_policy.py b/tests/test_skill_policy.py index 5e1a2b33c..f5c5d843a 100644 --- a/tests/test_skill_policy.py +++ b/tests/test_skill_policy.py @@ -147,7 +147,7 @@ def test_with_skill_rejects_unsafe_sandbox_path(tmp_path: Path) -> None: task = tmp_path / "task" _make_task_skills(task) - with pytest.raises(ValueError, match=r"environment\.skills_dir"): + with pytest.raises(ValueError, match=r"sandbox\.skills_dir"): resolve_task_skill_policy( task_path=task, skill_mode=SKILL_MODE_WITH_SKILL, diff --git a/tests/test_source_adapters.py b/tests/test_source_adapters.py index 72cce0d11..1162af8c0 100644 --- a/tests/test_source_adapters.py +++ b/tests/test_source_adapters.py @@ -69,7 +69,7 @@ def test_mcp_atlas_source_adapter_materializes_native_tasks( assert adapted.path != source assert task.config.task is not None assert task.config.task.name == "mcp-atlas/atlas-task-1" - assert task.config.environment.mcp_servers[0].tools == [ + assert task.config.sandbox.mcp_servers[0].tools == [ "search_query", "fetch_read", ] @@ -144,14 +144,14 @@ def test_toolathlon_source_adapter_materializes_mcp_and_setup( task = Task(generated) assert task.config.task is not None assert task.config.task.name == "toolathlon/arrange-workspace" - assert task.config.environment.workdir == "/workspace/agent_workspace" - assert task.config.environment.setup_commands - assert task.config.environment.mcp_servers[0].exclude_tags == [ + assert task.config.sandbox.workdir == "/workspace/agent_workspace" + assert task.config.sandbox.setup_commands + assert task.config.sandbox.mcp_servers[0].exclude_tags == [ "__benchflow_exclude_no_tools__" ] # The real server is wrapped in the container launcher so ${token.X} in # argv/env resolves at spawn time against the per-task token file. - word = task.config.environment.mcp_servers[1] + word = task.config.sandbox.mcp_servers[1] assert word.command == "/usr/bin/python3" assert word.args == [ "/workspace/.toolathlon/toolathlon_container.py", @@ -168,10 +168,10 @@ def test_toolathlon_source_adapter_materializes_mcp_and_setup( ) # First setup command stages the container helper and writes the global # token_key_session.py; the preprocess command runs last. - token_setup = task.config.environment.setup_commands[0].command + token_setup = task.config.sandbox.setup_commands[0].command assert "toolathlon_container.py write-config" in token_setup assert "TOOLATHLON_CONTAINER_MODULE_B64" in token_setup - setup_command = task.config.environment.setup_commands[-1].command + setup_command = task.config.sandbox.setup_commands[-1].command dockerfile = (generated / "environment" / "Dockerfile").read_text() task_toml = (generated / "task.toml").read_text() test_sh = (generated / "tests" / "test.sh").read_text() @@ -266,8 +266,8 @@ def test_toolathlon_email_task_gets_poste_sidecar(tmp_path: Path, monkeypatch) - task = Task(email) # Extra headroom for the DinD compose + sidecar image. - assert task.config.environment.memory_mb == 12288 - commands = [c.command for c in task.config.environment.setup_commands] + assert task.config.sandbox.memory_mb == 12288 + commands = [c.command for c in task.config.sandbox.setup_commands] rewrite_idx = next( i for i, c in enumerate(commands) if "poste" in c and "imap_server" in c ) @@ -299,7 +299,7 @@ def test_toolathlon_notion_task_gets_extended_preprocess_timeout( ) task = Task(adapted.path / "notion-hr") - commands = task.config.environment.setup_commands + commands = task.config.sandbox.setup_commands assert any("TOOLATHLON_NOTION_MCP_AUTH_B64" in c.command for c in commands) assert commands[-1].timeout_sec == 3600.0 assert commands[-1].host_lock == "toolathlon-notion-official-mcp" @@ -456,10 +456,10 @@ def test_toolathlon_k8s_task_gets_host_network_runtime( assert "helm-v3.15.4-linux" in dockerfile task = Task(generated) - assert task.config.environment.cpus == 6 - assert task.config.environment.memory_mb == 16384 - assert task.config.environment.storage_mb == 49152 - commands = [c.command for c in task.config.environment.setup_commands] + assert task.config.sandbox.cpus == 6 + assert task.config.sandbox.memory_mb == 16384 + assert task.config.sandbox.storage_mb == 49152 + commands = [c.command for c in task.config.sandbox.setup_commands] assert any("docker info" in c and "helm version" in c for c in commands) preprocess_idx = next(i for i, c in enumerate(commands) if "preprocess.main" in c) kubeconfig_idx = next(i for i, c in enumerate(commands) if "k8s_configs" in c) @@ -513,10 +513,8 @@ def test_toolathlon_woocommerce_task_gets_first_boot_sidecar( assert "$site_uploads/wc-logs" in entry assert "$site_uploads/woocommerce_uploads" in entry task = Task(generated) - assert task.config.environment.build_timeout_sec == 2400 - assert any( - "woo-rewrite" in c.command for c in task.config.environment.setup_commands - ) + assert task.config.sandbox.build_timeout_sec == 2400 + assert any("woo-rewrite" in c.command for c in task.config.sandbox.setup_commands) def test_toolathlon_canvas_task_reseeds_tokens_on_first_boot( @@ -560,10 +558,10 @@ def test_toolathlon_canvas_task_reseeds_tokens_on_first_boot( assert "Role.get_built_in_role('AccountAdmin')" in seed assert "account_user.workflow_state = 'active'" in seed task = Task(adapted.path / "canvas-list-test") - assert task.config.environment.build_timeout_sec == 2400 + assert task.config.sandbox.build_timeout_sec == 2400 rewrite = next( c.command - for c in task.config.environment.setup_commands + for c in task.config.sandbox.setup_commands if "canvas-rewrite" in c.command ) assert "/workspace/utils/app_specific/canvas" in rewrite @@ -617,7 +615,7 @@ def test_toolathlon_gym_adapter_normalizes_postgres_env( generated = adapted.path / "salary-report" task = Task(generated) - env = task.config.environment.mcp_servers[0].env + env = task.config.sandbox.mcp_servers[0].env assert env["PG_HOST"] == "postgres" assert env["PG_USER"] == "eigent" assert env["PG_PASSWORD"] == "camel" @@ -630,7 +628,7 @@ def test_toolathlon_gym_adapter_normalizes_postgres_env( "chmod -R a+rwX /workspace/agent_workspace" in (generated / "task.toml").read_text() ) - setup_command = task.config.environment.setup_commands[-1].command + setup_command = task.config.sandbox.setup_commands[-1].command assert 'chmod -R go-rwx "$private"' in setup_command assert ( "postgres:" in (generated / "environment" / "docker-compose.yaml").read_text() @@ -759,12 +757,12 @@ def test_toolathlon_adapter_materializes_tasks_with_missing_repo_configs( ] assert not (adapted.path / ".benchflow-source-adapter-skipped.json").exists() # setup_commands: [token-setup, credential-inject, preprocess]. - assert len(task.config.environment.setup_commands) == 3 + assert len(task.config.sandbox.setup_commands) == 3 assert ( "toolathlon_container.py write-config" - in task.config.environment.setup_commands[0].command + in task.config.sandbox.setup_commands[0].command ) - credential_setup = task.config.environment.setup_commands[1] + credential_setup = task.config.sandbox.setup_commands[1] assert credential_setup.env == { "TOOLATHLON_GCP_SERVICE_ACCOUNT_JSON": "${TOOLATHLON_GCP_SERVICE_ACCOUNT_JSON:-}", "TOOLATHLON_GCP_SERVICE_ACCOUNT_JSON_B64": "${TOOLATHLON_GCP_SERVICE_ACCOUNT_JSON_B64:-}", @@ -782,7 +780,7 @@ def test_toolathlon_adapter_materializes_tasks_with_missing_repo_configs( assert sheet.config.metadata["required_credential_files"] == [ "configs/google_credentials.json" ] - sheet_server = sheet.config.environment.mcp_servers[0] + sheet_server = sheet.config.sandbox.mcp_servers[0] assert sheet_server.command == "/usr/bin/python3" assert sheet_server.args[:3] == [ "/workspace/.toolathlon/toolathlon_container.py", @@ -802,7 +800,7 @@ def test_toolathlon_adapter_materializes_tasks_with_missing_repo_configs( "configs/gcp-oauth.keys.json", "configs/google_credentials.json", ] - calendar_server = calendar.config.environment.mcp_servers[0] + calendar_server = calendar.config.sandbox.mcp_servers[0] assert "HOME" not in calendar_server.env assert ( calendar_server.env["CALENDAR_OAUTH_PATH"] @@ -825,7 +823,7 @@ def test_toolathlon_adapter_materializes_tasks_with_missing_repo_configs( (workspace / "configs" / "gcp-oauth.keys.json").write_text( json.dumps(oauth_payload) ) - calendar_command = calendar.config.environment.setup_commands[1].command.replace( + calendar_command = calendar.config.sandbox.setup_commands[1].command.replace( "/usr/local/bin/uv run python", sys.executable ) result = subprocess.run( @@ -1185,7 +1183,7 @@ def test_toolathlon_arxiv_server_declares_ensure_dirs( _resolved(repo, repo="hkust-nlp/Toolathlon", path="tasks/finalpool") ) task = Task(adapted.path / "find-alita-paper") - arxiv = task.config.environment.mcp_servers[0] + arxiv = task.config.sandbox.mcp_servers[0] assert ( arxiv.env["TOOLATHLON_ENSURE_DIRS"] == "/workspace/agent_workspace/arxiv_local_storage" diff --git a/tests/test_task_config.py b/tests/test_task_config.py index 98ff8c4ea..db1dd1646 100644 --- a/tests/test_task_config.py +++ b/tests/test_task_config.py @@ -7,7 +7,7 @@ NetworkMode, TaskConfig, TaskOS, - VerifierEnvironmentMode, + VerifierSandboxMode, ) @@ -79,7 +79,13 @@ def test_verifier_timeout_sec_rejects_unusable_budgets(value): def test_task_config_accepts_current_harbor_task_toml_surface(): - """Guards commit 67378ddd's 2026-06-04 parity pass against schema shrinkage.""" + """Guards commit 67378ddd's 2026-06-04 parity pass against schema shrinkage. + + Harbor task.toml spells the sandbox spec ``[environment]`` (and + ``[verifier.environment]``); the toml import path converts both to the + native ``sandbox`` key, so the assertions below read ``cfg.sandbox`` / + ``cfg.verifier.sandbox``. + """ cfg = TaskConfig.model_validate_toml( """ schema_version = "1.3" @@ -174,16 +180,16 @@ def test_task_config_accepts_current_harbor_task_toml_surface(): assert cfg.metadata["custom"]["kept"] is True assert cfg.agent.network_mode == NetworkMode.ALLOWLIST assert cfg.agent.allowed_hosts == ["api.example.com"] - assert cfg.verifier.environment_mode == VerifierEnvironmentMode.SEPARATE + assert cfg.verifier.sandbox_mode == VerifierSandboxMode.SEPARATE assert cfg.verifier.hardening.cleanup_conftests is False - assert cfg.verifier.environment is not None - assert cfg.verifier.environment.allow_internet is False - assert cfg.environment.network_mode == NetworkMode.ALLOWLIST - assert cfg.environment.os == TaskOS.LINUX - assert cfg.environment.tpu is not None - assert cfg.environment.tpu.chip_count == 8 - assert cfg.environment.healthcheck is not None - assert cfg.environment.healthcheck.retries == 5 + assert cfg.verifier.sandbox is not None + assert cfg.verifier.sandbox.allow_internet is False + assert cfg.sandbox.network_mode == NetworkMode.ALLOWLIST + assert cfg.sandbox.os == TaskOS.LINUX + assert cfg.sandbox.tpu is not None + assert cfg.sandbox.tpu.chip_count == 8 + assert cfg.sandbox.healthcheck is not None + assert cfg.sandbox.healthcheck.retries == 5 assert cfg.multi_step_reward_strategy == MultiStepRewardStrategy.FINAL assert cfg.artifacts[0].source == "/logs/artifacts" assert cfg.steps is not None @@ -210,7 +216,11 @@ def test_task_config_accepts_native_oracle_alias(): def test_task_config_accepts_current_skillsbench_task_toml_surface(): - """Guards SkillsBench main a8eefb4 against parser-only run failures.""" + """Guards SkillsBench main a8eefb4 against parser-only run failures. + + The legacy ``[environment]`` table converts to ``cfg.sandbox`` on the + toml import path. + """ cfg = TaskConfig.model_validate_toml( """ version = "1.0" @@ -243,10 +253,10 @@ def test_task_config_accepts_current_skillsbench_task_toml_surface(): assert cfg.schema_version == "1.0" assert cfg.reward["reward"].startswith("(P0_passed") - assert cfg.environment.docker_image == "python:3.12-slim" - assert cfg.environment.bugswarm_image_tag == "google-auto-101506036" - assert cfg.environment.memory_mb == 4096 - assert cfg.environment.storage_mb == 8192 + assert cfg.sandbox.docker_image == "python:3.12-slim" + assert cfg.sandbox.bugswarm_image_tag == "google-auto-101506036" + assert cfg.sandbox.memory_mb == 4096 + assert cfg.sandbox.storage_mb == 8192 assert cfg.solution.timeout_sec == 1800.0 @@ -267,7 +277,7 @@ def test_task_config_accepts_mcp_tool_filter_surface(): """ ) - (server,) = cfg.environment.mcp_servers + (server,) = cfg.sandbox.mcp_servers assert server.name == "atlas" assert server.transport == "streamable-http" assert server.headers == {"x_run": "smoke"} @@ -291,7 +301,7 @@ def test_task_config_accepts_mcp_stdio_cwd(): """ ) - (server,) = cfg.environment.mcp_servers + (server,) = cfg.sandbox.mcp_servers assert server.cwd == "/workspace/agent_workspace" @@ -314,7 +324,7 @@ def test_task_config_accepts_environment_setup_commands(): """ ) - (command,) = cfg.environment.setup_commands + (command,) = cfg.sandbox.setup_commands assert command.command == "python preprocess.py" assert command.cwd == "/workspace" assert command.timeout_sec == 120 @@ -367,3 +377,107 @@ def test_task_config_rejects_conflicting_solution_inline_env(): MODAL_TOKEN_ID = "nested" """ ) + + +def test_task_config_rejects_renamed_environment_key_with_actionable_message(): + """The native surface accepts only 'sandbox'; 'environment' is a rename, + not an alias, and the error says exactly how to fix the file.""" + with pytest.raises( + ValueError, + match=r"the 'environment' key was renamed to 'sandbox' — rename " + r"'environment:' to 'sandbox:'", + ): + TaskConfig.model_validate({"environment": {"cpus": 2}}) + + +def test_task_config_rejects_renamed_verifier_environment_key(): + with pytest.raises( + ValueError, + match=r"the 'verifier\.environment' key was renamed to " + r"'verifier\.sandbox'", + ): + TaskConfig.model_validate({"verifier": {"environment": {"cpus": 2}}}) + + +def test_task_config_toml_converts_legacy_environment_table(): + """task.toml is the legacy/Harbor surface: '[environment]' converts.""" + cfg = TaskConfig.model_validate_toml('version = "1.0"\n[environment]\ncpus = 2\n') + assert cfg.sandbox.cpus == 2 + + +def test_task_config_toml_rejects_both_environment_and_sandbox(): + with pytest.raises( + ValueError, + match=r"declares both 'environment' and 'sandbox'", + ): + TaskConfig.model_validate_toml( + 'version = "1.0"\n[environment]\ncpus = 2\n[sandbox]\ncpus = 3\n' + ) + + +def test_task_config_toml_rejects_both_verifier_environment_and_sandbox(): + with pytest.raises( + ValueError, + match=r"declares both 'verifier\.environment' and 'verifier\.sandbox'", + ): + TaskConfig.model_validate_toml( + 'version = "1.0"\n[verifier.environment]\ncpus = 2\n' + "[verifier.sandbox]\ncpus = 3\n" + ) + + +def test_task_config_dump_emits_sandbox_key(): + """Emitters follow the rename: dumps carry 'sandbox', never 'environment'.""" + cfg = TaskConfig.model_validate({"sandbox": {"cpus": 2}}) + dumped = cfg.model_dump_toml() + assert "[sandbox]" in dumped + assert "[environment]" not in dumped + + +def test_task_config_rejects_renamed_verifier_environment_mode_key(): + with pytest.raises( + ValueError, + match=r"the 'verifier\.environment_mode' key was renamed to " + r"'verifier\.sandbox_mode'", + ): + TaskConfig.model_validate({"verifier": {"environment_mode": "separate"}}) + + +def test_task_config_toml_converts_legacy_verifier_environment_mode(): + cfg = TaskConfig.model_validate_toml( + 'version = "1.0"\n[verifier]\nenvironment_mode = "separate"\n' + ) + assert cfg.verifier.sandbox_mode == VerifierSandboxMode.SEPARATE + + +def test_task_config_toml_rejects_both_environment_mode_and_sandbox_mode(): + with pytest.raises( + ValueError, + match=r"declares both 'verifier\.environment_mode' and " + r"'verifier\.sandbox_mode'", + ): + TaskConfig.model_validate_toml( + 'version = "1.0"\n[verifier]\nenvironment_mode = "separate"\n' + 'sandbox_mode = "separate"\n' + ) + + +def test_task_config_toml_converts_step_verifier_environment_with_indexed_error(): + cfg = TaskConfig.model_validate_toml( + 'version = "1.0"\n[[steps]]\nname = "one"\n' + "[steps.verifier.environment]\ncpus = 2\n" + ) + assert cfg.steps is not None + assert cfg.steps[0].verifier.sandbox is not None + assert cfg.steps[0].verifier.sandbox.cpus == 2 + + with pytest.raises( + ValueError, + match=r"declares both 'steps\[0\]\.verifier\.environment' and " + r"'steps\[0\]\.verifier\.sandbox'", + ): + TaskConfig.model_validate_toml( + 'version = "1.0"\n[[steps]]\nname = "one"\n' + "[steps.verifier.environment]\ncpus = 2\n" + "[steps.verifier.sandbox]\ncpus = 3\n" + ) diff --git a/tests/test_task_document.py b/tests/test_task_document.py index 5b52d4ac4..0a396db57 100644 --- a/tests/test_task_document.py +++ b/tests/test_task_document.py @@ -16,7 +16,7 @@ MultiStepRewardStrategy, NetworkMode, TaskOS, - VerifierEnvironmentMode, + VerifierSandboxMode, ) from benchflow.task.document import ( dump_frontmatter_yaml, @@ -49,7 +49,11 @@ def test_task_document_preserves_demo_task_config_and_prompt() -> None: def test_render_task_md_from_legacy_emits_only_declared_frontmatter( tmp_path: Path, ) -> None: - """Migration must not materialize runtime defaults the author never wrote.""" + """Migration must not materialize runtime defaults the author never wrote. + + The legacy ``[environment]`` table is emitted under the native + ``sandbox`` key — migration follows the rename. + """ (tmp_path / "task.toml").write_text( '[metadata]\ndifficulty = "easy"\n\n' "[agent]\ntimeout_sec = 120\n\n" @@ -64,7 +68,7 @@ def test_render_task_md_from_legacy_emits_only_declared_frontmatter( "schema_version": "1.3", "metadata": {"difficulty": "easy"}, "agent": {"timeout_sec": 120}, - "environment": {"cpus": 2}, + "sandbox": {"cpus": 2}, } @@ -118,8 +122,8 @@ def test_task_document_minimal_profile_authoring_parses() -> None: assert document.config.task is not None assert document.config.task.name == "benchflow/runtime-capability-gate" - assert document.config.environment.docker_image == "ghcr.io/example/task:latest" - assert document.config.environment.network_mode == NetworkMode.NO_NETWORK + assert document.config.sandbox.docker_image == "ghcr.io/example/task:latest" + assert document.config.sandbox.network_mode == NetworkMode.NO_NETWORK assert document.config.verifier.timeout_sec == 1200 assert document.benchflow["verifier"]["spec"] == "verifier/verifier.md" assert document.benchflow["oracle"]["path"] == "oracle/" @@ -270,7 +274,7 @@ def test_task_document_explicit_fields_override_profile_defaults() -> None: image: ubuntu:24.04 agent: timeout_sec: 42 -environment: +sandbox: docker_image: ghcr.io/example/explicit:latest cpus: 9 verifier: @@ -281,8 +285,8 @@ def test_task_document_explicit_fields_override_profile_defaults() -> None: ) assert document.config.agent.timeout_sec == 42 - assert document.config.environment.docker_image == "ghcr.io/example/explicit:latest" - assert document.config.environment.cpus == 9 + assert document.config.sandbox.docker_image == "ghcr.io/example/explicit:latest" + assert document.config.sandbox.cpus == 9 assert document.config.verifier.timeout_sec == 33 @@ -326,7 +330,7 @@ def test_task_document_parses_roles_scenes_and_user_persona() -> None: timeout_sec: 300 verifier: timeout_sec: 120 -environment: +sandbox: cpus: 2 memory_mb: 4096 agents: @@ -370,7 +374,7 @@ def test_task_document_parses_roles_scenes_and_user_persona() -> None: ) assert document.instruction == "Handle the refund request." - assert document.config.environment.memory_mb == 4096 + assert document.config.sandbox.memory_mb == 4096 assert document.roles["planner"].capabilities == ["tool-use"] assert document.user["stop_rule"] == "done-or-3-rounds" assert document.user_persona == ( @@ -418,16 +422,16 @@ def test_task_document_frontmatter_matches_current_harbor_task_config_surface() JUDGE_API_KEY: ${JUDGE_API_KEY:-test} user: root network_mode: public - environment_mode: separate + sandbox_mode: separate pytest_plugins: [pytest_playwright] hardening: cleanup_conftests: false - environment: + sandbox: docker_image: ghcr.io/example/grader:latest cpus: 2 memory_mb: 1024 network_mode: no-network -environment: +sandbox: network_mode: allowlist allowed_hosts: [datasets.example.com] build_timeout_sec: 600 @@ -489,16 +493,16 @@ def test_task_document_frontmatter_matches_current_harbor_task_config_surface() assert cfg.task.name == "benchflow/harbor-parity" assert cfg.metadata["custom"]["kept"] is True assert cfg.agent.network_mode == NetworkMode.ALLOWLIST - assert cfg.verifier.environment_mode == VerifierEnvironmentMode.SEPARATE + assert cfg.verifier.sandbox_mode == VerifierSandboxMode.SEPARATE assert cfg.verifier.hardening.cleanup_conftests is False - assert cfg.verifier.environment is not None - assert cfg.verifier.environment.allow_internet is False - assert cfg.environment.network_mode == NetworkMode.ALLOWLIST - assert cfg.environment.os == TaskOS.LINUX - assert cfg.environment.tpu is not None - assert cfg.environment.tpu.chip_count == 8 - assert cfg.environment.healthcheck is not None - assert cfg.environment.healthcheck.retries == 5 + assert cfg.verifier.sandbox is not None + assert cfg.verifier.sandbox.allow_internet is False + assert cfg.sandbox.network_mode == NetworkMode.ALLOWLIST + assert cfg.sandbox.os == TaskOS.LINUX + assert cfg.sandbox.tpu is not None + assert cfg.sandbox.tpu.chip_count == 8 + assert cfg.sandbox.healthcheck is not None + assert cfg.sandbox.healthcheck.retries == 5 assert cfg.solution.env == {"SOLUTION_MODE": "oracle"} assert cfg.multi_step_reward_strategy == MultiStepRewardStrategy.FINAL assert cfg.artifacts[0].source == "/logs/artifacts" @@ -512,7 +516,7 @@ def test_task_document_rejects_unknown_task_config_fields() -> None: with pytest.raises(ValueError, match="unknown_harbor_field"): TaskDocument.from_text( """--- -environment: +sandbox: unknown_harbor_field: true --- ## prompt @@ -660,7 +664,7 @@ def test_task_loads_task_md_without_legacy_pair(tmp_path: Path) -> None: category: demo agent: timeout_sec: 120 -environment: +sandbox: cpus: 1 --- ## prompt @@ -1346,3 +1350,40 @@ def test_sidecar_prompt_files_take_precedence_over_legacy_headings( assert doc.instruction == "Base prompt." assert doc.role_prompts == {"reviewer": "File version wins."} + + +def test_task_document_rejects_renamed_environment_frontmatter_key() -> None: + """task.md is the native format: only 'sandbox:' validates, and the + 'environment:' failure names the rename instead of a bare extra-key error.""" + with pytest.raises( + ValueError, + match=r"the 'environment' key was renamed to 'sandbox' — rename " + r"'environment:' to 'sandbox:'", + ): + TaskDocument.from_text( + """--- +version: "1.0" +environment: + cpus: 2 +--- +Do it. +""" + ) + + +def test_task_document_rejects_renamed_verifier_environment_frontmatter_key() -> None: + with pytest.raises( + ValueError, + match=r"the 'verifier\.environment' key was renamed to " + r"'verifier\.sandbox'", + ): + TaskDocument.from_text( + """--- +version: "1.0" +verifier: + environment: + cpus: 2 +--- +Do it. +""" + ) diff --git a/tests/test_task_export.py b/tests/test_task_export.py index 322df5442..b00dc2ce3 100644 --- a/tests/test_task_export.py +++ b/tests/test_task_export.py @@ -71,7 +71,7 @@ def _write_native_task(task_dir: Path) -> None: timeout_sec: 300 verifier: timeout_sec: 120 - environment: + sandbox: network_mode: no-network agents: roles: @@ -314,7 +314,7 @@ def test_migrate_minimal_frontmatter_round_trips_equivalent_config( text = result.task_md.read_text() frontmatter = yaml.safe_load(text.split("---\n")[1]) - assert sorted(frontmatter) == ["agent", "environment", "metadata", "schema_version"] + assert sorted(frontmatter) == ["agent", "metadata", "sandbox", "schema_version"] assert "judge" not in text document = TaskDocument.from_path(result.task_md) assert document.config.model_dump() == ( @@ -345,8 +345,8 @@ def test_export_rehydrates_preserved_foreign_extensions(tmp_path: Path) -> None: task_md = task_dir / "task.md" task_md_text = task_md.read_text() task_md_text = task_md_text.replace( - "environment:\n network_mode: no-network\n", - """environment: + "sandbox:\n network_mode: no-network\n", + """sandbox: network_mode: no-network steps: - name: phase-one @@ -359,13 +359,13 @@ def test_export_rehydrates_preserved_foreign_extensions(tmp_path: Path) -> None: compat: source: harbor extra_paths: - - environment.modal.image - harbor_ext + - sandbox.modal.image - steps[0].runner - verifier.reward_kit.metric extra: harbor_ext: kept - environment: + sandbox: modal: image: registry.example.com/task:latest steps: @@ -380,8 +380,11 @@ def test_export_rehydrates_preserved_foreign_extensions(tmp_path: Path) -> None: report = export_task_to_split_layout(task_dir, out_dir) + # The harbor target emits Harbor's spelling: the merged native 'sandbox' + # table (including restored extras) leaves as '[environment]'. exported = tomllib.loads((out_dir / "task.toml").read_text()) assert exported["harbor_ext"] == "kept" + assert "sandbox" not in exported assert exported["environment"]["modal"] == { "image": "registry.example.com/task:latest" } @@ -389,8 +392,8 @@ def test_export_rehydrates_preserved_foreign_extensions(tmp_path: Path) -> None: assert exported["steps"][0]["runner"] == "harbor-step-runner" assert exported["verifier"]["reward_kit"] == {"metric": "exact_match"} assert report.restored_extension_paths == [ - "environment.modal.image", "harbor_ext", + "sandbox.modal.image", "steps[0].runner", "verifier.reward_kit.metric", ] @@ -398,6 +401,57 @@ def test_export_rehydrates_preserved_foreign_extensions(tmp_path: Path) -> None: assert "benchflow.compat" not in loss_paths +def test_export_converts_pre_rename_compat_envelope_environment_extras( + tmp_path: Path, +) -> None: + """A pre-rename compat envelope spelling extras 'environment.*' must not + export 'environment' NEXT TO 'sandbox' (a file that hard-errors on + re-import): the envelope is normalized to 'sandbox' before the merge, and + the harbor target then emits a single '[environment]' table.""" + task_dir = tmp_path / "native" + out_dir = tmp_path / "exported" + _write_native_task(task_dir) + task_md = task_dir / "task.md" + task_md.write_text( + task_md.read_text().replace( + "benchflow:\n teams:\n", + """benchflow: + compat: + source: harbor + extra_paths: + - environment.modal.image + extra: + environment: + modal: + image: registry.example.com/task:latest + teams: +""", + ) + ) + + export_task_to_split_layout(task_dir, out_dir) + + exported_text = (out_dir / "task.toml").read_text() + exported = tomllib.loads(exported_text) + assert "sandbox" not in exported + assert exported["environment"]["network_mode"] == "no-network" + assert exported["environment"]["modal"] == { + "image": "registry.example.com/task:latest" + } + # The exported file re-imports cleanly (no both-spellings hard error); + # the foreign 'modal' extra goes back into a compat envelope, so the + # re-import path is the compat importer rather than the strict loader. + from benchflow.task import import_task_config_toml + + reimported = import_task_config_toml(exported_text, source="harbor") + assert reimported.config.sandbox.network_mode is not None + assert reimported.report.extra_paths == ("sandbox.modal.image",) + # The parsed source document is not corrupted by the export-side + # conversion (the converter must operate on a copy of the envelope). + document = TaskDocument.from_path(task_md) + assert "environment" in document.benchflow["compat"]["extra"] + + def test_build_report_detects_alias_collisions(tmp_path: Path) -> None: """Mixed native and split aliases cannot silently collapse during export.""" task_dir = tmp_path / "native" @@ -487,8 +541,8 @@ def test_harbor_roundtrip_reports_restored_foreign_extensions( assert report.status == "lossless" assert report.restored_extension_paths == [ - "environment.modal.image", "harbor_ext", + "sandbox.modal.image", ] diff --git a/tests/test_task_package.py b/tests/test_task_package.py index 67b752344..973114291 100644 --- a/tests/test_task_package.py +++ b/tests/test_task_package.py @@ -44,7 +44,7 @@ def _write_native_task(task_dir: Path) -> None: schema_version: "1.3" task: name: benchflow/package-boundary - environment: + sandbox: network_mode: no-network workdir: /repo benchflow: @@ -119,7 +119,7 @@ def test_task_package_compiles_append_prompt_plan_and_redacts_user_facts( schema_version: "1.3" task: name: benchflow/prompt-plan - environment: + sandbox: network_mode: no-network agents: roles: @@ -194,7 +194,7 @@ def test_task_package_marks_scripted_user_runtime_supported(tmp_path: Path) -> N schema_version: "1.3" task: name: benchflow/supported-user - environment: + sandbox: network_mode: no-network agents: roles: @@ -246,7 +246,7 @@ def test_task_package_marks_model_user_runtime_supported(tmp_path: Path) -> None schema_version: "1.3" task: name: benchflow/model-user - environment: + sandbox: network_mode: no-network agents: roles: @@ -304,7 +304,7 @@ def test_task_package_marks_sequential_team_handoff_supported(tmp_path: Path) -> schema_version: "1.3" task: name: benchflow/team-handoff-user - environment: + sandbox: network_mode: no-network agents: roles: @@ -412,7 +412,7 @@ def test_task_package_compiles_explicit_replace_prompt_plan(tmp_path: Path) -> N schema_version: "1.3" task: name: benchflow/replace-plan - environment: + sandbox: network_mode: no-network agents: roles: diff --git a/tests/test_traces_task_gen.py b/tests/test_traces_task_gen.py index 128113db5..800d63cb5 100644 --- a/tests/test_traces_task_gen.py +++ b/tests/test_traces_task_gen.py @@ -146,8 +146,8 @@ def test_task_config_content( assert task.config.metadata["source_trace_id"] == "test-trace-001" assert task.config.agent.timeout_sec == 300 assert task.config.verifier.timeout_sec == 60 - assert task.config.environment.build_timeout_sec == 600 - assert task.config.environment.storage_mb == 10240 + assert task.config.sandbox.build_timeout_sec == 600 + assert task.config.sandbox.storage_mb == 10240 def test_task_md_prompt_content( self, simple_trace: ParsedTrace, tmp_path: Path diff --git a/tests/test_verifier_document.py b/tests/test_verifier_document.py index 6168917bb..e354943e4 100644 --- a/tests/test_verifier_document.py +++ b/tests/test_verifier_document.py @@ -39,7 +39,7 @@ def test_task_paths_accepts_selected_reward_kit_without_test_sh( """\ --- schema_version: "1.3" - environment: + sandbox: network_mode: no-network --- @@ -322,7 +322,7 @@ def test_check_task_reports_malformed_verifier_document(tmp_path: Path) -> None: """\ --- schema_version: "1.3" - environment: + sandbox: network_mode: no-network --- diff --git a/tests/test_verify.py b/tests/test_verify.py index 5282c4658..b82831e21 100644 --- a/tests/test_verify.py +++ b/tests/test_verify.py @@ -204,7 +204,7 @@ async def test_verifier_timeout_reads_task_name_not_config_name(self, tmp_path): "metadata": {"author_name": "benchflow"}, "agent": {"timeout_sec": 30}, "verifier": {"timeout_sec": 0.1}, - "environment": {"cpus": 1, "memory_mb": 1024}, + "sandbox": {"cpus": 1, "memory_mb": 1024}, } ) assert not hasattr(config, "name") # the bug condition