feat(sandbox): add Apple Container backend for macOS Virtualization.framework

README.md

@@ -19,7 +19,7 @@ BenchFlow is a universal environment framework: it runs AI agents against task e
 - **Loop strategies** — wrap any agent in a `--loop-strategy` (`verify-retry`, `self-review`); every rollout captures a per-iteration reward + token trajectory, so you can plot capability against cost (can a cheap model + loops match an expensive one at equal token spend?)
 - **`task.md` tasks** — one file (YAML frontmatter + prompt body) replaces the split `task.toml` + `instruction.md` layout; author with `bench tasks init` / `check` / `migrate` / `export`
 - **Hosted environments** — run external PrimeIntellect / Verifiers environments through `--source-env`, without converting them to BenchFlow tasks
-- **Sandboxes** — Docker locally, Daytona for parallel cloud runs (orphaned sandboxes auto-reaped at eval start), Modal for serverless/GPU-backed task environments
+- **Sandboxes** — Docker locally, Apple Container on Apple Silicon Macs, Daytona for parallel cloud runs (orphaned sandboxes auto-reaped at eval start), Modal for serverless/GPU-backed task environments
 - **Hardened verifier** — defaults block BenchJack/Meerkat-style reward-hacking; tasks opt out per-feature
 - **Training-ready output** — every scored rollout emits ATIF (`trainer/atif.json`) and ADP (`trainer/adp.jsonl`) trajectory records next to the Verifiers/ORS (OpenReward) reward record
 

docs/running-benchmarks.md

@@ -344,9 +344,18 @@ The **Harvey LAB harness** agent is special — it runs Harvey LAB's own agent l
 | Sandbox | Flag | Best for |
 |---------|------|----------|
 | Docker | `--sandbox docker` | Local development, small runs (≤10 tasks) |
+| Apple Container | `--sandbox apple-container` | Local Apple Silicon macOS runs without Docker Desktop |
 | Daytona | `--sandbox daytona` | Cloud runs with concurrency (needs `DAYTONA_API_KEY`) |
 | Modal | `--sandbox modal` | Serverless, high concurrency (needs Modal auth) |
 
+Apple Container requires Apple Container 1.1+ on Apple Silicon and runs the model
+proxy inside each VM. It supports public-network, single-container arm64 tasks and
+has no snapshot support. BenchFlow serializes Apple rollouts within each process
+and blocks new VMs when the live `data.kalloc.1024` headroom is unsafe. Avoid
+running concurrent BenchFlow processes, because the macOS allocation leak is
+system-wide. Use Docker, Daytona, or Modal for `network_mode = "no-network"`,
+multi-service, snapshot, or high-concurrency runs.
+
 For large-scale runs (100+ tasks), use Daytona or Modal with high concurrency:
 
 ```bash

src/benchflow/acp/runtime.py

@@ -46,7 +46,12 @@
     build_priv_drop_cmd,
     enforce_agent_egress_firewall,
 )
-from benchflow.sandbox.process import DaytonaProcess, DaytonaPtyProcess, DockerProcess
+from benchflow.sandbox.process import (
+    AppleContainerProcess,
+    DaytonaProcess,
+    DaytonaPtyProcess,
+    DockerProcess,
+)
 from benchflow.trajectories._capture import _capture_session_trajectory
 
 # Re-exported for backwards compatibility — tests and downstream code
@@ -574,6 +579,8 @@ async def connect_acp(
         try:
             if environment == "docker":
                 live_proc = DockerProcess.from_sandbox_env(env)
+            elif environment == "apple-container":
+                live_proc = AppleContainerProcess.from_sandbox_env(env)
             elif environment == "daytona":
                 transport_name = selected_acp_transport(
                     agent=agent,

src/benchflow/cli/main.py

@@ -588,7 +588,7 @@ def eval_run(
 ) -> None:
     """Run an evaluation — single task or batch.
 
-    Sandbox: docker, daytona, or modal.
+    Sandbox: docker, daytona, modal, or apple-container.
     """
     _apply_dotenv_to_process_env()
 

src/benchflow/continue_run/orchestrator.py

@@ -37,7 +37,7 @@
     sandbox_replay_base_url,
 )
 from benchflow.contracts import AgentProtocolError, SandboxStartupFailure
-from benchflow.sandbox.providers import OFF_BOX_MODEL_PROVIDERS
+from benchflow.sandbox.providers import SANDBOX_MODEL_PROXY_PROVIDERS
 from benchflow.scenes import compile_scenes_to_steps
 from benchflow.trajectories.types import LLMExchange, redact_trajectory_text
 
@@ -49,9 +49,9 @@
 # OpenAI-compatible endpoint.
 _REPLAY_API_KEY = "sk-benchflow-replay"
 _REPLAY_MODEL = "openai/replay"
-# Off-box-model providers (≡ non-docker) — derived from the canonical registry
-# so this replay-routing subset can't drift from litellm_runtime's copy.
-_SANDBOX_LOCAL_REPLAY_ENVIRONMENTS = OFF_BOX_MODEL_PROVIDERS
+# Providers whose replay and model proxies must run inside the sandbox. This is
+# the same placement contract used by the normal LiteLLM runtime.
+_SANDBOX_LOCAL_REPLAY_ENVIRONMENTS = SANDBOX_MODEL_PROXY_PROVIDERS
 _PROXY_MODES = frozenset({"auto", "host", "sandbox"})
 
 

src/benchflow/providers/litellm_runtime.py

@@ -49,7 +49,7 @@
     extract_usage_from_trajectory,
     trajectory_from_litellm_callback_log,
 )
-from benchflow.sandbox.providers import OFF_BOX_MODEL_PROVIDERS
+from benchflow.sandbox.providers import SANDBOX_MODEL_PROXY_PROVIDERS
 from benchflow.trajectories._llm_capture import LiveLLMTrajectoryWriter
 from benchflow.trajectories.types import Trajectory
 from benchflow.usage_tracking import UsageTrackingConfig, usage_unavailable
@@ -80,9 +80,10 @@
 # GenerateContent format), so they talk to their provider directly and report
 # usage_source='unavailable'. ``oracle`` has no model at all.
 _NATIVE_PROTOCOL_AGENTS = frozenset({"oracle", "gemini"})
-# Providers whose model traffic exits the sandbox to the host proxy (≡ non-docker);
-# derived from the canonical registry so it can't drift from the provider set.
-_SANDBOX_LOCAL_ENVIRONMENTS = OFF_BOX_MODEL_PROVIDERS
+# Providers whose mandatory LiteLLM proxy runs inside the sandbox. Keeping this
+# placement policy in the canonical provider registry prevents a new backend from
+# accidentally handing an in-sandbox agent a host-loopback endpoint.
+_SANDBOX_LOCAL_ENVIRONMENTS = SANDBOX_MODEL_PROXY_PROVIDERS
 
 
 @dataclass(frozen=True)