diff --git a/benchmarks/harbor-buzz-orchestra/scripts/benchmark.py b/benchmarks/harbor-buzz-orchestra/scripts/benchmark.py index f1e91d022b..b6f5601a82 100755 --- a/benchmarks/harbor-buzz-orchestra/scripts/benchmark.py +++ b/benchmarks/harbor-buzz-orchestra/scripts/benchmark.py @@ -84,51 +84,75 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: ) problems = parser.add_mutually_exclusive_group() problems.add_argument( - "--dataset", "-d", default=None, + "--dataset", + "-d", + default=None, help=f"Registry dataset (default: {DEFAULT_DATASET})", ) problems.add_argument( "--path", "-p", type=Path, help="Local task or dataset directory" ) parser.add_argument( - "--include-task", "-i", action="append", default=[], + "--include-task", + "-i", + action="append", + default=[], help="Task name to include (glob, repeatable)", ) parser.add_argument( - "--exclude-task", "-x", action="append", default=[], + "--exclude-task", + "-x", + action="append", + default=[], help="Task name to exclude (glob, repeatable)", ) parser.add_argument( - "--attempts", "-k", type=int, default=DEFAULT_ATTEMPTS, + "--attempts", + "-k", + type=int, + default=DEFAULT_ATTEMPTS, help=f"Runs per problem (default: {DEFAULT_ATTEMPTS}, the leaderboard requirement)", ) parser.add_argument( - "--manifest", type=Path, default=DEFAULT_MANIFEST, + "--manifest", + type=Path, + default=DEFAULT_MANIFEST, help=f"Team manifest YAML (default: {DEFAULT_MANIFEST.name})", ) parser.add_argument( - "--endpoint-config", type=Path, default=DEFAULT_ENDPOINTS, + "--endpoint-config", + type=Path, + default=DEFAULT_ENDPOINTS, help=f"Endpoint provider/API-key mapping (default: {DEFAULT_ENDPOINTS.name})", ) - parser.add_argument("--n-concurrent", "-n", type=int, default=4, help="Concurrent trials") + parser.add_argument( + "--n-concurrent", "-n", type=int, default=4, help="Concurrent trials" + ) parser.add_argument( "--jobs-dir", type=Path, default=PACKAGE_ROOT / "jobs", help="Job output root" ) - parser.add_argument("--job-name", default=None, help="Job name (default: lb--)") parser.add_argument( - "--upload", action="store_true", help="Upload to Harbor Hub when the job finishes" + "--job-name", default=None, help="Job name (default: lb--)" + ) + parser.add_argument( + "--upload", + action="store_true", + help="Upload to Harbor Hub when the job finishes", ) parser.add_argument( - "--gui", action="store_true", + "--gui", + action="store_true", help="Open the Buzz desktop app as the benchmark user to watch the run live", ) parser.add_argument( - "--fresh", action="store_true", + "--fresh", + action="store_true", help="Reset first: drop the stack's Docker volumes and the benchmark " - "GUI's app state (keys in state.json are kept)", + "GUI's app state (keys in state.json are kept)", ) parser.add_argument( - "--dry-run", action="store_true", + "--dry-run", + action="store_true", help="Print the underlying harbor command and exit (no stack bring-up)", ) return parser.parse_args(argv) @@ -153,7 +177,6 @@ def load_state() -> dict[str, str]: "user_secret_key": user.secret_key, "postgres_password": secrets.token_urlsafe(24), "redis_password": secrets.token_urlsafe(24), - "typesense_api_key": secrets.token_hex(16), "s3_access_key": secrets.token_hex(10), "s3_secret_key": secrets.token_hex(20), "git_hook_hmac_secret": secrets.token_hex(32), @@ -202,7 +225,6 @@ def write_env_file(state: dict[str, str]) -> Path: "POSTGRES_USER": "buzz", "POSTGRES_PASSWORD": state["postgres_password"], "REDIS_PASSWORD": state["redis_password"], - "TYPESENSE_API_KEY": state["typesense_api_key"], "BUZZ_S3_ACCESS_KEY": state["s3_access_key"], "BUZZ_S3_SECRET_KEY": state["s3_secret_key"], "BUZZ_S3_BUCKET": "buzz-media", @@ -217,14 +239,11 @@ def write_env_file(state: dict[str, str]) -> Path: def postgres_dsn(state: dict[str, str]) -> str: return ( - f"postgresql://buzz:{state['postgres_password']}" - f"@127.0.0.1:{PG_HOST_PORT}/buzz" + f"postgresql://buzz:{state['postgres_password']}@127.0.0.1:{PG_HOST_PORT}/buzz" ) -def write_provisioner_config( - state: dict[str, str], endpoint_config: Path -) -> Path: +def write_provisioner_config(state: dict[str, str], endpoint_config: Path) -> Path: """Resolve per-endpoint API keys from the environment and write the provisioner config: pinned user, keep-channels teardown.""" endpoints = json.loads(endpoint_config.read_text()) @@ -262,10 +281,14 @@ def write_provisioner_config( def compose_command(*args: str) -> list[str]: command = [ - "docker", "compose", - "--project-name", COMPOSE_PROJECT, - "--project-directory", str(STATE_DIR), - "--env-file", str(STATE_DIR / ".env"), + "docker", + "compose", + "--project-name", + COMPOSE_PROJECT, + "--project-directory", + str(STATE_DIR), + "--env-file", + str(STATE_DIR / ".env"), ] for file in COMPOSE_FILES: command += ["-f", str(file)] @@ -360,7 +383,9 @@ def linux_triple() -> str: """The musl triple matching the Docker engine that runs task containers.""" arch = subprocess.run( ["docker", "version", "--format", "{{.Server.Arch}}"], - capture_output=True, text=True, check=True, + capture_output=True, + text=True, + check=True, ).stdout.strip() try: return { @@ -385,22 +410,32 @@ def ensure_agent_binaries() -> Path: targets = AGENT_BINARIES + (FORWARDER_BINARY,) if all((bin_dir / name).is_file() for name in targets): return bin_dir - print(f"Linux agent binaries missing — cross-building for {triple} " - f"in {RUST_IMAGE} (first run only, ~2 min)...") + print( + f"Linux agent binaries missing — cross-building for {triple} " + f"in {RUST_IMAGE} (first run only, ~2 min)..." + ) LINUX_TARGET_DIR.mkdir(parents=True, exist_ok=True) (STATE_DIR / "cargo-registry").mkdir(exist_ok=True) packages = [arg for name in AGENT_BINARIES for arg in ("-p", name)] forwarder_src = FORWARDER_SOURCE.relative_to(REPO_ROOT) subprocess.run( [ - "docker", "run", "--rm", - "-v", f"{REPO_ROOT}:/src:ro", - "-v", f"{LINUX_TARGET_DIR}:/target", - "-v", f"{STATE_DIR / 'cargo-registry'}:/usr/local/cargo/registry", - "-e", "CARGO_TARGET_DIR=/target", - "-w", "/src", + "docker", + "run", + "--rm", + "-v", + f"{REPO_ROOT}:/src:ro", + "-v", + f"{LINUX_TARGET_DIR}:/target", + "-v", + f"{STATE_DIR / 'cargo-registry'}:/usr/local/cargo/registry", + "-e", + "CARGO_TARGET_DIR=/target", + "-w", + "/src", RUST_IMAGE, - "sh", "-c", + "sh", + "-c", "apk add --no-cache musl-dev >/dev/null && " f"cargo build --release --locked --target {triple} " + " ".join(packages) @@ -429,8 +464,13 @@ def launch_gui(state: dict[str, str]) -> subprocess.Popen: """ subprocess.run( compose_command( - "exec", "-T", "relay", - "buzz-admin", "add-member", "--pubkey", state["user_pubkey"], + "exec", + "-T", + "relay", + "buzz-admin", + "add-member", + "--pubkey", + state["user_pubkey"], ), check=True, ) @@ -445,12 +485,20 @@ def launch_gui(state: dict[str, str]) -> subprocess.Popen: ["rustc", "-vV"], capture_output=True, text=True, check=True ).stdout triple = next( - line.split(": ", 1)[1] for line in target.splitlines() if line.startswith("host: ") + line.split(": ", 1)[1] + for line in target.splitlines() + if line.startswith("host: ") ) sidecar_dir = desktop_dir / "src-tauri" / "binaries" sidecar_dir.mkdir(parents=True, exist_ok=True) binaries = ensure_binaries() - for name in ("buzz-acp", "buzz-agent", "buzz-dev-mcp", "git-credential-nostr", "buzz"): + for name in ( + "buzz-acp", + "buzz-agent", + "buzz-dev-mcp", + "git-credential-nostr", + "buzz", + ): stub = sidecar_dir / f"{name}-{triple}" if not stub.exists(): stub.touch() @@ -498,21 +546,30 @@ def leaderboard_argv( for pattern in args.exclude_task: argv += ["--exclude-task", pattern] argv += [ - "--attempts", str(args.attempts), - "--manifest", str(args.manifest), - "--endpoint-config", str(args.endpoint_config), - "--provisioner-config", str(provisioner_config), - "--agent-bin-dir", str(agent_bin_dir), + "--attempts", + str(args.attempts), + "--manifest", + str(args.manifest), + "--endpoint-config", + str(args.endpoint_config), + "--provisioner-config", + str(provisioner_config), + "--agent-bin-dir", + str(agent_bin_dir), # The relay as reachable from inside a task container: Docker's # host alias, bridged to the canonical localhost address by the # uploaded forwarder. Override the alias with # BUZZ_BENCHMARK_DOCKER_HOST if your engine exposes the host # differently. "--relay-gateway", - f"{os.environ.get('BUZZ_BENCHMARK_DOCKER_HOST', 'host.docker.internal')}" - f":{RELAY_HTTP_PORT}", - "--n-concurrent", str(args.n_concurrent), - "--jobs-dir", str(args.jobs_dir), + ( + f"{os.environ.get('BUZZ_BENCHMARK_DOCKER_HOST', 'host.docker.internal')}" + f":{RELAY_HTTP_PORT}" + ), + "--n-concurrent", + str(args.n_concurrent), + "--jobs-dir", + str(args.jobs_dir), ] if args.job_name: argv += ["--job-name", args.job_name] diff --git a/benchmarks/harbor-buzz-orchestra/scripts/run_leaderboard.py b/benchmarks/harbor-buzz-orchestra/scripts/run_leaderboard.py index 6fd43ea6fb..6eaf8d6af0 100755 --- a/benchmarks/harbor-buzz-orchestra/scripts/run_leaderboard.py +++ b/benchmarks/harbor-buzz-orchestra/scripts/run_leaderboard.py @@ -45,64 +45,101 @@ # host-header tenant-bound, so agents must present its canonical Host). FORWARDER_BINARY = "relay-forwarder" -PROVIDER_ORGS = {"anthropic": "Anthropic", "openai": "OpenAI", "databricks": "Databricks"} +PROVIDER_ORGS = { + "anthropic": "Anthropic", + "openai": "OpenAI", + "databricks": "Databricks", +} def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser = argparse.ArgumentParser( - description=__doc__.splitlines()[0], formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__.splitlines()[0], + formatter_class=argparse.RawDescriptionHelpFormatter, ) problems = parser.add_mutually_exclusive_group(required=True) problems.add_argument( - "--dataset", "-d", help="Registry dataset (e.g. terminal-bench/terminal-bench-2-1)" + "--dataset", + "-d", + help="Registry dataset (e.g. terminal-bench/terminal-bench-2-1)", ) problems.add_argument( "--path", "-p", type=Path, help="Local task or dataset directory" ) parser.add_argument( - "--include-task", "-i", action="append", default=[], + "--include-task", + "-i", + action="append", + default=[], help="Task name to include from the dataset (glob, repeatable)", ) parser.add_argument( - "--exclude-task", "-x", action="append", default=[], + "--exclude-task", + "-x", + action="append", + default=[], help="Task name to exclude from the dataset (glob, repeatable)", ) parser.add_argument( - "--attempts", "-k", type=int, required=True, + "--attempts", + "-k", + type=int, + required=True, help="Runs per problem (leaderboards require 5)", ) - parser.add_argument("--manifest", type=Path, required=True, help="Team manifest YAML") parser.add_argument( - "--endpoint-config", type=Path, required=True, + "--manifest", type=Path, required=True, help="Team manifest YAML" + ) + parser.add_argument( + "--endpoint-config", + type=Path, + required=True, help="JSON mapping manifest endpoint names to providers/API keys", ) parser.add_argument( - "--provisioner-config", type=Path, required=True, + "--provisioner-config", + type=Path, + required=True, help="JSON config for the Buzz relay/Postgres provisioner", ) parser.add_argument( - "--buzz-bin-dir", type=Path, default=None, + "--buzz-bin-dir", + type=Path, + default=None, help="Directory with the host buzz CLI (default: repo target/release, then target/debug)", ) parser.add_argument( - "--agent-bin-dir", type=Path, required=True, + "--agent-bin-dir", + type=Path, + required=True, help="Directory with Linux builds of buzz-acp/buzz-agent/buzz-dev-mcp " "to upload into each task container", ) parser.add_argument( - "--relay-gateway", default="", + "--relay-gateway", + default="", help="host:port of the benchmark relay as reachable from inside the " "task container (e.g. host.docker.internal:3600). When set, a " "loopback forwarder from --agent-bin-dir bridges the canonical " "relay address to this gateway", ) - parser.add_argument("--n-concurrent", "-n", type=int, default=4, help="Concurrent trials") - parser.add_argument("--jobs-dir", type=Path, default=Path("jobs"), help="Job output root") - parser.add_argument("--job-name", default=None, help="Job name (default: lb--)") parser.add_argument( - "--upload", action="store_true", help="Upload to Harbor Hub when the job finishes" + "--n-concurrent", "-n", type=int, default=4, help="Concurrent trials" + ) + parser.add_argument( + "--jobs-dir", type=Path, default=Path("jobs"), help="Job output root" + ) + parser.add_argument( + "--job-name", default=None, help="Job name (default: lb--)" + ) + parser.add_argument( + "--upload", + action="store_true", + help="Upload to Harbor Hub when the job finishes", + ) + parser.add_argument( + "--dry-run", action="store_true", help="Print the harbor command and exit" ) - parser.add_argument("--dry-run", action="store_true", help="Print the harbor command and exit") return parser.parse_args(argv) @@ -110,7 +147,9 @@ def find_binaries(bin_dir: Path | None) -> dict[str, Path]: candidates = ( [bin_dir] if bin_dir is not None - else [PACKAGE_ROOT.parents[1] / "target" / kind for kind in ("release", "debug")] + else [ + PACKAGE_ROOT.parents[1] / "target" / kind for kind in ("release", "debug") + ] ) for candidate in candidates: found = {name: candidate / name for name in BINARIES} @@ -146,11 +185,17 @@ def build_command( resource override would fail leaderboard static validation, so none are accepted or forwarded.""" command = [ - "harbor", "run", "--yes", - "--job-name", args.job_name, - "--jobs-dir", str(args.jobs_dir), - "-k", str(args.attempts), - "--n-concurrent", str(args.n_concurrent), + "harbor", + "run", + "--yes", + "--job-name", + args.job_name, + "--jobs-dir", + str(args.jobs_dir), + "-k", + str(args.attempts), + "--n-concurrent", + str(args.n_concurrent), ] if args.dataset: command += ["--dataset", args.dataset] @@ -250,7 +295,7 @@ def main(argv: list[str] | None = None) -> int: f"{PACKAGE_ROOT / 'testbed'} {Path(__file__).resolve()} ..." ) - result = subprocess.run(command) + result = subprocess.run(command, check=False) job_dir = args.jobs_dir / args.job_name if result.returncode != 0: print(f"harbor run failed (exit {result.returncode}); job dir: {job_dir}") diff --git a/benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/__init__.py b/benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/__init__.py index b423e8aa47..1b79d233b9 100644 --- a/benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/__init__.py +++ b/benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/__init__.py @@ -1,25 +1,25 @@ """Buzz orchestra custom agent for Harbor.""" from .agent import BuzzOrchestraAgent -from .manifest import ExperimentManifest, ManifestError -from .provisioning import AgentCredential, TrialHandle, TrialProvisioner -from .runtime import OrchestraRuntime, RuntimeResult from .container_runtime import ( BuzzContainerRuntime, EndpointLaunchConfig, RuntimeLaunchError, ) +from .manifest import ExperimentManifest, ManifestError +from .provisioning import AgentCredential, TrialHandle, TrialProvisioner +from .runtime import OrchestraRuntime, RuntimeResult __all__ = [ "AgentCredential", - "BuzzOrchestraAgent", "BuzzContainerRuntime", + "BuzzOrchestraAgent", "EndpointLaunchConfig", "ExperimentManifest", "ManifestError", "OrchestraRuntime", - "RuntimeResult", "RuntimeLaunchError", + "RuntimeResult", "TrialHandle", "TrialProvisioner", ] diff --git a/benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/agent.py b/benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/agent.py index 6354e9a587..3d1c81364f 100644 --- a/benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/agent.py +++ b/benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/agent.py @@ -9,10 +9,10 @@ from harbor.environments.base import BaseEnvironment from harbor.models.agent.context import AgentContext +from .container_runtime import BuzzContainerRuntime, EndpointLaunchConfig from .manifest import ExperimentManifest from .provisioning import TrialProvisioner from .runtime import OrchestraRuntime -from .container_runtime import BuzzContainerRuntime, EndpointLaunchConfig class BuzzOrchestraAgent(BaseAgent): @@ -83,7 +83,7 @@ def _load_mapping( except (OSError, json.JSONDecodeError) as error: raise ValueError(f"cannot load JSON config {path}: {error}") from error if not isinstance(value, dict): - raise ValueError(f"JSON config {path} must contain an object") + raise TypeError(f"JSON config {path} must contain an object") return value @classmethod diff --git a/benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/container_runtime.py b/benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/container_runtime.py index 3909f081f5..149a5295a7 100644 --- a/benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/container_runtime.py +++ b/benchmarks/harbor-buzz-orchestra/src/harbor_buzz_orchestra/container_runtime.py @@ -24,7 +24,6 @@ from .provisioning import AgentCredential, TrialHandle from .runtime import RuntimeResult - DEFAULT_MAX_AGENT_ROUNDS = 32 # Container-side layout for the uploaded Buzz stack. REMOTE_ROOT = "/opt/buzz" @@ -128,12 +127,20 @@ async def run( if forwarder is not None: infra.append(forwarder) await self._buzz_json( - trial.user, trial, "users", "set-profile", "--name", + trial.user, + trial, + "users", + "set-profile", + "--name", trial.user.agent_id, ) for credential in trial.credentials: await self._buzz_json( - credential, trial, "users", "set-profile", "--name", + credential, + trial, + "users", + "set-profile", + "--name", credential.agent_id, ) agents.append( @@ -244,11 +251,17 @@ async def _start_forwarder( ) from error forwarder = _Agent( AgentCredential( - agent_id="relay-forwarder", role="infra", - nostr_secret_key="", nostr_pubkey="", nostr_auth_tag="", - llm_endpoint="", llm_api_key="", + agent_id="relay-forwarder", + role="infra", + nostr_secret_key="", + nostr_pubkey="", + nostr_auth_tag="", + llm_endpoint="", + llm_api_key="", ), - pid, log, log, + pid, + log, + log, ) deadline = asyncio.get_running_loop().time() + self.readiness_timeout_seconds while True: @@ -418,9 +431,14 @@ async def _wait_for_done( await self._raise_for_dead_agents(environment, agents) polls += 1 messages = await self._buzz_json( - trial.user, trial, - "messages", "get", "--channel", trial.channel_id, - "--limit", "100", + trial.user, + trial, + "messages", + "get", + "--channel", + trial.channel_id, + "--limit", + "100", ) for message in messages: if message.get("pubkey") == orchestrator.nostr_pubkey and str( @@ -451,9 +469,7 @@ async def _raise_for_dead_agents( ) @staticmethod - async def _stop_agents( - environment: BaseEnvironment, agents: list[_Agent] - ) -> None: + async def _stop_agents(environment: BaseEnvironment, agents: list[_Agent]) -> None: """Terminate every process of the uploaded stack (acp, agent, mcp).""" if not agents: return @@ -461,14 +477,14 @@ async def _stop_agents( # to exist in task images, the /proc filesystem is. sweep = ( "for d in /proc/[0-9]*; do " - f"grep -aq {REMOTE_BIN} \"$d/cmdline\" 2>/dev/null " - "&& kill -TERM \"${d#/proc/}\" 2>/dev/null; done; true" + f'grep -aq {REMOTE_BIN} "$d/cmdline" 2>/dev/null ' + '&& kill -TERM "${d#/proc/}" 2>/dev/null; done; true' ) try: await environment.exec(sweep) await asyncio.sleep(2) await environment.exec(sweep.replace("-TERM", "-KILL")) - except Exception: # noqa: BLE001 — environment may already be gone + except Exception: # noqa: S110, BLE001 — environment may already be gone pass async def _collect_logs( @@ -476,7 +492,7 @@ async def _collect_logs( ) -> None: try: await environment.download_dir(REMOTE_LOGS, trial_dir) - except Exception: # noqa: BLE001 — best effort; env may be torn down + except Exception: # noqa: S110, BLE001 — best effort; env may be torn down pass # -- Buzz CLI as the trial user / provisioning identities ------------------- @@ -506,9 +522,14 @@ async def _send( self, credential: AgentCredential, trial: TrialHandle, content: str ) -> None: await self._buzz_json( - credential, trial, - "messages", "send", "--channel", trial.channel_id, - "--content", content, + credential, + trial, + "messages", + "send", + "--channel", + trial.channel_id, + "--content", + content, ) async def _buzz_json( @@ -614,9 +635,11 @@ def _compose_system_prompt( "", f"You are `{credential.agent_id}` (pubkey `{credential.nostr_pubkey}`).", f"The team coordinates in Buzz channel `{trial.channel_id}`.", - f"Tasks come from the user `{trial.user.agent_id}` " - f"(pubkey `{trial.user.nostr_pubkey}`); address your final report " - "to them.", + ( + f"Tasks come from the user `{trial.user.agent_id}` " + f"(pubkey `{trial.user.nostr_pubkey}`); address your final report " + "to them." + ), "", "| Name | Role | Pubkey |", "|------|------|--------|", @@ -625,8 +648,7 @@ def _compose_system_prompt( if teammate.agent_id == credential.agent_id: continue lines.append( - f"| {teammate.agent_id} | {teammate.role} " - f"| `{teammate.nostr_pubkey}` |" + f"| {teammate.agent_id} | {teammate.role} | `{teammate.nostr_pubkey}` |" ) composed = persona + "\n".join(lines) + "\n" path = trial_dir / f"{credential.agent_id}.system-prompt.md" diff --git a/benchmarks/harbor-buzz-orchestra/testbed/src/harbor_buzz_testbed/buzz_cli.py b/benchmarks/harbor-buzz-orchestra/testbed/src/harbor_buzz_testbed/buzz_cli.py index ed2bb31cc9..bd11f193ca 100644 --- a/benchmarks/harbor-buzz-orchestra/testbed/src/harbor_buzz_testbed/buzz_cli.py +++ b/benchmarks/harbor-buzz-orchestra/testbed/src/harbor_buzz_testbed/buzz_cli.py @@ -38,6 +38,7 @@ def run(self, *args: str) -> Any: capture_output=True, text=True, timeout=self._timeout, + check=False, env={ "BUZZ_RELAY_URL": self._relay_url, "BUZZ_PRIVATE_KEY": self._secret_key, diff --git a/benchmarks/harbor-buzz-orchestra/testbed/src/harbor_buzz_testbed/provisioner.py b/benchmarks/harbor-buzz-orchestra/testbed/src/harbor_buzz_testbed/provisioner.py index cfda6b59fa..d8f380387d 100644 --- a/benchmarks/harbor-buzz-orchestra/testbed/src/harbor_buzz_testbed/provisioner.py +++ b/benchmarks/harbor-buzz-orchestra/testbed/src/harbor_buzz_testbed/provisioner.py @@ -44,7 +44,7 @@ class TestbedConfig: archive_on_teardown: bool = True -def provisioner_from_dict(config: dict[str, object]) -> "BuzzTrialProvisioner": +def provisioner_from_dict(config: dict[str, object]) -> BuzzTrialProvisioner: """Harbor CLI factory for a JSON-decoded testbed configuration.""" return BuzzTrialProvisioner(TestbedConfig(**config)) @@ -100,7 +100,7 @@ def teardown(self, handle: TrialHandle) -> None: cli = self._cli_for(handle.credentials[0]) try: cli.archive_channel(handle.channel_id) - except Exception as error: # noqa: BLE001 — idempotent re-teardown + except Exception as error: if "archived" not in str(error).lower(): raise with psycopg.connect(self._config.postgres_dsn) as conn: diff --git a/benchmarks/harbor-buzz-orchestra/testbed/tests/test_benchmark.py b/benchmarks/harbor-buzz-orchestra/testbed/tests/test_benchmark.py index 2d88e339f5..e0c6d32ec4 100644 --- a/benchmarks/harbor-buzz-orchestra/testbed/tests/test_benchmark.py +++ b/benchmarks/harbor-buzz-orchestra/testbed/tests/test_benchmark.py @@ -37,8 +37,19 @@ def test_defaults_are_leaderboard_eligible(): def test_selectors_pass_through(): args = benchmark.parse_args( - ["--path", "/tmp/task", "-i", "cobol*", "-x", "flaky*", "-k", "1", - "--job-name", "smoke", "--dry-run"] + [ + "--path", + "/tmp/task", + "-i", + "cobol*", + "-x", + "flaky*", + "-k", + "1", + "--job-name", + "smoke", + "--dry-run", + ] ) argv = benchmark.leaderboard_argv(args, Path("p.json"), Path("b")) assert argv[argv.index("--path") + 1] == "/tmp/task" @@ -59,11 +70,15 @@ def test_state_is_generated_once_and_reused(state_dir): assert "user_pubkey" not in stored # derived, never persisted -def test_provisioner_config_pins_user_and_keeps_channels(state_dir, tmp_path, monkeypatch): +def test_provisioner_config_pins_user_and_keeps_channels( + state_dir, tmp_path, monkeypatch +): monkeypatch.setenv("FAKE_KEY_ENV", "sk-test") endpoints = tmp_path / "endpoints.json" endpoints.write_text( - json.dumps({"model-a": {"provider": "anthropic", "api_key_env": "FAKE_KEY_ENV"}}) + json.dumps( + {"model-a": {"provider": "anthropic", "api_key_env": "FAKE_KEY_ENV"}} + ) ) state = benchmark.load_state() path = benchmark.write_provisioner_config(state, endpoints) @@ -78,7 +93,9 @@ def test_provisioner_config_pins_user_and_keeps_channels(state_dir, tmp_path, mo assert config["relay_http_url"].startswith("http://localhost:") -def test_provisioner_config_missing_api_key_is_explicit(state_dir, tmp_path, monkeypatch): +def test_provisioner_config_missing_api_key_is_explicit( + state_dir, tmp_path, monkeypatch +): monkeypatch.delenv("MISSING_KEY_ENV", raising=False) endpoints = tmp_path / "endpoints.json" endpoints.write_text( @@ -91,9 +108,7 @@ def test_provisioner_config_missing_api_key_is_explicit(state_dir, tmp_path, mon def test_env_file_wires_owner_and_ports(state_dir): state = benchmark.load_state() env_path = benchmark.write_env_file(state) - env = dict( - line.split("=", 1) for line in env_path.read_text().splitlines() if line - ) + env = dict(line.split("=", 1) for line in env_path.read_text().splitlines() if line) assert env["RELAY_OWNER_PUBKEY"] == state["owner_pubkey"] assert env["BUZZ_HTTP_PORT"] == str(benchmark.RELAY_HTTP_PORT) assert env["BUZZ_PG_HOST_PORT"] == str(benchmark.PG_HOST_PORT) diff --git a/benchmarks/harbor-buzz-orchestra/testbed/tests/test_keys.py b/benchmarks/harbor-buzz-orchestra/testbed/tests/test_keys.py index 0f82578369..0ac794e0fa 100644 --- a/benchmarks/harbor-buzz-orchestra/testbed/tests/test_keys.py +++ b/benchmarks/harbor-buzz-orchestra/testbed/tests/test_keys.py @@ -6,6 +6,7 @@ import json import coincurve + from harbor_buzz_testbed.keys import ( compute_auth_tag, encode_nsec, @@ -21,8 +22,10 @@ "auth", "f9308a019258c31049344f85f89d5229b531c845836f99b08601f113bce036f9", "", - "20105c618d6e5d8f559cffb6f0d7a7b4f44f3a567e1be94c96378d45ac3625da" - "34c2e7357ea1d3ce980978334546b3e740c155e81b833ebe140d519d39ed8867", + ( + "20105c618d6e5d8f559cffb6f0d7a7b4f44f3a567e1be94c96378d45ac3625da" + "34c2e7357ea1d3ce980978334546b3e740c155e81b833ebe140d519d39ed8867" + ), ] diff --git a/benchmarks/harbor-buzz-orchestra/testbed/tests/test_provisioner_live.py b/benchmarks/harbor-buzz-orchestra/testbed/tests/test_provisioner_live.py index 5a6b40d8cf..711b877ab4 100644 --- a/benchmarks/harbor-buzz-orchestra/testbed/tests/test_provisioner_live.py +++ b/benchmarks/harbor-buzz-orchestra/testbed/tests/test_provisioner_live.py @@ -14,6 +14,7 @@ import psycopg import pytest + from harbor_buzz_testbed.buzz_cli import BuzzCli, BuzzCliError from harbor_buzz_testbed.provisioner import ( BuzzTrialProvisioner, diff --git a/benchmarks/harbor-buzz-orchestra/testbed/tests/test_provisioner_unit.py b/benchmarks/harbor-buzz-orchestra/testbed/tests/test_provisioner_unit.py index 9620be4bc8..e784de5825 100644 --- a/benchmarks/harbor-buzz-orchestra/testbed/tests/test_provisioner_unit.py +++ b/benchmarks/harbor-buzz-orchestra/testbed/tests/test_provisioner_unit.py @@ -7,6 +7,7 @@ import coincurve import pytest + from harbor_buzz_testbed.provisioner import ( BuzzTrialProvisioner, ProvisioningError, @@ -17,13 +18,13 @@ def config(**overrides) -> TestbedConfig: - defaults = dict( - relay_http_url="http://localhost:3000", - relay_ws_url="ws://host.docker.internal:3000", - owner_secret_key=OWNER_SECRET, - postgres_dsn="postgresql://unused", - llm_api_keys={"databricks/glm": "glm-key", "databricks/opus": "opus-key"}, - ) + defaults = { + "relay_http_url": "http://localhost:3000", + "relay_ws_url": "ws://host.docker.internal:3000", + "owner_secret_key": OWNER_SECRET, + "postgres_dsn": "postgresql://unused", + "llm_api_keys": {"databricks/glm": "glm-key", "databricks/opus": "opus-key"}, + } defaults.update(overrides) return TestbedConfig(**defaults) diff --git a/benchmarks/harbor-buzz-orchestra/tests/conftest.py b/benchmarks/harbor-buzz-orchestra/tests/conftest.py index bd0bcaf2dd..b1de094d76 100644 --- a/benchmarks/harbor-buzz-orchestra/tests/conftest.py +++ b/benchmarks/harbor-buzz-orchestra/tests/conftest.py @@ -1,4 +1,5 @@ from typing import Any + import pytest diff --git a/benchmarks/harbor-buzz-orchestra/tests/test_agent.py b/benchmarks/harbor-buzz-orchestra/tests/test_agent.py index 62c6047ab3..b305344c51 100644 --- a/benchmarks/harbor-buzz-orchestra/tests/test_agent.py +++ b/benchmarks/harbor-buzz-orchestra/tests/test_agent.py @@ -1,7 +1,9 @@ from types import SimpleNamespace from uuid import uuid4 + import pytest from harbor.models.agent.context import AgentContext + from harbor_buzz_orchestra import ( AgentCredential, BuzzOrchestraAgent, @@ -73,9 +75,7 @@ async def run(self, **kwargs): async def test_agent_lifecycle_and_context(tmp_path, manifest_data): provisioner, runtime, context_id = Provisioner(), Runtime(), uuid4() - environment = SimpleNamespace( - context_id=context_id, environment_name="hello-world" - ) + environment = SimpleNamespace(context_id=context_id, environment_name="hello-world") agent = BuzzOrchestraAgent( logs_dir=tmp_path, manifest=manifest_data, diff --git a/benchmarks/harbor-buzz-orchestra/tests/test_container_runtime.py b/benchmarks/harbor-buzz-orchestra/tests/test_container_runtime.py index 8669fe980c..ebf0eb4b5d 100644 --- a/benchmarks/harbor-buzz-orchestra/tests/test_container_runtime.py +++ b/benchmarks/harbor-buzz-orchestra/tests/test_container_runtime.py @@ -8,8 +8,6 @@ import pytest from harbor.environments.base import ExecResult -from harbor_buzz_orchestra.manifest import ExperimentManifest -from harbor_buzz_orchestra.provisioning import AgentCredential, TrialHandle from harbor_buzz_orchestra.container_runtime import ( REMOTE_BIN, REMOTE_LOGS, @@ -17,6 +15,8 @@ EndpointLaunchConfig, RuntimeLaunchError, ) +from harbor_buzz_orchestra.manifest import ExperimentManifest +from harbor_buzz_orchestra.provisioning import AgentCredential, TrialHandle def write_manifest(tmp_path: Path) -> ExperimentManifest: @@ -33,10 +33,20 @@ def write_manifest(tmp_path: Path) -> ExperimentManifest: { "condition": "test", "roster": [ - {"id": "orch", "kind": "orchestrator", "role": "lead", - "endpoint": "orch-model", **roster_entry}, - {"id": "worker", "kind": "worker", "role": "implementer", - "endpoint": "worker-model", **roster_entry}, + { + "id": "orch", + "kind": "orchestrator", + "role": "lead", + "endpoint": "orch-model", + **roster_entry, + }, + { + "id": "worker", + "kind": "worker", + "role": "implementer", + "endpoint": "worker-model", + **roster_entry, + }, ], "prices": { name: { @@ -162,10 +172,7 @@ def test_user_relay_url_prefers_host_view(tmp_path): == "http://localhost:3600" ) # pre-v1.2 handles fall back to deriving http from the agents' ws view. - assert ( - rt._user_relay_url(trial_handle(())) - == "http://host.docker.internal:3600" - ) + assert rt._user_relay_url(trial_handle(())) == "http://host.docker.internal:3600" with pytest.raises(RuntimeLaunchError, match="ws://"): rt._cli_relay_url("http://relay") @@ -209,16 +216,21 @@ async def test_forwarder_bridges_the_canonical_relay_address(tmp_path): forwarder_binary=str(forwarder), ) trial = TrialHandle( - run_id="run", trial_id="trial", manifest_hash="hash", - relay_ws_url="ws://localhost:3600", channel_id="channel", - credentials=(), user=user_credential(), + run_id="run", + trial_id="trial", + manifest_hash="hash", + relay_ws_url="ws://localhost:3600", + channel_id="channel", + credentials=(), + user=user_credential(), ) environment = Environment( responses={ FORWARDER: ExecResult(stdout="99\n", stderr="", return_code=0), "cat ": ExecResult( stdout="forwarding 127.0.0.1:3600 -> host.docker.internal:3600", - stderr="", return_code=0, + stderr="", + return_code=0, ), } ) @@ -295,9 +307,7 @@ class ReadyEnvironment(Environment): async def exec(self, command, env=None, **kwargs): if command.startswith("cat "): agent_id = re.search(r"([\w-]+)\.stdout\.log", command).group(1) - return ExecResult( - stdout=logs[agent_id], stderr="", return_code=0 - ) + return ExecResult(stdout=logs[agent_id], stderr="", return_code=0) return ExecResult(stdout="", stderr="", return_code=0) from harbor_buzz_orchestra.container_runtime import _Agent @@ -326,9 +336,7 @@ async def exec(self, command, env=None, **kwargs): async def test_dead_agent_processes_fail_the_trial(tmp_path): from harbor_buzz_orchestra.container_runtime import _Agent - agents = [ - _Agent(credential("worker-1", "worker", "worker-model"), 7, "o", "e") - ] + agents = [_Agent(credential("worker-1", "worker", "worker-model"), 7, "o", "e")] environment = Environment( responses={ "kill -0": ExecResult(stdout="DEAD:worker-1\n", stderr="", return_code=0) diff --git a/benchmarks/harbor-buzz-orchestra/tests/test_manifest.py b/benchmarks/harbor-buzz-orchestra/tests/test_manifest.py index 36533db3bf..f8230036b3 100644 --- a/benchmarks/harbor-buzz-orchestra/tests/test_manifest.py +++ b/benchmarks/harbor-buzz-orchestra/tests/test_manifest.py @@ -1,6 +1,8 @@ import copy + import pytest import yaml + from harbor_buzz_orchestra import ExperimentManifest, ManifestError diff --git a/benchmarks/harbor-buzz-orchestra/tests/test_run_leaderboard.py b/benchmarks/harbor-buzz-orchestra/tests/test_run_leaderboard.py index ed048ee5d9..451de72e79 100644 --- a/benchmarks/harbor-buzz-orchestra/tests/test_run_leaderboard.py +++ b/benchmarks/harbor-buzz-orchestra/tests/test_run_leaderboard.py @@ -109,8 +109,16 @@ def test_forbidden_flags_are_not_accepted(tmp_path): for flag in FORBIDDEN_FLAGS: with pytest.raises(SystemExit): run_leaderboard.parse_args( - ["--dataset", "d", "--attempts", "5", - "--agent-bin-dir", str(tmp_path), flag, "1"] + [ + "--dataset", + "d", + "--attempts", + "5", + "--agent-bin-dir", + str(tmp_path), + flag, + "1", + ] ) diff --git a/deploy/compose/.env.example b/deploy/compose/.env.example index cebe879da0..838824c17f 100644 --- a/deploy/compose/.env.example +++ b/deploy/compose/.env.example @@ -30,7 +30,6 @@ POSTGRES_DB=buzz POSTGRES_USER=buzz POSTGRES_PASSWORD=CHANGE_ME_RANDOM_PASSWORD REDIS_PASSWORD=CHANGE_ME_RANDOM_PASSWORD -TYPESENSE_API_KEY=CHANGE_ME_RANDOM_API_KEY BUZZ_S3_ACCESS_KEY=CHANGE_ME_RANDOM_ACCESS_KEY BUZZ_S3_SECRET_KEY=CHANGE_ME_RANDOM_SECRET_KEY BUZZ_S3_BUCKET=buzz-media @@ -45,7 +44,6 @@ CADDY_HTTPS_PORT=443 # Dev override ports. Only used with compose.dev.yml. POSTGRES_PORT=5432 REDIS_PORT=6379 -TYPESENSE_PORT=8108 MINIO_API_PORT=9000 MINIO_CONSOLE_PORT=9001 ADMINER_PORT=8082