From ceecf43cd92241aee01fe44e923a5ed4366d746e Mon Sep 17 00:00:00 2001 From: Xiangyi Li Date: Sun, 9 Aug 2026 13:28:58 -0700 Subject: [PATCH] release: 0.6.7 Version 0.6.7 (from 0.6.7.dev0), CITATION date 2026-08-09, and the 0.6.7 CHANGELOG section covering #949-#967. Also in this cut: - Point the wheel-shipped env0 pins at the org-owned ghcr.io/benchflow-ai/env0:0.2.0 base image (they named a personal Docker Hub image while their own comments and the docs declared ghcr authoritative) and drop the PR-archaeology comments from files that ship to PyPI. Validated on Daytona: 8/8 services ready, reward 1.0. - Document mean_reward in the summary.json layout, the per-task failure block and live-token footer in the CLI reference, and soften the external-agents cost figure (the underlying rate table is a placeholder). --- CHANGELOG.md | 83 ++++++++++++++++++- CITATION.cff | 4 +- docs/external-agents.md | 4 +- docs/getting-started.md | 2 +- docs/reference/cli.md | 15 ++++ pyproject.toml | 2 +- .../environment/_registry/env0@outage.toml | 15 ++-- .../environment/_registry/env0@prod.toml | 24 +++--- uv.lock | 2 +- 9 files changed, 121 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c657f638..2bf58785 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## [Unreleased] +## 0.6.7 — 2026-08-09 + ### Added - **Built-in environment registry.** The committed env-axis pins (`env0@prod`, `env0@outage`) moved from `benchmarks/_environments/` into @@ -10,13 +12,71 @@ `pip install benchflow` with no checkout and no env vars. `$BENCHFLOW_ENV_REGISTRY`, when set, still wins entirely; resolution stays content-addressed (sha256 logged), and unknown names now error - listing the available specs. + listing the available specs. (#961) - **Console progress heartbeat.** Single-concurrency eval runs print a throttled progress line (`… 6.2min, 12 tool calls (last: …)`) about every 45 seconds while the agent works, so a long prompt is distinguishable from a hang. The heartbeat is auto-gated off for multi-concurrency jobs; `bench eval run --quiet` suppresses it, and `BENCHFLOW_PROGRESS=on`/`off` overrides the auto-gate. (#951) +- **Live per-task activity in the eval dashboard.** Under a TTY the + running-now table gains an activity column ("38 calls · last: + file_editor", plus tokens once a usage snapshot exists), polled from the + ACP session's existing heartbeat counters. The agents-manifest autoload + also clones quietly — one "Cloning …" summary line instead of raw git + progress. (#956) +- **Phase labels and per-task failure reasons.** The activity cell is never + blank while a row exists: sandbox create, agent install, and verify show + dim phase labels ("creating sandbox…", "installing agent…", + "verifying…"). The final block prints one dim "✗ task: reason" line per + failed task — verifier error first, else a compact reward/metric + breakdown, else the reward — capped at 5 lines. `--quiet` now silences + the dashboard as well as the heartbeat. (#957) +- **Failure reasons mined from verifier artifacts.** When a displayed + failure would read as a bare "reward X", the CLI reads a small bounded + artifact from the rollout's own verifier dir — the CTRF report (first + failed test plus its assertion line) or a tail of `test-stdout.txt` — and + a dim "(details: …/verifier)" pointer names the artifact directory. + Artifacts resolve by the recorded rollout name, never by glob. (#959) +- **Fractional rewards in console summaries.** Per-task lines carry the + scored reward (`✗ task (reward=0.30, tools=47)`), the Score line renders + ", mean reward 0.30" alongside the binarized counts, and + `EvaluationResult` / `summary.json` gain a `mean_reward` field (mean over + scored rollouts; errors excluded, not zeroed). (#960) +- **Full failure counts on per-task console lines.** A CTRF report with + more than one failed test rolls up as " (+N more failure(s); P/T checks + passed)" after the first failure — the count suffix is never truncated + away — and parametrized test names keep their `[param]` ids whenever the + report carries them. (#962) +- **Live token usage in the dashboard footer.** The footer sums completed + tasks' trusted telemetry plus every running rollout's live ACP session + usage, so spend is visible while the run executes (usage lands per + completed prompt; cost stays scoring-time from the gateway log). + Single-tool agents' activity cell drops the redundant "last:" suffix in + favor of "38 calls · 412.0k tok" once tokens are available. (#963) +- **env0-shaped failure breakdowns.** The failure-reason tiers understand + metrics nested one level under `metrics` / `details`, pair + `_found` with `_total` into fractions ("deadlines 1/5", + lowest-signal first), probe `verifier/reward.json` on disk between the + CTRF and stdout tiers, and every failure block with on-disk verifier + artifacts gets one "(details: …)" pointer. (#964) +- **Mid-prompt live token usage via the gateway's live capture.** The #963 + live tokens stepped forward only when an ACP prompt completed, so a + single-prompt rollout showed "— tokens" for its whole agent phase. The + proxy runtime's existing live-capture loop (which already tails the + sandbox gateway's callback log every second) now also accumulates + provider tokens into an O(1) counter, and the dashboard reconciles the + two non-decreasing live signals as max(ACP, gateway) — display-only, + still replaced by the trusted scoring import at completion, and any + gateway-side failure degrades to the ACP-only behavior. (#965) + +- **Compact flow-style arrays in emitted task.md frontmatter.** Short + scalar-only lists render on one line (`tags: [parsing, nlp]`) instead of + multiline bullets, so hand-written flow arrays survive `bench tasks + migrate` / normalize round-trips. The style predicate measures each list + with the real YAML emitter (so it can never disagree with PyYAML's own + quoting) and falls back to block style for long, nested, or multiline + items. Parsing is unchanged — both styles were always accepted. (#967) ### Changed - Migrated the clawsbench `archive-amazon-shipping` task to the native @@ -51,6 +111,27 @@ environment now fails loud when it declares services but none are startable in the image, instead of passing a vacuous readiness gate and dying at the verifier. (#954) +- Pointed the built-in `env0@prod` / `env0@outage` pins at the org-owned + `ghcr.io/benchflow-ai/env0:0.2.0` base image. The wheel-shipped pins named a + personal Docker Hub image while their own comments and the environment docs + declared the ghcr image authoritative; `base_image` is recorded as rollout + provenance, so every env0 result carried the wrong base. Verified on Daytona + (env0 `auth-least-privilege-summary`, 8/8 services ready, reward 1.0). +- Restored the `env0@outage` perturbation (gmail and slack removed relative + to `env0@prod`) that the #954 upstream sync had erased by mirroring the + full service list into both pins, and corrected the pin header's stale + slack port. (#955) +- **ACP protocol JSON glued to PTY shell noise now decodes.** On PTY + transports, an agent's initialize response could arrive on the same line + as the shell prompt (ANSI/OSC-prefixed), fail the strict whole-line JSON + parse, and get filed as noise — the handshake then "timed out" at any + window with the answer sitting in the agent log. The PTY-facing transport + now retries from the first `{` and once more after an ANSI scrub, but + only until the first successfully decoded protocol message per + connection; afterwards the strict contract rules, so log-echoed envelopes + cannot impersonate protocol traffic mid-session. The pre-prompt handshake + window is also env-configurable via `BENCHFLOW_ACP_HANDSHAKE_TIMEOUT` + (seconds; default 60). (#958) ## 0.6.6 — 2026-08-04 diff --git a/CITATION.cff b/CITATION.cff index 53aa595e..24bf7eeb 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -9,8 +9,8 @@ authors: repository-code: "https://github.com/benchflow-ai/benchflow" url: "https://github.com/benchflow-ai/benchflow" license: Apache-2.0 -version: 0.6.6 -date-released: 2026-08-04 +version: 0.6.7 +date-released: 2026-08-09 keywords: - benchmark - llm-agents diff --git a/docs/external-agents.md b/docs/external-agents.md index 88b7c1be..156ca4fe 100644 --- a/docs/external-agents.md +++ b/docs/external-agents.md @@ -11,8 +11,8 @@ first one is all there is to know. Using an agent name BenchFlow doesn't recognize triggers a one-shot fetch of the declarative manifests from `benchflow-ai/agents@main`. Nothing to install or configure. End to end, from an empty directory to a scored rollout -(verified as written: skillsbench `edit-pdf`, reward 1.0, ~23 min, ~$0.02 on -`deepseek-v4-flash`): +(verified as written: skillsbench `edit-pdf`, reward 1.0, ~23 min, a few +cents on `deepseek-v4-flash` — rates approximate): ```bash # Install. `uv tool install --python 3.12 'benchflow[sandbox-daytona]'` (the diff --git a/docs/getting-started.md b/docs/getting-started.md index d81b7269..62450002 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -193,7 +193,7 @@ Each run writes under `--jobs-dir` (default `jobs/`): / summary.json # copy of the latest job summary (overwritten by the next run) / # job directory, named by start time - summary.json # job-level aggregate + summary.json # job-level aggregate (pass counts plus mean_reward — mean over scored rollouts) __/ # one rollout: task name + 8-char id result.json # rollout summary: rewards, errors, token usage/cost results.jsonl # Verifiers/Prime-RL shaped rollout row diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 85abfdca..03246adf 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -301,6 +301,21 @@ alike (so it also wins over an exported `on`). Note that on a TTY, mutes INFO logging while it owns the screen; pair it with `BENCHFLOW_NO_PROGRESS=1` to get plain heartbeat lines on a TTY. +The dashboard footer also carries a live token total: completed tasks' +trusted telemetry plus every running rollout's live usage (ACP session +counters reconciled with the sandbox gateway's live capture), so spend is +visible mid-run. Cost stays completed-tasks-only — `$` comes from the +gateway log imported at scoring time. + +After the run, each failed task gets one dim `✗ task: reason` line — +verifier error first, else a compact reward/metric breakdown, else the +scored reward, upgraded from small on-disk verifier artifacts (the CTRF +report, `reward.json`, or a `test-stdout.txt` tail) when the in-memory +reason is a bare reward. Multi-failure CTRF reports roll up as +`(+N more failure(s); P/T checks passed)`, and a dim +`(details: …/verifier)` pointer names the artifact directory whenever one +exists on disk. + The final `Score: P/T (…%)` line is pass-threshold aggregation — a task counts as passed only at reward 1.0 — while `mean reward` beside it is the average raw verifier reward, so `0/1 (0.0%)` next to `mean reward 0.80` means partial diff --git a/pyproject.toml b/pyproject.toml index acf26f3e..2de1d9de 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "benchflow" -version = "0.6.7.dev0" +version = "0.6.7" description = "Multi-turn agent benchmarking with ACP — run any agent, any model, any provider." readme = "README.md" requires-python = ">=3.12" diff --git a/src/benchflow/environment/_registry/env0@outage.toml b/src/benchflow/environment/_registry/env0@outage.toml index 47598b05..da3a9362 100644 --- a/src/benchflow/environment/_registry/env0@outage.toml +++ b/src/benchflow/environment/_registry/env0@outage.toml @@ -9,10 +9,13 @@ # resolves on a bare pip install with no $BENCHFLOW_ENV_REGISTRY set: # # bench eval create --tasks-dir --environment-manifest env0@outage ... +# +# NOTE: ghcr.io/benchflow-ai/env0:0.2.0 is amd64-only — run env0 on Daytona +# (x86_64), not local Docker on Apple Silicon. [environment] name = "env-0" -base_image = "xdotli/env0-base:latest" +base_image = "ghcr.io/benchflow-ai/env0:0.2.0" owns_lifecycle = false isolation = "per_task" @@ -22,13 +25,9 @@ mechanism = "image" [environment.readiness] timeout_sec = 60 -# Services below are copied VERBATIM from the upstream env-0.toml pin -# (2026-08-09 sync). The previous pin had drifted: bare CLI names -# (auth/gmail/...) that do not exist in ghcr.io/benchflow-ai/env0:0.2.0 -# images (which ship mock-auth/mock-gmail/... console scripts), plus a -# phantom slack service and shifted ports — so ManifestEnvironment -# probed, found ZERO startable services, and readiness passed vacuously -# while every rollout died at the verifier with connection-refused. +# Services below are copied VERBATIM from the upstream env-0.toml pin, +# minus this variant's deliberate outage (mock-gmail and mock-slack +# removed). [[environment.services]] name = "mock-auth" diff --git a/src/benchflow/environment/_registry/env0@prod.toml b/src/benchflow/environment/_registry/env0@prod.toml index 624766ac..553f5932 100644 --- a/src/benchflow/environment/_registry/env0@prod.toml +++ b/src/benchflow/environment/_registry/env0@prod.toml @@ -1,7 +1,7 @@ # env0@prod — pinned, content-addressed Environment-plane manifest for env-0. # # This is a built-in registry entry, shipped inside the benchflow wheel -# (benchflow/environment/_registry/) so PR #790's env-axis pin resolves on a +# (benchflow/environment/_registry/) so the env-axis pin resolves on a # bare pip install — no checkout, no env vars: # # bench eval create --tasks-dir --environment-manifest env0@prod ... @@ -9,16 +9,18 @@ # Set $BENCHFLOW_ENV_REGISTRY to a local directory of name@version.toml files # to override the built-in registry entirely. # -# Mirrors benchflow-ai/env0 tasks/_manifests/env-0.toml (the source of truth). env-0's per-task images build -# FROM the shared base and bake seed data; benchflow's ManifestEnvironment -# probes each service's CLI and starts only the ones installed in the per-task -# image, gating the agent on each service's /health. +# Mirrors benchflow-ai/env0 tasks/_manifests/env-0.toml (the source of truth). +# env-0's per-task images build FROM the shared base and bake seed data; +# benchflow's ManifestEnvironment probes each service's CLI and starts only +# the ones installed in the per-task image, gating the agent on each +# service's /health. # -# NOTE: xdotli/env0-base:latest is amd64-only — run env0 on Daytona (x86_64). +# NOTE: ghcr.io/benchflow-ai/env0:0.2.0 is amd64-only — run env0 on Daytona +# (x86_64), not local Docker on Apple Silicon. [environment] name = "env-0" -base_image = "xdotli/env0-base:latest" +base_image = "ghcr.io/benchflow-ai/env0:0.2.0" owns_lifecycle = false isolation = "per_task" @@ -28,13 +30,7 @@ mechanism = "image" [environment.readiness] timeout_sec = 60 -# Services below are copied VERBATIM from the upstream env-0.toml pin -# (2026-08-09 sync). The previous pin had drifted: bare CLI names -# (auth/gmail/...) that do not exist in ghcr.io/benchflow-ai/env0:0.2.0 -# images (which ship mock-auth/mock-gmail/... console scripts), plus a -# phantom slack service and shifted ports — so ManifestEnvironment -# probed, found ZERO startable services, and readiness passed vacuously -# while every rollout died at the verifier with connection-refused. +# Services below are copied VERBATIM from the upstream env-0.toml pin. [[environment.services]] name = "mock-auth" diff --git a/uv.lock b/uv.lock index 89804801..037089e3 100644 --- a/uv.lock +++ b/uv.lock @@ -329,7 +329,7 @@ wheels = [ [[package]] name = "benchflow" -version = "0.6.7.dev0" +version = "0.6.7" source = { editable = "." } dependencies = [ { name = "agent-client-protocol" },