From eb9e7bda420983c2137c9f523d8d73350b15f1a2 Mon Sep 17 00:00:00 2001 From: Bingran You Date: Tue, 14 Jul 2026 23:08:43 -0700 Subject: [PATCH] Pin merged BenchFlow OpenCode harness --- docs/opencode-grpo.md | 6 +++--- docs/qwen35-data-agent-e2e-canary.md | 3 ++- .../benchflow-task-posttrain/pyproject.toml | 4 ++-- .../benchflow_pipeline/config.py | 2 +- .../tests/test_config.py | 20 ++++++++++++++++++- 5 files changed, 27 insertions(+), 8 deletions(-) diff --git a/docs/opencode-grpo.md b/docs/opencode-grpo.md index 3a3b453..7c7f1fd 100644 --- a/docs/opencode-grpo.md +++ b/docs/opencode-grpo.md @@ -118,9 +118,9 @@ containing the exact prompt IDs, completion IDs, and sampled logprobs returned by the TRL server. Ordinary evaluation requests do not create sidecars. The executable pipeline pins BenchFlow -`cbc295464e62aa39f84e0daa675aa939c0e72f00`, which includes sampled-token -logprob capture, the native TRL SFT converter, and Qwen3.5 generation-prefix -validation. +`2a97db55947d6742b765ad34ddd91d74c20d625f`, which includes sampled-token +logprob capture, the native TRL SFT converter, Qwen3.5 generation-prefix +validation, and the reproducible OpenCode `1.17.20` harness pin. The rollout parser reconstructs one causal sequence across all model turns using the exact served prompt IDs from the bridge sidecar rather than diff --git a/docs/qwen35-data-agent-e2e-canary.md b/docs/qwen35-data-agent-e2e-canary.md index a33b68b..9eae5a2 100644 --- a/docs/qwen35-data-agent-e2e-canary.md +++ b/docs/qwen35-data-agent-e2e-canary.md @@ -37,7 +37,8 @@ evidence on the 14-task canary, not a competition-scale statistical claim. PostTrainArena commit `cf824b214e5ae08d6fc21becbcba7aae55e5109e` produced the run. The runtime used BenchFlow commit `6d6d2ee0965bdc7fe1e38555d1f7c4c21ee8a840`, whose OpenCode -`1.17.20` pin is tracked in BenchFlow PR #931. +`1.17.20` pin was merged from BenchFlow PR #931 as +`2a97db55947d6742b765ad34ddd91d74c20d625f`. ## Historical soccer canary diff --git a/pipelines/benchflow-task-posttrain/pyproject.toml b/pipelines/benchflow-task-posttrain/pyproject.toml index 8c652a2..26de9ed 100644 --- a/pipelines/benchflow-task-posttrain/pyproject.toml +++ b/pipelines/benchflow-task-posttrain/pyproject.toml @@ -19,7 +19,7 @@ Documentation = "https://github.com/benchflow-ai/posttrainarena/blob/main/docs/t [project.optional-dependencies] train = [ - "benchflow[trl,sandbox-daytona] @ git+https://github.com/benchflow-ai/benchflow.git@cbc295464e62aa39f84e0daa675aa939c0e72f00", + "benchflow[trl,sandbox-daytona] @ git+https://github.com/benchflow-ai/benchflow.git@2a97db55947d6742b765ad34ddd91d74c20d625f", "openenv @ git+https://github.com/huggingface/OpenEnv.git@6823135a714814e3efb3e39c4a9edff01e1a2a98", "openai>=2.0", "peft>=0.19,<0.20", @@ -34,7 +34,7 @@ hf = [ "tomli-w>=1.2,<2", ] test = [ - "benchflow[trl] @ git+https://github.com/benchflow-ai/benchflow.git@cbc295464e62aa39f84e0daa675aa939c0e72f00", + "benchflow[trl] @ git+https://github.com/benchflow-ai/benchflow.git@2a97db55947d6742b765ad34ddd91d74c20d625f", "huggingface_hub>=0.36,<2", "openenv @ git+https://github.com/huggingface/OpenEnv.git@6823135a714814e3efb3e39c4a9edff01e1a2a98", "peft>=0.19,<0.20", diff --git a/pipelines/benchflow-task-posttrain/src/posttrainarena/benchflow_pipeline/config.py b/pipelines/benchflow-task-posttrain/src/posttrainarena/benchflow_pipeline/config.py index e3b4290..055cf6e 100644 --- a/pipelines/benchflow-task-posttrain/src/posttrainarena/benchflow_pipeline/config.py +++ b/pipelines/benchflow-task-posttrain/src/posttrainarena/benchflow_pipeline/config.py @@ -9,7 +9,7 @@ from typing import Any, Literal -BENCHFLOW_COMMIT = "cbc295464e62aa39f84e0daa675aa939c0e72f00" +BENCHFLOW_COMMIT = "2a97db55947d6742b765ad34ddd91d74c20d625f" GrpoRunPolicy = Literal["on_reward", "always"] HarnessSkillMode = Literal["no-skill", "with-skill"] UsageTrackingPolicy = Literal["required"] diff --git a/pipelines/benchflow-task-posttrain/tests/test_config.py b/pipelines/benchflow-task-posttrain/tests/test_config.py index b73fbda..d7d45a7 100644 --- a/pipelines/benchflow-task-posttrain/tests/test_config.py +++ b/pipelines/benchflow-task-posttrain/tests/test_config.py @@ -1,17 +1,35 @@ from __future__ import annotations import hashlib +import tomllib from dataclasses import replace from pathlib import Path import pytest +from benchflow.agents.registry import AGENTS -from posttrainarena.benchflow_pipeline.config import load_config +from posttrainarena.benchflow_pipeline.config import BENCHFLOW_COMMIT, load_config ROOT = Path(__file__).resolve().parents[1] +def test_benchflow_dependency_pins_match_runtime_commit() -> None: + project = tomllib.loads((ROOT / "pyproject.toml").read_text())["project"] + optional = project["optional-dependencies"] + benchflow_dependencies = [ + dependency + for extra in ("train", "test") + for dependency in optional[extra] + if dependency.startswith("benchflow[") + ] + + assert len(benchflow_dependencies) == 2 + assert all(f"@{BENCHFLOW_COMMIT}" in item for item in benchflow_dependencies) + assert "opencode-ai@1.17.20" in AGENTS["opencode"].install_cmd + assert "opencode-ai@latest" not in AGENTS["opencode"].install_cmd + + def test_example_config_is_valid_and_pinned() -> None: config = load_config(ROOT / "configs/qwen3-4b-data-agent-smoke.toml")