Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions docs/opencode-grpo.md
Original file line number Diff line number Diff line change
Expand Up @@ -118,9 +118,9 @@ containing the exact prompt IDs, completion IDs, and sampled logprobs returned
by the TRL server. Ordinary evaluation requests do not create sidecars.

The executable pipeline pins BenchFlow
`cbc295464e62aa39f84e0daa675aa939c0e72f00`, which includes sampled-token
logprob capture, the native TRL SFT converter, and Qwen3.5 generation-prefix
validation.
`2a97db55947d6742b765ad34ddd91d74c20d625f`, which includes sampled-token
logprob capture, the native TRL SFT converter, Qwen3.5 generation-prefix
validation, and the reproducible OpenCode `1.17.20` harness pin.

The rollout parser reconstructs one causal sequence across all model turns
using the exact served prompt IDs from the bridge sidecar rather than
Expand Down
3 changes: 2 additions & 1 deletion docs/qwen35-data-agent-e2e-canary.md
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,8 @@ evidence on the 14-task canary, not a competition-scale statistical claim.
PostTrainArena commit
`cf824b214e5ae08d6fc21becbcba7aae55e5109e` produced the run. The runtime used
BenchFlow commit `6d6d2ee0965bdc7fe1e38555d1f7c4c21ee8a840`, whose OpenCode
`1.17.20` pin is tracked in BenchFlow PR #931.
`1.17.20` pin was merged from BenchFlow PR #931 as
`2a97db55947d6742b765ad34ddd91d74c20d625f`.

## Historical soccer canary

Expand Down
4 changes: 2 additions & 2 deletions pipelines/benchflow-task-posttrain/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ Documentation = "https://github.com/benchflow-ai/posttrainarena/blob/main/docs/t

[project.optional-dependencies]
train = [
"benchflow[trl,sandbox-daytona] @ git+https://github.com/benchflow-ai/benchflow.git@cbc295464e62aa39f84e0daa675aa939c0e72f00",
"benchflow[trl,sandbox-daytona] @ git+https://github.com/benchflow-ai/benchflow.git@2a97db55947d6742b765ad34ddd91d74c20d625f",
"openenv @ git+https://github.com/huggingface/OpenEnv.git@6823135a714814e3efb3e39c4a9edff01e1a2a98",
"openai>=2.0",
"peft>=0.19,<0.20",
Expand All @@ -34,7 +34,7 @@ hf = [
"tomli-w>=1.2,<2",
]
test = [
"benchflow[trl] @ git+https://github.com/benchflow-ai/benchflow.git@cbc295464e62aa39f84e0daa675aa939c0e72f00",
"benchflow[trl] @ git+https://github.com/benchflow-ai/benchflow.git@2a97db55947d6742b765ad34ddd91d74c20d625f",
"huggingface_hub>=0.36,<2",
"openenv @ git+https://github.com/huggingface/OpenEnv.git@6823135a714814e3efb3e39c4a9edff01e1a2a98",
"peft>=0.19,<0.20",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
from typing import Any, Literal


BENCHFLOW_COMMIT = "cbc295464e62aa39f84e0daa675aa939c0e72f00"
BENCHFLOW_COMMIT = "2a97db55947d6742b765ad34ddd91d74c20d625f"
GrpoRunPolicy = Literal["on_reward", "always"]
HarnessSkillMode = Literal["no-skill", "with-skill"]
UsageTrackingPolicy = Literal["required"]
Expand Down
20 changes: 19 additions & 1 deletion pipelines/benchflow-task-posttrain/tests/test_config.py
Original file line number Diff line number Diff line change
@@ -1,17 +1,35 @@
from __future__ import annotations

import hashlib
import tomllib
from dataclasses import replace
from pathlib import Path

import pytest
from benchflow.agents.registry import AGENTS

from posttrainarena.benchflow_pipeline.config import load_config
from posttrainarena.benchflow_pipeline.config import BENCHFLOW_COMMIT, load_config


ROOT = Path(__file__).resolve().parents[1]


def test_benchflow_dependency_pins_match_runtime_commit() -> None:
project = tomllib.loads((ROOT / "pyproject.toml").read_text())["project"]
optional = project["optional-dependencies"]
benchflow_dependencies = [
dependency
for extra in ("train", "test")
for dependency in optional[extra]
if dependency.startswith("benchflow[")
]

assert len(benchflow_dependencies) == 2
assert all(f"@{BENCHFLOW_COMMIT}" in item for item in benchflow_dependencies)
assert "opencode-ai@1.17.20" in AGENTS["opencode"].install_cmd
assert "opencode-ai@latest" not in AGENTS["opencode"].install_cmd


def test_example_config_is_valid_and_pinned() -> None:
config = load_config(ROOT / "configs/qwen3-4b-data-agent-smoke.toml")

Expand Down