From 427c7a70eb95deea7331e6fdb5d65ee4a284220f Mon Sep 17 00:00:00 2001 From: David Rhodus Date: Thu, 9 Jul 2026 19:48:10 -0700 Subject: [PATCH] hi-eval: per-turn context-growth harness + a long interp task MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Pi-efficiency gap lives in long multi-turn sessions where context accumulates, but the eval was blind to that regime: the multi-turn drive (HI_EVAL_GOAL=1 HI_EVAL_TURNS=N) overwrote the report each turn, so only the last turn's tokens survived — no growth curve. Instrument it. A new `TurnMetric` captures, after every session-continuing turn, the context (input) tokens sent, total tokens, tool calls, and goal progress (done/total, read from the report's goal summary block). It's threaded Candidate -> RunResult -> RunArtifact (into the trial JSON and runs.jsonl) and printed as a per-turn growth curve. `input_tokens` per turn is now a first-class signal — the "how fast does context accumulate vs. how far does the goal get" axis the curation/compaction levers move. Also add bench/long/interp: a ~10-requirement expression interpreter (tokenizer + recursive-descent parser — precedence, right-assoc **, unary, variables, comparisons, min/max/abs, typed errors) with a 30-assertion offline verify and a tested reference. Joins the existing multi-file CLI builds (kanban-cli, library-cli) in bench/long. Verified live (pipe/glm-5.2-fast, goal-driven multi-turn): ctx 9450 (t1, 0/8) -> 28407 (t2, 8/8), PASS — context accumulation and goal progress both captured and displayed. Tests: new read_turn_metric unit test (real report shape + fail-open on a missing report); fmt/clippy clean; 14 hi-eval tests pass. Co-Authored-By: Claude Opus 4.8 (1M context) --- bench/long/interp/fixed/interp.py | 160 ++++++++++++++++++++++++++++ bench/long/interp/fixture/interp.py | 2 + bench/long/interp/task.toml | 47 ++++++++ crates/hi-eval/src/artifacts.rs | 1 + crates/hi-eval/src/reporting.rs | 24 +++++ crates/hi-eval/src/results.rs | 27 +++++ crates/hi-eval/src/runner.rs | 72 ++++++++++++- 7 files changed, 331 insertions(+), 2 deletions(-) create mode 100644 bench/long/interp/fixed/interp.py create mode 100644 bench/long/interp/fixture/interp.py create mode 100644 bench/long/interp/task.toml diff --git a/bench/long/interp/fixed/interp.py b/bench/long/interp/fixed/interp.py new file mode 100644 index 00000000..f4074cda --- /dev/null +++ b/bench/long/interp/fixed/interp.py @@ -0,0 +1,160 @@ +"""A small infix expression interpreter: tokenizer + recursive-descent parser.""" + + +def _tokenize(s): + toks = [] + i, n = 0, len(s) + while i < n: + c = s[i] + if c.isspace(): + i += 1 + continue + if c.isdigit() or (c == "." and i + 1 < n and s[i + 1].isdigit()): + j = i + while j < n and (s[j].isdigit() or s[j] == "."): + j += 1 + text = s[i:j] + toks.append(("num", float(text) if "." in text else int(text))) + i = j + continue + if c.isalpha() or c == "_": + j = i + while j < n and (s[j].isalnum() or s[j] == "_"): + j += 1 + toks.append(("name", s[i:j])) + i = j + continue + two = s[i : i + 2] + if two in ("**", "<=", ">=", "==", "!="): + toks.append(("op", two)) + i += 2 + continue + if c in "+-*/%<>(),": + toks.append(("op", c)) + i += 1 + continue + raise ValueError("bad character: " + repr(c)) + toks.append(("end", None)) + return toks + + +_FUNCS = {"min": min, "max": max, "abs": lambda x: abs(x)} + + +class _Parser: + def __init__(self, toks, env): + self.toks = toks + self.pos = 0 + self.env = env + + def _peek(self): + return self.toks[self.pos] + + def _advance(self): + t = self.toks[self.pos] + self.pos += 1 + return t + + def _match(self, *ops): + t = self._peek() + if t[0] == "op" and t[1] in ops: + self.pos += 1 + return t[1] + return None + + def _expect(self, op): + if self._match(op) is None: + raise ValueError("expected " + op) + + def parse(self): + v = self.comparison() + if self._peek()[0] != "end": + raise ValueError("trailing tokens") + return v + + def comparison(self): + left = self.additive() + op = self._match("<", ">", "<=", ">=", "==", "!=") + if op is None: + return left + right = self.additive() + if op == "<": + return left < right + if op == ">": + return left > right + if op == "<=": + return left <= right + if op == ">=": + return left >= right + if op == "==": + return left == right + return left != right + + def additive(self): + left = self.multiplicative() + while True: + op = self._match("+", "-") + if op is None: + return left + right = self.multiplicative() + left = left + right if op == "+" else left - right + + def multiplicative(self): + left = self.power() + while True: + op = self._match("*", "/", "%") + if op is None: + return left + right = self.power() + if op == "*": + left = left * right + elif op == "/": + left = left / right + else: + left = left % right + + def power(self): + left = self.unary() + if self._match("**") is not None: + right = self.power() # right-associative + return left**right + return left + + def unary(self): + op = self._match("+", "-") + if op == "-": + return -self.unary() + if op == "+": + return self.unary() + return self.primary() + + def primary(self): + t = self._peek() + if t[0] == "num": + self._advance() + return t[1] + if self._match("(") is not None: + v = self.comparison() + self._expect(")") + return v + if t[0] == "name": + self._advance() + name = t[1] + if self._match("(") is not None: + args = [] + if self._peek() != ("op", ")"): + args.append(self.comparison()) + while self._match(",") is not None: + args.append(self.comparison()) + self._expect(")") + if name not in _FUNCS: + raise NameError("unknown function: " + name) + return _FUNCS[name](*args) + if name in self.env: + return self.env[name] + raise NameError("undefined variable: " + name) + raise ValueError("unexpected token: " + repr(t)) + + +def evaluate(expr, env=None): + return _Parser(_tokenize(expr), env or {}).parse() diff --git a/bench/long/interp/fixture/interp.py b/bench/long/interp/fixture/interp.py new file mode 100644 index 00000000..2577ce2c --- /dev/null +++ b/bench/long/interp/fixture/interp.py @@ -0,0 +1,2 @@ +def evaluate(expr, env=None): + raise NotImplementedError diff --git a/bench/long/interp/task.toml b/bench/long/interp/task.toml new file mode 100644 index 00000000..f681cd21 --- /dev/null +++ b/bench/long/interp/task.toml @@ -0,0 +1,47 @@ +name = "interp" +prompt = """Build an infix expression interpreter in interp.py. Implement `evaluate(expr, env=None)` that parses and evaluates an arithmetic/logic expression string and returns the result. The file currently raises NotImplementedError. Edit only interp.py. Implement all of these requirements: + +1. Integer and float literals (e.g. `42`, `1.5`). +2. Binary `+ - * /` with standard precedence (`*` and `/` bind tighter than `+` and `-`), left-associative. +3. Parentheses for grouping. +4. Unary minus and unary plus (e.g. `-3`, `-(2+3)`, `- -5`). +5. Modulo `%` (same precedence as `*`/`/`). +6. Exponentiation `**`, right-associative and binding tighter than unary minus's operand — i.e. `2 ** 3 ** 2 == 512`. +7. Variables resolved from the `env` dict argument, e.g. `evaluate("x + 1", {"x": 3}) == 4`. +8. Comparison operators `< > <= >= == !=` (lowest precedence) returning a bool. +9. Built-in functions `min`, `max` (variadic), and `abs`, callable with parenthesised comma-separated args, e.g. `evaluate("max(2, 3) + 1") == 4`. +10. Errors: division or modulo by zero raises ZeroDivisionError; an undefined variable or unknown function name raises NameError; any malformed/unparseable input (bad character, trailing tokens, unbalanced parens, missing operand) raises ValueError. + +Write a real tokenizer and parser — do not use eval() or the ast module. Verify your work as you go.""" +verify = '''python3 -c " +import interp as m +E = m.evaluate +assert E('1 + 2 * 3') == 7 +assert E('(1 + 2) * 3') == 9 +assert E('10 - 2 - 3') == 5 +assert E('7 % 3') == 1 +assert E('2 ** 3 ** 2') == 512 +assert E('6 / 2') == 3 +assert E('-3 + 4') == 1 +assert E('-(2 + 3)') == -5 +assert E('- -5') == 5 +assert E('1.5 + 2') == 3.5 +assert E('x + 1', {'x': 3}) == 4 +assert E('a * b', {'a': 4, 'b': 5}) == 20 +assert E('2 < 3') is True +assert E('3 <= 3') is True +assert E('2 == 2') is True +assert E('2 != 3') is True +assert E('5 > 9') is False +assert E('max(2, 3) + 1') == 4 +assert E('min(4, 2, 7)') == 2 +assert E('abs(-8)') == 8 +assert E('max(abs(-3), 2)') == 3 +import sys +for expr, exc in [('1 / 0', ZeroDivisionError), ('5 % 0', ZeroDivisionError), ('y + 1', NameError), ('1 +', ValueError), ('2 @ 3', ValueError), ('(1 + 2', ValueError)]: + try: + E(expr); sys.exit('no error for: ' + expr) + except exc: + pass +print('ok') +"''' diff --git a/crates/hi-eval/src/artifacts.rs b/crates/hi-eval/src/artifacts.rs index 0cdecefb..b13ed317 100644 --- a/crates/hi-eval/src/artifacts.rs +++ b/crates/hi-eval/src/artifacts.rs @@ -39,6 +39,7 @@ pub fn write_artifact( mcp_model: result.mcp_model.clone(), verify_output_summary: result.verify_output_summary.clone(), trajectory: result.trajectory.clone(), + growth: result.growth.clone(), }; let name = format!( "trial-{:03}-{}-{}-{}.json", diff --git a/crates/hi-eval/src/reporting.rs b/crates/hi-eval/src/reporting.rs index 6d365bf0..438a9a29 100644 --- a/crates/hi-eval/src/reporting.rs +++ b/crates/hi-eval/src/reporting.rs @@ -154,6 +154,30 @@ pub fn print_summary(results: &[RunResult], task_count: usize, active: &[&Config " parallel: {avg_max:.1} max concurrent batch · {serial_pct}% of {total_calls} calls serial", ); } + + // Context-growth curve (multi-turn drives only): the representative + // trial's per-turn input-token trajectory — "how fast does context + // accumulate vs. how far does the goal get". The exemplar is the row + // that drove the most turns. Watch for ctx climbing while goal/done + // stalls: that's the bloat the compaction/curation levers should flatten. + if let Some(row) = rows + .iter() + .filter(|r| !r.growth.is_empty()) + .max_by_key(|r| r.growth.len()) + { + println!(" growth (ctx tok · tools · sub-goals done):"); + for m in &row.growth { + let prog = if m.goal_total > 0 { + format!("{}/{}", m.goal_done, m.goal_total) + } else { + "-".to_string() + }; + println!( + " t{:<2} ctx={:>8} tools={:>3} goal={}", + m.turn, m.input_tokens, m.tool_calls, prog + ); + } + } } } } diff --git a/crates/hi-eval/src/results.rs b/crates/hi-eval/src/results.rs index 9732adbc..ded2b261 100644 --- a/crates/hi-eval/src/results.rs +++ b/crates/hi-eval/src/results.rs @@ -95,6 +95,25 @@ impl Trajectory { } } +/// One turn of a multi-turn (long-task) drive: how much context that +/// session-continuing turn sent and how far the goal had progressed by its end. +/// Empty for single-turn runs. `input_tokens` per turn is the growth-curve axis — +/// "how fast does context accumulate" — that the compaction/curation levers move. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct TurnMetric { + pub turn: u32, + /// Context (input) tokens sent to the model on this turn. + pub input_tokens: u64, + /// Total tokens (input + output) booked this turn. + pub total_tokens: u64, + /// Tool calls this turn. + pub tool_calls: u32, + /// Sub-goals marked `Done` by the end of this turn (progress axis). + pub goal_done: u32, + /// Total sub-goals in the plan (0 when not goal-driven). + pub goal_total: u32, +} + pub struct RunResult { pub config: String, pub model: String, @@ -118,6 +137,9 @@ pub struct RunResult { /// Trajectory of the representative (furthest-progressing) candidate — /// verify rounds, recovery retries, nudges, last verify attribution. pub trajectory: Trajectory, + /// Per-turn context-growth series of the representative candidate. Non-empty + /// only for multi-turn (`HI_EVAL_TURNS>1`) drives. + pub growth: Vec, } #[derive(Clone, Debug, Serialize)] @@ -142,6 +164,8 @@ pub struct Candidate { pub input_tokens: u64, pub seconds: f64, pub trajectory: Trajectory, + /// Per-turn context-growth series (multi-turn drive only; empty otherwise). + pub growth: Vec, } /// Why a candidate failed — so the summary shows *where* hi loses, not just how @@ -186,6 +210,9 @@ pub struct RunArtifact { pub mcp_model: Option, pub verify_output_summary: String, pub trajectory: Trajectory, + /// Per-turn context-growth series (multi-turn drive only; empty otherwise). + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub growth: Vec, } impl FailKind { diff --git a/crates/hi-eval/src/runner.rs b/crates/hi-eval/src/runner.rs index e849fee9..cb688483 100644 --- a/crates/hi-eval/src/runner.rs +++ b/crates/hi-eval/src/runner.rs @@ -8,7 +8,7 @@ use crate::artifacts::{copy_dir, make_workdir, verify_output_in}; use crate::config::{EvalProfile, Task}; use crate::results::{ Candidate, FailKind, RunResult, Trajectory, TrajectoryAttribution, TrajectoryToolCall, - classify, looks_like_build_error, + TurnMetric, classify, looks_like_build_error, }; /// A content snapshot of `dir` (relative path → bytes), excluding eval/run and @@ -90,6 +90,7 @@ pub async fn run_config( seconds: 0.0, mcp_model: None, trajectory: Trajectory::default(), + growth: Vec::new(), }; // Run candidates in parallel — each gets its own temp workdir. @@ -136,6 +137,7 @@ pub async fn run_config( // trajectory is surfaced, mirroring how `result.fail` is chosen. let mut best_rank: i32 = -1; let mut representative_trajectory: Option = None; + let mut representative_growth: Vec = Vec::new(); for candidate in candidates { let cand_rank = candidate .fail @@ -144,6 +146,7 @@ pub async fn run_config( if cand_rank > best_rank { best_rank = cand_rank; representative_trajectory = Some(candidate.trajectory.clone()); + representative_growth = candidate.growth.clone(); } result.passed |= candidate.passed; if let Some(k) = candidate.fail { @@ -170,6 +173,7 @@ pub async fn run_config( result.fail = fails.into_iter().max_by_key(|k| k.rank()); } result.trajectory = representative_trajectory.unwrap_or_default(); + result.growth = representative_growth; result.changed_files.sort(); result.changed_files.dedup(); result.compat_fallbacks_used.sort(); @@ -186,6 +190,31 @@ fn read_report_goal(report: &Path) -> Option { Some(goal.to_string()) } +/// A single turn's context-growth snapshot from the just-written report: context +/// (input) tokens sent this turn, total booked, tool calls, and goal progress. +/// Best-effort — a missing/short report yields zeroed fields, never an error. +fn read_turn_metric(report: &Path, turn: u32) -> TurnMetric { + let mut m = TurnMetric { + turn, + ..Default::default() + }; + let Ok(text) = std::fs::read_to_string(report) else { + return m; + }; + let Ok(v) = serde_json::from_str::(&text) else { + return m; + }; + m.input_tokens = v["input_tokens"].as_u64().unwrap_or(0); + m.total_tokens = v["total_tokens"].as_u64().unwrap_or(0); + m.tool_calls = v["telemetry"]["tool_calls"].as_u64().unwrap_or(0) as u32; + // The report's `goal` block is a summary: {objective, status, done, total}. + if let Some(goal) = v.get("goal").filter(|g| !g.is_null()) { + m.goal_done = goal["done"].as_u64().unwrap_or(0) as u32; + m.goal_total = goal["total"].as_u64().unwrap_or(0) as u32; + } + m +} + /// One independent attempt in an isolated copy of the fixture. #[allow(clippy::too_many_arguments)] pub fn run_candidate( @@ -275,12 +304,18 @@ sub-goal now, then update the plan with update_plan — including any newly disc ) .output() .context("failed to launch hi")?; + // Per-turn context-growth series (multi-turn drive only). Turn 1 first, then + // one snapshot per session-continuing drive turn — the growth curve. + let mut growth: Vec = Vec::new(); + if turns > 1 { + growth.push(read_turn_metric(&report, 1)); + } if goal_mode && turns > 1 { // Drive while the report says the goal is still active (stall guard: two // consecutive turns with an unchanged goal park the drive). let mut last_goal = read_report_goal(&report); let mut stall = 0u32; - for _ in 1..turns { + for turn in 1..turns { let active = last_goal.as_ref().is_some_and(|g| { g.contains("\"status\":\"Active\"") || g.contains("\"status\": \"Active\"") }); @@ -290,6 +325,7 @@ sub-goal now, then update the plan with update_plan — including any newly disc output = build_cmd(GOAL_CONTINUE_PROMPT, false, Some(turn_steps)) .output() .context("failed to launch hi (drive turn)")?; + growth.push(read_turn_metric(&report, turn + 1)); let goal = read_report_goal(&report); if goal == last_goal { stall += 1; @@ -352,6 +388,7 @@ sub-goal now, then update the plan with update_plan — including any newly disc input_tokens: report.input_tokens, seconds, trajectory: report.trajectory, + growth, }) } @@ -565,4 +602,35 @@ mod tests { let _ = std::fs::remove_file(path); } + + #[test] + fn read_turn_metric_captures_tokens_and_goal_progress() { + use super::read_turn_metric; + let path = + std::env::temp_dir().join(format!("hi-eval-turn-metric-{}.json", std::process::id())); + let report = serde_json::json!({ + "total_tokens": 9000, + "input_tokens": 8000, + "telemetry": { "tool_calls": 7 }, + // The report's goal block is a summary, not the full sub_goals list. + "goal": { "objective": "x", "status": "Active", "done": 2, "total": 4 } + }); + std::fs::write(&path, serde_json::to_string(&report).unwrap()).unwrap(); + + let m = read_turn_metric(&path, 3); + assert_eq!(m.turn, 3); + assert_eq!(m.input_tokens, 8000); + assert_eq!(m.total_tokens, 9000); + assert_eq!(m.tool_calls, 7); + assert_eq!(m.goal_total, 4); + assert_eq!(m.goal_done, 2); + + let _ = std::fs::remove_file(&path); + + // Missing report → zeroed metric, never a panic (fail-open). + let absent = read_turn_metric(std::path::Path::new("/no/such/report.json"), 5); + assert_eq!(absent.turn, 5); + assert_eq!(absent.input_tokens, 0); + assert_eq!(absent.goal_total, 0); + } }