hi-eval: per-turn context-growth harness + long interp task

bench/long/interp/fixed/interp.py

@@ -0,0 +1,160 @@
+"""A small infix expression interpreter: tokenizer + recursive-descent parser."""
+
+
+def _tokenize(s):
+    toks = []
+    i, n = 0, len(s)
+    while i < n:
+        c = s[i]
+        if c.isspace():
+            i += 1
+            continue
+        if c.isdigit() or (c == "." and i + 1 < n and s[i + 1].isdigit()):
+            j = i
+            while j < n and (s[j].isdigit() or s[j] == "."):
+                j += 1
+            text = s[i:j]
+            toks.append(("num", float(text) if "." in text else int(text)))
+            i = j
+            continue
+        if c.isalpha() or c == "_":
+            j = i
+            while j < n and (s[j].isalnum() or s[j] == "_"):
+                j += 1
+            toks.append(("name", s[i:j]))
+            i = j
+            continue
+        two = s[i : i + 2]
+        if two in ("**", "<=", ">=", "==", "!="):
+            toks.append(("op", two))
+            i += 2
+            continue
+        if c in "+-*/%<>(),":
+            toks.append(("op", c))
+            i += 1
+            continue
+        raise ValueError("bad character: " + repr(c))
+    toks.append(("end", None))
+    return toks
+
+
+_FUNCS = {"min": min, "max": max, "abs": lambda x: abs(x)}
+
+
+class _Parser:
+    def __init__(self, toks, env):
+        self.toks = toks
+        self.pos = 0
+        self.env = env
+
+    def _peek(self):
+        return self.toks[self.pos]
+
+    def _advance(self):
+        t = self.toks[self.pos]
+        self.pos += 1
+        return t
+
+    def _match(self, *ops):
+        t = self._peek()
+        if t[0] == "op" and t[1] in ops:
+            self.pos += 1
+            return t[1]
+        return None
+
+    def _expect(self, op):
+        if self._match(op) is None:
+            raise ValueError("expected " + op)
+
+    def parse(self):
+        v = self.comparison()
+        if self._peek()[0] != "end":
+            raise ValueError("trailing tokens")
+        return v
+
+    def comparison(self):
+        left = self.additive()
+        op = self._match("<", ">", "<=", ">=", "==", "!=")
+        if op is None:
+            return left
+        right = self.additive()
+        if op == "<":
+            return left < right
+        if op == ">":
+            return left > right
+        if op == "<=":
+            return left <= right
+        if op == ">=":
+            return left >= right
+        if op == "==":
+            return left == right
+        return left != right
+
+    def additive(self):
+        left = self.multiplicative()
+        while True:
+            op = self._match("+", "-")
+            if op is None:
+                return left
+            right = self.multiplicative()
+            left = left + right if op == "+" else left - right
+
+    def multiplicative(self):
+        left = self.power()
+        while True:
+            op = self._match("*", "/", "%")
+            if op is None:
+                return left
+            right = self.power()
+            if op == "*":
+                left = left * right
+            elif op == "/":
+                left = left / right
+            else:
+                left = left % right
+
+    def power(self):
+        left = self.unary()
+        if self._match("**") is not None:
+            right = self.power()  # right-associative
+            return left**right
+        return left
+
+    def unary(self):
+        op = self._match("+", "-")
+        if op == "-":
+            return -self.unary()
+        if op == "+":
+            return self.unary()
+        return self.primary()
+
+    def primary(self):
+        t = self._peek()
+        if t[0] == "num":
+            self._advance()
+            return t[1]
+        if self._match("(") is not None:
+            v = self.comparison()
+            self._expect(")")
+            return v
+        if t[0] == "name":
+            self._advance()
+            name = t[1]
+            if self._match("(") is not None:
+                args = []
+                if self._peek() != ("op", ")"):
+                    args.append(self.comparison())
+                    while self._match(",") is not None:
+                        args.append(self.comparison())
+                self._expect(")")
+                if name not in _FUNCS:
+                    raise NameError("unknown function: " + name)
+                return _FUNCS[name](*args)
+            if name in self.env:
+                return self.env[name]
+            raise NameError("undefined variable: " + name)
+        raise ValueError("unexpected token: " + repr(t))
+
+
+def evaluate(expr, env=None):
+    return _Parser(_tokenize(expr), env or {}).parse()

bench/long/interp/fixture/interp.py

@@ -0,0 +1,2 @@
+def evaluate(expr, env=None):
+    raise NotImplementedError

bench/long/interp/task.toml

@@ -0,0 +1,47 @@
+name = "interp"
+prompt = """Build an infix expression interpreter in interp.py. Implement `evaluate(expr, env=None)` that parses and evaluates an arithmetic/logic expression string and returns the result. The file currently raises NotImplementedError. Edit only interp.py. Implement all of these requirements:
+
+1. Integer and float literals (e.g. `42`, `1.5`).
+2. Binary `+ - * /` with standard precedence (`*` and `/` bind tighter than `+` and `-`), left-associative.
+3. Parentheses for grouping.
+4. Unary minus and unary plus (e.g. `-3`, `-(2+3)`, `- -5`).
+5. Modulo `%` (same precedence as `*`/`/`).
+6. Exponentiation `**`, right-associative and binding tighter than unary minus's operand — i.e. `2 ** 3 ** 2 == 512`.
+7. Variables resolved from the `env` dict argument, e.g. `evaluate("x + 1", {"x": 3}) == 4`.
+8. Comparison operators `< > <= >= == !=` (lowest precedence) returning a bool.
+9. Built-in functions `min`, `max` (variadic), and `abs`, callable with parenthesised comma-separated args, e.g. `evaluate("max(2, 3) + 1") == 4`.
+10. Errors: division or modulo by zero raises ZeroDivisionError; an undefined variable or unknown function name raises NameError; any malformed/unparseable input (bad character, trailing tokens, unbalanced parens, missing operand) raises ValueError.
+
+Write a real tokenizer and parser — do not use eval() or the ast module. Verify your work as you go."""
+verify = '''python3 -c "
+import interp as m
+E = m.evaluate
+assert E('1 + 2 * 3') == 7
+assert E('(1 + 2) * 3') == 9
+assert E('10 - 2 - 3') == 5
+assert E('7 % 3') == 1
+assert E('2 ** 3 ** 2') == 512
+assert E('6 / 2') == 3
+assert E('-3 + 4') == 1
+assert E('-(2 + 3)') == -5
+assert E('- -5') == 5
+assert E('1.5 + 2') == 3.5
+assert E('x + 1', {'x': 3}) == 4
+assert E('a * b', {'a': 4, 'b': 5}) == 20
+assert E('2 < 3') is True
+assert E('3 <= 3') is True
+assert E('2 == 2') is True
+assert E('2 != 3') is True
+assert E('5 > 9') is False
+assert E('max(2, 3) + 1') == 4
+assert E('min(4, 2, 7)') == 2
+assert E('abs(-8)') == 8
+assert E('max(abs(-3), 2)') == 3
+import sys
+for expr, exc in [('1 / 0', ZeroDivisionError), ('5 % 0', ZeroDivisionError), ('y + 1', NameError), ('1 +', ValueError), ('2 @ 3', ValueError), ('(1 + 2', ValueError)]:
+    try:
+        E(expr); sys.exit('no error for: ' + expr)
+    except exc:
+        pass
+print('ok')
+"'''

crates/hi-eval/src/artifacts.rs

@@ -39,6 +39,7 @@ pub fn write_artifact(
         mcp_model: result.mcp_model.clone(),
         verify_output_summary: result.verify_output_summary.clone(),
         trajectory: result.trajectory.clone(),
+        growth: result.growth.clone(),
     };
     let name = format!(
         "trial-{:03}-{}-{}-{}.json",

crates/hi-eval/src/reporting.rs

@@ -154,6 +154,30 @@ pub fn print_summary(results: &[RunResult], task_count: usize, active: &[&Config
                     "           parallel: {avg_max:.1} max concurrent batch · {serial_pct}% of {total_calls} calls serial",
                 );
             }
+
+            // Context-growth curve (multi-turn drives only): the representative
+            // trial's per-turn input-token trajectory — "how fast does context
+            // accumulate vs. how far does the goal get". The exemplar is the row
+            // that drove the most turns. Watch for ctx climbing while goal/done
+            // stalls: that's the bloat the compaction/curation levers should flatten.
+            if let Some(row) = rows
+                .iter()
+                .filter(|r| !r.growth.is_empty())
+                .max_by_key(|r| r.growth.len())
+            {
+                println!("           growth (ctx tok · tools · sub-goals done):");
+                for m in &row.growth {
+                    let prog = if m.goal_total > 0 {
+                        format!("{}/{}", m.goal_done, m.goal_total)
+                    } else {
+                        "-".to_string()
+                    };
+                    println!(
+                        "             t{:<2} ctx={:>8} tools={:>3} goal={}",
+                        m.turn, m.input_tokens, m.tool_calls, prog
+                    );
+                }
+            }
         }
     }
 }

crates/hi-eval/src/results.rs

@@ -95,6 +95,25 @@ impl Trajectory {
     }
 }
 
+/// One turn of a multi-turn (long-task) drive: how much context that
+/// session-continuing turn sent and how far the goal had progressed by its end.
+/// Empty for single-turn runs. `input_tokens` per turn is the growth-curve axis —
+/// "how fast does context accumulate" — that the compaction/curation levers move.
+#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
+pub struct TurnMetric {
+    pub turn: u32,
+    /// Context (input) tokens sent to the model on this turn.
+    pub input_tokens: u64,
+    /// Total tokens (input + output) booked this turn.
+    pub total_tokens: u64,
+    /// Tool calls this turn.
+    pub tool_calls: u32,
+    /// Sub-goals marked `Done` by the end of this turn (progress axis).
+    pub goal_done: u32,
+    /// Total sub-goals in the plan (0 when not goal-driven).
+    pub goal_total: u32,
+}
+
 pub struct RunResult {
     pub config: String,
     pub model: String,
@@ -118,6 +137,9 @@ pub struct RunResult {
     /// Trajectory of the representative (furthest-progressing) candidate —
     /// verify rounds, recovery retries, nudges, last verify attribution.
     pub trajectory: Trajectory,
+    /// Per-turn context-growth series of the representative candidate. Non-empty
+    /// only for multi-turn (`HI_EVAL_TURNS>1`) drives.
+    pub growth: Vec<TurnMetric>,
 }
 
 #[derive(Clone, Debug, Serialize)]
@@ -142,6 +164,8 @@ pub struct Candidate {
     pub input_tokens: u64,
     pub seconds: f64,
     pub trajectory: Trajectory,
+    /// Per-turn context-growth series (multi-turn drive only; empty otherwise).
+    pub growth: Vec<TurnMetric>,
 }
 
 /// Why a candidate failed — so the summary shows *where* hi loses, not just how
@@ -186,6 +210,9 @@ pub struct RunArtifact {
     pub mcp_model: Option<McpModelArtifact>,
     pub verify_output_summary: String,
     pub trajectory: Trajectory,
+    /// Per-turn context-growth series (multi-turn drive only; empty otherwise).
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    pub growth: Vec<TurnMetric>,
 }
 
 impl FailKind {