diff --git a/bench/plan/README.md b/bench/plan/README.md new file mode 100644 index 00000000..fe69bbff --- /dev/null +++ b/bench/plan/README.md @@ -0,0 +1,94 @@ +# Plan-completion evals (`/goal`) + +These fixtures measure the thing `/goal` is *for*: given a multi-section `plan.md` +and "build all of it", **what fraction of the plan does the agent actually +deliver?** Each fixture's hidden oracle scores per section and prints a fraction, +so a run that finishes 4 of 6 sections scores **0.67**, not a flat fail — which +is what you need to see an improvement like *20% → 75%*, rather than pass/fail. + +Unlike the single-turn `bench/tasks`, these run in **goal-drive mode**: the +candidate sets a long-horizon goal from the prompt and the harness drives it +across `HI_EVAL_TURNS` session-continuing turns (the real `/goal` cadence). + +## Fixtures + +- **`textkit/`** — a small Python text-utilities library: six independent + modules (`slugify`, `wordcount`, `roman`, `caesar`, `rpn`, `csvmini`), each an + exact contract in `fixture/plan.md`. Score = modules delivered / 6. + +## Running it + +Build the `hi` binary you want to measure, then point the harness at it: + +```sh +# From the repo root. Uses the binary at $HI_BIN (or target/{debug,release}/hi). +HI_BIN=./target/release/hi \ +HI_MODEL= HI_API_KEY= \ +HI_EVAL_GOAL=1 HI_EVAL_TURNS=12 \ +cargo run -p hi-eval -- bench/plan +``` + +- `HI_EVAL_GOAL=1` turns on goal-drive mode (`--goal ` on turn 1). +- `HI_EVAL_TURNS=12` gives the goal up to 12 drive turns to finish the plan. + Raise it for larger plans. + +## Reading the score + +The hidden scorer (`oracle/score.py`) prints, in the run's `final_oracle` +artifact: + +``` +=== textkit plan-completion score === +[PASS] slugify +[FAIL] roman: ... +... +sections delivered: 4/6 (67%) +HI_EVAL_SCORE=0.6667 +``` + +`HI_EVAL_SCORE` (0..1) is the "% of the plan delivered." A fully-delivered plan +also exits 0, so it registers as a normal harness **pass**. + +### Baseline vs. candidate + +Run the same fixture against two binaries and compare the fractions — e.g. the +old goal engine vs. the completion fixes: + +```sh +HI_BIN=./hi-baseline ... cargo run -p hi-eval -- bench/plan # e.g. 0.17 +HI_BIN=./target/release/hi ... cargo run -p hi-eval -- bench/plan # e.g. 0.83 +``` + +## Validating a fixture (no model, deterministic) + +The harness self-checks each fixture — the empty `fixture/` must score below +100% and the `fixed/` reference must score 100%: + +```sh +cargo run -p hi-eval -- --validate bench/plan +``` + +You can also score any built workspace directly: + +```sh +mkdir -p /path/to/built/.hi-eval-oracle +cp bench/plan/textkit/oracle/score.py /path/to/built/.hi-eval-oracle/ +( cd /path/to/built && python3 .hi-eval-oracle/score.py ) +``` + +## Adding a fixture + +Mirror `textkit/`: + +- `fixture/plan.md` — the spec the candidate reads (sections with exact + contracts + examples). Do **not** put the oracle here. +- `oracle/score.py` — the hidden scorer: check each section independently, + print `sections delivered: X/N` and `HI_EVAL_SCORE=<0..1>`, exit 0 iff all + delivered. It runs from the candidate's workspace (`sys.path` includes cwd). +- `fixed/` — a reference implementation that scores 100% (proves the plan is + buildable; used by `--validate`). +- `task.toml` — `prompt` = the objective, `allowed_changes` globs, and + `[final_oracle]` with `bundle = "oracle"` + the scorer command. + +Keep sections small, independent, and standard-library-only so scoring is fast +and deterministic. diff --git a/bench/plan/textkit/fixed/caesar.py b/bench/plan/textkit/fixed/caesar.py new file mode 100644 index 00000000..64e1d3cf --- /dev/null +++ b/bench/plan/textkit/fixed/caesar.py @@ -0,0 +1,20 @@ +def _shift(text, shift): + out = [] + for ch in text: + if "a" <= ch <= "z": + out.append(chr((ord(ch) - ord("a") + shift) % 26 + ord("a"))) + elif "A" <= ch <= "Z": + out.append(chr((ord(ch) - ord("A") + shift) % 26 + ord("A"))) + else: + out.append(ch) + return "".join(out) + + +def encrypt(text, shift): + """Caesar-shift letters forward by `shift`; non-letters unchanged.""" + return _shift(text, shift) + + +def decrypt(text, shift): + """Inverse of encrypt for the same shift.""" + return _shift(text, -shift) diff --git a/bench/plan/textkit/fixed/csvmini.py b/bench/plan/textkit/fixed/csvmini.py new file mode 100644 index 00000000..399b99b8 --- /dev/null +++ b/bench/plan/textkit/fixed/csvmini.py @@ -0,0 +1,54 @@ +def parse_csv(text): + """Parse simple CSV into a list of rows (each a list of string fields). + + Rules: fields are comma-separated; a field may be double-quoted, in which + case it may contain commas and newlines, and a doubled quote ("") inside a + quoted field is a literal quote. Rows are newline-separated. A trailing + newline does not produce an empty final row.""" + rows = [] + field = [] + row = [] + i = 0 + n = len(text) + in_quotes = False + saw_any = False + while i < n: + ch = text[i] + if in_quotes: + if ch == '"': + if i + 1 < n and text[i + 1] == '"': + field.append('"') + i += 2 + continue + in_quotes = False + i += 1 + continue + field.append(ch) + i += 1 + continue + if ch == '"': + in_quotes = True + saw_any = True + i += 1 + continue + if ch == ",": + row.append("".join(field)) + field = [] + saw_any = True + i += 1 + continue + if ch == "\n": + row.append("".join(field)) + rows.append(row) + field = [] + row = [] + saw_any = False + i += 1 + continue + field.append(ch) + saw_any = True + i += 1 + if saw_any or field: + row.append("".join(field)) + rows.append(row) + return rows diff --git a/bench/plan/textkit/fixed/roman.py b/bench/plan/textkit/fixed/roman.py new file mode 100644 index 00000000..37008672 --- /dev/null +++ b/bench/plan/textkit/fixed/roman.py @@ -0,0 +1,44 @@ +_VALUES = [ + (1000, "M"), + (900, "CM"), + (500, "D"), + (400, "CD"), + (100, "C"), + (90, "XC"), + (50, "L"), + (40, "XL"), + (10, "X"), + (9, "IX"), + (5, "V"), + (4, "IV"), + (1, "I"), +] + + +def to_roman(n): + """Integer 1..3999 -> Roman numeral string.""" + if not isinstance(n, int) or not (1 <= n <= 3999): + raise ValueError("n must be an integer in 1..3999") + out = [] + for value, symbol in _VALUES: + while n >= value: + out.append(symbol) + n -= value + return "".join(out) + + +def from_roman(s): + """Roman numeral string -> integer.""" + symbols = {sym: val for val, sym in [(1, "I"), (5, "V"), (10, "X"), (50, "L"), (100, "C"), (500, "D"), (1000, "M")]} + total = 0 + prev = 0 + for ch in reversed(s.upper()): + if ch not in symbols: + raise ValueError(f"bad roman numeral: {s!r}") + val = symbols[ch] + if val < prev: + total -= val + else: + total += val + prev = val + return total diff --git a/bench/plan/textkit/fixed/rpn.py b/bench/plan/textkit/fixed/rpn.py new file mode 100644 index 00000000..d58b268a --- /dev/null +++ b/bench/plan/textkit/fixed/rpn.py @@ -0,0 +1,27 @@ +_OPS = { + "+": lambda a, b: a + b, + "-": lambda a, b: a - b, + "*": lambda a, b: a * b, + "/": lambda a, b: a / b, +} + + +def eval_rpn(expr): + """Evaluate a space-separated reverse-Polish expression and return a float. + Supports + - * / on numbers. Raises ValueError on a malformed expression.""" + stack = [] + for token in expr.split(): + if token in _OPS: + if len(stack) < 2: + raise ValueError(f"not enough operands for {token!r}") + b = stack.pop() + a = stack.pop() + stack.append(_OPS[token](a, b)) + else: + try: + stack.append(float(token)) + except ValueError: + raise ValueError(f"bad token: {token!r}") + if len(stack) != 1: + raise ValueError("malformed expression") + return stack[0] diff --git a/bench/plan/textkit/fixed/slugify.py b/bench/plan/textkit/fixed/slugify.py new file mode 100644 index 00000000..630a59b6 --- /dev/null +++ b/bench/plan/textkit/fixed/slugify.py @@ -0,0 +1,9 @@ +import re + + +def slugify(s): + """Lowercase; collapse each run of non-[a-z0-9] characters into one + hyphen; strip leading/trailing hyphens.""" + s = s.lower() + s = re.sub(r"[^a-z0-9]+", "-", s) + return s.strip("-") diff --git a/bench/plan/textkit/fixed/wordcount.py b/bench/plan/textkit/fixed/wordcount.py new file mode 100644 index 00000000..8bc3bcf5 --- /dev/null +++ b/bench/plan/textkit/fixed/wordcount.py @@ -0,0 +1,10 @@ +import re + + +def word_count(s): + """Return a dict mapping each word to its count. Words are maximal runs of + [a-z0-9], compared case-insensitively.""" + counts = {} + for word in re.findall(r"[a-z0-9]+", s.lower()): + counts[word] = counts.get(word, 0) + 1 + return counts diff --git a/bench/plan/textkit/fixture/plan.md b/bench/plan/textkit/fixture/plan.md new file mode 100644 index 00000000..a153cb2f --- /dev/null +++ b/bench/plan/textkit/fixture/plan.md @@ -0,0 +1,73 @@ +# textkit — a small Python text-utilities library + +Build **textkit**, a Python library of six independent, self-contained modules in +this directory. Each section below is one module with an exact behavioral +contract and examples. Implement every section fully; a section counts as +delivered only when its module imports and its functions behave exactly as +specified. Pure standard library only (no third-party packages). Do not create a +package directory — put each module as a top-level `.py` file in this directory. + +## 1. `slugify.py` + +`slugify(s: str) -> str`: lowercase the string, replace every maximal run of +characters that are not `a`–`z` or `0`–`9` with a single hyphen `-`, then strip +leading and trailing hyphens. + +- `slugify("Hello, World!")` → `"hello-world"` +- `slugify(" A_B c ")` → `"a-b-c"` +- `slugify("--Rock & Roll--")` → `"rock-roll"` + +## 2. `wordcount.py` + +`word_count(s: str) -> dict`: return a dict mapping each word to how many times +it occurs. A word is a maximal run of `a`–`z` or `0`–`9`, compared +case-insensitively (lowercase the keys). + +- `word_count("The the THE cat")` → `{"the": 3, "cat": 1}` +- `word_count("Hi, hi! Bye.")` → `{"hi": 2, "bye": 1}` +- `word_count("")` → `{}` + +## 3. `roman.py` + +Two functions for classic Roman numerals (values 1..3999): + +- `to_roman(n: int) -> str`: integer → numeral. `to_roman(4)` → `"IV"`, + `to_roman(1994)` → `"MCMXCIV"`, `to_roman(2023)` → `"MMXXIII"`. Raise + `ValueError` if `n` is not an integer in `1..3999`. +- `from_roman(s: str) -> int`: numeral → integer (accept upper case). + `from_roman("MCMXCIV")` → `1994`, `from_roman("XLII")` → `42`. + +`from_roman(to_roman(n)) == n` for every `n` in `1..3999`. + +## 4. `caesar.py` + +A Caesar cipher over ASCII letters: + +- `encrypt(text: str, shift: int) -> str`: shift each letter forward by `shift` + (wrapping within `a`–`z` and within `A`–`Z`); leave every non-letter + unchanged. `encrypt("abc XYZ!", 3)` → `"def ABC!"`. +- `decrypt(text: str, shift: int) -> str`: the inverse, so + `decrypt(encrypt(t, k), k) == t` for any text `t` and shift `k`. + +## 5. `rpn.py` + +`eval_rpn(expr: str) -> float`: evaluate a space-separated reverse-Polish +expression supporting `+ - * /` over numbers, returning a float. Division is +floating-point. Raise `ValueError` on a malformed expression (bad token, too few +operands, or leftover operands). + +- `eval_rpn("3 4 + 2 *")` → `14.0` +- `eval_rpn("10 2 /")` → `5.0` +- `eval_rpn("5 1 2 + 4 * + 3 -")` → `14.0` + +## 6. `csvmini.py` + +`parse_csv(text: str) -> list`: parse simple CSV into a list of rows, each a list +of string fields. Fields are comma-separated; a field may be double-quoted, in +which case it may contain commas and newlines; a doubled quote `""` inside a +quoted field is a literal `"`. Rows are newline-separated, and a trailing newline +does not create an empty final row. + +- `parse_csv("a,b,c")` → `[["a", "b", "c"]]` +- `parse_csv('a,b\n"c,d",e')` → `[["a", "b"], ["c,d", "e"]]` +- `parse_csv('x,"he said ""hi"""')` → `[["x", 'he said "hi"']]` diff --git a/bench/plan/textkit/oracle/score.py b/bench/plan/textkit/oracle/score.py new file mode 100644 index 00000000..b5bc5d69 --- /dev/null +++ b/bench/plan/textkit/oracle/score.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +"""Hidden scorer for the `textkit` /goal plan-completion fixture. + +Each of the six plan sections is one module with an exact contract. A section +counts as *delivered* only if its module imports and every check passes. The +score is `delivered / 6` — a fractional "% of the plan actually built", so a run +that finishes 4 of 6 sections scores 0.67 instead of a flat fail. + +Runs against the candidate's modules in the current working directory. Prints a +per-section PASS/FAIL breakdown and a machine-readable `HI_EVAL_SCORE=<0..1>` +line, and exits 0 only when all six sections are delivered. +""" +import importlib +import os +import sys +import traceback + +# Import the candidate's modules from the workspace, not this scorer's dir. +sys.path.insert(0, os.getcwd()) + + +def check_slugify(): + m = importlib.import_module("slugify") + assert m.slugify("Hello, World!") == "hello-world" + assert m.slugify(" A_B c ") == "a-b-c" + assert m.slugify("--Rock & Roll--") == "rock-roll" + assert m.slugify("") == "" + + +def check_wordcount(): + m = importlib.import_module("wordcount") + assert m.word_count("The the THE cat") == {"the": 3, "cat": 1} + assert m.word_count("Hi, hi! Bye.") == {"hi": 2, "bye": 1} + assert m.word_count("") == {} + + +def check_roman(): + m = importlib.import_module("roman") + assert m.to_roman(4) == "IV" + assert m.to_roman(1994) == "MCMXCIV" + assert m.to_roman(2023) == "MMXXIII" + assert m.from_roman("MCMXCIV") == 1994 + assert m.from_roman("XLII") == 42 + for n in (1, 9, 40, 90, 400, 900, 3999, 1888): + assert m.from_roman(m.to_roman(n)) == n + for bad in (0, 4000, -1): + try: + m.to_roman(bad) + raise AssertionError(f"to_roman({bad}) should raise ValueError") + except ValueError: + pass + + +def check_caesar(): + m = importlib.import_module("caesar") + assert m.encrypt("abc XYZ!", 3) == "def ABC!" + for text, k in (("Hello, World!", 5), ("zZ aA", 1), ("nothing here 123", 13)): + assert m.decrypt(m.encrypt(text, k), k) == text + + +def check_rpn(): + m = importlib.import_module("rpn") + assert m.eval_rpn("3 4 + 2 *") == 14.0 + assert m.eval_rpn("10 2 /") == 5.0 + assert m.eval_rpn("5 1 2 + 4 * + 3 -") == 14.0 + for bad in ("1 +", "1 2 3", "1 x +"): + try: + m.eval_rpn(bad) + raise AssertionError(f"eval_rpn({bad!r}) should raise ValueError") + except ValueError: + pass + + +def check_csvmini(): + m = importlib.import_module("csvmini") + assert m.parse_csv("a,b,c") == [["a", "b", "c"]] + assert m.parse_csv('a,b\n"c,d",e') == [["a", "b"], ["c,d", "e"]] + assert m.parse_csv('x,"he said ""hi"""') == [["x", 'he said "hi"']] + assert m.parse_csv("a,b\n") == [["a", "b"]] + + +SECTIONS = [ + ("slugify", check_slugify), + ("wordcount", check_wordcount), + ("roman", check_roman), + ("caesar", check_caesar), + ("rpn", check_rpn), + ("csvmini", check_csvmini), +] + + +def main(): + delivered = 0 + print("=== textkit plan-completion score ===") + for name, check in SECTIONS: + try: + check() + delivered += 1 + print(f"[PASS] {name}") + except Exception as exc: # noqa: BLE001 — a broken/missing module must not crash scoring + first = "".join(traceback.format_exception_only(type(exc), exc)).strip() + print(f"[FAIL] {name}: {first}") + total = len(SECTIONS) + pct = round(100 * delivered / total) + print(f"sections delivered: {delivered}/{total} ({pct}%)") + print(f"HI_EVAL_SCORE={delivered / total:.4f}") + sys.exit(0 if delivered == total else 1) + + +if __name__ == "__main__": + main() diff --git a/bench/plan/textkit/task.toml b/bench/plan/textkit/task.toml new file mode 100644 index 00000000..147cb02d --- /dev/null +++ b/bench/plan/textkit/task.toml @@ -0,0 +1,17 @@ +schema_version = 2 +name = "textkit" +allowed_changes = ["*.py", "tests/**", "*.md"] +prompt = "Review plan.md and fully build every section it specifies. It defines six independent Python modules in this directory; implement each one exactly as its contract and examples require, using only the standard library." + +[timeouts] +candidate_seconds = 1200 +visible_feedback_seconds = 120 +oracle_seconds = 120 + +# The score is the FRACTION of plan sections delivered — the scorer prints +# `HI_EVAL_SCORE=<0..1>` and a per-section PASS/FAIL breakdown. Read that as the +# "% of the plan actually built"; a fully-delivered plan also exits 0 (a harness +# pass). Run in goal-drive mode: HI_EVAL_GOAL=1 HI_EVAL_TURNS=. +[final_oracle] +bundle = "oracle" +command = "python3 .hi-eval-oracle/score.py"