From 2db160ef798b817c9a4df7454bd0d7c3f27e2c0f Mon Sep 17 00:00:00 2001 From: "Voight-Kampff (bot)" <151556158+sunholo-voight-kampff@users.noreply.github.com> Date: Fri, 31 Jul 2026 12:22:26 +0200 Subject: [PATCH 1/2] fix(eval): agent cost was the wrong model's price, and the budget was the wrong unit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two defects found auditing OpenAI's 2026-07-30 GPT-5.6 price cut, plus the end-of-run tally that would have surfaced either one sooner. 1. PER-HARNESS PRICING. Executors that compute cost from token counts used their own hardcoded table, so the codex harness billed EVERY model it ran at gpt-5-codex's $1.25/$10 per 1M — gpt5-6-luna and gpt5-4-mini banked at identical rates despite really costing $0.20/$1.20 and $0.75/$4.50. Task.Pricing now carries the per-model rates and ResolveCostModel prefers them; a present-but-zero rate is honoured so free local models are not billed at cloud prices. The cost-KILL path was already per-model correct, which meant banked cost and kill threshold came from two different price tables: luna/graph_bfs banked $0.34259 at codex rates while the $0.30 budget that spared it saw $0.26980 at luna's. 2. COST PROVENANCE. cost_usd said nothing about whether anyone was charged. codex (auth_mode chatgpt) and claude (OAuth) run on subscriptions, so their dollars are list-price equivalents, not spend — yet they summed with metered OpenRouter/Vertex spend under one label, directly beneath a v1.0 KPI whose numerator is attributable METERED dollars. Each executor now classifies its own auth lane; banked as cost_provenance, stored on chain stages (schema v17), split out by ClassifyStageCost into a new `subscription` status that CostRollup keeps OUT of TotalKnownCost. Absent labels read as unknown, never metered — nothing is backfilled. 3. WORK GATES. A dollar ceiling buys tokens in inverse proportion to price, so nominally-uniform $0.30/$0.50 caps spanned 24x in real work: claude-sonnet-4-6, the longitudinal ANCHOR, got 0.14M tokens against deepseek-v4-flash's 3.40M — it had no budgets block at all and fell through to a formula that buys almost nothing at $3/$15. A suite asking "does the agent loop rescue weak models" gave the weakest the most rope. budgets.max_tokens_per_bench sets one shared 3.0M ceiling; subscription lanes gate on tokens alone and now sit identical. Task.MaxTokensPerBench had been plumbed since M-EVAL-OS-LONGITUDINAL but ONLY opencode honoured it; codex and claude now enforce it too. Metered opencode lanes keep their dollar ceiling as a real spend control with tokens as a second bound — still cost-limited, but for a stated reason. eval-suite now prints a cost tally at the end, split by provenance, so the metered figure is never conflated with subscription arithmetic again. Verified: build, go vet, make check-boundaries, and tests across executor, eval_harness, observatory, eval_analysis, coordinator, server and storage. Tally rendered against the real v0.30.0 baseline (2,090 rows, all correctly classified unknown — they predate the label). Co-Authored-By: Claude Opus 5 --- changelogs/v0.18-current.md | 123 ++++++++++++ cmd/ailang/eval_benchmark.go | 8 +- cmd/ailang/eval_suite_finalize.go | 11 ++ internal/coordinator/daemon_tasks_chain.go | 2 +- internal/coordinator/provider.go | 19 +- internal/coordinator/provider_executor.go | 1 + internal/eval_analysis/export_json.go | 6 +- internal/eval_analysis/loader_chains_test.go | 2 +- internal/eval_analysis/matrix.go | 9 + internal/eval_analysis/sweet_spot.go | 13 +- internal/eval_analysis/types.go | 31 ++- internal/eval_harness/agent_runner.go | 20 +- internal/eval_harness/agent_runner_multi.go | 16 ++ internal/eval_harness/cost_provenance_test.go | 148 +++++++++++++++ internal/eval_harness/cost_tally.go | 179 ++++++++++++++++++ internal/eval_harness/cost_tally_test.go | 104 ++++++++++ internal/eval_harness/metrics.go | 77 +++++--- internal/eval_harness/models.go | 32 ++++ internal/eval_harness/models.yml | 165 ++++++++++++++-- internal/executor/claude/claude.go | 66 ++++++- internal/executor/codex/codex.go | 100 +++++++++- internal/executor/cost.go | 74 ++++++++ internal/executor/cost_test.go | 69 +++++++ internal/executor/executor.go | 33 +++- .../executor/managed_agents/managed_agents.go | 4 +- internal/executor/motoko/motoko.go | 19 +- internal/executor/opencode/opencode.go | 2 + internal/executor/pi/pi.go | 2 + internal/observatory/backend.go | 6 +- internal/observatory/backend_composite.go | 4 +- internal/observatory/backend_gcp.go | 2 +- internal/observatory/backend_jaeger.go | 2 +- internal/observatory/backend_sqlite.go | 4 +- internal/observatory/cost_classify.go | 56 +++++- internal/observatory/cost_classify_test.go | 51 +++++ .../cost_per_verified_success_test.go | 2 +- .../observatory/cost_rollup_store_test.go | 2 +- internal/observatory/iteration_post.go | 3 +- internal/observatory/migrate.go | 8 + internal/observatory/migrate_test.go | 13 +- internal/observatory/migrate_v17.go | 32 ++++ internal/observatory/models_chains.go | 17 +- internal/observatory/store_chains.go | 14 +- internal/server/handlers_chains_cvs_test.go | 2 +- .../storage/firestore/observatory_chains.go | 13 +- 45 files changed, 1435 insertions(+), 131 deletions(-) create mode 100644 internal/eval_harness/cost_provenance_test.go create mode 100644 internal/eval_harness/cost_tally.go create mode 100644 internal/eval_harness/cost_tally_test.go create mode 100644 internal/observatory/migrate_v17.go diff --git a/changelogs/v0.18-current.md b/changelogs/v0.18-current.md index 8a3f3cdc3..72797c6ba 100644 --- a/changelogs/v0.18-current.md +++ b/changelogs/v0.18-current.md @@ -2,6 +2,129 @@ > Current changelog (v0.18.0 onward). Earlier series: [v0.10–v0.17](v0.10-v0.17-bytecode-vm.md). Index: [CHANGELOG.md](../CHANGELOG.md). +## [Unreleased] + +### Added + +- `eval-suite` now prints a **cost tally** when a run finishes — per model/mode, + then totals split by provenance. The split is the point: a single grand total + is misleading on a rig where agent-mode `codex` and `claude` authenticate by + subscription, so their `cost_usd` is real arithmetic over real tokens that + nobody paid. `METERED (actually billed)` is the only line that answers "what + did this cost us"; list-price-equivalent and unlabelled rows are surfaced + alongside it and never folded in. Best-effort by construction — a malformed + result file is skipped and a cost report never fails an otherwise-complete run + (`eval_harness.TallyCosts`). + +### Changed + +- Agent-eval budgets are now gated on **work**, not dollars. A dollar ceiling + buys tokens in inverse proportion to model price, so the nominally-uniform + `$0.30`/`$0.50` caps across `agent_suite` spanned **24x** in actual work + (audited 2026-07-31): `opencode-or-deepseek-v4-flash` got 3.40M tokens while + `claude-sonnet-4-6` — the longitudinal **anchor** every other model is scored + against — got 0.14M, the tightest gate in the suite. It had no `budgets:` + block at all, so it fell through to the default formula, which at `$3/$15` + buys almost nothing. A suite asking "does the agent loop rescue weak models" + was handing the weakest models the most iteration and the reference the least. + + `budgets.max_tokens_per_bench` (new, per-model, overrides the global + `--max-tokens-per-bench` flag) sets one shared **3.0M blended-token** ceiling + across the suite. Subscription lanes — `codex` on `auth_mode: chatgpt`, + `claude` on OAuth — gate on tokens alone and now sit at an identical 3.00M; + their dollar caps were raised purely so the token gate is the one that binds + (`claude-sonnet-4-6` `$0.50` → `$10.80`, `claude-haiku-4-5` `$0.30` → `$3.60`, + `gpt5-6-luna` `$0.30` → `$0.75`). This is **not** a spend increase: those lanes + are subscription-authenticated, so the figure is a list-price equivalent. + ⚠️ Under `AILANG_AUTH_MODE=apikey` (cloud dispatch) `claude` *is* metered and + those numbers become a real per-benchmark exposure ceiling — noted inline. + + Metered `opencode` lanes keep their existing dollar ceiling as a genuine spend + control and take the token gate as a *second* bound, whichever binds first. + Their comparison stays cost-limited, but now for a stated reason rather than + by accident. Net effect: the metered lane that was over-favoured + (`deepseek-v4-flash`, 3.40M) is pulled into line at 3.00M with no spend change. + +- `Task.MaxTokensPerBench` is now enforced by `codex` and `claude`. The field has + been plumbed to `executor.Task` since M-EVAL-OS-LONGITUDINAL Phase 1, but only + `opencode` ever honoured it — the other executors silently ignored it. Both now + abort on the cumulative-token ceiling, report `ThrashKilledAt`, and set + `FinishThrashAborted`, which outranks `FinishCostExhausted` because on a + subscription lane the dollar figure is notional and "did too much work" is the + truthful reason a run stopped. + +### Added + +- Cost provenance: every cost figure now records whether anyone was actually + charged. `executor.CostProvenance` is one of `metered` (API key, OpenRouter + credits, Vertex ADC), `list-price-equivalent` (real arithmetic over real + tokens on a subscription/OAuth lane — never billed), `free-local` (on-device, + no marginal cost), or `unknown`. Each executor classifies its own auth lane: + `codex` reads `~/.codex/auth.json` (`auth_mode: chatgpt` → subscription), + `claude` mirrors the `AILANG_AUTH_MODE` branch, `managed_agents` is metered + via ADC, and `opencode`/`motoko`/`pi` report provider-billed cost. Zero + resolved rates win over the lane, so a free local model is never labelled as + spend. Standard mode reaches metered HTTP APIs and is labelled accordingly. + + This closes the gap where a subscription run and a metered run were + indistinguishable once both landed in the same column — the v1.0 + `cost-per-verified-success` KPI defines its numerator as attributable + **metered** dollars, and on this rig most agent-mode dollars are not that. + + Surfaced end to end: banked as `cost_provenance` on eval result rows; stored + on chain stages (schema v17, `chain_stages.cost_provenance`); classified by + `ClassifyStageCost` into a new `subscription` status that `CostRollup` keeps + **out** of `TotalKnownCost` and reports separately; and exported as a + `costProvenance` breakdown alongside `totalCostUSD`. An absent label — every + row banked before 2026-07-30 — reads as `unknown`, never as metered; no + provenance is backfilled by guess. `dollars_per_pass`, `p90_cost_per_success` + and `cost_overhead_vs_best` are documented as list-price figures rather than + silently re-scoped. + +### Fixed + +- Agent-mode `cost_usd` now bills the model that actually ran. Executors that + compute cost from token counts were using their own hardcoded table — the + codex executor billed **every** model it ran at gpt-5-codex's `$1.25/$10` per + 1M, so `gpt5-6-luna` and `gpt5-4-mini` were banked at identical rates despite + really costing `$0.20/$1.20` and `$0.75/$4.50`. `Task.Pricing` now carries the + per-model rates from `models.yml` and `executor.ResolveCostModel` prefers them, + falling back to the executor's table only when a caller supplies none (a + present-but-zero rate is honoured, so free local Ollama models are not billed + at cloud prices). Applied to the two executors that compute cost — `codex` and + `managed_agents`; `claude`, `pi`, `motoko` and `opencode` bank the figure their + CLI reports and were never affected, and their `CostModel()` doc comments now + say so. + + The cost-**kill** path was already correct — `executor.CostBudget` is built + from `models.yml` pricing in `agent_runner_multi.go` — which means banked cost + and kill threshold were computed from two different price tables. A v0.30.0 + row shows the split directly: `gpt5-6-luna`/`graph_bfs` banked `$0.34259` at + codex rates while the `$0.30` budget that spared it saw `$0.26980` at luna's. + Do not compare `cost_usd` against `max_cost_usd` on any pre-fix agent row. + +- `codex` HealthCheck no longer treats a missing `OPENAI_API_KEY` as a warning. + Codex authenticates from `~/.codex/auth.json` (`codex login`); an env key does + not override it — probe-verified against codex-cli 0.145.0 with + `auth_mode: "chatgpt"`, where a deliberately invalid key in the environment + still ran clean. Its presence was never evidence that runs are metered. + +### Changed + +- Eval pricing: OpenAI cut the GPT-5.6 family on 2026-07-30 and `models.yml` now + matches. `gpt5-6-luna` -80% (`$1.00/$6.00` → `$0.20/$1.20` per 1M) and + `gpt5-6-terra` -20% (`$2.50/$15.00` → `$2.00/$12.00` per 1M); `gpt5-6-sol` is + unchanged at `$5/$30`. Verified against + `developers.openai.com/api/docs/models/gpt-5.6-{sol,terra,luna}`. Two + consequences are recorded inline: Luna's explicit `max_cost_usd: $0.30` is now + load-bearing (the default formula resolves to `$0.051` at the new prices, down + from `$0.256`), and every banked Terra/Luna cost figure dated before 2026-07-30 + was measured at the old prices, so cross-boundary cost comparisons need + normalizing. Luna is now 3.75x cheaper than the `gpt5-4-mini` incumbent it lost + to on 2026-07-11 — the quality half of that gate (23/26 vs 25/26, N=1) still + stands, so the `dev_models`/`lang_harness_suite` slot is unchanged pending an + N≥3 `--tier core` re-gate. + ## [v0.31.0] - 2026-07-29 ### Added diff --git a/cmd/ailang/eval_benchmark.go b/cmd/ailang/eval_benchmark.go index ecaebf931..f0ba4e336 100644 --- a/cmd/ailang/eval_benchmark.go +++ b/cmd/ailang/eval_benchmark.go @@ -291,6 +291,10 @@ func runSingleBenchmark(ctx context.Context, model, benchmarkID, lang, condition // NOT recomputed from tokens: agent CLIs report their own billed cost, // which already includes reasoning. Deriving it here would double-count. CostUSD: result.Cost, + // Whether that cost was actually billed. The rig's codex and claude + // lanes authenticate by subscription, so a non-zero cost there is a + // list-price equivalent, not spend (see executor.CostProvenance). + CostProvenance: result.CostProvenance, // Use standard validation fields from agent runner CompileOk: result.CompileOk, RuntimeOk: result.RuntimeOk, @@ -417,7 +421,7 @@ func runSingleBenchmark(ctx context.Context, model, benchmarkID, lang, condition _ = evalChain.Store.UpdateStageEvalAssessment(ctx, stageID, assessment) tokensIn := result.Usage.InputTokens + result.Usage.CacheCreationInputTokens + result.Usage.CacheReadInputTokens - _ = evalChain.Store.UpdateStageMetrics(ctx, stageID, result.Cost, tokensIn, result.Usage.OutputTokens, result.NumTurns, result.ToolCallCount, int64(result.DurationMS)) + _ = evalChain.Store.UpdateStageMetrics(ctx, stageID, result.Cost, tokensIn, result.Usage.OutputTokens, result.NumTurns, result.ToolCallCount, int64(result.DurationMS), result.CostProvenance) stageStatus := observatory.StageStatusCompleted if !result.Success { @@ -693,7 +697,7 @@ func runSingleBenchmark(ctx context.Context, model, benchmarkID, lang, condition } _ = evalChain.Store.UpdateStageEvalAssessment(ctx, stageID, assessment) _ = evalChain.Store.UpdateStageMetrics(ctx, stageID, metrics.CostUSD, - metrics.InputTokens, metrics.OutputTokens, 0, 0, metrics.DurationMs) + metrics.InputTokens, metrics.OutputTokens, 0, 0, metrics.DurationMs, metrics.CostProvenance) stageStatus := observatory.StageStatusCompleted if !metrics.StdoutOk { diff --git a/cmd/ailang/eval_suite_finalize.go b/cmd/ailang/eval_suite_finalize.go index 2988e25b2..f3227fb27 100644 --- a/cmd/ailang/eval_suite_finalize.go +++ b/cmd/ailang/eval_suite_finalize.go @@ -79,6 +79,17 @@ func finalizeSuiteRun(p suiteSummaryParams) { } fmt.Println() + // Cost tally, split by whether the dollars were actually charged. A single + // grand total would be misleading here: agent-mode codex and claude run on + // subscriptions, so their cost_usd is real arithmetic over real tokens that + // nobody paid. Best-effort — a cost report must never fail a completed run. + if tally, err := eval_harness.TallyCosts(p.outputDir); err == nil { + if s := tally.Format(); s != "" { + fmt.Print(s) + fmt.Println() + } + } + // M-EVAL-OS-LONGITUDINAL Phase 3: write summary.json that aggregates // per-(benchmark, model, lang, condition) pass rate and token distribution // across trials. Required for Phase 4 candidates command + Phase 5 diff --git a/internal/coordinator/daemon_tasks_chain.go b/internal/coordinator/daemon_tasks_chain.go index 838c6e9c1..8052cb5a4 100644 --- a/internal/coordinator/daemon_tasks_chain.go +++ b/internal/coordinator/daemon_tasks_chain.go @@ -48,7 +48,7 @@ func (d *Daemon) updateStageMetrics(ctx context.Context, task *TaskRecord, resul return } durationMs := result.Duration.Milliseconds() - if err := d.obsBackend.UpdateStageMetrics(ctx, task.StageID, result.Cost, result.InputTokens, result.OutputTokens, result.NumTurns, result.ToolCallCount, durationMs); err != nil { + if err := d.obsBackend.UpdateStageMetrics(ctx, task.StageID, result.Cost, result.InputTokens, result.OutputTokens, result.NumTurns, result.ToolCallCount, durationMs, result.CostProvenance); err != nil { d.logger.Printf("Warning: Failed to update stage %s metrics: %v", task.StageID, err) } } diff --git a/internal/coordinator/provider.go b/internal/coordinator/provider.go index 3d261c381..8214ad401 100644 --- a/internal/coordinator/provider.go +++ b/internal/coordinator/provider.go @@ -105,13 +105,18 @@ func DefaultExecuteOptions() *ExecuteOptions { // ExecuteResult contains the result of task execution type ExecuteResult struct { - Success bool - Output string - Error string - Provider string - Duration time.Duration - Cost float64 - TokensUsed int // Total tokens (InputTokens + OutputTokens) + Success bool + Output string + Error string + Provider string + Duration time.Duration + Cost float64 + // CostProvenance says whether Cost was actually billed — "metered", + // "list-price-equivalent" (subscription lane), "free-local", or "" for + // unclassified. Carried through to the chain stage so cost rollups can + // keep notional and real dollars apart. + CostProvenance string + TokensUsed int // Total tokens (InputTokens + OutputTokens) // Detailed token breakdown InputTokens int OutputTokens int diff --git a/internal/coordinator/provider_executor.go b/internal/coordinator/provider_executor.go index fc5457552..476b2974e 100644 --- a/internal/coordinator/provider_executor.go +++ b/internal/coordinator/provider_executor.go @@ -144,6 +144,7 @@ func (p *ExecutorProvider) Execute(ctx context.Context, task *AnalyzedTask, opts result.Output = execResult.Output result.Error = execResult.Error result.Cost = execResult.CostUSD + result.CostProvenance = string(execResult.CostProvenance) result.InputTokens = execResult.InputTokens result.OutputTokens = execResult.OutputTokens result.TokensUsed = execResult.InputTokens + execResult.OutputTokens diff --git a/internal/eval_analysis/export_json.go b/internal/eval_analysis/export_json.go index c9e078971..a448bb315 100644 --- a/internal/eval_analysis/export_json.go +++ b/internal/eval_analysis/export_json.go @@ -130,7 +130,11 @@ func ExportBenchmarkJSON(matrix *PerformanceMatrix, history []*Baseline, results "cacheCreationTokens": matrix.Aggregates.CacheCreationTokens, "cacheHitRate": matrix.Aggregates.CacheHitRate, "totalCostUSD": matrix.Aggregates.TotalCostUSD, - "avgDurationMs": matrix.Aggregates.AvgDurationMs, + // How those dollars arose. Absent/"unknown" dominates pre-2026-07-30 + // baselines; a large list-price-equivalent share means the cohort's cost + // column is notional, not spend. + "costProvenance": matrix.Aggregates.CostProvenance, + "avgDurationMs": matrix.Aggregates.AvgDurationMs, // Agent metrics (M-EVAL-AGENT) "agentRuns": len(agentResults), "agentSuccessRate": agentSuccessRate, diff --git a/internal/eval_analysis/loader_chains_test.go b/internal/eval_analysis/loader_chains_test.go index d9428c3fc..5e25a292b 100644 --- a/internal/eval_analysis/loader_chains_test.go +++ b/internal/eval_analysis/loader_chains_test.go @@ -129,7 +129,7 @@ func TestLoadResultsFromChain(t *testing.T) { if err := store.UpdateStageEvalAssessment(ctx, stage.ID, assessment); err != nil { t.Fatalf("failed to update assessment: %v", err) } - if err := store.UpdateStageMetrics(ctx, stage.ID, 0.003, 500, 200, 3, 5, 8000); err != nil { + if err := store.UpdateStageMetrics(ctx, stage.ID, 0.003, 500, 200, 3, 5, 8000, ""); err != nil { t.Fatalf("failed to update metrics: %v", err) } } diff --git a/internal/eval_analysis/matrix.go b/internal/eval_analysis/matrix.go index 50d2aade0..c47775b37 100644 --- a/internal/eval_analysis/matrix.go +++ b/internal/eval_analysis/matrix.go @@ -58,6 +58,7 @@ func calculateAggregates(results []*BenchmarkResult) Aggregates { totalDuration := int64(0) cacheRead := 0 cacheCreate := 0 + costProvenance := map[string]int{} for _, r := range results { if r.FirstAttemptOk { @@ -78,6 +79,13 @@ func calculateAggregates(results []*BenchmarkResult) Aggregates { totalDuration += r.DurationMs cacheRead += r.CacheReadInputTokens cacheCreate += r.CacheCreationInputTokens + + // An absent label is unknown provenance, never an assumption of metering. + prov := r.CostProvenance + if prov == "" { + prov = "unknown" + } + costProvenance[prov]++ } agg.ZeroShotSuccess = safeDiv(float64(firstAttemptSuccess), float64(len(results))) @@ -86,6 +94,7 @@ func calculateAggregates(results []*BenchmarkResult) Aggregates { agg.RepairSuccessRate = safeDiv(float64(repairSuccess), float64(repairUsed)) agg.TotalTokens = totalTokens agg.TotalCostUSD = totalCost + agg.CostProvenance = costProvenance agg.AvgDurationMs = safeDiv(float64(totalDuration), float64(len(results))) agg.CacheReadTokens = cacheRead agg.CacheCreationTokens = cacheCreate diff --git a/internal/eval_analysis/sweet_spot.go b/internal/eval_analysis/sweet_spot.go index 4932b4c8e..93f3768c6 100644 --- a/internal/eval_analysis/sweet_spot.go +++ b/internal/eval_analysis/sweet_spot.go @@ -56,12 +56,18 @@ type SweetSpotRow struct { // Efficiency (mirrors EfficiencyAggregates, but flattened for table output). MedianTTSMs float64 `json:"median_tts_ms"` MedianTokensPerSec float64 `json:"median_tokens_per_sec"` - P90CostPerSuccess float64 `json:"p90_cost_per_success"` - SpeedEfficiency float64 `json:"speed_efficiency"` + // P90CostPerSuccess is list-price, same caveat as DollarsPerPass below. + P90CostPerSuccess float64 `json:"p90_cost_per_success"` + SpeedEfficiency float64 `json:"speed_efficiency"` // DollarsPerPass is total $ across all runs / number of passes. The // headline economic metric for the dashboard. 0 when no passes. // M-EVAL-SWEET-SPOT-WEBSITE-INTEGRATION (v0.19.0). + // + // LIST-PRICE dollars, not spend. Agent-mode rows from a subscription lane + // (codex auth_mode chatgpt, claude OAuth) contribute a cost nobody was + // billed. Read alongside Aggregates.CostProvenance before calling this + // money; the v1.0 metered-dollars KPI must not take it at face value. DollarsPerPass float64 `json:"dollars_per_pass"` // CostOverheadVsBest is the median ratio (this_model_cost / best_passer_cost) @@ -71,6 +77,9 @@ type SweetSpotRow struct { // per-benchmark min(CostUSD) across all models that passed THAT benchmark. // Captures "if a perfect router picked the cheapest model per benchmark, how // much more would this model cost than that router?" + // + // Ratio of LIST-PRICE figures. It stays meaningful across provenances (both + // sides are list price), but it does NOT say the overhead was paid. CostOverheadVsBest float64 `json:"cost_overhead_vs_best"` // TokenOverheadVsBest is the same shape but for TotalTokens. Distinguishes diff --git a/internal/eval_analysis/types.go b/internal/eval_analysis/types.go index 5c6c2b78f..048ce8eb8 100644 --- a/internal/eval_analysis/types.go +++ b/internal/eval_analysis/types.go @@ -25,13 +25,18 @@ type BenchmarkResult struct { CacheReadInputTokens int `json:"cache_read_input_tokens,omitempty"` CacheCreationInputTokens int `json:"cache_creation_input_tokens,omitempty"` CostUSD float64 `json:"cost_usd"` - CompileOk bool `json:"compile_ok"` - RuntimeOk bool `json:"runtime_ok"` - StdoutOk bool `json:"stdout_ok"` - DurationMs int64 `json:"duration_ms"` - CompileMs int64 `json:"compile_ms"` - ExecuteMs int64 `json:"execute_ms"` - ErrorCategory string `json:"error_category"` + // CostProvenance says whether CostUSD was actually billed: "metered", + // "list-price-equivalent" (subscription lane — real arithmetic, zero spend), + // "free-local", or "unknown". Absent in baselines banked before 2026-07-30, + // where it reads "" — unmeasured, NOT metered. + CostProvenance string `json:"cost_provenance,omitempty"` + CompileOk bool `json:"compile_ok"` + RuntimeOk bool `json:"runtime_ok"` + StdoutOk bool `json:"stdout_ok"` + DurationMs int64 `json:"duration_ms"` + CompileMs int64 `json:"compile_ms"` + ExecuteMs int64 `json:"execute_ms"` + ErrorCategory string `json:"error_category"` // Validity marks whether this row is a MEASUREMENT at all (vs a failure to // measure: dead subject, harness error, wrong config). NIL means valid — @@ -172,8 +177,16 @@ type Aggregates struct { RepairUsed int `json:"repair_used"` // Number of repairs attempted RepairSuccessRate float64 `json:"repair_success_rate"` // Repair success rate TotalTokens int `json:"total_tokens"` - TotalCostUSD float64 `json:"total_cost_usd"` - AvgDurationMs float64 `json:"avg_duration_ms"` + // TotalCostUSD sums cost_usd across EVERY run in the cohort regardless of + // provenance, so it is a LIST-PRICE total, not spend. On a subscription rig + // most agent-mode dollars here were never billed. Use CostProvenance to see + // the split before quoting this as money. + TotalCostUSD float64 `json:"total_cost_usd"` + // CostProvenance counts runs by how their cost arose (metered / + // list-price-equivalent / free-local / unknown). "unknown" includes every + // row banked before 2026-07-30, when the label did not exist. + CostProvenance map[string]int `json:"cost_provenance,omitempty"` + AvgDurationMs float64 `json:"avg_duration_ms"` // Prompt-cache aggregates. CacheHitRate is cache reads as a share of ALL // input tokens the model saw (reads + writes + uncached), so it answers "how // much of our input did we avoid paying full price for". 0 means either no diff --git a/internal/eval_harness/agent_runner.go b/internal/eval_harness/agent_runner.go index 9e38d7fcd..e3d56f9d6 100644 --- a/internal/eval_harness/agent_runner.go +++ b/internal/eval_harness/agent_runner.go @@ -79,14 +79,18 @@ func (c AgentBenchmarkConfig) ResolvedVerifyTimeout() time.Duration { // AgentBenchmarkResult captures agent evaluation outcome type AgentBenchmarkResult struct { - BenchmarkID string - Executor string // Executor used: "claude", "gemini", etc. - Success bool - Iterations int // Number of agent turns - Cost float64 // Total cost in USD - DurationMS int // Total time in milliseconds - NumTurns int // Conversation turns - ToolCallCount int // Number of tool invocations (validates agentic behavior) + BenchmarkID string + Executor string // Executor used: "claude", "gemini", etc. + Success bool + Iterations int // Number of agent turns + Cost float64 // Total cost in USD + // CostProvenance says whether Cost is money anyone was charged + // (metered | list-price-equivalent | free-local | unknown). Empty means + // the executor did not classify it; read as unknown, never as metered. + CostProvenance string + DurationMS int // Total time in milliseconds + NumTurns int // Conversation turns + ToolCallCount int // Number of tool invocations (validates agentic behavior) // ToolCalls is the per-tool-name histogram (nil if the executor didn't capture // names). Answers "did the agent actually invoke `ailang fmt`" — which the // scalar ToolCallCount cannot. diff --git a/internal/eval_harness/agent_runner_multi.go b/internal/eval_harness/agent_runner_multi.go index e9e15df5e..556cf5957 100644 --- a/internal/eval_harness/agent_runner_multi.go +++ b/internal/eval_harness/agent_runner_multi.go @@ -281,6 +281,21 @@ func RunAgentBenchmarkWithExecutor(spec *BenchmarkSpec, config MultiExecutorConf if maxCost := cfg.ResolvedMaxCostUSD(); maxCost > 0 { task.Budget = executor.NewCostBudget(maxCost, cfg.Pricing.InputPer1K, cfg.Pricing.OutputPer1K) } + // Report cost at the SAME per-model rates the budget enforces at. + // Set unconditionally: Budget exists only when enforcement is on + // (maxCost > 0), but a cost_usd is banked either way, and before + // this the two came from different price tables — a codex row could + // bank $0.34 while the budget that spared it saw $0.27. + task.Pricing = &executor.CostModel{ + ProviderName: cfg.Provider, + InputTokenCost: cfg.Pricing.InputPer1K, + OutputTokenCost: cfg.Pricing.OutputPer1K, + } + // The WORK gate. A per-model budgets:max_tokens_per_bench overrides + // the global --max-tokens-per-bench flag; this is what makes the + // agent suite comparable, because the dollar gate above buys work + // in inverse proportion to price (see Budgets doc). + task.MaxTokensPerBench = cfg.ResolvedMaxTokensPerBench(config.MaxTokensPerBench) // M-EVAL-LOCAL-OLLAMA (v0.22.0): take the MAX of spec.Timeout and // model.HardTimeoutSecs rather than letting spec.Timeout veto. The // benchmark spec timeout is cloud-tuned (Sonnet 4.6 speeds); local @@ -422,6 +437,7 @@ func RunAgentBenchmarkWithExecutor(spec *BenchmarkSpec, config MultiExecutorConf Success: success, Iterations: result.NumTurns, Cost: result.CostUSD, + CostProvenance: string(result.CostProvenance), DurationMS: result.DurationMS, NumTurns: result.NumTurns, ToolCallCount: result.ToolCallCount, diff --git a/internal/eval_harness/cost_provenance_test.go b/internal/eval_harness/cost_provenance_test.go new file mode 100644 index 000000000..68fb40747 --- /dev/null +++ b/internal/eval_harness/cost_provenance_test.go @@ -0,0 +1,148 @@ +package eval_harness + +import ( + "encoding/json" + "testing" + + "github.com/sunholo-data/ailang/internal/executor" +) + +// TestStandardModeCostProvenance: standard mode reaches providers over metered +// HTTP APIs, so a priced model is genuinely billed — unlike agent mode, where +// the codex/claude CLIs run on subscriptions. Unresolvable must not guess. +func TestStandardModeCostProvenance(t *testing.T) { + saved := GlobalModelsConfig + t.Cleanup(func() { GlobalModelsConfig = saved }) + + GlobalModelsConfig = &ModelsConfig{Models: map[string]ModelConfig{ + "gpt5-6-luna": {Pricing: Pricing{InputPer1K: 0.0002, OutputPer1K: 0.0012}}, + "local-gemma": {Pricing: Pricing{}}, + }} + + tests := []struct{ model, want string }{ + {"gpt5-6-luna", string(executor.CostMetered)}, + {"local-gemma", string(executor.CostFreeLocal)}, + {"never-heard-of-it", string(executor.CostProvenanceUnknown)}, + } + for _, tt := range tests { + if got := standardModeCostProvenance(tt.model); got != tt.want { + t.Errorf("%s: got %q, want %q", tt.model, got, tt.want) + } + } + + GlobalModelsConfig = nil + if got := standardModeCostProvenance("gpt5-6-luna"); got != string(executor.CostProvenanceUnknown) { + t.Errorf("no config loaded: got %q, want unknown", got) + } +} + +// TestRunMetrics_CostProvenanceRoundTrip pins the banked JSON contract: the +// label ships with the row, and its ABSENCE on a pre-2026-07-30 baseline +// decodes as unknown rather than silently reading as metered. +func TestRunMetrics_CostProvenanceRoundTrip(t *testing.T) { + saved := GlobalModelsConfig + t.Cleanup(func() { GlobalModelsConfig = saved }) + GlobalModelsConfig = &ModelsConfig{Models: map[string]ModelConfig{ + "gpt5-6-luna": {Pricing: Pricing{InputPer1K: 0.0002, OutputPer1K: 0.0012}}, + }} + + m := NewRunMetrics("fizzbuzz", "ailang", "gpt5-6-luna", 42) + blob, err := json.Marshal(m) + if err != nil { + t.Fatalf("marshal: %v", err) + } + var back RunMetrics + if err := json.Unmarshal(blob, &back); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if back.CostProvenance != string(executor.CostMetered) { + t.Errorf("round-tripped provenance = %q, want %q", back.CostProvenance, executor.CostMetered) + } + + // A pre-fix banked row has no cost_provenance key at all. + var legacy RunMetrics + if err := json.Unmarshal([]byte(`{"id":"fizzbuzz","cost_usd":0.34259375}`), &legacy); err != nil { + t.Fatalf("legacy unmarshal: %v", err) + } + if legacy.CostProvenance != "" { + t.Errorf("legacy row provenance = %q, want empty (reads as unknown)", legacy.CostProvenance) + } +} + +// TestResolvedMaxTokensPerBench: the per-model work gate overrides the global +// --max-tokens-per-bench flag, and 0 from both means no enforcement. +func TestResolvedMaxTokensPerBench(t *testing.T) { + tests := []struct { + name string + perModel int + flag int + want int + }{ + {"per-model wins over the flag", 3_000_000, 500_000, 3_000_000}, + {"flag applies when no per-model budget", 0, 500_000, 500_000}, + {"both unset = unlimited", 0, 0, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + m := &ModelConfig{Budgets: Budgets{MaxTokensPerBench: tt.perModel}} + if got := m.ResolvedMaxTokensPerBench(tt.flag); got != tt.want { + t.Errorf("got %d, want %d", got, tt.want) + } + }) + } +} + +// TestAgentSuiteGatesAreConsistent is the guard for the 2026-07-31 audit: a +// dollar-only gate buys work in inverse proportion to price, which let the +// suite span 24x in actual tokens and left claude-sonnet-4-6 — the longitudinal +// ANCHOR — with the tightest budget of all six. Every agent_suite model must +// now carry an explicit token gate, and the subscription lanes must all land on +// the SAME effective work ceiling. +func TestAgentSuiteGatesAreConsistent(t *testing.T) { + cfg, err := LoadModelsConfig("models.yml") + if err != nil { + t.Fatalf("load models.yml: %v", err) + } + subscription := map[string]bool{"codex": true, "claude": true} + var subWork []float64 + + for _, name := range cfg.AgentSuite { + m, ok := cfg.Models[name] + if !ok { + t.Fatalf("agent_suite references unknown model %q", name) + } + tok := m.Budgets.MaxTokensPerBench + if tok == 0 { + t.Errorf("%s: no max_tokens_per_bench — a dollar-only gate is not a work gate", name) + continue + } + // Blended rate at the ~95/5 input/output mix agent runs actually show. + blended := m.Pricing.InputPer1K*0.95 + m.Pricing.OutputPer1K*0.05 + byCost := m.ResolvedMaxCostUSD() / blended * 1000 + binds := byCost + if float64(tok) < binds { + binds = float64(tok) + } + cli := "" + if m.AgentCLI != nil { + cli = *m.AgentCLI + } + if subscription[cli] { + // No spend to control here, so the token gate MUST be the binding one. + if byCost < float64(tok) { + t.Errorf("%s: dollar cap binds first (%.2fM tok) — token gate never fires on a subscription lane", + name, byCost/1e6) + } + subWork = append(subWork, binds) + } + } + for i := 1; i < len(subWork); i++ { + if subWork[i] != subWork[0] { + t.Errorf("subscription lanes disagree on work ceiling: %.2fM vs %.2fM", + subWork[0]/1e6, subWork[i]/1e6) + } + } + if len(subWork) < 2 { + t.Fatalf("expected at least 2 subscription-lane models in agent_suite, got %d", len(subWork)) + } +} diff --git a/internal/eval_harness/cost_tally.go b/internal/eval_harness/cost_tally.go new file mode 100644 index 000000000..9289e6d3e --- /dev/null +++ b/internal/eval_harness/cost_tally.go @@ -0,0 +1,179 @@ +package eval_harness + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/sunholo-data/ailang/internal/executor" +) + +// CostTally is the end-of-run cost report for a suite, split by whether the +// dollars were actually charged. +// +// The split is the whole point. A single grand total is misleading on this rig: +// agent-mode codex and claude authenticate by subscription, so their cost_usd is +// real arithmetic over real tokens that nobody paid. Summing that with metered +// OpenRouter/OpenAI spend produces a number that is neither a bill nor a +// list price. See executor.CostProvenance. +type CostTally struct { + // Metered is spend an account was genuinely charged for — the only figure + // that answers "what did this run cost us". + Metered float64 `json:"metered_usd"` + // ListPriceEquivalent is what the subscription lanes would have cost at list + // price. Useful for comparing models on equal terms; not money. + ListPriceEquivalent float64 `json:"list_price_equivalent_usd"` + // FreeLocal is always 0 by construction (on-device models have no marginal + // token cost); the run count is what carries the signal. + FreeLocalRuns int `json:"free_local_runs"` + // UnknownRuns counts rows with no provenance label — every row banked before + // 2026-07-30, plus any executor that could not classify its auth lane. Their + // dollars are held in UnknownCost rather than silently added to Metered. + UnknownRuns int `json:"unknown_runs"` + UnknownCost float64 `json:"unknown_usd"` + + TotalRuns int `json:"total_runs"` + ByMode map[string]int `json:"runs_by_mode"` + perModel map[string]*modelTally + ModelCount int `json:"model_count"` +} + +type modelTally struct { + model string + cost float64 + runs int + provenance string +} + +// TallyCosts walks a finished suite's banked result files and aggregates cost by +// provenance. Malformed files are skipped — a cost report must never be the +// thing that fails a completed run. +func TallyCosts(outputDir string) (*CostTally, error) { + files, err := resultFilesIn(outputDir) + if err != nil { + return nil, err + } + t := &CostTally{ByMode: map[string]int{}, perModel: map[string]*modelTally{}} + for _, f := range files { + data, err := os.ReadFile(f) + if err != nil { + continue + } + var m RunMetrics + if err := json.Unmarshal(data, &m); err != nil { + continue + } + t.TotalRuns++ + mode := m.EvalMode + if mode == "" { + mode = "standard" + } + t.ByMode[mode]++ + + switch m.CostProvenance { + case string(executor.CostMetered): + t.Metered += m.CostUSD + case string(executor.CostListPriceEquivalent): + t.ListPriceEquivalent += m.CostUSD + case string(executor.CostFreeLocal): + t.FreeLocalRuns++ + default: + // Absent or unrecognised. Held apart deliberately: assuming metered + // is exactly the error the provenance field exists to prevent. + t.UnknownRuns++ + t.UnknownCost += m.CostUSD + } + + key := m.Model + "\x00" + mode + mt := t.perModel[key] + if mt == nil { + mt = &modelTally{model: m.Model + " (" + mode + ")"} + t.perModel[key] = mt + } + mt.cost += m.CostUSD + mt.runs++ + if mt.provenance == "" { + mt.provenance = m.CostProvenance + } + } + t.ModelCount = len(t.perModel) + return t, nil +} + +// Format renders the tally for the end of a suite run. Returns "" when there is +// nothing to report, so callers can print unconditionally. +func (t *CostTally) Format() string { + if t == nil || t.TotalRuns == 0 { + return "" + } + var b strings.Builder + b.WriteString("Cost tally\n") + + rows := make([]*modelTally, 0, len(t.perModel)) + for _, mt := range t.perModel { + rows = append(rows, mt) + } + sort.Slice(rows, func(i, j int) bool { return rows[i].cost > rows[j].cost }) + for _, mt := range rows { + label := provenanceLabel(mt.provenance) + fmt.Fprintf(&b, " %-38s %8.4f %4d runs %s\n", mt.model, mt.cost, mt.runs, label) + } + + fmt.Fprintf(&b, " %s\n", strings.Repeat("-", 72)) + fmt.Fprintf(&b, " %-38s $%.2f\n", "METERED (actually billed)", t.Metered) + if t.ListPriceEquivalent > 0 { + fmt.Fprintf(&b, " %-38s $%.2f subscription — not billed\n", + "list-price-equivalent", t.ListPriceEquivalent) + } + if t.UnknownRuns > 0 { + fmt.Fprintf(&b, " %-38s $%.2f %d runs unlabelled — NOT counted as spend\n", + "unknown provenance", t.UnknownCost, t.UnknownRuns) + } + if t.FreeLocalRuns > 0 { + fmt.Fprintf(&b, " %-38s %d runs, no marginal cost\n", "free-local", t.FreeLocalRuns) + } + return b.String() +} + +func provenanceLabel(p string) string { + switch p { + case string(executor.CostMetered): + return "metered" + case string(executor.CostListPriceEquivalent): + return "list-price-equiv" + case string(executor.CostFreeLocal): + return "free-local" + default: + return "unknown" + } +} + +// resultFilesIn returns every banked result JSON under outputDir, skipping the +// summary files the suite writes alongside them. Mirrors SummarizeRotation's +// walk so the two never disagree about what counts as a result. +func resultFilesIn(outputDir string) ([]string, error) { + var files []string + for _, mode := range []string{"standard", "agent"} { + direct, _ := filepath.Glob(filepath.Join(outputDir, mode, "*.json")) + files = append(files, direct...) + condDirs, _ := filepath.Glob(filepath.Join(outputDir, mode, "*")) + for _, cd := range condDirs { + if info, err := os.Stat(cd); err == nil && info.IsDir() { + condFiles, _ := filepath.Glob(filepath.Join(cd, "*.json")) + files = append(files, condFiles...) + } + } + } + out := files[:0] + for _, f := range files { + base := filepath.Base(f) + if base == "summary.json" || base == "baseline.json" { + continue + } + out = append(out, f) + } + return out, nil +} diff --git a/internal/eval_harness/cost_tally_test.go b/internal/eval_harness/cost_tally_test.go new file mode 100644 index 000000000..cadeaafcf --- /dev/null +++ b/internal/eval_harness/cost_tally_test.go @@ -0,0 +1,104 @@ +package eval_harness + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/sunholo-data/ailang/internal/executor" +) + +func writeRow(t *testing.T, dir, name string, m RunMetrics) { + t.Helper() + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + b, err := json.Marshal(m) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, name+".json"), b, 0o644); err != nil { + t.Fatal(err) + } +} + +// TestTallyCosts_SplitsByProvenance is the guard on the headline number: a +// subscription lane's cost must never be added to metered spend, and an +// unlabelled legacy row must not be assumed metered either. +func TestTallyCosts_SplitsByProvenance(t *testing.T) { + root := t.TempDir() + writeRow(t, filepath.Join(root, "standard"), "a", RunMetrics{ + Model: "gpt5-6-sol", CostUSD: 2.00, CostProvenance: string(executor.CostMetered)}) + writeRow(t, filepath.Join(root, "agent"), "b", RunMetrics{ + Model: "gpt5-6-luna", EvalMode: "agent", CostUSD: 8.36, + CostProvenance: string(executor.CostListPriceEquivalent)}) + writeRow(t, filepath.Join(root, "agent"), "c", RunMetrics{ + Model: "local-gemma", EvalMode: "agent", CostUSD: 0, + CostProvenance: string(executor.CostFreeLocal)}) + // A row banked before the label existed. + writeRow(t, filepath.Join(root, "standard"), "d", RunMetrics{ + Model: "or-glm-5-2", CostUSD: 1.50}) + + tally, err := TallyCosts(root) + if err != nil { + t.Fatalf("TallyCosts: %v", err) + } + if tally.Metered != 2.00 { + t.Errorf("Metered = %v, want 2.00 (subscription and unknown must stay out)", tally.Metered) + } + if tally.ListPriceEquivalent != 8.36 { + t.Errorf("ListPriceEquivalent = %v, want 8.36", tally.ListPriceEquivalent) + } + if tally.UnknownRuns != 1 || tally.UnknownCost != 1.50 { + t.Errorf("unknown = %d runs / $%v, want 1 / $1.50 — held apart, not summed into metered", + tally.UnknownRuns, tally.UnknownCost) + } + if tally.FreeLocalRuns != 1 { + t.Errorf("FreeLocalRuns = %d, want 1", tally.FreeLocalRuns) + } + if tally.TotalRuns != 4 { + t.Errorf("TotalRuns = %d, want 4", tally.TotalRuns) + } + if got := tally.ByMode["agent"]; got != 2 { + t.Errorf("agent runs = %d, want 2", got) + } + + out := tally.Format() + for _, want := range []string{"METERED (actually billed)", "$2.00", "subscription — not billed", "NOT counted as spend"} { + if !strings.Contains(out, want) { + t.Errorf("Format() missing %q:\n%s", want, out) + } + } +} + +// TestTallyCosts_EmptyDirIsSilent: a run that banked nothing must not print a +// tally, and must not error. +func TestTallyCosts_EmptyDirIsSilent(t *testing.T) { + tally, err := TallyCosts(t.TempDir()) + if err != nil { + t.Fatalf("TallyCosts on empty dir: %v", err) + } + if s := tally.Format(); s != "" { + t.Errorf("Format() on empty tally = %q, want empty", s) + } +} + +// TestTallyCosts_SkipsMalformed: a corrupt result file must not take down the +// cost report at the end of an otherwise-complete run. +func TestTallyCosts_SkipsMalformed(t *testing.T) { + root := t.TempDir() + dir := filepath.Join(root, "standard") + writeRow(t, dir, "good", RunMetrics{Model: "m", CostUSD: 1, CostProvenance: string(executor.CostMetered)}) + if err := os.WriteFile(filepath.Join(dir, "bad.json"), []byte("{not json"), 0o644); err != nil { + t.Fatal(err) + } + tally, err := TallyCosts(root) + if err != nil { + t.Fatalf("TallyCosts: %v", err) + } + if tally.TotalRuns != 1 || tally.Metered != 1 { + t.Errorf("got %d runs / $%v, want 1 / $1 (malformed row skipped)", tally.TotalRuns, tally.Metered) + } +} diff --git a/internal/eval_harness/metrics.go b/internal/eval_harness/metrics.go index a621eee80..24e39cb9f 100644 --- a/internal/eval_harness/metrics.go +++ b/internal/eval_harness/metrics.go @@ -7,6 +7,8 @@ import ( "path/filepath" "strings" "time" + + "github.com/sunholo-data/ailang/internal/executor" ) // RunMetrics captures the results of a single benchmark run @@ -27,22 +29,31 @@ type RunMetrics struct { // // omitempty keeps pre-v0.31.0 baselines parsing unchanged (absent reads as 0) // and keeps rows for providers without cache reporting free of noise. - CacheReadInputTokens int `json:"cache_read_input_tokens,omitempty"` - CacheCreationInputTokens int `json:"cache_creation_input_tokens,omitempty"` - TotalTokens int `json:"total_tokens"` // Total for billing (includes reasoning) - CostUSD float64 `json:"cost_usd"` - CompileOk bool `json:"compile_ok"` - RuntimeOk bool `json:"runtime_ok"` - StdoutOk bool `json:"stdout_ok"` - DurationMs int64 `json:"duration_ms"` // Total time (startup + compile + execution) - CompileMs int64 `json:"compile_ms"` // Time spent in compilation (if separate) - ExecuteMs int64 `json:"execute_ms"` // Time spent in execution (if measurable) - ErrorCategory string `json:"error_category"` // compile_error | runtime_error | logic_error | none - Stdout string `json:"stdout,omitempty"` - Stderr string `json:"stderr,omitempty"` - ExpectedStdout string `json:"expected_stdout,omitempty"` - Timestamp time.Time `json:"timestamp"` - Code string `json:"code,omitempty"` // Generated code (optional, for debugging) + CacheReadInputTokens int `json:"cache_read_input_tokens,omitempty"` + CacheCreationInputTokens int `json:"cache_creation_input_tokens,omitempty"` + TotalTokens int `json:"total_tokens"` // Total for billing (includes reasoning) + CostUSD float64 `json:"cost_usd"` + // CostProvenance labels CostUSD: "metered" (an account was genuinely + // charged), "list-price-equivalent" (real arithmetic over real tokens, but + // a subscription/OAuth lane covered the run and nobody was billed), + // "free-local" (on-device, no marginal cost), or "unknown". + // + // Added 2026-07-30. omitempty keeps earlier baselines parsing unchanged — + // but an ABSENT value means unmeasured, NOT metered. Any aggregate that + // claims metered dollars must filter on this, not assume it. + CostProvenance string `json:"cost_provenance,omitempty"` + CompileOk bool `json:"compile_ok"` + RuntimeOk bool `json:"runtime_ok"` + StdoutOk bool `json:"stdout_ok"` + DurationMs int64 `json:"duration_ms"` // Total time (startup + compile + execution) + CompileMs int64 `json:"compile_ms"` // Time spent in compilation (if separate) + ExecuteMs int64 `json:"execute_ms"` // Time spent in execution (if measurable) + ErrorCategory string `json:"error_category"` // compile_error | runtime_error | logic_error | none + Stdout string `json:"stdout,omitempty"` + Stderr string `json:"stderr,omitempty"` + ExpectedStdout string `json:"expected_stdout,omitempty"` + Timestamp time.Time `json:"timestamp"` + Code string `json:"code,omitempty"` // Generated code (optional, for debugging) // ResolvedProfile / ResolvedExtensions record what the SUBJECT reports it // actually loaded, from its own step-0 broadcast — not what we asked for. @@ -325,16 +336,38 @@ func CalculateCostWithBreakdown(model string, inputTokens, outputTokens int) flo return cost } +// standardModeCostProvenance classifies a standard-mode row's cost. +// +// Standard mode reaches providers over their metered HTTP APIs via an API key +// (internal/ai), so a priced model is genuinely billed — unlike agent mode, +// where the codex and claude CLIs run on subscriptions. A zero-rate model is +// on-device and free. An unresolvable model yields unknown rather than a guess, +// matching CalculateCostWithBreakdown's no-silent-fallback stance. +func standardModeCostProvenance(model string) string { + if GlobalModelsConfig == nil { + return string(executor.CostProvenanceUnknown) + } + cfg, ok := GlobalModelsConfig.Models[model] + if !ok { + return string(executor.CostProvenanceUnknown) + } + if cfg.Pricing.InputPer1K == 0 && cfg.Pricing.OutputPer1K == 0 { + return string(executor.CostFreeLocal) + } + return string(executor.CostMetered) +} + // NewRunMetrics creates a new RunMetrics with timestamp and error category. // MicroragState is auto-populated from the inherited env so every metrics // emission honours the eval-suite --microrag flag (M-BRAIN-MICRORAG). func NewRunMetrics(id, lang, model string, seed int64) *RunMetrics { return &RunMetrics{ - ID: id, - Lang: lang, - Model: model, - Seed: seed, - Timestamp: time.Now(), - MicroragState: MicroragModeAuto.ResolvedState(), + ID: id, + Lang: lang, + Model: model, + Seed: seed, + Timestamp: time.Now(), + MicroragState: MicroragModeAuto.ResolvedState(), + CostProvenance: standardModeCostProvenance(model), } } diff --git a/internal/eval_harness/models.go b/internal/eval_harness/models.go index 78b1327f1..5774cbe2a 100644 --- a/internal/eval_harness/models.go +++ b/internal/eval_harness/models.go @@ -54,13 +54,45 @@ type Pricing struct { // // Cost is the primary gate; wall-clock HardTimeoutSecs is a safety net for // hung connections, not a cost proxy. +// +// MaxTokensPerBench is the WORK gate, and it exists because a dollar gate is +// not one. A dollar ceiling buys work in inverse proportion to model price, so +// a flat $0.30 across the agent suite spanned 24x in actual tokens (audited +// 2026-07-31): claude-sonnet-4-6 — the longitudinal ANCHOR — got 0.14M tokens +// while opencode-or-deepseek-v4-flash got 3.40M. That silently handed the +// weakest models the most iteration and the reference model the least, which +// is backwards for a suite whose question is "does the agent loop rescue weak +// models". It also means a vendor price change silently re-scopes the gate, +// which is exactly what OpenAI's 2026-07-30 cut did to gpt5-6-luna. +// +// Policy (Mark, 2026-07-31): subscription lanes (codex, claude-on-OAuth) gate +// on TOKENS alone — equal work, and no spend to control. Metered lanes keep a +// dollar ceiling as a real spend control WITH the token gate as a second bound; +// whichever binds first wins. Their comparison stays confounded, but for a +// stated reason rather than by accident. type Budgets struct { MaxCostUSD float64 `yaml:"max_cost_usd"` + MaxTokensPerBench int `yaml:"max_tokens_per_bench"` HardTimeoutSecs int `yaml:"hard_timeout_secs"` ExpectedTTFTSecs int `yaml:"expected_ttft_secs"` ExpectedTTFSolutionSecs int `yaml:"expected_ttf_solution_secs"` } +// ResolvedMaxTokensPerBench returns the effective cumulative-token ceiling for +// a model: the explicit per-model budget when set, otherwise flagFallback (the +// global --max-tokens-per-bench value, 0 = unlimited). +// +// Per-model wins because the flag is one number for a whole run, and the point +// of the token gate is that every model in a cohort gets the SAME work — which +// a shared flag already gives, but a mixed cohort (cloud + local) may want to +// split. 0 from both = no token enforcement (legacy behaviour). +func (m *ModelConfig) ResolvedMaxTokensPerBench(flagFallback int) int { + if m.Budgets.MaxTokensPerBench > 0 { + return m.Budgets.MaxTokensPerBench + } + return flagFallback +} + // ResolvedMaxCostUSD returns the effective cost ceiling for a model: // the explicit Budgets.MaxCostUSD when set, otherwise the default formula. // Returns 0 if pricing is also zero (free local models — no enforcement). diff --git a/internal/eval_harness/models.yml b/internal/eval_harness/models.yml index 2597ca94d..1783f9a14 100644 --- a/internal/eval_harness/models.yml +++ b/internal/eval_harness/models.yml @@ -249,6 +249,15 @@ models: # knowledge cutoff Feb 16 2026, programmatic tool calling (Responses API), # explicit prompt-cache breakpoints (30-min minimum cache life), # reasoning.effort none/low/medium/high/xhigh/max. + # + # 💰 PRICE CUT 2026-07-30 (OpenAI, ~3 weeks post-launch): Luna -80% + # ($1/$6 → $0.20/$1.20), Terra -20% ($2.50/$15 → $2/$12), Sol UNCHANGED + # ($5/$30). Cached input is 10% of input across all three tiers + # (Sol $0.50, Terra $0.20, Luna $0.02 per 1M); cache WRITES bill at 1.25x + # uncached input. All tiers: >272K-token prompts bill at 2x input / 1.5x + # output for the whole request. Source: developers.openai.com/api/docs/models/. + # Any banked Terra/Luna cost figure dated before 2026-07-30 used the old + # prices — normalize before comparing across that boundary. gpt5-6-sol: api_name: "gpt-5.6-sol" provider: "openai" @@ -262,7 +271,10 @@ models: output_per_1k: 0.030 # $30.00 per 1M = $0.030 per 1K notes: | GPT-5.6 Sol — flagship tier of the GPT-5.6 family (July 9, 2026). - Same pricing as gpt-5.5 ($5/$30 per 1M). + Same pricing as gpt-5.5 ($5/$30 per 1M) — NOT cut on 2026-07-30 when + Terra/Luna were. That announcement did add a "fast" service tier for Sol + (up to 2.5x faster) at 2x the price; this row is standard-tier pricing, so + if a fast-tier Sol is ever wired up it needs its own entry at $10/$60. Gated 2026-07-11: smoke 21/23 (anchor-normalized clean — sonnet-4-6 also failed the same 2), core 26/26 N=1 → 25/26 after N=3 de-flake of explicit_dataflow_ssa (both Sol and gpt5-5 fail it 3/3). Core TIE with @@ -281,15 +293,22 @@ models: agent_cli: "codex" max_output_tokens: 65536 # M-EVAL-TOKEN-HEADROOM: was 128000 pricing: - input_per_1k: 0.0025 # $2.50 per 1M = $0.0025 per 1K - output_per_1k: 0.015 # $15.00 per 1M = $0.015 per 1K + # PRICE CUT 2026-07-30 (-20%): was $2.50/$15 per 1M. + input_per_1k: 0.002 # $2.00 per 1M = $0.002 per 1K + output_per_1k: 0.012 # $12.00 per 1M = $0.012 per 1K notes: | GPT-5.6 Terra — balanced tier (July 9, 2026). OpenAI claims competitive - with GPT-5.5 at 2x cheaper. Same pricing as gpt-5.4 ($2.50/$15 per 1M). + with GPT-5.5 at 2x cheaper. + 💰 2026-07-30: OpenAI cut Terra 20% → $2/$12 per 1M (cached input $0.20/1M, + per developers.openai.com/api/docs/models/gpt-5.6-terra). Now cheaper than + gpt-5.4 ($2.50/$15), which it previously matched. Gated 2026-07-11: smoke 21/23 (anchor-normalized clean), core 24/26 at $1.82 — ties the claude-sonnet-4-6 anchor (24/26, $2.52) at ~70% of its cost, 1 behind gpt5-5/sol. Strong value. In extended_suite since 2026-07-11 (new-release tracking, per Mark). + ⚠️ That $1.82 was measured at PRE-CUT prices — the same run is ~$1.46 now + (~58% of the anchor). Do not compare post-07-30 costs against pre-07-30 + banked Terra costs without normalizing. Uses max_completion_tokens instead of max_tokens. gpt5-6-luna: @@ -301,19 +320,71 @@ models: agent_cli: "codex" max_output_tokens: 65536 # M-EVAL-TOKEN-HEADROOM: was 128000 pricing: - input_per_1k: 0.001 # $1.00 per 1M = $0.001 per 1K - output_per_1k: 0.006 # $6.00 per 1M = $0.006 per 1K + # PRICE CUT 2026-07-30 (-80%): was $1.00/$6.00 per 1M. + input_per_1k: 0.0002 # $0.20 per 1M = $0.0002 per 1K + output_per_1k: 0.0012 # $1.20 per 1M = $0.0012 per 1K budgets: - max_cost_usd: 0.30 # tier parity: every other cheap-tier agent model has an - hard_timeout_secs: 600 # explicit $0.30; the default formula gave luna only $0.256 - # and cost-killed 3 agent benchmarks in the v0.30.0 baseline + # RAISED 2026-07-31 (Mark): $0.30 -> $0.75. NOT a spend increase — codex on this + # rig authenticates by ChatGPT subscription (auth_mode: chatgpt), so this gate buys + # WORK, not dollars. At the post-07-30 rates $0.75 is ~3.0M blended tokens, sized + # from the largest observed completion profile for the benchmarks luna was killed + # on (gauntlet_10, ~2.26M) plus ~33% headroom. + # + # Why the old $0.30 was wrong TWICE over: it cost-killed 18 of 56 agent benchmarks + # in the v0.30.0 baseline (all 19 failures were the cap, not the model — see notes), + # and it was calibrated when luna cost 5x more, so the price cut silently changed + # how much work it bought. Never read this number as spend; read it as tokens. + # Do NOT delete the override: the default formula resolves to $0.051 at these rates. + # WORK GATE (2026-07-31, Mark): one shared 3.0M blended-token ceiling across + # agent_suite, so every model gets the same number of iterations. A dollar cap + # cannot do this — it buys work in inverse proportion to price, which is how a + # flat $0.30/$0.50 came to span 24x in actual tokens across this suite. + max_tokens_per_bench: 3000000 + # $0.75 is the dollar equivalent of that 3.0M gate at the post-07-30 rates, + # kept as a redundant backstop. Subscription lane — not spend. + max_cost_usd: 0.75 + hard_timeout_secs: 600 notes: | GPT-5.6 Luna — fast/affordable tier (July 9, 2026). + 💰 2026-07-30: OpenAI cut Luna 80% → $0.20/$1.20 per 1M (cached input + $0.02/1M, per developers.openai.com/api/docs/models/gpt-5.6-luna). Gated 2026-07-11: smoke 21/23 (anchor-normalized clean), core 23/26 at $0.73 — LOST to incumbent gpt5-4-mini (25/26, $0.55) on both quality and cost, so dev_models/lang_harness_suite keep mini. In extended_suite since - 2026-07-11 for new-release longitudinal tracking (per Mark); re-gate the - mini slot if OpenAI retires the 5.4 line. + 2026-07-11 for new-release longitudinal tracking (per Mark). + ⚠️ RE-GATE PENDING (2026-07-30): the price cut flips half that verdict. + Luna is now $0.20/$1.20 vs mini's $0.75/$4.50 — 3.75x cheaper on both + axes, so the $0.73 core run re-prices to ~$0.15 vs mini's $0.55. Quality + is still 23/26 vs 25/26 on N=1, which remains the reason mini holds the + slot. Per model-manager step 5.5, settle it with an N>=3 `--tier core` + run of luna+mini+anchor in ONE command before moving the dev_models / + lang_harness_suite slot. Also re-gate if OpenAI retires the 5.4 line. + The cost half of that comparison is robust to the v0.30.0 cost defect + (CAVEATS.md): luna is cheaper than mini on BOTH axes at near-identical + token usage, and the unrecorded reasoning bills at the output rate where + luna is 3.75x cheaper — so the mini/luna ratio holds at ~3.8x for any + reasoning volume. Correcting the defect widens luna's lead, never flips it. + ⚠️ AGENT-MODE 37/56 IS NOT A CAPABILITY NUMBER (found 2026-07-30). All 19 + v0.30.0 agent failures were 18x `cost budget exceeded` + 1 logic_error — + i.e. the $0.30 cap fired, not the model. Do NOT quote 37/56 as luna's + agent capability; only a re-run can produce one. + The price cut DOES move those kills. The kill threshold is enforced by + `executor.CostBudget`, built in agent_runner_multi.go from THIS `pricing:` + block — so the 18 kills were assessed at luna's real (then $1/$6) rates, + and at $0.20/$1.20 the same token volumes land at $0.065–$0.222, all under + the unchanged $0.30 cap. Re-run before quoting any agent number. + Separately, the banked `cost_usd` on those rows was NOT from this block: + the codex executor billed every model it ran at hardcoded gpt-5-codex rates + ($1.25/$10 per 1M) until fixed 2026-07-30 (executor.ResolveCostModel now + prefers Task.Pricing). v0.30.0 agent rows therefore carry a cost from one + price table and a kill threshold from another — luna/graph_bfs banked + $0.34259 (codex rates) while the budget that spared it saw $0.26980 + (luna rates). Do not compare `cost_usd` against `max_cost_usd` on any + pre-fix agent row. + Also: codex on this rig authenticates via `~/.codex/auth.json` + (`auth_mode: chatgpt`, ChatGPT subscription — probe-verified 2026-07-30 that + a deliberately invalid OPENAI_API_KEY in the env does NOT override it), so + agent-mode `cost_usd` is a list-price equivalent, never billed spend. Uses max_completion_tokens instead of max_tokens. # OpenAI GPT-5.4 (released March 5, 2026) @@ -514,6 +585,26 @@ models: pricing: input_per_1k: 0.003 # $3 per million = $0.003 per 1K output_per_1k: 0.015 # $15 per million = $0.015 per 1K + budgets: + # WORK GATE (2026-07-31, Mark): one shared 3.0M blended-token ceiling across + # agent_suite, so every model gets the same number of iterations. A dollar cap + # cannot do this — it buys work in inverse proportion to price, which is how a + # flat $0.30/$0.50 came to span 24x in actual tokens across this suite. + max_tokens_per_bench: 3000000 + # ⚠️ THIS MODEL WAS THE WORST-HIT BY THE DOLLAR GATE. It had no budgets block at + # all, so it fell through to the default formula — min($0.50, in*64 + out*32) — + # which at $3/$15 bought just 0.14M tokens: the TIGHTEST gate in the suite, 4% + # of what opencode-or-deepseek-v4-flash got from the same $0.50. It is the + # LONGITUDINAL ANCHOR every other model is scored against, so it was the single + # most iteration-starved row in a suite asking "does the agent loop rescue weak + # models". Purely accidental; found in the 2026-07-31 gate audit. + # + # $10.80 is the dollar equivalent of the 3.0M token gate at $3/$15, set so the + # TOKEN gate binds first. Not spend on this rig (OAuth). ⚠️ Under + # AILANG_AUTH_MODE=apikey (cloud dispatch) claude IS metered and this is the + # per-benchmark exposure ceiling — revisit before running the suite that way. + max_cost_usd: 10.80 + hard_timeout_secs: 600 notes: | Anthropic's best combination of speed and intelligence (Feb 17, 2026). 200K context window (1M available in beta). @@ -556,10 +647,19 @@ models: output_per_1k: 0.005 # $5.00 per million = $0.005 per 1K # M-EVAL-COST-AND-SPEED-BUDGETS (v0.15.1 retro): cheap-model agent budget. # Default formula = $0.224 — close to ceiling but multi-turn loops blow - # past it. Explicit $0.30 keeps haiku-class agents from being killed + # past it. Explicit $0.30 kept haiku-class agents from being killed # mid-stream while still capping runaway costs. budgets: - max_cost_usd: 0.30 + # WORK GATE (2026-07-31, Mark): one shared 3.0M blended-token ceiling across + # agent_suite, so every model gets the same number of iterations. A dollar cap + # cannot do this — it buys work in inverse proportion to price, which is how a + # flat $0.30/$0.50 came to span 24x in actual tokens across this suite. + max_tokens_per_bench: 3000000 + # $3.60 is the dollar equivalent of that 3.0M gate at $1/$5 — raised from $0.30 + # (which bought only 0.25M) so the TOKEN gate is the one that binds. On this rig + # claude runs on OAuth, so this is not spend. ⚠️ Under AILANG_AUTH_MODE=apikey + # (cloud dispatch) it IS metered — this is the per-benchmark exposure ceiling. + max_cost_usd: 3.60 hard_timeout_secs: 600 notes: | Anthropic's fastest and most cost-effective model. @@ -1146,7 +1246,17 @@ models: # Default formula = $0.056 — too tight. opencode harness compounds the # issue with internal Haiku compaction calls. Explicit $0.30. budgets: - max_cost_usd: 0.30 + # WORK GATE (2026-07-31, Mark): one shared 3.0M blended-token ceiling across + # agent_suite, so every model gets the same number of iterations. A dollar cap + # cannot do this — it buys work in inverse proportion to price, which is how a + # flat $0.30/$0.50 came to span 24x in actual tokens across this suite. + max_tokens_per_bench: 3000000 + # $3.60 is the dollar equivalent of the 3.0M token gate at $1/$5 — raised from + # $0.30 (which bought only 0.25M) so the TOKEN gate is the one that binds. + # On this rig claude runs on OAuth, so this is not spend. ⚠️ Under + # AILANG_AUTH_MODE=apikey (cloud dispatch) it IS metered — this is the + # per-benchmark exposure ceiling in that configuration. + max_cost_usd: 3.60 hard_timeout_secs: 600 notes: | Claude Haiku 4.5 evaluated via the opencode CLI harness. @@ -2057,6 +2167,15 @@ models: input_per_1k: 0.0014 # $1.40 per 1M output_per_1k: 0.0044 # $4.40 per 1M budgets: + # WORK GATE (2026-07-31, Mark): one shared 3.0M blended-token ceiling across + # agent_suite, so every model gets the same number of iterations. A dollar cap + # cannot do this — it buys work in inverse proportion to price, which is how a + # flat $0.30/$0.50 came to span 24x in actual tokens across this suite. + max_tokens_per_bench: 3000000 + # METERED lane (OpenRouter credits). The dollar cap stays a REAL spend control + # and is deliberately NOT raised to match the token gate — 3.0M tokens here + # would cost real money. Whichever bound binds first wins, so this model's + # comparison stays cost-limited, but for a stated reason rather than by accident. max_cost_usd: 0.30 hard_timeout_secs: 600 notes: | @@ -2155,6 +2274,15 @@ models: # Cheap model (~$0.003/benchmark) — 50¢ cap is 150x typical cost. # hard_timeout_secs is a network-hang safety net, not a cost proxy. budgets: + # WORK GATE (2026-07-31, Mark): one shared 3.0M blended-token ceiling across + # agent_suite, so every model gets the same number of iterations. A dollar cap + # cannot do this — it buys work in inverse proportion to price, which is how a + # flat $0.30/$0.50 came to span 24x in actual tokens across this suite. + max_tokens_per_bench: 3000000 + # METERED lane (OpenRouter credits). The dollar cap stays a REAL spend control + # and is deliberately NOT raised to match the token gate — 3.0M tokens here + # would cost real money. Whichever bound binds first wins, so this model's + # comparison stays cost-limited, but for a stated reason rather than by accident. max_cost_usd: 0.50 hard_timeout_secs: 3600 notes: | @@ -2180,6 +2308,15 @@ models: input_per_1k: 0.00043 output_per_1k: 0.00087 budgets: + # WORK GATE (2026-07-31, Mark): one shared 3.0M blended-token ceiling across + # agent_suite, so every model gets the same number of iterations. A dollar cap + # cannot do this — it buys work in inverse proportion to price, which is how a + # flat $0.30/$0.50 came to span 24x in actual tokens across this suite. + max_tokens_per_bench: 3000000 + # METERED lane (OpenRouter credits). The dollar cap stays a REAL spend control + # and is deliberately NOT raised to match the token gate — 3.0M tokens here + # would cost real money. Whichever bound binds first wins, so this model's + # comparison stays cost-limited, but for a stated reason rather than by accident. max_cost_usd: 0.50 hard_timeout_secs: 3600 notes: | diff --git a/internal/executor/claude/claude.go b/internal/executor/claude/claude.go index aaeabe876..d89f6f721 100644 --- a/internal/executor/claude/claude.go +++ b/internal/executor/claude/claude.go @@ -320,6 +320,13 @@ func (e *ClaudeExecutor) ExecuteStreaming(ctx context.Context, task *executor.Ta var firstAttemptMs int64 = -1 var firstStreamEventAt time.Time var costKilled bool + // The WORK gate (M-EVAL-OS-LONGITUDINAL Phase 1, extended to claude 2026-07-31). + // A dollar budget buys work in inverse proportion to model price: at $3/$15, + // claude-sonnet-4-6 — the suite's longitudinal ANCHOR — got 0.14M tokens from + // the same $0.50 that bought deepseek-v4-flash 3.40M. On the rig's OAuth lane + // there is also no spend to control, so tokens are the only meaningful gate. + var thrashKilled bool + var thrashKilledAtTokens int // runningInputTokens / runningOutputTokens track the cumulative usage // reported in message_delta events; we feed deltas into Budget.Add(). // Claude emits cumulative output_tokens in message_delta and the full @@ -457,6 +464,12 @@ func (e *ClaudeExecutor) ExecuteStreaming(ctx context.Context, task *executor.Ta _ = cmd.Process.Kill() } } + if task.MaxTokensPerBench > 0 && !thrashKilled && + runningInputTokens+runningOutputTokens > task.MaxTokensPerBench { + thrashKilled = true + thrashKilledAtTokens = runningInputTokens + runningOutputTokens + _ = cmd.Process.Kill() + } } case "content_block_stop": @@ -517,13 +530,14 @@ func (e *ClaudeExecutor) ExecuteStreaming(ctx context.Context, task *executor.Ta // This lets cloud handlers broadcast metrics before the executor returns. if mh, ok := handler.(executor.MetricsHandler); ok && finalResult != nil { mh.OnMetrics(executor.ExecutionMetrics{ - NumTurns: finalResult.NumTurns, - InputTokens: finalResult.Usage.InputTokens, - OutputTokens: finalResult.Usage.OutputTokens, - CostUSD: finalResult.TotalCostUSD, - DurationMS: finalResult.DurationMS, - SessionID: finalResult.SessionID, - Success: !finalResult.IsError, + NumTurns: finalResult.NumTurns, + InputTokens: finalResult.Usage.InputTokens, + OutputTokens: finalResult.Usage.OutputTokens, + CostUSD: finalResult.TotalCostUSD, + CostProvenance: executor.ResolveCostProvenance(task, e.authLane()), + DurationMS: finalResult.DurationMS, + SessionID: finalResult.SessionID, + Success: !finalResult.IsError, }) } } @@ -565,6 +579,7 @@ func (e *ClaudeExecutor) ExecuteStreaming(ctx context.Context, task *executor.Ta InputTokens: runningInputTokens, OutputTokens: runningOutputTokens, CostKilledAt: task.Budget.KilledAt(), + ThrashKilledAt: thrashKilledAtTokens, FirstAttemptMs: firstAttemptMs, SuccessAtMs: -1, }, nil @@ -584,6 +599,11 @@ func (e *ClaudeExecutor) ExecuteStreaming(ctx context.Context, task *executor.Ta if costKilled { errMsg = fmt.Sprintf("cost budget exceeded ($%.4f) — %s", task.Budget.KilledAt(), errMsg) } + // Token gate outranks the cost gate — on OAuth the dollars are notional. + if thrashKilled { + errMsg = fmt.Sprintf("thrash abort: cumulative tokens %d exceeded MaxTokensPerBench=%d — %s", + thrashKilledAtTokens, task.MaxTokensPerBench, errMsg) + } return &executor.Result{ Success: false, Error: errMsg, @@ -595,6 +615,7 @@ func (e *ClaudeExecutor) ExecuteStreaming(ctx context.Context, task *executor.Ta InputTokens: runningInputTokens, OutputTokens: runningOutputTokens, CostKilledAt: task.Budget.KilledAt(), + ThrashKilledAt: thrashKilledAtTokens, FirstAttemptMs: firstAttemptMs, SuccessAtMs: -1, }, nil @@ -618,6 +639,7 @@ func (e *ClaudeExecutor) ExecuteStreaming(ctx context.Context, task *executor.Ta InputTokens: runningInputTokens, OutputTokens: runningOutputTokens, CostKilledAt: task.Budget.KilledAt(), + ThrashKilledAt: thrashKilledAtTokens, FirstAttemptMs: firstAttemptMs, SuccessAtMs: -1, }, nil @@ -649,6 +671,11 @@ func (e *ClaudeExecutor) ExecuteStreaming(ctx context.Context, task *executor.Ta errorMsg = fmt.Sprintf("cost budget exceeded ($%.4f)", task.Budget.KilledAt()) success = false } + if thrashKilled { + errorMsg = fmt.Sprintf("thrash abort: cumulative tokens %d exceeded MaxTokensPerBench=%d", + thrashKilledAtTokens, task.MaxTokensPerBench) + success = false + } if !success { span.SetStatus(codes.Error, errorMsg) @@ -677,6 +704,7 @@ func (e *ClaudeExecutor) ExecuteStreaming(ctx context.Context, task *executor.Ta NumTurns: finalResult.NumTurns, ToolCallCount: toolCallCount, CostUSD: finalResult.TotalCostUSD, + CostProvenance: executor.ResolveCostProvenance(task, e.authLane()), InputTokens: finalResult.Usage.InputTokens, OutputTokens: finalResult.Usage.OutputTokens, CacheReadInputTokens: finalResult.Usage.CacheReadInputTokens, @@ -684,6 +712,7 @@ func (e *ClaudeExecutor) ExecuteStreaming(ctx context.Context, task *executor.Ta SessionID: sessionID, Transcript: transcriptBuf.String(), CostKilledAt: task.Budget.KilledAt(), + ThrashKilledAt: thrashKilledAtTokens, FirstAttemptMs: firstAttemptMs, SuccessAtMs: -1, TokensPerSec: tokensPerSec, @@ -725,7 +754,28 @@ func (e *ClaudeExecutor) Capabilities() []executor.Capability { } } -// CostModel returns pricing information for cost calculations +// authLane reports whether claude runs are charged per token. +// +// Mirrors the M-CLOUD-DUAL-AUTH branch in Execute: AILANG_AUTH_MODE=apikey means +// ANTHROPIC_API_KEY drives a metered account; anything else is the OAuth +// subscription lane, where the CLI still emits a non-zero total_cost_usd that +// nobody is charged. On the eval rig the key is deliberately stripped, so the +// default is the common case, not an edge case. +func (e *ClaudeExecutor) authLane() executor.AuthLane { + if os.Getenv("AILANG_AUTH_MODE") == "apikey" { + return executor.AuthLaneBilled + } + return executor.AuthLaneSubscription +} + +// CostModel returns pricing information for cost calculations. +// +// NOT used for Result.CostUSD: the claude CLI reports its own +// total_cost_usd and the executor banks that figure directly. Kept because +// the Executor interface requires it and callers may use it for pre-flight +// estimates. Audited 2026-07-30 — do not assume this table is what gets +// banked. Note the CLI's figure is itself a list-price equivalent when the +// rig authenticates via OAuth subscription, not metered spend. func (e *ClaudeExecutor) CostModel() *executor.CostModel { // Default to Haiku pricing return &executor.CostModel{ diff --git a/internal/executor/codex/codex.go b/internal/executor/codex/codex.go index d79144b26..783321b0a 100644 --- a/internal/executor/codex/codex.go +++ b/internal/executor/codex/codex.go @@ -9,9 +9,11 @@ package codex import ( "bufio" "context" + "encoding/json" "fmt" "os" "os/exec" + "path/filepath" "strings" "sync/atomic" "time" @@ -216,6 +218,13 @@ func (e *CodexExecutor) ExecuteStreaming(ctx context.Context, task *executor.Tas var firstAttemptMs int64 = -1 var firstStreamEventAt time.Time var costKilled bool + // The WORK gate (M-EVAL-OS-LONGITUDINAL Phase 1, extended to codex 2026-07-31). + // A dollar budget buys work in inverse proportion to model price, so it cannot + // give two models an equal number of iterations. On subscription lanes — codex + // with auth_mode chatgpt — there is no spend to control at all, and this is the + // only gate that means anything. + var thrashKilled bool + var thrashKilledAtTokens int var prevBudgetIn, prevBudgetOut int go func() { @@ -335,6 +344,12 @@ func (e *CodexExecutor) ExecuteStreaming(ctx context.Context, task *executor.Tas _ = cmd.Process.Kill() } } + if task.MaxTokensPerBench > 0 && !thrashKilled && + inputTokens+outputTokens > task.MaxTokensPerBench { + thrashKilled = true + thrashKilledAtTokens = inputTokens + outputTokens + _ = cmd.Process.Kill() + } } } // New format has no separate "result" event — turn.completed @@ -403,6 +418,12 @@ func (e *CodexExecutor) ExecuteStreaming(ctx context.Context, task *executor.Tas _ = cmd.Process.Kill() } } + if task.MaxTokensPerBench > 0 && !thrashKilled && + inputTokens+outputTokens > task.MaxTokensPerBench { + thrashKilled = true + thrashKilledAtTokens = inputTokens + outputTokens + _ = cmd.Process.Kill() + } } case "tool_use", "tool_call": @@ -491,6 +512,7 @@ func (e *CodexExecutor) ExecuteStreaming(ctx context.Context, task *executor.Tas OutputTokens: outputTokens, CacheReadInputTokens: cachedInput, CostKilledAt: task.Budget.KilledAt(), + ThrashKilledAt: thrashKilledAtTokens, FirstAttemptMs: firstAttemptMs, SuccessAtMs: -1, }, nil @@ -547,6 +569,7 @@ func (e *CodexExecutor) ExecuteStreaming(ctx context.Context, task *executor.Tas OutputTokens: outputTokens, CacheReadInputTokens: cachedInput, CostKilledAt: task.Budget.KilledAt(), + ThrashKilledAt: thrashKilledAtTokens, FirstAttemptMs: firstAttemptMs, SuccessAtMs: -1, }, nil @@ -584,6 +607,14 @@ func (e *CodexExecutor) ExecuteStreaming(ctx context.Context, task *executor.Tas errMsg = fmt.Sprintf("cost budget exceeded ($%.4f) — %s", task.Budget.KilledAt(), errMsg) finishReason = executor.FinishCostExhausted } + // The token gate outranks the cost gate: on a subscription lane the + // dollar figure is notional, so "thrash_aborted" is the truthful + // reason a run was stopped for doing too much work. + if thrashKilled { + errMsg = fmt.Sprintf("thrash abort: cumulative tokens %d exceeded MaxTokensPerBench=%d — %s", + thrashKilledAtTokens, task.MaxTokensPerBench, errMsg) + finishReason = executor.FinishThrashAborted + } freshInput, cachedInput := splitCodexInputTokens(inputTokens, cachedInputTokens) return &executor.Result{ Success: false, @@ -599,6 +630,7 @@ func (e *CodexExecutor) ExecuteStreaming(ctx context.Context, task *executor.Tas OutputTokens: outputTokens, CacheReadInputTokens: cachedInput, CostKilledAt: task.Budget.KilledAt(), + ThrashKilledAt: thrashKilledAtTokens, FirstAttemptMs: firstAttemptMs, SuccessAtMs: -1, TokensPerSec: tokensPerSec, @@ -606,7 +638,10 @@ func (e *CodexExecutor) ExecuteStreaming(ctx context.Context, task *executor.Tas } freshInput, cachedInput := splitCodexInputTokens(inputTokens, cachedInputTokens) - cost := e.CostModel().CalculateCost(executor.TokenUsage{ + // Bill the model that actually ran, not codex's default. The harness + // supplies per-model rates via Task.Pricing; CostModel() is the + // fallback for callers that don't (see executor.ResolveCostModel). + cost := executor.ResolveCostModel(task, e.CostModel()).CalculateCost(executor.TokenUsage{ InputTokens: freshInput, OutputTokens: outputTokens, CacheReadInputTokens: cachedInput, @@ -626,6 +661,10 @@ func (e *CodexExecutor) ExecuteStreaming(ctx context.Context, task *executor.Tas success = false finishReason = executor.FinishCostExhausted } + if thrashKilled { + success = false + finishReason = executor.FinishThrashAborted + } span.SetAttributes( attribute.Int("task.turns", turnNum), @@ -649,6 +688,7 @@ func (e *CodexExecutor) ExecuteStreaming(ctx context.Context, task *executor.Tas NumTurns: turnNum, ToolCallCount: toolCallCount, CostUSD: cost, + CostProvenance: executor.ResolveCostProvenance(task, e.authLane()), InputTokens: freshInput, OutputTokens: outputTokens, CacheReadInputTokens: cachedInput, @@ -656,6 +696,7 @@ func (e *CodexExecutor) ExecuteStreaming(ctx context.Context, task *executor.Tas Transcript: transcriptBuf.String(), ProviderData: providerData(rawEvents), CostKilledAt: task.Budget.KilledAt(), + ThrashKilledAt: thrashKilledAtTokens, FirstAttemptMs: firstAttemptMs, SuccessAtMs: -1, TokensPerSec: tokensPerSec, @@ -672,9 +713,53 @@ func (e *CodexExecutor) Capabilities() []executor.Capability { } } +// authLane reports whether codex runs are charged per token. +// +// Codex authenticates from ~/.codex/auth.json, written by `codex login`. An +// OPENAI_API_KEY in the environment does NOT override it — probe-verified +// 2026-07-30 against codex-cli 0.145.0 with auth_mode "chatgpt", where a +// deliberately invalid env key still ran clean. Reading the env var here would +// therefore report "billed" for a run the ChatGPT plan covered. +// +// Unreadable or unrecognised → Unknown. A wrong "metered" is the failure mode +// this exists to prevent, so it is never the fallback. +func (e *CodexExecutor) authLane() executor.AuthLane { + home, err := os.UserHomeDir() + if err != nil { + return executor.AuthLaneUnknown + } + data, err := os.ReadFile(filepath.Join(home, ".codex", "auth.json")) + if err != nil { + return executor.AuthLaneUnknown + } + var auth struct { + AuthMode string `json:"auth_mode"` + APIKey string `json:"OPENAI_API_KEY"` + } + if err := json.Unmarshal(data, &auth); err != nil { + return executor.AuthLaneUnknown + } + switch auth.AuthMode { + case "chatgpt": + return executor.AuthLaneSubscription + case "apikey": + return executor.AuthLaneBilled + } + // Older codex releases wrote the key with no auth_mode discriminator. + if auth.APIKey != "" { + return executor.AuthLaneBilled + } + return executor.AuthLaneUnknown +} + // CostModel returns pricing for gpt-5-codex (the default Codex model). // Source: https://platform.openai.com/docs/pricing // gpt-5-codex: $1.25/$10.00 per 1M tokens = $0.00125/$0.01 per 1K. +// +// FALLBACK ONLY. The codex CLI runs whatever `--model` it is handed, so this +// table is correct for exactly one of them. Result.CostUSD is billed via +// executor.ResolveCostModel, which prefers Task.Pricing (the per-model rates +// from models.yml). This is reached only when a caller supplies no pricing. func (e *CodexExecutor) CostModel() *executor.CostModel { return &executor.CostModel{ ProviderName: "openai", @@ -696,11 +781,14 @@ func (e *CodexExecutor) HealthCheck(ctx context.Context) error { if err := checkCmd.Run(); err != nil { return fmt.Errorf("codex --version failed: %w", err) } - if os.Getenv("OPENAI_API_KEY") == "" { - // Auth may also come from `codex login` cache; warn but do not fail. - if os.Getenv("DEBUG_AGENT") != "" { - fmt.Fprintf(os.Stderr, "[DEBUG_CODEX] OPENAI_API_KEY unset; relying on codex login cache\n") - } + // Auth comes from ~/.codex/auth.json, written by `codex login`. OPENAI_API_KEY + // in the environment does NOT override it — probe-verified 2026-07-30 against + // codex-cli 0.145.0 with auth_mode "chatgpt": a deliberately invalid key in the + // env still ran clean. So its absence is not a warning condition, and its + // presence is not proof that runs are metered (this rig is on a ChatGPT + // subscription, where cost_usd is a list-price equivalent, never billed spend). + if os.Getenv("DEBUG_AGENT") != "" { + fmt.Fprintf(os.Stderr, "[DEBUG_CODEX] auth: ~/.codex/auth.json (codex login); OPENAI_API_KEY is not consulted\n") } return nil } diff --git a/internal/executor/cost.go b/internal/executor/cost.go index 01408ef26..2766d553a 100644 --- a/internal/executor/cost.go +++ b/internal/executor/cost.go @@ -93,6 +93,80 @@ func (b *CostBudget) KilledAt() float64 { return math.Float64frombits(bits) } +// CostProvenance records whether a Result's CostUSD is money anyone was +// actually charged. Without it, a subscription run and a metered run are +// indistinguishable once both land in the same column — and the v1.0 +// cost-per-verified-success KPI, whose numerator is "attributable METERED +// dollars", silently aggregates both. +// +// This is orthogonal to accuracy. A list-price-equivalent figure can be +// perfectly computed and still represent zero spend. +type CostProvenance string + +const ( + // CostMetered: the account is genuinely charged per token (API key, + // OpenRouter credits, Vertex ADC). The only provenance admissible in a + // metered-dollars KPI. + CostMetered CostProvenance = "metered" + // CostListPriceEquivalent: the figure is real arithmetic over real tokens, + // but the run went through a subscription/OAuth lane and was never billed. + // Reproducible and comparable; just not spend. + CostListPriceEquivalent CostProvenance = "list-price-equivalent" + // CostFreeLocal: an on-device model with zero marginal token cost. + // Distinct from a $0 metered figure, which would mean "billed nothing". + CostFreeLocal CostProvenance = "free-local" + // CostProvenanceUnknown: the auth lane could not be determined. Surfaced + // rather than assumed — guessing "metered" is what this type exists to stop. + CostProvenanceUnknown CostProvenance = "unknown" +) + +// AuthLane is an executor's determination of how the current run authenticates. +type AuthLane int + +const ( + // AuthLaneUnknown: could not be determined; do not assume either way. + AuthLaneUnknown AuthLane = iota + // AuthLaneBilled: per-token charges land on an account. + AuthLaneBilled + // AuthLaneSubscription: a seat/plan covers the run; no per-token charge. + AuthLaneSubscription +) + +// ResolveCostProvenance classifies a task's cost given the executor's auth lane. +// +// Zero resolved rates win over the lane: a local Ollama model is free-local +// whatever the auth story, because there is no per-token charge to attribute. +func ResolveCostProvenance(task *Task, lane AuthLane) CostProvenance { + if task != nil && task.Pricing != nil && + task.Pricing.InputTokenCost == 0 && task.Pricing.OutputTokenCost == 0 { + return CostFreeLocal + } + switch lane { + case AuthLaneBilled: + return CostMetered + case AuthLaneSubscription: + return CostListPriceEquivalent + default: + return CostProvenanceUnknown + } +} + +// ResolveCostModel picks the pricing an executor should bill a task at. +// +// Task.Pricing (the per-model rates from models.yml) wins whenever it is +// present; fallback is the executor's own CostModel(), which names a single +// default model and is therefore only correct when that is what actually ran. +// +// A present-but-zero Task.Pricing is honoured, not treated as missing: local +// Ollama models are genuinely free, and falling back to a cloud price table +// for them would invent spend that never happened. +func ResolveCostModel(task *Task, fallback *CostModel) *CostModel { + if task != nil && task.Pricing != nil { + return task.Pricing + } + return fallback +} + // DefaultMaxCostUSD computes the fallback budget used when models.yml // omits a per-model `budgets:` block. // diff --git a/internal/executor/cost_test.go b/internal/executor/cost_test.go index ea4e56e04..8f5b84a38 100644 --- a/internal/executor/cost_test.go +++ b/internal/executor/cost_test.go @@ -232,3 +232,72 @@ func approxEqual(a, b, tol float64) bool { } return d < tol } + +// TestResolveCostModel pins the 2026-07-30 defect: every model routed through +// a CLI harness was billed at that harness's single hardcoded table (each +// codex-run model at gpt-5-codex's $1.25/$10 per 1M), while Task.Budget +// enforced the kill at the model's real models.yml rates. The two numbers came +// from different price tables and were not comparable. +func TestResolveCostModel(t *testing.T) { + // The hardcoded fallback each CLI executor declares. + codexTable := &CostModel{ProviderName: "openai", InputTokenCost: 0.00125, OutputTokenCost: 0.01} + // gpt-5.6-luna after OpenAI's 2026-07-30 cut: $0.20/$1.20 per 1M. + luna := &CostModel{ProviderName: "openai", InputTokenCost: 0.0002, OutputTokenCost: 0.0012} + + tests := []struct { + name string + task *Task + want *CostModel + }{ + {"per-model pricing wins over the harness table", &Task{Pricing: luna}, luna}, + {"no pricing supplied falls back", &Task{}, codexTable}, + {"nil task falls back", nil, codexTable}, + { + // Local Ollama models are genuinely free. Falling back here would + // invent cloud spend that never happened. + name: "present-but-zero pricing is honoured, not treated as missing", + task: &Task{Pricing: &CostModel{ProviderName: "ollama"}}, + want: &CostModel{ProviderName: "ollama"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := ResolveCostModel(tt.task, codexTable) + if got.InputTokenCost != tt.want.InputTokenCost || got.OutputTokenCost != tt.want.OutputTokenCost { + t.Errorf("rates = $%g/$%g, want $%g/$%g", + got.InputTokenCost, got.OutputTokenCost, tt.want.InputTokenCost, tt.want.OutputTokenCost) + } + }) + } +} + +// TestResolveCostModel_MatchesBudgetPricing is the regression proper: the cost +// an executor BANKS must agree with the cost the budget ENFORCES on, for the +// same token counts. Uses the real banked row that exposed the split — +// luna/graph_bfs, 256,987 in / 2,136 out, banked $0.34259375 at codex rates +// while the $0.30 budget spared it because it saw $0.26980 at luna's rates. +func TestResolveCostModel_MatchesBudgetPricing(t *testing.T) { + const ( + inTok, outTok = 256987, 2136 + lunaIn, lunaOut = 0.001, 0.006 // luna's models.yml rates at the time + codexIn, codexOut = 0.00125, 0.01 + maxUSD = 0.30 + ) + task := &Task{ + Budget: NewCostBudget(maxUSD, lunaIn, lunaOut), + Pricing: &CostModel{ProviderName: "openai", InputTokenCost: lunaIn, OutputTokenCost: lunaOut}, + } + enforced, exceeded := task.Budget.Add(inTok, outTok) + if exceeded { + t.Fatalf("budget reported exceeded at $%.5f; this row passed in the real run", enforced) + } + banked := ResolveCostModel(task, &CostModel{InputTokenCost: codexIn, OutputTokenCost: codexOut}). + CalculateCost(TokenUsage{InputTokens: inTok, OutputTokens: outTok}) + + if diff := banked - enforced; diff > 1e-9 || diff < -1e-9 { + t.Errorf("banked $%.5f != enforced $%.5f — the two price tables have split again", banked, enforced) + } + if banked >= maxUSD { + t.Errorf("banked $%.5f >= cap $%.2f, but the run was not killed — banked cost is billing the wrong model", banked, maxUSD) + } +} diff --git a/internal/executor/executor.go b/internal/executor/executor.go index c37b76b3a..c4eac06f9 100644 --- a/internal/executor/executor.go +++ b/internal/executor/executor.go @@ -96,6 +96,24 @@ type Task struct { // Nil = legacy behaviour (wall-clock-only). Budget *CostBudget + // Pricing carries the PER-MODEL rates from models.yml so an executor that + // computes Result.CostUSD from token counts bills the model it actually ran. + // + // Without this, such an executor can only fall back to its own hardcoded + // table, which names one model but is applied to every model the harness + // routes through that CLI — e.g. every codex-run model was billed at + // gpt-5-codex's $1.25/$10 per 1M regardless of its real price (found + // 2026-07-30). Task.Budget already carried the correct per-model rates, so + // the kill threshold and the banked cost were computed from two different + // price tables and were not comparable. + // + // Deliberately separate from Budget: Budget exists only when cost + // ENFORCEMENT is on (ResolvedMaxCostUSD() > 0), while pricing is needed for + // REPORTING either way. Nil = no per-model rates supplied; executors fall + // back to their own CostModel(). Zero rates are meaningful, not missing + // (local Ollama models are genuinely free) — see ResolveCostModel. + Pricing *CostModel + // MaxTokensPerBench (M-EVAL-OS-LONGITUDINAL Phase 1, v0.23.0): hard token // ceiling per benchmark for thrash detection on free (pricing=0) local // models. When cumulative input+output tokens exceed this value mid-stream, @@ -148,6 +166,10 @@ type Result struct { // discoverable tool like `ailang fmt`, which the scalar count can't answer. ToolCalls map[string]int CostUSD float64 // Total cost in USD + // CostProvenance says whether CostUSD is money anyone was charged. + // Empty means the executor did not classify it — read as "unknown", + // never as "metered". See executor.ResolveCostProvenance. + CostProvenance CostProvenance // Token usage InputTokens int @@ -364,9 +386,14 @@ type ExecutionMetrics struct { InputTokens int OutputTokens int CostUSD float64 - DurationMS int - SessionID string - Success bool + // CostProvenance labels CostUSD for live observers the same way + // Result.CostProvenance does for banked rows — these metrics reach the + // control plane and dashboard, which must not render subscription + // arithmetic as metered spend. + CostProvenance CostProvenance + DurationMS int + SessionID string + Success bool } // CostModel contains pricing information diff --git a/internal/executor/managed_agents/managed_agents.go b/internal/executor/managed_agents/managed_agents.go index 979d16e8f..057400677 100644 --- a/internal/executor/managed_agents/managed_agents.go +++ b/internal/executor/managed_agents/managed_agents.go @@ -275,7 +275,9 @@ func (e *Executor) ExecuteStreaming( // rate because Vertex's gemini-3-5-flash pricing model doesn't separate // them — so they must be added back here even though the fields are now // stored separately. - cm := e.CostModel() + cm := executor.ResolveCostModel(task, e.CostModel()) + // Vertex ADC bills a real GCP project — the one agent lane that is metered. + res.CostProvenance = executor.ResolveCostProvenance(task, executor.AuthLaneBilled) res.CostUSD = cm.CalculateCost(executor.TokenUsage{ InputTokens: res.InputTokens, OutputTokens: res.OutputTokens + res.ReasonTokens, diff --git a/internal/executor/motoko/motoko.go b/internal/executor/motoko/motoko.go index c5221823e..70d19cf6c 100644 --- a/internal/executor/motoko/motoko.go +++ b/internal/executor/motoko/motoko.go @@ -538,17 +538,22 @@ func (e *MotokoExecutor) ExecuteStreaming(ctx context.Context, task *executor.Ta if result.SessionID == "" { result.SessionID = sessionID } + // motoko's run_summary cost comes from whatever provider it routed to. + // OpenRouter-routed models bill real credits; local ollama models carry + // zero rates in models.yml and resolve to free-local regardless of lane. + result.CostProvenance = executor.ResolveCostProvenance(task, executor.AuthLaneBilled) // Surface metrics for any MetricsHandler observers. if mh, ok := handler.(executor.MetricsHandler); ok { mh.OnMetrics(executor.ExecutionMetrics{ - NumTurns: result.NumTurns, - InputTokens: result.InputTokens, - OutputTokens: result.OutputTokens, - CostUSD: result.CostUSD, - DurationMS: result.DurationMS, - SessionID: result.SessionID, - Success: result.Success, + NumTurns: result.NumTurns, + InputTokens: result.InputTokens, + OutputTokens: result.OutputTokens, + CostUSD: result.CostUSD, + CostProvenance: executor.ResolveCostProvenance(task, executor.AuthLaneBilled), + DurationMS: result.DurationMS, + SessionID: result.SessionID, + Success: result.Success, }) } diff --git a/internal/executor/opencode/opencode.go b/internal/executor/opencode/opencode.go index c64e3afc0..b1c75ccc9 100644 --- a/internal/executor/opencode/opencode.go +++ b/internal/executor/opencode/opencode.go @@ -421,6 +421,7 @@ func (e *OpenCodeExecutor) ExecuteStreaming(ctx context.Context, task *executor. CacheReadInputTokens: cacheReadTokens, CacheCreationInputTokens: cacheWriteTokens, CostUSD: totalCostUSD, + CostProvenance: executor.ResolveCostProvenance(task, executor.AuthLaneBilled), NumTurns: numSteps, ToolCallCount: toolCallCount, ToolCalls: toolCalls, @@ -452,6 +453,7 @@ func (e *OpenCodeExecutor) ExecuteStreaming(ctx context.Context, task *executor. CacheReadInputTokens: cacheReadTokens, CacheCreationInputTokens: cacheWriteTokens, CostUSD: totalCostUSD, + CostProvenance: executor.ResolveCostProvenance(task, executor.AuthLaneBilled), NumTurns: numSteps, ToolCallCount: toolCallCount, ToolCalls: toolCalls, diff --git a/internal/executor/pi/pi.go b/internal/executor/pi/pi.go index ac4950770..187c16d69 100644 --- a/internal/executor/pi/pi.go +++ b/internal/executor/pi/pi.go @@ -376,6 +376,7 @@ func (e *PiExecutor) ExecuteStreaming(ctx context.Context, task *executor.Task, CacheReadInputTokens: cacheReadTokens, CacheCreationInputTokens: cacheWriteTokens, CostUSD: totalCostUSD, + CostProvenance: executor.ResolveCostProvenance(task, executor.AuthLaneBilled), NumTurns: numTurns, ToolCallCount: toolCallCount, ToolCalls: toolCalls, @@ -412,6 +413,7 @@ func (e *PiExecutor) ExecuteStreaming(ctx context.Context, task *executor.Task, CacheReadInputTokens: cacheReadTokens, CacheCreationInputTokens: cacheWriteTokens, CostUSD: totalCostUSD, + CostProvenance: executor.ResolveCostProvenance(task, executor.AuthLaneBilled), NumTurns: numTurns, ToolCallCount: toolCallCount, ToolCalls: toolCalls, diff --git a/internal/observatory/backend.go b/internal/observatory/backend.go index 285c72170..7e00a27e2 100644 --- a/internal/observatory/backend.go +++ b/internal/observatory/backend.go @@ -139,7 +139,11 @@ type Backend interface { UpdateStageStatus(ctx context.Context, stageID string, status ChainStageStatus) error UpdateStageSession(ctx context.Context, stageID, sessionID string) error UpdateStageApproval(ctx context.Context, stageID string, status ApprovalStatus, approvalType ApprovalType, feedback string) error - UpdateStageMetrics(ctx context.Context, stageID string, cost float64, tokensIn, tokensOut, turns, toolCalls int, durationMs int64) error + // UpdateStageMetrics accumulates a stage's denormalized metrics. + // costProvenance labels whether `cost` was actually billed (see + // executor.CostProvenance); pass "" when the caller cannot classify it — + // that reads as unknown and never overwrites a label already recorded. + UpdateStageMetrics(ctx context.Context, stageID string, cost float64, tokensIn, tokensOut, turns, toolCalls int, durationMs int64, costProvenance string) error UpdateStageError(ctx context.Context, stageID, errorMessage string) error GetSpansByStageID(ctx context.Context, stageID string) ([]*Span, error) // GetSpanLitesByStageID returns lightweight spans without attributes (M-PERF-OBSERVATORY). diff --git a/internal/observatory/backend_composite.go b/internal/observatory/backend_composite.go index 1fd04ddc2..041ae9636 100644 --- a/internal/observatory/backend_composite.go +++ b/internal/observatory/backend_composite.go @@ -428,8 +428,8 @@ func (b *CompositeBackend) UpdateStageSession(ctx context.Context, stageID, sess return b.local.UpdateStageSession(ctx, stageID, sessionID) } -func (b *CompositeBackend) UpdateStageMetrics(ctx context.Context, stageID string, cost float64, tokensIn, tokensOut, turns, toolCalls int, durationMs int64) error { - return b.local.UpdateStageMetrics(ctx, stageID, cost, tokensIn, tokensOut, turns, toolCalls, durationMs) +func (b *CompositeBackend) UpdateStageMetrics(ctx context.Context, stageID string, cost float64, tokensIn, tokensOut, turns, toolCalls int, durationMs int64, costProvenance string) error { + return b.local.UpdateStageMetrics(ctx, stageID, cost, tokensIn, tokensOut, turns, toolCalls, durationMs, costProvenance) } func (b *CompositeBackend) UpdateStageApproval(ctx context.Context, stageID string, status ApprovalStatus, approvalType ApprovalType, feedback string) error { diff --git a/internal/observatory/backend_gcp.go b/internal/observatory/backend_gcp.go index 936a03e54..e618f82f2 100644 --- a/internal/observatory/backend_gcp.go +++ b/internal/observatory/backend_gcp.go @@ -695,7 +695,7 @@ func (b *GCPTraceBackend) UpdateStageSession(ctx context.Context, stageID, sessi return nil } -func (b *GCPTraceBackend) UpdateStageMetrics(ctx context.Context, stageID string, cost float64, tokensIn, tokensOut, turns, toolCalls int, durationMs int64) error { +func (b *GCPTraceBackend) UpdateStageMetrics(ctx context.Context, stageID string, cost float64, tokensIn, tokensOut, turns, toolCalls int, durationMs int64, costProvenance string) error { return nil } diff --git a/internal/observatory/backend_jaeger.go b/internal/observatory/backend_jaeger.go index 89113f9b2..3f4faabfd 100644 --- a/internal/observatory/backend_jaeger.go +++ b/internal/observatory/backend_jaeger.go @@ -338,7 +338,7 @@ func (b *JaegerBackend) UpdateStageSession(ctx context.Context, stageID, session return nil } -func (b *JaegerBackend) UpdateStageMetrics(ctx context.Context, stageID string, cost float64, tokensIn, tokensOut, turns, toolCalls int, durationMs int64) error { +func (b *JaegerBackend) UpdateStageMetrics(ctx context.Context, stageID string, cost float64, tokensIn, tokensOut, turns, toolCalls int, durationMs int64, costProvenance string) error { return nil } diff --git a/internal/observatory/backend_sqlite.go b/internal/observatory/backend_sqlite.go index f476942f0..926e859fd 100644 --- a/internal/observatory/backend_sqlite.go +++ b/internal/observatory/backend_sqlite.go @@ -460,8 +460,8 @@ func (b *SQLiteBackend) UpdateStageSession(ctx context.Context, stageID, session return b.store.UpdateStageSession(ctx, stageID, sessionID) } -func (b *SQLiteBackend) UpdateStageMetrics(ctx context.Context, stageID string, cost float64, tokensIn, tokensOut, turns, toolCalls int, durationMs int64) error { - return b.store.UpdateStageMetrics(ctx, stageID, cost, tokensIn, tokensOut, turns, toolCalls, durationMs) +func (b *SQLiteBackend) UpdateStageMetrics(ctx context.Context, stageID string, cost float64, tokensIn, tokensOut, turns, toolCalls int, durationMs int64, costProvenance string) error { + return b.store.UpdateStageMetrics(ctx, stageID, cost, tokensIn, tokensOut, turns, toolCalls, durationMs, costProvenance) } func (b *SQLiteBackend) UpdateStageApproval(ctx context.Context, stageID string, status ApprovalStatus, approvalType ApprovalType, feedback string) error { diff --git a/internal/observatory/cost_classify.go b/internal/observatory/cost_classify.go index 3e047c884..afffd1d75 100644 --- a/internal/observatory/cost_classify.go +++ b/internal/observatory/cost_classify.go @@ -1,5 +1,7 @@ package observatory +import "github.com/sunholo-data/ailang/internal/executor" + // Cost-attribution classifier (M-MISSION-COST-CHAINS, M1). // // The chains rollup historically summed the self-reported `cost` column with a SQL @@ -9,9 +11,15 @@ package observatory // // This file adds a READ-SIDE, per-stage classifier under Mark's SCOPED-INFERENCE // rule (2026-07-24, `4e1348adb`). It NEVER mutates stored data and NEVER guesses a -// model or a rate. The four statuses are mutually exclusive: +// model or a rate. The five statuses are mutually exclusive: // -// - reported : cost > 0 in storage → the sender attributed cost. Left untouched. +// - reported : cost > 0 in storage and NOT labelled subscription → the sender +// attributed cost. Left untouched. Note this means "self-reported", not +// "proven metered": a stage with no cost_provenance (every row banked before +// 2026-07-30) lands here regardless of whether anyone was billed. +// - subscription: cost > 0 AND cost_provenance == "list-price-equivalent" → +// real arithmetic over real tokens on a lane nobody was charged for (codex +// `auth_mode: chatgpt`, claude OAuth). Kept OUT of metered-dollar totals. // - estimated: tokens > 0, cost == 0, and the stage's model resolves to a // NON-ZERO metered rate → cost is computed tokens×rate and flagged. // - unknown : tokens > 0, cost == 0, and the model is unresolvable → surfaced as @@ -36,6 +44,12 @@ const ( CostStatusUnknown CostStatus = "unknown" // CostStatusQuota: quota/subscription lane (no tokens) — $0-by-design, never estimated. CostStatusQuota CostStatus = "quota" + // CostStatusSubscription: the stage reported a non-zero cost AND labelled it + // list-price-equivalent — real arithmetic over real tokens on a lane where + // nobody was billed (codex `auth_mode: chatgpt`, claude OAuth). Distinct from + // `quota`, which has no tokens at all, and from `reported`, which a + // metered-dollars KPI may legitimately sum. Added 2026-07-30. + CostStatusSubscription CostStatus = "subscription" ) // StageCost is the classified cost of a single stage. @@ -43,6 +57,7 @@ type StageCost struct { Status CostStatus `json:"status"` // CostUSD is the effective dollar cost for the rollup: // - reported : the stored cost. + // - subscription: the stored cost (notional — never billed). // - estimated: the computed tokens×rate (may be $0 for a free model). // - unknown : 0 (but MUST be surfaced as unknown, not as $0 metered spend). // - quota : 0. @@ -80,8 +95,15 @@ func ClassifyStageCost(stage *ChainStage) StageCost { return StageCost{Status: CostStatusQuota} } - // 1. Self-reported cost is authoritative and never re-estimated. + // 1. Self-reported cost is never re-estimated — but "self-reported" is not + // the same as "billed". An agent CLI on a subscription lane (codex with + // auth_mode chatgpt, claude on OAuth) emits a non-zero cost that nobody + // was charged. When the stage carries that provenance, split it out so a + // metered-dollars total can exclude it; otherwise behave as before. if stage.Cost > 0 { + if stage.CostProvenance == string(executor.CostListPriceEquivalent) { + return StageCost{Status: CostStatusSubscription, CostUSD: stage.Cost} + } return StageCost{Status: CostStatusReported, CostUSD: stage.Cost} } @@ -130,21 +152,31 @@ func ClassifyStageCost(stage *ChainStage) StageCost { type CostRollup struct { ReportedCost float64 `json:"reported_cost"` EstimatedCost float64 `json:"estimated_cost"` + // SubscriptionCost is real arithmetic over real tokens that nobody paid. + // Deliberately EXCLUDED from TotalKnownCost so a metered-dollars KPI cannot + // pick it up by accident; surface it alongside, never inside. + SubscriptionCost float64 `json:"subscription_cost"` // UnknownCost is always 0 by construction (unknown is never given a dollar figure); // UnknownStages is the count that MUST trigger an incomplete-data warning. - ReportedStages int `json:"reported_stages"` - EstimatedStages int `json:"estimated_stages"` - QuotaStages int `json:"quota_stages"` - UnknownStages int `json:"unknown_stages"` + ReportedStages int `json:"reported_stages"` + EstimatedStages int `json:"estimated_stages"` + QuotaStages int `json:"quota_stages"` + SubscriptionStages int `json:"subscription_stages"` + UnknownStages int `json:"unknown_stages"` // Token counts split the same way (quota lanes carry no tokens by definition). - ReportedTokens int64 `json:"reported_tokens"` - EstimatedTokens int64 `json:"estimated_tokens"` - UnknownTokens int64 `json:"unknown_tokens"` + ReportedTokens int64 `json:"reported_tokens"` + EstimatedTokens int64 `json:"estimated_tokens"` + UnknownTokens int64 `json:"unknown_tokens"` + SubscriptionTokens int64 `json:"subscription_tokens"` } // TotalKnownCost returns reported + estimated dollars (the credible total). // Unknown stages contribute NO dollars — callers must surface UnknownStages // separately as an incomplete-data warning rather than silently reading $0. +// +// SubscriptionCost is deliberately NOT included: it is money nobody spent, and +// the v1.0 cost-per-verified-success KPI counts attributable metered dollars. +// Callers wanting the full list-price picture add SubscriptionCost explicitly. func (r CostRollup) TotalKnownCost() float64 { return r.ReportedCost + r.EstimatedCost } @@ -176,6 +208,10 @@ func (r *CostRollup) AddStage(stage *ChainStage) { r.UnknownTokens += tokens case CostStatusQuota: r.QuotaStages++ + case CostStatusSubscription: + r.SubscriptionCost += sc.CostUSD + r.SubscriptionStages++ + r.SubscriptionTokens += tokens } } diff --git a/internal/observatory/cost_classify_test.go b/internal/observatory/cost_classify_test.go index 423e8a7d1..047690358 100644 --- a/internal/observatory/cost_classify_test.go +++ b/internal/observatory/cost_classify_test.go @@ -274,3 +274,54 @@ func TestRollupStages_SplitsTotals(t *testing.T) { t.Fatalf("TotalKnownCost must be reported+estimated only") } } + +// TestClassifyStageCost_Subscription pins the 2026-07-30 fix: a non-zero cost +// from a subscription lane is real arithmetic over real tokens that NOBODY was +// billed for, and must not land in a metered-dollars total. +func TestClassifyStageCost_Subscription(t *testing.T) { + tests := []struct { + name string + provenance string + wantStatus CostStatus + }{ + {"subscription lane splits out", "list-price-equivalent", CostStatusSubscription}, + {"metered stays reported", "metered", CostStatusReported}, + // Every row banked before the column existed. Unlabelled is NOT proof of + // metering — but downgrading all history to unknown would destroy the + // rollup, so it stays `reported` and the doc says what that means. + {"unlabelled legacy row stays reported", "", CostStatusReported}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := ClassifyStageCost(&ChainStage{ + Cost: 0.34259375, TokensIn: 256987, TokensOut: 2136, + CostProvenance: tt.provenance, + }) + if got.Status != tt.wantStatus { + t.Errorf("status = %q, want %q", got.Status, tt.wantStatus) + } + if got.CostUSD != 0.34259375 { + t.Errorf("cost = %v, want the stored figure preserved", got.CostUSD) + } + }) + } +} + +// TestCostRollup_SubscriptionExcludedFromKnownCost is the KPI guard: a cohort +// mixing a subscription stage with a metered one must not report their sum as +// spend. TotalKnownCost is the metered-dollars numerator. +func TestCostRollup_SubscriptionExcludedFromKnownCost(t *testing.T) { + r := RollupStages([]*ChainStage{ + {Cost: 2.00, TokensIn: 1000, TokensOut: 100, CostProvenance: "metered"}, + {Cost: 8.36, TokensIn: 6241312, TokensOut: 55679, CostProvenance: "list-price-equivalent"}, + }) + if r.TotalKnownCost() != 2.00 { + t.Errorf("TotalKnownCost = %v, want 2.00 (subscription must be excluded)", r.TotalKnownCost()) + } + if r.SubscriptionCost != 8.36 { + t.Errorf("SubscriptionCost = %v, want 8.36 (surfaced, not discarded)", r.SubscriptionCost) + } + if r.SubscriptionStages != 1 || r.ReportedStages != 1 { + t.Errorf("stage split = %d subscription / %d reported, want 1/1", r.SubscriptionStages, r.ReportedStages) + } +} diff --git a/internal/observatory/cost_per_verified_success_test.go b/internal/observatory/cost_per_verified_success_test.go index f4cb80d59..f7cdaeea6 100644 --- a/internal/observatory/cost_per_verified_success_test.go +++ b/internal/observatory/cost_per_verified_success_test.go @@ -75,7 +75,7 @@ func bankCohort(t *testing.T, store *Store, sourceRef string, specs []cvsStageSp if err := store.UpdateStageEvalAssessment(ctx, stage.ID, assessment); err != nil { t.Fatalf("failed to bank assessment %d: %v", i, err) } - if err := store.UpdateStageMetrics(ctx, stage.ID, s.cost, s.tokensIn, s.tokensOut, 0, 0, 0); err != nil { + if err := store.UpdateStageMetrics(ctx, stage.ID, s.cost, s.tokensIn, s.tokensOut, 0, 0, 0, ""); err != nil { t.Fatalf("failed to bank metrics %d: %v", i, err) } } diff --git a/internal/observatory/cost_rollup_store_test.go b/internal/observatory/cost_rollup_store_test.go index b34db8912..a9eb77e49 100644 --- a/internal/observatory/cost_rollup_store_test.go +++ b/internal/observatory/cost_rollup_store_test.go @@ -15,7 +15,7 @@ func seedStage(t *testing.T, store *Store, chainID, agentID string, cost float64 if err != nil { t.Fatalf("CreateStage: %v", err) } - if err := store.UpdateStageMetrics(ctx, stage.ID, cost, tokensIn, tokensOut, 0, 0, 0); err != nil { + if err := store.UpdateStageMetrics(ctx, stage.ID, cost, tokensIn, tokensOut, 0, 0, 0, ""); err != nil { t.Fatalf("UpdateStageMetrics: %v", err) } if model != "" { diff --git a/internal/observatory/iteration_post.go b/internal/observatory/iteration_post.go index 47ec2cf39..b62aeaf06 100644 --- a/internal/observatory/iteration_post.go +++ b/internal/observatory/iteration_post.go @@ -111,7 +111,8 @@ func PostIteration(ctx context.Context, backend *SQLiteBackend, p *IterationPost } // Metrics: cost + tokens (quota lanes post zeros, which is a no-op add). if st.CostUSD != 0 || st.TokensIn != 0 || st.TokensOut != 0 { - if err := store.UpdateStageMetrics(ctx, stage.ID, st.CostUSD, st.TokensIn, st.TokensOut, 0, 0, 0); err != nil { + // "" = provenance not classified by this poster; reads as unknown. + if err := store.UpdateStageMetrics(ctx, stage.ID, st.CostUSD, st.TokensIn, st.TokensOut, 0, 0, 0, ""); err != nil { return chain.ID, fmt.Errorf("update stage %d metrics: %w", i, err) } } diff --git a/internal/observatory/migrate.go b/internal/observatory/migrate.go index af0b1260f..422b664f0 100644 --- a/internal/observatory/migrate.go +++ b/internal/observatory/migrate.go @@ -496,6 +496,14 @@ func MigrateWithVersion(db *sql.DB) (int, error) { } } + // Migration v17: chain_stages.cost_provenance (metered vs subscription). + if currentVersion < 17 { + currentVersion, err = migrateV17(db, currentVersion) + if err != nil { + return currentVersion, err + } + } + return currentVersion, nil } diff --git a/internal/observatory/migrate_test.go b/internal/observatory/migrate_test.go index 86a4ff73d..3092e61e7 100644 --- a/internal/observatory/migrate_test.go +++ b/internal/observatory/migrate_test.go @@ -65,7 +65,7 @@ func TestMigrateWithVersion(t *testing.T) { if err != nil { t.Fatalf("MigrateWithVersion failed: %v", err) } - // Current schema version is 16: + // Current schema version is 17: // v1=base, v2=parent_task_id, v3=sessions, v4=remove unused tables, // v5=metrics+cache tokens, v6=chat_messages (M-CHAT-HISTORY-DB), // v7=execution_chains (M-CHAINS-SIMPLIFY), @@ -78,7 +78,8 @@ func TestMigrateWithVersion(t *testing.T) { // v14=trace_summaries workspace column (M-PERF-OBSERVATORY) // v15=eval_baselines table backfill (M-EVAL-OS-LONGITUDINAL Phase 2) // v16=ELO rating tables (M-EVAL-RATING-EFFICIENCY part 2) - expectedVersion := 16 + // v17=chain_stages.cost_provenance (metered vs subscription cost) + expectedVersion := 17 if version != expectedVersion { t.Errorf("expected version %d, got %d", expectedVersion, version) } @@ -88,8 +89,8 @@ func TestMigrateWithVersion(t *testing.T) { if err != nil { t.Fatalf("second MigrateWithVersion failed: %v", err) } - if version != 16 { - t.Errorf("expected version 16 on second call, got %d", version) + if version != 17 { + t.Errorf("expected version 17 on second call, got %d", version) } } @@ -131,8 +132,8 @@ func TestMigrateWithVersion_V15BackfillsEvalBaselines(t *testing.T) { if err != nil { t.Fatalf("v15 MigrateWithVersion failed: %v", err) } - if version != 16 { - t.Errorf("expected version 16 after backfill, got %d", version) + if version != 17 { + t.Errorf("expected version 17 after backfill, got %d", version) } if !tableExists(t, db, "eval_baselines") { t.Error("eval_baselines table should exist after the v15 migration") diff --git a/internal/observatory/migrate_v17.go b/internal/observatory/migrate_v17.go new file mode 100644 index 000000000..1342e6356 --- /dev/null +++ b/internal/observatory/migrate_v17.go @@ -0,0 +1,32 @@ +package observatory + +import ( + "database/sql" + "fmt" +) + +// migrateV17 adds chain_stages.cost_provenance — whether a stage's `cost` +// is money anyone was actually charged. +// +// Why (found 2026-07-30): ClassifyStageCost treated any `cost > 0` as +// authoritative reported spend, but the agent CLIs that authenticate by +// subscription (codex on `auth_mode: chatgpt`, claude on OAuth) still emit a +// non-zero cost figure that is never billed. A cohort mixing those with +// genuinely metered OpenRouter/Vertex stages blended notional and real dollars +// under one label — directly under the v1.0 `cost-per-verified-success` KPI, +// whose numerator is defined as attributable METERED dollars. +// +// Values mirror executor.CostProvenance: "metered", "list-price-equivalent", +// "free-local", "unknown". NULL/empty means the stage predates this column; +// it reads as unknown provenance, NEVER as metered — backfilling a guess is +// exactly the fabrication this column exists to stop. +func migrateV17(db *sql.DB, currentVersion int) (int, error) { + _, err := db.Exec("ALTER TABLE chain_stages ADD COLUMN cost_provenance TEXT") + if err != nil && !isColumnAlreadyExists(err) { + return currentVersion, fmt.Errorf("v17 add chain_stages.cost_provenance: %w", err) + } + if _, err := db.Exec("INSERT INTO schema_version (version) VALUES (17)"); err != nil { + return currentVersion, fmt.Errorf("failed to record version 17: %w", err) + } + return 17, nil +} diff --git a/internal/observatory/models_chains.go b/internal/observatory/models_chains.go index 055c2830b..82fbd0572 100644 --- a/internal/observatory/models_chains.go +++ b/internal/observatory/models_chains.go @@ -108,12 +108,17 @@ type ChainStage struct { CompletedAt *time.Time `json:"completed_at,omitempty"` // Summary (denormalized from spans) - Cost float64 `json:"cost"` - TokensIn int `json:"tokens_in"` - TokensOut int `json:"tokens_out"` - Turns int `json:"turns"` - ToolCalls int `json:"tool_calls"` - DurationMs int64 `json:"duration_ms"` + Cost float64 `json:"cost"` + // CostProvenance says whether Cost was actually billed: "metered", + // "list-price-equivalent" (subscription lane — real arithmetic, zero spend), + // "free-local", or "unknown". Empty = the stage predates the column and its + // provenance is unknown; it must NOT be read as metered. + CostProvenance string `json:"cost_provenance,omitempty"` + TokensIn int `json:"tokens_in"` + TokensOut int `json:"tokens_out"` + Turns int `json:"turns"` + ToolCalls int `json:"tool_calls"` + DurationMs int64 `json:"duration_ms"` // Error tracking ErrorMessage string `json:"error_message,omitempty"` diff --git a/internal/observatory/store_chains.go b/internal/observatory/store_chains.go index 555458b15..147dec1d6 100644 --- a/internal/observatory/store_chains.go +++ b/internal/observatory/store_chains.go @@ -352,6 +352,7 @@ func (s *Store) GetChainStages(ctx context.Context, chainID string, opts ChainRe handoff_to, iteration, human_feedback, started_at, completed_at, cost, tokens_in, tokens_out, turns, tool_calls, duration_ms, + COALESCE(cost_provenance, ''), error_message, error_count, eval_assessment FROM chain_stages @@ -379,6 +380,7 @@ func (s *Store) GetChainStages(ctx context.Context, chainID string, opts ChainRe &handoffTo, &stage.Iteration, &humanFeedback, &startedAt, &completedAt, &stage.Cost, &stage.TokensIn, &stage.TokensOut, &stage.Turns, &stage.ToolCalls, &stage.DurationMs, + &stage.CostProvenance, &errorMessage, &stage.ErrorCount, &evalAssessmentJSON, ) @@ -467,6 +469,7 @@ func (s *Store) GetStage(ctx context.Context, id string) (*ChainStage, error) { handoff_to, iteration, human_feedback, started_at, completed_at, cost, tokens_in, tokens_out, turns, tool_calls, duration_ms, + COALESCE(cost_provenance, ''), error_message, error_count FROM chain_stages WHERE id = ? `, id).Scan( @@ -476,6 +479,7 @@ func (s *Store) GetStage(ctx context.Context, id string) (*ChainStage, error) { &handoffTo, &stage.Iteration, &humanFeedback, &startedAt, &completedAt, &stage.Cost, &stage.TokensIn, &stage.TokensOut, &stage.Turns, &stage.ToolCalls, &stage.DurationMs, + &stage.CostProvenance, &errorMessage, &stage.ErrorCount, ) if err == sql.ErrNoRows { @@ -632,7 +636,7 @@ func (s *Store) UpdateStageApproval(ctx context.Context, stageID string, status } // UpdateStageMetrics updates the denormalized metrics on a stage. -func (s *Store) UpdateStageMetrics(ctx context.Context, stageID string, cost float64, tokensIn, tokensOut, turns, toolCalls int, durationMs int64) error { +func (s *Store) UpdateStageMetrics(ctx context.Context, stageID string, cost float64, tokensIn, tokensOut, turns, toolCalls int, durationMs int64, costProvenance string) error { if stageID == "" { return fmt.Errorf("stage_id is required") } @@ -644,9 +648,13 @@ func (s *Store) UpdateStageMetrics(ctx context.Context, stageID string, cost flo tokens_out = tokens_out + ?, turns = turns + ?, tool_calls = tool_calls + ?, - duration_ms = duration_ms + ? + duration_ms = duration_ms + ?, + -- Keep the first non-empty label. Stage metrics accumulate across + -- calls; a later caller that cannot classify must not erase what an + -- earlier one established. + cost_provenance = COALESCE(NULLIF(cost_provenance, ''), NULLIF(?, '')) WHERE id = ? - `, cost, tokensIn, tokensOut, turns, toolCalls, durationMs, stageID) + `, cost, tokensIn, tokensOut, turns, toolCalls, durationMs, costProvenance, stageID) if err != nil { return fmt.Errorf("failed to update stage metrics: %w", err) } diff --git a/internal/server/handlers_chains_cvs_test.go b/internal/server/handlers_chains_cvs_test.go index 14ac05c1f..ffd1a974f 100644 --- a/internal/server/handlers_chains_cvs_test.go +++ b/internal/server/handlers_chains_cvs_test.go @@ -74,7 +74,7 @@ func newKPITestServer(t *testing.T, sourceRef string, stages []cvsStage) (*Serve if err := store.UpdateStageEvalAssessment(ctx, stage.ID, a); err != nil { t.Fatalf("bank assessment %d: %v", i, err) } - if err := store.UpdateStageMetrics(ctx, stage.ID, s.cost, s.tokensIn, s.tokensOut, 0, 0, 0); err != nil { + if err := store.UpdateStageMetrics(ctx, stage.ID, s.cost, s.tokensIn, s.tokensOut, 0, 0, 0, ""); err != nil { t.Fatalf("bank metrics %d: %v", i, err) } } diff --git a/internal/storage/firestore/observatory_chains.go b/internal/storage/firestore/observatory_chains.go index a317949c9..287a54d27 100644 --- a/internal/storage/firestore/observatory_chains.go +++ b/internal/storage/firestore/observatory_chains.go @@ -337,15 +337,22 @@ func (s *ObservatoryStore) UpdateStageApproval(ctx context.Context, stageID stri return err } -func (s *ObservatoryStore) UpdateStageMetrics(ctx context.Context, stageID string, cost float64, tokensIn, tokensOut, turns, toolCalls int, durationMs int64) error { - _, err := s.client.Doc(collObsChainStages, stageID).Update(ctx, []firestore.Update{ +func (s *ObservatoryStore) UpdateStageMetrics(ctx context.Context, stageID string, cost float64, tokensIn, tokensOut, turns, toolCalls int, durationMs int64, costProvenance string) error { + updates := []firestore.Update{ {Path: "cost", Value: cost}, {Path: "tokens_in", Value: tokensIn}, {Path: "tokens_out", Value: tokensOut}, {Path: "turns", Value: turns}, {Path: "tool_calls", Value: toolCalls}, {Path: "duration_ms", Value: durationMs}, - }) + } + // Only write a label we actually have. An unclassified caller must not + // clear provenance an earlier one established (mirrors the SQLite + // COALESCE(NULLIF(...)) behaviour). + if costProvenance != "" { + updates = append(updates, firestore.Update{Path: "cost_provenance", Value: costProvenance}) + } + _, err := s.client.Doc(collObsChainStages, stageID).Update(ctx, updates) return err } From 3fafe53ed45d4169396bfcb001774620a200c857 Mon Sep 17 00:00:00 2001 From: "Voight-Kampff (bot)" <151556158+sunholo-voight-kampff@users.noreply.github.com> Date: Fri, 31 Jul 2026 12:48:12 +0200 Subject: [PATCH 2/2] refactor(executor): split cost/auth out of codex.go and claude.go MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The provenance and token-gate work pushed both executors past the 800-line check-file-sizes gate (codex 817, claude 843) — a CI failure, not a warning. Extracted into a cost.go per package: authLane() + CostModel() for both, plus getModel() for claude. These belong together — what a run costs and whether anyone was charged for it are the same question asked twice. codex 817 -> 761, claude 843 -> 778. No behaviour change. Co-Authored-By: Claude Opus 5 --- internal/executor/claude/claude.go | 65 ------------------------ internal/executor/claude/cost.go | 79 ++++++++++++++++++++++++++++++ internal/executor/codex/codex.go | 58 ---------------------- internal/executor/codex/cost.go | 71 +++++++++++++++++++++++++++ 4 files changed, 150 insertions(+), 123 deletions(-) create mode 100644 internal/executor/claude/cost.go create mode 100644 internal/executor/codex/cost.go diff --git a/internal/executor/claude/claude.go b/internal/executor/claude/claude.go index d89f6f721..43b42e2be 100644 --- a/internal/executor/claude/claude.go +++ b/internal/executor/claude/claude.go @@ -754,38 +754,6 @@ func (e *ClaudeExecutor) Capabilities() []executor.Capability { } } -// authLane reports whether claude runs are charged per token. -// -// Mirrors the M-CLOUD-DUAL-AUTH branch in Execute: AILANG_AUTH_MODE=apikey means -// ANTHROPIC_API_KEY drives a metered account; anything else is the OAuth -// subscription lane, where the CLI still emits a non-zero total_cost_usd that -// nobody is charged. On the eval rig the key is deliberately stripped, so the -// default is the common case, not an edge case. -func (e *ClaudeExecutor) authLane() executor.AuthLane { - if os.Getenv("AILANG_AUTH_MODE") == "apikey" { - return executor.AuthLaneBilled - } - return executor.AuthLaneSubscription -} - -// CostModel returns pricing information for cost calculations. -// -// NOT used for Result.CostUSD: the claude CLI reports its own -// total_cost_usd and the executor banks that figure directly. Kept because -// the Executor interface requires it and callers may use it for pre-flight -// estimates. Audited 2026-07-30 — do not assume this table is what gets -// banked. Note the CLI's figure is itself a list-price equivalent when the -// rig authenticates via OAuth subscription, not metered spend. -func (e *ClaudeExecutor) CostModel() *executor.CostModel { - // Default to Haiku pricing - return &executor.CostModel{ - ProviderName: "anthropic", - InputTokenCost: 0.001, // $1.00 per 1M - OutputTokenCost: 0.005, // $5.00 per 1M - CacheReadCost: 0.0001, // $0.10 per 1M - } -} - // HealthCheck verifies the executor is configured and accessible func (e *ClaudeExecutor) HealthCheck(ctx context.Context) error { // Check if claude binary exists @@ -799,39 +767,6 @@ func (e *ClaudeExecutor) HealthCheck(ctx context.Context) error { return nil } -// Close releases any resources held by the executor -func (e *ClaudeExecutor) Close() error { - return nil -} - -func (e *ClaudeExecutor) getModel(task *executor.Task) string { - if task.Model != "" { - return task.Model - } - return e.model -} - -// claudeHeadlessResult matches Claude CLI output structure -type claudeHeadlessResult struct { - Type string `json:"type"` - Subtype string `json:"subtype"` - IsError bool `json:"is_error"` - Result string `json:"result"` - NumTurns int `json:"num_turns"` - DurationMS int `json:"duration_ms"` - TotalCostUSD float64 `json:"total_cost_usd"` - SessionID string `json:"session_id"` - Usage claudeUsage `json:"usage"` -} - -type claudeUsage struct { - InputTokens int `json:"input_tokens"` - OutputTokens int `json:"output_tokens"` - CacheReadInputTokens int `json:"cache_read_input_tokens"` - CacheCreationInputTokens int `json:"cache_creation_input_tokens"` -} - -// Register registers the Claude executor with the global factory func Register() { executor.GlobalFactory().Register("claude", func(cfg *executor.Config) (executor.Executor, error) { return New(cfg) diff --git a/internal/executor/claude/cost.go b/internal/executor/claude/cost.go new file mode 100644 index 000000000..0e6ee6909 --- /dev/null +++ b/internal/executor/claude/cost.go @@ -0,0 +1,79 @@ +package claude + +import ( + "os" + + "github.com/sunholo-data/ailang/internal/executor" +) + +// Cost, auth-lane and model resolution for the claude harness. +// +// Split out of claude.go 2026-07-31 when the executor crossed the 800-line +// check-file-sizes gate. Grouped because all three answer "which model ran, +// under which account, at what price". + +// authLane reports whether claude runs are charged per token. +// +// Mirrors the M-CLOUD-DUAL-AUTH branch in Execute: AILANG_AUTH_MODE=apikey means +// ANTHROPIC_API_KEY drives a metered account; anything else is the OAuth +// subscription lane, where the CLI still emits a non-zero total_cost_usd that +// nobody is charged. On the eval rig the key is deliberately stripped, so the +// default is the common case, not an edge case. +func (e *ClaudeExecutor) authLane() executor.AuthLane { + if os.Getenv("AILANG_AUTH_MODE") == "apikey" { + return executor.AuthLaneBilled + } + return executor.AuthLaneSubscription +} + +// CostModel returns pricing information for cost calculations. +// +// NOT used for Result.CostUSD: the claude CLI reports its own +// total_cost_usd and the executor banks that figure directly. Kept because +// the Executor interface requires it and callers may use it for pre-flight +// estimates. Audited 2026-07-30 — do not assume this table is what gets +// banked. Note the CLI's figure is itself a list-price equivalent when the +// rig authenticates via OAuth subscription, not metered spend. +func (e *ClaudeExecutor) CostModel() *executor.CostModel { + // Default to Haiku pricing + return &executor.CostModel{ + ProviderName: "anthropic", + InputTokenCost: 0.001, // $1.00 per 1M + OutputTokenCost: 0.005, // $5.00 per 1M + CacheReadCost: 0.0001, // $0.10 per 1M + } +} + +// Close releases any resources held by the executor +func (e *ClaudeExecutor) Close() error { + return nil +} + +func (e *ClaudeExecutor) getModel(task *executor.Task) string { + if task.Model != "" { + return task.Model + } + return e.model +} + +// claudeHeadlessResult matches Claude CLI output structure +type claudeHeadlessResult struct { + Type string `json:"type"` + Subtype string `json:"subtype"` + IsError bool `json:"is_error"` + Result string `json:"result"` + NumTurns int `json:"num_turns"` + DurationMS int `json:"duration_ms"` + TotalCostUSD float64 `json:"total_cost_usd"` + SessionID string `json:"session_id"` + Usage claudeUsage `json:"usage"` +} + +type claudeUsage struct { + InputTokens int `json:"input_tokens"` + OutputTokens int `json:"output_tokens"` + CacheReadInputTokens int `json:"cache_read_input_tokens"` + CacheCreationInputTokens int `json:"cache_creation_input_tokens"` +} + +// Register registers the Claude executor with the global factory diff --git a/internal/executor/codex/codex.go b/internal/executor/codex/codex.go index 783321b0a..9f86ab947 100644 --- a/internal/executor/codex/codex.go +++ b/internal/executor/codex/codex.go @@ -9,11 +9,9 @@ package codex import ( "bufio" "context" - "encoding/json" "fmt" "os" "os/exec" - "path/filepath" "strings" "sync/atomic" "time" @@ -713,62 +711,6 @@ func (e *CodexExecutor) Capabilities() []executor.Capability { } } -// authLane reports whether codex runs are charged per token. -// -// Codex authenticates from ~/.codex/auth.json, written by `codex login`. An -// OPENAI_API_KEY in the environment does NOT override it — probe-verified -// 2026-07-30 against codex-cli 0.145.0 with auth_mode "chatgpt", where a -// deliberately invalid env key still ran clean. Reading the env var here would -// therefore report "billed" for a run the ChatGPT plan covered. -// -// Unreadable or unrecognised → Unknown. A wrong "metered" is the failure mode -// this exists to prevent, so it is never the fallback. -func (e *CodexExecutor) authLane() executor.AuthLane { - home, err := os.UserHomeDir() - if err != nil { - return executor.AuthLaneUnknown - } - data, err := os.ReadFile(filepath.Join(home, ".codex", "auth.json")) - if err != nil { - return executor.AuthLaneUnknown - } - var auth struct { - AuthMode string `json:"auth_mode"` - APIKey string `json:"OPENAI_API_KEY"` - } - if err := json.Unmarshal(data, &auth); err != nil { - return executor.AuthLaneUnknown - } - switch auth.AuthMode { - case "chatgpt": - return executor.AuthLaneSubscription - case "apikey": - return executor.AuthLaneBilled - } - // Older codex releases wrote the key with no auth_mode discriminator. - if auth.APIKey != "" { - return executor.AuthLaneBilled - } - return executor.AuthLaneUnknown -} - -// CostModel returns pricing for gpt-5-codex (the default Codex model). -// Source: https://platform.openai.com/docs/pricing -// gpt-5-codex: $1.25/$10.00 per 1M tokens = $0.00125/$0.01 per 1K. -// -// FALLBACK ONLY. The codex CLI runs whatever `--model` it is handed, so this -// table is correct for exactly one of them. Result.CostUSD is billed via -// executor.ResolveCostModel, which prefers Task.Pricing (the per-model rates -// from models.yml). This is reached only when a caller supplies no pricing. -func (e *CodexExecutor) CostModel() *executor.CostModel { - return &executor.CostModel{ - ProviderName: "openai", - InputTokenCost: 0.00125, - OutputTokenCost: 0.01, - CacheReadCost: 0.000125, - } -} - // HealthCheck verifies the codex binary exists on PATH and responds. func (e *CodexExecutor) HealthCheck(ctx context.Context) error { codexPath := e.codexPath diff --git a/internal/executor/codex/cost.go b/internal/executor/codex/cost.go new file mode 100644 index 000000000..a33833e77 --- /dev/null +++ b/internal/executor/codex/cost.go @@ -0,0 +1,71 @@ +package codex + +import ( + "encoding/json" + "os" + "path/filepath" + + "github.com/sunholo-data/ailang/internal/executor" +) + +// Cost and auth-lane resolution for the codex harness. +// +// Split out of codex.go 2026-07-31 when the executor crossed the 800-line +// check-file-sizes gate. These two concerns belong together: what a run costs +// and whether anyone was charged for it are the same question asked twice. + +// authLane reports whether codex runs are charged per token. +// +// Codex authenticates from ~/.codex/auth.json, written by `codex login`. An +// OPENAI_API_KEY in the environment does NOT override it — probe-verified +// 2026-07-30 against codex-cli 0.145.0 with auth_mode "chatgpt", where a +// deliberately invalid env key still ran clean. Reading the env var here would +// therefore report "billed" for a run the ChatGPT plan covered. +// +// Unreadable or unrecognised → Unknown. A wrong "metered" is the failure mode +// this exists to prevent, so it is never the fallback. +func (e *CodexExecutor) authLane() executor.AuthLane { + home, err := os.UserHomeDir() + if err != nil { + return executor.AuthLaneUnknown + } + data, err := os.ReadFile(filepath.Join(home, ".codex", "auth.json")) + if err != nil { + return executor.AuthLaneUnknown + } + var auth struct { + AuthMode string `json:"auth_mode"` + APIKey string `json:"OPENAI_API_KEY"` + } + if err := json.Unmarshal(data, &auth); err != nil { + return executor.AuthLaneUnknown + } + switch auth.AuthMode { + case "chatgpt": + return executor.AuthLaneSubscription + case "apikey": + return executor.AuthLaneBilled + } + // Older codex releases wrote the key with no auth_mode discriminator. + if auth.APIKey != "" { + return executor.AuthLaneBilled + } + return executor.AuthLaneUnknown +} + +// CostModel returns pricing for gpt-5-codex (the default Codex model). +// Source: https://platform.openai.com/docs/pricing +// gpt-5-codex: $1.25/$10.00 per 1M tokens = $0.00125/$0.01 per 1K. +// +// FALLBACK ONLY. The codex CLI runs whatever `--model` it is handed, so this +// table is correct for exactly one of them. Result.CostUSD is billed via +// executor.ResolveCostModel, which prefers Task.Pricing (the per-model rates +// from models.yml). This is reached only when a caller supplies no pricing. +func (e *CodexExecutor) CostModel() *executor.CostModel { + return &executor.CostModel{ + ProviderName: "openai", + InputTokenCost: 0.00125, + OutputTokenCost: 0.01, + CacheReadCost: 0.000125, + } +}