From 7f0379a870c39f7e18b3a2479861fd2805f4e3ab Mon Sep 17 00:00:00 2001 From: Osher Elhadad Date: Sat, 25 Jul 2026 11:50:59 +0000 Subject: [PATCH 1/3] feat(release): cache-aware compaction, LLM extract, and production hardening MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The next-release cut of context-guru. Ships the cache-aware compaction redesign and the LLM relevance-trimmer, then hardens the whole path for production based on a multi-agent review (security, generalizability, bugs, dead code, tests, docs). Presets & defaults - New default presets shipping the SWE-bench-winning config: `codesmart` (cache-aware `[format, dedup, failed_run, cmdfilter, extract_llm, extract, cacheinject]`, extract_llm routed to CHEAP_MODEL and gated so most turns make no model call) and `codesafe` (deterministic-only, zero model calls by policy). Presets can now carry tuned per-component config (presetConfigs); the proxy default is `codesmart`. Without a cheap model, extract_llm safely no-ops. - Removed the `phi_evict` component (superseded by `mask`). Cache stability (the release's core invariant) - `mask` now freezes its decision and replays it byte-identically every turn, so a masked output that slides into the cached prefix no longer reverts to full and churns the provider KV cache (previously only `failed_run` did this). - Every offloader now honors kept-verbatim + existing-marker skips uniformly, so content the agent just expanded is never re-compacted into an expand-loop. - GET /expand is scoped to the owning session (per-(session,key) ownership record), closing a cross-session/tenant disclosure of stashed originals. Correctness fixes - stripTerminalNoise no longer blanks CRLF-terminated tool output (the trailing CR is the line separator, not an in-line progress redraw). - AggregateSSE reconstructs extended-thinking blocks (thinking_delta + signature_delta), so the expand continuation builds a valid assistant turn. - UTF-8-safe truncation in headPeek and the dump trace (no split runes). - resolveCacheAware detects a real cache_control/cachePoint breakpoint structurally instead of substring-scanning the whole body (no false positives from tool text). Reliability / security hardening - Top-level fail-open recover around apply.BodyFull, the proxy pre-forward path, and emitter callbacks — a panic outside a component now forwards the original request. - Request bodies are bounded (http.MaxBytesReader) to prevent memory-exhaustion DoS. - Starlark sandbox gains an input-size cap and a heap-growth watchdog (the step/time limits can't preempt a native op); skeleton's tree walk gains a depth bound (an unbounded recursion is an uncatchable stack-overflow crash). - modelinfo resolves the model window via a single-flighted, non-blocking background fetch with negative caching (was a synchronous hot-path GitHub GET, refetch-on-fail). - Debug/dump files created 0600 (they persist raw payloads). Cleanup - Removed dead code (internal/extract/cache.go, RunExtractionBatch/buildBatchPrompt, lastUserText, overlap, writeBuffered, modelinfo.Default + unused `once`) and internal-only docs (docs/superpowers/). Retired stale phi_evict references and visualizations. Tests & docs - New regression tests: mask byte-stability across turns, kept-verbatim skip, CRLF preservation + interior-CR redraw, headPeek UTF-8, AggregateSSE thinking-block preservation, modelinfo single-flight/non-blocking, /expand session scoping, and rich-preset config resolution. Full suite + race detector green; mkdocs --strict clean. - README + docs overhaul with the three-way SWE-bench results (cheapest arm, beats headroom on cost/cache/steps/reward), per-component live-captured before/after examples, and corrected build/quickstart. Assisted-By: Claude (Anthropic) Signed-off-by: Osher Elhadad --- README.md | 137 ++- apply/apply.go | 145 ++- cmd/context-guru-proxy/main.go | 25 +- components/all/llm_test.go | 47 +- components/all/more_test.go | 107 ++ components/all/p4_test.go | 55 +- components/all/reuse_test.go | 7 +- components/component.go | 28 + components/offload/cmdfilter.go | 5 + components/offload/collapse.go | 37 +- components/offload/common.go | 203 +++- components/offload/common_test.go | 93 ++ components/offload/dedup.go | 12 +- components/offload/extract.go | 180 +--- components/offload/extract_llm.go | 318 ++++++ components/offload/failed_run.go | 55 +- components/offload/marker.go | 52 + components/offload/mask.go | 59 +- components/offload/ownership_test.go | 30 + components/offload/phi_evict.go | 136 --- components/offload/skeleton.go | 33 +- components/offload/smartcrush.go | 13 +- components/offload/state.go | 127 +++ components/offload/summarize.go | 2 +- components/pipeline.go | 12 +- components/trigger.go | 61 +- components/trigger_test.go | 47 +- config/config.go | 85 +- config/config_test.go | 61 ++ deploy/harbor/analyze.py | 144 +++ deploy/harbor/analyze_content.py | 116 +++ deploy/harbor/cachecost.py | 171 +++ deploy/harbor/deep_analysis.py | 133 +++ deploy/harbor/deep_plots.py | 159 +++ deploy/harbor/diffsum.py | 104 ++ deploy/harbor/dump_unique.py | 79 ++ deploy/harbor/gen_result_docs.py | 66 ++ deploy/harbor/measure.py | 154 +++ deploy/harbor/plots.py | 196 ++++ deploy/harbor/replay.py | 137 +++ deploy/harbor/replay2.py | 203 ++++ deploy/harbor/run-bench.sh | 15 + deploy/harbor/run-proxy.sh | 18 + deploy/harbor/swebench.py | 344 ++++++ deploy/harbor/sweep.py | 213 ++++ docs/RESULTS.md | 317 +----- docs/components.md | 124 ++- docs/components/extract.md | 103 +- docs/components/extract_llm.md | 91 ++ docs/components/phi_evict.md | 57 - docs/design.md | 2 +- docs/examples/live-captures.md | 169 +++ docs/get-started/overview.md | 2 +- docs/get-started/quickstart-proxy.md | 82 +- docs/how-to/choose-a-preset.md | 5 +- docs/img/benchmark/components.png | Bin 0 -> 44170 bytes docs/img/benchmark/cost_decomposition.png | Bin 0 -> 32184 bytes docs/img/benchmark/headline.png | Bin 0 -> 105020 bytes docs/img/benchmark/per_task_cost.png | Bin 0 -> 130244 bytes docs/img/benchmark/per_task_dcost.png | Bin 0 -> 98014 bytes docs/img/benchmark/per_task_dsteps.png | Bin 0 -> 96637 bytes docs/javascripts/charts.js | 76 +- docs/javascripts/mathjax.js | 2 +- docs/reference/presets.md | 6 +- docs/results/REPRODUCE.md | 124 +++ docs/results/baseline.md | 64 ++ docs/results/comparison.md | 87 ++ docs/results/components.md | 172 +++ docs/results/context-guru.md | 66 ++ docs/results/headroom.md | 64 ++ docs/setup.md | 26 +- ...-06-24-deepen-components-with-libraries.md | 984 ------------------ ...-06-24-validate-config-measure-document.md | 69 -- expand/expand.go | 56 + expand/inject.go | 76 ++ expand/inject_test.go | 122 +++ expand/sse.go | 145 +++ expand/sse_test.go | 56 + internal/cheapmodel/anthropic.go | 5 + internal/cheapmodel/openai.go | 5 + internal/cheapmodel/usage.go | 30 + internal/extract/cache.go | 27 - internal/extract/extract.go | 17 +- internal/extract/prompt.go | 189 +++- internal/extract/starlark.go | 119 ++- internal/extract/starlark_reassign_test.go | 50 + internal/extract/starlark_test.go | 4 +- internal/extract/zz_live_test.go | 50 +- internal/modelinfo/modelinfo.go | 212 ++++ internal/modelinfo/modelinfo_test.go | 111 ++ internal/tokens/tokens.go | 59 +- internal/tokens/tokens_test.go | 20 + metrics/metrics.go | 104 +- metrics/metrics_test.go | 41 + mkdocs.yml | 16 +- overrides/home.html | 22 +- proxy/proxy.go | 215 +++- proxy/proxy_test.go | 70 ++ 98 files changed, 6735 insertions(+), 2202 deletions(-) create mode 100644 components/offload/common_test.go create mode 100644 components/offload/extract_llm.go create mode 100644 components/offload/ownership_test.go delete mode 100644 components/offload/phi_evict.go create mode 100644 deploy/harbor/analyze.py create mode 100644 deploy/harbor/analyze_content.py create mode 100644 deploy/harbor/cachecost.py create mode 100644 deploy/harbor/deep_analysis.py create mode 100644 deploy/harbor/deep_plots.py create mode 100644 deploy/harbor/diffsum.py create mode 100644 deploy/harbor/dump_unique.py create mode 100644 deploy/harbor/gen_result_docs.py create mode 100644 deploy/harbor/measure.py create mode 100644 deploy/harbor/plots.py create mode 100644 deploy/harbor/replay.py create mode 100644 deploy/harbor/replay2.py create mode 100755 deploy/harbor/run-bench.sh create mode 100755 deploy/harbor/run-proxy.sh create mode 100644 deploy/harbor/swebench.py create mode 100644 deploy/harbor/sweep.py create mode 100644 docs/components/extract_llm.md delete mode 100644 docs/components/phi_evict.md create mode 100644 docs/examples/live-captures.md create mode 100644 docs/img/benchmark/components.png create mode 100644 docs/img/benchmark/cost_decomposition.png create mode 100644 docs/img/benchmark/headline.png create mode 100644 docs/img/benchmark/per_task_cost.png create mode 100644 docs/img/benchmark/per_task_dcost.png create mode 100644 docs/img/benchmark/per_task_dsteps.png create mode 100644 docs/results/REPRODUCE.md create mode 100644 docs/results/baseline.md create mode 100644 docs/results/comparison.md create mode 100644 docs/results/components.md create mode 100644 docs/results/context-guru.md create mode 100644 docs/results/headroom.md delete mode 100644 docs/superpowers/plans/2026-06-24-deepen-components-with-libraries.md delete mode 100644 docs/superpowers/plans/2026-06-24-validate-config-measure-document.md create mode 100644 expand/inject.go create mode 100644 expand/inject_test.go create mode 100644 expand/sse.go create mode 100644 expand/sse_test.go create mode 100644 internal/cheapmodel/usage.go delete mode 100644 internal/extract/cache.go create mode 100644 internal/extract/starlark_reassign_test.go create mode 100644 internal/modelinfo/modelinfo.go create mode 100644 internal/modelinfo/modelinfo_test.go diff --git a/README.md b/README.md index e1e28bc..d703e81 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,53 @@ +
+ # context-guru -Provider-agnostic **context engineering** for LLM agents: a Go library that shrinks the -tokens a request carries — losslessly, or lossy-but-reversible — without touching the -agent. Same core runs as an **HTTP proxy/gateway** or an **in-process plugin**. +**Provider-agnostic context engineering for LLM agents.** +Shrink the tokens every request carries — losslessly, or lossy-but-reversibly — **without touching the agent.** + +[![Docs](https://img.shields.io/badge/docs-online-009688.svg)](https://rossoctl.github.io/context-guru/) +[![Go Reference](https://img.shields.io/badge/pkg.go.dev-reference-007d9c.svg)](https://pkg.go.dev/github.com/rossoctl/context-guru) +[![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](LICENSE) +[![Go 1.26](https://img.shields.io/badge/go-1.26-00ADD8.svg)](go.mod) + +[Quickstart](#quickstart-60-seconds) · [Why it wins](#benchmark-the-cheapest--highest-reward-arm-on-swe-bench-verified) · [Components](#the-pipeline) · [Docs](https://rossoctl.github.io/context-guru/) · [Reproduce](docs/results/REPRODUCE.md) + +
+ +--- -- **Fail open, always** — any component error/panic reverts *that component only*; the - original request is always a valid fallback. -- **Never worse** — a component that grows the request is reverted. +context-guru is a single Go core that reduces the token cost of LLM-agent traffic. The **same core** +runs as an **HTTP proxy/gateway** (drop-in, any language, zero agent changes) or as an **in-process +plugin**. It operates on the messages array — dropping redundant tool output, collapsing superseded +runs, projecting large reads down to what's relevant — and every reduction is **safe by construction**: + +- **Fail open, always** — any component error or panic reverts *that component only*; the original + request is always a valid fallback. +- **Never worse** — a component that would grow a message is reverted. You never pay to compact. - **Reversible** — every lossy drop leaves a `<>` marker and stashes the original, recoverable via a model-callable `context_guru_expand` tool or `GET /expand`. +## Benchmark: the cheapest & highest-reward arm on SWE-bench Verified + +Evaluated **live, end-to-end**, with the **claude-code** agent on **`aws/claude-sonnet-5`**, against a +no-compaction baseline and against [**headroom**](https://pypi.org/project/headroom-ai/) +(`headroom-ai` v0.32.1). 50 tasks, matched on the 48 that scored under all three arms. + +| dimension | baseline | **context-guru** | headroom | +|---|--:|--:|--:| +| reward (solved / 48) | 43 | **42** | 40 | +| **total billed cost** | $29.73 | **$25.71 (−13.5%)** | $28.19 (−5.2%) | +| cache-read tokens | 96.8M | **80.6M (−16.8%)** | 91.1M (−5.9%) | +| cache-write tokens | 1.77M | **1.70M** | 1.76M | +| mean steps / task | 35.5 | **31.0** | 34.6 | +| added latency / req | — | 117 ms | 63 ms | + +**context-guru is the cheapest arm and beats headroom on cost, cache usage, steps, and reward** — because +it *freezes each compaction and replays it byte-identically every turn*, compounding the cache-read saving +across the whole session while never mutating the cached prefix. headroom keeps an edge on added latency +(it is fully deterministic). Full three-way study, per-task/per-component breakdowns, real before→after +examples, and how to reproduce: **[docs/RESULTS.md](docs/RESULTS.md)**. + ## Architecture ```mermaid @@ -33,55 +71,96 @@ Components implement one of two lossiness-typed interfaces and are stacked in co ```mermaid flowchart TD C["Component — Name() · Enabled(ctx)"] - C --> R["Reformat: lossless repack
format · cacheinject"] - C --> O["Offload: drop + stash, returns cache_keys
skeleton · dedup · collapse · failed_run
cmdfilter · extract · smartcrush · mask · phi_evict"] + C --> R["Reformat: lossless repack
format · toon · cacheinject"] + C --> O["Offload: drop + stash, returns cache_keys
skeleton · dedup · collapse · failed_run
cmdfilter · extract · extract_llm · smartcrush · mask · summarize"] ``` ## Install -Requires **Go 1.26** and a **C toolchain** (`CGO_ENABLED=1`; the `skeleton` component uses -tree-sitter via cgo). The module pins bifrost with a local `replace` to `../bifrost/core`, -so build from the parent directory that holds both repos: +Requires **Go 1.26** and a **C toolchain** (`CGO_ENABLED=1`). Build from the repo root: ```sh -cd .../context-engineering # dir containing lab-context-engineering/ and bifrost/ -CGO_ENABLED=1 go build -o bin/context-guru-proxy \ - ./lab-context-engineering/cmd/context-guru-proxy +CGO_ENABLED=1 go build -tags cg_skeleton -o bin/context-guru-proxy ./cmd/context-guru-proxy ``` -Or build the gateway image (see [docs/setup.md](docs/setup.md)): +The `cg_skeleton` build tag pulls in tree-sitter (via cgo) so the `skeleton` component can parse code. +It is **optional** — omit the tag and the tree-sitter dependency for a pure-Go build; the `skeleton` +component is simply inert without it, everything else works. Or build the gateway image +(see [docs/setup.md](docs/setup.md)): ```sh -docker build -f lab-context-engineering/Dockerfile -t context-guru:local . +docker build -t context-guru:local . ``` -## Run the proxy +## Quickstart (60 seconds) ```sh -context-guru-proxy --preset balanced # or --config cg.yaml +# 1 — run the proxy (ships with the SWE-bench-winning cache-aware config by default) +./bin/context-guru-proxy # --preset codesmart; listens on :4000 (LISTEN_ADDR to change) + +# 2 — point any agent at it (one port serves both dialects) +export ANTHROPIC_BASE_URL=http://localhost:4000/anthropic +export OPENAI_BASE_URL=http://localhost:4000/openai/v1 +claude # e.g. Claude Code + +# 3 — watch the savings add up +curl -s localhost:4000/stats | jq # token-weighted savings rollup ``` -Point any agent at it (one port serves both dialects): +Or drive it directly with an Anthropic-style request (this is exactly how the quickstart is tested — see +[docs/get-started/quickstart-proxy.md](docs/get-started/quickstart-proxy.md)): ```sh -ANTHROPIC_BASE_URL=http://localhost:4000/anthropic -OPENAI_BASE_URL=http://localhost:4000/openai/v1 +curl -s localhost:4000/anthropic/v1/messages \ + -H 'content-type: application/json' \ + -H "Authorization: Bearer $YOUR_KEY" \ + -d '{"model":"...","max_tokens":64,"messages":[ ... ]}' ``` +Presets: **`codesmart`** (the default — the SWE-bench-winning cache-aware config +`[format, dedup, failed_run, cmdfilter, extract_llm, extract, cacheinject]`), **`codesafe`** (the same +minus the LLM pass — deterministic-only `[format, dedup, failed_run, cmdfilter, extract, collapse, cacheinject]`, +zero model calls by policy), plus `general`, `agent`, `coding`, `mcp`, `balanced`, `safe`, `summarize`, `off`. +`codesmart`'s LLM relevance-trimmer (`extract_llm`) engages only when a cheap model is configured +(`CHEAP_MODEL*`); without one it safely no-ops and behaves like `codesafe`. +See [docs/components.md](docs/components.md) and [docs/reference/presets.md](docs/reference/presets.md). + | Flag / env | Default | Purpose | |---|---|---| -| `--preset` / `PRESET` | `balanced` | pipeline preset when no `--config` | +| `--preset` / `PRESET` | `codesmart` | pipeline preset when no `--config` | | `--config` / `CONFIG` | — | YAML config (overrides preset) | | `LISTEN_ADDR` | `:4000` | listen address | -| `--openai-upstream` / `OPENAI_UPSTREAM` | `https://api.openai.com` | OpenAI upstream base | | `--anthropic-upstream` / `ANTHROPIC_UPSTREAM` | `https://api.anthropic.com` | Anthropic upstream base | +| `--openai-upstream` / `OPENAI_UPSTREAM` | `https://api.openai.com` | OpenAI upstream base | | `OPENAI_API_KEY` / `ANTHROPIC_API_KEY` | — | real key injected on forward (gateway mode); empty = pass client auth through | +| `CHEAP_MODEL` (+ `CHEAP_MODEL_*`) | — | dedicated cheap model for the LLM components (`extract_llm`, `summarize`) | | `FORCE_MODEL` | — | overwrite the request `model` (eval-containers `EVAL_MODEL`) | Routes: `POST /openai/v1/chat/completions`, `POST /anthropic/v1/messages`, `GET /healthz`, -`GET /stats` (savings rollups), `GET /expand?id=` (recover an offloaded original). -Per-request: header `x-context-guru-session` sets the session key; `x-context-guru-bypass: true` -skips the pipeline. +`GET /stats` (savings rollups), `GET /expand?id=` (recover an offloaded original). Per-request: header +`x-context-guru-session` sets the session key; `x-context-guru-bypass: true` skips the pipeline. + +## The pipeline + +Every component operates on tool-output messages. **Reformat** = lossless repack; **Offload** = drop +bytes, stash the original, leave a recoverable marker. Real, live-captured before→after examples for each +are in **[docs/components.md](docs/components.md)** and **[docs/results/components.md](docs/results/components.md)**. + +| Component | Kind | What it does | +|---|---|---| +| `format` | Reformat | re-encodes pretty JSON tool output as compact JSON | +| `toon` | Reformat | re-encodes a uniform JSON array as TOON (header once, one row per item) | +| `cacheinject` | Reformat | adds an Anthropic `cache_control` breakpoint on a stable prefix boundary | +| `dedup` | Offload | replaces a byte-identical earlier tool output with a pointer | +| `failed_run` | Offload | collapses superseded test/build runs, keeps the latest in full | +| `cmdfilter` | Offload | shrinks structured command output via declarative DSL filters | +| `extract` | Offload | deterministic noise collapse (repeated lines, blank runs, progress bars) | +| `extract_llm` | Offload (LLM) | a cheap model writes a sandboxed filter that trims to what's relevant | +| `collapse` | Offload | head/tail window on any oversized output (last-resort fallback) | +| `mask` | Offload | age-based GC — keep the newest N tool outputs, stash older ones | +| `skeleton` | Offload | replaces code-block function bodies with signatures (needs `cg_skeleton`) | +| `smartcrush` | Offload | keeps anchor items of a long JSON array, drops the middle | +| `summarize` | Offload (LLM) | compresses the middle of the trajectory into one summary (run alone) | ## Integrate @@ -96,11 +175,11 @@ Details in [docs/integrations.md](docs/integrations.md). ## Docs - [docs/design.md](docs/design.md) — architecture: component model, fail-open pipeline, store, session, expand loop, metrics. -- [docs/components.md](docs/components.md) — every registered component: how it works, before→after, lossiness, config, best use. +- [docs/components.md](docs/components.md) — every registered component: how it works, live before→after, lossiness, config, best use. - [docs/integrations.md](docs/integrations.md) — proxy gateway vs AuthBridge plugin, with request paths. - [docs/setup.md](docs/setup.md) — setup + a concrete SWE-bench run through the eval-containers gateway. -- [docs/RESULTS.md](docs/RESULTS.md) — per-component SWE-bench benchmark (Claude Code, claude-sonnet-4-6): `mask` ≈27% token savings, no reward loss. +- [docs/RESULTS.md](docs/RESULTS.md) — the live three-way SWE-bench Verified benchmark (Claude Code, `aws/claude-sonnet-5`): context-guru is the cheapest arm (−13.5% billed cost vs baseline) and beats headroom on cost, cache usage, steps, and reward. ## License -Apache-2.0. See [LICENSE](LICENSE). +Apache-2.0. See [LICENSE](LICENSE). A [Rossoctl](https://github.com/rossoctl) platform component. diff --git a/apply/apply.go b/apply/apply.go index abffd81..7584c30 100644 --- a/apply/apply.go +++ b/apply/apply.go @@ -26,6 +26,7 @@ import ( "reflect" "strconv" "strings" + "unicode/utf8" bschemas "github.com/maximhq/bifrost/core/schemas" "github.com/rossoctl/context-guru/components" @@ -95,6 +96,34 @@ func Body(ctx context.Context, pipe *components.Pipeline, st store.Store, provid // open. explicitSession is the host-supplied session id ("" -> content hash). // models carries the LLM clients that NeedsModel components may call. func BodyWithModel(ctx context.Context, pipe *components.Pipeline, st store.Store, provider bschemas.ModelProvider, body []byte, explicitSession string, bypass bool, models components.ModelSpec) ([]byte, bool) { + return BodyWithModelWindow(ctx, pipe, st, provider, body, explicitSession, bypass, models, 0) +} + +// BodyWithModelWindow is BodyWithModel plus the model's resolved context window +// (max input tokens, 0 = unknown) so fraction-based Trigger thresholds can scale +// with the model. Hosts that resolve the window (the proxy, via internal/modelinfo) +// call this; window=0 reproduces the pre-D3 behavior exactly. +func BodyWithModelWindow(ctx context.Context, pipe *components.Pipeline, st store.Store, provider bschemas.ModelProvider, body []byte, explicitSession string, bypass bool, models components.ModelSpec, window int) ([]byte, bool) { + return BodyFull(ctx, pipe, st, provider, body, explicitSession, bypass, models, window, "auto") +} + +// BodyFull is BodyWithModelWindow plus the cache mode ("auto"|"on"|"off", +// default "auto") controlling cache-aware compaction. "auto" turns on +// cache-awareness when the backend is a prompt-caching provider or the request +// already carries cache_control breakpoints; "on" forces it; "off" restores the +// legacy compact-everything behavior (correct for confirmed non-caching backends). +func BodyFull(ctx context.Context, pipe *components.Pipeline, st store.Store, provider bschemas.ModelProvider, body []byte, explicitSession string, bypass bool, models components.ModelSpec, window int, cacheMode string) (result []byte, changedBody bool) { + // Top-level fail-open backstop: the per-component recover in pipeline.runOne only + // covers component code. A panic anywhere else on the rewrite path (normalize, the + // sjson splice, rebuildCountChanged, a marshal) must NOT 500 the client — forward + // the original body unchanged. This makes CLAUDE.md's fail-open invariant hold for + // the whole entry point, not just inside components. + defer func() { + if r := recover(); r != nil { + slog.Error("context-guru: recovered from panic in BodyFull; forwarding original request", "panic", r) + result, changedBody = body, false + } + }() msgsRaw := gjson.GetBytes(body, "messages") if !msgsRaw.Exists() || !msgsRaw.IsArray() { return body, false @@ -110,12 +139,26 @@ func BodyWithModel(ctx context.Context, pipe *components.Pipeline, st store.Stor } chat := &bschemas.BifrostChatRequest{Provider: provider, Input: norm} sys, firstUser := systemAndFirstUser(norm) + sessionID := session.Resolve(explicitSession, sys, firstUser) + cacheAware := resolveCacheAware(cacheMode, provider, body) + maxCachedIdx := -1 + if cacheAware && !bypass { + // Messages present on the previous turn of this session are already committed + // to the provider cache; only the new tail is being cache-written this turn. + // Restrict supersession/age offloaders to that tail so they never mutate the + // cached prefix. Growth-based (dialect-agnostic; needs no cache_control mapping). + maxCachedIdx = prevLen(st, sessionID) - 1 + defer putLen(st, sessionID, len(norm)) + } c := &components.Ctx{ - Ctx: ctx, - Session: session.Resolve(explicitSession, sys, firstUser), - Store: st, - Model: models, - Bypass: bypass, + Ctx: ctx, + Session: sessionID, + Store: st, + Model: models, + Bypass: bypass, + CtxWindow: window, + CacheAware: cacheAware, + MaxCachedIdx: maxCachedIdx, } // Canonical form of each normalized message BEFORE the pipeline, so a @@ -181,6 +224,92 @@ func BodyWithModel(ctx context.Context, pipe *components.Pipeline, st store.Stor return out, changed } +// resolveCacheAware decides whether cache-aware compaction is active for this +// request. "off" disables it; "on" forces it; "auto" (default) enables it when the +// backend is a prompt-caching provider OR the request already carries cache_control +// breakpoints (so we assume caching even when the provider isn't in the static set — +// covers "backend caches but we can't see it from the provider name"). +func resolveCacheAware(mode string, provider bschemas.ModelProvider, body []byte) bool { + switch mode { + case "off": + return false + case "on": + return true + default: // "auto" / "" + switch provider { + case bschemas.Anthropic, bschemas.Bedrock, bschemas.BedrockMantle, bschemas.Vertex: + return true + } + return hasCacheBreakpoint(body) + } +} + +// hasCacheBreakpoint reports whether the request carries a REAL prompt-cache +// breakpoint — a structural cache_control / cachePoint field on a message, content +// block, system block, or tool. This is deliberately structural (gjson path queries), +// not a substring scan of the whole body: a tool output whose text merely contains the +// string "cache_control" must NOT flip the request into cache-aware mode (that would +// wrongly restrict offloaders to the tail on a non-caching backend). A key inside a +// JSON string value never matches these paths. +func hasCacheBreakpoint(body []byte) bool { + paths := []string{ + "messages.#.content.#.cache_control", + "messages.#.cache_control", + "system.#.cache_control", + "tools.#.cache_control", + "messages.#.content.#.cachePoint", + } + for _, p := range paths { + r := gjson.GetBytes(body, p) + if !r.Exists() { + continue + } + found := false + r.ForEach(func(_, v gjson.Result) bool { + // nested arrays (content-of-messages) surface as arrays here; recurse one level + if v.IsArray() { + v.ForEach(func(_, vv gjson.Result) bool { + if vv.IsObject() { + found = true + return false + } + return true + }) + } else if v.IsObject() { + found = true + } + return !found + }) + if found { + return true + } + } + return false +} + +// prevLen / putLen track, per session, how many normalized messages the previous +// turn carried — the boundary between the already-cached prefix and this turn's +// uncached tail. Stored in the same Store as offload state (TTL+LRU); a miss (first +// turn / expired) yields 0 so the whole request is treated as tail. +func prevLen(st store.Store, session string) int { + b, ok := st.Get("cg:len:" + session) + if !ok || len(b) == 0 { + return 0 + } + n := 0 + for _, ch := range b { + if ch < '0' || ch > '9' { + return 0 + } + n = n*10 + int(ch-'0') + } + return n +} + +func putLen(st store.Store, session string, n int) { + st.Put("cg:len:"+session, []byte(strconv.Itoa(n))) +} + // change is one rewritten message, captured for the CONTEXT_GURU_DUMP trace so a // human can see exactly what context-guru did to the wire. type change struct { @@ -202,6 +331,10 @@ func clip(s string, n int) string { if len(s) <= n { return s } + // truncate on a rune boundary so the trace stays valid UTF-8 + for n > 0 && !utf8.RuneStart(s[n]) { + n-- + } return s[:n] + "…[+" + strconv.Itoa(len(s)-n) + " bytes]" } @@ -209,7 +342,7 @@ var dumpPath = os.Getenv("CONTEXT_GURU_DUMP") // dumpChanges appends one JSON line describing this request's rewrites. func dumpChanges(session string, changes []change) { - f, err := os.OpenFile(dumpPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644) + f, err := os.OpenFile(dumpPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o600) if err != nil { return } diff --git a/cmd/context-guru-proxy/main.go b/cmd/context-guru-proxy/main.go index 06958b4..0a0d08d 100644 --- a/cmd/context-guru-proxy/main.go +++ b/cmd/context-guru-proxy/main.go @@ -22,6 +22,7 @@ import ( _ "github.com/rossoctl/context-guru/components/all" "github.com/rossoctl/context-guru/config" "github.com/rossoctl/context-guru/internal/cheapmodel" + "github.com/rossoctl/context-guru/internal/modelinfo" "github.com/rossoctl/context-guru/metrics" "github.com/rossoctl/context-guru/proxy" ) @@ -30,7 +31,7 @@ func main() { var ( addr = envOr("LISTEN_ADDR", ":4000") cfgPath = flag.String("config", envOr("CONFIG", ""), "path to context-guru YAML config") - preset = flag.String("preset", envOr("PRESET", "balanced"), "preset to use when --config is absent") + preset = flag.String("preset", envOr("PRESET", "codesmart"), "preset to use when --config is absent (codesmart = the SWE-bench-winning cache-aware config; codesafe = deterministic-only)") openai = flag.String("openai-upstream", envOr("OPENAI_UPSTREAM", "https://api.openai.com"), "OpenAI upstream base URL") anthropic = flag.String("anthropic-upstream", envOr("ANTHROPIC_UPSTREAM", "https://api.anthropic.com"), "Anthropic upstream base URL") bob = flag.String("bob-upstream", envOr("BOB_UPSTREAM", ""), "Bob (BobShell) backend base URL; enables the Bob gateway routes when set (e.g. https://api.us-east.bob.ibm.com)") @@ -61,8 +62,12 @@ func main() { // via env); the agent holds only a placeholder. Empty => pass client auth. OpenAIKey: os.Getenv("OPENAI_API_KEY"), AnthropicKey: os.Getenv("ANTHROPIC_API_KEY"), - ForceModel: os.Getenv("FORCE_MODEL"), // eval-containers pins EVAL_MODEL's model here - CheapModel: cheapModelFromEnv(), // static "config"-source LLM for NeedsModel components + ForceModel: os.Getenv("FORCE_MODEL"), // eval-containers pins EVAL_MODEL's model here + CheapModel: cheapModelFromEnv(), // static "config"-source LLM for NeedsModel components + InjectExpand: os.Getenv("INJECT_EXPAND"), // auto (default) | always | never + CacheMode: os.Getenv("CACHE_MODE"), // auto (default) | on | off — cache-aware compaction + Windows: modelWindows(), // dynamic context-window resolver (fraction triggers) + // Per-request /compact override: swap the pipeline (?preset / header) while // keeping this config's component blocks. nil-safe in the handler. PipelineFor: func(preset string, names []string) (*components.Pipeline, error) { @@ -113,6 +118,20 @@ func parseBool(s string) (v, ok bool) { return false, false } +// modelWindows builds the dynamic context-window resolver used for fraction-based +// triggers. Default chain: LiteLLM's public prices map (cached) -> small embedded +// fallback. MODEL_INFO_URL overrides the map source; MODEL_INFO=off disables it +// (windows unknown => fraction triggers ignored, absolutes apply). +func modelWindows() modelinfo.Resolver { + if strings.EqualFold(os.Getenv("MODEL_INFO"), "off") { + return nil + } + return modelinfo.Chain{ + modelinfo.NewLiteLLM(os.Getenv("MODEL_INFO_URL"), nil, 0), + modelinfo.DefaultStatic(), + } +} + // cheapModelFromEnv builds the static "config"-source LLM client for NeedsModel // components (extract code/rlm, summarize with model.source=config). Returns nil // when CHEAP_MODEL is unset, so those components fall back / no-op. diff --git a/components/all/llm_test.go b/components/all/llm_test.go index e57f524..a043be0 100644 --- a/components/all/llm_test.go +++ b/components/all/llm_test.go @@ -135,9 +135,10 @@ func TestSummarizeEmptyResponseSkips(t *testing.T) { // TestExtractRLMUsesModel: strategy=rlm currently maps to code and still runs the // model's filter (not silently deterministic). func TestExtractRLMUsesModel(t *testing.T) { - off := newComp(t, "extract", "strategy: rlm\nmin_tokens: 1\nmodel:\n source: config\n") + off := newComp(t, "extract_llm", "strategy: rlm\nmin_tokens: 1\nmodel:\n source: config\n") st := store.NewMemory(store.Options{}) - body := `[{"id":1,"name":"keep this one"},{"id":2,"name":"drop it"}]` + pad := strings.Repeat("padding ", 40) // so reduction beats the marker cost (D1 guard) + body := `[{"id":1,"name":"keep this one ` + pad + `"},{"id":2,"name":"drop it ` + pad + `"}]` req := &bschemas.BifrostChatRequest{Input: []bschemas.ChatMessage{ {Role: bschemas.ChatMessageRoleUser, Content: &bschemas.ChatMessageContent{ContentStr: strp("find keep")}}, toolMsg(body), @@ -156,9 +157,10 @@ func TestExtractRLMUsesModel(t *testing.T) { // TestExtractCodeUsesModel: the code strategy runs the model's Starlark filter and // keeps only the matching records (a contained subset), with a marker. func TestExtractCodeUsesModel(t *testing.T) { - off := newComp(t, "extract", "strategy: code\nmin_tokens: 1\nmodel:\n source: config\n") + off := newComp(t, "extract_llm", "strategy: code\nmin_tokens: 1\nmodel:\n source: config\n") st := store.NewMemory(store.Options{}) - body := `[{"id":1,"name":"keep this"},{"id":2,"name":"drop this"},{"id":3,"name":"keep that"}]` + pad := strings.Repeat("padding ", 40) // so reduction beats the marker cost (D1 guard) + body := `[{"id":1,"name":"keep this ` + pad + `"},{"id":2,"name":"drop this ` + pad + `"},{"id":3,"name":"keep that ` + pad + `"}]` req := &bschemas.BifrostChatRequest{Input: []bschemas.ChatMessage{ {Role: bschemas.ChatMessageRoleUser, Content: &bschemas.ChatMessageContent{ContentStr: strp("find the keep records")}}, toolMsg(body), @@ -182,9 +184,11 @@ func TestExtractCodeUsesModel(t *testing.T) { } } -// TestExtractCodeNilModelFallsBack: strategy=code but no model -> deterministic. -func TestExtractCodeNilModelFallsBack(t *testing.T) { - off := newComp(t, "extract", "strategy: code\nmin_tokens: 1\nhead_lines: 1\ntail_lines: 1\nmodel:\n source: config\n") +// With the extract split, the deterministic noise collapse is the separate `extract` +// component. Repeated identical lines are collapsed by it (no LLM needed), with a +// recovery marker. +func TestDeterministicExtractCollapsesRepeats(t *testing.T) { + off := newComp(t, "extract", "min_tokens: 1\n") st := store.NewMemory(store.Options{}) lines := make([]string, 30) for i := range lines { @@ -195,13 +199,36 @@ func TestExtractCodeNilModelFallsBack(t *testing.T) { {Role: bschemas.ChatMessageRoleUser, Content: &bschemas.ChatMessageContent{ContentStr: strp("find keep")}}, toolMsg(strings.Join(lines, "\n")), }} - c := &components.Ctx{Ctx: context.Background(), Store: st} // no model -> deterministic + c := &components.Ctx{Ctx: context.Background(), Store: st} var rep components.Report if _, err := off.Offload(req, &rep, c); err != nil { t.Fatal(err) } - // deterministic projection still runs and marks the message. if !strings.Contains(schema.MessageText(req.Input[1]), "<= before { - t.Fatalf("phi_evict should trim toward budget: before=%d after=%d", before, after) - } - // newest tool output (index 3) must survive. - if strings.Contains(schema.MessageText(req.Input[3]), "evicted") { - t.Fatal("most recent tool output must not be evicted") - } - // at least one earlier one evicted + recoverable. - var found bool - for _, i := range []int{1, 2} { - if keys := expand.ParseMarkers(schema.MessageText(req.Input[i])); len(keys) == 1 { - if _, ok := expand.Resolve(st, keys[0]); ok { - found = true - } - } - } - if !found { - t.Fatal("expected an evicted, recoverable earlier output") - } -} func TestExtractProjectsRelevantLines(t *testing.T) { var b strings.Builder diff --git a/components/all/reuse_test.go b/components/all/reuse_test.go index 076fba7..75ee5fa 100644 --- a/components/all/reuse_test.go +++ b/components/all/reuse_test.go @@ -78,11 +78,14 @@ func TestSummarizeReusesCheckpoint(t *testing.T) { // TestExtractReusesResultCache: the same large tool output re-sent on a later turn // reuses the prior compaction — no second model call — and is still reduced. func TestExtractReusesResultCache(t *testing.T) { - off := newComp(t, "extract", "strategy: code\nmin_tokens: 1\nmodel:\n source: config\n") + off := newComp(t, "extract_llm", "strategy: code\nmin_tokens: 1\nmodel:\n source: config\n") st := store.NewMemory(store.Options{}) filter := "data = json.decode(INPUT)\nOUTPUT = json.encode([r for r in data if \"keep\" in r[\"name\"]])\n" cm := &countingModel{resp: filter} - body := `[{"id":1,"name":"keep this"},{"id":2,"name":"drop this"},{"id":3,"name":"keep that"}]` + // Records padded so dropping the non-keep record shrinks the output by far more + // than the recovery marker costs (marker-inclusive never-worse guard, D1). + pad := strings.Repeat("padding ", 40) + body := `[{"id":1,"name":"keep this ` + pad + `"},{"id":2,"name":"drop this ` + pad + `"},{"id":3,"name":"keep that ` + pad + `"}]` run := func() *bschemas.BifrostChatRequest { req := &bschemas.BifrostChatRequest{Input: []bschemas.ChatMessage{ userMsg("find the keep records"), toolMsg(body), diff --git a/components/component.go b/components/component.go index ef1ed5c..34abf17 100644 --- a/components/component.go +++ b/components/component.go @@ -105,6 +105,34 @@ type Ctx struct { Model ModelSpec // Bypass short-circuits the whole pipeline (x-context-guru-bypass header). Bypass bool + // CtxWindow is the model's max input tokens for THIS request, resolved by the + // host (dynamically, via internal/modelinfo). 0 = unknown, in which case + // fraction-based Trigger thresholds are ignored and only absolutes apply. Stored + // as a resolved int so Trigger stays a pure, network-free, unit-testable function. + CtxWindow int + // CacheAware is true when this request goes to a prompt-caching backend and the + // pipeline should avoid mutating already-cached content. When true, supersession/ + // age-based offloaders (failed_run, mask, collapse) must restrict their + // mutations to the uncached tail (message index > MaxCachedIdx) so they don't + // invalidate the provider's KV cache at a full→collapsed transition. Deterministic + // tail/in-place offloaders (dedup, cmdfilter, extract) are already byte-stable on + // the unchanged prefix and ignore this. False = legacy compact-everything. + CacheAware bool + // MaxCachedIdx is the highest req.Input index considered already committed to the + // provider cache (the messages present on the previous turn of this session). + // -1 = unknown/first turn/cache off ⇒ no tail restriction. Only meaningful when + // CacheAware is true. + MaxCachedIdx int +} + +// TailOnly reports whether a supersession/age-based offloader may mutate the message +// at index i without risking the provider's cached prefix. When cache-awareness is +// off or the boundary is unknown, every index is fair game (legacy behavior). +func (c *Ctx) TailOnly(i int) bool { + if c == nil || !c.CacheAware || c.MaxCachedIdx < 0 { + return true + } + return i > c.MaxCachedIdx } // Report is the per-component result, modelled after lean-ctx's ToolOutput diff --git a/components/offload/cmdfilter.go b/components/offload/cmdfilter.go index 338205f..289bf2f 100644 --- a/components/offload/cmdfilter.go +++ b/components/offload/cmdfilter.go @@ -73,6 +73,10 @@ func (f *Cmdfilter) Offload(req *schemas.BifrostChatRequest, rep *components.Rep if content == "" { continue } + if skipReduce(c, content) { + continue // marker-bearing (a filter rule could drop the marker line and orphan + // the stash) or expanded by the agent — leave it verbatim + } filt := f.reg.Match(selectorKey(content)) if filt == nil { continue @@ -105,6 +109,7 @@ func (f *Cmdfilter) Offload(req *schemas.BifrostChatRequest, rep *components.Rep } if mode == markerFull { c.Store.Put(key, []byte(content)) + recordOwner(c, key) // scope GET /expand retrieval to this session keys = append(keys, key) } else { rep.Irreversible = true diff --git a/components/offload/collapse.go b/components/offload/collapse.go index 76ffe8c..aae3b06 100644 --- a/components/offload/collapse.go +++ b/components/offload/collapse.go @@ -20,16 +20,18 @@ func init() { components.Register("collapse", newCollapse) } // double-collapses. type Collapse struct { maxTokens int + maxFrac float64 headLines int tailLines int mode markerMode } type collapseConfig struct { - MaxTokens int `yaml:"max_tokens"` - HeadLines int `yaml:"head_lines"` - TailLines int `yaml:"tail_lines"` - MarkerMode string `yaml:"marker_mode"` // full (default) | summary | off + MaxTokens int `yaml:"max_tokens"` + MaxFrac float64 `yaml:"max_frac"` // optional: threshold as a fraction of the model window (wins when window known) + HeadLines int `yaml:"head_lines"` + TailLines int `yaml:"tail_lines"` + MarkerMode string `yaml:"marker_mode"` // full (default) | summary | off } func newCollapse(raw []byte) (components.Component, error) { @@ -39,13 +41,14 @@ func newCollapse(raw []byte) (components.Component, error) { return nil, err } } - return &Collapse{maxTokens: cfg.MaxTokens, headLines: cfg.HeadLines, tailLines: cfg.TailLines, mode: parseMarkerMode(cfg.MarkerMode)}, nil + return &Collapse{maxTokens: cfg.MaxTokens, maxFrac: cfg.MaxFrac, headLines: cfg.HeadLines, tailLines: cfg.TailLines, mode: parseMarkerMode(cfg.MarkerMode)}, nil } func (Collapse) Name() string { return "collapse" } func (Collapse) Enabled(*components.Ctx) bool { return true } func (cl *Collapse) Offload(req *schemas.BifrostChatRequest, rep *components.Report, c *components.Ctx) ([]string, error) { + maxTokens := resolveBudget(cl.maxTokens, cl.maxFrac, c.CtxWindow) // frac of window wins when known var keys []string changed := 0 for i := range req.Input { @@ -57,24 +60,28 @@ func (cl *Collapse) Offload(req *schemas.BifrostChatRequest, rep *components.Rep continue // non-text blocks would be dropped by a text rewrite } content := schema.MessageText(*m) - if content == "" || schema.TextTokens(content) <= cl.maxTokens { + if content == "" || schema.TextTokens(content) <= maxTokens { continue } - if expand.HasPlaceholder(content) { - continue // already offloaded by an earlier component + if skipReduce(c, content) { + continue // already offloaded by an earlier component, or expanded by the agent } lines := strings.Split(content, "\n") if len(lines) <= cl.headLines+cl.tailLines { continue // few long lines; head/tail wouldn't help } omitted := len(lines) - cl.headLines - cl.tailLines - tok, key := mark(c, rep, cl.mode, content, " [full output: call "+expand.ToolName+"]") - var b strings.Builder - b.WriteString(strings.Join(lines[:cl.headLines], "\n")) - fmt.Fprintf(&b, "\n... (%d lines omitted) ", omitted) - b.WriteString(tok + "\n") - b.WriteString(strings.Join(lines[len(lines)-cl.tailLines:], "\n")) - schema.SetMessageText(m, b.String()) + head := strings.Join(lines[:cl.headLines], "\n") + tail := strings.Join(lines[len(lines)-cl.tailLines:], "\n") + newText, key, eff, ok := tryMark(c, cl.mode, content, " [full output: call "+expand.ToolName+"]", + func(tok string) string { + return fmt.Sprintf("%s\n... (%d lines omitted) %s\n%s", head, omitted, tok, tail) + }) + if !ok { + continue // head/tail window+marker wouldn't shrink this output; leave it verbatim + } + commitMark(c, rep, eff, key, content) + schema.SetMessageText(m, newText) changed++ if key != "" { keys = append(keys, key) diff --git a/components/offload/common.go b/components/offload/common.go index 93140c1..8274420 100644 --- a/components/offload/common.go +++ b/components/offload/common.go @@ -1,12 +1,190 @@ package offload import ( + "math" + "regexp" "strings" + "unicode/utf8" bschemas "github.com/maximhq/bifrost/core/schemas" "github.com/rossoctl/context-guru/schema" ) +// resolveBudget converts an absolute token knob + an optional fraction-of-window +// into an effective token count: the fraction (ceil(frac*window)) wins when set and +// the window is known, else the absolute. Lets size knobs (collapse.max_tokens) +// scale with the model context window while staying backward compatible when no +// fraction is configured or the window is unknown. +func resolveBudget(absolute int, frac float64, window int) int { + if frac > 0 && window > 0 { + return int(math.Ceil(frac * float64(window))) + } + return absolute +} + +// headPeek returns a single-line, whitespace-collapsed, clipped snippet of the +// start of content — the cue an age/budget-based offloader (mask) can +// leave inside its marker so the model knows WHAT was hidden without a blind +// expand round-trip. Replay/diff analysis showed a bare "[older tool output +// masked]" marker gives the model zero signal (e.g. it masked a still-relevant +// source-file read on SWE), forcing needless expand calls; a ~1-line peek fixes +// that at negligible token cost (the marker-inclusive never-worse guard in +// tryMark still drops the rewrite if the peek would make it not shrink). +// maxChars<=0 disables the peek (returns ""). +func headPeek(content string, maxChars int) string { + if maxChars <= 0 { + return "" + } + // Peek across the START of the content (not just the first line): a bounded + // prefix, newlines→spaces, whitespace collapsed. This keeps the cue useful for + // line-numbered file reads (first line is a bare number) as well as command + // output (first line "Exit code 1 / Traceback ..."). Slice first so we never + // scan a multi-KB output. + head := clipRunes(content, maxChars*4) + head = strings.Join(strings.Fields(head), " ") // newlines+runs → single spaces + if head == "" { + return "" + } + // never let a peek carry a marker sentinel back into the request + head = strings.ReplaceAll(head, "< maxChars { + head = string(r[:maxChars]) + "…" + } + return head +} + +// clipRunes returns at most maxBytes bytes of s truncated on a UTF-8 rune boundary, +// so a multibyte rune is never split into invalid UTF-8 (which would corrupt the +// marker text spliced back into the request body). +func clipRunes(s string, maxBytes int) string { + if maxBytes <= 0 || len(s) <= maxBytes { + if maxBytes <= 0 { + return "" + } + return s + } + b := maxBytes + for b > 0 && !utf8.RuneStart(s[b]) { + b-- + } + return s[:b] +} + +// progressNoiseRe matches a line that is ENTIRELY a progress indicator (tqdm/pip +// bars, percentage-only lines, spinner/byte-counter redraws) — safe to drop because +// it carries no information the agent acts on. Deliberately narrow to avoid eating +// real content: the whole line must be bar/percent glyphs + digits + separators. +var progressNoiseRe = regexp.MustCompile(`^[\s\d./%|=><:\-\[\]()x,]*[%█━▏▎▍▌▋▊▉►][\s\d./%|=><:\-\[\]()x,█━▏▎▍▌▋▊▉►]*$`) + +// ansiRe matches ANSI/VT100 escape sequences (colors, cursor moves, etc.). Stripping +// them is universally safe — they are pure terminal display control, never content — +// and general across any tool/agent/benchmark. Reversible (the original is stashed). +var ansiRe = regexp.MustCompile("\x1b\\[[0-9;?]*[ -/]*[@-~]|\x1b\\][^\x07\x1b]*(?:\x07|\x1b\\\\)") + +// stripTerminalNoise removes ANSI escapes and collapses carriage-return progress +// redraws (a line rewritten in place by \r keeps only its final rendered segment). +// Returns (cleaned, changed). Deterministic, content-preserving except for pure +// terminal control — the safe, zero-cost first pass before structural collapsing. +func stripTerminalNoise(s string) (string, bool) { + changed := false + if ansiRe.MatchString(s) { + s = ansiRe.ReplaceAllString(s, "") + changed = true + } + if strings.Contains(s, "\r") { + lines := strings.Split(s, "\n") + for i, ln := range lines { + // A trailing "\r" is just the CRLF (\r\n) line separator surfacing after the + // split on "\n" — NOT a redraw. Treating it as one would keep only ln[len:] = "" + // and blank every CRLF line (silent content loss). Only a CR *inside* the line + // is an in-place progress redraw; there we keep the final rendered segment and + // re-attach the original trailing CR so line endings stay byte-identical. + core := strings.TrimSuffix(ln, "\r") + j := strings.LastIndexByte(core, '\r') + if j < 0 { + continue // no interior CR: pure content (possibly CRLF) — leave untouched + } + seg := core[j+1:] + if strings.HasSuffix(ln, "\r") { + seg += "\r" + } + lines[i] = seg + changed = true + } + s = strings.Join(lines, "\n") + } + return s, changed +} + +// collapseObviousNoise is the CONSERVATIVE deterministic reducer: it deletes ONLY +// provably-redundant noise and keeps every unique informative line verbatim, so it +// can never hide content the agent needs and force a redo (the failure mode of blind +// head/tail truncation). It removes: runs of blank lines (→ one), consecutively +// repeated single lines or multi-line blocks (keeping one copy — e.g. a traceback +// dumped 50× by a retry loop), and pure progress-bar/spinner lines. Everything else +// is kept. Relevance-aware trimming is the LLM strategy's job, not this one. Returns +// changed=false when there was no obvious noise to drop (leave the output untouched). +func collapseObviousNoise(content string) (string, bool) { + content, termChanged := stripTerminalNoise(content) // universally-safe first pass + lines := strings.Split(content, "\n") + n := len(lines) + out := make([]string, 0, n) + changed := false + blocksEqual := func(a, b, k int) bool { + for j := 0; j < k; j++ { + if lines[a+j] != lines[b+j] { + return false + } + } + return true + } + const maxBlock = 12 + for i := 0; i < n; { + ln := lines[i] + if strings.TrimSpace(ln) != "" && progressNoiseRe.MatchString(ln) && strings.ContainsAny(ln, "%█━▏▎▍▌▋▊▉►") { + changed = true + i++ + continue + } + if strings.TrimSpace(ln) == "" { // collapse blank runs to one + if len(out) > 0 && strings.TrimSpace(out[len(out)-1]) == "" { + changed = true + i++ + continue + } + out = append(out, ln) + i++ + continue + } + // consecutively repeated block of size k (k=1 covers duplicate lines): + // keep one copy, drop the immediate repeats. + collapsed := false + for k := 1; k <= maxBlock && i+2*k <= n; k++ { + if !blocksEqual(i, i+k, k) { + continue + } + reps := 2 + for i+(reps+1)*k <= n && blocksEqual(i, i+reps*k, k) { + reps++ + } + out = append(out, lines[i:i+k]...) + i += reps * k + changed = true + collapsed = true + break + } + if collapsed { + continue + } + out = append(out, ln) + i++ + } + if !changed { + return content, termChanged // terminal-noise stripping may have changed it alone + } + return strings.Join(out, "\n"), true +} + // errWords mark a tool output (or an item) as carrying a failure — such items // are preserved by smartcrush and prioritized elsewhere, since dropping the one // error in a haystack is exactly the accuracy loss to avoid. @@ -22,17 +200,6 @@ func hasError(s string) bool { return false } -// lastUserText returns the text of the most recent user message — the query -// relevance is scored against (extract, phi_evict). -func lastUserText(req *bschemas.BifrostChatRequest) string { - for i := len(req.Input) - 1; i >= 0; i-- { - if req.Input[i].Role == bschemas.ChatMessageRoleUser { - return schema.MessageText(req.Input[i]) - } - } - return "" -} - // goalCap bounds the conversational context handed to an LLM component so a huge // task statement can't blow up every prompt. Generous — the point is to pass the // real task, not one trailing sentence. @@ -100,20 +267,6 @@ func keywords(s string) map[string]struct{} { return out } -// overlap is the fraction of query terms present in text (0..1). -func overlap(query map[string]struct{}, text string) float64 { - if len(query) == 0 { - return 0 - } - tk := keywords(text) - hit := 0 - for w := range query { - if _, ok := tk[w]; ok { - hit++ - } - } - return float64(hit) / float64(len(query)) -} // toolIndices returns the indices of tool-role messages, in order. func toolIndices(req *bschemas.BifrostChatRequest) []int { diff --git a/components/offload/common_test.go b/components/offload/common_test.go new file mode 100644 index 0000000..d257a89 --- /dev/null +++ b/components/offload/common_test.go @@ -0,0 +1,93 @@ +package offload + +import ( + "strings" + "testing" + "unicode/utf8" +) + +// The deterministic reducer must drop ONLY obvious noise (repeated blocks, blank +// runs, progress bars) and keep every unique informative line — so it can never +// hide content the agent needs and force a redo. +func TestCollapseObviousNoise(t *testing.T) { + // a traceback dumped 4× by a retry loop + blank runs + a progress bar + block := "Traceback (most recent call last):\n File \"x\", line 4\nModuleNotFoundError: No module named 'numpy'" + in := block + "\n" + block + "\n" + block + "\n" + block + + "\n\n\n\nkeep me unique line A\n 45%|████████ | 45/100\nkeep me unique line B" + out, changed := collapseObviousNoise(in) + if !changed { + t.Fatal("expected noise to be collapsed") + } + // repeated block kept exactly once + if n := strings.Count(out, "ModuleNotFoundError"); n != 1 { + t.Fatalf("repeated block should collapse to 1 copy, got %d\n%s", n, out) + } + // unique lines preserved + for _, want := range []string{"keep me unique line A", "keep me unique line B"} { + if !strings.Contains(out, want) { + t.Fatalf("unique line dropped: %q\n%s", want, out) + } + } + // progress bar line removed + if strings.Contains(out, "45%") { + t.Fatalf("progress bar should be dropped\n%s", out) + } + // blank run collapsed (no triple newline remains) + if strings.Contains(out, "\n\n\n") { + t.Fatalf("blank run not collapsed\n%s", out) + } + if len(out) >= len(in) { + t.Fatalf("output should be smaller: %d -> %d", len(in), len(out)) + } +} + +// CRLF (\r\n) content must be preserved: the terminating CR is the line separator, +// not an in-line progress redraw. A regression once treated it as a redraw and blanked +// every CRLF line (keeping only the empty text after the trailing CR). +func TestStripTerminalNoisePreservesCRLF(t *testing.T) { + in := "line one\r\nline two\r\nline three\r\n" + out, _ := stripTerminalNoise(in) + for _, want := range []string{"line one", "line two", "line three"} { + if !strings.Contains(out, want) { + t.Fatalf("CRLF content blanked — %q missing from %q", want, out) + } + } +} + +// An INTERIOR carriage return IS a progress redraw: keep only the final rendered +// segment (and preserve the line's own trailing CR). +func TestStripTerminalNoiseCollapsesRedraw(t *testing.T) { + in := "downloading: 10%\rdownloading: 55%\rdownloading: 100%\ndone\n" + out, changed := stripTerminalNoise(in) + if !changed { + t.Fatal("interior-CR redraw should be collapsed") + } + if strings.Contains(out, "10%") || strings.Contains(out, "55%") { + t.Fatalf("redraw should keep only the final segment: %q", out) + } + if !strings.Contains(out, "100%") || !strings.Contains(out, "done") { + t.Fatalf("final segment + following lines must survive: %q", out) + } +} + +// headPeek must never split a multibyte rune and emit invalid UTF-8 into the marker. +func TestHeadPeekUTF8Safe(t *testing.T) { + // many multibyte runes so the byte cut lands mid-rune at several offsets + content := strings.Repeat("日本語テキスト絵文字😀 ", 50) + for _, n := range []int{1, 3, 7, 16, 40, 96} { + peek := headPeek(content, n) + if !utf8.ValidString(peek) { + t.Fatalf("headPeek(%d) produced invalid UTF-8: %q", n, peek) + } + } +} + +// No obvious noise → returns the content unchanged (conservative: never touch +// unique content). +func TestCollapseObviousNoiseNoop(t *testing.T) { + in := "line one\nline two\nline three\nresult: 42\nerror: none" + out, changed := collapseObviousNoise(in) + if changed || out != in { + t.Fatalf("clean content must be left verbatim, changed=%v\n%s", changed, out) + } +} diff --git a/components/offload/dedup.go b/components/offload/dedup.go index 002e2af..4d71df0 100644 --- a/components/offload/dedup.go +++ b/components/offload/dedup.go @@ -51,14 +51,22 @@ func (d *Dedup) Offload(req *schemas.BifrostChatRequest, rep *components.Report, if content == "" || schema.TextTokens(content) < d.minTokens { continue } + if skipReduce(c, content) { + continue // already carries a marker, or was expanded by the agent — don't re-reduce + } h := hashKey(content) if _, dup := seen[h]; !dup { seen[h] = i continue } // Later duplicate: collapse to a pointer (stash+marker in full mode). - tok, key := mark(c, rep, d.mode, content, "") - schema.SetMessageText(m, "[identical to an earlier tool output] "+tok) + newText, key, eff, ok := tryMark(c, d.mode, content, "", + func(tok string) string { return "[identical to an earlier tool output] " + tok }) + if !ok { + continue // pointer+marker wouldn't shrink this duplicate; leave it verbatim + } + commitMark(c, rep, eff, key, content) + schema.SetMessageText(m, newText) changed++ if key != "" { keys = append(keys, key) diff --git a/components/offload/extract.go b/components/offload/extract.go index bdd96e7..c31d0ed 100644 --- a/components/offload/extract.go +++ b/components/offload/extract.go @@ -1,116 +1,59 @@ package offload import ( - "context" - "strings" - "time" - bschemas "github.com/maximhq/bifrost/core/schemas" "github.com/rossoctl/context-guru/components" "github.com/rossoctl/context-guru/expand" - "github.com/rossoctl/context-guru/internal/extract" "github.com/rossoctl/context-guru/schema" "gopkg.in/yaml.v3" ) -// llmCallTimeout bounds a single in-request extract model call so a slow/hung -// model fails open (the component falls back / reverts) instead of stalling the -// agent. extract can make several calls per request (one per large tool output), -// so this ceiling stays modest; the per-content result cache keeps repeats free. -const llmCallTimeout = 60 * time.Second - func init() { components.Register("extract", newExtract) } -// Extract projects a large tool output down to the part relevant to the current -// query, stashing the full original. Three strategies: -// - deterministic (default, no LLM): keep query-keyword + head/tail + error lines. -// - code: a cheap LLM writes a Starlark `extract_relevant_data` filter, run in a -// sandbox (no imports/IO, step+time limited); the output is accepted only if a -// containment + sanity check proves it's a lossless projection, else it falls -// back to deterministic. (winnow's llm_compact, ported.) -// - rlm: reserved for very large outputs; currently maps to code. +// Extract is the DETERMINISTIC (no-LLM) tool-output reducer. It runs every request +// (cheap) and is deliberately CONSERVATIVE: it removes only OBVIOUS, provably +// redundant noise (exactly repeated lines/blocks, runs of blank lines, progress +// bars/spinners) via collapseObviousNoise, keeping every unique informative line +// verbatim. This guarantees it can never hide content the agent needs and force it +// to redo work. Relevance-aware, aggressive trimming (regex rewrites, summaries) is +// the job of the separate `extract_llm` component — configure them together +// (`[extract, extract_llm]`) so the cheap pass runs every step and the LLM pass only +// every few steps. // -// The LLM strategies need a Model (Ctx.Model, per model.source); when none is -// available they degrade to deterministic. The full original is always stashed -// under a marker, so any reduction is reversible via the expand tool. +// It is byte-stable on unchanged content (deterministic per-message), so it is +// cache-safe and needs no cache-boundary restriction. The full original is stashed +// under a marker (reversible via the expand tool). type Extract struct { - minTokens int - head int - tail int - strategy string - modelSource string - modelClient components.Model // config-pinned client (model: block), or nil - trigger components.Trigger - mode markerMode - rewrite bool + minTokens int + trigger components.Trigger + mode markerMode } type extractConfig struct { MinTokens int `yaml:"min_tokens"` - Head int `yaml:"head_lines"` - Tail int `yaml:"tail_lines"` - Strategy string `yaml:"strategy"` // deterministic | code | rlm - Model modelConfig `yaml:"model"` Trigger components.Trigger `yaml:"trigger"` MarkerMode string `yaml:"marker_mode"` // full (default) | summary | off - // Rewrite (code strategy only): drop the deletion-only containment proof so the - // model may reword/summarize/rewrite. Lossy + unverified — pair with a non-full - // marker_mode. Default false keeps the verified deletion-only guarantee. - Rewrite bool `yaml:"rewrite"` } func newExtract(raw []byte) (components.Component, error) { - cfg := extractConfig{MinTokens: 300, Head: 5, Tail: 5, Strategy: "deterministic"} + cfg := extractConfig{MinTokens: 300} if len(raw) > 0 { if err := yaml.Unmarshal(raw, &cfg); err != nil { return nil, err } } - // Legacy min_tokens is the per-output floor; the canonical knob is - // trigger.min_output_tokens. Fold one into the other so both work. - if cfg.Trigger.MinOutputTokens == 0 { - cfg.Trigger.MinOutputTokens = cfg.MinTokens - } - return &Extract{minTokens: cfg.MinTokens, head: cfg.Head, tail: cfg.Tail, strategy: cfg.Strategy, modelSource: cfg.Model.Source, modelClient: cfg.Model.Client(), trigger: cfg.Trigger, mode: parseMarkerMode(cfg.MarkerMode), rewrite: cfg.Rewrite}, nil -} - -// outputFloor is the minimum tokens a single tool output must have to be worth -// offloading (the "large output" trigger). -func (e *Extract) outputFloor() int { - if e.trigger.MinOutputTokens > 0 { - return e.trigger.MinOutputTokens - } - return e.minTokens + return &Extract{minTokens: cfg.MinTokens, trigger: cfg.Trigger, mode: parseMarkerMode(cfg.MarkerMode)}, nil } func (Extract) Name() string { return "extract" } func (Extract) Enabled(*components.Ctx) bool { return true } -// NeedsModel reports whether the configured strategy calls an LLM. -func (e *Extract) NeedsModel() bool { return e.strategy == "code" || e.strategy == "rlm" } +func (e *Extract) outputFloor(window int) int { + return e.trigger.OutputFloor(window, e.minTokens) +} func (e *Extract) Offload(req *bschemas.BifrostChatRequest, rep *components.Report, c *components.Ctx) ([]string, error) { - // Request-level trigger: for the LLM strategies, don't spend a model call - // until the request is genuinely large / deep. Deterministic runs always - // (it's cheap). Zero thresholds fire always (backward compatible). - if e.NeedsModel() && !e.trigger.Fires(req) { - rep.Skipped = true - return nil, nil - } - goal := conversationGoal(req) // full task + recent turns, not one trailing sentence - query := keywords(goal) - if len(query) == 0 { - rep.Skipped = true - return nil, nil // nothing to condition relevance on - } - var model components.Model - if e.NeedsModel() { - if model = e.modelClient; model == nil { // config-pinned client wins - model = c.Model.For(e.modelSource) - } - } - floor := e.outputFloor() - keepIDs := extract.HarvestIdentifiers(goal, 40) + floor := e.outputFloor(c.CtxWindow) var keys []string changed := 0 for _, i := range toolIndices(req) { @@ -122,23 +65,20 @@ func (e *Extract) Offload(req *bschemas.BifrostChatRequest, rep *components.Repo if content == "" || schema.TextTokens(content) < floor { continue } - if expand.HasPlaceholder(content) { - continue - } - // Reuse a prior compaction of this exact output (marker/whitespace - // insensitive) — no LLM call, and the same bytes keep the prefix stable. - id := extract.ContentKey(content) - projected, ok := "", false - if cached, hit := getResult(c, id); hit { - projected, ok = string(cached), true - } else if projected, ok = e.reduce(c, content, goal, keepIDs, query, model); ok { - putResult(c, id, []byte(projected)) + if skipReduce(c, content) { + continue // already offloaded, or expanded by the agent — don't re-reduce } + projected, ok := collapseObviousNoise(content) if !ok || schema.TextTokens(projected) >= schema.TextTokens(content) { continue } - tok, key := mark(c, rep, e.mode, content, " [full output: call "+expand.ToolName+"]") - schema.SetMessageText(msg, projected+"\n"+tok) + newText, key, eff, ok2 := tryMark(c, e.mode, content, " [full output: call "+expand.ToolName+"]", + func(tok string) string { return projected + "\n" + tok }) + if !ok2 { + continue // projection+marker wouldn't shrink this message; leave it verbatim + } + commitMark(c, rep, eff, key, content) + schema.SetMessageText(msg, newText) changed++ if key != "" { keys = append(keys, key) @@ -149,61 +89,3 @@ func (e *Extract) Offload(req *bschemas.BifrostChatRequest, rep *components.Repo } return keys, nil } - -// reduce picks the strategy: for code/rlm with a model, run the sandboxed -// LLM-generated filter (containment-validated inside RunExtraction, with its own -// deterministic fallback); otherwise the deterministic line projection. Always -// fail-open — any miss falls through to project(). -func (e *Extract) reduce(c *components.Ctx, content, goal string, keepIDs []string, query map[string]struct{}, model components.Model) (string, bool) { - if model != nil && e.NeedsModel() { - cfg := extract.DefaultCfg() - cfg.Mode = "code" // rlm is deferred → use the Starlark code strategy - cfg.Floor = e.outputFloor() - cfg.Rewrite = e.rewrite - ctx, cancel := context.WithTimeout(c.Ctx, llmCallTimeout) - res, _ := extract.RunExtraction(ctx, content, goal, keepIDs, schema.TextTokens(content), cfg, model) - cancel() - if res != "" && res != content { - return res, true - } - } - return e.project(content, query) -} - -// project keeps head/tail context plus every line relevant to the query or -// carrying an error, collapsing the runs between with an ellipsis marker. -func (e *Extract) project(content string, query map[string]struct{}) (string, bool) { - lines := strings.Split(content, "\n") - if len(lines) <= e.head+e.tail+1 { - return "", false - } - keep := make([]bool, len(lines)) - for i := 0; i < e.head && i < len(lines); i++ { - keep[i] = true - } - for i := len(lines) - e.tail; i < len(lines); i++ { - if i >= 0 { - keep[i] = true - } - } - for i, ln := range lines { - if hasError(ln) || overlap(query, ln) > 0 { - keep[i] = true - } - } - var b strings.Builder - gap := false - for i, ln := range lines { - if keep[i] { - if gap { - b.WriteString("…\n") - gap = false - } - b.WriteString(ln) - b.WriteByte('\n') - } else { - gap = true - } - } - return strings.TrimRight(b.String(), "\n"), true -} diff --git a/components/offload/extract_llm.go b/components/offload/extract_llm.go new file mode 100644 index 0000000..b1618eb --- /dev/null +++ b/components/offload/extract_llm.go @@ -0,0 +1,318 @@ +package offload + +import ( + "context" + "log/slog" + "os" + "regexp" + "strings" + "sync" + "time" + + bschemas "github.com/maximhq/bifrost/core/schemas" + "github.com/rossoctl/context-guru/components" + "github.com/rossoctl/context-guru/expand" + "github.com/rossoctl/context-guru/internal/extract" + "github.com/rossoctl/context-guru/schema" + "gopkg.in/yaml.v3" +) + +// debugExtractLLM logs per-request candidate accounting when CONTEXT_GURU_DEBUG is set. +var debugExtractLLM = os.Getenv("CONTEXT_GURU_DEBUG") != "" + +// llmCallTimeout bounds a SINGLE in-request extract model call. Kept tight so a slow +// or rate-limited compaction model fails open FAST (leave the output verbatim this +// turn) instead of stalling the agent's request — synchronous compaction is on the +// hot path, so a long timeout here can push the agent's own request past its deadline. +const llmCallTimeout = 15 * time.Second + +// llmConcurrency bounds how many of a request's candidate compactions run at once. +// Independent per-output calls run concurrently so a turn's parallel tool outputs cost +// ~one call's wall time instead of the sum. Bounded so a burst can't overwhelm the +// cheap-model endpoint. (A single-call batch alternative was measured ~3× worse on +// tokens saved with no latency win — see docs/CACHE_AWARE_ITERATIONS.md.) +const llmConcurrency = 4 + +func init() { components.Register("extract_llm", newExtractLLM) } + +// ExtractLLM is the relevance-aware, LLM-driven tool-output reducer. A cheap model +// writes a small Starlark program that trims ONE tool output down to what the agent +// needs next (it may delete OR rewrite via regex, preserving ids/paths/errors +// verbatim, and may emit a one-line SUMMARY that goes into the marker). The program +// runs in a sandbox (no imports/IO, step+time limited) and the result must pass a +// sanity check (non-empty, strictly smaller, keep-ids present); on any miss the item +// is left verbatim. The full original is always stashed (reversible via expand). +// +// It is the EXPENSIVE pass, so it is throttled (per-session cadence + per-request +// cap) and — in cache-aware mode — only rewrites tool outputs in the uncached tail +// that are still medium/large AFTER the deterministic components ran. Prior +// compactions are reused byte-for-byte from state so the request prefix stays stable. +type ExtractLLM struct { + minTokens int + strategy string + modelSource string + modelClient components.Model + trigger components.Trigger + mode markerMode + rewrite bool + llmEveryN int + llmMaxPerReq int + skipFileReads *bool // nil = auto (skip when cache-aware); true/false = force + mu sync.Mutex + llmSeen map[string]int // session -> count of qualifying (LLM-eligible) requests +} + +type extractLLMConfig struct { + MinTokens int `yaml:"min_tokens"` + Strategy string `yaml:"strategy"` // code (default) | single | rlm | auto + LLMEveryN int `yaml:"llm_every_n_requests"` // throttle LLM path: fire once per N requests/session + LLMMaxPerReq int `yaml:"llm_max_per_request"` // cap LLM calls per firing request (0 = unlimited) + Model modelConfig `yaml:"model"` + Trigger components.Trigger `yaml:"trigger"` + MarkerMode string `yaml:"marker_mode"` // full (default) | summary | off + // Rewrite lets the program reword/summarize/collapse (not just delete), dropping + // the strict deletion-only containment proof; ids/paths/errors/keep-ids are still + // required verbatim by the sanity check. Default true (the powerful mode) — set + // false to force verified deletion-only. + Rewrite *bool `yaml:"rewrite"` + // SkipFileReads controls whether line-numbered source-file dumps are left verbatim. + // Tri-state: unset = AUTO (skip when the request is prompt-cached, reduce otherwise); + // true = always skip; false = always reduce. Rationale (measured, SWE-bench 50): + // on a ~98%-cached agent, file reads already bill at the cheap cache-read rate, so + // skeletonizing them saves almost nothing yet costs the compaction LLM + one-time + // cache-write transitions → +30% billed cost. On a NON-caching backend the same + // reduction is a direct saving. So AUTO skips file reads exactly when caching makes + // them cheap. See docs/CACHE_AWARE_ITERATIONS.md. + SkipFileReads *bool `yaml:"skip_file_reads"` +} + +func newExtractLLM(raw []byte) (components.Component, error) { + cfg := extractLLMConfig{MinTokens: 300, Strategy: "code"} + if len(raw) > 0 { + if err := yaml.Unmarshal(raw, &cfg); err != nil { + return nil, err + } + } + rewrite := true + if cfg.Rewrite != nil { + rewrite = *cfg.Rewrite + } + if cfg.Strategy == "" { + cfg.Strategy = "code" + } + return &ExtractLLM{ + minTokens: cfg.MinTokens, strategy: cfg.Strategy, + modelSource: cfg.Model.Source, modelClient: cfg.Model.Client(), + trigger: cfg.Trigger, mode: parseMarkerMode(cfg.MarkerMode), rewrite: rewrite, + llmEveryN: cfg.LLMEveryN, llmMaxPerReq: cfg.LLMMaxPerReq, + skipFileReads: cfg.SkipFileReads, llmSeen: map[string]int{}, + }, nil +} + +func (*ExtractLLM) Name() string { return "extract_llm" } +func (*ExtractLLM) Enabled(*components.Ctx) bool { return true } + +func (e *ExtractLLM) outputFloor(window int) int { + return e.trigger.OutputFloor(window, e.minTokens) +} + +// llmAllowedThisRequest applies the per-session cadence: true on the 1st qualifying +// request and every Nth after, so the LLM path fires "every multiple steps". +func (e *ExtractLLM) llmAllowedThisRequest(session string) bool { + if e.llmEveryN <= 1 { + return true + } + e.mu.Lock() + defer e.mu.Unlock() + e.llmSeen[session]++ + return (e.llmSeen[session]-1)%e.llmEveryN == 0 +} + +var lineNumberedRe = regexp.MustCompile(`^\s{0,6}\d+[\t ]`) + +// looksLikeFileRead reports whether content is a line-numbered source-file dump (a +// read/cat -n output): most non-empty lines begin with a line number. Such outputs +// are whole files the agent is working with — irreducible — so skip the model call. +func looksLikeFileRead(content string) bool { + checked, numbered := 0, 0 + for _, ln := range strings.Split(content, "\n") { + if strings.TrimSpace(ln) == "" { + continue + } + checked++ + if lineNumberedRe.MatchString(ln) { + numbered++ + } + if checked >= 40 { + break + } + } + return checked >= 8 && numbered*100/checked >= 60 +} + +func (e *ExtractLLM) Offload(req *bschemas.BifrostChatRequest, rep *components.Report, c *components.Ctx) ([]string, error) { + fires := e.trigger.Fires(req, c.CtxWindow) + goal := conversationGoal(req) + query := keywords(goal) + if len(query) == 0 { + rep.Skipped = true + return nil, nil + } + model := e.modelClient + if model == nil { + model = c.Model.For(e.modelSource) + } + // Per-session cadence: on throttled steps drop the model (skip this request). + if model != nil && fires && !e.llmAllowedThisRequest(c.Session) { + model = nil + } + floor := e.outputFloor(c.CtxWindow) + keepIDs := extract.HarvestIdentifiers(goal, 40) + tools := toolIndices(req) + var keys []string + changed := 0 + + // apply splices a compacted projection + marker into message i (serial: store writes + // and message mutation are not concurrency-safe). + apply := func(i int, content, projected, summary string) { + if projected == "" || schema.TextTokens(projected) >= schema.TextTokens(content) { + return + } + hint := " [full output: call " + expand.ToolName + "]" + newText, key, eff, ok := tryMark(c, e.mode, content, hint, func(tok string) string { + if summary != "" { + return projected + "\n[" + summary + "] " + tok + } + return projected + "\n" + tok + }) + if !ok { + return + } + commitMark(c, rep, eff, key, content) + schema.SetMessageText(&req.Input[i], newText) + changed++ + if key != "" { + keys = append(keys, key) + } + } + + // Phase 1 (serial, cheap): reapply frozen compactions on every turn (keeps the + // request prefix byte-stable so the provider cache stays warm), and collect the NEW + // candidates that still need a model call. + type cand struct { + i int + content string + id string + } + var cands []cand + skipFR := false + if e.skipFileReads != nil { + skipFR = *e.skipFileReads + } + var dbgTail, dbgFloor, dbgPlace, dbgReapply, dbgBigTailBlocked int + for _, i := range tools { + msg := &req.Input[i] + if !schema.Rewritable(*msg) { + continue + } + content := schema.MessageText(*msg) + if content == "" || expand.HasPlaceholder(content) { + dbgPlace++ + continue + } + id := extract.ContentKey(content) + // If the agent recently EXPANDED this content, leave it verbatim (re-compacting it + // would just trigger another expand — a loop). The expand handler marks it. + if isKeptVerbatim(c, id) { + continue + } + if cached, hit := getResult(c, id); hit { + summary, _ := getSummary(c, id) + apply(i, content, string(cached), summary) + dbgReapply++ + continue + } + // A NEW compaction, on the UNCACHED region only (cache-safe): when cache-aware that + // is every message newer than last turn (index > MaxCachedIdx) — catching ALL of a + // turn's tool outputs including PARALLEL tool calls, never the cached prefix. When + // caching is off, any message is fair game. File reads included (largest mass); + // safe because we never touch already-cached content and freeze+reapply the result. + sz := schema.TextTokens(content) + if c.CacheAware && !c.TailOnly(i) { + dbgTail++ + if sz >= floor { + dbgBigTailBlocked++ // a large output we skipped ONLY because it's not in the tail + } + continue + } + if sz < floor { + dbgFloor++ + continue // only medium/large outputs are worth a model call + } + if huge := e.trigger.IsHuge(sz, c.CtxWindow); !c.CacheAware && !fires && !huge { + continue + } + if model == nil { + continue + } + if skipFR && looksLikeFileRead(content) { + continue + } + cands = append(cands, cand{i, content, id}) + } + if debugExtractLLM && len(tools) > 0 { + slog.Info("cg.debug.extract_llm", "tools", len(tools), "cands", len(cands), + "reapplied", dbgReapply, "skip_placeholder", dbgPlace, "skip_tail", dbgTail, + "skip_floor", dbgFloor, "big_but_not_tail", dbgBigTailBlocked, + "cacheAware", c.CacheAware, "maxCachedIdx", c.MaxCachedIdx, "floor", floor, + "nInput", len(req.Input)) + } + if e.llmMaxPerReq > 0 && len(cands) > e.llmMaxPerReq { + cands = cands[:e.llmMaxPerReq] // cap model calls per request + } + + // Phase 2 (parallel): the candidate compactions are independent. A focused per-output + // prompt ("trim THIS one output") gives a much better reduction than a single program + // over the whole heterogeneous batch (measured on SWE: ~3× more tokens saved), and + // running them concurrently (bounded) keeps a turn's cost to ~one call's wall time — + // so parallel beats a single-call batch on tokens AND latency. Each output fails open + // independently (a miss leaves that one verbatim). + if len(cands) > 0 { + type outT struct{ projected, summary string } + out := make([]outT, len(cands)) + sem := make(chan struct{}, llmConcurrency) + var wg sync.WaitGroup + for k := range cands { + wg.Add(1) + go func(k int) { + defer wg.Done() + sem <- struct{}{} + defer func() { <-sem }() + cfg := extract.DefaultCfg() + cfg.Mode, cfg.Floor, cfg.Rewrite = e.strategy, floor, e.rewrite + ctx, cancel := context.WithTimeout(c.Ctx, llmCallTimeout) + defer cancel() + res, sum, _ := extract.RunExtractionSummary(ctx, cands[k].content, goal, keepIDs, schema.TextTokens(cands[k].content), cfg, model) + if res != "" && res != cands[k].content { + out[k] = outT{res, sum} + } + }(k) + } + wg.Wait() + for k := range cands { // Phase 3 (serial): freeze + splice. + if out[k].projected == "" { + continue + } + putResult(c, cands[k].id, []byte(out[k].projected)) + if out[k].summary != "" { + putSummary(c, cands[k].id, out[k].summary) + } + apply(cands[k].i, cands[k].content, out[k].projected, out[k].summary) + } + } + + if changed == 0 { + rep.Skipped = true + } + return keys, nil +} diff --git a/components/offload/failed_run.go b/components/offload/failed_run.go index 3db3b4a..690a58d 100644 --- a/components/offload/failed_run.go +++ b/components/offload/failed_run.go @@ -17,6 +17,17 @@ func init() { components.Register("failed_run", newFailedRun) } // positives only cost an expand round-trip, never data (the original is stashed). var runMarkers = regexp.MustCompile(`(?i)(\d+ (passed|failed|error)|BUILD (SUCCESS|FAIL)|=+ (FAILURES|test session)|Traceback \(most recent|\bFAILED\b|\bpanic:|\bnpm ERR!)`) +// failMarkers identify a run that FAILED. Only a failed earlier run is safely +// "superseded" by a later run (the agent fixed it and moved on); a PASSED/successful +// earlier run is a distinct result the agent may still reference (e.g. `pytest test_a` +// passing, then `pytest test_b`), so it is kept verbatim. Restricting collapse to +// failures is what keeps failed_run from hiding a still-relevant successful result — +// a general, agent-agnostic safety rule. +// Note: the count must be NON-ZERO ("0 failed" is a PASS, not a failure); the bare +// pytest "FAILED " token is matched CASE-SENSITIVELY so a lowercase "0 failed" +// summary doesn't trip it. +var failMarkers = regexp.MustCompile(`(?i)([1-9]\d* (failed|error(s|ed)?)\b|build fail|=+ failures|traceback \(most recent|\bpanic:|\bnpm err!|\bexit(ed with)? (code )?[1-9])|(?-i:\bFAILED\b)`) + // FailedRun collapses earlier test/build runs that a later run supersedes: only // the most recent run-like tool output is kept in full; earlier ones become a // pointer + stash. This is the "provable-reason" collapse — a superseded run is @@ -70,16 +81,54 @@ func (fr *FailedRun) Offload(req *schemas.BifrostChatRequest, rep *components.Re rep.Skipped = true return nil, nil } - // Keep the last run in full; collapse every earlier one. + // Keep the last run in full; collapse every earlier one THAT FAILED. A passed/ + // successful earlier run is a distinct result the agent may still reference, so + // it stays verbatim (only genuinely-superseded failures are collapsed). var keys []string + changed := 0 for _, i := range runs[:len(runs)-1] { m := &req.Input[i] content := schema.MessageText(*m) - tok, key := mark(c, rep, fr.mode, content, " [full output: call "+expand.ToolName+"]") - schema.SetMessageText(m, "[superseded by a later run] "+tok) + // Reapply a previously-frozen collapse on EVERY turn (cache-stable), regardless + // of the tail boundary — the agent re-sends the original, so we must re-collapse + // it to the same bytes or it reverts to full and churns the cache. + if fk, saved, ok := reapplyFrozen(c, fr.Name(), m); ok { + rep.TokensBefore += saved // (report best-effort; pipeline recomputes exact) + changed++ + keys = append(keys, fk...) + continue + } + if isKeptVerbatim(c, contentKey(content)) { + continue // agent expanded this superseded run; leave it verbatim (no bounce) + } + if !failMarkers.MatchString(content) { + continue // earlier run succeeded — not superseded, keep it + } + // A NEW collapse mutates an OLDER message (the superseded run), which on a + // prompt-cached agent flips already-cached content full→collapsed and forces a + // cache-write of the whole suffix — the dominant +cost we measured (121 such + // transitions on SWE-50). On a cached agent the superseded run already bills at + // the cheap cache-read rate, so collapsing it doesn't pay: skip NEW collapses + // entirely (frozen ones are still reapplied above for stability). With caching + // OFF, collapse freely — there the content cut is a direct saving. + if c.CacheAware { + continue + } + newText, key, eff, ok := tryMark(c, fr.mode, content, " [full output: call "+expand.ToolName+"]", + func(tok string) string { return "[superseded by a later failed→re-run] " + tok }) + if !ok { + continue // collapse+marker wouldn't shrink this run; leave it verbatim + } + commitMark(c, rep, eff, key, content) + schema.SetMessageText(m, newText) + freeze(c, fr.Name(), content, newText) // freeze so later turns replay it (no churn) + changed++ if key != "" { keys = append(keys, key) } } + if changed == 0 { + rep.Skipped = true + } return keys, nil } diff --git a/components/offload/marker.go b/components/offload/marker.go index a9de545..621bd69 100644 --- a/components/offload/marker.go +++ b/components/offload/marker.go @@ -3,6 +3,7 @@ package offload import ( "github.com/rossoctl/context-guru/components" "github.com/rossoctl/context-guru/expand" + "github.com/rossoctl/context-guru/schema" ) // markerMode selects what an Offload component leaves behind in place of the @@ -61,6 +62,13 @@ func effectiveMode(c *components.Ctx, mode markerMode) markerMode { // // hint is the component-specific recovery hint (e.g. " [full output: call // context_guru_expand]"); it is only emitted in full mode, where expand works. +// +// Deprecated: mark stashes eagerly, before the caller knows whether the marker- +// bearing rewrite is actually smaller than the original. New offloaders should use +// tryMark + commitMark, which apply a marker-INCLUSIVE never-worse check per message +// and only stash on a committed win (avoids growing a single message by the marker's +// ~10-15 tokens, and avoids orphan store entries for rewrites that are then rejected). +// Retained only for tests; all shipped offloaders use tryMark. func mark(c *components.Ctx, rep *components.Report, mode markerMode, original, hint string) (token, key string) { if effectiveMode(c, mode) == markerFull { key = hashKey(original) @@ -73,3 +81,47 @@ func mark(c *components.Ctx, rep *components.Report, mode markerMode, original, } return "", "" // off } + +// markToken computes the marker token and store key for a mode WITHOUT stashing. +// It returns the effective mode (full degrades to off when the store can't persist). +// This is the "plan" half of the split that lets a component build its candidate +// rewrite and size-check it (marker included) before committing any side effect. +func markToken(c *components.Ctx, mode markerMode, original, hint string) (token, key string, eff markerMode) { + eff = effectiveMode(c, mode) + switch eff { + case markerFull: + key = hashKey(original) + return expand.Marker(key) + hint, key, eff + case markerSummary: + return expand.SummaryMarker, "", eff + default: // off + return "", "", eff + } +} + +// tryMark builds the candidate replacement text an Offload wants to write — +// assemble(token), where the component's layout closure places the marker token — +// and reports whether that text is strictly smaller than the original, MARKER +// INCLUDED. It performs no side effects (no stash), so a caller that gets ok=false +// leaves the message verbatim. This is the shared marker-inclusive never-worse guard +// (the aggregate pipeline guard is per-request, not per-message, so without this a +// single small output could grow by the marker's tokens while the request still net-shrinks). +func tryMark(c *components.Ctx, mode markerMode, original, hint string, assemble func(token string) string) (newText, key string, eff markerMode, ok bool) { + token, key, eff := markToken(c, mode, original, hint) + newText = assemble(token) + ok = schema.TextTokens(newText) < schema.TextTokens(original) + return newText, key, eff, ok +} + +// commitMark performs the side effects once a caller accepts a tryMark candidate: +// stash the original under key (full mode) or record the deliberate lossy drop +// (summary/off set rep.Irreversible so the pipeline's "dropped without stashing" +// guard doesn't revert them). Call only when tryMark returned ok. +func commitMark(c *components.Ctx, rep *components.Report, eff markerMode, key, original string) { + if eff == markerFull { + c.Store.Put(key, []byte(original)) + recordOwner(c, key) // scope GET /expand retrieval to this session + return + } + rep.Irreversible = true +} diff --git a/components/offload/mask.go b/components/offload/mask.go index 8f711cd..9867304 100644 --- a/components/offload/mask.go +++ b/components/offload/mask.go @@ -15,15 +15,19 @@ func init() { components.Register("mask", newMask) } // older ones are replaced with a short marker + stash. Age-based, complementary // to the content-based offloaders. type Mask struct { - keepRecent int - minTokens int - mode markerMode + keepRecent int + minTokens int + keepHeadChars int + mode markerMode } type maskConfig struct { - KeepRecent int `yaml:"keep_recent"` - MinTokens int `yaml:"min_tokens"` - MarkerMode string `yaml:"marker_mode"` // full (default) | summary | off + KeepRecent int `yaml:"keep_recent"` + MinTokens int `yaml:"min_tokens"` + // KeepHeadChars leaves a one-line peek of the masked output inside the marker + // so the model knows what was hidden (cuts blind expand round-trips); 0 disables. + KeepHeadChars *int `yaml:"keep_head_chars"` + MarkerMode string `yaml:"marker_mode"` // full (default) | summary | off } func newMask(raw []byte) (components.Component, error) { @@ -33,7 +37,11 @@ func newMask(raw []byte) (components.Component, error) { return nil, err } } - return &Mask{keepRecent: cfg.KeepRecent, minTokens: cfg.MinTokens, mode: parseMarkerMode(cfg.MarkerMode)}, nil + keepHead := 96 // default: one-line cue in the marker + if cfg.KeepHeadChars != nil { + keepHead = *cfg.KeepHeadChars + } + return &Mask{keepRecent: cfg.KeepRecent, minTokens: cfg.MinTokens, keepHeadChars: keepHead, mode: parseMarkerMode(cfg.MarkerMode)}, nil } func (Mask) Name() string { return "mask" } @@ -54,14 +62,43 @@ func (m *Mask) Offload(req *bschemas.BifrostChatRequest, rep *components.Report, continue // non-text blocks would be dropped by a text rewrite } content := schema.MessageText(*msg) - if content == "" || schema.TextTokens(content) < m.minTokens { + if content == "" { + continue + } + // Reapply a previously-frozen mask on EVERY turn (cache-stable), regardless of + // the tail boundary: the agent re-sends the original, so we must re-mask it to the + // same bytes or it reverts full→masked→full and churns the provider KV cache. This + // also skips kept-verbatim content (see reapplyFrozen). + if fk, saved, ok := reapplyFrozen(c, m.Name(), msg); ok { + rep.TokensBefore += saved // (report best-effort; pipeline recomputes exact) + changed++ + keys = append(keys, fk...) + continue + } + if skipReduce(c, content) { + continue // already offloaded, or expanded by the agent — don't re-hide + } + if schema.TextTokens(content) < m.minTokens { continue } - if expand.HasPlaceholder(content) { + // A NEW mask only in the uncached tail: masking content the provider already + // cached flips it full→masked and forces a cache-write of the suffix. Frozen masks + // are replayed everywhere above; new ones stay in the tail. + if !c.TailOnly(i) { continue } - tok, key := mark(c, rep, m.mode, content, " [full output: call "+expand.ToolName+"]") - schema.SetMessageText(msg, "[older tool output masked] "+tok) + prefix := "[older tool output masked] " + if peek := headPeek(content, m.keepHeadChars); peek != "" { + prefix = "[older tool output masked; starts: " + peek + "] " + } + newText, key, eff, ok := tryMark(c, m.mode, content, " [full output: call "+expand.ToolName+"]", + func(tok string) string { return prefix + tok }) + if !ok { + continue // marker-inclusive rewrite wouldn't shrink this message; leave it verbatim + } + commitMark(c, rep, eff, key, content) + schema.SetMessageText(msg, newText) + freeze(c, m.Name(), content, newText) // freeze so later turns replay it (no churn) changed++ if key != "" { keys = append(keys, key) diff --git a/components/offload/ownership_test.go b/components/offload/ownership_test.go new file mode 100644 index 0000000..08e5465 --- /dev/null +++ b/components/offload/ownership_test.go @@ -0,0 +1,30 @@ +package offload + +import ( + "testing" + + "github.com/rossoctl/context-guru/components" + "github.com/rossoctl/context-guru/store" +) + +// A stash is keyed by a global content hash, so GET /expand must be scoped to the +// session that produced it — otherwise any caller could fetch another session's +// offloaded original by supplying its id (cross-session/tenant disclosure). +func TestOwnsKeyScopesBySession(t *testing.T) { + st := store.NewMemory(store.Options{}) + cA := &components.Ctx{Session: "A", Store: st} + recordOwner(cA, "hash123") + + if !OwnsKey(st, "A", "hash123") { + t.Fatal("session A must own the key it stashed") + } + if OwnsKey(st, "B", "hash123") { + t.Fatal("session B must NOT own session A's stash (IDOR)") + } + if OwnsKey(st, "", "hash123") { + t.Fatal("an empty session must own nothing") + } + if OwnsKey(st, "A", "") { + t.Fatal("an empty key must never be owned") + } +} diff --git a/components/offload/phi_evict.go b/components/offload/phi_evict.go deleted file mode 100644 index 920ccca..0000000 --- a/components/offload/phi_evict.go +++ /dev/null @@ -1,136 +0,0 @@ -package offload - -import ( - "sort" - - bschemas "github.com/maximhq/bifrost/core/schemas" - "github.com/rossoctl/context-guru/components" - "github.com/rossoctl/context-guru/expand" - "github.com/rossoctl/context-guru/schema" - "gopkg.in/yaml.v3" -) - -func init() { components.Register("phi_evict", newPhiEvict) } - -// PhiEvict ranks tool outputs by a lean-ctx-style context-field score Φ and -// offloads the lowest-scoring ones until the transcript fits a token budget. -// -// Φ = wR·relevance + wH·recency − wC·cost − wD·redundancy -// -// This is the scalar-ranking essence of lean-ctx's Context Field; the full -// heat-diffusion/PageRank/Thompson-bandit machinery is a documented refinement. -// The MMR/Lost-in-the-Middle reorder is a separate ordering concern (deferred); -// here Φ drives eviction, the reduction. The most recent tool output is never -// evicted (the agent is most likely to need it). -type PhiEvict struct { - budget int - weights weights - mode markerMode -} - -type weights struct{ R, H, C, D float64 } - -type phiConfig struct { - BudgetTokens int `yaml:"budget_tokens"` - Weights string `yaml:"weights"` // balanced | aggressive | conservative - MarkerMode string `yaml:"marker_mode"` // full (default) | summary | off -} - -func newPhiEvict(raw []byte) (components.Component, error) { - cfg := phiConfig{BudgetTokens: 120000, Weights: "balanced"} - if len(raw) > 0 { - if err := yaml.Unmarshal(raw, &cfg); err != nil { - return nil, err - } - } - return &PhiEvict{budget: cfg.BudgetTokens, weights: presetWeights(cfg.Weights), mode: parseMarkerMode(cfg.MarkerMode)}, nil -} - -func presetWeights(name string) weights { - switch name { - case "aggressive": // punish cost harder - return weights{R: 0.30, H: 0.10, C: 0.45, D: 0.15} - case "conservative": // trust relevance/recency, light cost - return weights{R: 0.45, H: 0.20, C: 0.05, D: 0.05} - default: // balanced (lean-ctx defaults, cost/redundancy folded) - return weights{R: 0.40, H: 0.20, C: 0.30, D: 0.10} - } -} - -func (PhiEvict) Name() string { return "phi_evict" } -func (PhiEvict) Enabled(*components.Ctx) bool { return true } - -type scored struct { - idx int - tokens int - phi float64 -} - -func (p *PhiEvict) Offload(req *bschemas.BifrostChatRequest, rep *components.Report, c *components.Ctx) ([]string, error) { - tools := toolIndices(req) - total := 0 - for _, i := range tools { - total += schema.TextTokens(schema.MessageText(req.Input[i])) - } - if total <= p.budget || len(tools) <= 1 { - rep.Skipped = true - return nil, nil - } - - query := keywords(lastUserText(req)) - items := make([]scored, 0, len(tools)) - seen := map[string]struct{}{} - for pos, i := range tools { - content := schema.MessageText(req.Input[i]) - tk := schema.TextTokens(content) - recency := float64(pos) / float64(len(tools)-1) // 0..1, newest = 1 - cost := 0.0 - if total > 0 { - cost = float64(tk) / float64(total) - } - redundancy := 0.0 - if h := hashKey(content); h != "" { - if _, dup := seen[h]; dup { - redundancy = 1 - } - seen[h] = struct{}{} - } - phi := p.weights.R*overlap(query, content) + p.weights.H*recency - - p.weights.C*cost - p.weights.D*redundancy - items = append(items, scored{idx: i, tokens: tk, phi: phi}) - } - - // Evict lowest Φ first; never the most recent tool output. - newest := tools[len(tools)-1] - sort.SliceStable(items, func(a, b int) bool { return items[a].phi < items[b].phi }) - - var keys []string - changed := 0 - for _, it := range items { - if total <= p.budget { - break - } - if it.idx == newest { - continue - } - msg := &req.Input[it.idx] - if !schema.Rewritable(*msg) { - continue // non-text blocks would be dropped by a text rewrite - } - content := schema.MessageText(*msg) - if expand.HasPlaceholder(content) { - continue - } - tok, key := mark(c, rep, p.mode, content, " [full output: call "+expand.ToolName+"]") - schema.SetMessageText(msg, "[evicted to fit context budget] "+tok) - if key != "" { - keys = append(keys, key) - } - changed++ - total -= it.tokens - } - if changed == 0 { - rep.Skipped = true - } - return keys, nil -} diff --git a/components/offload/skeleton.go b/components/offload/skeleton.go index 765fd74..5c4f0ad 100644 --- a/components/offload/skeleton.go +++ b/components/offload/skeleton.go @@ -83,6 +83,9 @@ func (s *Skeleton) Offload(req *schemas.BifrostChatRequest, rep *components.Repo if content == "" || !strings.Contains(content, "```") { continue } + if skipReduce(c, content) { + continue // already carries a marker, or was expanded by the agent — don't re-reduce + } matches := fenceRe.FindAllStringSubmatchIndex(content, -1) if matches == nil { continue @@ -108,8 +111,14 @@ func (s *Skeleton) Offload(req *schemas.BifrostChatRequest, rep *components.Repo continue } out.WriteString(content[last:]) - tok, key := mark(c, rep, s.mode, content, " [full source: call "+expand.ToolName+"]") - schema.SetMessageText(m, out.String()+"\n"+tok) + body := out.String() + newText, key, eff, ok := tryMark(c, s.mode, content, " [full source: call "+expand.ToolName+"]", + func(tok string) string { return body + "\n" + tok }) + if !ok { + continue // skeleton+marker wouldn't shrink this message; leave it verbatim + } + commitMark(c, rep, eff, key, content) + schema.SetMessageText(m, newText) emitted++ if key != "" { keys = append(keys, key) @@ -121,6 +130,10 @@ func (s *Skeleton) Offload(req *schemas.BifrostChatRequest, rep *components.Repo return keys, nil } +// maxParseDepth bounds skeleton's tree-walk recursion over untrusted parse trees so +// pathologically nested input can't overflow the Go stack (an uncatchable crash). +const maxParseDepth = 5000 + // skeletonize parses src and replaces function/method/constructor bodies with a // placeholder, keeping everything else. Returns ok=false on parse failure or // when there is nothing to elide (fail-open: caller leaves the block untouched). @@ -135,8 +148,16 @@ func skeletonize(src []byte, grammar string) (string, bool) { return "", false } var ranges [][2]uint - var walk func(n *sitter.Node, parentKind string) - walk = func(n *sitter.Node, parentKind string) { + var walk func(n *sitter.Node, parentKind string, depth int) + walk = func(n *sitter.Node, parentKind string, depth int) { + // Bound recursion depth: the parse tree comes from untrusted tool-output text, + // and a deeply nested input (thousands of nested blocks/parens) would overflow + // the Go stack — a FATAL runtime throw that recover() cannot catch, crashing the + // whole proxy. Past the limit we stop descending (fail-open: leave that subtree + // un-skeletonized). maxParseDepth is far beyond any real source nesting. + if depth > maxParseDepth { + return + } kind := n.Kind() if isBodyKind(kind) && isDeclKind(parentKind) { ranges = append(ranges, [2]uint{n.StartByte(), n.EndByte()}) @@ -144,11 +165,11 @@ func skeletonize(src []byte, grammar string) (string, bool) { } for i := uint(0); i < n.ChildCount(); i++ { if ch := n.Child(i); ch != nil { - walk(ch, kind) + walk(ch, kind, depth+1) } } } - walk(root, "") + walk(root, "", 0) if len(ranges) == 0 { return "", false } diff --git a/components/offload/smartcrush.go b/components/offload/smartcrush.go index fd59a6e..c77f621 100644 --- a/components/offload/smartcrush.go +++ b/components/offload/smartcrush.go @@ -57,8 +57,8 @@ func (s *SmartCrush) Offload(req *bschemas.BifrostChatRequest, rep *components.R continue // non-text blocks would be dropped by a text rewrite } content := schema.MessageText(*msg) - if expand.HasPlaceholder(content) { - continue // already offloaded by an earlier component/turn + if skipReduce(c, content) { + continue // already offloaded by an earlier component/turn, or expanded by the agent } trimmed := strings.TrimSpace(content) if len(trimmed) == 0 || trimmed[0] != '[' || schema.TextTokens(content) < s.minTokens { @@ -82,9 +82,14 @@ func (s *SmartCrush) Offload(req *bschemas.BifrostChatRequest, rep *components.R if err != nil { continue } - tok, key := mark(c, rep, s.mode, content, " [full array: call "+expand.ToolName+"]") note := fmt.Sprintf(" [%d of %d items shown] ", len(kept), len(items)) - schema.SetMessageText(msg, string(crushed)+note+tok) + newText, key, eff, ok := tryMark(c, s.mode, content, " [full array: call "+expand.ToolName+"]", + func(tok string) string { return string(crushed) + note + tok }) + if !ok { + continue // crushed array+marker wouldn't shrink this message; leave it verbatim + } + commitMark(c, rep, eff, key, content) + schema.SetMessageText(msg, newText) changed++ if key != "" { keys = append(keys, key) diff --git a/components/offload/state.go b/components/offload/state.go index 33ed3ef..957e004 100644 --- a/components/offload/state.go +++ b/components/offload/state.go @@ -7,6 +7,10 @@ import ( bschemas "github.com/maximhq/bifrost/core/schemas" "github.com/rossoctl/context-guru/components" + "github.com/rossoctl/context-guru/expand" + "github.com/rossoctl/context-guru/internal/extract" + "github.com/rossoctl/context-guru/schema" + "github.com/rossoctl/context-guru/store" ) // State reuse over the generic Store (key→bytes), session-scoped by key prefix @@ -29,6 +33,129 @@ func putResult(c *components.Ctx, id string, v []byte) { c.Store.Put(resultKey(c.Session, id), v) } +// --- Freeze + reapply (cache stability) ------------------------------------- +// +// The cache-safety invariant: once an offloader compacts an output, it must send the +// SAME bytes for that output on every later turn — otherwise the agent (which re-sends +// the ORIGINAL each turn) makes the output flip compacted→full→compacted, churning the +// provider KV cache. A tail gate alone is not enough: it compacts in the tail then +// skips once the content is in the prefix, which is exactly that churn. So a tail-gated +// offloader FREEZES its decision here (keyed by component + original-content hash) and +// REAPPLIES it on every turn, regardless of the tail boundary. New decisions are still +// gated to the tail; frozen ones are replayed everywhere. + +func frozenKey(session, comp, ck string) string { return "cg:frz:" + session + ":" + comp + ":" + ck } + +// freeze records the replacement text a component produced for an original content, so +// later turns replay it byte-for-byte. +func freeze(c *components.Ctx, comp, original, replacement string) { + c.Store.Put(frozenKey(c.Session, comp, contentKey(original)), []byte(replacement)) +} + +// reapplyFrozen replays a component's frozen decision for the message at m, if one +// exists and still shrinks it. It also refreshes the expand originals for any markers +// in the replacement (the agent re-sent the full original as m's content), so +// restoration keeps working across turns. Returns the marker keys + whether it acted. +func reapplyFrozen(c *components.Ctx, comp string, m *bschemas.ChatMessage) ([]string, int, bool) { + content := schema.MessageText(*m) + if isKeptVerbatim(c, contentKey(content)) { + return nil, 0, false // agent expanded this; replaying the collapse would loop + } + repl, ok := c.Store.Get(frozenKey(c.Session, comp, contentKey(content))) + if !ok { + return nil, 0, false + } + rs := string(repl) + saved := schema.TextTokens(content) - schema.TextTokens(rs) + if saved <= 0 { + return nil, 0, false + } + keys := expand.ParseMarkers(rs) + for _, k := range keys { + c.Store.Put(k, []byte(content)) // refresh the stashed original for expand + } + schema.SetMessageText(m, rs) + return keys, saved, true +} + +// contentKey is a marker/whitespace-insensitive content hash (shared with extract's +// result cache), so the same output re-sent across turns maps to one frozen decision. +func contentKey(s string) string { return extract.ContentKey(s) } + +// --- Kept-verbatim (expanded content) ------------------------------------- +// +// When the agent expands an offloaded output, re-compacting it on the next turn would +// just make the agent expand it again — an expand loop (wasted round-trips + cache +// churn). The expand handler marks the restored content's key kept-verbatim so the +// offloaders leave it alone thereafter. + +func keptKey(ck string) string { return "cg:keep:" + ck } + +// MarkKeptVerbatim records that this original content was expanded and must not be +// re-compacted (keyed by content hash, session-independent). Exported for the proxy's +// expand loop, which has the restored original but not the offload Ctx. +func MarkKeptVerbatim(st store.Store, original string) { + st.Put(keptKey(contentKey(original)), []byte{1}) +} + +func isKeptVerbatim(c *components.Ctx, ck string) bool { + _, ok := c.Store.Get(keptKey(ck)) + return ok +} + +// skipReduce reports whether an offloader must leave this content untouched: it +// already carries an offload marker (reducing again would double-compact and can +// orphan the earlier stash), or the agent expanded it and re-compacting would just +// trigger another expand — a per-turn bounce loop. Every offloader consults this on +// each candidate so the kept-verbatim / never-double-reduce guarantees hold uniformly. +func skipReduce(c *components.Ctx, content string) bool { + return expand.HasPlaceholder(content) || isKeptVerbatim(c, contentKey(content)) +} + +// --- Stash ownership (scoping GET /expand by session) ---------------------- +// +// A rewind stash is keyed by a content HASH, which is global (the same content in two +// sessions hashes the same). The model-driven expand loop is inherently same-session (a +// request only ever contains markers minted from its own content), but the management +// GET /expand endpoint takes an arbitrary id — so without a scope check any client that +// reaches the proxy could fetch another session's stashed original. We record, per +// (session, key), that this session stashed that key, and GET /expand only resolves a +// key the caller's session actually owns. + +func ownerKey(session, key string) string { return "cg:own:" + session + ":" + key } + +// recordOwner marks that this session stashed key (no-op for empty key / summary-off). +func recordOwner(c *components.Ctx, key string) { + if key != "" { + c.Store.Put(ownerKey(c.Session, key), []byte{1}) + } +} + +// OwnsKey reports whether session stashed key. Exported for the proxy's GET /expand +// handler to scope retrieval to the caller's session (prevents cross-session/tenant +// disclosure of offloaded originals). Returns false when either is empty. +func OwnsKey(st store.Store, session, key string) bool { + if session == "" || key == "" { + return false + } + _, ok := st.Get(ownerKey(session, key)) + return ok +} + +// summaryKey namespaces the one-line SUMMARY the LLM extract emitted for a content +// id, so a later turn reusing the cached reduction also re-emits the same marker +// digest (byte-stable) without re-calling the model. +func summaryKey(session, id string) string { return "cg:sum1:" + session + ":" + id } + +func getSummary(c *components.Ctx, id string) (string, bool) { + b, ok := c.Store.Get(summaryKey(c.Session, id)) + return string(b), ok +} + +func putSummary(c *components.Ctx, id, s string) { + c.Store.Put(summaryKey(c.Session, id), []byte(s)) +} + // sumCheckpoint is the per-session summarize state: the exact summary message // text produced last time (re-emitted verbatim so the prefix stays byte-stable), // how many leading span messages it subsumed, a hash of that span to prove the diff --git a/components/offload/summarize.go b/components/offload/summarize.go index 101f0ce..0739fe7 100644 --- a/components/offload/summarize.go +++ b/components/offload/summarize.go @@ -94,7 +94,7 @@ func (s *Summarize) Offload(req *bschemas.BifrostChatRequest, rep *components.Re start, end := 1, len(msgs)-s.keepLast // Request-level trigger: don't summarize (an LLM call) until the transcript // is genuinely large / deep. Zero thresholds fire always (back-compat). - if !s.trigger.Fires(req) || end <= start { + if !s.trigger.Fires(req, c.CtxWindow) || end <= start { rep.Skipped = true return nil, nil } diff --git a/components/pipeline.go b/components/pipeline.go index 5c246e3..07c1a8a 100644 --- a/components/pipeline.go +++ b/components/pipeline.go @@ -41,13 +41,21 @@ func (p *Pipeline) Run(req *schemas.BifrostChatRequest, c *Ctx) *RunReport { rep := p.runOne(comp, req, c) rr.Components = append(rr.Components, rep) rr.DurationMs += rep.DurationMs - p.emitter.Component(rep) + safeEmit(func() { p.emitter.Component(rep) }) } rr.TokensAfter = schema.MessagesTokens(req) - p.emitter.Run(*rr) + safeEmit(func() { p.emitter.Run(*rr) }) return rr } +// safeEmit runs an emitter callback under recover: metrics/observability must never +// break a request. A panicking emitter is swallowed (fail-open) rather than propagating +// out of Run, where — unlike component code — no other recover would catch it. +func safeEmit(fn func()) { + defer func() { _ = recover() }() + fn() +} + // runOne executes a single component with snapshot/restore isolation and the // never-worse guard. It never returns an error — failures are recorded on the // Report and the request is reverted. diff --git a/components/trigger.go b/components/trigger.go index 3f8cd72..ee96a60 100644 --- a/components/trigger.go +++ b/components/trigger.go @@ -1,6 +1,8 @@ package components import ( + "math" + "github.com/maximhq/bifrost/core/schemas" "github.com/rossoctl/context-guru/schema" ) @@ -21,17 +23,66 @@ type Trigger struct { MinRequestTokens int `yaml:"min_request_tokens"` // whole request must be at least this many tokens MinMessages int `yaml:"min_messages"` // …and carry at least this many messages (≈ steps) MinOutputTokens int `yaml:"min_output_tokens"` // per-item floor: only offload an output at least this big + + // Context-window fractions (0 = unset) make triggers general across models: + // each is resolved against Ctx.CtxWindow (the model's max input tokens, obtained + // dynamically). When the window is unknown (0) fractions are ignored and only the + // absolute thresholds apply — fully backward compatible. + MinRequestFrac float64 `yaml:"min_request_frac"` // fire when request >= frac*window (e.g. 0.6) + MinOutputFrac float64 `yaml:"min_output_frac"` // per-item: only offload an output >= frac*window + HugeOutputFrac float64 `yaml:"huge_output_frac"` // HARD per-item trigger: a single output >= frac*window +} + +// frac converts a fraction of the window to an absolute token count (0 if either is unset). +func frac(f float64, window int) int { + if f <= 0 || window <= 0 { + return 0 + } + return int(math.Ceil(f * float64(window))) } -// Fires reports whether the request-level thresholds are met. Both are ANDed; -// a zero threshold imposes no constraint. It does not consider MinOutputTokens -// (that is a per-item floor the component applies itself). -func (t Trigger) Fires(req *schemas.BifrostChatRequest) bool { +// Fires reports whether the request-level thresholds are met, given the resolved +// model context window (0 = unknown). The effective request-token threshold is the +// MAX of the absolute MinRequestTokens and the fraction MinRequestFrac*window; the +// message-count threshold is unchanged. Thresholds are ANDed; a zero threshold +// imposes no constraint. Does not consider the per-item floors (OutputFloor/IsHuge). +func (t Trigger) Fires(req *schemas.BifrostChatRequest, window int) bool { if t.MinMessages > 0 && len(req.Input) < t.MinMessages { return false } - if t.MinRequestTokens > 0 && schema.MessagesTokens(req) < t.MinRequestTokens { + reqFloor := t.MinRequestTokens + if f := frac(t.MinRequestFrac, window); f > reqFloor { + reqFloor = f + } + if reqFloor > 0 && schema.MessagesTokens(req) < reqFloor { return false } return true } + +// OutputFloor is the per-item minimum size an Offload should act on: the absolute +// MinOutputTokens if set, else MinOutputFrac*window, else legacyDefault (the +// component's pre-trigger min_tokens default). Lets a component keep firing sensibly +// whether configured absolutely, as a fraction, or not at all. +func (t Trigger) OutputFloor(window, legacyDefault int) int { + // The absolute floor is the base; the window fraction only RAISES it (never replaces + // it). Returning the fraction outright was a footgun: on a large-window model + // (e.g. 1M) a small frac like 0.0075 resolved to 7500, silently overriding a 1500 + // absolute and suppressing nearly all compaction. max() keeps both meaningful. + base := legacyDefault + if t.MinOutputTokens > 0 { + base = t.MinOutputTokens + } + if f := frac(t.MinOutputFrac, window); f > base { + return f + } + return base +} + +// IsHuge reports whether a single tool output is large enough (>= HugeOutputFrac*window) +// to be a "huge tool call" hard trigger — worth acting on regardless of the request-level +// Fires gate. Returns false when the window is unknown or HugeOutputFrac is unset. +func (t Trigger) IsHuge(outputTokens, window int) bool { + h := frac(t.HugeOutputFrac, window) + return h > 0 && outputTokens >= h +} diff --git a/components/trigger_test.go b/components/trigger_test.go index 20ed569..7c3482a 100644 --- a/components/trigger_test.go +++ b/components/trigger_test.go @@ -34,8 +34,53 @@ func TestTriggerFires(t *testing.T) { {"MinOutputTokens does not affect Fires", Trigger{MinOutputTokens: 99999}, deep, true}, } for _, c := range cases { - if got := c.tr.Fires(c.req); got != c.want { + if got := c.tr.Fires(c.req, 0); got != c.want { // window 0 = only absolutes apply t.Errorf("%s: Fires=%v want %v", c.name, got, c.want) } } } + +func TestTriggerFractions(t *testing.T) { + // big ~ 4000 "word " tokens; window 200k. + big := &schemas.BifrostChatRequest{Input: []schemas.ChatMessage{mkMsg(strings.Repeat("word ", 4000))}} + const W = 200000 + + // MinRequestFrac 0.6*200k=120k > big(~4k) => does not fire. + if (Trigger{MinRequestFrac: 0.6}).Fires(big, W) { + t.Error("frac 0.6 of 200k should not fire on a ~4k request") + } + // A tiny frac fires. + if !(Trigger{MinRequestFrac: 0.001}).Fires(big, W) { + t.Error("frac 0.001 of 200k (=200) should fire on a ~4k request") + } + // Unknown window (0) => fraction ignored, so a big frac still fires. + if !(Trigger{MinRequestFrac: 0.9}).Fires(big, 0) { + t.Error("fraction must be ignored when window unknown (0)") + } + // Absolute wins when larger than the fraction term. + if !(Trigger{MinRequestTokens: 1, MinRequestFrac: 0.9}).Fires(big, 0) { + t.Error("with window 0, only the (met) absolute applies") + } + + // OutputFloor precedence: absolute > frac > legacy. + if got := (Trigger{MinOutputTokens: 500}).OutputFloor(W, 300); got != 500 { + t.Errorf("absolute floor should win: got %d", got) + } + if got := (Trigger{MinOutputFrac: 0.01}).OutputFloor(W, 300); got != 2000 { + t.Errorf("frac floor 0.01*200k=2000: got %d", got) + } + if got := (Trigger{}).OutputFloor(0, 300); got != 300 { + t.Errorf("legacy default when nothing set / window unknown: got %d", got) + } + + // IsHuge: 0.15*200k=30k threshold. + if !(Trigger{HugeOutputFrac: 0.15}).IsHuge(31000, W) { + t.Error("31k output should be huge at 0.15*200k") + } + if (Trigger{HugeOutputFrac: 0.15}).IsHuge(31000, 0) { + t.Error("huge must be false when window unknown") + } + if (Trigger{}).IsHuge(999999, W) { + t.Error("huge must be false when HugeOutputFrac unset") + } +} diff --git a/config/config.go b/config/config.go index e36c594..11334ae 100644 --- a/config/config.go +++ b/config/config.go @@ -52,12 +52,39 @@ func LoadBytes(b []byte) (*Config, error) { return &c, nil } -// applyPreset fills an empty Pipeline from the named preset. Explicit fields in -// the document always win (they were already decoded). +// applyPreset fills an empty Pipeline (and, for rich presets, default component +// configs) from the named preset. Explicit fields in the document always win — they +// were already decoded, so the preset only supplies what the user left unset. func (c *Config) applyPreset() error { if c.Preset == "" { return nil } + // Rich preset: a full config doc carrying tuned per-component settings a bare + // pipeline name-list can't express (e.g. extract_llm's cheap-model routing + + // thresholds). Decode it into a scratch Config, then use its values only where the + // user didn't specify (pipeline; per-component config merged with user override). + if doc, ok := presetConfigs[c.Preset]; ok { + var pc Config + dec := yaml.NewDecoder(bytes.NewReader([]byte(doc))) + dec.KnownFields(true) + if err := dec.Decode(&pc); err != nil { + return fmt.Errorf("config: preset %q: %w", c.Preset, err) + } + if len(c.Pipeline) == 0 { + c.Pipeline = pc.Pipeline + } + if len(pc.Components) > 0 { + merged := make(map[string]yaml.Node, len(pc.Components)+len(c.Components)) + for k, v := range pc.Components { + merged[k] = v + } + for k, v := range c.Components { // user config wins per component + merged[k] = v + } + c.Components = merged + } + return nil + } p, ok := presets[c.Preset] if !ok { return fmt.Errorf("config: unknown preset %q", c.Preset) @@ -75,7 +102,7 @@ var presets = map[string][]string{ "off": {}, // passthrough: no components (baseline / A-B control) "safe": {"format", "cacheinject"}, "balanced": {"format", "dedup", "failed_run", "cmdfilter", "cacheinject"}, - "aggressive": {"format", "dedup", "failed_run", "cmdfilter", "smartcrush", "extract", "cacheinject"}, + "aggressive": {"format", "dedup", "failed_run", "cmdfilter", "smartcrush", "extract", "extract_llm", "cacheinject"}, "coding": {"format", "skeleton", "cmdfilter", "cacheinject"}, "mcp": {"format", "smartcrush", "cacheinject"}, // agent: tuned for long agentic sessions (e.g. Claude Code on SWE-bench), @@ -85,10 +112,60 @@ var presets = map[string][]string{ // eval-containers SWE-bench sweep (see docs/RESULTS.md); extract + failed_run // + dedup add relevance/supersession/dup wins; cacheinject keeps the prefix // cacheable. Order: lossless first, then offload old-then-large, cache last. - "agent": {"format", "dedup", "failed_run", "mask", "extract", "cacheinject"}, + "agent": {"format", "dedup", "failed_run", "mask", "extract", "extract_llm", "cacheinject"}, + // general: the recommended all-round pipeline, safe+effective for any agent/ + // benchmark. Ordered by pipeline semantics: lossless repack first (format, toon) + // so downstream token counts are honest; cheap structural offloaders next (dedup, + // failed_run, cmdfilter); age-based mask; relevance-based extract; the blind + // head/tail collapse as the last-resort catch-all for anything still oversized; + // cacheinject last so the cache breakpoint sits on the final bytes. Every offloader + // defaults to marker_mode:full (reversible via the injected expand tool) and skips + // content already carrying a placeholder, so they never double-reduce. Combines the + // levers that proved reward-neutral in the benchmark sweeps without stacking the + // two overlapping old-context reducers (mask is the one kept; summarize + // is its own preset — see docs/components.md redundancy notes). + "general": {"format", "toon", "dedup", "failed_run", "cmdfilter", "mask", "extract", "extract_llm", "collapse", "cacheinject"}, // summarize restructures the whole transcript (changes the message count) — run // it alone so no other component's in-place edits race apply's rebuild. "summarize": {"summarize"}, + // codesmart / codesafe are the SWE-bench study's winning configs, shipped as the + // recommended defaults (codesmart is the proxy default). Their tuned per-component + // settings live in presetConfigs; the name-lists here keep PresetPipeline (used by + // /compact?preset=) resolving them. + "codesmart": {"format", "dedup", "failed_run", "cmdfilter", "extract_llm", "extract", "cacheinject"}, + "codesafe": {"format", "dedup", "failed_run", "cmdfilter", "extract", "collapse", "cacheinject"}, +} + +// presetConfigs carries FULL config docs for presets whose behavior depends on tuned +// per-component settings a bare pipeline name-list cannot express. Kept verbatim from +// the SWE-bench study (deploy/harbor/swebench.py): +// - codesmart (the winning cache-aware config, and the proxy default): the LLM +// relevance-trimmer extract_llm routed to the CHEAP model (model.source: config, +// nil-when-unset ⇒ it silently no-ops to deterministic — see docs), gated at 3000 +// tok so most turns make no model call, ≤4 calls/req; the free deterministic extract +// catches smaller noise; cacheinject keeps the prefix warm. +// - codesafe: the deterministic-only variant (NO LLM, by policy) — same structural +// offloaders plus a blind collapse fallback, zero model calls. +// +// Component defaults are left untouched, so general/agent/aggressive are unaffected. +var presetConfigs = map[string]string{ + "codesmart": `pipeline: [format, dedup, failed_run, cmdfilter, extract_llm, extract, cacheinject] +components: + extract: + min_tokens: 400 + extract_llm: + strategy: code + model: + source: config + min_tokens: 3000 + trigger: + min_request_tokens: 3000 + llm_every_n_requests: 1 + llm_max_per_request: 4`, + "codesafe": `pipeline: [format, dedup, failed_run, cmdfilter, extract, collapse, cacheinject] +components: + collapse: + max_tokens: 3000`, } // PresetPipeline returns the default pipeline (ordered component names) for a diff --git a/config/config_test.go b/config/config_test.go index a934238..4b5d127 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -38,6 +38,67 @@ func TestUnknownPresetErrors(t *testing.T) { } } +// TestRichPresetCarriesComponentConfig verifies the codesmart preset expands to both +// its pipeline AND its tuned per-component config (which a bare name-list can't carry) — +// specifically that extract_llm is routed to the cheap "config" model, not the default. +func TestRichPresetCarriesComponentConfig(t *testing.T) { + c, err := LoadBytes([]byte("preset: codesmart\n")) + if err != nil { + t.Fatal(err) + } + want := []string{"format", "dedup", "failed_run", "cmdfilter", "extract_llm", "extract", "cacheinject"} + if strings.Join(c.Pipeline, ",") != strings.Join(want, ",") { + t.Fatalf("codesmart pipeline = %v, want %v", c.Pipeline, want) + } + node, ok := c.Components["extract_llm"] + if !ok { + t.Fatal("codesmart must carry extract_llm component config") + } + var got struct { + Model struct { + Source string `yaml:"source"` + } `yaml:"model"` + MinTokens int `yaml:"min_tokens"` + } + if err := node.Decode(&got); err != nil { + t.Fatal(err) + } + if got.Model.Source != "config" { + t.Fatalf("extract_llm.model.source = %q, want config (cheap model)", got.Model.Source) + } + if got.MinTokens != 3000 { + t.Fatalf("extract_llm.min_tokens = %d, want 3000", got.MinTokens) + } +} + +// TestRichPresetUserOverrideWins: an explicit component config overrides the preset's. +func TestRichPresetUserOverrideWins(t *testing.T) { + c, err := LoadBytes([]byte("preset: codesmart\ncomponents:\n extract_llm:\n min_tokens: 9999\n")) + if err != nil { + t.Fatal(err) + } + var got struct { + MinTokens int `yaml:"min_tokens"` + } + node := c.Components["extract_llm"] + if err := node.Decode(&got); err != nil { + t.Fatal(err) + } + if got.MinTokens != 9999 { + t.Fatalf("user min_tokens should win: got %d, want 9999", got.MinTokens) + } + // and codesafe (deterministic-only) must contain NO extract_llm. + cs, err := LoadBytes([]byte("preset: codesafe\n")) + if err != nil { + t.Fatal(err) + } + for _, name := range cs.Pipeline { + if name == "extract_llm" { + t.Fatal("codesafe must be deterministic-only (no extract_llm)") + } + } +} + func TestStoreOptionsParse(t *testing.T) { c, err := LoadBytes([]byte("store: {ttl_seconds: 60, max_entries: 5}\n")) if err != nil { diff --git a/deploy/harbor/analyze.py b/deploy/harbor/analyze.py new file mode 100644 index 0000000..76ba871 --- /dev/null +++ b/deploy/harbor/analyze.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 +"""Deep per-task + per-component analysis of a baseline-vs-config SWE run. + +The point: separate the two things that were being conflated — + (1) CONTENT tokens context-guru removed from the request body (the "savings"), and + (2) the BILLED cost delta once Anthropic prompt-cache tiers are applied + (cache-read $0.20/M, cache-write $2.50/M, fresh input $2/M, output $10/M). +On a heavily-cached agent (2) can go the opposite way from (1): removing old content +mutates the cached prefix, forcing cache-WRITES that outweigh the content saved. + +Outputs: matched per-task table, aggregate, a cost-delta DECOMPOSITION (how much of +the delta is cache-write vs cache-read vs output/steps), and per-component content +savings + CG's own LLM cost. Writes /analysis.json for plotting. + +Usage: analyze.py [--out DIR] [--label NAME] +""" +import argparse, json, statistics as st +from pathlib import Path + +# litellm claude-sonnet-5 rates ($/token) +IN, OUT, CREAD, CWRITE = 2e-6, 10e-6, 0.2e-6, 2.5e-6 + + +def billed(r): + """Recompute billed input-side cost from the trial's token tiers (cache-aware).""" + return (r.get("fresh_input", 0) * IN + r.get("cache_read", 0) * CREAD + + r.get("cache_write", 0) * CWRITE + r.get("completion_tokens", 0) * OUT) + + +def load(p): + return {r["task"]: r for r in json.load(open(p))} + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("baseline"); ap.add_argument("config"); ap.add_argument("summary") + ap.add_argument("--out", default="/tmp/cg-runs/analysis") + ap.add_argument("--label", default="config") + ap.add_argument("--dump", default="", help="CONTEXT_GURU_DUMP jsonl for UNIQUE savings") + ap.add_argument("--baseline-summary", default="", help="baseline summary for cache-hit/latency A/B") + a = ap.parse_args() + base, cfg = load(a.baseline), load(a.config) + allc = json.load(open(a.summary))["configs"] + # pick the config summary (the one with per-component data / not the 'off' baseline) + summ = next((x for x in allc if x.get("per_component")), allc[-1]) + bsumm = next((x for x in allc if x.get("config") == "off"), None) + common = [t for t in base if t in cfg and not base[t].get("exception") + and not cfg[t].get("exception") and base[t]["reward"] is not None + and cfg[t]["reward"] is not None] + common.sort() + rows = [] + for t in common: + b, c = base[t], cfg[t] + rows.append(dict(task=t.split("__")[-1], + b_reward=b["reward"], c_reward=c["reward"], + b_steps=b["steps"], c_steps=c["steps"], + b_prompt=b["prompt_tokens"], c_prompt=c["prompt_tokens"], + b_cread=b["cache_read"], c_cread=c["cache_read"], + b_cwrite=b["cache_write"], c_cwrite=c["cache_write"], + b_fresh=b["fresh_input"], c_fresh=c["fresh_input"], + b_cost=round(billed(b), 4), c_cost=round(billed(c), 4))) + + def col(rows, k): return [r[k] for r in rows] + def s(k): return sum(col(rows, k)) + bsolved = sum(1 for r in rows if r["b_reward"] >= 1) + csolved = sum(1 for r in rows if r["c_reward"] >= 1) + print(f"\n=== {a.label}: matched {len(rows)} tasks (scored in both, no exceptions) ===") + print(f"REWARD: baseline {bsolved}/{len(rows)} {a.label} {csolved}/{len(rows)}") + print(f" {a.label}-only solves: {[r['task'] for r in rows if r['c_reward']>r['b_reward']]}") + print(f" baseline-only solves : {[r['task'] for r in rows if r['c_reward'] {a.label} total ${ccost+cg_llm:.2f} ({100*(ccost+cg_llm-bcost)/bcost:+.0f}%)") + print("cost-delta decomposition (why it moved):") + print(f" cache_write: {s('c_cwrite')-s('b_cwrite'):+,} tok = ${d_cwrite:+.2f} <-- offload re-caches the mutated prefix") + print(f" cache_read : {s('c_cread')-s('b_cread'):+,} tok = ${d_cread:+.2f}") + print(f" fresh_input: {s('c_fresh')-s('b_fresh'):+,} tok = ${d_fresh:+.2f}") + print(f" CG own LLM : ${cg_llm:+.2f}") + ch = lambda pre: 100*s(pre+"cread")/(s(pre+"cread")+s(pre+"cwrite")+s(pre+"fresh")) + print(f"cache-hit: baseline {ch('b_'):.1f}% {a.label} {ch('c_'):.1f}% steps {st.mean(col(rows,'b_steps')):.1f}->{st.mean(col(rows,'c_steps')):.1f}") + + # content savings the proxy achieved (separate from billed cost!) + print(f"\nCONTENT removed by CE (proxy /stats): {summ.get('proxy_savings_pct')}% per-component:") + for k, v in sorted((summ.get("per_component") or {}).items(), key=lambda x: -(x[1]["saved_tokens"] or 0)): + if v["saved_tokens"]: + print(f" {k:<12} {v['saved_tokens']:>10,} tok {v['runs']} runs own-latency {round((v['duration_ms'] or 0)/1000,1)}s") + print(f" CG LLM: {summ.get('cg_llm_calls')} calls, ${summ.get('cg_llm_cost')}") + + # UNIQUE vs cumulative savings (dedup by content) — the honest per-component number + uniq = {} + if a.dump: + import subprocess + jp = f"{a.out}/dump_unique.json" + Path(a.out).mkdir(parents=True, exist_ok=True) + subprocess.run(["python3", str(Path(__file__).parent / "dump_unique.py"), a.dump, "--json", jp]) + try: + uniq = json.load(open(jp)) + except Exception: + uniq = {} + + # latency + cache-hit + restoration A/B (with vs without CG) + print("\n=== LATENCY / CACHE / RESTORATION (with vs without CG) ===") + print(f" CG-added latency/req: {round(summ.get('cg_added_ms_avg') or 0,1)} ms " + f"(baseline {round((bsumm or {}).get('cg_added_ms_avg') or 0,1)} ms)") + print(f" agent wall/task: baseline {round((bsumm or {}).get('mean_agent_wall_s') or 0,1)}s " + f"{a.label} {round(summ.get('mean_agent_wall_s') or 0,1)}s") + print(f" cache_hit_rate: baseline {(bsumm or {}).get('cache_hit_rate')} {a.label} {summ.get('cache_hit_rate')}") + print(f" expand bounces (restoration fired): {summ.get('proxy_bounces')}") + + print("\n=== PER-TASK (sorted by cost delta) ===") + rows_sorted = sorted(rows, key=lambda r: (r["c_cost"] - r["b_cost"])) + print(f"{'task':<22}{'rew b/c':>8}{'steps b/c':>11}{'cost_b':>8}{'cost_c':>8}{'Δcost':>8}{'Δcwrite':>10}") + for r in rows_sorted: + rew = f"{int(r['b_reward'])}/{int(r['c_reward'])}" + steps = f"{r['b_steps']}/{r['c_steps']}" + dcost = r["c_cost"] - r["b_cost"] + dcw = r["c_cwrite"] - r["b_cwrite"] + print(f"{r['task']:<22}{rew:>8}{steps:>11}{r['b_cost']:>8.3f}{r['c_cost']:>8.3f}{dcost:>+8.3f}{dcw:>+10,}") + Path(a.out).mkdir(parents=True, exist_ok=True) + Path(f"{a.out}/analysis.json").write_text(json.dumps(dict( + label=a.label, matched=len(rows), reward=[bsolved, csolved], + cost=[round(bcost, 3), round(ccost, 3), round(cg_llm, 3)], + decomposition=dict(cache_write=round(d_cwrite, 3), cache_read=round(d_cread, 3), + fresh=round(d_fresh, 3), cg_llm=round(cg_llm, 3)), + per_component=summ.get("per_component"), + unique_savings=uniq.get("components", {}) if uniq else {}, + latency=dict(cg_added_ms=summ.get("cg_added_ms_avg"), + agent_wall_base=(bsumm or {}).get("mean_agent_wall_s"), + agent_wall_cfg=summ.get("mean_agent_wall_s")), + cache_hit=[(bsumm or {}).get("cache_hit_rate"), summ.get("cache_hit_rate")], + bounces=summ.get("proxy_bounces"), + proxy_savings_pct=summ.get("proxy_savings_pct"), + rows=rows), indent=1)) + print(f"\nwrote {a.out}/analysis.json") + + +if __name__ == "__main__": + main() diff --git a/deploy/harbor/analyze_content.py b/deploy/harbor/analyze_content.py new file mode 100644 index 0000000..1062651 --- /dev/null +++ b/deploy/harbor/analyze_content.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +"""Where are the tokens? Categorize tool-output token mass in real captured requests +so component investment is evidence-driven, not guessed. Also measures exact-duplicate +mass and same-resource re-read mass (the ceilings for dedup / a supersede-reads lever). + +Usage: analyze_content.py [more.jsonl ...] +""" +import json, sys, re, hashlib +from collections import defaultdict + +TOK = lambda s: max(1, len(s) // 4) # cheap ~4 chars/token proxy (consistent across categories) +LINE_NO = re.compile(r"^\s{0,6}\d+[\t ]") +TEST = re.compile(r"(\d+ (passed|failed|error)|=+ (FAILURES|test session)|Traceback \(most recent|PASSED|FAILED)") +SEARCH = re.compile(r"^\S+:\d+:", re.M) +DIFF = re.compile(r"^(diff --git|@@ |commit [0-9a-f]{7})", re.M) +INSTALL = re.compile(r"(Requirement already satisfied|Collecting |Installing collected|pip install)") + + +def looks_file_read(s): + lines = [l for l in s.split("\n") if l.strip()] + if len(lines) < 8: + return False + numbered = sum(1 for l in lines[:40] if LINE_NO.match(l)) + return numbered * 100 // min(len(lines), 40) >= 60 + + +def categorize(s): + if looks_file_read(s): + return "file_read" + if INSTALL.search(s): + return "install_log" + if DIFF.search(s): + return "diff" + if TEST.search(s): + return "test_log" + if len(SEARCH.findall(s)) >= 3: + return "search" + return "other" + + +def tool_texts(body, provider): + """Yield (text, tool_name_hint) for each tool output in the request.""" + out = [] + if provider == "anthropic": + for m in body.get("messages", []): + c = m.get("content") + if m.get("role") == "user" and isinstance(c, list): + for b in c: + if isinstance(b, dict) and b.get("type") == "tool_result": + t = b.get("content") + if isinstance(t, str): + out.append(t) + elif isinstance(t, list): + out.append("".join(x.get("text", "") for x in t if isinstance(x, dict))) + else: # openai + for m in body.get("messages", []): + if m.get("role") == "tool": + c = m.get("content") + out.append(c if isinstance(c, str) else json.dumps(c)) + return out + + +def main(): + files = sys.argv[1:] + # group requests by conversation (first user msg), take the LARGEST (latest) request per conv + for f in files: + recs = [json.loads(l) for l in open(f) if l.strip()] + by_conv = defaultdict(list) + for r in recs: + b = r["body"] + key = None + for m in b.get("messages", []): + if m.get("role") == "user": + key = hashlib.sha1(json.dumps(m.get("content"))[:200].encode()).hexdigest()[:8] + break + by_conv[key].append(r) + cat_tok = defaultdict(int) + total = 0 + exact_dup_tok = 0 + reread_tok = 0 # same file_read path seen more than once (only extra copies) + largest = None + for conv, rs in by_conv.items(): + rs.sort(key=lambda r: len(r["body"].get("messages", []))) + r = rs[-1] # largest/latest request in the conversation + prov = r.get("provider", "anthropic") + texts = tool_texts(r["body"], prov) + seen_hash = {} + seen_path = {} + for t in texts: + if not t: + continue + tk = TOK(t) + total += tk + cat_tok[categorize(t)] += tk + h = hashlib.sha1(t.encode()).hexdigest() + if h in seen_hash: + exact_dup_tok += tk + seen_hash[h] = 1 + if looks_file_read(t): + # crude "resource id" = first ~120 chars (path banners differ per tool) + rid = t[:120] + if rid in seen_path: + reread_tok += tk + seen_path[rid] = 1 + if largest is None or tk > 0 and len(r["body"].get("messages", [])) > largest[0]: + largest = (len(r["body"].get("messages", [])), conv, texts) + print(f"\n===== {f} =====") + print(f"tool-output tokens (largest req per conv, summed): {total:,}") + for c, v in sorted(cat_tok.items(), key=lambda x: -x[1]): + print(f" {c:12s} {v:9,} ({100*v//max(total,1):2d}%)") + print(f" exact-duplicate tool-output tokens (dedup ceiling): {exact_dup_tok:,} ({100*exact_dup_tok//max(total,1)}%)") + print(f" same-file re-read extra tokens (supersede-reads ceiling): {reread_tok:,} ({100*reread_tok//max(total,1)}%)") + + +if __name__ == "__main__": + main() diff --git a/deploy/harbor/cachecost.py b/deploy/harbor/cachecost.py new file mode 100644 index 0000000..a374bf2 --- /dev/null +++ b/deploy/harbor/cachecost.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python3 +"""Deterministic, cache-aware cost model of a context-guru config on a FIXED request +stream — the honest "does CE save cost" measure, with ZERO agent nondeterminism. + +Why: a single live baseline-vs-config run is dominated by step-count variance (the +agent takes a different path), so the billed-cost delta is noise, not the CE effect. +Here we take ONE captured request stream and compute, for the SAME turns, the billed +input cost (a) as the agent sent them (baseline) vs (b) after each is rewritten by the +config via /compact — applying Anthropic prompt-cache tiers turn-over-turn. + +Cache model (faithful simplification of Anthropic prompt caching): within a +conversation, order requests by turn (growing message count). For turn t, the longest +token PREFIX byte-identical to turn t-1's sent content is a cache-READ (0.1x input); +everything after the first divergence is re-processed and cache-WRITTEN (1.25x). This +captures BOTH effects that matter: (1) mutating an OLD tool output shifts the divergence +point earlier -> more cache-write (the penalty), and (2) a smaller compacted prefix means +every later turn re-reads fewer tokens (the benefit). Output tokens are equal for both +sides on identical traffic, so they're excluded. + +Usage: cachecost.py --config codesmart [--port 4021] [--limit N] +""" +import argparse, json, subprocess, time, urllib.request, os, hashlib +from pathlib import Path + +CG = Path("/home/vpcuser/projects/context-engineering/context-guru") +BIN = "/tmp/cg-runs/cg-proxy-rp" +IN_RATE, CREAD, CWRITE = 2e-6, 0.2e-6, 2.5e-6 # $/token: input, cache-read, cache-write + +# reuse the swebench harness's custom configs +import importlib.util +spec = importlib.util.spec_from_file_location("swb", str(CG / "deploy/harbor/swebench.py")) +swb = importlib.util.module_from_spec(spec); spec.loader.exec_module(swb) + + +def creds(): + e = json.load(open(os.path.expanduser("~/.claude/settings.json")))["env"] + return e["ANTHROPIC_BASE_URL"], e["ANTHROPIC_AUTH_TOKEN"] + + +def content_text(body): + """The full request text the provider sees (system + messages), as one string — + the thing the cache keys on.""" + parts = [] + sysv = body.get("system") + if isinstance(sysv, str): parts.append(sysv) + elif isinstance(sysv, list): + parts += [b.get("text", "") for b in sysv if isinstance(b, dict)] + for m in body.get("messages", []): + c = m.get("content") + if isinstance(c, str): parts.append(c) + elif isinstance(c, list): + for b in c: + if isinstance(b, dict): + parts.append(b.get("text") or (b.get("content") if isinstance(b.get("content"), str) else json.dumps(b.get("content", ""))) or json.dumps(b.get("input", ""))) + return "\n".join(p for p in parts if p) + + +def toks(s): return max(1, len(s) // 4) # ~4 chars/token + + +def common_prefix_tokens(a, b): + n = min(len(a), len(b)); i = 0 + while i < n and a[i] == b[i]: + i += 1 + return i // 4 + + +def conv_key(body): + # group by the FIRST user message (the task instruction — stable & unique per task) + for m in body.get("messages", []): + if m.get("role") == "user": + c = m.get("content"); t = c if isinstance(c, str) else json.dumps(c) + return hashlib.sha1(t.encode()).hexdigest()[:12] + return "none" + + +def stream_cost(bodies_by_conv): + """bodies_by_conv: {conv: [content_text ordered by turn]}. Returns total $ and token tiers.""" + cread = cwrite = 0 + for conv, texts in bodies_by_conv.items(): + prev = "" + for t in texts: + cp = common_prefix_tokens(prev, t) if prev else 0 + total = toks(t) + cread += cp + cwrite += max(0, total - cp) # new/changed -> processed + cached + prev = t + cost = cread * CREAD + cwrite * CWRITE + return cost, cread, cwrite + + +def start_proxy(config, base, token, port): + subprocess.run(f"pkill -x {os.path.basename(BIN)}", shell=True); time.sleep(1) + env = dict(os.environ, ANTHROPIC_UPSTREAM=base, ANTHROPIC_API_KEY=token, + OPENAI_UPSTREAM=base, OPENAI_API_KEY=token, LISTEN_ADDR=f":{port}", + CHEAP_MODEL="aws/claude-sonnet-5", CHEAP_MODEL_PROVIDER="anthropic", + CHEAP_MODEL_BASE=base, CHEAP_MODEL_KEY=token, CHEAP_MODEL_AUTH="bearer") + cfgp = f"/tmp/cg-runs/cachecost-{config}.yaml" + Path(cfgp).write_text(swb.CUSTOM_CONFIGS[config]) + p = subprocess.Popen([BIN, "--config", cfgp], cwd=str(CG), stdout=open(f"/tmp/cg-runs/cachecost-proxy.log", "w"), + stderr=subprocess.STDOUT, env=env, preexec_fn=os.setsid) + for _ in range(30): + try: + urllib.request.urlopen(f"http://localhost:{port}/healthz", timeout=3).read(); return p + except Exception: + time.sleep(0.5) + raise RuntimeError("proxy did not come up") + + +def rewrite(body, prov, port): + data = json.dumps(body).encode() + req = urllib.request.Request(f"http://localhost:{port}/compact?provider={prov}", data=data, headers={"content-type": "application/json"}) + return json.loads(urllib.request.urlopen(req, timeout=120).read()) + + +def group(recs, key_fn): + from collections import defaultdict + g = defaultdict(list) + for r in recs: g[conv_key(r["body"])].append(r) + for k in g: + g[k].sort(key=lambda r: len(r["body"].get("messages", []))) # turn order + return g + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("capture"); ap.add_argument("--config", default="codesmart") + ap.add_argument("--port", type=int, default=4021); ap.add_argument("--limit", type=int, default=0) + a = ap.parse_args() + base, token = creds() + recs = [json.loads(l) for l in open(a.capture) if l.strip()] + if a.limit: recs = recs[:a.limit] + convs = group(recs, conv_key) + print(f"{len(recs)} requests, {len(convs)} conversations") + # baseline: cost of the stream as sent + base_by_conv = {c: [content_text(r["body"]) for r in rs] for c, rs in convs.items()} + b_cost, b_cr, b_cw = stream_cost(base_by_conv) + # config: rewrite each request via /compact, then model cache cost + start_proxy(a.config, base, token, a.port) + cfg_by_conv = {} + n = 0 + for c, rs in convs.items(): + texts = [] + for r in rs: + prov = r.get("provider", "anthropic") + try: + out = rewrite(r["body"], prov, a.port) + texts.append(content_text(out)) + except Exception as e: + texts.append(content_text(r["body"])) # fail-open + n += 1 + if n % 100 == 0: print(f" rewritten {n}/{len(recs)}", flush=True) + cfg_by_conv[c] = texts + subprocess.run(f"pkill -x {os.path.basename(BIN)}", shell=True) + # CG's own LLM cost from /stats (captured before kill) + c_cost, c_cr, c_cw = stream_cost(cfg_by_conv) + print(f"\n=== cache-aware cost (deterministic, identical traffic) ===") + print(f"baseline: ${b_cost:.3f} (cache_read {b_cr:,} tok, cache_write {b_cw:,} tok)") + print(f"{a.config}: ${c_cost:.3f} (cache_read {c_cr:,} tok, cache_write {c_cw:,} tok)") + delta = 100 * (c_cost - b_cost) / b_cost if b_cost else 0 + print(f"delta: {delta:+.1f}% ({'SAVES' if delta<0 else 'COSTS MORE'})") + print(f"content: baseline {b_cr+b_cw:,} tok -> {a.config} {c_cr+c_cw:,} tok ({100*(1-(c_cr+c_cw)/(b_cr+b_cw)):.1f}% fewer content tokens)") + Path("/tmp/cg-runs/cachecost.json").write_text(json.dumps(dict( + baseline=dict(cost=b_cost, cache_read=b_cr, cache_write=b_cw), + config=dict(name=a.config, cost=c_cost, cache_read=c_cr, cache_write=c_cw), + delta_pct=delta), indent=1)) + print("wrote /tmp/cg-runs/cachecost.json") + + +if __name__ == "__main__": + main() diff --git a/deploy/harbor/deep_analysis.py b/deploy/harbor/deep_analysis.py new file mode 100644 index 0000000..8fb7436 --- /dev/null +++ b/deploy/harbor/deep_analysis.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python3 +"""Deep three-way analysis: baseline vs context-guru vs headroom on SWE-bench Verified. +Computes EVERY dimension per-task, aggregated, and per-component, and emits one JSON +(deep_analysis.json) that the plotting + doc scripts consume. + +Dimensions: reward, steps, cache_read, cache_write, fresh_input, output, cache-aware +billed cost (+ decomposition), cache-hit rate, agent wall, tool's own LLM cost, added +latency, tokens removed (cumulative + unique), per-component / per-strategy savings. + +Usage: deep_analysis.py --out /tmp/cg-runs/deep +""" +import argparse, json, subprocess +from pathlib import Path +from statistics import mean + +IN, OUT, CREAD, CWRITE = 2e-6, 10e-6, 0.2e-6, 2.5e-6 +CG = Path("/home/vpcuser/projects/context-engineering/context-guru") + +SRC = { + "baseline": ("/tmp/cg-runs/final50/rows-off.json", None), + "context-guru": ("/tmp/cg-runs/final50-v6/rows-codesmart.json", "/tmp/cg-runs/final50-v6/summary.json"), + "headroom": ("/tmp/hd-runs/swe50/rows-hd-cache.json", None), +} + + +def billed(r): + return (r.get("fresh_input", 0) * IN + r.get("cache_read", 0) * CREAD + + r.get("cache_write", 0) * CWRITE + r.get("completion_tokens", 0) * OUT) + + +def load(p): + return {r["task"]: r for r in json.load(open(p))} + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--out", default="/tmp/cg-runs/deep") + a = ap.parse_args() + Path(a.out).mkdir(parents=True, exist_ok=True) + + rows = {k: load(v[0]) for k, v in SRC.items()} + # tasks scored (no exception, reward not None) in ALL three + common = sorted(t for t in rows["baseline"] + if all(t in rows[c] and not rows[c][t].get("exception") + and rows[c][t].get("reward") is not None for c in rows)) + configs = list(SRC) + + # per-task records + per_task = [] + for t in common: + rec = {"task": t.split("__")[-1]} + for c in configs: + r = rows[c][t] + rec[c] = dict(reward=int(r.get("reward") or 0), steps=r.get("steps", 0), + cache_read=r.get("cache_read", 0), cache_write=r.get("cache_write", 0), + fresh=r.get("fresh_input", 0), out=r.get("completion_tokens", 0), + cost=round(billed(r), 4), wall=r.get("agent_wall_s") or 0) + per_task.append(rec) + + def agg(c, key): + return sum(rows[c][t][{"cache_read": "cache_read", "cache_write": "cache_write", + "fresh": "fresh_input", "out": "completion_tokens", + "steps": "steps"}[key]] or 0 for t in common) + + # per-config aggregate over matched tasks + aggregate = {} + for c in configs: + solved = sum(1 for t in common if (rows[c][t].get("reward") or 0) >= 1) + cr, cw, fr = agg(c, "cache_read"), agg(c, "cache_write"), agg(c, "fresh") + cost = sum(billed(rows[c][t]) for t in common) + cachecost = cr * CREAD + cw * CWRITE + fr * IN + aggregate[c] = dict( + n=len(common), solved=solved, rate=round(solved / len(common), 3), + mean_steps=round(mean(rows[c][t]["steps"] for t in common), 1), + cache_read=cr, cache_write=cw, fresh=fr, out=agg(c, "out"), + cache_hit=round(100 * cr / max(cr + cw + fr, 1), 2), + billed_cost=round(cost, 2), input_cache_cost=round(cachecost, 2), + cache_read_cost=round(cr * CREAD, 2), cache_write_cost=round(cw * CWRITE, 2), + mean_wall_s=round(mean((rows[c][t].get("agent_wall_s") or 0) for t in common), 1), + ) + + # tool's own LLM cost + added latency + content% + per-component (context-guru) + tool = {c: dict(llm_cost=0.0, added_ms=0.0, content_pct=0.0, bounces=0, per_component={}) for c in configs} + cs = json.load(open(SRC["context-guru"][1]))["configs"] + cg = next((x for x in cs if x.get("per_component")), cs[0]) + tool["context-guru"] = dict( + llm_cost=cg.get("cg_llm_cost", 0), added_ms=round(cg.get("cg_added_ms_avg") or 0, 1), + content_pct=cg.get("proxy_savings_pct", 0), bounces=cg.get("proxy_bounces", 0), + per_component={k: {"cum": v.get("saved_tokens"), "runs": v.get("runs"), + "ms": round(v.get("duration_ms") or 0, 1)} + for k, v in (cg.get("per_component") or {}).items()}) + # unique per-component (context-guru) from the dump + try: + subprocess.run(["python3", str(CG / "deploy/harbor/dump_unique.py"), + "/tmp/cg-runs/dump-swebench-codesmart.jsonl", "--json", f"{a.out}/cg_unique.json"], check=False) + uq = json.load(open(f"{a.out}/cg_unique.json")) + tool["context-guru"]["unique"] = uq + except Exception: + pass + # headroom per-strategy + tool metrics from its RESULTS.json + try: + hr = json.load(open("/tmp/hd-runs/RESULTS.json"))["headroom_hd_cache"] + tool["headroom"] = dict(llm_cost=hr.get("added_llm_cost", 0), + added_ms=hr.get("proxy_overhead_avg_ms", 0), + content_pct=hr.get("content_savings_pct", 0), bounces=0, + per_strategy=hr.get("per_strategy_saved", {}), + saved_tokens=hr.get("saved_tokens")) + except Exception: + pass + + out = dict(matched_tasks=len(common), configs=configs, aggregate=aggregate, + tool=tool, per_task=per_task) + Path(f"{a.out}/deep_analysis.json").write_text(json.dumps(out, indent=1)) + # console summary + print(f"matched {len(common)} tasks (scored+no-exception in all 3)\n") + hdr = f"{'metric':<22}" + "".join(f"{c:>16}" for c in configs) + print(hdr); print("-" * len(hdr)) + for m, fn in [("solved", lambda x: f"{x['solved']}/{x['n']}"), ("rate", lambda x: f"{x['rate']:.0%}"), + ("mean_steps", lambda x: x["mean_steps"]), ("billed_cost $", lambda x: f"${x['billed_cost']}"), + ("cache_read (M)", lambda x: f"{x['cache_read']/1e6:.1f}"), + ("cache_write (M)", lambda x: f"{x['cache_write']/1e6:.2f}"), + ("cache_read_cost $", lambda x: f"${x['cache_read_cost']}"), + ("cache_write_cost $", lambda x: f"${x['cache_write_cost']}"), + ("cache_hit %", lambda x: x["cache_hit"]), ("mean_wall_s", lambda x: x["mean_wall_s"])]: + print(f"{m:<22}" + "".join(f"{str(fn(aggregate[c])):>16}" for c in configs)) + print(f"\n{'tool llm $':<22}" + "".join(f"{str(tool[c]['llm_cost']):>16}" for c in configs)) + print(f"{'added ms/req':<22}" + "".join(f"{str(tool[c]['added_ms']):>16}" for c in configs)) + print(f"{'content %':<22}" + "".join(f"{str(tool[c]['content_pct']):>16}" for c in configs)) + print(f"\nwrote {a.out}/deep_analysis.json") + + +if __name__ == "__main__": + main() diff --git a/deploy/harbor/deep_plots.py b/deploy/harbor/deep_plots.py new file mode 100644 index 0000000..4af6d62 --- /dev/null +++ b/deploy/harbor/deep_plots.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python3 +"""Comprehensive 3-way figures (baseline / context-guru / headroom) from deep_analysis.json. +Validated CVD-safe palette, one axis per chart, thin marks, recessive grid, legend for +multi-series. Run with /tmp/cg-runs/plotenv/bin/python. + +Usage: deep_plots.py deep_analysis.json --out DIR +""" +import argparse, json +from pathlib import Path +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt + +SURFACE = "#fcfcfb"; INK = "#0b0b0b"; INK2 = "#52514e"; GRID = "#e6e5e2" +# baseline grey, context-guru blue, headroom orange +COL = {"baseline": "#8a8985", "context-guru": "#2a78d6", "headroom": "#eb6834"} +COMP = ["#2a78d6", "#eb6834", "#1baf7a", "#eda100", "#e87ba4", "#008300"] +plt.rcParams.update({ + "figure.facecolor": SURFACE, "axes.facecolor": SURFACE, "savefig.facecolor": SURFACE, + "text.color": INK, "axes.labelcolor": INK2, "xtick.color": INK2, "ytick.color": INK2, + "axes.edgecolor": GRID, "font.size": 10, "axes.titlesize": 12, "axes.titleweight": "bold", + "axes.grid": True, "grid.color": GRID, "grid.linewidth": 0.8, "axes.axisbelow": True, "figure.dpi": 130, +}) + + +def style(ax): + for s in ("top", "right"): + ax.spines[s].set_visible(False) + ax.tick_params(length=0) + + +def bars(ax, agg, fn, title, ylabel, fmt="{:.0f}", cfgs=None): + cfgs = cfgs or list(agg) + vals = [fn(agg[c]) for c in cfgs] + xs = range(len(cfgs)) + ax.bar(xs, vals, color=[COL[c] for c in cfgs], width=0.6, zorder=2) + for x, v in zip(xs, vals): + ax.text(x, v, " " + fmt.format(v), ha="center", va="bottom", fontsize=9, color=INK) + ax.set_xticks(list(xs)); ax.set_xticklabels(cfgs, fontsize=9) + ax.set_title(title); ax.set_ylabel(ylabel) + ax.grid(axis="x", visible=False); style(ax) + ax.set_ylim(0, max(vals) * 1.18) + + +def fig_headline(A, out): + agg, cfgs = A["aggregate"], A["configs"] + fig, ax = plt.subplots(2, 3, figsize=(12, 6.4)) + bars(ax[0][0], agg, lambda x: x["solved"], f"Reward (solved / {A['matched_tasks']})", "tasks", "{:.0f}") + bars(ax[0][1], agg, lambda x: x["billed_cost"], "Billed input cost (matched total)", "$", "${:.1f}") + bars(ax[0][2], agg, lambda x: x["mean_steps"], "Mean agent steps / task", "steps", "{:.1f}") + bars(ax[1][0], agg, lambda x: x["cache_read"] / 1e6, "Cache-read tokens", "M tokens", "{:.1f}") + bars(ax[1][1], agg, lambda x: x["cache_write"] / 1e6, "Cache-write tokens", "M tokens", "{:.2f}") + tool = A["tool"] + lat = {c: tool[c]["added_ms"] for c in cfgs} + ax[1][2].bar(range(len(cfgs)), [lat[c] for c in cfgs], color=[COL[c] for c in cfgs], width=0.6, zorder=2) + for x, c in enumerate(cfgs): + ax[1][2].text(x, lat[c], f" {lat[c]:.0f}", ha="center", va="bottom", fontsize=9, color=INK) + ax[1][2].set_xticks(range(len(cfgs))); ax[1][2].set_xticklabels(cfgs, fontsize=9) + ax[1][2].set_title("Compaction latency added / request"); ax[1][2].set_ylabel("ms") + ax[1][2].grid(axis="x", visible=False); style(ax[1][2]) + fig.suptitle("baseline vs context-guru vs headroom — SWE-bench Verified (matched %d tasks)" % A["matched_tasks"], + fontsize=13, fontweight="bold") + fig.tight_layout(); fig.savefig(out); plt.close(fig) + + +def fig_cost_decomp(A, out): + agg, cfgs = A["aggregate"], A["configs"] + fig, ax = plt.subplots(figsize=(8, 4)) + import numpy as np + x = np.arange(len(cfgs)); w = 0.6 + cr = [agg[c]["cache_read_cost"] for c in cfgs] + cw = [agg[c]["cache_write_cost"] for c in cfgs] + llm = [A["tool"][c]["llm_cost"] for c in cfgs] + ax.bar(x, cr, w, label="cache-read", color="#2a78d6", zorder=2) + ax.bar(x, cw, w, bottom=cr, label="cache-write", color="#eb6834", zorder=2) + ax.bar(x, llm, w, bottom=[a + b for a, b in zip(cr, cw)], label="tool LLM", color="#eda100", zorder=2) + for xi, c in enumerate(cfgs): + tot = cr[xi] + cw[xi] + llm[xi] + ax.text(xi, tot, f" ${tot:.1f}", ha="center", va="bottom", fontsize=9, color=INK) + ax.set_xticks(x); ax.set_xticklabels(cfgs, fontsize=9) + ax.set_ylabel("$ (matched total)"); ax.set_title("Cost decomposition: cache-read + cache-write + tool LLM") + ax.legend(frameon=False, fontsize=8); ax.grid(axis="x", visible=False); style(ax) + fig.tight_layout(); fig.savefig(out); plt.close(fig) + + +def fig_per_task_cost(A, out): + rows = sorted(A["per_task"], key=lambda r: r["baseline"]["cost"]) + n = len(rows) + fig, ax = plt.subplots(figsize=(8, max(6, n * 0.22))) + for i, r in enumerate(rows): + ax.plot([r["headroom"]["cost"], r["context-guru"]["cost"]], [i, i], color=GRID, lw=1.5, zorder=1) + for c, mk in [("baseline", "|"), ("headroom", "o"), ("context-guru", "o")]: + ax.scatter([r[c]["cost"] for r in rows], range(n), s=30, color=COL[c], zorder=3, label=c, + marker=mk if c != "baseline" else "D") + ax.set_yticks(range(n)); ax.set_yticklabels([r["task"] for r in rows], fontsize=6) + ax.set_xlabel("billed input cost per task ($)"); ax.set_title("Per-task cost") + ax.legend(loc="lower right", frameon=False, fontsize=8); ax.grid(axis="y", visible=False); style(ax) + fig.tight_layout(); fig.savefig(out); plt.close(fig) + + +def fig_per_task_delta(A, key, title, xlabel, out): + rows = sorted(A["per_task"], key=lambda r: r["context-guru"][key] - r["baseline"][key]) + n = len(rows) + fig, ax = plt.subplots(figsize=(8, max(6, n * 0.22))) + for i, r in enumerate(rows): + d = r["context-guru"][key] - r["baseline"][key] + ax.barh(i, d, color=(COL["context-guru"] if d < 0 else COL["headroom"]), height=0.6, zorder=2) + ax.axvline(0, color=INK2, lw=1) + ax.set_yticks(range(n)); ax.set_yticklabels([r["task"] for r in rows], fontsize=6) + ax.set_xlabel(xlabel); ax.set_title(title + " (context-guru − baseline)") + ax.text(0.98, 0.02, "blue = context-guru lower (better)", transform=ax.transAxes, ha="right", + va="bottom", fontsize=8, color=INK2) + ax.grid(axis="y", visible=False); style(ax) + fig.tight_layout(); fig.savefig(out); plt.close(fig) + + +def fig_components(A, out): + cg = A["tool"]["context-guru"] + uq = (cg.get("unique") or {}).get("components", {}) + hr = A["tool"]["headroom"].get("per_strategy", {}) + fig, axes = plt.subplots(1, 2, figsize=(11, 3.6)) + # context-guru: cumulative vs unique + items = [(k, v.get("cum", 0), (uq.get(k, {}) or {}).get("uniq_saved", 0)) for k, v in cg["per_component"].items() if v.get("cum")] + # map dump categories to component names best-effort + items.sort(key=lambda x: x[1]) + ax = axes[0]; y = range(len(items)) + ax.barh([i + 0.2 for i in y], [c for _, c, _ in items], height=0.36, color="#8a8985", label="cumulative", zorder=2) + ax.barh([i - 0.2 for i in y], [u for _, _, u in items], height=0.36, color="#2a78d6", label="unique", zorder=2) + ax.set_yticks(list(y)); ax.set_yticklabels([k for k, _, _ in items], fontsize=8) + ax.set_xlabel("tokens removed"); ax.set_title("context-guru — per component"); ax.set_xscale("symlog") + ax.legend(frameon=False, fontsize=8, loc="lower right"); ax.grid(axis="y", visible=False); style(ax) + # headroom per-strategy + hs = sorted([(k, v) for k, v in hr.items() if v], key=lambda x: x[1]) + ax = axes[1] + ax.barh(range(len(hs)), [v for _, v in hs], color="#eb6834", height=0.6, zorder=2) + for i, (k, v) in enumerate(hs): + ax.text(v, i, f" {v:,}", va="center", fontsize=8, color=INK) + ax.set_yticks(range(len(hs))); ax.set_yticklabels([k for k, _ in hs], fontsize=8) + ax.set_xlabel("tokens removed"); ax.set_title("headroom — per compressor") + ax.grid(axis="y", visible=False); style(ax) + fig.tight_layout(); fig.savefig(out); plt.close(fig) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("analysis"); ap.add_argument("--out", default="/tmp/cg-runs/deep/img") + a = ap.parse_args() + A = json.load(open(a.analysis)); Path(a.out).mkdir(parents=True, exist_ok=True) + fig_headline(A, f"{a.out}/headline.png") + fig_cost_decomp(A, f"{a.out}/cost_decomposition.png") + fig_per_task_cost(A, f"{a.out}/per_task_cost.png") + fig_per_task_delta(A, "cost", "Per-task cost delta", "Δ billed cost ($)", f"{a.out}/per_task_dcost.png") + fig_per_task_delta(A, "steps", "Per-task step delta", "Δ steps", f"{a.out}/per_task_dsteps.png") + fig_components(A, f"{a.out}/components.png") + print(f"wrote 6 figures to {a.out}") + + +if __name__ == "__main__": + main() diff --git a/deploy/harbor/diffsum.py b/deploy/harbor/diffsum.py new file mode 100644 index 0000000..dd50498 --- /dev/null +++ b/deploy/harbor/diffsum.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +"""Summarize what a component actually did to the messages: given before/after +request bodies (as written by replay2 under /tmp/cg-runs/diffs///), find +the messages whose text changed and print a compact before->after snippet for each. +Lets a reviewer judge correctness / over-aggression without wading through the full +100KB+ bodies. Also flags structural changes (msg/tool_result/tool_use counts). + +Usage: diffsum.py [after.json] [--head 220] [--tail 160] +""" +import argparse, json, sys +from pathlib import Path + + +def msg_texts(body): + """Return list of (role, kind, text) for each content-bearing block, so we can + align before/after and show only what changed.""" + out = [] + for mi, m in enumerate(body.get("messages", [])): + role = m.get("role") + c = m.get("content") + if isinstance(c, str): + out.append((mi, role, "text", c)) + elif isinstance(c, list): + for bi, b in enumerate(c): + if not isinstance(b, dict): + continue + t = b.get("type") + if t == "text": + out.append((mi, role, "text", b.get("text", ""))) + elif t == "tool_result": + cc = b.get("content") + if isinstance(cc, str): + out.append((mi, role, "tool_result", cc)) + elif isinstance(cc, list): + for x in cc: + if isinstance(x, dict) and x.get("type") == "text": + out.append((mi, role, "tool_result", x.get("text", ""))) + return out + + +def struct(body): + msgs = body.get("messages", []) + tr = tu = 0 + for m in msgs: + c = m.get("content") + if isinstance(c, list): + tr += sum(1 for b in c if isinstance(b, dict) and b.get("type") == "tool_result") + tu += sum(1 for b in c if isinstance(b, dict) and b.get("type") == "tool_use") + if isinstance(m.get("tool_calls"), list): + tu += len(m["tool_calls"]) + return len(msgs), tr, tu + + +def clip(s, head, tail): + s = s.replace("\n", "\\n") + if len(s) <= head + tail + 20: + return s + return s[:head] + f" …[{len(s)-head-tail} chars]… " + s[-tail:] + + +def summarize(before_p, after_p, head, tail): + b = json.loads(Path(before_p).read_bytes()) + a = json.loads(Path(after_p).read_bytes()) + sb, sa = struct(b), struct(a) + print(f"# {Path(before_p).parent.name}/{Path(before_p).stem}") + print(f"struct (msgs,tool_results,tool_uses): {sb} -> {sa} bytes {len(json.dumps(b))}->{len(json.dumps(a))}") + if sb != sa: + print(f" ** STRUCTURE CHANGED ** {sb}->{sa}") + tb, ta = msg_texts(b), msg_texts(a) + # align by (msg_index, role, kind) position; report changed texts + changed = 0 + n = min(len(tb), len(ta)) + for i in range(n): + (mi, role, kind, xb) = tb[i] + (_, _, _, xa) = ta[i] + if xb != xa: + changed += 1 + print(f"\n [{role}/{kind} #{mi}] {len(xb)}->{len(xa)} chars") + print(f" BEFORE: {clip(xb, head, tail)}") + print(f" AFTER : {clip(xa, head, tail)}") + if len(tb) != len(ta): + print(f"\n (block count changed {len(tb)}->{len(ta)})") + if changed == 0 and sb == sa: + print(" (no text-block changes detected at aligned positions)") + print() + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("target", help="a before.json file, or a dir of reqN.before/after pairs") + ap.add_argument("after", nargs="?", default=None) + ap.add_argument("--head", type=int, default=220) + ap.add_argument("--tail", type=int, default=160) + a = ap.parse_args() + p = Path(a.target) + if p.is_dir(): + for bf in sorted(p.glob("*.before.json")): + summarize(bf, str(bf).replace(".before.", ".after."), a.head, a.tail) + else: + summarize(a.target, a.after or str(a.target).replace(".before.", ".after."), a.head, a.tail) + + +if __name__ == "__main__": + main() diff --git a/deploy/harbor/dump_unique.py b/deploy/harbor/dump_unique.py new file mode 100644 index 0000000..93bcf9b --- /dev/null +++ b/deploy/harbor/dump_unique.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +"""Per-component CUMULATIVE-vs-UNIQUE savings from a CONTEXT_GURU_DUMP change log. + +The agent re-sends its history verbatim every turn, and the proxy re-compacts it, so +/stats sums the same compaction many times (cumulative). This dedups each distinct +compaction by the hash of its BEFORE content to report the honest UNIQUE tokens saved +and the over-count ratio. Category is inferred from the marker text (the dump has no +component field). + +Usage: dump_unique.py [--json out.json] +""" +import argparse, json, hashlib +from collections import defaultdict + + +def category(after): + if "superseded by a later failed" in after: + return "failed_run" + if "identical to an earlier" in after: + return "dedup" + if "older tool output masked" in after: + return "mask" + if "evicted to fit context" in after: + return "phi_evict" + if "lines omitted)" in after: + return "collapse" + if "<8} {'distinct':>9} {'cum_saved':>11} {'uniq_saved':>11} {'overcount':>10}") + for c in sorted(cum, key=lambda k: -uniq[k]): + ratio = cum[c] / uniq[c] if uniq[c] else 0 + print(f"{c:22s} {cnt[c]:8d} {len(seen[c]):9d} {cum[c]:11,} {uniq[c]:11,} {ratio:9.1f}x") + out["components"][c] = {"changes": cnt[c], "distinct": len(seen[c]), + "cum_saved": cum[c], "uniq_saved": uniq[c], "overcount_ratio": round(ratio, 2)} + tot_cum = sum(cum.values()); tot_uniq = sum(uniq.values()) + print(f"{'TOTAL':22s} {'':>8} {'':>9} {tot_cum:11,} {tot_uniq:11,}") + out["total_cum_saved"] = tot_cum + out["total_uniq_saved"] = tot_uniq + if a.json: + json.dump(out, open(a.json, "w"), indent=1) + print(f"wrote {a.json}") + + +if __name__ == "__main__": + main() diff --git a/deploy/harbor/gen_result_docs.py b/deploy/harbor/gen_result_docs.py new file mode 100644 index 0000000..178bae8 --- /dev/null +++ b/deploy/harbor/gen_result_docs.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +"""Generate a per-config results doc (per-task table + totals) from a rows.json. +Usage: gen_result_docs.py