diff --git a/.github/workflows/build-agent.yaml b/.github/workflows/build-agent.yaml deleted file mode 100644 index b1cea6f..0000000 --- a/.github/workflows/build-agent.yaml +++ /dev/null @@ -1,51 +0,0 @@ -name: Build and Push Agent Image - -on: - push: - tags: - - 'release-*' - -jobs: - build: - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - - steps: - - name: Checkout code - uses: actions/checkout@v3 - - - name: Set up Python environment - uses: actions/setup-python@v4 - with: - python-version: '3.13' - - - name: Install Kagenti CLI - run: pip install kagenti-cli - - - name: Login to GitHub Container Registry - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Set up Docker - uses: docker/setup-docker-action@v4 - with: - daemon-config: '{"features": {"containerd-snapshotter": true}}' - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Extract version from tag - id: extract_version - run: | - # Extract version from tag (e.g., release-1.2.3 -> 1.2.3) - VERSION=${GITHUB_REF#refs/tags/release-} - echo "version=$VERSION" >> $GITHUB_OUTPUT - - - name: Build and Push - run: | - kagenti-adk client-side-build ./ --tag ghcr.io/${{ github.repository }}/my-agent:${{ steps.extract_version.outputs.version }} --no-import --multi-platform --push diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000..a8e3784 --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,39 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + +permissions: + contents: read + +jobs: + build-test: + runs-on: ubuntu-latest + env: + CGO_ENABLED: "1" + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: '1.25' + check-latest: true + - name: Lint + run: make lint + - name: Test + run: make test + - name: Build + run: make build + + trivy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Trivy filesystem scan + uses: aquasecurity/trivy-action@0.28.0 + with: + scan-type: fs + ignore-unfixed: true + severity: CRITICAL,HIGH + exit-code: '1' diff --git a/.github/workflows/weekly-docker-test.yaml b/.github/workflows/weekly-docker-test.yaml deleted file mode 100644 index 51744aa..0000000 --- a/.github/workflows/weekly-docker-test.yaml +++ /dev/null @@ -1,44 +0,0 @@ -name: Weekly Docker Build and Test - -on: - schedule: - # Run every Monday at 9:00 AM UTC - - cron: '0 9 * * 1' - workflow_dispatch: - -jobs: - docker-test: - runs-on: ubuntu-latest - - steps: - - name: Checkout code - uses: actions/checkout@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Build Docker image - run: docker build -t test-image . - - - name: Run Docker container and check startup - run: | - set +e - - timeout 10s docker run --rm test-image > container_output.log 2>&1 - EXIT_CODE=$? - - if grep -q "Application startup complete." container_output.log; then - echo "✓ Success: Application started successfully" - cat container_output.log - exit 0 - elif [ $EXIT_CODE -eq 124 ]; then - echo "✗ Error: Timeout reached (10s) without seeing 'Application startup complete.'" - echo "Container output:" - cat container_output.log - exit 1 - else - echo "✗ Error: Container exited with code $EXIT_CODE without completing startup" - echo "Container output:" - cat container_output.log - exit 1 - fi diff --git a/.gitignore b/.gitignore index a38dbc0..a715f63 100644 --- a/.gitignore +++ b/.gitignore @@ -162,3 +162,9 @@ cython_debug/ # option (not recommended) you can uncomment the following to ignore the entire idea folder. .idea/ + +# Go / lab-context-engineering +/bin/ +*.test +*.out +coverage.txt diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..16465e0 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,21 @@ +# Run `make pre-commit` once to install these hooks (including commit-msg / DCO). +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: check-added-large-files + + - repo: https://github.com/dnephin/pre-commit-golang + rev: v0.5.1 + hooks: + - id: go-fmt + - id: go-vet + - id: go-mod-tidy + + - repo: https://github.com/jorisroovers/gitlint + rev: v0.19.1 + hooks: + - id: gitlint diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..5549ad0 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,35 @@ +# CLAUDE.md + +Guidance for working in `lab-context-engineering`, a Kagenti platform component. + +## What this repo is + +A single **Go** core that reduces the token cost of LLM agent traffic. It ships as a +standalone proxy binary (`cmd/proxy`), an importable library (`engine`, `surfaces`), and +eval-containers wiring. Its lineage is the Python `winnow` prototype (`../winnow`), which +is the behavioral reference — port its *logic*, re-implement its transport in Go. + +## Hard boundaries + +- **No AuthBridge / kagenti-extensions code lives here.** That plugin is built in + `kagenti-extensions` and depends on this repo. Keep the public API (`engine`, + `surfaces`, `config`) clean and importable; never reach into another repo. +- **Fail open, always.** Any error in any stage forwards the original request untouched. + Reductions must be reversible (markers + rewind store). Never drop content that is only + *predicted* unused — `provable_only` is on by default. + +## Conventions + +- Go 1.25, module `github.com/kagenti/lab-context-engineering`. `make fmt lint test build`. +- Match the surrounding code's style; keep packages small and single-purpose. +- **Commits: DCO sign-off is mandatory** — `git commit -s`. Author as the repo owner. + AI attribution uses `Assisted-By:` — never `Co-Authored-By`, never a "Generated with" + line. Conventional-commit titles. +- Observability follows OpenTelemetry **GenAI semantic conventions** (`gen_ai.*`). + +## Layout + +`cmd/proxy` (binary) · `engine` (Transform/Expand + stages) · `surfaces` (wire⇄canonical) +· `internal/*` (types, extract, relevance, signals, taxonomy, actions, cache, markers, +rewind, zones, session, compaction, tokens) · `config` · `observability` · `deploy` · +`docs`. diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..e8792a0 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,8 @@ +# Code of Conduct + +This project adheres to the Kagenti community Code of Conduct, which is based on the +[Contributor Covenant](https://www.contributor-covenant.org/version/2/1/code_of_conduct/). + +By participating, you are expected to uphold this code. Please report unacceptable +behavior to the maintainers listed in [MAINTAINERS.md](MAINTAINERS.md) or via the process +described in the main [Kagenti](https://github.com/kagenti/kagenti) repository. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..210bc5f --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,41 @@ +# Contributing + +Thanks for your interest in `lab-context-engineering`, a component of the +[Kagenti](https://github.com/kagenti/kagenti) platform. + +## Developer Certificate of Origin (DCO) + +All commits must be signed off under the [DCO](https://developercertificate.org/): + +```sh +git commit -s -m "feat: ..." +``` + +This adds a `Signed-off-by:` trailer certifying you wrote or have the right to submit the +patch. PRs with unsigned commits will not be merged. + +When a commit was assisted by an AI agent, attribute it with an `Assisted-By:` trailer — +do **not** use `Co-Authored-By` and do not add "Generated with" lines. + +## Development + +```sh +make fmt # format + go mod tidy +make lint # go vet + gofmt check +make test # go test -race ./... +make build # build ./bin/lab-cx +``` + +- Go 1.25+. +- Keep packages small and single-purpose; the engine core must stay transport-agnostic. +- Every reduction must be reversible and fail-open. Add tests for both the happy path and + the fault-injection (fail-open) path. + +## Pull requests + +Use conventional-commit titles (`feat:`, `fix:`, `docs:`, `refactor:`, `test:`, `chore:`). +Keep PRs focused. CI (lint, test, build, Trivy) must be green. + +## Reporting security issues + +See [SECURITY.md](SECURITY.md). Do not open public issues for vulnerabilities. diff --git a/Dockerfile b/Dockerfile index ceffa77..adedd3d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,16 @@ -FROM python:3.13-slim -COPY --from=ghcr.io/astral-sh/uv:0.9.5 /uv /bin/uv -WORKDIR /app +# Build the lab-cx proxy. CGO is required (tree-sitter), so the final image is +# glibc-based (distroless/base), not static. +FROM golang:1.25 AS build +WORKDIR /src COPY . . -RUN UV_COMPILE_BYTECODE=1 HOME=/tmp uv sync --no-cache --link-mode copy -ENV PRODUCTION_MODE=True -CMD ["/app/.venv/bin/server"] +ARG VERSION=dev +ARG COMMIT=none +RUN CGO_ENABLED=1 go build \ + -ldflags "-s -w -X github.com/kagenti/lab-context-engineering/internal/buildinfo.Version=${VERSION} -X github.com/kagenti/lab-context-engineering/internal/buildinfo.Commit=${COMMIT}" \ + -o /out/lab-cx ./cmd/proxy + +FROM gcr.io/distroless/base-debian12:nonroot +COPY --from=build /out/lab-cx /usr/local/bin/lab-cx +EXPOSE 8080 +ENTRYPOINT ["/usr/local/bin/lab-cx"] +CMD ["proxy"] diff --git a/GOVERNANCE.md b/GOVERNANCE.md new file mode 100644 index 0000000..cb93ae0 --- /dev/null +++ b/GOVERNANCE.md @@ -0,0 +1,9 @@ +# Governance + +`lab-context-engineering` is a component of the +[Kagenti](https://github.com/kagenti/kagenti) platform and follows Kagenti's project +governance. See the Kagenti community +[GOVERNANCE](https://github.com/i-am-bee/community/blob/main/GOVERNANCE.md) document for +the roles, decision-making process, and the path to becoming a maintainer. + +Maintainers of this repository are listed in [MAINTAINERS.md](MAINTAINERS.md). diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/MAINTAINERS.md b/MAINTAINERS.md index 17acd49..bf0563d 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -4,7 +4,7 @@ A repository maintainer is a committer with the additional privilege of merging ## Current Maintainers -This template repository shares the same maintainers as the main Kagenti ADK repository. You can find the full list in the main repository's [MAINTAINERS.md](https://github.com/kagenti/adk/blob/main/MAINTAINERS.md) file. +This repository is part of the [Kagenti](https://github.com/kagenti/kagenti) platform and shares the platform's maintainers. You can find the full list in the main repository's [MAINTAINERS.md](https://github.com/kagenti/kagenti/blob/main/MAINTAINERS.md) file. ## Becoming a Maintainer diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..08ceea0 --- /dev/null +++ b/Makefile @@ -0,0 +1,45 @@ +BINARY := lab-cx +PKG := github.com/kagenti/lab-context-engineering +VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null || echo dev) +COMMIT ?= $(shell git rev-parse --short HEAD 2>/dev/null || echo none) +LDFLAGS := -s -w \ + -X $(PKG)/internal/buildinfo.Version=$(VERSION) \ + -X $(PKG)/internal/buildinfo.Commit=$(COMMIT) + +# CGO is required to compile the tree-sitter binding; a C toolchain (gcc/clang) +# must be present for make test/build/lint. +export CGO_ENABLED=1 + +.DEFAULT_GOAL := help + +.PHONY: help +help: ## Display this help + @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | \ + awk 'BEGIN {FS = ":.*?## "}; {printf " \033[36m%-12s\033[0m %s\n", $$1, $$2}' + +.PHONY: build +build: ## Build the lab-cx binary into ./bin + @mkdir -p bin + go build -ldflags "$(LDFLAGS)" -o bin/$(BINARY) ./cmd/proxy + +.PHONY: test +test: ## Run all tests with the race detector + go test -race ./... + +.PHONY: fmt +fmt: ## Format all Go source + gofmt -w . + go mod tidy + +.PHONY: lint +lint: ## Run go vet and gofmt check + go vet ./... + @test -z "$$(gofmt -l .)" || (echo "gofmt needed:" && gofmt -l . && exit 1) + +.PHONY: pre-commit +pre-commit: ## Install pre-commit hooks + pre-commit install --install-hooks --hook-type commit-msg + +.PHONY: clean +clean: ## Remove build artifacts + rm -rf bin diff --git a/README.md b/README.md index 2721ab8..5f14954 100644 --- a/README.md +++ b/README.md @@ -1,31 +1,165 @@ +# lab-context-engineering -## Executive Summary +Context engineering for LLM agents — reduce token cost without changing the agent or +hurting the result. -Context engineering manages an agentic system's context: persisting and efficiently retrieving conversations and trajectories for the purpose of optimizing LLM context, this may include compacting and summarizing context to reduce cost and latency while preserving accuracy and overall agent behavior. +`lab-context-engineering` is a [Kagenti](https://github.com/kagenti/kagenti) platform +component: a single Go core that sits between an LLM coding agent and the model API, +reduces the context the request carries (caching, lossless reduction, and +verified-lossless extraction), and forwards a cheaper request upstream. It runs from one +codebase as a **standalone proxy** (`lab-cx proxy`), an **importable Go library** +(`engine`, `surfaces`, `config`), or **inside eval-containers**. -## Project Structure +It is **fail-open** (any error in any stage forwards the original request untouched) and +every reduction is **reversible** (a namespaced marker plus a content-addressed rewind +store, so a downstream consumer or the agent can always recover the original bytes). + +## Install + +Prerequisites: + +- **Go 1.25** +- **A C toolchain** (`gcc`/`clang`) — the code skeletonizer uses tree-sitter via cgo, so + `CGO_ENABLED=1` is required to build, test, and lint. The Makefile exports it for you. + +```sh +make build # builds ./bin/lab-cx (CGO_ENABLED=1) +./bin/lab-cx version # prints: lab-cx () +``` + +## Run + +Point any agent's base URL at the proxy — no agent changes: + +```sh +./bin/lab-cx proxy --preset balanced +# then, for any agent: +# ANTHROPIC_BASE_URL=http://localhost:8080 +# OPENAI_BASE_URL=http://localhost:8080/v1 +``` + +Works with Claude Code, Bob/OpenClaw, Codex, Cursor, Aider. With a config file and the +cheap-model extractor enabled, forwarding all traffic to an upstream gateway: + +```sh +./bin/lab-cx proxy --config configs/lab-cx.yaml \ + --upstream https://gateway.example/v1 \ + --extract-model claude-haiku-4-5 --extract-provider anthropic --extract-auth bearer \ + --extract-base https://gateway.example/v1 +``` + +`proxy` flags: `--addr` (default `:8080`), `--preset` +(`safe|balanced|aggressive|cache|coding|mcp`), `--config` (a YAML file that names which +components run, overrides `--preset`), `--upstream` (forward ALL requests here; default +routes by provider), `--extract-model/-provider/-auth/-base` (enable + configure the +cheap-model extractor; these override the config's transport block), `--max-body-bytes` +(default 32 MiB; `0` = no cap), `--upstream-timeout` (default `0s` = none; a non-zero +value caps the whole request and can truncate long SSE streams). + +Read the live aggregate metrics: ```sh -├── src/ # Source code directory -│ └── context-engineering/ # Python package directory -│ ├── __init__.py # Package initialization (empty) -│ └── TDB.... # Implementation -├── Dockerfile # Container configuration to build the agent -├── pyproject.toml # Package metadata and dependencies -├── uv.lock # Dependency lock file (generated by UV) -└── README.md # Project documentation +./bin/lab-cx stats # GETs /stats from a running proxy + prints a summary +./bin/lab-cx stats --addr http://localhost:8080 ``` -**Key files to focus on:** -**TBD - Add key files...** +Endpoints the proxy serves: + +- `GET /stats` — process-wide reduction snapshot as JSON (tokens before/after/saved, + cache injected, extracted, stage errors, added-latency p50/p95). +- `GET /winnow/expand?id=` — returns the original bytes behind a reversibility + marker. +- `GET /health`, `GET /ready` — liveness/readiness. + +Bypass / disable: + +- `WINNOW_DISABLE=1` env var — run the proxy as a transparent passthrough. +- `x-winnow-bypass: true` request header — skip reduction for that single request. + +## Components + +- **Cache** — injects Anthropic `cache_control` breakpoints on the stable prefix + (lossless). Stands down when the client already self-caches (e.g. Claude Code). +- **Reduce** (deterministic, no model) — `relevance` scoring + `collapse` of + stale/duplicate/empty content, `dedup`, `cmdfilter` (trims noisy shell-command output), + `skeleton` (drops function bodies, keeps signatures, via tree-sitter), and `format` + re-encoders that re-express bulky structured output in cheaper lossless forms + (`json_compact`, `toon`, `jsonl`, `markdown_kv`, `tsv`, `csv`). +- **Extract** (cheap model) — a cheap model proposes a structured projection of a huge + tool/MCP output; accepted only if **structurally contained** in the original. Strategies + `code` (model writes a filter run in a **Starlark sandbox** over the full body), `single` + (one-shot JSON-return filter), `rlm` (chunked), and a `deterministic` fallback. +- **Tokenizer** — real BPE token counts via tiktoken `o200k_base` (`internal/tokens`); the + same counter gates every reduction (a component never inflates an output). +- **Reversibility** — namespaced markers + a content-addressed rewind store; recover via + `engine.FindMarkers` + `engine.Expand`, or the `/winnow/expand` endpoint. +- **Observability** — OpenTelemetry GenAI semantic conventions (`gen_ai.*`) plus the live + `/stats` aggregate. + +## Config + +The example config is [`configs/lab-cx.yaml`](configs/lab-cx.yaml). It folds onto a base +`preset`, then selects which stages/reducers/encoders/extract-strategies run, **purely by +name**. Extension path — no core edit beyond one registration: -- `pyproject.toml`: Update this if you need to add dependencies -- `Dockerfile`: Modify if you need special build configuration +1. **Add** a new encoder/reducer/extract-strategy/stage, and **register by name** in its + registry (encoders in `internal/reduce/actions.go`, reducers via + `reduce.RegisterReducer`, strategies in `internal/extract` `RunExtraction`, stages in + `engine` `builtinStages`). +2. **List it by name** in the matching list in `configs/lab-cx.yaml` (an empty/omitted list + means "all built-in defaults"; list order sets priority/run order). -## Requirements -**TBD - Add requirements..** -- Python 3.11 or higher +Full detail in [docs/RUNNING.md](docs/RUNNING.md). +## Integrations +- [docs/integration/claude-code.md](docs/integration/claude-code.md) — base-URL swap; + real runs measured (**−50% tokens** on a large-file task with correctness preserved; + do-no-harm on a trivial task). +- [docs/integration/bob.md](docs/integration/bob.md) — IBM Bob Shell (OpenAI-compatible + CLI); real run, **5/5 tasks correct**, cache-injection lever fires on every request. +- [docs/integration/swe-bench.md](docs/integration/swe-bench.md) — proxy before the + eval-containers gateway; wiring validated, full run is a documented runbook. +- [docs/integration/authbridge.md](docs/integration/authbridge.md) — import the engine + in-process from a Kagenti AuthBridge plugin (the plugin lives in `kagenti-extensions`). + +## Results + +Real measured numbers (2026-06-24). See [docs/RESULTS.md](docs/RESULTS.md) for the index. + +| Measurement | Result | Source | +|---|---|---| +| `cmdfilter` on verbose command output | **−94%** tokens (reversible) | [RESULTS-offline.md](docs/RESULTS-offline.md) | +| `format` re-encode on structured output | **−35%** best / **−29%** TOON | [RESULTS-offline.md](docs/RESULTS-offline.md) | +| `skeleton` on a code read | **−78%** tokens | [RESULTS-offline.md](docs/RESULTS-offline.md) | +| Full deterministic pipeline, 10 real fixtures | **−93%** aggregate (all reversible) | [RESULTS-offline.md](docs/RESULTS-offline.md) | +| Cheap-model extractor (`claude-haiku-4-5`) | **−56%…−80%** on 4/6 structured fixtures; 2 honest declines; all contained | [RESULTS-extract.md](docs/RESULTS-extract.md) | +| **Claude Code, large-file task (haiku)** | **−50.2%** tokens (34,954 saved), answer correct (42 funcs) | [claude-code.md](docs/integration/claude-code.md) | +| **Bob (haiku), 5-task suite** | **5/5 correct** incl. a file edit; cache injected 6/6; 0 errors | [bob.md](docs/integration/bob.md) | + +These are real measured numbers on real tool-output fixtures and live agent runs, with a +real BPE tokenizer. **Claude Code** (self-caching): −50% on a large-file task with +`reduce_cached_prefix`, answer unchanged — and do-no-harm on a trivial task. **Bob** +(non-self-caching): 5/5 tasks correct, the cache-injection lever fires every request. The +end-to-end **SWE-bench** run is a **documented runbook, not yet executed** +([swe-bench.md](docs/integration/swe-bench.md)). + +## Repository layout + +``` +cmd/proxy/ the lab-cx binary (proxy | stats | version) +cmd/labcx-bench/ offline + --extract measurement harness +engine/ Engine: Transform / Expand, stage orchestration (builtinStages) +surfaces/ anthropic | openai | gemini wire <-> canonical request +config/ Settings + presets, YAML config loader +canon/ canonical request type +observability/ OpenTelemetry GenAI emitter + /stats aggregator +internal/ tokens, reduce, extract, cheapmodel, cache, markers, store, proxyhttp, treesitter +configs/ example config (lab-cx.yaml) +docs/ RUNNING, RESULTS, integration guides +deploy/ eval-containers wiring +``` +## License +Apache-2.0. See [LICENSE](LICENSE). diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..4b3b7fb --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,18 @@ +# Security Policy + +## Reporting a vulnerability + +Please report security vulnerabilities privately. Do **not** open a public GitHub issue. + +Use GitHub's private vulnerability reporting ("Report a vulnerability" under the +repository's Security tab), or follow the disclosure process described in the main +[Kagenti](https://github.com/kagenti/kagenti/blob/main/SECURITY.md) repository. + +We will acknowledge your report and work with you on a coordinated disclosure. + +## Scope notes + +This component intercepts and rewrites LLM agent traffic. It is designed to **fail open**: +on any internal error it forwards the original request unmodified. It does not persist +credentials. Reduction state (the rewind store) is content-addressed and node-local by +default; treat any configured shared backend (e.g. Redis) as sensitive. diff --git a/canon/request.go b/canon/request.go new file mode 100644 index 0000000..325b035 --- /dev/null +++ b/canon/request.go @@ -0,0 +1,107 @@ +// Package canon defines lab-context-engineering's canonical request model: an +// Anthropic-shaped chat request. Every surface (Anthropic, OpenAI, Gemini) maps a +// provider's wire format onto this model, the stage pipeline reads and mutates it, +// and the surface renders it back. It is the one shape the engine understands. +// +// The model is intentionally a generic decoded-JSON container (Root) rather than a +// fully-typed struct: an LLM request carries many provider-specific top-level fields +// (max_tokens, temperature, stream, metadata, ...) and per-block extensions +// (cache_control). Keeping the whole object as decoded JSON preserves every field +// across a decode→encode round-trip and lets stages add breakpoints freely — exactly +// what the winnow prototype relied on. Typed helpers give ergonomic access to the +// parts stages actually touch (messages and content blocks). +package canon + +import ( + "bytes" + "encoding/json" +) + +// Request is a canonical (Anthropic-shaped) chat request. Root is the decoded +// top-level JSON object and is the source of truth; the typed accessors below read +// and mutate it in place. +type Request struct { + Root map[string]any +} + +// Decode parses an Anthropic-shaped request body into a Request. Numbers are +// preserved as json.Number so an integer field re-encodes as "1", not "1.0". +func Decode(body []byte) (Request, error) { + dec := json.NewDecoder(bytes.NewReader(body)) + dec.UseNumber() + var root map[string]any + if err := dec.Decode(&root); err != nil { + return Request{}, err + } + if root == nil { + root = map[string]any{} + } + return Request{Root: root}, nil +} + +// Encode serializes the request back to JSON bytes. +func (r Request) Encode() ([]byte, error) { + return json.Marshal(r.Root) +} + +// Clone returns a deep copy so a surface can mutate without touching the caller's +// data. It round-trips through JSON, preserving json.Number fidelity. +func (r Request) Clone() Request { + b, err := json.Marshal(r.Root) + if err != nil { + return Request{Root: map[string]any{}} + } + out, err := Decode(b) + if err != nil { + return Request{Root: map[string]any{}} + } + return out +} + +// Model returns the "model" field, or "" if absent. +func (r Request) Model() string { + s, _ := r.Root["model"].(string) + return s +} + +// Messages returns the raw message list as a slice of maps. Mutating an element +// mutates the underlying request. Returns nil if there are no messages. +func (r Request) Messages() []map[string]any { + raw, ok := r.Root["messages"].([]any) + if !ok { + return nil + } + out := make([]map[string]any, 0, len(raw)) + for _, m := range raw { + if mm, ok := m.(map[string]any); ok { + out = append(out, mm) + } + } + return out +} + +// Blocks returns the content blocks of a message as a slice of maps. Anthropic +// content may be a plain string (returned as a single synthesized text block) or a +// list of block objects. +func Blocks(msg map[string]any) []map[string]any { + switch c := msg["content"].(type) { + case string: + return []map[string]any{{"type": "text", "text": c}} + case []any: + out := make([]map[string]any, 0, len(c)) + for _, b := range c { + if bb, ok := b.(map[string]any); ok { + out = append(out, bb) + } + } + return out + default: + return nil + } +} + +// BlockType returns a block's "type" field. +func BlockType(block map[string]any) string { + s, _ := block["type"].(string) + return s +} diff --git a/cmd/labcx-bench/main.go b/cmd/labcx-bench/main.go new file mode 100644 index 0000000..0b4d17d --- /dev/null +++ b/cmd/labcx-bench/main.go @@ -0,0 +1,718 @@ +// Command labcx-bench is an OFFLINE measurement harness: it proves, on REAL tool-output +// fixtures, how much each DETERMINISTIC reduction component reduces tokens/bytes — with +// no model in the loop (the Extract stage is OFF). For every fixture it builds a minimal +// canonical request that surfaces the fixture as a tool_result, runs the engine with one +// component enabled at a time (and the full deterministic pipeline), and reports +// tokens/bytes before+after, % saved, whether the reduction is reversible (the original +// recovers byte-for-byte via the rewind store), and the added latency. +// +// The token counts come from the same real BPE tokenizer the engine uses (o200k_base, +// internal/tokens). Run it and paste the table into docs/RESULTS-offline.md: +// +// CGO_ENABLED=1 go run ./cmd/labcx-bench # human table +// CGO_ENABLED=1 go run ./cmd/labcx-bench --json # machine-readable rows +// +// With --extract the harness flips into ONLINE mode: it enables cheap-model extraction, +// wires a REAL Anthropic-compatible gateway (claude-haiku-4-5) via cheapmodel.Anthropic, +// and measures the LLM extractor on the large structured fixtures. See docs/RESULTS-extract.md +// for the recorded run. The env vars ANTHROPIC_BASE_URL + ANTHROPIC_AUTH_TOKEN must be set: +// +// source /tmp/lcx_env.sh && CGO_ENABLED=1 go run ./cmd/labcx-bench --extract +package main + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "flag" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "github.com/kagenti/lab-context-engineering/canon" + "github.com/kagenti/lab-context-engineering/config" + "github.com/kagenti/lab-context-engineering/engine" + "github.com/kagenti/lab-context-engineering/internal/cheapmodel" + "github.com/kagenti/lab-context-engineering/internal/extract" + "github.com/kagenti/lab-context-engineering/internal/markers" + "github.com/kagenti/lab-context-engineering/internal/reduce" + "github.com/kagenti/lab-context-engineering/internal/tokens" +) + +// fixture is one real tool output plus how to surface it as a canonical request. +type fixture struct { + name string // table label + relPath string // path under testdata/fixtures + toolName string // tool that produced it (read → skeleton path; "" → generic) + filePath string // input.file_path for read tools (drives FilePath/skeleton) + command string // input.command for Bash-style tools (drives cmdfilter) +} + +// fixtureSet is the committed corpus. Each entry names the component it primarily +// exercises (see testdata/fixtures/README.md for provenance of every file). +var fixtureSet = []fixture{ + // cmd_outputs → command-output filter + {"pytest_failures", "cmd_outputs/pytest_failures.txt", "Bash", "", "pytest -q tests/"}, + {"cargo_test_failure", "cmd_outputs/cargo_test_failure.txt", "Bash", "", "cargo test"}, + {"cargo_build", "cmd_outputs/cargo_build.txt", "Bash", "", "cargo build"}, + // structured_json → format re-encoder (toon / tsv / csv / json_compact) + {"flights_search", "structured_json/flights_search.json", "search_flights", "", ""}, + {"users_directory", "structured_json/users_directory.json", "list_users", "", ""}, + {"products_inventory", "structured_json/products_inventory.json", "list_products", "", ""}, + {"oc_pods_slice", "structured_json/oc_pods_slice.json", "Bash", "", "oc get pods -o json"}, + // search_results → format re-encoder on nested record arrays + {"glab_issue_list", "search_results/glab_issue_list.json", "Bash", "", "glab issue list -F json"}, + {"glab_mr_list", "search_results/glab_mr_list.json", "Bash", "", "glab mr list -F json"}, + // file_reads → code skeletonizer + {"runner_py", "file_reads/runner.py", "Read", "scripts/benchmark-sessions/lib/runner.py", ""}, +} + +// component is one deterministic configuration to measure in isolation, plus the full +// deterministic pipeline. extract is always OFF (no model). +type component struct { + name string + settings func() config.Settings +} + +// isolate builds a settings value that turns the reduce stage on but enables only the +// named reducers / encoders, with caching off and extraction off — so a measured +// reduction is attributable to that one component. +func isolate(reducers, encoders []string, cmdFilter bool) config.Settings { + return config.Settings{ + CacheEnabled: false, + ReduceEnabled: true, + ProtectRecent: 2, + // ProvableOnly off so an unused tool_result collapses to reason "unused" (a + // collapse-eligible reason); reductions stay reversible regardless. + ProvableOnly: false, + CollapseOutputs: true, + CmdFilter: cmdFilter, + Reducers: reducers, + Encoders: encoders, + Stages: []string{"reduce"}, + ExtractEnabled: false, + } +} + +func components() []component { + return []component{ + {"cmdfilter", func() config.Settings { + // Only the command-output filter: no reducers in the routing loop. + s := isolate([]string{"__none__"}, nil, true) + return s + }}, + {"format_toon", func() config.Settings { + // Only the format re-encoder, restricted to TOON. + return isolate([]string{"format"}, []string{"toon"}, false) + }}, + {"format_best", func() config.Settings { + // Only the format re-encoder, all encoders (it picks the smallest faithful). + return isolate([]string{"format"}, nil, false) + }}, + {"skeleton", func() config.Settings { + // Only the code skeletonizer. + return isolate([]string{"skeleton"}, nil, false) + }}, + {"collapse", func() config.Settings { + // Only the collapse reducer (reversible marker for unused outputs). + return isolate([]string{"collapse"}, nil, false) + }}, + {"pipeline_full", func() config.Settings { + // Full deterministic pipeline: cmdfilter + all reducers/encoders + cache, + // extraction OFF. This is the "balanced" preset minus the model. + s := config.Default() + s.ExtractEnabled = false + s.ProvableOnly = false + return s + }}, + } +} + +// row is one measured (fixture, component) result. +type row struct { + Fixture string `json:"fixture"` + Component string `json:"component"` + TokensBefore int `json:"tokens_before"` + TokensAfter int `json:"tokens_after"` + PctSaved float64 `json:"pct_saved"` + BytesBefore int `json:"bytes_before"` + BytesAfter int `json:"bytes_after"` + Reversible bool `json:"reversible"` + AddedLatency float64 `json:"added_latency_ms"` + changed bool // internal: did this component touch the fixture at all +} + +func main() { + jsonOut := flag.Bool("json", false, "emit machine-readable JSON rows instead of a table") + extractMode := flag.Bool("extract", false, "ONLINE: measure the real cheap-model extractor against the live gateway (claude-haiku-4-5)") + flag.Parse() + + if *extractMode { + runExtract(*jsonOut) + return + } + + rows := measureAll() + + if *jsonOut { + enc := json.NewEncoder(os.Stdout) + enc.SetIndent("", " ") + if err := enc.Encode(rows); err != nil { + fmt.Fprintln(os.Stderr, "labcx-bench:", err) + os.Exit(1) + } + return + } + fmt.Print(markdownTable(rows)) +} + +// fixturesDir locates testdata/fixtures by walking up from the working directory for the +// go.mod that marks the module root, so the harness runs from any subdirectory (and from +// `go test`, whose cwd is the package dir). +func fixturesDir() (string, error) { + dir, err := os.Getwd() + if err != nil { + return "", err + } + for { + if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil { + return filepath.Join(dir, "testdata", "fixtures"), nil + } + parent := filepath.Dir(dir) + if parent == dir { + return "", fmt.Errorf("module root (go.mod) not found above %s", dir) + } + dir = parent + } +} + +// measureAll runs every component against every fixture and returns the rows in a stable +// order (fixture order, then component order). +func measureAll() []row { + base, err := fixturesDir() + if err != nil { + fmt.Fprintln(os.Stderr, "labcx-bench:", err) + os.Exit(1) + } + comps := components() + var rows []row + for _, fx := range fixtureSet { + body, err := os.ReadFile(filepath.Join(base, filepath.FromSlash(fx.relPath))) + if err != nil { + fmt.Fprintf(os.Stderr, "labcx-bench: read %s: %v\n", fx.relPath, err) + os.Exit(1) + } + fixtureText := string(body) + for _, c := range comps { + rows = append(rows, measure(fx, fixtureText, c)) + } + } + return rows +} + +// measure runs one component against one fixture in isolation and verifies reversibility. +func measure(fx fixture, fixtureText string, c component) row { + settings := c.settings() + eng := engine.New(settings, nil, nil) + + req := buildRequest(fx, fixtureText) + before := req.Clone() + + start := time.Now() + out, _ := eng.Transform(context.Background(), req) + elapsed := time.Since(start) + + beforeText := toolResultText(before, fx) + afterText := toolResultText(out, fx) + + r := row{ + Fixture: fx.name, + Component: c.name, + TokensBefore: tokens.Count(beforeText), + TokensAfter: tokens.Count(afterText), + BytesBefore: len(beforeText), + BytesAfter: len(afterText), + AddedLatency: float64(elapsed.Microseconds()) / 1000.0, + changed: afterText != beforeText, + } + if r.TokensBefore > 0 { + r.PctSaved = round2(float64(r.TokensBefore-r.TokensAfter) / float64(r.TokensBefore) * 100) + } + r.Reversible = verifyReversible(eng, beforeText, afterText) + return r +} + +// verifyReversible confirms the reduction is recoverable: if the component left a winnow +// marker, the rewind store must return the exact original; if it left the text untouched +// (a no-op for this fixture/component), that is trivially reversible. A reduction that +// changed the text but exposes no recoverable original would be a (disallowed) lossy drop. +func verifyReversible(eng *engine.Engine, before, after string) bool { + if after == before { + return true // no-op: nothing to recover + } + ids := engine.FindMarkers(after) + if len(ids) == 0 { + // The format re-encoder rewrites in place behind a recovery note that DOES carry + // a marker; if no marker is present and the text changed, treat as not provably + // reversible. + return false + } + for _, id := range ids { + orig, ok := eng.Expand(id) + if !ok || !strings.Contains(before, orig) { + return false + } + } + return true +} + +// buildRequest wraps a fixture as a canonical Anthropic-shaped request: an assistant +// tool_use (so command-aware and read-aware passes see the command / file path) followed +// by the tool_result carrying the fixture, then two neutral trailing turns so the +// fixture sits OUTSIDE the protect-recent window and is eligible for reduction. +func buildRequest(fx fixture, fixtureText string) canon.Request { + input := map[string]any{} + if fx.command != "" { + input["command"] = fx.command + } + if fx.filePath != "" { + input["file_path"] = fx.filePath + } + root := map[string]any{ + "model": "claude-sonnet-4-6", + "messages": []any{ + map[string]any{"role": "user", "content": "Run the tool and report back."}, + map[string]any{"role": "assistant", "content": []any{ + map[string]any{"type": "tool_use", "id": "tu_1", "name": fx.toolName, "input": input}, + }}, + map[string]any{"role": "user", "content": []any{ + map[string]any{"type": "tool_result", "tool_use_id": "tu_1", "content": fixtureText}, + }}, + map[string]any{"role": "assistant", "content": "Acknowledged; continuing."}, + map[string]any{"role": "user", "content": "Please proceed with the next step."}, + }, + } + return canon.Request{Root: root} +} + +// toolResultText pulls the (possibly reduced) text of the fixture's tool_result block +// back out of a request, so before/after are compared on the SAME block. +func toolResultText(req canon.Request, fx fixture) string { + for _, m := range req.Messages() { + for _, b := range canon.Blocks(m) { + if canon.BlockType(b) != "tool_result" { + continue + } + switch c := b["content"].(type) { + case string: + return c + case []any: + var parts []string + for _, x := range c { + if bb, ok := x.(map[string]any); ok && bb["type"] == "text" { + if t, ok := bb["text"].(string); ok { + parts = append(parts, t) + } + } + } + return strings.Join(parts, "\n") + } + } + } + return "" +} + +func round2(f float64) float64 { + return float64(int(f*100+0.5*sign(f))) / 100 +} + +func sign(f float64) float64 { + if f < 0 { + return -1 + } + return 1 +} + +// markdownTable renders the measured rows as the table pasted into RESULTS-offline.md. +func markdownTable(rows []row) string { + var b strings.Builder + b.WriteString("| fixture | component | tokens_before | tokens_after | %saved | bytes_before | bytes_after | reversible | added_latency_ms |\n") + b.WriteString("| --- | --- | ---: | ---: | ---: | ---: | ---: | :---: | ---: |\n") + // Stable order: fixtures as declared, components as declared. + for _, r := range rows { + rev := "yes" + if !r.Reversible { + rev = "no" + } + note := "" + if !r.changed { + note = " (no-op)" + } + fmt.Fprintf(&b, "| %s | %s%s | %d | %d | %.2f | %d | %d | %s | %.3f |\n", + r.Fixture, r.Component, note, r.TokensBefore, r.TokensAfter, r.PctSaved, + r.BytesBefore, r.BytesAfter, rev, r.AddedLatency) + } + return b.String() +} + +// --------------------------------------------------------------------------- +// --extract: ONLINE measurement of the REAL cheap-model extractor. +// --------------------------------------------------------------------------- + +// extractFloor is the LLMCompactFloor used for the --extract measurement. The committed +// structured fixtures are only a few hundred tokens, well below the production default of +// 3000, so we lower the floor to 200 here purely so that extraction actually FIRES on the +// corpus. This is a measurement-only setting; it does not change any default. +const extractFloor = 200 + +// extractFixture is a large/structured fixture surfaced as a tool_result, paired with a +// realistic recent goal that references specific records — so HarvestIdentifiers builds a +// non-trivial keep-set and the extractor has something concrete to filter toward. +type extractFixture struct { + name string + relPath string + toolName string + command string + goal string +} + +// extractFixtureSet is the structured_json + search_results corpus (the large, record-shaped +// fixtures extraction targets). Each goal names records that exist in the fixture. +var extractFixtureSet = []extractFixture{ + {"flights_search", "structured_json/flights_search.json", "search_flights", "", + "Book the cheapest nonstop SFO->JFK flight. I only care about flights FL003 and FL004 and their prices; ignore the rest."}, + {"users_directory", "structured_json/users_directory.json", "list_users", "", + "I need the admin account only — find user_0037 (alice.target@corp.com) and confirm the role is admin."}, + {"products_inventory", "structured_json/products_inventory.json", "list_products", "", + "Reorder the out-of-stock items. I care about SKU0001 and SKU0004 (in_stock=false) and their prices."}, + {"oc_pods_slice", "structured_json/oc_pods_slice.json", "Bash", "oc get pods -o json", + "Find pods that are NOT Running or that have restarts. I care about alertmanager-main-0 and any pod with restartCount > 0."}, + {"glab_issue_list", "search_results/glab_issue_list.json", "Bash", "glab issue list -F json", + "Triage open issues. I only care about issue iid 156 (Support glab CI pipeline filtering) — its title, state, and labels."}, + {"glab_mr_list", "search_results/glab_mr_list.json", "Bash", "glab mr list -F json", + "Review the glab MR. I only care about merge request iid 314 (feat(glab): add GitLab CLI support) — its state, merge_status, and source_branch."}, +} + +// extractRow is one measured (fixture) extraction result. +type extractRow struct { + Fixture string `json:"fixture"` + TokensBefore int `json:"tokens_before"` + TokensAfter int `json:"tokens_after"` + PctSaved float64 `json:"pct_saved"` + Strategy string `json:"strategy"` + Contained bool `json:"contained"` + LatencyMs float64 `json:"latency_ms"` + Model string `json:"model"` +} + +// runExtract enables real cheap-model extraction and measures it against the structured +// corpus, printing a Markdown table (or JSON with --json). It requires the gateway env. +func runExtract(jsonOut bool) { + base := os.Getenv("ANTHROPIC_BASE_URL") + token := os.Getenv("ANTHROPIC_AUTH_TOKEN") + if base == "" || token == "" { + fmt.Fprintln(os.Stderr, "labcx-bench --extract: ANTHROPIC_BASE_URL and ANTHROPIC_AUTH_TOKEN must be set (source /tmp/lcx_env.sh)") + os.Exit(1) + } + dir, err := fixturesDir() + if err != nil { + fmt.Fprintln(os.Stderr, "labcx-bench:", err) + os.Exit(1) + } + + model := cheapmodel.Anthropic{ + BaseURL: base, + APIKey: token, + Model: "claude-haiku-4-5", + AuthScheme: "bearer", + } + + var rows []extractRow + for _, fx := range extractFixtureSet { + body, err := os.ReadFile(filepath.Join(dir, filepath.FromSlash(fx.relPath))) + if err != nil { + fmt.Fprintf(os.Stderr, "labcx-bench: read %s: %v\n", fx.relPath, err) + os.Exit(1) + } + rows = append(rows, measureExtract(fx, string(body), model)) + } + + if jsonOut { + enc := json.NewEncoder(os.Stdout) + enc.SetIndent("", " ") + if err := enc.Encode(rows); err != nil { + fmt.Fprintln(os.Stderr, "labcx-bench:", err) + os.Exit(1) + } + return + } + fmt.Print(extractTable(rows)) +} + +// measureExtract runs the engine with extraction ENABLED against one fixture and records +// the real result. The engine is configured so the fixture qualifies as an LLM candidate +// (ExtractEnabled, low LLMCompactFloor, small ProtectRecent). To record the strategy the +// model actually chose — which engine.EnableExtract does not surface — we register a +// strategy-capturing extract func that mirrors EnableExtract's logic exactly (same +// goal/keep-set, RunExtraction, then reversible splice via the engine store + recovery +// marker) using cheapmodel.Anthropic as the model. There is exactly ONE model call path. +func measureExtract(fx extractFixture, fixtureText string, model engine.Model) extractRow { + settings := config.Settings{ + CacheEnabled: false, + ReduceEnabled: true, + ProtectRecent: 1, + ProvableOnly: false, + CollapseOutputs: true, + Reducers: []string{"__none__"}, // no deterministic reducer touches the body first + Stages: []string{"reduce", "extract"}, + ExtractEnabled: true, + LLMCompactFloor: extractFloor, + } + eng := engine.New(settings, nil, nil) + + cfg := engine.DefaultExtractConfig() + cfg.Floor = extractFloor + // EnableExtract performs the real wiring/splice; we then override its func with a + // behaviorally identical one that also records the chosen strategy. + eng.EnableExtract(model, cfg) + + var chosen string + eng.SetExtract(capturingExtractFunc(eng, model, cfg.Floor, &chosen)) + + req := buildExtractRequest(fx, fixtureText) + before := req.Clone() + beforeText := toolResultTextAny(before) + + start := time.Now() + out, _ := eng.Transform(context.Background(), req) + elapsed := time.Since(start) + + afterText := toolResultTextAny(out) + + r := extractRow{ + Fixture: fx.name, + TokensBefore: tokens.Count(beforeText), + TokensAfter: tokens.Count(afterText), + Strategy: chosen, + LatencyMs: float64(elapsed.Milliseconds()), + Model: "claude-haiku-4-5", + } + if r.TokensBefore > 0 { + r.PctSaved = round2(float64(r.TokensBefore-r.TokensAfter) / float64(r.TokensBefore) * 100) + } + if chosen == "" { + r.Strategy = "none" + } + // Contained: the engine only splices a result that left a reversible recovery marker + // whose stored original is a substring of the pre-extraction body. Verify via the + // public FindMarkers + Expand round-trip — a spliced block is lossless and reversible. + r.Contained = verifyContained(eng, beforeText, afterText) + return r +} + +// capturingExtractFunc returns an extract func equivalent to engine.EnableExtract's, but +// it records the strategy RunExtraction selected for the (single) candidate into *chosen +// and performs the reversible splice through the engine's public store + recovery marker. +func capturingExtractFunc(eng *engine.Engine, model engine.Model, floor int, chosen *string) engine.ExtractFunc { + icfg := extract.Cfg{Mode: "auto", Floor: floor, AllowDeterministic: true, MaxChars: 4000} + cache := extract.NewCache() + return func(ctx context.Context, req canon.Request, cands []reduce.Candidate) error { + goal := recentGoalText(req, 6) + keep := extract.HarvestIdentifiers(goal, 60) + for _, c := range cands { + func() { + defer func() { _ = recover() }() // fail-open per candidate + key := goalKey(extract.ContentKey(c.Text), goal, keep) + result, ok := cache.Get(key) + var strat string + if ok { + strat = "cache" + } else { + result, strat = extract.RunExtraction(ctx, c.Text, goal, keep, c.TokenEst, icfg, model) + if strat == "none" || result == "" { + *chosen = "none" + return + } + cache.Put(key, result) + } + *chosen = strat + spliceResult(eng, req, c, result) + }() + } + return nil + } +} + +// spliceResult mirrors engine.(*Engine).splice using the public API: store the original +// for reversal, write the extracted text plus a recovery marker, never inflate. +func spliceResult(eng *engine.Engine, req canon.Request, c reduce.Candidate, result string) { + block := blockAtExtract(req, c.MsgIndex, c.BlockIndex) + if block == nil { + return + } + rid := eng.Store().Put(c.Text) + label := c.FilePath + if label == "" { + label = c.ToolName + } + if label == "" { + label = "tool output" + } + newText := strings.TrimRight(result, "\n") + "\n" + markers.RecoveryNote(label, "extracted", rid) + if tokens.Count(newText) >= tokens.Count(c.Text) { + return // never inflate + } + switch block["type"] { + case "tool_result": + block["content"] = newText + case "text": + block["text"] = newText + } +} + +func blockAtExtract(req canon.Request, mi, bi int) map[string]any { + msgs := req.Messages() + if mi < 0 || mi >= len(msgs) { + return nil + } + list, ok := msgs[mi]["content"].([]any) + if !ok || bi < 0 || bi >= len(list) { + return nil + } + blk, _ := list[bi].(map[string]any) + return blk +} + +// goalKey mirrors engine.goalKey: a goal-aware composite of the body content key, goal, +// and keep-set, so a different goal is a cache miss. +func goalKey(contentKey, goal string, keep []string) string { + h := sha256.New() + h.Write([]byte(contentKey)) + h.Write([]byte{0}) + h.Write([]byte(goal)) + for _, k := range keep { + h.Write([]byte{0}) + h.Write([]byte(k)) + } + return hex.EncodeToString(h.Sum(nil))[:24] +} + +// recentGoalText concatenates the text of the last k turns (excluding tool_result content), +// mirroring the engine's goal-extraction window. +func recentGoalText(req canon.Request, k int) string { + msgs := req.Messages() + start := len(msgs) - k + if start < 0 { + start = 0 + } + var out []string + for _, m := range msgs[start:] { + switch c := m["content"].(type) { + case string: + out = append(out, c) + case []any: + for _, b := range c { + if bb, ok := b.(map[string]any); ok && bb["type"] == "text" { + if t, ok := bb["text"].(string); ok { + out = append(out, t) + } + } + } + } + } + s := strings.Join(out, "\n") + if len(s) > 6000 { + s = s[:6000] + } + return s +} + +// buildExtractRequest surfaces the fixture as a tool_result, with the fixture's realistic +// goal as the recent trailing user turn (so it sits OUTSIDE protect-recent=1 and the goal +// conditions the keep-set). +func buildExtractRequest(fx extractFixture, fixtureText string) canon.Request { + input := map[string]any{} + if fx.command != "" { + input["command"] = fx.command + } + root := map[string]any{ + "model": "claude-sonnet-4-6", + "messages": []any{ + map[string]any{"role": "user", "content": "Run the tool and report back."}, + map[string]any{"role": "assistant", "content": []any{ + map[string]any{"type": "tool_use", "id": "tu_1", "name": fx.toolName, "input": input}, + }}, + map[string]any{"role": "user", "content": []any{ + map[string]any{"type": "tool_result", "tool_use_id": "tu_1", "content": fixtureText}, + }}, + map[string]any{"role": "assistant", "content": "Acknowledged; continuing."}, + map[string]any{"role": "user", "content": fx.goal}, + }, + } + return canon.Request{Root: root} +} + +// toolResultTextAny pulls the (possibly extracted) tool_result text out of a request. +func toolResultTextAny(req canon.Request) string { + for _, m := range req.Messages() { + for _, b := range canon.Blocks(m) { + if canon.BlockType(b) != "tool_result" { + continue + } + switch c := b["content"].(type) { + case string: + return c + case []any: + var parts []string + for _, x := range c { + if bb, ok := x.(map[string]any); ok && bb["type"] == "text" { + if t, ok := bb["text"].(string); ok { + parts = append(parts, t) + } + } + } + return strings.Join(parts, "\n") + } + } + } + return "" +} + +// verifyContained confirms a spliced extraction is lossless + reversible: the marker's +// stored original must be a substring of the original body. A decline (no marker, text +// unchanged) is reported as not-contained=false honestly (nothing was spliced). +func verifyContained(eng *engine.Engine, before, after string) bool { + if after == before { + return false // no splice happened (decline) + } + ids := engine.FindMarkers(after) + if len(ids) == 0 { + return false + } + for _, id := range ids { + orig, ok := eng.Expand(id) + if !ok || !strings.Contains(before, orig) { + return false + } + } + return true +} + +// extractTable renders the measured extraction rows as the table pasted into RESULTS-extract.md. +func extractTable(rows []extractRow) string { + var b strings.Builder + b.WriteString("| fixture | tokens_before | tokens_after | %saved | strategy | contained | latency_ms | model |\n") + b.WriteString("| --- | ---: | ---: | ---: | :---: | :---: | ---: | :--- |\n") + for _, r := range rows { + contained := "yes" + if !r.Contained { + contained = "no" + } + fmt.Fprintf(&b, "| %s | %d | %d | %.2f | %s | %s | %.0f | %s |\n", + r.Fixture, r.TokensBefore, r.TokensAfter, r.PctSaved, r.Strategy, contained, r.LatencyMs, r.Model) + } + return b.String() +} diff --git a/cmd/labcx-bench/main_test.go b/cmd/labcx-bench/main_test.go new file mode 100644 index 0000000..34a6986 --- /dev/null +++ b/cmd/labcx-bench/main_test.go @@ -0,0 +1,130 @@ +package main + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/kagenti/lab-context-engineering/config" + "github.com/kagenti/lab-context-engineering/engine" +) + +// findFixture returns the committed fixture by name (fails the test if absent). +func findFixture(t *testing.T, name string) fixture { + t.Helper() + for _, fx := range fixtureSet { + if fx.name == name { + return fx + } + } + t.Fatalf("fixture %q not in fixtureSet", name) + return fixture{} +} + +// componentByName returns the settings constructor for a named component (fails if absent). +func componentByName(t *testing.T, name string) func() config.Settings { + t.Helper() + for _, c := range components() { + if c.name == name { + return c.settings + } + } + t.Fatalf("component %q not defined", name) + return nil +} + +// readFixture loads a committed fixture from disk via the same resolver the harness uses. +func readFixture(t *testing.T, fx fixture) string { + t.Helper() + base, err := fixturesDir() + if err != nil { + t.Fatalf("fixturesDir: %v", err) + } + body, err := os.ReadFile(filepath.Join(base, filepath.FromSlash(fx.relPath))) + if err != nil { + t.Fatalf("read %s: %v", fx.relPath, err) + } + return string(body) +} + +// TestHarnessReducesAndRecovers runs the harness end to end on one committed fixture and +// asserts the core promise: a deterministic component reduces tokens > 0 AND the original +// is recoverable byte-for-byte from the rewind store (reversible). +func TestHarnessReducesAndRecovers(t *testing.T) { + fx := findFixture(t, "runner_py") + text := readFixture(t, fx) + + r := measure(fx, text, component{name: "skeleton", settings: componentByName(t, "skeleton")}) + + if r.TokensBefore <= 0 { + t.Fatalf("expected non-empty fixture, got tokens_before=%d", r.TokensBefore) + } + if r.TokensAfter >= r.TokensBefore { + t.Fatalf("expected token reduction: tokens_after=%d not < tokens_before=%d", + r.TokensAfter, r.TokensBefore) + } + if r.BytesAfter >= r.BytesBefore { + t.Fatalf("expected byte reduction: bytes_after=%d not < bytes_before=%d", + r.BytesAfter, r.BytesBefore) + } + if r.PctSaved <= 0 { + t.Fatalf("expected pct_saved > 0, got %.2f", r.PctSaved) + } + if !r.Reversible { + t.Fatalf("skeleton reduction must be reversible (recoverable via rewind store)") + } +} + +// TestCmdFilterReducesRealCommandOutput proves the command-output filter reduces a real +// large command output (rtk's cargo build) and stays reversible. +func TestCmdFilterReducesRealCommandOutput(t *testing.T) { + fx := findFixture(t, "cargo_build") + text := readFixture(t, fx) + r := measure(fx, text, component{name: "cmdfilter", settings: componentByName(t, "cmdfilter")}) + if r.TokensAfter >= r.TokensBefore { + t.Fatalf("cmdfilter should reduce cargo build noise: before=%d after=%d", + r.TokensBefore, r.TokensAfter) + } + if !r.Reversible { + t.Fatalf("cmdfilter reduction must be reversible") + } +} + +// TestEveryRowReversible runs the whole matrix and asserts no component ever produces an +// irreversible (unrecoverable lossy) result — fail-open and reversibility are the repo's +// hard invariants. +func TestEveryRowReversible(t *testing.T) { + for _, r := range measureAll() { + if !r.Reversible { + t.Errorf("%s / %s: not reversible (tokens %d->%d)", + r.Fixture, r.Component, r.TokensBefore, r.TokensAfter) + } + } +} + +// TestFindMarkersExpandRoundTrip is a direct check that the engine's public recovery API +// (FindMarkers + Expand) round-trips an original through a reducing pass. +func TestFindMarkersExpandRoundTrip(t *testing.T) { + fx := findFixture(t, "glab_mr_list") + text := readFixture(t, fx) + + eng := engine.New(componentByName(t, "collapse")(), nil, nil) + req := buildRequest(fx, text) + before := toolResultText(req.Clone(), fx) + out, _ := eng.Transform(context.Background(), req) + after := toolResultText(out, fx) + + ids := engine.FindMarkers(after) + if len(ids) == 0 { + t.Fatalf("expected a recovery marker after collapse") + } + orig, ok := eng.Expand(ids[0]) + if !ok { + t.Fatalf("Expand(%q) returned not-found", ids[0]) + } + if !strings.Contains(before, orig) { + t.Fatalf("recovered original does not match the pre-reduction text") + } +} diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go new file mode 100644 index 0000000..cc507ad --- /dev/null +++ b/cmd/proxy/main.go @@ -0,0 +1,235 @@ +// Command lab-cx is the standalone context-engineering proxy: it sits between any +// LLM coding agent and the model API, reduces token cost on the request, and +// forwards it upstream. The same engine that powers this binary is importable as +// a library (see ../../engine) so other hosts — e.g. a Kagenti AuthBridge plugin +// — can wrap it without running this process. +package main + +import ( + "encoding/json" + "flag" + "fmt" + "io" + "net/http" + "os" + "time" + + "github.com/kagenti/lab-context-engineering/config" + "github.com/kagenti/lab-context-engineering/engine" + "github.com/kagenti/lab-context-engineering/internal/buildinfo" + "github.com/kagenti/lab-context-engineering/internal/cheapmodel" + "github.com/kagenti/lab-context-engineering/internal/proxyhttp" + "github.com/kagenti/lab-context-engineering/observability" +) + +func usage() { + fmt.Fprintf(os.Stderr, `lab-cx — context-engineering proxy for LLM agents + +Usage: + lab-cx proxy [--addr :8080] [--preset balanced] [--upstream URL] start the proxy + lab-cx stats [--addr http://localhost:8080] print live /stats + lab-cx version print version + +Point your agent at the proxy, e.g.: + ANTHROPIC_BASE_URL=http://localhost:8080 OPENAI_BASE_URL=http://localhost:8080/v1 + +Flags: + --addr listen address (default :8080) + --preset safe|balanced|aggressive|cache|coding|mcp (default balanced) + --config path to a YAML config file (overrides --preset; names which reducers/ + encoders/extract-strategies/stages run, by name; see configs/lab-cx.yaml). + --extract-model/-provider/-auth/-base still override the file's transport. + --upstream forward ALL requests here (e.g. an eval gateway); default routes by + provider (api.anthropic.com / api.openai.com) + --max-body-bytes cap request body size in bytes (default 33554432 = 32 MiB); + 0 means no cap + --upstream-timeout max total time per upstream request (e.g. 30s; default 0s = no + timeout). WARNING: a non-zero value caps the WHOLE request + including streamed responses and can truncate long LLM SSE streams +`) +} + +func main() { + if len(os.Args) < 2 { + usage() + os.Exit(2) + } + switch os.Args[1] { + case "version", "-v", "--version": + fmt.Printf("lab-cx %s (%s)\n", buildinfo.Version, buildinfo.Commit) + case "proxy": + runProxy(os.Args[2:]) + case "stats": + runStats(os.Args[2:]) + case "help", "-h", "--help": + usage() + default: + fmt.Fprintf(os.Stderr, "unknown command %q\n\n", os.Args[1]) + usage() + os.Exit(2) + } +} + +func runProxy(args []string) { + fs := flag.NewFlagSet("proxy", flag.ExitOnError) + addr := fs.String("addr", ":8080", "listen address") + preset := fs.String("preset", "balanced", "settings preset") + configPath := fs.String("config", "", "path to a YAML config file (overrides --preset; see configs/lab-cx.yaml)") + upstream := fs.String("upstream", "", "forward all requests to this base URL") + extractModel := fs.String("extract-model", "", "enable cheap-model extraction with this model (e.g. claude-haiku-4-5)") + extractProvider := fs.String("extract-provider", "anthropic", "extraction model provider: anthropic|openai") + extractBase := fs.String("extract-base", "", "base URL for the extraction model (default per provider)") + extractAuth := fs.String("extract-auth", "x-api-key", "anthropic auth scheme: x-api-key|bearer (bearer for gateway endpoints)") + maxBodyBytes := fs.Int64("max-body-bytes", 33554432, "cap request body size in bytes (32 MiB default); 0 means no cap") + upstreamTimeout := fs.String("upstream-timeout", "0s", "max total time per upstream request (e.g. 30s); 0 means no timeout. WARNING: a non-zero value caps the WHOLE request including streamed responses and can truncate long LLM SSE streams") + _ = fs.Parse(args) + + upstreamTimeoutDur, err := time.ParseDuration(*upstreamTimeout) + if err != nil { + fmt.Fprintf(os.Stderr, "lab-cx: invalid --upstream-timeout %q: %v\n", *upstreamTimeout, err) + os.Exit(2) + } + + if os.Getenv("WINNOW_DISABLE") == "1" { + fmt.Fprintln(os.Stderr, "lab-cx: WINNOW_DISABLE=1 — running as a transparent passthrough") + } + + // Resolve settings: a --config file (with named component selection) takes + // precedence over --preset. The file may also name the extraction transport + // (provider/model/auth/base); the matching --extract-* flags still override it. + var settings config.Settings + var tr config.Transport + settingsSrc := "preset=" + *preset + if *configPath != "" { + var loadErr error + settings, tr, loadErr = config.LoadWithTransport(*configPath) + if loadErr != nil { + fmt.Fprintf(os.Stderr, "lab-cx: %v\n", loadErr) + os.Exit(2) + } + settingsSrc = "config=" + *configPath + } else { + settings = config.Preset(*preset) + } + // --extract-* flags override the config's transport block (flag set explicitly wins; + // otherwise fall back to the config value, then the flag default). + if v := flagOrConfig(fs, "extract-model", *extractModel, tr.Model); v != "" { + *extractModel = v + } + if v := flagOrConfig(fs, "extract-provider", *extractProvider, tr.Provider); v != "" { + *extractProvider = v + } + if v := flagOrConfig(fs, "extract-auth", *extractAuth, tr.Auth); v != "" { + *extractAuth = v + } + if v := flagOrConfig(fs, "extract-base", *extractBase, tr.Base); v != "" { + *extractBase = v + } + + if os.Getenv("WINNOW_DISABLE") == "1" { + settings.Disabled = true + } + eng := engine.New(settings, nil, nil) + if *extractModel != "" { + key := os.Getenv("WINNOW_EXTRACT_KEY") + var model engine.Model + switch *extractProvider { + case "openai": + if key == "" { + key = os.Getenv("OPENAI_API_KEY") + } + model = cheapmodel.OpenAI{ + BaseURL: *extractBase, APIKey: key, Model: *extractModel, + } + case "anthropic", "": + if key == "" { + key = os.Getenv("ANTHROPIC_API_KEY") + } + if key == "" { + key = os.Getenv("ANTHROPIC_AUTH_TOKEN") + } + model = cheapmodel.Anthropic{ + BaseURL: *extractBase, APIKey: key, Model: *extractModel, + AuthScheme: *extractAuth, + } + default: + fmt.Fprintf(os.Stderr, "lab-cx: unknown --extract-provider %q (want anthropic|openai)\n", *extractProvider) + os.Exit(2) + } + eng.EnableExtract(model, engine.DefaultExtractConfig()) + fmt.Fprintf(os.Stderr, "lab-cx: extraction enabled (provider=%s model=%s)\n", *extractProvider, *extractModel) + } + agg := observability.NewAggregator(defaultCostRates()) + handler := proxyhttp.New(proxyhttp.Config{ + Engine: eng, Upstream: *upstream, + // Stream each event via slog AND fold it into the Aggregator served at /stats. + Emitter: observability.Tee{observability.SlogEmitter{}, agg}, + Aggregator: agg, + MaxBodyBytes: *maxBodyBytes, + UpstreamTimeout: upstreamTimeoutDur, + }) + + fmt.Fprintf(os.Stderr, "lab-cx proxy listening on %s (%s)\n", *addr, settingsSrc) + if err := http.ListenAndServe(*addr, handler); err != nil { + fmt.Fprintln(os.Stderr, "lab-cx:", err) + os.Exit(1) + } +} + +// defaultCostRates is a small input/output price table ($/MTok) used to estimate the +// dollar value of saved tokens. Savings are priced on input rates (reduction removes +// input tokens). DefaultCostKey prices any model not listed here. Edit to match your +// provider's published pricing; values here are illustrative defaults. +func defaultCostRates() map[string]observability.CostRate { + return map[string]observability.CostRate{ + "claude-haiku-4-5": {InputPerMTok: 1.00, OutputPerMTok: 5.00}, + observability.DefaultCostKey: {InputPerMTok: 3.00, OutputPerMTok: 15.00}, + } +} + +// runStats GETs /stats from a running proxy and prints the JSON plus a one-line summary. +func runStats(args []string) { + fs := flag.NewFlagSet("stats", flag.ExitOnError) + addr := fs.String("addr", "http://localhost:8080", "base URL of a running lab-cx proxy") + _ = fs.Parse(args) + + url := *addr + "/stats" + resp, err := http.Get(url) + if err != nil { + fmt.Fprintf(os.Stderr, "lab-cx: GET %s: %v\n", url, err) + os.Exit(1) + } + defer resp.Body.Close() + body, _ := io.ReadAll(resp.Body) + if resp.StatusCode != http.StatusOK { + fmt.Fprintf(os.Stderr, "lab-cx: %s returned %d: %s\n", url, resp.StatusCode, string(body)) + os.Exit(1) + } + + var snap observability.Snapshot + if err := json.Unmarshal(body, &snap); err != nil { + fmt.Fprintf(os.Stderr, "lab-cx: bad /stats response: %v\n", err) + os.Exit(1) + } + fmt.Println(string(body)) + fmt.Println(observability.SummaryOf(snap)) +} + +// flagOrConfig resolves an extract-* knob: an explicitly-set flag wins; otherwise the +// config value (if non-empty); otherwise the flag's current value (its default). This +// lets a --config file name the extraction transport while --extract-* flags override. +func flagOrConfig(fs *flag.FlagSet, name, flagVal, cfgVal string) string { + set := false + fs.Visit(func(f *flag.Flag) { + if f.Name == name { + set = true + } + }) + if set { + return flagVal + } + if cfgVal != "" { + return cfgVal + } + return flagVal +} diff --git a/config/config.go b/config/config.go new file mode 100644 index 0000000..13e217d --- /dev/null +++ b/config/config.go @@ -0,0 +1,79 @@ +// Package config holds the engine's settings and named presets. Settings are plain +// data; the proxy resolves them from WINNOW_*-style env / preset and hands them to the +// engine. Mirrors the knobs winnow exposes, trimmed to what the Go core uses. +package config + +// Settings configures a reduction pass end to end. +type Settings struct { + Disabled bool // global kill switch: forward untouched + + // Cache stage (cache_control injection). + CacheEnabled bool + CacheBreakpoints int + CacheStableGap int + CacheToolsBreakpoint bool + + // Reduce stage. + ReduceEnabled bool + ProtectRecent int + ProtectRecentToolUses int + ProvableOnly bool + CollapseOutputs bool + ContextLimit int + ReduceCachedPrefix bool + CmdFilter bool + RehydrateOnCompaction bool + + // Extract stage (cheap-model projection of large structured outputs). + ExtractEnabled bool + ExtractMode string // "" | auto | single | rlm | deterministic + LLMCompactFloor int + LLMCompactStructuredOnly bool + + // Named component selection. Each list is referenced BY NAME so that a component + // added tomorrow (a new reducer/encoder/extract strategy/stage) is controlled purely + // from config — register it by name in its package, then list it here. An empty list + // means "use all built-in defaults" (no filtering), preserving prior behavior. + // + // Reducers filters internal/reduce reducers by Reducer.Name (e.g. collapse, skeleton, + // format). Encoders filters AND orders the format re-encoders by name (e.g. + // json_compact, toon, jsonl, markdown_kv, tsv, csv). ExtractStrategies restricts the + // internal/extract strategy order (e.g. code, single, rlm, deterministic). Stages + // selects which engine stages run and in what order (reduce, extract, cache). + Reducers []string + Encoders []string + ExtractStrategies []string + Stages []string +} + +// Default is the "balanced" preset: lossless caching + accuracy-safe reduction on, +// LLM extraction off (it needs an injected model client; the proxy enables it). +func Default() Settings { + return Settings{ + CacheEnabled: true, CacheBreakpoints: 4, CacheStableGap: 2, CacheToolsBreakpoint: true, + ReduceEnabled: true, ProtectRecent: 2, ProvableOnly: true, CollapseOutputs: true, + CmdFilter: true, RehydrateOnCompaction: true, + LLMCompactFloor: 3000, LLMCompactStructuredOnly: true, + } +} + +// Preset returns a named settings preset, or Default() for an unknown name. +func Preset(name string) Settings { + s := Default() + switch name { + case "safe": + // Lossless only: no predicted drops, no LLM, caching + lossless re-encode. + s.ProvableOnly = true + s.CollapseOutputs = false + case "balanced", "": + // defaults + case "aggressive": + s.ProvableOnly = false + s.ProtectRecent = 1 + case "cache": + s.ReduceEnabled = false + case "coding", "mcp": + s.ProtectRecentToolUses = 8 + } + return s +} diff --git a/config/file.go b/config/file.go new file mode 100644 index 0000000..fcae72b --- /dev/null +++ b/config/file.go @@ -0,0 +1,154 @@ +package config + +import ( + "bytes" + "fmt" + "os" + + "gopkg.in/yaml.v3" +) + +// fileConfig is the on-disk YAML shape. It is intentionally separate from Settings: +// the file groups knobs under stage sections (reduce/extract/cache) and carries the +// named component-selection lists, which we then fold onto a Settings derived from the +// chosen preset. Pointers distinguish "key absent" (keep preset/default) from "key set +// to the zero value" (an explicit override). +// +// Extending the file is a one-line change: a new reducer/encoder/extract-strategy is +// referenced purely by NAME in the relevant list (reduce.reducers, reduce.encoders, +// extract.strategies) — see configs/lab-cx.yaml. No new struct field is needed to wire +// a newly-registered component on or off. +type fileConfig struct { + Preset string `yaml:"preset"` + Stages []string `yaml:"stages"` + + Reduce *struct { + Enabled *bool `yaml:"enabled"` + ProtectRecent *int `yaml:"protect_recent"` + ProtectRecentToolUses *int `yaml:"protect_recent_tool_uses"` + ProvableOnly *bool `yaml:"provable_only"` + CollapseOutputs *bool `yaml:"collapse_outputs"` + ContextLimit *int `yaml:"context_limit"` + ReduceCachedPrefix *bool `yaml:"reduce_cached_prefix"` + CmdFilter *bool `yaml:"cmd_filter"` + RehydrateOnCompaction *bool `yaml:"rehydrate_on_compaction"` + Reducers []string `yaml:"reducers"` + Encoders []string `yaml:"encoders"` + } `yaml:"reduce"` + + Extract *struct { + Enabled *bool `yaml:"enabled"` + Mode string `yaml:"mode"` + Strategies []string `yaml:"strategies"` + Floor *int `yaml:"floor"` + StructuredOnly *bool `yaml:"structured_only"` + // Provider/model/auth/base are transport concerns the proxy reads off the loaded + // config; the engine core ignores them. They live here so a single file fully + // describes a run. + Provider string `yaml:"provider"` + Model string `yaml:"model"` + Auth string `yaml:"auth"` + Base string `yaml:"base"` + } `yaml:"extract"` + + Cache *struct { + Enabled *bool `yaml:"enabled"` + Breakpoints *int `yaml:"breakpoints"` + StableGap *int `yaml:"stable_gap"` + ToolsBreakpoint *bool `yaml:"tools_breakpoint"` + } `yaml:"cache"` +} + +// Transport holds the extraction model's transport knobs parsed from a config file. +// The engine ignores these; the proxy uses them to construct the cheap-model client. +// Empty fields mean "fall back to flag/default". +type Transport struct { + Provider string + Model string + Auth string + Base string +} + +// Load reads a YAML config file and folds it onto the preset-derived Settings. Unknown +// top-level keys are an error (strict). Missing keys fall back to the preset/defaults. +// It returns the resolved Settings; use LoadWithTransport when the extraction model's +// provider/model/auth/base are also needed. +func Load(path string) (Settings, error) { + s, _, err := LoadWithTransport(path) + return s, err +} + +// LoadWithTransport is Load plus the extraction transport block (provider/model/auth/ +// base) the proxy needs to construct the cheap-model client. +func LoadWithTransport(path string) (Settings, Transport, error) { + raw, err := os.ReadFile(path) + if err != nil { + return Settings{}, Transport{}, fmt.Errorf("config: read %s: %w", path, err) + } + var fc fileConfig + dec := yaml.NewDecoder(bytes.NewReader(raw)) + dec.KnownFields(true) // strict: unknown keys are an error + if err := dec.Decode(&fc); err != nil { + return Settings{}, Transport{}, fmt.Errorf("config: parse %s: %w", path, err) + } + + s := Preset(fc.Preset) + if fc.Stages != nil { + s.Stages = fc.Stages + } + + if r := fc.Reduce; r != nil { + setBool(&s.ReduceEnabled, r.Enabled) + setInt(&s.ProtectRecent, r.ProtectRecent) + setInt(&s.ProtectRecentToolUses, r.ProtectRecentToolUses) + setBool(&s.ProvableOnly, r.ProvableOnly) + setBool(&s.CollapseOutputs, r.CollapseOutputs) + setInt(&s.ContextLimit, r.ContextLimit) + setBool(&s.ReduceCachedPrefix, r.ReduceCachedPrefix) + setBool(&s.CmdFilter, r.CmdFilter) + setBool(&s.RehydrateOnCompaction, r.RehydrateOnCompaction) + if r.Reducers != nil { + s.Reducers = r.Reducers + } + if r.Encoders != nil { + s.Encoders = r.Encoders + } + } + + if e := fc.Extract; e != nil { + setBool(&s.ExtractEnabled, e.Enabled) + if e.Mode != "" { + s.ExtractMode = e.Mode + } + setInt(&s.LLMCompactFloor, e.Floor) + setBool(&s.LLMCompactStructuredOnly, e.StructuredOnly) + if e.Strategies != nil { + s.ExtractStrategies = e.Strategies + } + } + + if c := fc.Cache; c != nil { + setBool(&s.CacheEnabled, c.Enabled) + setInt(&s.CacheBreakpoints, c.Breakpoints) + setInt(&s.CacheStableGap, c.StableGap) + setBool(&s.CacheToolsBreakpoint, c.ToolsBreakpoint) + } + + var tr Transport + if e := fc.Extract; e != nil { + tr = Transport{Provider: e.Provider, Model: e.Model, Auth: e.Auth, Base: e.Base} + } + return s, tr, nil +} + +func setBool(dst *bool, v *bool) { + if v != nil { + *dst = *v + } +} + +func setInt(dst *int, v *int) { + if v != nil { + *dst = *v + } +} diff --git a/config/file_test.go b/config/file_test.go new file mode 100644 index 0000000..6735461 --- /dev/null +++ b/config/file_test.go @@ -0,0 +1,129 @@ +package config + +import ( + "os" + "path/filepath" + "reflect" + "testing" +) + +// writeTmp writes content to a temp YAML file and returns its path. +func writeTmp(t *testing.T, content string) string { + t.Helper() + dir := t.TempDir() + p := filepath.Join(dir, "cfg.yaml") + if err := os.WriteFile(p, []byte(content), 0o644); err != nil { + t.Fatalf("write temp config: %v", err) + } + return p +} + +// TestLoadDisablesExtractAndSelectsComponents is the spec'd acceptance test: a YAML +// that disables extract and enables only the toon encoder + only the collapse reducer +// must produce Settings/Components that reflect exactly that. +func TestLoadDisablesExtractAndSelectsComponents(t *testing.T) { + p := writeTmp(t, ` +preset: balanced +reduce: + reducers: [collapse] + encoders: [toon] +extract: + enabled: false +`) + s, err := Load(p) + if err != nil { + t.Fatalf("Load: %v", err) + } + if s.ExtractEnabled { + t.Errorf("ExtractEnabled = true, want false") + } + if !reflect.DeepEqual(s.Encoders, []string{"toon"}) { + t.Errorf("Encoders = %v, want [toon]", s.Encoders) + } + if !reflect.DeepEqual(s.Reducers, []string{"collapse"}) { + t.Errorf("Reducers = %v, want [collapse]", s.Reducers) + } +} + +// TestLoadFullShape exercises every documented key and the preset base + overrides. +func TestLoadFullShape(t *testing.T) { + p := writeTmp(t, ` +preset: balanced +stages: [reduce, extract, cache] +reduce: + protect_recent: 5 + provable_only: false + reducers: [collapse, skeleton, format, dedup, cmdfilter] + encoders: [json_compact, toon, csv] +extract: + enabled: true + mode: single + strategies: [code, single, deterministic] + floor: 1234 +cache: + enabled: true + breakpoints: 7 +`) + s, err := Load(p) + if err != nil { + t.Fatalf("Load: %v", err) + } + if s.ProtectRecent != 5 { + t.Errorf("ProtectRecent = %d, want 5", s.ProtectRecent) + } + if s.ProvableOnly { + t.Errorf("ProvableOnly = true, want false (override)") + } + if !reflect.DeepEqual(s.Stages, []string{"reduce", "extract", "cache"}) { + t.Errorf("Stages = %v", s.Stages) + } + if !reflect.DeepEqual(s.ExtractStrategies, []string{"code", "single", "deterministic"}) { + t.Errorf("ExtractStrategies = %v", s.ExtractStrategies) + } + if !s.ExtractEnabled { + t.Errorf("ExtractEnabled = false, want true") + } + if s.ExtractMode != "single" { + t.Errorf("ExtractMode = %q, want single", s.ExtractMode) + } + if s.LLMCompactFloor != 1234 { + t.Errorf("LLMCompactFloor = %d, want 1234", s.LLMCompactFloor) + } + if s.CacheBreakpoints != 7 { + t.Errorf("CacheBreakpoints = %d, want 7", s.CacheBreakpoints) + } +} + +// TestLoadDefaultsFromPreset: missing keys fall back to the preset/defaults. +func TestLoadDefaultsFromPreset(t *testing.T) { + p := writeTmp(t, `preset: safe`) + s, err := Load(p) + if err != nil { + t.Fatalf("Load: %v", err) + } + want := Preset("safe") + if s.ProvableOnly != want.ProvableOnly || s.CollapseOutputs != want.CollapseOutputs { + t.Errorf("safe preset not applied: got ProvableOnly=%v CollapseOutputs=%v", + s.ProvableOnly, s.CollapseOutputs) + } + if s.CacheBreakpoints != want.CacheBreakpoints { + t.Errorf("CacheBreakpoints = %d, want preset %d", s.CacheBreakpoints, want.CacheBreakpoints) + } +} + +// TestLoadStrictUnknownKey: an unknown top-level key is an error. +func TestLoadStrictUnknownKey(t *testing.T) { + p := writeTmp(t, ` +preset: balanced +bogus_key: 1 +`) + if _, err := Load(p); err == nil { + t.Fatal("expected error on unknown top-level key, got nil") + } +} + +func TestLoadMissingFile(t *testing.T) { + if _, err := Load("/no/such/file.yaml"); err == nil { + t.Fatal("expected error for missing file") + } +} diff --git a/configs/lab-cx.yaml b/configs/lab-cx.yaml new file mode 100644 index 0000000..ba38b18 --- /dev/null +++ b/configs/lab-cx.yaml @@ -0,0 +1,49 @@ +# lab-cx example config — pass with: lab-cx proxy --config configs/lab-cx.yaml +# +# This file controls which components run and in what order, purely by NAME. It folds +# onto a base `preset`, then applies the overrides below. Unknown top-level keys are a +# hard error (strict). Any key you omit falls back to the preset/defaults. +# +# ── HOW TO ADD A NEW COMPONENT (one-line config, no core edit needed) ─────────────── +# • new ENCODER : register it in internal/reduce/actions.go `allEncoders` +# (unique name + rank), then list its name under `reduce.encoders`. +# • new REDUCER : register it via reduce.RegisterReducer (its Reducer.Name), then +# list that name under `reduce.reducers`. +# • new EXTRACT STRATEGY : add it to internal/extract `RunExtraction`'s switch + +# `rawStrategyOrder`, then list its name under `extract.strategies`. +# • new STAGE : register it in engine `builtinStages`, then list it under `stages`. +# Each list is referenced by NAME; enabling/disabling/ordering is pure config. +# An empty or omitted list means "use all built-in defaults". +# ───────────────────────────────────────────────────────────────────────────────────── + +preset: balanced # base defaults: safe|balanced|aggressive|cache|coding|mcp + +stages: [reduce, extract, cache] # which engine stages run, in order + +reduce: + protect_recent: 2 # keep the N most recent turns at full fidelity + provable_only: true # never drop merely-predicted-unused content + # Enabled reducers, by Reducer.Name. Built-ins: collapse, skeleton, format. + # (cmdfilter and dedup are separate pipeline passes toggled by their own keys below.) + reducers: [collapse, skeleton, format] + cmd_filter: true # dedup/trim noisy shell command output + # Format re-encoders, by name — this list also sets PRIORITY (first listed wins ties). + # Built-ins: json_compact, toon, jsonl, markdown_kv, tsv, csv. + encoders: [json_compact, toon, jsonl, markdown_kv, tsv, csv] + +extract: + enabled: true # cheap-model projection of large structured outputs + mode: auto # auto | single | rlm | deterministic + # Allowed strategies, by name; intersected with the computed order. Built-ins: + # code, single, rlm, deterministic. + strategies: [code, single, rlm, deterministic] + floor: 3000 # token floor before extraction is considered + # Extraction model transport (read by the proxy; --extract-* flags override these): + provider: anthropic # anthropic | openai + model: claude-haiku-4-5 + auth: x-api-key # x-api-key | bearer (bearer for gateway endpoints) + # base: https://gateway.example/v1 # optional: override the model base URL + +cache: + enabled: true + breakpoints: 4 # ephemeral cache_control breakpoints on the stable prefix diff --git a/deploy/eval-containers/README.md b/deploy/eval-containers/README.md new file mode 100644 index 0000000..b593bda --- /dev/null +++ b/deploy/eval-containers/README.md @@ -0,0 +1,54 @@ +# Running lab-cx inside eval-containers + +[eval-containers](https://github.com/exgentic/eval-containers) runs `one benchmark + +one agent + one model`, with the agent's LLM traffic always flowing through a +**gateway** (bifrost/litellm) on port 4000. `lab-cx` slots into that request path as a +plain HTTP proxy — no changes to the agent images, no changes to the gateway. + +The image is built from this repo's root `Dockerfile`: + +```sh +docker build -t ghcr.io/kagenti/lab-context-engineering:latest . +``` + +## Placement A — before the gateway (recommended) + +``` +runner (agent) ──▶ lab-cx ──▶ gateway ──▶ provider +``` + +lab-cx sees what the agent sends, in its native protocol, before any model-name +rewrite — the right place to control the agent's context. Use +[`compose.override.yaml`](compose.override.yaml): + +```sh +docker compose -f compose.yaml -f .../deploy/eval-containers/compose.override.yaml up \ + -y --abort-on-container-exit +``` + +It adds a `winnow` service on the `internal` network, repoints the runner's +`ANTHROPIC_BASE_URL` / `OPENAI_BASE_URL` / `GOOGLE_GEMINI_BASE_URL` at it, and sets +`--upstream http://gateway:4000` so lab-cx forwards the full (prefixed) path on to the +gateway. The agent still holds only `sk-proxy`; the real key stays in the gateway. + +## Placement B — after the gateway + +``` +runner (agent) ──▶ gateway ──▶ lab-cx ──▶ provider +``` + +lab-cx sees normalized (OpenAI-shaped) traffic. Point the gateway's `OPENAI_API_BASE` +at lab-cx (`http://winnow:8080`) and put `winnow` on the `upstream` network, with +`--upstream` set to the real provider base. + +## Notes + +- **Suffix routing**: lab-cx reduces any path ending in `/v1/messages` or + `/chat/completions`, so the eval prefixes (`/anthropic`, `/openai/v1`) work + unchanged. The Gemini prefix (`/genai`) is forwarded untouched (fail-open) until the + Gemini surface lands. +- **Isolation/cost invariants are preserved**: lab-cx adds no egress (before-gateway it + lives on `internal` only), doesn't touch `sk-proxy`/`TASK_ID`, and the gateway keeps + emitting OTel + enforcing `EVAL_MODEL_MAX_BUDGET`. +- **Measuring the effect**: compare `trajectory.jsonl` / `result.json` token + cost + totals with and without the override on the same `EVAL_TASK_ID`. diff --git a/deploy/eval-containers/compose.override.yaml b/deploy/eval-containers/compose.override.yaml new file mode 100644 index 0000000..b8073e8 --- /dev/null +++ b/deploy/eval-containers/compose.override.yaml @@ -0,0 +1,29 @@ +# Drop-in override that inserts lab-cx BEFORE the eval-containers gateway, so it sees +# exactly what the agent sends (native protocol, pre model-name rewrite) and controls +# the context before it reaches the gateway: +# +# runner (agent) ──▶ lab-cx ──▶ gateway ──▶ provider +# +# Usage (from an eval-containers benchmark dir): +# docker compose -f compose.yaml -f /path/to/this/compose.override.yaml up ... +# +# It repoints the runner's *_BASE_URL at lab-cx and points lab-cx at the gateway. +# Nothing about the agent images or the gateway changes. For the AFTER-gateway +# placement instead, see README.md in this directory. + +services: + winnow: + image: ghcr.io/kagenti/lab-context-engineering:latest + command: ["proxy", "--addr", ":8080", "--preset", "balanced", "--upstream", "http://gateway:4000"] + networks: + - internal + + runner: + environment: + # Route the agent through lab-cx; it forwards the (full, prefixed) path to the + # gateway. Suffix routing means /anthropic/v1/messages etc. are still reduced. + ANTHROPIC_BASE_URL: http://winnow:8080/anthropic + OPENAI_BASE_URL: http://winnow:8080/openai/v1 + GOOGLE_GEMINI_BASE_URL: http://winnow:8080/genai + depends_on: + - winnow diff --git a/docs/RESULTS-extract.md b/docs/RESULTS-extract.md new file mode 100644 index 0000000..f32b6a8 --- /dev/null +++ b/docs/RESULTS-extract.md @@ -0,0 +1,89 @@ +# Real cheap-model extractor results (ONLINE, claude-haiku-4-5) + +- **Date measured:** 2026-06-24 +- **Mode:** ONLINE. The cheap-model **Extract** stage is **ON** for every row. Each row is + the result of a REAL call to a live model gateway — there is no mock and no canned + output anywhere in this measurement. +- **Model:** `claude-haiku-4-5`, served via an Anthropic-compatible LiteLLM gateway + (`ANTHROPIC_BASE_URL`), bearer-authenticated (`cheapmodel.Anthropic{AuthScheme: "bearer"}`). + The token is never printed or committed. +- **`LLMCompactFloor` used:** **200** tokens. The production default is 3000. The committed + structured fixtures are only a few hundred tokens each, so the floor is lowered to 200 + **purely for this measurement** so that extraction actually fires on the corpus. A + fixture under 200 tokens is below the floor and never becomes an extraction candidate — + it shows up honestly as `strategy=none`, `0% saved`. This is a measurement-only knob and + changes no default. +- **`ProtectRecent`:** 1 (the goal turn is the only protected trailing turn; the + `tool_result` sits outside the protect window so it is eligible). +- **Latency:** `latency_ms` is **wall-clock of the full `engine.Transform`, including the + real network round-trip to the gateway model** (plus, for the `code` strategy, local + Starlark execution of the model-written filter). It is therefore dominated by network + + inference time, not engine overhead. +- **Tokenizer:** `o200k_base` via `internal/tokens` — the same counter the engine uses to + gate reductions. +- **Fixtures:** the large/structured corpus only — the `structured_json/` and + `search_results/` record-shaped fixtures under `testdata/fixtures/`. Each is surfaced as + a `tool_result` paired with a realistic recent goal that names specific records (so the + keep-set the model filters toward is concrete and realistic). Provenance: + `testdata/fixtures/README.md`. + +## Reproduce + +```sh +# from the repo root; loads ANTHROPIC_BASE_URL + ANTHROPIC_AUTH_TOKEN + the haiku model id +source /tmp/lcx_env.sh +CGO_ENABLED=1 go run ./cmd/labcx-bench --extract # the table below +CGO_ENABLED=1 go run ./cmd/labcx-bench --extract --json # machine-readable rows +``` + +`--extract` requires `ANTHROPIC_BASE_URL` and `ANTHROPIC_AUTH_TOKEN` in the environment; +it fails fast if they are unset. Without `--extract` the harness stays fully offline and +deterministic (see `RESULTS-offline.md`) — no model is called. + +## Results + +These are the exact rows printed by the command above on 2026-06-24. They are not edited. +Because they come from a live model, exact `%saved`, `strategy`, and `latency_ms` will vary +run to run (the model is non-deterministic); the qualitative outcome — which fixtures shrink +and which decline — is stable. + +| fixture | tokens_before | tokens_after | %saved | strategy | contained | latency_ms | model | +| --- | ---: | ---: | ---: | :---: | :---: | ---: | :--- | +| flights_search | 607 | 122 | 79.90 | code | yes | 1921 | claude-haiku-4-5 | +| users_directory | 194 | 194 | 0.00 | none | no | 1 | claude-haiku-4-5 | +| products_inventory | 254 | 93 | 63.39 | code | yes | 2492 | claude-haiku-4-5 | +| oc_pods_slice | 811 | 258 | 68.19 | single | yes | 8533 | claude-haiku-4-5 | +| glab_issue_list | 2106 | 2106 | 0.00 | none | no | 3138 | claude-haiku-4-5 | +| glab_mr_list | 2934 | 1270 | 56.71 | deterministic | yes | 8081 | claude-haiku-4-5 | + +## Analysis (honest) + +**Shrank (4 of 6).** The extractor reduced four fixtures, each via a different strategy: + +- `flights_search` **−79.9%** and `products_inventory` **−63.4%** via the **code** strategy + — the model wrote a Starlark filter that ran locally over the *full* body and kept only + the records named in the goal (FL003/FL004; SKU0001/SKU0004). +- `oc_pods_slice` **−68.2%** via the **single** strategy — a one-shot JSON-return filter + keeping the pod(s) the goal asked about. +- `glab_mr_list` **−56.7%** via the **deterministic** strategy — the model strategies did + not beat the deterministic projection here, so the ordered fallback (a deterministic, + keep-set-driven projection) won. This is still a real run: the model strategies were + attempted first and declined, and the safe deterministic fallback produced the kept set. + +**Declined (2 of 6) — both shown honestly as `strategy=none`, `0% saved`:** + +- `users_directory` (194 tokens) is **below the 200-token floor**, so it never became an + extraction candidate. The 1 ms latency reflects that no model call was made. This is the + floor working as designed, not a model failure. +- `glab_issue_list` (2106 tokens) **did** qualify (above the floor) and the model was + called, but no strategy produced a result that was both strictly smaller AND passed the + containment/validation gate, so the engine kept the original untouched. A decline is a + valid, **safe** outcome: fail-open means the original is forwarded unchanged. + +**Containment / reversibility.** Every spliced result is marked `contained=yes`, verified +by the harness via the public `engine.FindMarkers` + `engine.Expand` round-trip: each +spliced block carries a recovery marker whose stored original is a substring of the +pre-extraction body. So every reduction here is **lossless and reversible** — the engine +only splices a result it can fully recover from its rewind store, and only when the result +is strictly smaller (it never inflates). Declined fixtures show `contained=no` simply +because nothing was spliced (the original is intact). diff --git a/docs/RESULTS-offline.md b/docs/RESULTS-offline.md new file mode 100644 index 0000000..1bb6aba --- /dev/null +++ b/docs/RESULTS-offline.md @@ -0,0 +1,144 @@ +# Offline measurement results (deterministic, no model) + +- **Date measured:** 2026-06-24 +- **Mode:** OFFLINE / deterministic only. The cheap-model Extract stage is **OFF** for + every row here; nothing in this document calls a model. Token reduction comes entirely + from the deterministic components (command-output filter, format re-encoder, code + skeletonizer, reversible collapse) and the full deterministic pipeline. +- **Tokenizer:** `o200k_base` (the modern GPT-family BPE), via `internal/tokens` — the + same counter the engine uses to gate reductions. Counts are real BPE token counts, not + a chars/4 estimate. +- **Fixtures:** 10 real tool outputs committed under `testdata/fixtures/` (rtk command + outputs + winnow structured tool outputs). Provenance for every file is in + `testdata/fixtures/README.md`. + +## Reproduce + +```sh +# from the repo root +CGO_ENABLED=1 go run ./cmd/labcx-bench # the table below +CGO_ENABLED=1 go run ./cmd/labcx-bench --json # machine-readable rows +``` + +The harness wraps each fixture as a canonical Anthropic-shaped request (a `tool_result` +block, paired with the `tool_use` that produced it so command/file-aware passes can see +the command or path), places it outside the protect-recent window, and runs the engine +with **one component enabled at a time** plus the **full deterministic pipeline**. For +every row it verifies the reduction is **reversible** — either it is a no-op, or the +original is recoverable from the rewind store via `engine.FindMarkers` + `engine.Expand`. + +## Measured table (verbatim harness output) + +The numbers below are pasted directly from `go run ./cmd/labcx-bench`. Rows tagged +`(no-op)` are components that did not apply to that fixture (e.g. the skeletonizer on a +JSON array); they are kept in the table to show the fail-safe — a component never inflates +a fixture it does not understand. + +| fixture | component | tokens_before | tokens_after | %saved | bytes_before | bytes_after | reversible | added_latency_ms | +| --- | --- | ---: | ---: | ---: | ---: | ---: | :---: | ---: | +| pytest_failures | cmdfilter (no-op) | 94 | 94 | 0.00 | 324 | 324 | yes | 13.065 | +| pytest_failures | format_toon (no-op) | 94 | 94 | 0.00 | 324 | 324 | yes | 0.625 | +| pytest_failures | format_best (no-op) | 94 | 94 | 0.00 | 324 | 324 | yes | 0.615 | +| pytest_failures | skeleton (no-op) | 94 | 94 | 0.00 | 324 | 324 | yes | 0.521 | +| pytest_failures | collapse (no-op) | 94 | 94 | 0.00 | 324 | 324 | yes | 0.520 | +| pytest_failures | pipeline_full (no-op) | 94 | 94 | 0.00 | 324 | 324 | yes | 0.979 | +| cargo_test_failure | cmdfilter (no-op) | 95 | 95 | 0.00 | 292 | 292 | yes | 0.983 | +| cargo_test_failure | format_toon (no-op) | 95 | 95 | 0.00 | 292 | 292 | yes | 0.569 | +| cargo_test_failure | format_best (no-op) | 95 | 95 | 0.00 | 292 | 292 | yes | 0.568 | +| cargo_test_failure | skeleton (no-op) | 95 | 95 | 0.00 | 292 | 292 | yes | 0.500 | +| cargo_test_failure | collapse (no-op) | 95 | 95 | 0.00 | 292 | 292 | yes | 0.484 | +| cargo_test_failure | pipeline_full (no-op) | 95 | 95 | 0.00 | 292 | 292 | yes | 0.949 | +| cargo_build | cmdfilter | 2687 | 155 | 94.23 | 6743 | 409 | yes | 11.652 | +| cargo_build | format_toon (no-op) | 2687 | 2687 | 0.00 | 6743 | 6743 | yes | 7.597 | +| cargo_build | format_best (no-op) | 2687 | 2687 | 0.00 | 6743 | 6743 | yes | 7.866 | +| cargo_build | skeleton (no-op) | 2687 | 2687 | 0.00 | 6743 | 6743 | yes | 7.736 | +| cargo_build | collapse | 2687 | 54 | 97.99 | 6743 | 133 | yes | 7.667 | +| cargo_build | pipeline_full | 2687 | 155 | 94.23 | 6743 | 409 | yes | 10.966 | +| flights_search | cmdfilter (no-op) | 607 | 607 | 0.00 | 1356 | 1356 | yes | 2.349 | +| flights_search | format_toon | 607 | 291 | 52.06 | 1356 | 535 | yes | 3.353 | +| flights_search | format_best | 607 | 263 | 56.67 | 1356 | 504 | yes | 4.178 | +| flights_search | skeleton (no-op) | 607 | 607 | 0.00 | 1356 | 1356 | yes | 2.588 | +| flights_search | collapse | 607 | 55 | 90.94 | 1356 | 133 | yes | 2.282 | +| flights_search | pipeline_full | 607 | 55 | 90.94 | 1356 | 133 | yes | 2.285 | +| users_directory | cmdfilter (no-op) | 194 | 194 | 0.00 | 533 | 533 | yes | 0.855 | +| users_directory | format_toon | 194 | 140 | 27.84 | 533 | 382 | yes | 1.228 | +| users_directory | format_best | 194 | 124 | 36.08 | 533 | 362 | yes | 1.577 | +| users_directory | skeleton (no-op) | 194 | 194 | 0.00 | 533 | 533 | yes | 0.860 | +| users_directory | collapse (no-op) | 194 | 194 | 0.00 | 533 | 533 | yes | 0.831 | +| users_directory | pipeline_full | 194 | 124 | 36.08 | 533 | 362 | yes | 2.756 | +| products_inventory | cmdfilter (no-op) | 254 | 254 | 0.00 | 643 | 643 | yes | 1.140 | +| products_inventory | format_toon | 254 | 147 | 42.13 | 643 | 345 | yes | 1.677 | +| products_inventory | format_best | 254 | 136 | 46.46 | 643 | 323 | yes | 1.998 | +| products_inventory | skeleton (no-op) | 254 | 254 | 0.00 | 643 | 643 | yes | 1.097 | +| products_inventory | collapse | 254 | 50 | 80.31 | 643 | 133 | yes | 1.124 | +| products_inventory | pipeline_full | 254 | 50 | 80.31 | 643 | 133 | yes | 1.150 | +| oc_pods_slice | cmdfilter (no-op) | 811 | 811 | 0.00 | 3216 | 3216 | yes | 3.836 | +| oc_pods_slice | format_toon | 811 | 511 | 36.99 | 3216 | 1791 | yes | 5.226 | +| oc_pods_slice | format_best | 811 | 511 | 36.99 | 3216 | 1791 | yes | 5.721 | +| oc_pods_slice | skeleton (no-op) | 811 | 811 | 0.00 | 3216 | 3216 | yes | 3.837 | +| oc_pods_slice | collapse | 811 | 50 | 93.83 | 3216 | 133 | yes | 3.208 | +| oc_pods_slice | pipeline_full | 811 | 50 | 93.83 | 3216 | 133 | yes | 3.556 | +| glab_issue_list | cmdfilter (no-op) | 2106 | 2106 | 0.00 | 6748 | 6748 | yes | 8.076 | +| glab_issue_list | format_toon | 2106 | 1992 | 5.41 | 6748 | 6482 | yes | 14.328 | +| glab_issue_list | format_best | 2106 | 1799 | 14.58 | 6748 | 6180 | yes | 17.478 | +| glab_issue_list | skeleton (no-op) | 2106 | 2106 | 0.00 | 6748 | 6748 | yes | 7.020 | +| glab_issue_list | collapse | 2106 | 54 | 97.44 | 6748 | 133 | yes | 6.747 | +| glab_issue_list | pipeline_full | 2106 | 54 | 97.44 | 6748 | 133 | yes | 7.337 | +| glab_mr_list | cmdfilter (no-op) | 2934 | 2934 | 0.00 | 9463 | 9463 | yes | 10.454 | +| glab_mr_list | format_toon | 2934 | 2713 | 7.53 | 9463 | 9004 | yes | 19.472 | +| glab_mr_list | format_best | 2934 | 2417 | 17.62 | 9463 | 8483 | yes | 22.569 | +| glab_mr_list | skeleton (no-op) | 2934 | 2934 | 0.00 | 9463 | 9463 | yes | 9.973 | +| glab_mr_list | collapse | 2934 | 50 | 98.30 | 9463 | 133 | yes | 9.687 | +| glab_mr_list | pipeline_full | 2934 | 50 | 98.30 | 9463 | 133 | yes | 9.715 | +| runner_py | cmdfilter (no-op) | 1401 | 1401 | 0.00 | 5399 | 5399 | yes | 7.418 | +| runner_py | format_toon (no-op) | 1401 | 1401 | 0.00 | 5399 | 5399 | yes | 6.184 | +| runner_py | format_best (no-op) | 1401 | 1401 | 0.00 | 5399 | 5399 | yes | 6.681 | +| runner_py | skeleton | 1401 | 303 | 78.37 | 5399 | 1099 | yes | 8.803 | +| runner_py | collapse | 1401 | 59 | 95.79 | 5399 | 162 | yes | 6.218 | +| runner_py | pipeline_full | 1401 | 59 | 95.79 | 5399 | 162 | yes | 6.271 | + +(The ~13 ms on the first row is the one-time lazy initialization of the BPE tokenizer +amortized into that measurement; steady-state per-fixture latency is sub-millisecond to a +few milliseconds, scaling with input size.) + +## What this shows + +Each deterministic component reduces tokens on exactly the fixture types it targets, and +is a safe no-op everywhere else: + +- **Command-output filter (`cmdfilter`)** is the biggest single-component win on verbose + CLI noise: on a real `cargo build` of rtk's 203-crate dependency graph it cut **94.2%** + of tokens (2687 → 155) by dropping the `Compiling …` chatter while keeping the + `Finished` line — reversibly. It is a deliberate **no-op** on the two small pytest/cargo + fixtures: those are mostly failure/summary signal already, and below the size where the + recovery-marker overhead is worth paying, so the filter declines to touch them (it never + inflates an output). + +- **Format re-encoder** is the right tool for structured tool/MCP outputs. On flat + uniform record arrays (`flights_search`, `users_directory`, `products_inventory`) the + best-encoding mode saved **36–57%** (TOON alone **28–52%**) by switching JSON to a + delimited/TOON table — fully lossless, the data is identical. On nested record arrays + (`glab_issue_list`, `glab_mr_list`, `oc_pods_slice`) the savings are smaller + (**15–37%**) because the nesting limits the table encoders, but it still helps and never + hurts. + +- **Code skeletonizer** targets file reads: on the real Python source `runner.py` it + dropped function bodies for **78.4%** token savings, keeping every signature. + +- **Reversible collapse** is the universal fallback for an *unused* tool output (one whose + content is not referenced later): it replaces the body with a recovery marker for + **80–98%** savings. It is the strongest per-fixture reducer but the most aggressive — it + hides content behind a marker rather than re-expressing it — so the pipeline prefers a + signal-preserving reducer when one applies (see below). + +- **Full deterministic pipeline** combines them in the engine's real order. Across all 10 + fixtures it took **11,183 → 786 tokens, a 93.0% reduction**, all reversible. Note the + pipeline does **not** simply take the max of the columns: on `cargo_build` it reports + 94.2% (the cmdfilter result, 155 tokens) rather than collapse's 98.0% (54 tokens), + because cmdfilter runs first as a pre-pass and keeps the failure/summary signal a model + would actually need — the engine intentionally trades a few tokens for fidelity. + +The headline: **cmdfilter −94% on verbose command output, format re-encode −35% (best) / +−29% (TOON) on structured outputs, skeleton −78% on a code read, and the full +deterministic pipeline −93% aggregate across 10 real fixtures — every reduction +reversible, with no model in the loop.** diff --git a/docs/RESULTS.md b/docs/RESULTS.md new file mode 100644 index 0000000..65fc558 --- /dev/null +++ b/docs/RESULTS.md @@ -0,0 +1,61 @@ +# Results index + +Every number here is real and reproducible — links to the source docs that hold the +verbatim harness output. No numbers are invented; this page only indexes them. + +## Offline deterministic components + +- **Headline:** `cmdfilter` **−94%** on verbose command output, `format` re-encode + **−35%** best / **−29%** TOON on structured output, `skeleton` **−78%** on a code read, + full deterministic pipeline **−93%** aggregate across 10 real fixtures — every reduction + reversible, no model in the loop. +- **Date:** 2026-06-24. **How produced:** `CGO_ENABLED=1 go run ./cmd/labcx-bench` over + the committed `testdata/fixtures/` corpus, `o200k_base` token counts, reversibility + verified per row. +- **Full table + analysis:** [RESULTS-offline.md](RESULTS-offline.md). + +## Cheap-model extractor (online) + +- **Headline:** `claude-haiku-4-5` extractor reduced **4 of 6** structured fixtures + **−56%…−80%** (via `code`/`single`/`deterministic` strategies), with **2 honest + declines** shown as `strategy=none`; every spliced result is `contained=yes` (lossless + + reversible). +- **Date:** 2026-06-24. **How produced:** `CGO_ENABLED=1 go run ./cmd/labcx-bench + --extract` against a live Anthropic-compatible gateway (`LLMCompactFloor=200` for the + small fixtures), real network round-trips, no mocks. +- **Full table + honest analysis:** [RESULTS-extract.md](RESULTS-extract.md). + +## Claude Code integration (real runs) + +- **Headline (real savings):** on a large-file task with `reduce_cached_prefix`, **−50.2% + tokens** (69,683 → 34,729; 34,954 saved; ~$0.035), and the answer was **correct** (42 + top-level funcs — exact — plus an accurate summary). `skeleton` dropped function bodies + while keeping signatures, so the task was preserved → real savings, **no harm**. +- **Headline (trivial task):** `cache_injected: 0` / `tokens_saved: 0` — **do-no-harm on a + self-caching client, as designed** (lab-cx stands down rather than fight Claude Code's + own `cache_control`). +- **Date:** 2026-06-24. **How produced:** `claude -p` routed through `lab-cx proxy` via a + settings file, reading `/stats` (`scripts/cc-demo.sh` for the trivial run). +- **Detail:** [integration/claude-code.md](integration/claude-code.md). + +## Bob (IBM Bob Shell) integration (real run) + +- **Headline:** **5/5 verifiable tasks correct** through the proxy (incl. a file edit and + a 300-record JSON analysis); `cache_injected` on **every** request (Bob is not + self-caching, so the cache lever fires — the one Claude Code declines); 0 stage errors, + ~3ms p50 added latency. Content reduction was 0 because Bob condensed files itself; + content-reduction value is shown by the component results above and the Claude Code + large-file run. +- **Date:** 2026-06-24. **How produced:** real `bob` CLI with `CUSTOM_BASE_URL` → proxy → + Bob backend; `/stats` after a 3-task suite. +- **Detail:** [integration/bob.md](integration/bob.md). + +## SWE-bench / eval-containers integration + +- **Headline:** wiring **validated** — the merged compose renders `winnow` before the + gateway and the lab-cx image builds (CGO, distroless/base). The **full SWE-bench run was + NOT executed** this session (multi-GB pulls + real model spend); the doc is the runbook + to execute, with no fabricated benchmark numbers. +- **Date:** 2026-06-24. **How produced:** structural `docker compose ... config` merge + validation (no image pull). +- **Detail + runbook:** [integration/swe-bench.md](integration/swe-bench.md). diff --git a/docs/RUNNING.md b/docs/RUNNING.md new file mode 100644 index 0000000..9d7f52a --- /dev/null +++ b/docs/RUNNING.md @@ -0,0 +1,123 @@ +# Running and measuring lab-cx + +How to run each component and how to measure it. Every command here is real against the +current flags/targets. CGO is required throughout (`CGO_ENABLED=1`) — the skeletonizer +uses tree-sitter via cgo. + +## Build + +```sh +make build # ./bin/lab-cx (exports CGO_ENABLED=1) +./bin/lab-cx version +``` + +## Run the proxy + +```sh +# preset mode (default :8080), routes by provider +./bin/lab-cx proxy --preset balanced + +# config mode, forwarding all traffic to one upstream gateway, extractor on +./bin/lab-cx proxy --config configs/lab-cx.yaml \ + --upstream https://gateway.example/v1 \ + --extract-model claude-haiku-4-5 --extract-provider anthropic --extract-auth bearer \ + --extract-base https://gateway.example/v1 +``` + +Point an agent at it: + +```sh +ANTHROPIC_BASE_URL=http://localhost:8080 # Anthropic-wire agents +OPENAI_BASE_URL=http://localhost:8080/v1 # OpenAI-wire agents +``` + +Flags: `--addr`, `--preset` (`safe|balanced|aggressive|cache|coding|mcp`), `--config`, +`--upstream`, `--extract-model/-provider/-auth/-base`, `--max-body-bytes` (default +33554432 = 32 MiB; `0` = no cap), `--upstream-timeout` (default `0s`; non-zero caps the +whole request incl. streamed responses). + +Extractor API key comes from the env: `WINNOW_EXTRACT_KEY`, else `ANTHROPIC_API_KEY` / +`ANTHROPIC_AUTH_TOKEN` (anthropic provider) or `OPENAI_API_KEY` (openai provider). + +Disable / bypass: + +- `WINNOW_DISABLE=1` — transparent passthrough for the whole process. +- `x-winnow-bypass: true` header — skip reduction for one request. + +## Read /stats + +```sh +curl -s http://localhost:8080/stats # raw JSON +./bin/lab-cx stats # JSON + one-line summary (default addr) +./bin/lab-cx stats --addr http://localhost:8080 +``` + +`/stats` fields: `requests`, `tokens_before`, `tokens_after`, `tokens_saved`, +`cache_injected`, `extracted`, `stage_errors`, `added_latency_p50_ms`, +`added_latency_p95_ms`. Recover an omitted block: `GET /winnow/expand?id=`. + +## Measure: offline deterministic harness + +No model in the loop — proves each deterministic component on real fixtures under +`testdata/fixtures/`. Token counts use the same `o200k_base` BPE counter the engine uses. + +```sh +CGO_ENABLED=1 go run ./cmd/labcx-bench # human-readable Markdown table +CGO_ENABLED=1 go run ./cmd/labcx-bench --json # machine-readable rows +``` + +It wraps each fixture as a canonical Anthropic-shaped request, runs the engine with one +component enabled at a time plus the full deterministic pipeline, and verifies every row +is reversible. Recorded run: [RESULTS-offline.md](RESULTS-offline.md). + +## Measure: cheap-model extractor harness (online) + +`--extract` flips the harness into ONLINE mode against a real Anthropic-compatible +gateway (`claude-haiku-4-5`, bearer auth). Requires the gateway env: + +```sh +export ANTHROPIC_BASE_URL=https://gateway.example/v1 # the model gateway +export ANTHROPIC_AUTH_TOKEN= # never printed/committed +CGO_ENABLED=1 go run ./cmd/labcx-bench --extract # Markdown table +CGO_ENABLED=1 go run ./cmd/labcx-bench --extract --json # machine-readable rows +``` + +It fails fast if `ANTHROPIC_BASE_URL` / `ANTHROPIC_AUTH_TOKEN` are unset. Recorded run: +[RESULTS-extract.md](RESULTS-extract.md). + +## Run the Claude Code demo + +```sh +ANTHROPIC_BASE_URL=https://gateway.example/v1 ANTHROPIC_AUTH_TOKEN= \ + scripts/cc-demo.sh +``` + +Starts the proxy with the extractor on, routes a real `claude -p` task through it via a +settings file (so Claude Code's `env.ANTHROPIC_BASE_URL` precedence is handled), and +prints `/stats` before and after. Detail + measured result: +[integration/claude-code.md](integration/claude-code.md). + +## Config-extension guide + +`configs/lab-cx.yaml` selects components purely by **name**. Each list folds onto the base +`preset`; an empty/omitted list means "all built-in defaults"; list order sets +priority/run order. To add a component, register it once in its registry, then list its +name in the config — no other core edit needed. + +| Component | Register it in | Then list it under | +|---|---|---| +| **Stage** | `engine` `builtinStages` map (`engine/engine.go`) | `stages:` | +| **Reducer** | `reduce.RegisterReducer(...)` (its `Reducer.Name`) | `reduce.reducers:` | +| **Encoder** (format re-encoder) | `allEncoders` table in `internal/reduce/actions.go` (unique name + rank) | `reduce.encoders:` (order = tie-break priority) | +| **Extract strategy** | `RunExtraction`'s switch + `rawStrategyOrder` in `internal/extract/extract.go` | `extract.strategies:` | + +Built-ins: stages `reduce, extract, cache`; reducers `collapse, skeleton, format` +(`cmdfilter` and `dedup` are separate passes toggled by their own keys); encoders +`json_compact, toon, jsonl, markdown_kv, tsv, csv`; strategies +`code, single, rlm, deterministic`. + +Other `reduce` keys: `protect_recent` (keep N most recent turns at full fidelity), +`provable_only` (never drop merely-predicted-unused content), `cmd_filter` (bool). +`extract` keys: `enabled`, `mode` (`auto|single|rlm|deterministic`), `floor` (token floor +before extraction is considered), and the transport block (`provider`, `model`, `auth`, +`base`) which the `--extract-*` flags override. `cache` keys: `enabled`, `breakpoints`. diff --git a/docs/integration/authbridge.md b/docs/integration/authbridge.md new file mode 100644 index 0000000..132e826 --- /dev/null +++ b/docs/integration/authbridge.md @@ -0,0 +1,69 @@ +# Integrating with Kagenti AuthBridge + +The AuthBridge plugin lives in +[kagenti-extensions](https://github.com/kagenti/kagenti-extensions), **not** in this +repo. It depends on this module and wraps the engine in-process (the `sparc` plugin is +the structural precedent: a thin pipeline plugin delegating to a focused capability). +This repo's job is to expose a clean, importable Go API; it carries no AuthBridge code. + +## The contract + +```go +import ( + "github.com/kagenti/lab-context-engineering/config" + "github.com/kagenti/lab-context-engineering/engine" + "github.com/kagenti/lab-context-engineering/surfaces" +) + +// Construct once (e.g. in the plugin's Init), reuse across requests. +eng := engine.New(config.Default(), nil, nil) // nil → in-memory rewind store + evictions +``` + +Per request, on the **outbound** LLM call (AuthBridge's `OnRequest` for an +`inference`-classified request), feed the body you already have through the matching +surface, transform, and write the rendered bytes back with `pctx.SetBody`: + +```go +surface := surfaces.Anthropic{} // or surfaces.OpenAI{} per the detected wire format +req, token, err := surface.ToInternal(pctx.Body) +if err == surfaces.ErrUnsupported || err != nil { + return // fail open: leave pctx.Body untouched +} +reduced, _ := eng.Transform(ctx, req) // never errors; worst case is a no-op +out, err := surface.Render(reduced, token) +if err == nil { + pctx.SetBody(out) // the only mutation AuthBridge needs +} +``` + +Declare `Capabilities{ReadsBody: true, WritesBody: true}` and +`RequiresAny: ["inference-parser"]` so a parser runs first. Ship it `on_error: observe` +to roll out in shadow mode. + +## Recovering omitted content + +Reductions are reversible. To serve an expand request, or to rehydrate before the +agent's own summarization turn: + +```go +for _, id := range engine.FindMarkers(text) { + if original, ok := eng.Expand(id); ok { + // splice original back in + } +} +``` + +## Notes + +- **Fail open is the contract.** `Transform` never returns an error; on any internal + fault it forwards the request unchanged. `ToInternal` returning `ErrUnsupported` + (e.g. Gemini today) also means "forward untouched". +- **Why raw bytes, not parsed inference?** Feeding `pctx.Body` is the simplest path and + avoids coupling to a parser's types. A `canon.Request` constructor from + already-parsed messages can be added later if double-parsing ever shows up in a + profile — it does not today. +- **State.** The default rewind store is in-memory (per process). For multi-replica + recovery, supply a shared `store.Rewind` to `engine.New` (Redis/SQLite backends are a + planned addition). +- **Observability.** Emit the engine's `Report` (tokens before/after/saved) via + AuthBridge's OpenTelemetry GenAI conventions. diff --git a/docs/integration/bob.md b/docs/integration/bob.md new file mode 100644 index 0000000..513490a --- /dev/null +++ b/docs/integration/bob.md @@ -0,0 +1,84 @@ +# Integrating with Bob (IBM Bob Shell) — real run + +[Bob Shell](https://bob.ibm.com) is an OpenAI-compatible CLI coding agent. It integrates +with `lab-cx` by pointing its `CUSTOM_BASE_URL` at the proxy; the proxy reduces the +OpenAI-shaped traffic and forwards to Bob's backend. Bob is **not self-caching**, so +unlike Claude Code lab-cx's cache-injection lever fires on every request. + +## Setup + +```sh +# Bob's API key (here from winnow/.env: BOBSHELL_API_KEY=bob_...) +export BOBSHELL_API_KEY=... + +# Start lab-cx: agent upstream = Bob's backend; cheap-model extractor = your gateway. +./bin/lab-cx proxy --addr 127.0.0.1:8093 --preset balanced \ + --upstream https://api.us-east.bob.ibm.com \ + --extract-model claude-haiku-4-5 --extract-provider anthropic \ + --extract-auth bearer --extract-base "$ANTHROPIC_BASE_URL" +``` + +Run Bob routed through the proxy (Bob's `custom` auth mode treats `CUSTOM_BASE_URL` as a +plain OpenAI provider and does its own auth handshake — drive the real `bob` CLI, a raw +curl with the `bob_` key is rejected by the backend as "invalid jwt"): + +```sh +CUSTOM_BASE_URL=http://127.0.0.1:8093/v1 \ +BOBSHELL_DEFAULT_AUTH_TYPE=custom \ +BOBSHELL_API_KEY="$BOBSHELL_API_KEY" \ +bob "List the function names defined in calc.py, comma-separated." \ + --accept-license --yolo --hide-intermediary-output +``` + +Then read `curl -s http://127.0.0.1:8093/stats`. + +## Results (real, 2026-06-24, Bob → lab-cx → Bob backend) + +**Correctness: 5/5 verifiable tasks correct, including a file edit — no harm.** + +| Task | Expected | Bob's answer | OK | +|---|---|---|---| +| `17 * 23` | 391 | `391` | ✓ | +| function names in calc.py | add, sub | `add, sub` | ✓ | +| add `multiply(a,b)` to calc.py | file edited | `multiply` added to calc.py | ✓ | +| analyze 300-record data.json | 200 active, first inactive id 1000 | `200 active … id 1000` | ✓ | +| functions in calc.py (variant) | add, sub | correct | ✓ | + +`/stats` after the 3-task suite: + +```json +{ "requests": 6, "tokens_before": 2919, "tokens_after": 2919, "tokens_saved": 0, + "reduction_ratio": 0, "cache_injected": 6, "extracted": 0, "stage_errors": 0, + "added_latency_p50_ms": 3, "added_latency_p95_ms": 12 } +``` + +**What this shows:** +- **The integration works end-to-end** and **preserves correctness** (5/5, including a + write) with negligible added latency (~3ms p50, 0 stage errors). +- **`cache_injected: 6/6`** — lab-cx injects `cache_control` breakpoints Bob never sends. + On a multi-turn Bob session this bills the growing transcript prefix at provider + cache-read rates (~10× cheaper) from turn 2 on — the lever a self-caching client like + Claude Code declines. (Our content-token metric doesn't price the provider-side + cache-read discount; `cache_injected` is the signal that it's engaged.) +- **`tokens_saved: 0` (content) here** because Bob *condensed the files itself* (it ran + logic over `data.json` rather than dumping its 43 KB into the model context), so the + Reduce/Extract content levers had no large raw output to act on. + +## Where Bob content-reduction savings come from + +Content reduction (Reduce/Extract) needs large **raw** tool outputs in the model context +— e.g. an MCP tool returning a big JSON array, or a `Read` of a large file kept across +turns. That path is proven: +- Component level on real fixtures: −93% aggregate ([../RESULTS-offline.md](../RESULTS-offline.md)), + cheap-model extractor −56%…−80% ([../RESULTS-extract.md](../RESULTS-extract.md)). +- End-to-end on a self-caching agent reading a large file: + [claude-code.md](claude-code.md) shows **50% token reduction** with correctness + preserved. The same Reduce levers apply to Bob when it surfaces large raw outputs; + enable extraction with a lower `floor` (see `configs/lab-cx.yaml`) to catch + medium-size structured results. + +## Reproduce + +The full flow (proxy + 5-task suite + a large-file read) is scriptable exactly as above; +swap the `bob ""` line for your own. Keep `--extract-*` pointed at a cheap model to +also exercise the extractor on large structured tool outputs. diff --git a/docs/integration/claude-code.md b/docs/integration/claude-code.md new file mode 100644 index 0000000..65250c4 --- /dev/null +++ b/docs/integration/claude-code.md @@ -0,0 +1,124 @@ +# Integrating with Claude Code (real run) + +`lab-cx` integrates with Claude Code by the standard base-URL swap: point Claude Code's +`ANTHROPIC_BASE_URL` at the proxy, and the proxy forwards upstream (a provider or, here, +an Anthropic-compatible gateway). No Claude Code changes; it works unmodified. + +## How it was actually run (2026-06-24, claude-haiku-4-5 via an Anthropic-compatible gateway) + +1. Start the proxy, forwarding to the model endpoint, extraction on: + +```sh +./bin/lab-cx proxy --addr 127.0.0.1:8090 --preset balanced \ + --upstream "$ANTHROPIC_BASE_URL" \ + --extract-model claude-haiku-4-5 --extract-provider anthropic \ + --extract-auth bearer --extract-base "$ANTHROPIC_BASE_URL" +``` + +2. Run Claude Code routed through the proxy. **Precedence note:** if your + `~/.claude/settings.json` sets `env.ANTHROPIC_BASE_URL`, Claude Code applies it over + an inherited shell variable — so an inline `ANTHROPIC_BASE_URL=...` is ignored. Route + it explicitly with a settings file instead: + +```sh +# /tmp/cc-settings.json: {"env":{"ANTHROPIC_BASE_URL":"http://127.0.0.1:8090", +# "ANTHROPIC_AUTH_TOKEN":"","ANTHROPIC_MODEL":"claude-haiku-4-5", +# "ANTHROPIC_SMALL_FAST_MODEL":"claude-haiku-4-5"}} +claude -p "Read main.go and README.md and summarize each in one line." \ + --settings /tmp/cc-settings.json --dangerously-skip-permissions +``` + +3. Read the proxy's `/stats`: + +```sh +curl -s http://127.0.0.1:8090/stats +``` + +## Result (verbatim, honest) + +The task completed correctly through the proxy. `/stats` after the run: + +```json +{ "requests": 2, "tokens_before": 12246, "tokens_after": 12246, + "tokens_saved": 0, "cache_injected": 0, "extracted": 0, + "stage_errors": 0, "added_latency_p50_ms": 41, "added_latency_p95_ms": 65 } +``` + +**Interpretation (do-no-harm, as designed):** +- **Traffic traversed lab-cx** (`requests: 2`) and the agent's task succeeded — the + integration works. +- **`cache_injected: 0`** — Claude Code is a *self-caching* client (it sends its own + `cache_control` breakpoints), so lab-cx's cache stage correctly **stands down** rather + than fight the client's cache. This matches the winnow finding that on Claude Code the + proxy is cost-neutral by design. +- **`tokens_saved: 0`** — a tiny 2-file, 2-turn task has no large/stale/duplicate tool + outputs to reduce and nothing over the extraction floor. lab-cx safely did nothing. +- **`added_latency_p50_ms: 41`** — the reduction pass adds ~40ms even when it changes + nothing (parse + score + render); acceptable, and dwarfed by model latency. + +**Where Claude Code savings actually come from** (not exercised by this toy task): +long sessions that re-read large files / re-run noisy commands / accumulate big tool +outputs — there the Reduce pre-passes (cmdfilter, dedup, skeleton, format) and, with +`reduce_cached_prefix` enabled, prefix re-caching kick in. For per-component evidence on +real large outputs see [../RESULTS-offline.md](../RESULTS-offline.md) (deterministic, +−93% aggregate) and [../RESULTS-extract.md](../RESULTS-extract.md) (haiku extractor, +−56%…−80% on structured outputs). A non-self-caching tool-calling agent additionally +gets the cache-injection lever that Claude Code declines. + +## Real savings on Claude Code (and a harm check) + +The do-no-harm run above changed nothing because the task was tiny and Claude Code +self-caches. To get **real reduction on a self-caching client**, enable +`reduce_cached_prefix` (lab-cx then reduces even the client's cached prefix — a one-time +re-cache of a smaller prefix) and give it a workload with a large file read. Config: + +```yaml +# /tmp/cc-save-cfg.yaml +preset: balanced +reduce: + protect_recent: 1 + provable_only: true + reduce_cached_prefix: true + reducers: [collapse, skeleton, format] +extract: { enabled: true, mode: auto, floor: 400 } +``` + +Run (a 1,179-line / 32 KB real Go file, 42 top-level funcs), routed through the proxy +with haiku, then read `/stats`: + +```sh +./bin/lab-cx proxy --addr 127.0.0.1:8094 --config /tmp/cc-save-cfg.yaml \ + --upstream "$ANTHROPIC_BASE_URL" --extract-model claude-haiku-4-5 \ + --extract-provider anthropic --extract-auth bearer --extract-base "$ANTHROPIC_BASE_URL" +claude -p "Read bigfile.go, then tell me in two lines: (1) roughly how many top-level + 'func' declarations it has, and (2) what the code does overall." \ + --settings /tmp/cc-save-settings.json --dangerously-skip-permissions +``` + +**Result (real, 2026-06-24, claude-haiku-4-5):** + +```json +{ "requests": 4, "tokens_before": 69683, "tokens_after": 34729, + "tokens_saved": 34954, "reduction_ratio": 0.502, "cost_saved_usd": 0.0350, + "cache_injected": 0, "added_latency_p50_ms": 86, "added_latency_p95_ms": 191 } +``` + +- **−50.2% tokens** (34,954 saved across 4 requests; up to −70% on the turn carrying the + full file read), ~$0.035 saved on this short interaction. +- **No harm — the task was answered correctly:** Claude Code reported **42 top-level + `func` declarations** (exactly matches `grep -c '^func '`) and an accurate summary of + the file. The `skeleton` reducer dropped function *bodies* while keeping every + signature, so the agent could still count and describe the functions — reduction that + preserves the task. +- `cache_injected: 0` because Claude Code self-caches; with `reduce_cached_prefix` lab-cx + reduces content instead of fighting the client's cache. + +So: on a real workload (large file read) lab-cx cuts Claude Code's tokens ~50% with the +answer unchanged; on a trivial task it safely does nothing (above). + +## Metric note + +`/stats` reports `reduction_ratio` = `tokens_saved / tokens_before` (fraction of input +removed; 0 = no savings, higher = more reduction). On this self-caching do-no-harm run +that value is 0, matching `tokens_saved: 0`. (Earlier builds exposed `ratio` as +`after/before`, which read as 1.0 at zero savings — fixed.) diff --git a/docs/integration/swe-bench.md b/docs/integration/swe-bench.md new file mode 100644 index 0000000..a1ee450 --- /dev/null +++ b/docs/integration/swe-bench.md @@ -0,0 +1,77 @@ +# Integrating with eval-containers (SWE-bench) + +[eval-containers](https://github.com/exgentic/eval-containers) runs `one benchmark + one +agent + one model`, with all agent→model traffic flowing through a **gateway** on port +4000. `lab-cx` inserts as a plain proxy *before* that gateway, so it sees and reduces +exactly what the agent sends — here with **claude-code** as the agent and +**claude-haiku-4-5** as the model. + +## Wiring (validated) + +The before-gateway placement is [`deploy/eval-containers/compose.override.yaml`](../../deploy/eval-containers/compose.override.yaml). +Merging it with the real swe-bench compose was validated structurally (no image pull): + +```sh +cd eval-containers/containers/benchmarks/swe-bench +EVAL_TASK_ID=astropy__astropy-12907 EVAL_AGENT=claude-code EVAL_MODEL=anthropic/claude-haiku-4-5 \ +OPENAI_API_BASE=... OPENAI_API_KEY=... \ +docker compose -f compose.yaml -f /path/to/lab-context-engineering/deploy/eval-containers/compose.override.yaml \ + config --services +# -> otelcol, gateway, winnow, runner (winnow inserted before the gateway) +``` + +The override adds a `winnow` service on the `internal` network, repoints the runner's +`ANTHROPIC_BASE_URL` (`http://gateway:4000/anthropic`) to `http://winnow:8080/anthropic`, +and sets `winnow --upstream http://gateway:4000`. The agent still holds only `sk-proxy`; +the real key stays in the gateway. lab-cx's suffix routing reduces the `/anthropic/...` +prefixed path and forwards the full path on. + +## Runbook (full run) + +1. Build the lab-cx image (CGO; final image is `distroless/base`): + ```sh + cd lab-context-engineering && docker build -t ghcr.io/kagenti/lab-context-engineering:latest . + ``` +2. Point the eval **gateway** at your model provider. For a standard provider set + `OPENAI_API_BASE`/`OPENAI_API_KEY` + `EVAL_MODEL` per the eval-containers docs. For an + **Anthropic-compatible gateway** (e.g. the IBM LiteLLM endpoint used in this repo's + other tests), either configure the eval gateway's provider to that endpoint, or set + `winnow --upstream` directly to it (bypassing the eval gateway) — winnow forwards the + agent's `Authorization` header through. Use `--extract-auth bearer` for bearer-token + gateways. +3. Run a task (or a slice from `tasks.txt`) with and without the override and compare: + ```sh + # with lab-cx + EVAL_TASK_ID=astropy__astropy-12907 EVAL_AGENT=claude-code EVAL_MODEL=anthropic/claude-haiku-4-5 \ + docker compose -f compose.yaml -f .../compose.override.yaml up -y --abort-on-container-exit + ``` +4. Read the metrics: + - **eval-containers**: `result.json` (resolve status) + `trajectory.jsonl` (per-call + `total_tokens`, `cost_usd`) in the shared `output` volume. + - **lab-cx**: `curl http://winnow:8080/stats` (tokens_before/after/saved, cache, + extracted, added latency) — run from inside the compose network, or expose the port. + - Compare with-vs-without lab-cx on the same `EVAL_TASK_ID` for the token/cost delta; + `result.json` resolve status confirms accuracy is preserved. + +## Status (honest) + +- **Wiring: validated** — the merged compose renders correctly (`winnow` before + `gateway`) and the lab-cx image builds (CGO, distroless/base). +- **Full SWE-bench run: NOT executed in this session.** It requires multi-GB image + pulls, configuring the eval gateway against the model provider, and real model spend + + long agent-loop runtime — out of scope for a single session, and we do not fabricate + benchmark numbers. The runbook above is what to execute. +- The measured evidence we DO have is real and reproducible: deterministic components + −93% aggregate on real fixtures ([../RESULTS-offline.md](../RESULTS-offline.md)) and + the haiku extractor −56%…−80% on structured outputs + ([../RESULTS-extract.md](../RESULTS-extract.md)), plus a real Claude Code run through + the proxy ([claude-code.md](claude-code.md)). SWE-bench is the end-to-end validation to + run next with a provisioned gateway. + +## Note for self-caching agents + +Claude Code self-caches, so on SWE-bench lab-cx's biggest lever is the **Reduce** +pre-passes (cmdfilter/dedup/skeleton/format) and the **extractor** on large tool +outputs, not cache injection (it defers — see [claude-code.md](claude-code.md)). Enable +`reduce_cached_prefix` in the config to also re-cache a smaller prefix on self-caching +clients. diff --git a/docs/superpowers/plans/2026-06-24-deepen-components-with-libraries.md b/docs/superpowers/plans/2026-06-24-deepen-components-with-libraries.md new file mode 100644 index 0000000..1948420 --- /dev/null +++ b/docs/superpowers/plans/2026-06-24-deepen-components-with-libraries.md @@ -0,0 +1,984 @@ +# Deepen Naive Components With Established Libraries — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Replace the from-scratch / stubbed pieces of `lab-context-engineering` with established libraries — real tokenizer, tree-sitter symbol+skeleton extraction, TOON encoding, and a real LLM-writes-code-in-a-sandbox extractor — and fix the verified non-dependency bugs. + +**Architecture:** Keep the engine/stage/surface boundaries intact. Swap implementations behind the existing internal package APIs so callers don't change: `tokens.Count`, `reduce` signals/skeleton/format, `extract` strategies, `cheapmodel`. New leaf package `internal/treesitter` wraps the CGO binding. The Starlark extractor has the cheap model write a Starlark filter run over the FULL parsed output, verified by the existing `extract.IsContained` (fail-open). + +**Tech Stack:** Go 1.25 (now **CGO-enabled**), `github.com/tiktoken-go/tokenizer`, `github.com/tree-sitter/go-tree-sitter` + `github.com/alexaandru/go-sitter-forest`, `github.com/toon-format/toon-go` (pinned), `go.starlark.net/starlark`. + +## Global Constraints + +- Go module `github.com/kagenti/lab-context-engineering`, Go 1.25. `make fmt lint test build` must stay green; lint = `go vet` + `gofmt -l` clean. +- **CGO is now required** (tree-sitter): CI sets `CGO_ENABLED=1` with a C toolchain; the Docker final stage moves from `distroless/static` to `distroless/base-debian12:nonroot` (glibc present). +- **Fail-open is non-negotiable:** any error in any stage/strategy forwards the original content untouched. Every reduction stays reversible (markers + rewind store). +- DCO sign-off on every commit (`git commit -s`); author Osher Elhadad; `Assisted-By:` trailer, never `Co-Authored-By`. Conventional-commit titles. +- New dependencies are pinned in `go.mod`; `toon-go` has no releases — pin an exact commit (`go get github.com/toon-format/toon-go@`). +- Public API (`engine`, `surfaces`, `config`, `canon`, `observability`) signatures must not break — these tasks change internals only. + +--- + +### Task 1: Accurate tokenizer (replace chars/4) + +**Files:** +- Modify: `internal/tokens/tokens.go` +- Test: `internal/tokens/tokens_test.go` +- Modify: `go.mod`, `go.sum` + +**Interfaces:** +- Consumes: nothing. +- Produces: `tokens.Count(text string) int` (unchanged signature) — now BPE-accurate. + +- [ ] **Step 1: Add the dependency** + +Run: `go get github.com/tiktoken-go/tokenizer@latest` +Expected: `go.mod` gains the require line. + +- [ ] **Step 2: Write the failing test** + +```go +package tokens + +import "testing" + +func TestCountIsBPENotCharQuarter(t *testing.T) { + // "hello world" is 2 BPE tokens in cl100k/o200k, not len/4 == 2..3 by luck; + // use a case where chars/4 is clearly wrong: repeated punctuation. + got := Count("!!!!!!!!!!!!!!!!") + if got == len("!!!!!!!!!!!!!!!!")/4 { + t.Fatalf("Count still looks like chars/4: %d", got) + } + if got <= 0 { + t.Fatalf("Count returned %d", got) + } +} + +func TestCountEmpty(t *testing.T) { + if Count("") != 0 { + t.Fatal("empty must be 0") + } +} + +func TestCountStableAcrossCalls(t *testing.T) { + a, b := Count("the quick brown fox"), Count("the quick brown fox") + if a != b { + t.Fatalf("non-deterministic: %d vs %d", a, b) + } +} +``` + +- [ ] **Step 3: Run test to verify it fails** + +Run: `go test ./internal/tokens/ -run TestCountIsBPE -v` +Expected: FAIL (current impl is chars/4). + +- [ ] **Step 4: Implement with tiktoken-go, lazy-initialized** + +```go +// Package tokens estimates token counts using a real BPE tokenizer (o200k_base, +// the modern GPT family encoding) — an accurate offline proxy. The provider's +// usage remains authoritative; this drives reduction gating and never-inflate guards. +package tokens + +import ( + "sync" + + "github.com/tiktoken-go/tokenizer" +) + +var ( + encOnce sync.Once + enc tokenizer.Codec +) + +func codec() tokenizer.Codec { + encOnce.Do(func() { + // o200k_base is embedded in the binary (pure-Go, offline, no CGO). + enc, _ = tokenizer.Get(tokenizer.O200kBase) + }) + return enc +} + +// Count returns the BPE token count of text (0 for empty). Falls back to a +// chars/4 estimate only if the tokenizer failed to initialize. +func Count(text string) int { + if text == "" { + return 0 + } + c := codec() + if c == nil { + return (len(text) + 3) / 4 + } + ids, _, err := c.Encode(text) + if err != nil { + return (len(text) + 3) / 4 + } + return len(ids) +} +``` + +- [ ] **Step 5: Run tests + vet** + +Run: `go test ./internal/tokens/ -v && go vet ./internal/tokens/` +Expected: PASS. Then `go test ./...` to confirm thresholds elsewhere still pass (adjust any test that hard-coded a chars/4 number — none currently assert absolute token counts). + +- [ ] **Step 6: Commit** + +```bash +git add go.mod go.sum internal/tokens/ +git commit -s -m "feat(tokens): use tiktoken-go o200k_base instead of chars/4 estimate" +``` + +--- + +### Task 2: tree-sitter foundation package + +**Files:** +- Create: `internal/treesitter/treesitter.go` +- Test: `internal/treesitter/treesitter_test.go` +- Modify: `go.mod`, `go.sum` + +**Interfaces:** +- Produces: + - `treesitter.LangForExt(path string) string` — returns a grammar name ("go","python",...) or "". + - `treesitter.Parse(lang string, src []byte) (*sitter.Tree, *sitter.Language, bool)` — parsed tree + language, ok=false on unknown lang / parse failure (fail-open). + - Callers must `defer tree.Close()`. + +- [ ] **Step 1: Add dependencies** + +Run: +``` +go get github.com/tree-sitter/go-tree-sitter@latest +go get github.com/alexaandru/go-sitter-forest@latest +``` +Expected: both in `go.mod`. (forest provides `forest.GetLanguage(name)` across ~490 grammars; import the root package — binary-size note below.) + +- [ ] **Step 2: Write the failing test** + +```go +package treesitter + +import "testing" + +func TestLangForExt(t *testing.T) { + cases := map[string]string{"a.go": "go", "b.py": "python", "c.ts": "typescript", "d.txt": ""} + for path, want := range cases { + if got := LangForExt(path); got != want { + t.Fatalf("LangForExt(%q)=%q want %q", path, got, want) + } + } +} + +func TestParseGo(t *testing.T) { + tree, lang, ok := Parse("go", []byte("package p\nfunc Foo() int { return 1 }\n")) + if !ok { + t.Fatal("expected parse ok for valid Go") + } + defer tree.Close() + if lang == nil || tree.RootNode().HasError() { + t.Fatal("unexpected nil language or parse error") + } +} + +func TestParseUnknownLangFailsOpen(t *testing.T) { + if _, _, ok := Parse("klingon", []byte("x")); ok { + t.Fatal("unknown language must return ok=false") + } +} +``` + +- [ ] **Step 3: Run test to verify it fails** + +Run: `CGO_ENABLED=1 go test ./internal/treesitter/ -v` +Expected: FAIL (package doesn't exist). + +- [ ] **Step 4: Implement the wrapper** + +```go +// Package treesitter wraps the tree-sitter binding for the reduce signals and +// skeletonizer. CGO-backed; fail-open (unknown language / parse error => ok=false). +package treesitter + +import ( + "path" + "strings" + + forest "github.com/alexaandru/go-sitter-forest" + sitter "github.com/tree-sitter/go-tree-sitter" +) + +var extToLang = map[string]string{ + ".go": "go", ".py": "python", ".js": "javascript", ".jsx": "javascript", + ".ts": "typescript", ".tsx": "tsx", ".rs": "rust", ".java": "java", + ".c": "c", ".h": "c", ".cc": "cpp", ".cpp": "cpp", ".hpp": "cpp", + ".rb": "ruby", ".php": "php", ".cs": "c_sharp", ".kt": "kotlin", + ".swift": "swift", ".scala": "scala", +} + +// LangForExt maps a file path to a tree-sitter grammar name, or "" if unsupported. +func LangForExt(p string) string { + return extToLang[strings.ToLower(path.Ext(p))] +} + +// Parse parses src under the named grammar. ok=false on unknown grammar or a nil +// language; the caller treats that as "skip" (fail-open). Caller must Close the tree. +func Parse(lang string, src []byte) (*sitter.Tree, *sitter.Language, bool) { + raw := forest.GetLanguage(lang) + if raw == nil { + return nil, nil, false + } + language := sitter.NewLanguage(raw) + if language == nil { + return nil, nil, false + } + parser := sitter.NewParser() + defer parser.Close() + if err := parser.SetLanguage(language); err != nil { + return nil, nil, false + } + tree := parser.Parse(src, nil) + if tree == nil { + return nil, nil, false + } + return tree, language, true +} +``` + +> Note: `forest.GetLanguage(name)` links all grammars (large binary). If binary size matters, replace with a `switch` over per-language subpackages (`.../go-sitter-forest/golang`, `/python`, …) — same `*sitter.Language`. Keep the root import for v1 simplicity; track size in the commit message. + +- [ ] **Step 5: Run tests + vet** + +Run: `CGO_ENABLED=1 go test ./internal/treesitter/ -v && go vet ./internal/treesitter/` +Expected: PASS. + +- [ ] **Step 6: Commit** + +```bash +git add go.mod go.sum internal/treesitter/ +git commit -s -m "feat(treesitter): CGO tree-sitter wrapper (LangForExt + Parse, fail-open)" +``` + +--- + +### Task 3: Real symbol extraction (replace regex `definedSymbols`) + +**Files:** +- Modify: `internal/reduce/signals.go` (replace `definedSymbols`; keep `salientLiterals`/`literalsUsed`/`pathReferenced`/`symbolsUsed`) +- Test: `internal/reduce/signals_test.go` + +**Interfaces:** +- Consumes: `treesitter.LangForExt`, `treesitter.Parse`. +- Produces: `definedSymbols(text, filePath string) map[string]struct{}` (unchanged signature) — now tree-sitter-based; `isCodePath` now delegates to `treesitter.LangForExt(fp) != ""`. + +- [ ] **Step 1: Write the failing test (Go method case the regex missed)** + +```go +package reduce + +import "testing" + +func TestDefinedSymbolsCatchesGoMethod(t *testing.T) { + src := "package p\ntype T struct{}\nfunc (r *T) DoThing() {}\nfunc Helper() {}\n" + syms := definedSymbols(src, "x.go") + for _, want := range []string{"DoThing", "Helper"} { + if _, ok := syms[want]; !ok { + t.Fatalf("missing symbol %q in %v", want, syms) + } + } +} + +func TestDefinedSymbolsIgnoresComments(t *testing.T) { + src := "package p\n// func Ghost() {}\nfunc Real() {}\n" + syms := definedSymbols(src, "x.go") + if _, ok := syms["Ghost"]; ok { + t.Fatal("commented-out func must not be a defined symbol") + } +} + +func TestDefinedSymbolsNonCodeEmpty(t *testing.T) { + if len(definedSymbols("hello", "notes.txt")) != 0 { + t.Fatal("non-code path must yield no symbols") + } +} +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `CGO_ENABLED=1 go test ./internal/reduce/ -run TestDefinedSymbols -v` +Expected: FAIL (`DoThing` missed by the regex, or `Ghost` wrongly included). + +- [ ] **Step 3: Implement via tree-sitter query** + +Replace `defRe`, `isCodePath`, and `definedSymbols` in `signals.go` with: + +```go +// (top of file) import "github.com/kagenti/lab-context-engineering/internal/treesitter" +// and sitter "github.com/tree-sitter/go-tree-sitter" + +func isCodePath(fp string) bool { return treesitter.LangForExt(fp) != "" } + +// nameFieldKinds: definition node kinds whose "name" field (or identifier child) +// names a symbol, across grammars. +var defNodeKinds = map[string]bool{ + "function_declaration": true, "function_definition": true, "function_item": true, + "method_declaration": true, "method_definition": true, "method": true, + "class_declaration": true, "class_definition": true, "class": true, + "struct_item": true, "enum_item": true, "trait_item": true, + "type_spec": true, "interface_declaration": true, "module": true, +} + +// definedSymbols returns names of functions/classes/methods/types defined in code +// text. Empty for non-code paths or on any parse failure (fail-open). Drops names +// shorter than 3 chars. +func definedSymbols(text, filePath string) map[string]struct{} { + lang := treesitter.LangForExt(filePath) + if lang == "" { + return nil + } + src := []byte(text) + tree, _, ok := treesitter.Parse(lang, src) + if !ok { + return nil + } + defer tree.Close() + out := map[string]struct{}{} + var walk func(n *sitter.Node) + walk = func(n *sitter.Node) { + if defNodeKinds[n.Kind()] { + if name := n.ChildByFieldName("name"); name != nil { + if s := name.Utf8Text(src); len(s) >= 3 { + out[s] = struct{}{} + } + } + } + for i := uint(0); i < n.NamedChildCount(); i++ { + walk(n.NamedChild(i)) + } + } + walk(tree.RootNode()) + return out +} +``` + +Delete the now-unused `defRe` regex. + +- [ ] **Step 4: Run tests + the full reduce suite** + +Run: `CGO_ENABLED=1 go test ./internal/reduce/ -v` +Expected: PASS (new + existing relevance/pipeline tests). + +- [ ] **Step 5: Commit** + +```bash +git add internal/reduce/signals.go internal/reduce/signals_test.go +git commit -s -m "feat(reduce): tree-sitter symbol extraction (catches methods, ignores comments)" +``` + +--- + +### Task 4: Real code skeletonization (un-stub `skeletonize`) + +**Files:** +- Modify: `internal/reduce/actions.go` (replace `skeletonize` stub; `skeletonReduce` already calls it and passes the lang) +- Test: `internal/reduce/skeleton_test.go` + +**Interfaces:** +- Consumes: `treesitter`. +- Produces: `skeletonize(source, filePath string) (string, bool)` — **signature changes** from `(source, lang string)` to take the file path so it can resolve the grammar. Update `skeletonReduce` to call `skeletonize(item.Text, item.FilePath)`. + +- [ ] **Step 1: Write the failing test** + +```go +package reduce + +import ( + "strings" + "testing" +) + +func TestSkeletonizeDropsGoBodies(t *testing.T) { + src := "package p\nfunc Big() int {\n\t" + strings.Repeat("x := 1\n\t", 50) + "return x\n}\n" + sk, ok := skeletonize(src, "big.go") + if !ok { + t.Fatal("expected skeletonization to apply") + } + if !strings.Contains(sk, "func Big()") { + t.Fatal("signature must be kept") + } + if strings.Contains(sk, "x := 1\n\tx := 1") { + t.Fatal("body should have been elided") + } +} + +func TestSkeletonizeNonCodeFails(t *testing.T) { + if _, ok := skeletonize("hello", "n.txt"); ok { + t.Fatal("non-code must return ok=false") + } +} +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `CGO_ENABLED=1 go test ./internal/reduce/ -run TestSkeletonize -v` +Expected: FAIL (stub returns false). + +- [ ] **Step 3: Implement body elision by byte range** + +```go +// (imports) "sort"; treesitter; sitter "github.com/tree-sitter/go-tree-sitter"; tokens already imported + +var bodyDefKinds = map[string]bool{ + "function_declaration": true, "function_definition": true, "function_item": true, + "method_declaration": true, "method_definition": true, "method": true, + "constructor_declaration": true, +} + +// skeletonize keeps signatures and drops function/method BODIES (the "body" field), +// language-agnostic via tree-sitter. Returns ok=false for non-code, parse failure, +// no bodies, or no token savings. +func skeletonize(source, filePath string) (string, bool) { + lang := treesitter.LangForExt(filePath) + if lang == "" { + return "", false + } + src := []byte(source) + tree, _, ok := treesitter.Parse(lang, src) + if !ok { + return "", false + } + defer tree.Close() + type span struct{ start, end uint } + var bodies []span + var walk func(n *sitter.Node) + walk = func(n *sitter.Node) { + if bodyDefKinds[n.Kind()] { + if b := n.ChildByFieldName("body"); b != nil { + bodies = append(bodies, span{b.StartByte(), b.EndByte()}) + return // don't recurse into a body we're dropping (nested fns) + } + } + for i := uint(0); i < n.NamedChildCount(); i++ { + walk(n.NamedChild(i)) + } + } + walk(tree.RootNode()) + if len(bodies) == 0 { + return "", false + } + sort.Slice(bodies, func(i, j int) bool { return bodies[i].start > bodies[j].start }) + out := append([]byte(nil), src...) + for _, b := range bodies { + out = append(out[:b.start], append([]byte("{ ... }"), out[b.end:]...)...) + } + result := string(out) + if tokens.Count(result) >= tokens.Count(source) { + return "", false + } + return result, true +} +``` + +Update `skeletonReduce` to call `skeletonize(item.Text, item.FilePath)` (drop the second `lang` arg and its `skeletonize(item.Text, "")` call site). + +- [ ] **Step 4: Run tests** + +Run: `CGO_ENABLED=1 go test ./internal/reduce/ -v` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add internal/reduce/actions.go internal/reduce/skeleton_test.go +git commit -s -m "feat(reduce): tree-sitter code skeletonization (keep signatures, drop bodies)" +``` + +--- + +### Task 5: Add TOON to the format re-encoder + +**Files:** +- Modify: `internal/reduce/actions.go` (`bestEncoding` encoder table) +- Test: `internal/reduce/format_test.go` +- Modify: `go.mod`, `go.sum` + +**Interfaces:** +- Produces: a new encoder candidate `("toon", rank, encTOON)` in `bestEncoding`; selection still by `tokens.Count`, only returned when strictly smaller. + +- [ ] **Step 1: Pin the dependency** + +Run: `go get github.com/toon-format/toon-go@` +(No releases — pin an explicit commit; record the sha in the commit message.) + +- [ ] **Step 2: Write the failing test** + +```go +package reduce + +import ( + "strings" + "testing" +) + +func TestBestEncodingConsidersTOON(t *testing.T) { + // Uniform array with a nested field CSV can't represent; TOON should be a + // candidate and the result must round-trip-contain the data. + var recs []string + for i := 0; i < 40; i++ { + recs = append(recs, `{"id":`+itoaTest(i)+`,"meta":{"k":"v"},"name":"item"}`) + } + body := "[" + strings.Join(recs, ",") + "]" + enc, name := bestEncoding(body) + if enc == "" { + t.Fatal("expected a smaller encoding") + } + if len(enc) >= len(body) { + t.Fatal("encoding not smaller") + } + _ = name // may be toon or json_compact depending on token counts; just assert it ran +} +``` + +(Add `func itoaTest(n int) string` helper if not present in the test package.) + +- [ ] **Step 3: Run test to verify it fails or is incomplete** + +Run: `CGO_ENABLED=1 go test ./internal/reduce/ -run TestBestEncodingConsidersTOON -v` +Expected: passes only if TOON helps; first add the encoder, then confirm. (If it already passes via json_compact, still add TOON so it competes — verify with a uniform-flat case where TOON beats JSON.) + +- [ ] **Step 4: Implement `encTOON` and register it** + +```go +// import toon "github.com/toon-format/toon-go" + +func encTOON(data any) (string, bool) { + out, err := toon.MarshalString(data, toon.WithLengthMarkers(true)) + if err != nil || out == "" { + return "", false + } + return out, true +} +``` + +In `bestEncoding`'s `encoders` slice, insert `{"toon", 1, encTOON}` and renumber ranks so the model-friendly order is `json_compact(0) < toon(1) < jsonl(2) < markdown_kv(3) < tsv(4) < csv(5)`. TOON output must still pass the existing "strictly smaller" gate; the rewind store holds the original so re-encoding is reversible. + +- [ ] **Step 5: Run tests** + +Run: `CGO_ENABLED=1 go test ./internal/reduce/ -v` +Expected: PASS. + +- [ ] **Step 6: Commit** + +```bash +git add go.mod go.sum internal/reduce/actions.go internal/reduce/format_test.go +git commit -s -m "feat(reduce): add TOON to the format re-encoder candidates (pinned toon-go @)" +``` + +--- + +### Task 6: Starlark code-writing extractor strategy + +**Files:** +- Create: `internal/extract/starlark.go` +- Modify: `internal/extract/extract.go` (add a "code" strategy that uses Starlark; update prompt contract; `strategyOrder` auto picks "code" for mid-size, "rlm" for very large) +- Create: `internal/extract/starlark_test.go` +- Modify: `internal/extract/prompt.go` (a code-writing prompt variant) +- Modify: `go.mod`, `go.sum` + +**Interfaces:** +- Consumes: `extract.Model` (existing), `extract.IsContained` (existing). +- Produces: + - `runStarlark(ctx, body, goal string, keepIDs []string, model Model) string` — asks the model for a Starlark program (contract: read `INPUT` string, assign `OUTPUT` string), runs it sandboxed over the FULL body, returns the candidate text ("" on any failure). + - `runCode` becomes the new auto primary for mid-size bodies; `runSingle` (JSON-return) stays as a fallback strategy named "single". + +- [ ] **Step 1: Add the dependency** + +Run: `go get go.starlark.net/starlark@latest go.starlark.net/lib/json@latest` + +- [ ] **Step 2: Write the failing test (sandbox runs a model-written filter over the FULL body)** + +```go +package extract + +import ( + "context" + "strings" + "testing" +) + +// starlarkModel returns a fixed Starlark program that keeps records whose name +// contains "keep" — exercises real code execution over the full input. +type starlarkModel struct{} + +func (starlarkModel) Complete(_ context.Context, _ string) (string, error) { + return ` +data = json.decode(INPUT) +kept = [r for r in data if "keep" in r["name"]] +OUTPUT = json.encode(kept) +`, nil +} + +func TestRunStarlarkFiltersFullBody(t *testing.T) { + var recs []string + for i := 0; i < 100; i++ { + name := "drop" + if i%10 == 0 { + name = "keep" + } + recs = append(recs, `{"id":`+itoa(i)+`,"name":"`+name+`"}`) + } + body := "[" + strings.Join(recs, ",") + "]" + out := runStarlark(context.Background(), body, "find keep", nil, starlarkModel{}) + if out == "" { + t.Fatal("expected a Starlark result") + } + if !IsContained(parseBody(out), parseBody(body)) { + t.Fatalf("Starlark output must be a contained subset: %s", out) + } + if strings.Contains(out, "drop") { + t.Fatal("filter should have dropped non-keep records") + } + if !strings.Contains(out, "keep") { + t.Fatal("filter should have kept the keep records (recall, not truncation)") + } +} + +// malicious program must fail-open (no panic, returns ""). +type evilModel struct{} + +func (evilModel) Complete(_ context.Context, _ string) (string, error) { + return `load("os", "x")`, nil // imports disabled +} + +func TestRunStarlarkFailsOpenOnDisallowed(t *testing.T) { + if out := runStarlark(context.Background(), `[{"a":1}]`, "", nil, evilModel{}); out != "" { + t.Fatalf("disallowed program must fail open to \"\", got %q", out) + } +} +``` + +- [ ] **Step 3: Run test to verify it fails** + +Run: `CGO_ENABLED=1 go test ./internal/extract/ -run TestRunStarlark -v` +Expected: FAIL (`runStarlark` undefined). + +- [ ] **Step 4: Implement the Starlark sandbox** + +```go +// Package extract — starlark.go +package extract + +import ( + "context" + "time" + + starjson "go.starlark.net/lib/json" + "go.starlark.net/starlark" +) + +const ( + starlarkMaxSteps = 50_000_000 + starlarkTimeout = 2 * time.Second +) + +// runStarlark asks the model for a Starlark program whose contract is: read the +// global string INPUT (the full tool output), assign a string global OUTPUT (the +// filtered value). It runs sandboxed over the FULL body — no imports, no I/O, step + +// time limits — and returns OUTPUT, or "" on any failure (fail-open). Containment is +// verified by the caller (RunExtraction). +func runStarlark(ctx context.Context, body, goal string, keepIDs []string, model Model) (out string) { + defer func() { + if recover() != nil { + out = "" + } + }() + if model == nil { + return "" + } + src, err := model.Complete(ctx, buildCodePrompt(body, goal, keepIDs)) + if err != nil { + return "" + } + src = stripFences(src) + + ctx, cancel := context.WithTimeout(ctx, starlarkTimeout) + defer cancel() + thread := &starlark.Thread{Name: "extract"} // Load==nil => load() disabled + thread.SetMaxExecutionSteps(starlarkMaxSteps) + done := make(chan struct{}) + go func() { + select { + case <-ctx.Done(): + thread.Cancel(ctx.Err().Error()) + case <-done: + } + }() + defer close(done) + + predeclared := starlark.StringDict{ + "json": starjson.Module, + "INPUT": starlark.String(body), + } + globals, err := starlark.ExecFile(thread, "extract.star", src, predeclared) + if err != nil { + return "" + } + res, ok := globals["OUTPUT"].(starlark.String) + if !ok { + return "" + } + return string(res) +} +``` + +Add `buildCodePrompt` to `prompt.go` (a sibling of `buildPrompt`): instruct the model to "write a Starlark program: `data = json.decode(INPUT)`; select a smaller value of the same shape; `OUTPUT = json.encode(result)`; select/never summarize; recall-first; no imports/IO." Show the schema + a small sample (it does NOT need the full body — the program runs over the real INPUT), so this prompt is token-cheap. + +In `extract.go`: add strategy `"code"` calling `runStarlark`; in `strategyOrder` auto, primary becomes `"code"` for `tokenEst < max(floor*4,8000)` and `"rlm"` above it; keep `"single"` (JSON-return) and `"deterministic"` as ordered fallbacks. Add the case to `RunExtraction`'s switch. + +- [ ] **Step 5: Run tests** + +Run: `CGO_ENABLED=1 go test ./internal/extract/ -v` +Expected: PASS (new Starlark tests + existing containment/deterministic/rlm). + +- [ ] **Step 6: Commit** + +```bash +git add go.mod go.sum internal/extract/ +git commit -s -m "feat(extract): Starlark code-writing extractor over full output (sandboxed, containment-verified)" +``` + +--- + +### Task 7: Fix extractor recall + cache correctness + +**Files:** +- Modify: `internal/extract/extract.go` (`runSingle` no longer truncates to 4000 when used as fallback; document that "code"/"rlm" see the full body) +- Modify: `engine/extractfunc.go` (goal-aware cache key) +- Test: `engine/extract_engine_test.go` (add a cross-goal cache test) + +**Interfaces:** +- Produces: cache key = `extract.ContentKey(text) + ":" + hash(goal+keep)` so the same body under a different goal is a cache miss. + +- [ ] **Step 1: Write the failing test** + +```go +// in engine package test +func TestExtractionCacheIsGoalAware(t *testing.T) { + // Two requests, same large body, different goals -> the second must NOT reuse + // the first's goal-specific filtered result. + // (Construct two canon.Requests sharing the tool_result body but with different + // trailing user text; run Transform on each with a model that echoes the goal; + // assert the spliced results differ.) +} +``` + +(Write the concrete bodies as in the existing `TestExtractionEndToEnd`, differing only in the final user text; use a model whose output depends on the goal token so a shared cache would be observable.) + +- [ ] **Step 2: Run test to verify it fails** + +Run: `CGO_ENABLED=1 go test ./engine/ -run TestExtractionCacheIsGoalAware -v` +Expected: FAIL (current cache keys on body only → second goal reuses first result). + +- [ ] **Step 3: Implement goal-aware key** + +In `engine/extractfunc.go`, change the cache key from `extract.ContentKey(c.Text)` to a composite: + +```go +import "crypto/sha256"; import "encoding/hex" + +func goalKey(contentKey, goal string, keep []string) string { + h := sha256.New() + h.Write([]byte(contentKey)); h.Write([]byte{0}); h.Write([]byte(goal)) + for _, k := range keep { h.Write([]byte{0}); h.Write([]byte(k)) } + return hex.EncodeToString(h.Sum(nil))[:24] +} +// key := goalKey(extract.ContentKey(c.Text), goal, keep) +``` + +- [ ] **Step 4: Run tests** + +Run: `CGO_ENABLED=1 go test ./engine/ ./internal/extract/ -v` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add engine/extractfunc.go engine/extract_engine_test.go internal/extract/extract.go +git commit -s -m "fix(extract): goal-aware extraction cache key; full-body strategies avoid truncation" +``` + +--- + +### Task 8: OpenAI cheap-model client + robust content parsing + +**Files:** +- Create: `internal/cheapmodel/openai.go` +- Modify: `internal/cheapmodel/anthropic.go` (scan content blocks for first text, not `Content[0]`) +- Test: `internal/cheapmodel/cheapmodel_test.go` (httptest servers for both) +- Modify: `cmd/proxy/main.go` (`--extract-provider anthropic|openai`) + +**Interfaces:** +- Produces: `cheapmodel.OpenAI{BaseURL, APIKey, Model, MaxTokens, Client}` implementing `engine.Model` via `Complete`; both clients select the first text content block. + +- [ ] **Step 1: Write the failing tests (httptest upstreams)** + +```go +package cheapmodel + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "testing" +) + +func TestAnthropicSkipsNonTextLeadingBlock(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.WriteString(w, `{"content":[{"type":"thinking","text":""},{"type":"text","text":"RESULT"}]}`) + })) + defer srv.Close() + got, err := Anthropic{BaseURL: srv.URL, Model: "m"}.Complete(context.Background(), "p") + if err != nil || got != "RESULT" { + t.Fatalf("got %q err %v", got, err) + } +} + +func TestOpenAIComplete(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.WriteString(w, `{"choices":[{"message":{"content":"OUT"}}]}`) + })) + defer srv.Close() + got, err := OpenAI{BaseURL: srv.URL, Model: "m"}.Complete(context.Background(), "p") + if err != nil || got != "OUT" { + t.Fatalf("got %q err %v", got, err) + } +} +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `go test ./internal/cheapmodel/ -v` +Expected: FAIL (OpenAI undefined; Anthropic returns "" for leading non-text block). + +- [ ] **Step 3: Implement** + +In `anthropic.go`, replace `return out.Content[0].Text` with a loop returning the first block whose `Text != ""`. Create `openai.go` with an `OpenAI` struct POSTing to `{base}/v1/chat/completions` (`{model, max_tokens, messages:[{role:"user",content:prompt}]}`), parsing `choices[0].message.content`. Add `--extract-provider` to `cmd/proxy/main.go` selecting which client to construct. + +- [ ] **Step 4: Run tests** + +Run: `go test ./internal/cheapmodel/ -v && go build ./...` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add internal/cheapmodel/ cmd/proxy/main.go +git commit -s -m "feat(cheapmodel): OpenAI client + first-text-block selection; --extract-provider flag" +``` + +--- + +### Task 9: Non-dependency correctness/perf fixes + +**Files:** +- Modify: `internal/reduce/signals.go` (`pathReferenced` precompile) +- Modify: `internal/extract/deterministic.go` (rune-safe slicing) +- Modify: `internal/proxyhttp/proxy.go` (upstream client timeout + request body cap) +- Test: extend `internal/extract/extract_test.go`, `internal/proxyhttp/proxy_test.go` + +**Interfaces:** no signature changes. + +- [ ] **Step 1: Write failing tests** + +```go +// extract_test.go — rune-safe truncation +func TestTruncateValueDoesNotSplitRunes(t *testing.T) { + s := strings.Repeat("é", 10) // 2 bytes each + out := truncateValue(s, 5).(string) + if !utf8.ValidString(out) { + t.Fatalf("truncation split a rune: %q", out) + } +} +``` + +```go +// proxy_test.go — body size cap returns 413 +func TestRequestBodyCap(t *testing.T) { + h := New(Config{Engine: engine.New(config.Default(), nil, nil), Upstream: "http://unused", MaxBodyBytes: 10}) + req := httptest.NewRequest(http.MethodPost, "/v1/messages", strings.NewReader(strings.Repeat("x", 100))) + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + if rec.Code != http.StatusRequestEntityTooLarge { + t.Fatalf("status=%d want 413", rec.Code) + } +} +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `CGO_ENABLED=1 go test ./internal/extract/ ./internal/proxyhttp/ -run 'TestTruncateValue|TestRequestBodyCap' -v` +Expected: FAIL. + +- [ ] **Step 3: Implement** + +- `truncateValue`/`extractTextWindow`: convert to `[]rune` before slicing (`r := []rune(v); if len(r) > n { return string(r[:n]) }`). +- `pathReferenced`: precompute the two boundary regexes once per call from `file_path`+basename outside the per-item loop, or hoist to a small `sync.Map` cache keyed by file path. (Minimum: build the regexps once before the `for _, tok` loop instead of inside it; the existing call is already per-(item,candidate) — move compilation to the top of `pathReferenced`.) +- `proxyhttp.Config`: add `MaxBodyBytes int64` (default e.g. 32<<20) and `UpstreamTimeout time.Duration`; in `model`/`passthrough`, wrap `r.Body` with `http.MaxBytesReader` and return 413 on overflow; default `cfg.Client` to `&http.Client{Timeout: cfg.UpstreamTimeout}` when set. + +- [ ] **Step 4: Run tests** + +Run: `CGO_ENABLED=1 go test ./... -v 2>&1 | tail -20` +Expected: PASS across all packages. + +- [ ] **Step 5: Commit** + +```bash +git add internal/reduce/signals.go internal/extract/deterministic.go internal/proxyhttp/ +git commit -s -m "fix: rune-safe truncation, precompiled pathReferenced regex, proxy body cap + timeout" +``` + +--- + +### Task 10: CI + Docker for CGO + +**Files:** +- Modify: `.github/workflows/ci.yaml` +- Modify: `Dockerfile` +- Modify: `Makefile` + +**Interfaces:** none. + +- [ ] **Step 1: Update the Makefile** + +Set `export CGO_ENABLED=1` near the top so `make test`/`make build` compile the tree-sitter binding. (Document that a C toolchain is required.) + +- [ ] **Step 2: Update CI** + +In `.github/workflows/ci.yaml`, ensure the `build-test` job runs with `CGO_ENABLED=1` (ubuntu runners include gcc). Add an `env: { CGO_ENABLED: "1" }` to the job. + +- [ ] **Step 3: Update the Dockerfile** + +Build stage already uses `golang:1.25` (has gcc). Change `CGO_ENABLED=0` → `CGO_ENABLED=1` in the build command, and the final stage from `gcr.io/distroless/static-debian12:nonroot` to `gcr.io/distroless/base-debian12:nonroot` (provides glibc for the CGO binary). + +- [ ] **Step 4: Verify locally** + +Run: `make lint && CGO_ENABLED=1 go test ./... 2>&1 | tail -20 && make build && ./bin/lab-cx version` +Expected: lint clean, all tests pass, binary runs. + +- [ ] **Step 5: Commit** + +```bash +git add .github/workflows/ci.yaml Dockerfile Makefile +git commit -s -m "build: enable CGO for tree-sitter (CI + distroless/base + Makefile)" +``` + +--- + +## Out of scope (deferred, not naive-critical) + +- MinHash/LSH for dedup (`github.com/ekzhu/minhash-lsh`) — current O(n²) Jaccard is correct; revisit if a profiler flags it on large transcripts. +- cmdfilter as data-driven config / machine-readable (`go test -json`) parsing — current regex rules are correct for the enumerated tools; expand coverage later. +- Real RLM REPL loop (iterative `print`/`llm_query` fan-out) — Task 6's Starlark "code" strategy + chunked "rlm" cover the high-value cases; the full recursive REPL is a follow-up. +- Tier-2 real-Python subprocess sandbox — Starlark in-process covers JSON projection; add only if a hard memory ceiling or full Python semantics is needed. +- Gemini surface; tiktoken→true Claude token counts (needs Anthropic count_tokens API). diff --git a/docs/superpowers/plans/2026-06-24-validate-config-measure-document.md b/docs/superpowers/plans/2026-06-24-validate-config-measure-document.md new file mode 100644 index 0000000..5a328c4 --- /dev/null +++ b/docs/superpowers/plans/2026-06-24-validate-config-measure-document.md @@ -0,0 +1,69 @@ +# Validate, Configure, Measure & Document Implementation Plan + +> **For agentic workers:** Executed via superpowers:subagent-driven-development — fresh subagent per task, review between tasks. Real model runs use the IBM LiteLLM gateway (claude-haiku-4-5) already in the session env. + +**Goal:** Prove every component works (unit + real runs), add a generic/extendable config file, measure & expose real token/cost/latency reduction on real fixtures, integrate for real with Claude Code (haiku) and eval-containers/SWE-bench, and document install/run/results. + +**Architecture:** Keep public API stable. Add a config-file loader → `config.Settings` + component registry; a measurement harness reusing winnow/headroom/rtk fixtures; a `/stats` aggregation on the proxy. Real model calls go through `cheapmodel` (bearer-auth) and through Claude Code pointed at the proxy. + +## Global Constraints + +- Go 1.25.0, CGO enabled. `make lint test build` green. DCO sign-off (`git commit -s`), author Osher-Elhadad, no `Co-Authored-By`/"Generated with". +- **Honesty:** real numbers only where actually measured; runbooks (not invented numbers) where a run wasn't executed. Every results doc states exactly how it was produced (model, fixtures, command, date). +- Model: gateway base `ANTHROPIC_BASE_URL` + bearer `ANTHROPIC_AUTH_TOKEN` (in session env; also `/tmp/lcx_env.sh`). Haiku id `claude-haiku-4-5`. **Never echo the token** into logs/commits/docs. +- Fail-open preserved everywhere. + +--- + +### Task 1: cheapmodel bearer auth + header config +**Files:** `internal/cheapmodel/anthropic.go`, `openai.go`, `cheapmodel_test.go`; `cmd/proxy/main.go`. +- Add an `AuthScheme` ("x-api-key" default | "bearer") or auto: if the key looks like a gateway token / a `--extract-auth bearer` flag is set, send `Authorization: Bearer ` instead of `x-api-key`. OpenAI already uses Bearer. +- Add `--extract-auth` flag (default "x-api-key") + read base/token from env (`ANTHROPIC_AUTH_TOKEN` fallback for the key). +- Test: httptest asserts the bearer header path sends `Authorization: Bearer`. + +### Task 2: Generic, extendable config file +**Files:** `config/file.go` (+ test), `cmd/proxy/main.go`, example `configs/lab-cx.yaml`. +- A declarative config (YAML; use `gopkg.in/yaml.v3`) that maps to `Settings` AND a **component registry**: `stages: [reduce, extract, cache]` (order + enable), `reducers:` (collapse/skeleton/format/dedup/cmdfilter on/off), `encoders:` (json_compact/toon/csv/... on/off + rank), `extract: {mode, strategies:[code,single,rlm,deterministic], floor, provider, model}`. +- The registry is keyed by NAME so adding an extractor/encoder/stage tomorrow = a new registered name + a config entry, no core edit. Document the extension point in a doc comment + RUNNING.md. +- `lab-cx proxy --config configs/lab-cx.yaml`. Unknown keys error; missing keys fall back to preset/defaults. +- Test: load a config that disables extract + enables only TOON; assert resulting Settings + active component set. + +### Task 3: Metrics — measure & expose +**Files:** `observability/` (+ a cost rate table), `internal/proxyhttp/proxy.go` (`/stats`), `cmd/proxy` (`lab-cx stats`). +- Aggregate per-request `Report`s into a process-wide stat: requests, tokens_before/after/saved, ratio, by-stage savings, est. cost saved (USD via a small per-model rate table, configurable), p50/p95 added latency. +- Expose `GET /stats` (JSON) + a human summary line; `lab-cx stats` prints the latest. Document the metric definitions. +- Test: feed two Events, assert aggregation + /stats JSON shape. + +### Task 4: Fetch headroom/rtk + measurement harness +**Files:** fetch `rtk-ai/rtk` and the headroom source into `../` (or vendor fixtures under `testdata/fixtures/`); `cmd/labcx-bench/main.go` (or `scripts/measure.go`); `testdata/fixtures/*`. +- Pull REAL fixtures: rtk command-output samples (pytest/cargo/npm logs), headroom/winnow MCP-JSON + large file-read + search-result fixtures (mine `../winnow/benchmarks/data`). +- Harness runs each component in isolation AND the full pipeline over each fixture, emitting a table: fixture, component, tokens_before, tokens_after, %saved, bytes, lossless? (containment/expand check), latency. +- Run it for real on the deterministic components; commit the fixtures + the generated `docs/RESULTS-offline.md` table (real numbers). +- Test: harness has a unit test on one fixture asserting reduction > 0 and round-trip recoverable. + +### Task 5: Real LLM-extractor measurement (haiku) +**Files:** extend the harness with an `--extract` mode; `docs/RESULTS-extract.md`. +- Source the gateway env, run the extractor (code/single/rlm) with claude-haiku-4-5 over the structured/large fixtures; record tokens_before/after, %saved, strategy chosen, containment-verified, latency, and (sampled) that the result is a faithful subset. +- Write real numbers + the exact command to RESULTS-extract.md. Note cost. + +### Task 6: Real Claude Code + haiku integration +**Files:** `docs/integration/claude-code.md`, capture script `scripts/cc-demo.sh`. +- Start `lab-cx proxy` (extract on, haiku, bearer) pointing upstream at the gateway. Run a small real `claude -p ""` with `ANTHROPIC_BASE_URL=http://localhost:PORT` against a throwaway repo; capture the proxy `/stats` before/after (tokens saved, cache hits) and confirm the task still completes. +- Document exact env + commands + the measured reduction. Honest: if Claude Code's auth can't traverse the proxy, document that and fall back to a tool-calling agent demo. + +### Task 7: eval-containers SWE-bench (wire + runbook + tiny slice) +**Files:** `deploy/eval-containers/` (already has compose override + README); `docs/integration/swe-bench.md`. +- Finalize the before-gateway wiring so `lab-cx` reduces traffic with claude-code as the agent and claude-haiku-4-5 as the model via the IBM gateway. Build the lab-cx image. +- Write the exact runbook (env, compose command, where token/cost land in result.json/trajectory.jsonl). +- If Docker + time + budget allow, run **1–3 SWE-bench tasks** with and without lab-cx and record the real token/cost delta + resolve status; else clearly mark the runbook as not-yet-executed. + +### Task 8: Docs — README + RUNNING + RESULTS +**Files:** `README.md`, `docs/RUNNING.md`, `docs/RESULTS.md` (links the per-area results). +- README: what it is, install (incl. CGO/C toolchain, `make build`), every run mode (proxy, wrap, config file, stats), the three integrations, and a results summary with real numbers. +- RUNNING.md: per-component how-to + the config-extension guide. RESULTS.md: consolidated real measurements + how each was produced + caveats. + +--- + +## Out of scope / caveats +- Full SWE-bench (300+ tasks) — too costly; a tiny slice or runbook only. +- True Claude-token billing accuracy — tiktoken o200k is the offline proxy; gateway usage is authoritative where available. diff --git a/engine/engine.go b/engine/engine.go new file mode 100644 index 0000000..5ee1f72 --- /dev/null +++ b/engine/engine.go @@ -0,0 +1,167 @@ +// Package engine is the public entry point: it runs the stage pipeline over a +// canonical request and exposes Transform (reduce a request, fail-open) and Expand +// (recover a collapsed block). A host — the proxy binary, an eval-containers adapter, +// or a Kagenti AuthBridge plugin — wraps an Engine; the engine has no transport code. +package engine + +import ( + "context" + + "github.com/kagenti/lab-context-engineering/canon" + "github.com/kagenti/lab-context-engineering/config" + "github.com/kagenti/lab-context-engineering/internal/cache" + "github.com/kagenti/lab-context-engineering/internal/markers" + "github.com/kagenti/lab-context-engineering/internal/reduce" + "github.com/kagenti/lab-context-engineering/internal/store" +) + +// FindMarkers returns the rewind ids of any reversible markers in text. A host uses +// this to serve an expand request or to rehydrate collapsed blocks before forwarding +// (e.g. ahead of a summarization turn). Pair with Engine.Expand to recover originals. +func FindMarkers(text string) []string { return markers.FindIDs(text) } + +// ExtractFunc applies cheap-model extraction to large candidate outputs in place. +// It is injected by the host (the engine core has no model client). nil disables the +// Extract stage. It must fail open: on any error, leave candidates untouched. +type ExtractFunc func(ctx context.Context, req canon.Request, cands []reduce.Candidate) error + +// Context is the per-request state a stage may read. Plain services only — no +// transport types — so the same stages run in any host. +type Context struct { + Settings config.Settings + Store store.Rewind + Evictions *store.Eviction + ClientCacheFloor int // cache.FloorIndex of the incoming request + StickyIDs map[string]struct{} // ids reduced on prior turns (cache stability) + Extract ExtractFunc + GoCtx context.Context +} + +// Stage is one transformation over the canonical request. +type Stage interface { + Name() string + Enabled(*Context) bool + Run(req canon.Request, agg *Report, ctx *Context) error +} + +// Report aggregates what the pipeline did. Embeds the reduce report and adds +// cross-stage info. +type Report struct { + Skipped bool + StageErrors int + Candidates []reduce.Candidate + CacheInjected bool + Reduce reduce.Report +} + +// Engine runs stages over requests. Construct with New. +type Engine struct { + settings config.Settings + store store.Rewind + evict *store.Eviction + stages []Stage + extract ExtractFunc +} + +// builtinStages maps a stage NAME to its built-in implementation. Stage selection in a +// config file ("stages: [reduce, extract, cache]") is resolved through this table, so a +// stage added tomorrow is wired on/off purely by name — register it here, then list it. +var builtinStages = map[string]func() Stage{ + "reduce": func() Stage { return ReduceStage{} }, + "extract": func() Stage { return ExtractStage{} }, + "cache": func() Stage { return CacheStage{} }, +} + +// defaultStageOrder is the pipeline used when Settings.Stages is empty. +var defaultStageOrder = []string{"reduce", "extract", "cache"} + +// stagesFor resolves the stage list from names, falling back to the default order when +// the list is empty. Unknown names are skipped (fail-open: a typo can't crash startup). +func stagesFor(names []string) []Stage { + order := names + if len(order) == 0 { + order = defaultStageOrder + } + out := make([]Stage, 0, len(order)) + for _, n := range order { + if mk, ok := builtinStages[n]; ok { + out = append(out, mk()) + } + } + return out +} + +// New builds an engine with the configured stage pipeline (default: Reduce → Extract → +// Cache, or Settings.Stages by name when provided). +func New(settings config.Settings, st store.Rewind, ev *store.Eviction) *Engine { + if st == nil { + st = store.NewMemory(0) + } + if ev == nil { + ev = store.NewEviction() + } + return &Engine{ + settings: settings, store: st, evict: ev, + stages: stagesFor(settings.Stages), + } +} + +// SetExtract injects the cheap-model extraction function (enables the Extract stage). +func (e *Engine) SetExtract(fn ExtractFunc) { e.extract = fn } + +// RegisterStage inserts a custom stage at index (appended if index < 0 or too large). +func (e *Engine) RegisterStage(s Stage, index int) { + if index < 0 || index > len(e.stages) { + e.stages = append(e.stages, s) + return + } + e.stages = append(e.stages[:index], append([]Stage{s}, e.stages[index:]...)...) +} + +// Store exposes the rewind store so the host can serve expand requests. +func (e *Engine) Store() store.Rewind { return e.store } + +// Expand recovers the original content for a rewind id. +func (e *Engine) Expand(id string) (string, bool) { return e.store.Get(id) } + +// Transform reduces req in place and returns a Report. It never returns an error: +// any stage failure is isolated (the request is restored to its pre-stage state) and +// counted, so the worst case is a no-op forward. The returned request is the one the +// host should forward. +func (e *Engine) Transform(goctx context.Context, req canon.Request) (canon.Request, Report) { + if e.settings.Disabled { + return req, Report{Skipped: true} + } + ctx := &Context{ + Settings: e.settings, Store: e.store, Evictions: e.evict, + ClientCacheFloor: cache.FloorIndex(req.Root), + Extract: e.extract, GoCtx: goctx, + } + var agg Report + for _, st := range e.stages { + if !st.Enabled(ctx) { + continue + } + if err := safeRun(st, req, &agg, ctx); err != nil { + agg.StageErrors++ + } + } + return req, agg +} + +// safeRun isolates a stage: it snapshots the request and restores it if the stage +// returns an error or panics, so a faulty stage cannot corrupt the forwarded request. +func safeRun(st Stage, req canon.Request, agg *Report, ctx *Context) (err error) { + snapshot := req.Clone() + defer func() { + if r := recover(); r != nil { + req.Root = snapshot.Root + err = errFromPanic(r) + } + }() + if e := st.Run(req, agg, ctx); e != nil { + req.Root = snapshot.Root + return e + } + return nil +} diff --git a/engine/engine_test.go b/engine/engine_test.go new file mode 100644 index 0000000..5b29783 --- /dev/null +++ b/engine/engine_test.go @@ -0,0 +1,81 @@ +package engine + +import ( + "context" + "strings" + "testing" + + "github.com/kagenti/lab-context-engineering/canon" + "github.com/kagenti/lab-context-engineering/config" + "github.com/kagenti/lab-context-engineering/internal/markers" +) + +func bigText(n int) string { + return strings.Repeat("a.go some reasonably long line of file content here\n", n) +} + +// TestTransformReducesAndCaches: a stale read is collapsed (reduce) and the request +// gets cache breakpoints (cache stage, since the client didn't self-cache). +func TestTransformReducesAndCaches(t *testing.T) { + body := []byte(`{"system":"be helpful","tools":[{"name":"read"}],"messages":[ + {"role":"user","content":[{"type":"text","text":"fix bug"}]}, + {"role":"assistant","content":[{"type":"tool_use","id":"u1","name":"read","input":{"file_path":"a.go"}}]}, + {"role":"user","content":[{"type":"tool_result","tool_use_id":"u1","content":"` + strings.ReplaceAll(bigText(12), "\n", "\\n") + `"}]}, + {"role":"assistant","content":[{"type":"tool_use","id":"u2","name":"edit","input":{"file_path":"a.go"}}]}, + {"role":"user","content":[{"type":"tool_result","tool_use_id":"u2","content":"success: edited"}]}, + {"role":"user","content":[{"type":"text","text":"thanks"}]} + ]}`) + req, err := canon.Decode(body) + if err != nil { + t.Fatalf("decode: %v", err) + } + s := config.Default() + s.ProtectRecent = 1 + e := New(s, nil, nil) + + out, rep := e.Transform(context.Background(), req) + + if rep.Reduce.TokensSaved <= 0 { + t.Fatalf("expected reduction; before=%d after=%d", rep.Reduce.TokensBefore, rep.Reduce.TokensAfter) + } + if !rep.CacheInjected { + t.Fatalf("expected cache stage to run on a non-self-caching client") + } + // The collapsed block carries a marker recoverable via Expand. + out2, _ := out.Encode() + ids := markers.FindIDs(string(out2)) + if len(ids) == 0 { + t.Fatalf("expected a winnow marker in the reduced request") + } + if _, ok := e.Expand(ids[0]); !ok { + t.Fatalf("Expand could not recover marker %s", ids[0]) + } +} + +// TestTransformDisabledIsNoop verifies the global kill switch forwards untouched. +func TestTransformDisabledIsNoop(t *testing.T) { + body := []byte(`{"messages":[{"role":"user","content":[{"type":"text","text":"hi"}]}]}`) + req, _ := canon.Decode(body) + s := config.Default() + s.Disabled = true + e := New(s, nil, nil) + _, rep := e.Transform(context.Background(), req) + if !rep.Skipped { + t.Fatalf("disabled engine should skip") + } +} + +// TestSelfCachingClientSkipsCacheStage: when the client already set cache_control, +// the cache stage stands down (ClientCacheFloor >= 0). +func TestSelfCachingClientSkipsCacheStage(t *testing.T) { + body := []byte(`{"messages":[ + {"role":"user","content":[{"type":"text","text":"hi","cache_control":{"type":"ephemeral"}}]}, + {"role":"assistant","content":[{"type":"text","text":"hello"}]} + ]}`) + req, _ := canon.Decode(body) + e := New(config.Default(), nil, nil) + _, rep := e.Transform(context.Background(), req) + if rep.CacheInjected { + t.Fatalf("cache stage should stand down for a self-caching client") + } +} diff --git a/engine/extract_cache_test.go b/engine/extract_cache_test.go new file mode 100644 index 0000000..b59b13a --- /dev/null +++ b/engine/extract_cache_test.go @@ -0,0 +1,111 @@ +package engine + +import ( + "context" + "encoding/json" + "strings" + "sync" + "testing" + + "github.com/kagenti/lab-context-engineering/canon" + "github.com/kagenti/lab-context-engineering/config" +) + +// recordingModel returns the first two records of the JSON array embedded in the +// prompt (a contained subset, so extraction succeeds and is cached) and records +// every prompt it is asked to complete. The recorded count makes a wrongly-shared +// cache observable: a body-only key calls the model once across two goals; a +// goal-aware key calls it once per goal. +type recordingModel struct { + mu sync.Mutex + prompts []string +} + +func (m *recordingModel) Complete(_ context.Context, prompt string) (string, error) { + m.mu.Lock() + m.prompts = append(m.prompts, prompt) + m.mu.Unlock() + + const marker = "INPUT (return a smaller value of this same shape):\n" + i := strings.Index(prompt, marker) + if i < 0 { + return "", nil + } + rest := prompt[i+len(marker):] + if e := strings.Index(rest, "\n\n"); e >= 0 { + rest = rest[:e] + } + var arr []any + if json.Unmarshal([]byte(rest), &arr) == nil && len(arr) >= 2 { + b, _ := json.Marshal(arr[:2]) + return string(b), nil + } + return "", nil +} + +func (m *recordingModel) calls() int { + m.mu.Lock() + defer m.mu.Unlock() + return len(m.prompts) +} + +// reqWithGoal builds a request whose tool_result body is the shared large array +// and whose trailing user text is the goal. Using the "single" strategy forces the +// model to receive the body inline (so recordingModel can echo a contained subset) +// and keeps the per-request model-call count deterministic at one. +func reqWithGoal(t *testing.T, arr, goal string) canon.Request { + t.Helper() + body := []byte(`{"messages":[ + {"role":"assistant","content":[{"type":"tool_use","id":"u1","name":"list_items","input":{}}]}, + {"role":"user","content":[{"type":"tool_result","tool_use_id":"u1","content":` + jsonStr(arr) + `}]}, + {"role":"user","content":[{"type":"text","text":` + jsonStr(goal) + `}]} + ]}`) + req, err := canon.Decode(body) + if err != nil { + t.Fatalf("decode: %v", err) + } + return req +} + +// TestExtractionCacheIsGoalAware asserts that the same tool-output body re-read under +// a DIFFERENT agent goal does NOT reuse the first goal's filtered result. Observed at +// the cache level: with a body-only key the model is called once (goal B reuses goal +// A's entry); with the goal-aware key it is called twice (once per goal). +func TestExtractionCacheIsGoalAware(t *testing.T) { + var recs []string + for i := 0; i < 400; i++ { + recs = append(recs, `{"id":`+itoa(i)+`,"name":"item_`+itoa(i)+`","detail":"some detail text for this record"}`) + } + arr := "[" + strings.Join(recs, ",") + "]" + + model := &recordingModel{} + s := config.Default() + s.ProtectRecent = 1 + eng := New(s, nil, nil) + // "single" forces the inline-body strategy so each goal makes exactly one model + // call; the cache key is what decides whether goal B reuses goal A. + cfg := DefaultExtractConfig() + cfg.Mode = "single" + eng.EnableExtract(model, cfg) + + reqA := reqWithGoal(t, arr, "goal A: inspect the items for duplicate ids") + reqB := reqWithGoal(t, arr, "goal B: summarize the names of every record") + + if _, rep := eng.Transform(context.Background(), reqA); len(rep.Candidates) == 0 { + t.Fatalf("goal A: expected at least one extraction candidate") + } + afterA := model.calls() + if afterA == 0 { + t.Fatalf("goal A: expected the model to be called") + } + + if _, rep := eng.Transform(context.Background(), reqB); len(rep.Candidates) == 0 { + t.Fatalf("goal B: expected at least one extraction candidate") + } + afterB := model.calls() + + if afterB <= afterA { + t.Fatalf("goal B reused goal A's cached result (cache is body-only, not goal-aware): "+ + "model calls after A=%d, after B=%d", afterA, afterB) + } +} diff --git a/engine/extract_engine_test.go b/engine/extract_engine_test.go new file mode 100644 index 0000000..94c73f6 --- /dev/null +++ b/engine/extract_engine_test.go @@ -0,0 +1,84 @@ +package engine + +import ( + "context" + "encoding/json" + "strings" + "testing" + + "github.com/kagenti/lab-context-engineering/canon" + "github.com/kagenti/lab-context-engineering/config" +) + +// firstTwoModel returns the first two records of the JSON array embedded in the +// prompt — a content-aware, always-contained selection. +type firstTwoModel struct{} + +func (firstTwoModel) Complete(_ context.Context, prompt string) (string, error) { + const marker = "INPUT (return a smaller value of this same shape):\n" + i := strings.Index(prompt, marker) + if i < 0 { + return "", nil + } + rest := prompt[i+len(marker):] + if e := strings.Index(rest, "\n\n"); e >= 0 { + rest = rest[:e] + } + var arr []any + if json.Unmarshal([]byte(rest), &arr) == nil && len(arr) >= 2 { + b, _ := json.Marshal(arr[:2]) + return string(b), nil + } + return "", nil +} + +func TestExtractionEndToEnd(t *testing.T) { + // A large structured tool output (well over the 3000-token floor) that the + // deterministic path would only re-encode; extraction selects 2 records. + var recs []string + for i := 0; i < 400; i++ { + recs = append(recs, `{"id":`+itoa(i)+`,"name":"item_`+itoa(i)+`","detail":"some detail text for this record"}`) + } + arr := "[" + strings.Join(recs, ",") + "]" + + body := []byte(`{"messages":[ + {"role":"assistant","content":[{"type":"tool_use","id":"u1","name":"list_items","input":{}}]}, + {"role":"user","content":[{"type":"tool_result","tool_use_id":"u1","content":` + jsonStr(arr) + `}]}, + {"role":"user","content":[{"type":"text","text":"now inspect the results"}]} + ]}`) + req, err := canon.Decode(body) + if err != nil { + t.Fatalf("decode: %v", err) + } + + s := config.Default() + s.ProtectRecent = 1 + eng := New(s, nil, nil) + eng.EnableExtract(firstTwoModel{}, DefaultExtractConfig()) + + out, rep := eng.Transform(context.Background(), req) + if len(rep.Candidates) == 0 { + t.Fatalf("expected at least one extraction candidate") + } + outBody, _ := out.Encode() + ids := FindMarkers(string(outBody)) + if len(ids) == 0 { + t.Fatalf("expected extracted block to carry a recoverable marker") + } + if _, ok := eng.Expand(ids[0]); !ok { + t.Fatalf("Expand failed for extracted marker") + } + if len(outBody) >= len(body) { + t.Fatalf("extraction did not shrink the request: %d vs %d", len(outBody), len(body)) + } +} + +func itoa(n int) string { + b, _ := json.Marshal(n) + return string(b) +} + +func jsonStr(s string) string { + b, _ := json.Marshal(s) + return string(b) +} diff --git a/engine/extractfunc.go b/engine/extractfunc.go new file mode 100644 index 0000000..c1ff36e --- /dev/null +++ b/engine/extractfunc.go @@ -0,0 +1,172 @@ +package engine + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "strings" + + "github.com/kagenti/lab-context-engineering/canon" + "github.com/kagenti/lab-context-engineering/internal/extract" + "github.com/kagenti/lab-context-engineering/internal/markers" + "github.com/kagenti/lab-context-engineering/internal/reduce" + "github.com/kagenti/lab-context-engineering/internal/tokens" +) + +const goalTurns = 6 // recent turns whose text conditions extraction + keep-set + +// Model is the cheap-model client extraction calls. A host (the proxy, or a Kagenti +// AuthBridge plugin) implements it; the engine has no model client of its own. +type Model interface { + Complete(ctx context.Context, prompt string) (string, error) +} + +// ExtractConfig configures cheap-model extraction (public mirror of the internal cfg). +type ExtractConfig struct { + Mode string // "auto" | "single" | "rlm" | "deterministic" + Floor int // token floor; rlm kicks in at max(floor*4,8000) in auto + MinKeepRatio float64 // 0 disables the blunt ratio backstop + AllowDeterministic bool + MaxChars int + // Strategies, when non-empty, restricts the strategy order to these names by-name + // (code | single | rlm | deterministic). Empty means "all". + Strategies []string +} + +// DefaultExtractConfig returns sensible extraction defaults. +func DefaultExtractConfig() ExtractConfig { + c := extract.DefaultCfg() + return ExtractConfig{Mode: c.Mode, Floor: c.Floor, AllowDeterministic: c.AllowDeterministic, MaxChars: c.MaxChars} +} + +func (c ExtractConfig) internal() extract.Cfg { + return extract.Cfg{Mode: c.Mode, Floor: c.Floor, MinKeepRatio: c.MinKeepRatio, + AllowDeterministic: c.AllowDeterministic, MaxChars: c.MaxChars, + AllowedStrategies: c.Strategies} +} + +// EnableExtract turns on cheap-model extraction with the given model and config. It +// builds the candidate→extraction→reversible-splice adapter and registers it; the +// Extract stage then runs after Reduce. Fail-open per candidate. Settings that name +// component selection (ExtractMode, ExtractStrategies) override cfg when set, so a +// config file fully drives the run. +func (e *Engine) EnableExtract(model Model, cfg ExtractConfig) { + if e.settings.ExtractMode != "" { + cfg.Mode = e.settings.ExtractMode + } + if len(e.settings.ExtractStrategies) > 0 { + cfg.Strategies = e.settings.ExtractStrategies + } + icfg := cfg.internal() + cache := extract.NewCache() + e.settings.ExtractEnabled = true + e.extract = func(ctx context.Context, req canon.Request, cands []reduce.Candidate) error { + goal := recentGoalText(req, goalTurns) + keep := extract.HarvestIdentifiers(goal, 60) + for _, c := range cands { + func() { + defer func() { _ = recover() }() // fail-open per candidate + key := goalKey(extract.ContentKey(c.Text), goal, keep) + result, ok := cache.Get(key) + if !ok { + var strat string + result, strat = extract.RunExtraction(ctx, c.Text, goal, keep, c.TokenEst, icfg, model) + if strat == "none" || result == "" { + return + } + cache.Put(key, result) + } + e.splice(req, c, result) + }() + } + return nil + } +} + +// goalKey makes the extraction cache goal-aware: the cache value is the FILTERED +// result, which depends on the goal and keep-set, not just the body. Keying on body +// alone would let the same tool output re-read under a different goal reuse the first +// goal's filtered result. The key composites the body's content key with the goal and +// the keep-set so a different goal is a cache miss. +func goalKey(contentKey, goal string, keep []string) string { + h := sha256.New() + h.Write([]byte(contentKey)) + h.Write([]byte{0}) + h.Write([]byte(goal)) + for _, k := range keep { + h.Write([]byte{0}) + h.Write([]byte(k)) + } + return hex.EncodeToString(h.Sum(nil))[:24] +} + +// splice replaces the candidate block with the extracted result plus a reversible +// recovery marker (original stored), only if that is strictly smaller. +func (e *Engine) splice(req canon.Request, c reduce.Candidate, result string) { + block := blockAt(req, c.MsgIndex, c.BlockIndex) + if block == nil { + return + } + rid := e.store.Put(c.Text) + label := c.FilePath + if label == "" { + label = c.ToolName + } + if label == "" { + label = "tool output" + } + newText := strings.TrimRight(result, "\n") + "\n" + markers.RecoveryNote(label, "extracted", rid) + if tokens.Count(newText) >= tokens.Count(c.Text) { + return // never inflate + } + switch block["type"] { + case "tool_result": + block["content"] = newText + case "text": + block["text"] = newText + } +} + +func blockAt(req canon.Request, mi, bi int) map[string]any { + msgs := req.Messages() + if mi < 0 || mi >= len(msgs) { + return nil + } + list, ok := msgs[mi]["content"].([]any) + if !ok || bi < 0 || bi >= len(list) { + return nil + } + blk, _ := list[bi].(map[string]any) + return blk +} + +// recentGoalText concatenates the text of the last k turns — the agent's current +// focus — excluding tool_result content (harvesting ids from the very output being +// extracted would defeat extraction). +func recentGoalText(req canon.Request, k int) string { + msgs := req.Messages() + start := len(msgs) - k + if start < 0 { + start = 0 + } + var out []string + for _, m := range msgs[start:] { + switch c := m["content"].(type) { + case string: + out = append(out, c) + case []any: + for _, b := range c { + if bb, ok := b.(map[string]any); ok && bb["type"] == "text" { + if t, ok := bb["text"].(string); ok { + out = append(out, t) + } + } + } + } + } + s := strings.Join(out, "\n") + if len(s) > 6000 { + s = s[:6000] + } + return s +} diff --git a/engine/integration_test.go b/engine/integration_test.go new file mode 100644 index 0000000..a42b063 --- /dev/null +++ b/engine/integration_test.go @@ -0,0 +1,63 @@ +package engine_test + +// This file imports ONLY the public packages, exactly as an external consumer (e.g. +// the Kagenti AuthBridge plugin in kagenti-extensions) would. It stands in for that +// plugin: take the raw request body the host already holds, reduce it, forward the +// rendered bytes, and recover an omitted block via Expand. If this compiles and +// passes, the public integration surface is intact. + +import ( + "context" + "strings" + "testing" + + "github.com/kagenti/lab-context-engineering/canon" + "github.com/kagenti/lab-context-engineering/config" + "github.com/kagenti/lab-context-engineering/engine" + "github.com/kagenti/lab-context-engineering/surfaces" +) + +func TestPublicAPI_AuthBridgeStyleWrap(t *testing.T) { + // What a host plugin does, end to end. + s := config.Default() + s.ProtectRecent = 1 + eng := engine.New(s, nil, nil) + surface := surfaces.Anthropic{} // pick by the request's wire format + + rawBody := []byte(`{"messages":[ + {"role":"assistant","content":[{"type":"tool_use","id":"u1","name":"read","input":{"file_path":"a.go"}}]}, + {"role":"user","content":[{"type":"tool_result","tool_use_id":"u1","content":"` + + strings.Repeat("a.go a fairly long line of read content goes here\\n", 12) + `"}]}, + {"role":"assistant","content":[{"type":"tool_use","id":"u2","name":"edit","input":{"file_path":"a.go"}}]}, + {"role":"user","content":[{"type":"tool_result","tool_use_id":"u2","content":"success: edited"}]}, + {"role":"user","content":[{"type":"text","text":"thanks"}]} + ]}`) + + req, token, err := surface.ToInternal(rawBody) + if err != nil { + t.Fatalf("ToInternal: %v", err) + } + reduced, _ := eng.Transform(context.Background(), req) + outBody, err := surface.Render(reduced, token) + if err != nil { + t.Fatalf("Render: %v", err) + } + if len(outBody) >= len(rawBody) { + t.Fatalf("expected reduced body to be smaller: %d vs %d", len(outBody), len(rawBody)) + } + + // The host can recover any omitted block from the marker (e.g. to serve an + // expand tool call, or to rehydrate before a summarization turn). + ids := engine.FindMarkers(string(outBody)) + if len(ids) == 0 { + t.Fatalf("expected a recoverable marker in the reduced body") + } + if _, ok := eng.Expand(ids[0]); !ok { + t.Fatalf("Expand failed for %s", ids[0]) + } + + // A re-decode of the forwarded bytes is still a valid canonical request. + if _, err := canon.Decode(outBody); err != nil { + t.Fatalf("forwarded body is not valid JSON: %v", err) + } +} diff --git a/engine/stages.go b/engine/stages.go new file mode 100644 index 0000000..f1aeafd --- /dev/null +++ b/engine/stages.go @@ -0,0 +1,81 @@ +package engine + +import ( + "fmt" + + "github.com/kagenti/lab-context-engineering/canon" + "github.com/kagenti/lab-context-engineering/internal/cache" + "github.com/kagenti/lab-context-engineering/internal/reduce" +) + +func errFromPanic(r any) error { return fmt.Errorf("stage panic: %v", r) } + +// ReduceStage runs the deterministic, lossless-first reduction pass. +type ReduceStage struct{} + +func (ReduceStage) Name() string { return "reduce" } +func (ReduceStage) Enabled(c *Context) bool { return c.Settings.ReduceEnabled } + +func (ReduceStage) Run(req canon.Request, agg *Report, c *Context) error { + s := c.Settings + opts := reduce.DefaultOpts() + opts.ProtectRecent = s.ProtectRecent + opts.ProtectRecentToolUses = s.ProtectRecentToolUses + opts.ProvableOnly = s.ProvableOnly + opts.CollapseOutputs = s.CollapseOutputs + opts.ContextLimit = s.ContextLimit + opts.ReduceCachedPrefix = s.ReduceCachedPrefix + opts.CmdFilter = s.CmdFilter + opts.RehydrateOnCompaction = s.RehydrateOnCompaction + opts.EnabledReducers = s.Reducers + opts.EnabledEncoders = s.Encoders + opts.CacheFloor = c.ClientCacheFloor + opts.StickyIDs = c.StickyIDs + // Only mark candidates when an extractor is wired; otherwise leave large outputs + // for the deterministic actions to handle. + opts.LLMCompact = c.Extract != nil && s.ExtractEnabled + opts.LLMCompactFloor = s.LLMCompactFloor + opts.LLMCompactStructuredOnly = s.LLMCompactStructuredOnly + + rep := reduce.ReduceRequest(req, c.Store, c.Evictions, opts) + agg.Reduce = rep + agg.Candidates = rep.LLMCandidates + return nil +} + +// ExtractStage hands the large candidate outputs to the injected cheap-model +// extractor. No-op when no extractor is wired or there are no candidates. +type ExtractStage struct{} + +func (ExtractStage) Name() string { return "extract" } +func (ExtractStage) Enabled(c *Context) bool { + return c.Settings.ExtractEnabled && c.Extract != nil +} + +func (ExtractStage) Run(req canon.Request, agg *Report, c *Context) error { + if len(agg.Candidates) == 0 { + return nil + } + return c.Extract(c.GoCtx, req, agg.Candidates) +} + +// CacheStage injects ephemeral cache_control breakpoints on the stable prefix. Only +// active when the client did not already self-cache (ClientCacheFloor < 0), matching +// winnow: a self-caching client (e.g. Claude Code) keeps its own breakpoints. +type CacheStage struct{} + +func (CacheStage) Name() string { return "cache" } +func (CacheStage) Enabled(c *Context) bool { + return c.Settings.CacheEnabled && c.ClientCacheFloor < 0 +} + +func (CacheStage) Run(req canon.Request, agg *Report, c *Context) error { + s := c.Settings + anchor := -1 + if msgs, ok := req.Root["messages"].([]any); ok { + anchor = cache.ProtectedAnchorIndex(msgs, s.ProtectRecent, s.ProtectRecentToolUses) + } + cache.Inject(req.Root, s.CacheBreakpoints, s.CacheStableGap, anchor, s.CacheToolsBreakpoint) + agg.CacheInjected = true + return nil +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..67720d1 --- /dev/null +++ b/go.mod @@ -0,0 +1,32 @@ +module github.com/kagenti/lab-context-engineering + +go 1.25.0 + +require ( + github.com/alexaandru/go-sitter-forest/c v1.9.4 + github.com/alexaandru/go-sitter-forest/c_sharp v1.9.6 + github.com/alexaandru/go-sitter-forest/cpp v1.9.5 + github.com/alexaandru/go-sitter-forest/go v1.9.4 + github.com/alexaandru/go-sitter-forest/java v1.9.5 + github.com/alexaandru/go-sitter-forest/javascript v1.9.2 + github.com/alexaandru/go-sitter-forest/kotlin v1.9.4 + github.com/alexaandru/go-sitter-forest/php v1.9.5 + github.com/alexaandru/go-sitter-forest/python v1.9.10 + github.com/alexaandru/go-sitter-forest/ruby v1.9.3 + github.com/alexaandru/go-sitter-forest/rust v1.9.13 + github.com/alexaandru/go-sitter-forest/scala v1.9.8 + github.com/alexaandru/go-sitter-forest/swift v1.9.5 + github.com/alexaandru/go-sitter-forest/tsx v1.9.2 + github.com/alexaandru/go-sitter-forest/typescript v1.9.4 + github.com/tiktoken-go/tokenizer v0.7.0 + github.com/toon-format/toon-go v0.0.0-20251202084852-7ca0e27c4e8c + github.com/tree-sitter/go-tree-sitter v0.25.0 + go.starlark.net v0.0.0-20260613233743-8ba36ccb83fb + gopkg.in/yaml.v3 v3.0.1 +) + +require ( + github.com/dlclark/regexp2 v1.11.5 // indirect + github.com/mattn/go-pointer v0.0.1 // indirect + golang.org/x/sys v0.42.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..8ff308d --- /dev/null +++ b/go.sum @@ -0,0 +1,82 @@ +github.com/alexaandru/go-sitter-forest/c v1.9.4 h1:E7Qb0zN/DSQhsIaJvfKHIhk6QLxu+4DNi8OgoMpvyMM= +github.com/alexaandru/go-sitter-forest/c v1.9.4/go.mod h1:ycxxUak8lW4au1HrvwY93ztpiJyCjm0lUb0OYhXcvmk= +github.com/alexaandru/go-sitter-forest/c_sharp v1.9.6 h1:qGVeM/rs6ITL+WVZh0DVn6sHtDwuZpQIQwgmnVUbb9o= +github.com/alexaandru/go-sitter-forest/c_sharp v1.9.6/go.mod h1:Z7u9b/RpBzNY7Q17Rs0ASuRkDjXUn9eGOv0Hwsn8maw= +github.com/alexaandru/go-sitter-forest/cpp v1.9.5 h1:GudtwF4yPlT5NANI8+rU7xS3ilm03wVWghmuaorMzEw= +github.com/alexaandru/go-sitter-forest/cpp v1.9.5/go.mod h1:OVmJ399hre2mWHm+n0iOOZVcw9MHG2RuldxIvQKlbQ0= +github.com/alexaandru/go-sitter-forest/go v1.9.4 h1:Xpq8HijvTp2+AHL4UplLLiCOdIAA20u0JyM8NGBIBWs= +github.com/alexaandru/go-sitter-forest/go v1.9.4/go.mod h1:jCpJr1AOFPGq1DrAJevuurB04NJVk8jw3hYW3Mxc7G8= +github.com/alexaandru/go-sitter-forest/java v1.9.5 h1:ibPa56LZBAwQ9McU2CVEAwkaB3ManNfaGUDFcSOq6qs= +github.com/alexaandru/go-sitter-forest/java v1.9.5/go.mod h1:aWD8ZAZ6IbhsCnF7Jog1GWtx3ow//PiWkkAvVfnB8/I= +github.com/alexaandru/go-sitter-forest/javascript v1.9.2 h1:b8ZWUk/1bXLmGXN/tL26PQf0EQ6vSM7J6pnNjN8by2Q= +github.com/alexaandru/go-sitter-forest/javascript v1.9.2/go.mod h1:cWOFBHR7EffqM4sdVQ70WtTnFKPZxt5sAm1zS1tWaL8= +github.com/alexaandru/go-sitter-forest/kotlin v1.9.4 h1:H2cRqquwV3rbNsUGUvyRZKWwC4TMLEDjXs0jzbIZASE= +github.com/alexaandru/go-sitter-forest/kotlin v1.9.4/go.mod h1:QCAC6OJsnUIRMx1akoZNzKRe+slaQq4sGSLAVwMFTuQ= +github.com/alexaandru/go-sitter-forest/php v1.9.5 h1:t8FV0CrjobKKk941AJ5EZrLOeY6am25x/NR6iZx8emk= +github.com/alexaandru/go-sitter-forest/php v1.9.5/go.mod h1:LY33+NVll5+uKJ9YQiAFy/QcX02EHWlDlL/PPqAAjzg= +github.com/alexaandru/go-sitter-forest/python v1.9.10 h1:qKbso5wI4vlkQ0EjZBO5hRrxwqf0tmGLVdbzasZCcrg= +github.com/alexaandru/go-sitter-forest/python v1.9.10/go.mod h1:CXs3nuPcj4p3FqY8BnvJqsJ0vbsrda6Oj3sQDyRumNY= +github.com/alexaandru/go-sitter-forest/ruby v1.9.3 h1:3GdkatWtd0jXvhnxdqJCdM+9JwAQeTwtWStiDAjAgr0= +github.com/alexaandru/go-sitter-forest/ruby v1.9.3/go.mod h1:h+TaY3e2ayXHy1jgwLZ+Jnho97roJzmaefSZjFvUM9k= +github.com/alexaandru/go-sitter-forest/rust v1.9.13 h1:orVZVXQ+IkXkIvzeyO6YvHBwujEvbYFvpMRNAGulHmw= +github.com/alexaandru/go-sitter-forest/rust v1.9.13/go.mod h1:ZKZRL1Yfy6siuWMvv2h/F9lt/8kMKn6VHnGj00PPNmM= +github.com/alexaandru/go-sitter-forest/scala v1.9.8 h1:w3HtQ4aij4PnRTBsIHITHtym8WiPGBGNL5KIJ6qIofw= +github.com/alexaandru/go-sitter-forest/scala v1.9.8/go.mod h1:+CuFWqcheV8rHtOWWDADtP7az6uSZKQ6zE0DLhg+LeY= +github.com/alexaandru/go-sitter-forest/swift v1.9.5 h1:CCfvj4BRjvN7HtznqDbgU7ylHHO9ML34ezsJFbErjV0= +github.com/alexaandru/go-sitter-forest/swift v1.9.5/go.mod h1:EzSPcZpETNyJIoAyPdbQgFUxWM+vcO3y5eYh8kmNvNc= +github.com/alexaandru/go-sitter-forest/tsx v1.9.2 h1:R6CIvxcs6zGF/nwI7YbHzRM1HuRjL1g7nKYfDMR7ANE= +github.com/alexaandru/go-sitter-forest/tsx v1.9.2/go.mod h1:JIGhuMRPOeeUQbqrNiDh3YZV9iMtWTHUmTgSH8WMTIk= +github.com/alexaandru/go-sitter-forest/typescript v1.9.4 h1:k+zE1JbmcDjgqPxO0fVnCnsCFj0yWmRaLpp2sbC4MoA= +github.com/alexaandru/go-sitter-forest/typescript v1.9.4/go.mod h1:fzlkFeml5odd1gUkYOgiNXK4bF2M6hBcfTitiJPlso8= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZQ= +github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/mattn/go-pointer v0.0.1 h1:n+XhsuGeVO6MEAp7xyEukFINEa+Quek5psIR/ylA6o0= +github.com/mattn/go-pointer v0.0.1/go.mod h1:2zXcozF6qYGgmsG+SeTZz3oAbFLdD3OWqnUbNvJZAlc= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/tiktoken-go/tokenizer v0.7.0 h1:VMu6MPT0bXFDHr7UPh9uii7CNItVt3X9K90omxL54vw= +github.com/tiktoken-go/tokenizer v0.7.0/go.mod h1:6UCYI/DtOallbmL7sSy30p6YQv60qNyU/4aVigPOx6w= +github.com/toon-format/toon-go v0.0.0-20251202084852-7ca0e27c4e8c h1:D8lDFovBMZywze1eh9iwMLcYor5f11mHBocLhO7cBe8= +github.com/toon-format/toon-go v0.0.0-20251202084852-7ca0e27c4e8c/go.mod h1:j/BOnpF2ihnz4lELs99h9mwGJBx/zdleOUCnLLRPCsc= +github.com/tree-sitter/go-tree-sitter v0.25.0 h1:sx6kcg8raRFCvc9BnXglke6axya12krCJF5xJ2sftRU= +github.com/tree-sitter/go-tree-sitter v0.25.0/go.mod h1:r77ig7BikoZhHrrsjAnv8RqGti5rtSyvDHPzgTPsUuU= +github.com/tree-sitter/tree-sitter-c v0.23.4 h1:nBPH3FV07DzAD7p0GfNvXM+Y7pNIoPenQWBpvM++t4c= +github.com/tree-sitter/tree-sitter-c v0.23.4/go.mod h1:MkI5dOiIpeN94LNjeCp8ljXN/953JCwAby4bClMr6bw= +github.com/tree-sitter/tree-sitter-cpp v0.23.4 h1:LaWZsiqQKvR65yHgKmnaqA+uz6tlDJTJFCyFIeZU/8w= +github.com/tree-sitter/tree-sitter-cpp v0.23.4/go.mod h1:doqNW64BriC7WBCQ1klf0KmJpdEvfxyXtoEybnBo6v8= +github.com/tree-sitter/tree-sitter-embedded-template v0.23.2 h1:nFkkH6Sbe56EXLmZBqHHcamTpmz3TId97I16EnGy4rg= +github.com/tree-sitter/tree-sitter-embedded-template v0.23.2/go.mod h1:HNPOhN0qF3hWluYLdxWs5WbzP/iE4aaRVPMsdxuzIaQ= +github.com/tree-sitter/tree-sitter-go v0.23.4 h1:yt5KMGnTHS+86pJmLIAZMWxukr8W7Ae1STPvQUuNROA= +github.com/tree-sitter/tree-sitter-go v0.23.4/go.mod h1:Jrx8QqYN0v7npv1fJRH1AznddllYiCMUChtVjxPK040= +github.com/tree-sitter/tree-sitter-html v0.23.2 h1:1UYDV+Yd05GGRhVnTcbP58GkKLSHHZwVaN+lBZV11Lc= +github.com/tree-sitter/tree-sitter-html v0.23.2/go.mod h1:gpUv/dG3Xl/eebqgeYeFMt+JLOY9cgFinb/Nw08a9og= +github.com/tree-sitter/tree-sitter-java v0.23.5 h1:J9YeMGMwXYlKSP3K4Us8CitC6hjtMjqpeOf2GGo6tig= +github.com/tree-sitter/tree-sitter-java v0.23.5/go.mod h1:NRKlI8+EznxA7t1Yt3xtraPk1Wzqh3GAIC46wxvc320= +github.com/tree-sitter/tree-sitter-javascript v0.23.1 h1:1fWupaRC0ArlHJ/QJzsfQ3Ibyopw7ZfQK4xXc40Zveo= +github.com/tree-sitter/tree-sitter-javascript v0.23.1/go.mod h1:lmGD1EJdCA+v0S1u2fFgepMg/opzSg/4pgFym2FPGAs= +github.com/tree-sitter/tree-sitter-json v0.24.8 h1:tV5rMkihgtiOe14a9LHfDY5kzTl5GNUYe6carZBn0fQ= +github.com/tree-sitter/tree-sitter-json v0.24.8/go.mod h1:F351KK0KGvCaYbZ5zxwx/gWWvZhIDl0eMtn+1r+gQbo= +github.com/tree-sitter/tree-sitter-php v0.23.11 h1:iHewsLNDmznh8kgGyfWfujsZxIz1YGbSd2ZTEM0ZiP8= +github.com/tree-sitter/tree-sitter-php v0.23.11/go.mod h1:T/kbfi+UcCywQfUNAJnGTN/fMSUjnwPXA8k4yoIks74= +github.com/tree-sitter/tree-sitter-python v0.23.6 h1:qHnWFR5WhtMQpxBZRwiaU5Hk/29vGju6CVtmvu5Haas= +github.com/tree-sitter/tree-sitter-python v0.23.6/go.mod h1:cpdthSy/Yoa28aJFBscFHlGiU+cnSiSh1kuDVtI8YeM= +github.com/tree-sitter/tree-sitter-ruby v0.23.1 h1:T/NKHUA+iVbHM440hFx+lzVOzS4dV6z8Qw8ai+72bYo= +github.com/tree-sitter/tree-sitter-ruby v0.23.1/go.mod h1:kUS4kCCQloFcdX6sdpr8p6r2rogbM6ZjTox5ZOQy8cA= +github.com/tree-sitter/tree-sitter-rust v0.23.2 h1:6AtoooCW5GqNrRpfnvl0iUhxTAZEovEmLKDbyHlfw90= +github.com/tree-sitter/tree-sitter-rust v0.23.2/go.mod h1:hfeGWic9BAfgTrc7Xf6FaOAguCFJRo3RBbs7QJ6D7MI= +go.starlark.net v0.0.0-20260613233743-8ba36ccb83fb h1:NGUBN0jbH0IR3msRslALnoxlySm+6YvVKvVDjdDJrlA= +go.starlark.net v0.0.0-20260613233743-8ba36ccb83fb/go.mod h1:Iue6g6iirlfLoVi/DYCi5/x0h/bAOuWF3dULTKpt2Vo= +golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= +golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/buildinfo/buildinfo.go b/internal/buildinfo/buildinfo.go new file mode 100644 index 0000000..2679718 --- /dev/null +++ b/internal/buildinfo/buildinfo.go @@ -0,0 +1,9 @@ +// Package buildinfo exposes version metadata stamped at build time via -ldflags. +package buildinfo + +// Version is the released version, overridden at build time with +// -ldflags "-X github.com/kagenti/lab-context-engineering/internal/buildinfo.Version=...". +var Version = "dev" + +// Commit is the git commit the binary was built from. +var Commit = "none" diff --git a/internal/cache/cache.go b/internal/cache/cache.go new file mode 100644 index 0000000..3066782 --- /dev/null +++ b/internal/cache/cache.go @@ -0,0 +1,202 @@ +// Package cache injects Anthropic ephemeral cache_control breakpoints on the stable +// prefix of a request. Lossless: nothing is dropped or rewritten, only annotated +// where the provider may cache. Ported from winnow's cache.py. +// +// All functions operate on the canonical Root map (Anthropic shape) in place. +package cache + +const maxBreakpoints = 4 + +func ephemeral() map[string]any { return map[string]any{"type": "ephemeral"} } + +// asBlocks returns msg["content"] as a []any of block maps, or nil. +func msgContentList(msg map[string]any) ([]any, bool) { + c, ok := msg["content"].([]any) + return c, ok +} + +func hasCacheControl(block any) bool { + b, ok := block.(map[string]any) + if !ok { + return false + } + _, ok = b["cache_control"] + return ok +} + +// FloorIndex returns the message index of the last message carrying a cache_control +// breakpoint (-1 if none). Reducing any message at or before this index would bust +// the client's prompt cache, so reduction must stay below it. -1 means unrestricted. +func FloorIndex(root map[string]any) int { + floor := -1 + msgs, _ := root["messages"].([]any) + for i, m := range msgs { + mm, ok := m.(map[string]any) + if !ok { + continue + } + c, ok := msgContentList(mm) + if !ok { + continue + } + for _, b := range c { + if hasCacheControl(b) { + floor = i + break + } + } + } + return floor +} + +// ProtectedAnchorIndex returns the message index just behind the reducer's protected +// working set — the deepest place a stable breakpoint can sit and still be byte- +// identical next turn. Returns -1 if there's nothing to anchor. +func ProtectedAnchorIndex(msgs []any, protectRecent, protectRecentToolUses int) int { + n := len(msgs) + if n == 0 { + return -1 + } + protectFrom := n - protectRecent + if protectRecentToolUses > 0 { + var tu []int + for i, m := range msgs { + mm, ok := m.(map[string]any) + if !ok { + continue + } + c, ok := msgContentList(mm) + if !ok { + continue + } + for _, b := range c { + if bb, ok := b.(map[string]any); ok && bb["type"] == "tool_use" { + tu = append(tu, i) + break + } + } + } + if len(tu) >= protectRecentToolUses { + if cand := tu[len(tu)-protectRecentToolUses]; cand < protectFrom { + protectFrom = cand + } + } + } + if protectFrom-1 < 0 { + return 0 + } + return protectFrom - 1 +} + +func countBreakpoints(root map[string]any) int { + n := 0 + if sys, ok := root["system"].([]any); ok { + for _, b := range sys { + if hasCacheControl(b) { + n++ + } + } + } + msgs, _ := root["messages"].([]any) + for _, m := range msgs { + mm, ok := m.(map[string]any) + if !ok { + continue + } + if c, ok := msgContentList(mm); ok { + for _, b := range c { + if hasCacheControl(b) { + n++ + } + } + } + } + return n +} + +// markLastBlock adds cache_control to the last block of a message content, +// normalising a bare string into a single text block. Returns the new content. +func markLastBlock(content any) any { + switch c := content.(type) { + case string: + if c != "" { + return []any{map[string]any{"type": "text", "text": c, "cache_control": ephemeral()}} + } + case []any: + if len(c) > 0 { + if last, ok := c[len(c)-1].(map[string]any); ok { + if _, has := last["cache_control"]; !has { + last["cache_control"] = ephemeral() + } + } + } + } + return content +} + +// Inject annotates root in place with ephemeral cache breakpoints on the stable +// prefix. Safe/no-op once the provider cap (4) is reached. stableAnchorIdx (or -1 +// to fall back to a tail offset) pins the deep-stable breakpoint to recurring bytes. +func Inject(root map[string]any, breakpoints, stableGap, stableAnchorIdx int, toolsBreakpoint bool) { + budget := maxBreakpoints - countBreakpoints(root) + if budget <= 0 { + return + } + + // 0) tools: biggest static segment. + if toolsBreakpoint && budget > 0 { + if tools, ok := root["tools"].([]any); ok && len(tools) > 0 { + if last, ok := tools[len(tools)-1].(map[string]any); ok { + if _, has := last["cache_control"]; !has { + last["cache_control"] = ephemeral() + budget-- + } + } + } + } + + // 1) end of system: caches tools + system. + if budget > 0 { + switch sys := root["system"].(type) { + case string: + if sys != "" { + root["system"] = []any{map[string]any{"type": "text", "text": sys, "cache_control": ephemeral()}} + budget-- + } + case []any: + if len(sys) > 0 { + if last, ok := sys[len(sys)-1].(map[string]any); ok { + if _, has := last["cache_control"]; !has { + last["cache_control"] = ephemeral() + budget-- + } + } + } + } + } + + msgs, _ := root["messages"].([]any) + + // 2) deep-stable breakpoint, anchored to the protected-set boundary when given. + stableIdx := stableAnchorIdx + if !(stableAnchorIdx >= 0 && stableAnchorIdx < len(msgs)-1) { + gap := stableGap + if gap < 1 { + gap = 1 + } + stableIdx = len(msgs) - 1 - gap + } + if breakpoints >= 3 && budget > 0 && stableIdx >= 0 && stableIdx < len(msgs)-1 { + if m, ok := msgs[stableIdx].(map[string]any); ok { + m["content"] = markLastBlock(m["content"]) + budget-- + } + } + + // 3) rolling breakpoint on the last message. + if budget > 0 && len(msgs) > 0 { + if m, ok := msgs[len(msgs)-1].(map[string]any); ok { + m["content"] = markLastBlock(m["content"]) + } + } +} diff --git a/internal/cache/cache_test.go b/internal/cache/cache_test.go new file mode 100644 index 0000000..ccd4689 --- /dev/null +++ b/internal/cache/cache_test.go @@ -0,0 +1,52 @@ +package cache + +import ( + "testing" + + "github.com/kagenti/lab-context-engineering/canon" +) + +func TestInjectAddsBreakpointsAndRespectsCap(t *testing.T) { + req, _ := canon.Decode([]byte(`{ + "system":"you are helpful", + "tools":[{"name":"read"}], + "messages":[ + {"role":"user","content":[{"type":"text","text":"a"}]}, + {"role":"assistant","content":[{"type":"text","text":"b"}]}, + {"role":"user","content":[{"type":"text","text":"c"}]}, + {"role":"assistant","content":[{"type":"text","text":"d"}]} + ] + }`)) + Inject(req.Root, 4, 2, -1, true) + + if got := countBreakpoints(req.Root); got == 0 || got > maxBreakpoints { + t.Fatalf("breakpoints = %d, want 1..%d", got, maxBreakpoints) + } + // tools and system should each be marked. + if !hasCacheControl(req.Root["tools"].([]any)[0]) { + t.Fatal("tools breakpoint missing") + } + if _, ok := req.Root["system"].([]any); !ok { + t.Fatal("string system should be normalised to a marked block list") + } + + // Re-injecting must not exceed the cap (idempotent-ish, never over 4). + Inject(req.Root, 4, 2, -1, true) + if got := countBreakpoints(req.Root); got > maxBreakpoints { + t.Fatalf("breakpoints = %d exceeds cap", got) + } +} + +func TestFloorIndex(t *testing.T) { + req, _ := canon.Decode([]byte(`{"messages":[ + {"role":"user","content":[{"type":"text","text":"a","cache_control":{"type":"ephemeral"}}]}, + {"role":"assistant","content":[{"type":"text","text":"b"}]} + ]}`)) + if got := FloorIndex(req.Root); got != 0 { + t.Fatalf("FloorIndex = %d, want 0", got) + } + empty, _ := canon.Decode([]byte(`{"messages":[{"role":"user","content":[{"type":"text","text":"a"}]}]}`)) + if got := FloorIndex(empty.Root); got != -1 { + t.Fatalf("FloorIndex (no cache) = %d, want -1", got) + } +} diff --git a/internal/cheapmodel/anthropic.go b/internal/cheapmodel/anthropic.go new file mode 100644 index 0000000..f237c53 --- /dev/null +++ b/internal/cheapmodel/anthropic.go @@ -0,0 +1,87 @@ +// Package cheapmodel provides a minimal Anthropic Messages client used as the +// engine's injected extraction model. It implements engine.Model. (An OpenAI variant +// is a straightforward follow-up; the canonical model is Anthropic-shaped, so the +// Anthropic client is the natural first.) +package cheapmodel + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "strings" +) + +// Anthropic calls a small Anthropic model with a single user prompt and returns the +// text of the first content block. +type Anthropic struct { + BaseURL string // default https://api.anthropic.com + APIKey string + Model string // e.g. claude-haiku-4-5 + MaxTokens int // default 2048 + Client *http.Client + // AuthScheme selects how the API key is sent. "" or "x-api-key" sends the + // x-api-key header (Anthropic default). "bearer" sends + // Authorization: Bearer instead, for gateways (e.g. an IBM LiteLLM + // Anthropic-compatible endpoint) that authenticate with a bearer token. The + // anthropic-version header is sent in both cases. + AuthScheme string +} + +func (a Anthropic) Complete(ctx context.Context, prompt string) (string, error) { + base := a.BaseURL + if base == "" { + base = "https://api.anthropic.com" + } + maxTok := a.MaxTokens + if maxTok == 0 { + maxTok = 2048 + } + client := a.Client + if client == nil { + client = http.DefaultClient + } + reqBody, _ := json.Marshal(map[string]any{ + "model": a.Model, + "max_tokens": maxTok, + "messages": []any{map[string]any{"role": "user", "content": prompt}}, + }) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, + strings.TrimRight(base, "/")+"/v1/messages", bytes.NewReader(reqBody)) + if err != nil { + return "", err + } + req.Header.Set("content-type", "application/json") + if a.AuthScheme == "bearer" { + req.Header.Set("Authorization", "Bearer "+a.APIKey) + } else { + req.Header.Set("x-api-key", a.APIKey) + } + req.Header.Set("anthropic-version", "2023-06-01") + + resp, err := client.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("cheapmodel: status %d", resp.StatusCode) + } + var out struct { + Content []struct { + Text string `json:"text"` + } `json:"content"` + } + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + return "", err + } + // Return the first content block that carries text. A leading non-text block + // (e.g. "thinking") has an empty Text, so we skip it rather than returning "". + for _, c := range out.Content { + if c.Text != "" { + return c.Text, nil + } + } + return "", nil +} diff --git a/internal/cheapmodel/cheapmodel_test.go b/internal/cheapmodel/cheapmodel_test.go new file mode 100644 index 0000000..06df247 --- /dev/null +++ b/internal/cheapmodel/cheapmodel_test.go @@ -0,0 +1,75 @@ +package cheapmodel + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "testing" +) + +func TestAnthropicSkipsNonTextLeadingBlock(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.WriteString(w, `{"content":[{"type":"thinking","text":""},{"type":"text","text":"RESULT"}]}`) + })) + defer srv.Close() + got, err := Anthropic{BaseURL: srv.URL, Model: "m"}.Complete(context.Background(), "p") + if err != nil || got != "RESULT" { + t.Fatalf("got %q err %v", got, err) + } +} + +func TestAnthropicBearerAuth(t *testing.T) { + var gotAuth, gotKey, gotVersion string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotAuth = r.Header.Get("Authorization") + gotKey = r.Header.Get("x-api-key") + gotVersion = r.Header.Get("anthropic-version") + _, _ = io.WriteString(w, `{"content":[{"type":"text","text":"OK"}]}`) + })) + defer srv.Close() + _, err := Anthropic{BaseURL: srv.URL, APIKey: "tok", Model: "m", AuthScheme: "bearer"}.Complete(context.Background(), "p") + if err != nil { + t.Fatalf("err %v", err) + } + if gotAuth != "Bearer tok" { + t.Fatalf("Authorization = %q, want %q", gotAuth, "Bearer tok") + } + if gotKey != "" { + t.Fatalf("x-api-key = %q, want empty", gotKey) + } + if gotVersion != "2023-06-01" { + t.Fatalf("anthropic-version = %q, want 2023-06-01", gotVersion) + } +} + +func TestAnthropicDefaultAuthUsesAPIKey(t *testing.T) { + var gotAuth, gotKey string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotAuth = r.Header.Get("Authorization") + gotKey = r.Header.Get("x-api-key") + _, _ = io.WriteString(w, `{"content":[{"type":"text","text":"OK"}]}`) + })) + defer srv.Close() + _, err := Anthropic{BaseURL: srv.URL, APIKey: "tok", Model: "m"}.Complete(context.Background(), "p") + if err != nil { + t.Fatalf("err %v", err) + } + if gotKey != "tok" { + t.Fatalf("x-api-key = %q, want %q", gotKey, "tok") + } + if gotAuth != "" { + t.Fatalf("Authorization = %q, want empty", gotAuth) + } +} + +func TestOpenAIComplete(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.WriteString(w, `{"choices":[{"message":{"content":"OUT"}}]}`) + })) + defer srv.Close() + got, err := OpenAI{BaseURL: srv.URL, Model: "m"}.Complete(context.Background(), "p") + if err != nil || got != "OUT" { + t.Fatalf("got %q err %v", got, err) + } +} diff --git a/internal/cheapmodel/openai.go b/internal/cheapmodel/openai.go new file mode 100644 index 0000000..fe05a18 --- /dev/null +++ b/internal/cheapmodel/openai.go @@ -0,0 +1,70 @@ +package cheapmodel + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "strings" +) + +// OpenAI calls a small OpenAI chat-completions model with a single user prompt and +// returns the content of the first choice. It implements engine.Model. +type OpenAI struct { + BaseURL string // default https://api.openai.com + APIKey string + Model string // e.g. gpt-4o-mini + MaxTokens int // default 2048 + Client *http.Client +} + +func (o OpenAI) Complete(ctx context.Context, prompt string) (string, error) { + base := o.BaseURL + if base == "" { + base = "https://api.openai.com" + } + maxTok := o.MaxTokens + if maxTok == 0 { + maxTok = 2048 + } + client := o.Client + if client == nil { + client = http.DefaultClient + } + reqBody, _ := json.Marshal(map[string]any{ + "model": o.Model, + "max_tokens": maxTok, + "messages": []any{map[string]any{"role": "user", "content": prompt}}, + }) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, + strings.TrimRight(base, "/")+"/v1/chat/completions", bytes.NewReader(reqBody)) + if err != nil { + return "", err + } + req.Header.Set("content-type", "application/json") + req.Header.Set("authorization", "Bearer "+o.APIKey) + + resp, err := client.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("cheapmodel: status %d", resp.StatusCode) + } + var out struct { + Choices []struct { + Message struct { + Content string `json:"content"` + } `json:"message"` + } `json:"choices"` + } + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + return "", err + } + if len(out.Choices) == 0 { + return "", nil + } + return out.Choices[0].Message.Content, nil +} diff --git a/internal/extract/cache.go b/internal/extract/cache.go new file mode 100644 index 0000000..f128ded --- /dev/null +++ b/internal/extract/cache.go @@ -0,0 +1,27 @@ +package extract + +import "sync" + +// Cache memoizes extraction results by ContentKey, so the same large output re-sent +// on a later turn skips the paid model call. In-memory, process-local. +// +// ponytail: a plain map+mutex, unbounded. Add an LRU/TTL only if memory ever bites. +type Cache struct { + mu sync.Mutex + m map[string]string +} + +func NewCache() *Cache { return &Cache{m: map[string]string{}} } + +func (c *Cache) Get(key string) (string, bool) { + c.mu.Lock() + defer c.mu.Unlock() + v, ok := c.m[key] + return v, ok +} + +func (c *Cache) Put(key, val string) { + c.mu.Lock() + c.m[key] = val + c.mu.Unlock() +} diff --git a/internal/extract/contain.go b/internal/extract/contain.go new file mode 100644 index 0000000..a9c564b --- /dev/null +++ b/internal/extract/contain.go @@ -0,0 +1,101 @@ +// Package extract is the cheap-model tool-output extractor. A small model returns a +// SMALLER value of the same shape (selecting records/fields, never paraphrasing); the +// containment validator here PROVES the result is a lossless projection of the +// original, so a chatty model can never corrupt a value the agent relies on. On any +// failure it falls back to a deterministic projection, then to the original +// (fail-open). Ported from winnow's actions/llm_compact.py. +// +// ponytail: the model returns the selected JSON directly and containment verifies it — +// no model-generated code (winnow's Python sandbox) and no custom spec language. The +// containment proof is what makes the mechanism safe, whatever produced the subset. +package extract + +import ( + "encoding/json" + "strings" +) + +// IsContained reports whether result is a lossless projection of original (both +// parsed values): string → substring; list → order-preserving subsequence of +// contained items; dict → keys subset with contained values; numbers/bools/nil → +// equal where present; nil result → always allowed (dropping is fine). +func IsContained(result, original any) bool { + return checkContained(result, original) +} + +func checkContained(out, in any) bool { + if out == nil { + return true // dropping content is always allowed + } + if in == nil { + return false + } + switch o := out.(type) { + case string: + s, ok := in.(string) + return ok && strings.Contains(s, o) + case bool: + b, ok := in.(bool) + return ok && b == o + case json.Number: + return numbersEqual(o, in) + case float64: + return numbersEqual(o, in) + case []any: + il, ok := in.([]any) + if !ok || len(o) > len(il) { + return false + } + i := 0 + for _, item := range o { + matched := false + for i < len(il) { + if checkContained(item, il[i]) { + matched = true + i++ + break + } + i++ + } + if !matched { + return false + } + } + return true + case map[string]any: + im, ok := in.(map[string]any) + if !ok { + return false + } + for k, v := range o { + iv, present := im[k] + if !present { + return false // extra key not in input + } + if !checkContained(v, iv) { + return false + } + } + return true + default: + return out == in + } +} + +// numbersEqual compares two JSON-decoded numbers (json.Number or float64) by value. +func numbersEqual(a, b any) bool { + return numStr(a) == numStr(b) +} + +func numStr(v any) string { + switch n := v.(type) { + case json.Number: + return n.String() + case float64: + // json.Number-style rendering via json.Marshal keeps integers integral. + b, _ := json.Marshal(n) + return string(b) + default: + return "" + } +} diff --git a/internal/extract/deterministic.go b/internal/extract/deterministic.go new file mode 100644 index 0000000..a47dcd2 --- /dev/null +++ b/internal/extract/deterministic.go @@ -0,0 +1,180 @@ +package extract + +import ( + "regexp" + "strings" +) + +// Deterministic, model-free projection: filter a parsed value to the parts that match +// the keep-set (plus the important-key "spine"). Every leaf it emits is an unchanged +// value, a string prefix, or a contiguous window, so its output always passes +// IsContained by construction and is never empty. Ported from winnow's +// deterministic_project. + +var importantKeyTokens = []string{"id", "status", "state", "name", "error", "reason", "date", "time"} + +var jsonKeyRe = regexp.MustCompile(`"([A-Za-z_][A-Za-z0-9_\-]{1,})"\s*:`) + +func isImportantKey(keyLower string) bool { + for _, tok := range importantKeyTokens { + if strings.Contains(keyLower, tok) { + return true + } + } + return false +} + +func termMatches(term, keyLower string) bool { + return term == keyLower || strings.Contains(keyLower, term) || strings.Contains(term, keyLower) +} + +func truncateValue(value any, maxChars int) any { + if value == nil || maxChars <= 0 { + return value + } + switch v := value.(type) { + case string: + if r := []rune(v); len(r) > maxChars { + return string(r[:maxChars]) + } + return v + case []any: + out := make([]any, len(v)) + for i, x := range v { + out[i] = truncateValue(x, maxChars) + } + return out + case map[string]any: + out := make(map[string]any, len(v)) + for k, x := range v { + out[k] = truncateValue(x, maxChars) + } + return out + } + return value +} + +func extractTextWindow(text string, terms []string, maxChars int) string { + runes := []rune(text) + if maxChars <= 0 || len(runes) <= maxChars { + return text + } + loweredRunes := []rune(strings.ToLower(text)) + for _, term := range terms { + t := strings.TrimSpace(strings.ToLower(term)) + if t == "" { + continue + } + // Index into the rune slice (not bytes) so the window math stays rune-aligned. + if idx := runeIndex(loweredRunes, []rune(t)); idx >= 0 { + start := idx - maxChars/4 + if start < 0 { + start = 0 + } + if start > len(runes)-maxChars { + start = len(runes) - maxChars + } + return string(runes[start : start+maxChars]) + } + } + return string(runes[:maxChars]) +} + +// runeIndex returns the index (in runes) of the first occurrence of sub in s, or -1. +func runeIndex(s, sub []rune) int { + if len(sub) == 0 { + return 0 + } + for i := 0; i+len(sub) <= len(s); i++ { + match := true + for j := range sub { + if s[i+j] != sub[j] { + match = false + break + } + } + if match { + return i + } + } + return -1 +} + +func projectMapping(m map[string]any, terms map[string]struct{}, maxChars int) map[string]any { + out := map[string]any{} + for key, value := range m { + kl := strings.ToLower(key) + keep := isImportantKey(kl) + if !keep { + for t := range terms { + if termMatches(t, kl) { + keep = true + break + } + } + } + if !keep { + if s, ok := value.(string); ok { + vl := strings.ToLower(s) + for t := range terms { + if strings.Contains(vl, t) { + keep = true + break + } + } + } + } + if keep { + out[key] = truncateValue(value, maxChars) + } + } + return out +} + +// DeterministicProject filters a parsed value to the keep-set + important-key spine. +// Never empties: a value that projects to nothing falls back to a truncated copy. +func DeterministicProject(value any, keepIDs []string, maxChars int) any { + terms := map[string]struct{}{} + for _, k := range keepIDs { + if k != "" { + terms[strings.ToLower(k)] = struct{}{} + } + } + switch v := value.(type) { + case string: + winTerms := append([]string{}, keepIDs...) + for i, m := range jsonKeyRe.FindAllStringSubmatch(v, -1) { + if i >= 16 { + break + } + winTerms = append(winTerms, m[1]) + } + return extractTextWindow(v, winTerms, maxChars) + case map[string]any: + if len(terms) > 0 { + if p := projectMapping(v, terms, maxChars); len(p) > 0 { + return p + } + } + return truncateValue(v, maxChars) + case []any: + var kept []any + for _, item := range v { + m, ok := item.(map[string]any) + if !ok { + kept = append(kept, truncateValue(item, maxChars)) + continue + } + if len(terms) > 0 { + if p := projectMapping(m, terms, maxChars); len(p) > 0 { + kept = append(kept, p) + } + } + } + if len(kept) > 0 { + return kept + } + return truncateValue(v, maxChars) + } + return truncateValue(value, maxChars) +} diff --git a/internal/extract/extract.go b/internal/extract/extract.go new file mode 100644 index 0000000..7549728 --- /dev/null +++ b/internal/extract/extract.go @@ -0,0 +1,350 @@ +package extract + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "regexp" + "strings" + "sync" + + "github.com/kagenti/lab-context-engineering/internal/markers" + "github.com/kagenti/lab-context-engineering/internal/tokens" +) + +const sampleChars = 4000 + +// Model is the cheap-model client the extractor calls. The host (proxy / plugin) +// injects a concrete implementation; the extractor core stays transport-free. +type Model interface { + Complete(ctx context.Context, prompt string) (string, error) +} + +// Cfg configures extraction. +type Cfg struct { + Mode string // auto | single | rlm | deterministic + Floor int // token floor; rlm kicks in at max(floor*4, 8000) in auto + MinKeepRatio float64 // 0 disables the blunt ratio backstop (keep-set check governs) + AllowDeterministic bool + MaxChars int // deterministic projection window + // AllowedStrategies, when non-empty, restricts strategyOrder to these strategy names + // (code | single | rlm | deterministic) preserving the computed order. Empty means + // "all" — prior behavior. Lets config enable/disable strategies purely by name. + AllowedStrategies []string +} + +// DefaultCfg mirrors winnow's ExtractCfg defaults. +func DefaultCfg() Cfg { + return Cfg{Mode: "auto", Floor: 3000, AllowDeterministic: true, MaxChars: sampleChars} +} + +var wsRe = regexp.MustCompile(`\s+`) + +// ContentKey is a stable, marker- and whitespace-insensitive key for a body, so the +// same output re-sent on a later turn hits the extraction cache. +func ContentKey(text string) string { + s := wsRe.ReplaceAllString(markers.Strip(text), " ") + s = strings.TrimSpace(s) + sum := sha256.Sum256([]byte(s)) + return hex.EncodeToString(sum[:])[:24] +} + +var identRe = regexp.MustCompile(`[A-Za-z_][\w./-]{3,}|\b\d{3,}\b`) + +// HarvestIdentifiers pulls distinctive identifiers (paths, symbols, ids, numbers) +// from the agent's recent turns — the keep-set the extractor must retain. +func HarvestIdentifiers(text string, cap int) []string { + var seen []string + idx := map[string]struct{}{} + for _, m := range identRe.FindAllString(text, -1) { + if _, ok := idx[m]; ok { + continue + } + idx[m] = struct{}{} + seen = append(seen, m) + if len(seen) >= cap { + break + } + } + return seen +} + +// parseBody returns the parsed value handed to the extractor: JSON if possible, then +// NDJSON (as a list), else the raw string. +func parseBody(text string) any { + var v any + dec := json.NewDecoder(strings.NewReader(text)) + dec.UseNumber() + if err := dec.Decode(&v); err == nil { + return v + } + if recs := parseNDJSON(text); recs != nil { + return recs + } + return text +} + +func parseNDJSON(text string) []any { + var lines []string + for _, ln := range strings.Split(text, "\n") { + if strings.TrimSpace(ln) != "" { + lines = append(lines, ln) + } + } + if len(lines) < 2 { + return nil + } + var recs []any + for _, ln := range lines { + var v any + dec := json.NewDecoder(strings.NewReader(ln)) + dec.UseNumber() + if err := dec.Decode(&v); err != nil { + return nil + } + switch v.(type) { + case map[string]any, []any: + recs = append(recs, v) + default: + return nil + } + } + return recs +} + +func resultToText(v any) string { + switch v.(type) { + case map[string]any, []any: + // Compact, not indented: this is the reduced tool-output value, and + // indentation whitespace inflates the BPE token count (it can exceed the + // original), which trips the never-inflate gate and silently drops the + // extraction. Compact JSON keeps the projection a real reduction. + b, err := json.Marshal(v) + if err == nil { + return string(b) + } + } + return fmt.Sprint(v) +} + +func stripFences(s string) string { + c := strings.TrimSpace(s) + if !strings.HasPrefix(c, "```") { + return c + } + lines := strings.Split(c, "\n") + if strings.HasPrefix(lines[0], "```") { + lines = lines[1:] + } + if len(lines) > 0 && strings.TrimSpace(lines[len(lines)-1]) == "```" { + lines = lines[:len(lines)-1] + } + return strings.Join(lines, "\n") +} + +// --- sanity + validation gate --- + +func extractionIsSane(bodyText, resultText string, keepIDs []string, minKeepRatio float64) bool { + if resultText == "" { + return false + } + bodyN, resN := tokens.Count(bodyText), tokens.Count(resultText) + switch strings.TrimSpace(resultText) { + case "", "[]", "{}", "null", `""`: + if bodyN > 0 { + return false + } + } + if minKeepRatio > 0 && float64(resN) < minKeepRatio*float64(bodyN) { + return false + } + for _, kid := range keepIDs { + if len(kid) >= 5 && strings.ContainsFunc(kid, isLetter) && + strings.Contains(bodyText, kid) && !strings.Contains(resultText, kid) { + return false + } + } + return true +} + +func isLetter(r rune) bool { + return (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') +} + +func validateExtraction(resultText, bodyText string, keepIDs []string, cfg Cfg) bool { + if !extractionIsSane(bodyText, resultText, keepIDs, cfg.MinKeepRatio) { + return false + } + return IsContained(parseBody(resultText), parseBody(bodyText)) +} + +// intersectAllowed filters order to the allowed set (non-empty), preserving order. +// Empty allowed means no filtering. +func intersectAllowed(order, allowed []string) []string { + if len(allowed) == 0 { + return order + } + allow := make(map[string]struct{}, len(allowed)) + for _, a := range allowed { + allow[a] = struct{}{} + } + out := make([]string, 0, len(order)) + for _, s := range order { + if _, ok := allow[s]; ok { + out = append(out, s) + } + } + return out +} + +func strategyOrder(tokenEst int, cfg Cfg) []string { + return intersectAllowed(rawStrategyOrder(tokenEst, cfg), cfg.AllowedStrategies) +} + +func rawStrategyOrder(tokenEst int, cfg Cfg) []string { + switch cfg.Mode { + case "deterministic": + return []string{"deterministic"} + case "single", "rlm": + order := []string{cfg.Mode} + if cfg.AllowDeterministic { + order = append(order, "deterministic") + } + return order + case "code": + order := []string{"code"} + if cfg.AllowDeterministic { + order = append(order, "deterministic") + } + return order + default: // auto + // "code" (model-written Starlark filter over the full body) is primary for + // mid-size bodies; "rlm" (chunked) above the floor. "single" (JSON-return) and + // "deterministic" are ordered fallbacks behind the primary. + floor4 := cfg.Floor * 4 + if floor4 < 8000 { + floor4 = 8000 + } + var order []string + if tokenEst >= floor4 { + order = []string{"rlm", "code", "single"} + } else { + order = []string{"code", "single"} + } + if cfg.AllowDeterministic { + order = append(order, "deterministic") + } + return order + } +} + +// RunExtraction tries strategies in order, returning the first candidate that is +// strictly smaller AND passes the validation gate, else ("", "none"). Fail-open: the +// caller keeps the original on "none". +func RunExtraction(ctx context.Context, body, goal string, keepIDs []string, tokenEst int, cfg Cfg, model Model) (string, string) { + base := tokens.Count(body) + for _, name := range strategyOrder(tokenEst, cfg) { + var cand string + switch name { + case "code": + cand = runStarlark(ctx, body, goal, keepIDs, model) + case "single": + cand = runSingle(ctx, body, goal, keepIDs, model) + case "rlm": + cand = runRLMBatched(ctx, body, goal, keepIDs, model) + case "deterministic": + cand = resultToText(DeterministicProject(parseBody(body), keepIDs, cfg.MaxChars)) + } + if cand == "" || tokens.Count(cand) >= base { + continue + } + if validateExtraction(cand, body, keepIDs, cfg) { + return cand, name + } + } + return "", "none" +} + +// runSingle asks the model for the filtered subset in one call. It is a FALLBACK +// behind the primary full-body strategies: "code" (a model-written Starlark filter) +// and "rlm" (chunked) both run over the FULL body, so they never truncate. runSingle +// inlines the body into the prompt via buildPrompt, which truncates to sampleChars to +// bound prompt cost — acceptable for a fallback, since whatever the model returns is +// still containment-checked against the full body before it can be spliced in. +func runSingle(ctx context.Context, body, goal string, keepIDs []string, model Model) string { + if model == nil { + return "" + } + out, err := model.Complete(ctx, buildPrompt(body, goal, keepIDs)) + if err != nil { + return "" + } + return stripFences(out) +} + +const rlmChunkSize = 20 +const rlmConcurrency = 6 + +// runRLMBatched chunks a large list body and asks the model to filter each chunk +// concurrently, then merges the kept records (order-preserving). Containment over the +// merged result is checked by the caller. Non-list bodies fall back to a single call. +func runRLMBatched(ctx context.Context, body, goal string, keepIDs []string, model Model) string { + if model == nil { + return "" + } + parsed := parseBody(body) + list, ok := parsed.([]any) + if !ok || len(list) <= rlmChunkSize { + return runSingle(ctx, body, goal, keepIDs, model) + } + var chunks [][]any + for i := 0; i < len(list); i += rlmChunkSize { + end := i + rlmChunkSize + if end > len(list) { + end = len(list) + } + chunks = append(chunks, list[i:end]) + } + + results := make([][]any, len(chunks)) + sem := make(chan struct{}, rlmConcurrency) + var wg sync.WaitGroup + for ci, chunk := range chunks { + wg.Add(1) + sem <- struct{}{} + go func(ci int, chunk []any) { + defer wg.Done() + defer func() { <-sem }() + defer func() { _ = recover() }() + chunkJSON, err := json.Marshal(chunk) + if err != nil { + return + } + out, err := model.Complete(ctx, buildPrompt(string(chunkJSON), goal, keepIDs)) + if err != nil { + return + } + var kept []any + if json.Unmarshal([]byte(stripFences(out)), &kept) == nil { + results[ci] = kept + } + // On parse failure the chunk contributes nothing (safe drop; containment holds). + }(ci, chunk) + } + wg.Wait() + + var merged []any + for _, r := range results { + merged = append(merged, r...) + } + if len(merged) == 0 { + return "" + } + b, err := json.Marshal(merged) + if err != nil { + return "" + } + return string(b) +} diff --git a/internal/extract/extract_test.go b/internal/extract/extract_test.go new file mode 100644 index 0000000..edf839d --- /dev/null +++ b/internal/extract/extract_test.go @@ -0,0 +1,156 @@ +package extract + +import ( + "context" + "encoding/json" + "strings" + "testing" + "unicode/utf8" +) + +func parse(s string) any { return parseBody(s) } + +func TestContainment(t *testing.T) { + original := parse(`{"id":1,"name":"alpha","tags":["x","y","z"],"note":"hello world"}`) + cases := []struct { + name string + out string + want bool + }{ + {"subset keys", `{"id":1,"name":"alpha"}`, true}, + {"substring value", `{"note":"hello"}`, true}, + {"subsequence list", `{"tags":["x","z"]}`, true}, + {"drop everything (nil-ish)", `{}`, true}, + {"extra key", `{"id":1,"missing":true}`, false}, + {"paraphrased string", `{"name":"ALPHA"}`, false}, + {"changed number", `{"id":2}`, false}, + {"invented value", `{"name":"omega"}`, false}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + if got := IsContained(parse(c.out), original); got != c.want { + t.Fatalf("IsContained(%s) = %v, want %v", c.out, got, c.want) + } + }) + } +} + +func TestContainmentListSubsequenceOrder(t *testing.T) { + orig := parse(`[{"id":1},{"id":2},{"id":3}]`) + if !IsContained(parse(`[{"id":1},{"id":3}]`), orig) { + t.Fatal("order-preserving subsequence should be contained") + } + if IsContained(parse(`[{"id":3},{"id":1}]`), orig) { + t.Fatal("out-of-order subsequence must NOT be contained") + } +} + +func TestDeterministicProjectIsAlwaysContained(t *testing.T) { + body := `[{"id":1,"name":"alpha","blob":"` + strings.Repeat("x", 200) + `"}, + {"id":2,"name":"beta","blob":"` + strings.Repeat("y", 200) + `"}]` + proj := DeterministicProject(parseBody(body), []string{"alpha"}, 4000) + if !IsContained(proj, parseBody(body)) { + t.Fatalf("deterministic projection must be contained: %v", proj) + } +} + +// fakeModel parses the INPUT embedded in the prompt and returns its first element +// (for arrays) — a content-aware, always-contained selection, like a well-behaved +// cheap model. Used to exercise the single + RLM paths. +type firstElemModel struct{} + +func (firstElemModel) Complete(_ context.Context, prompt string) (string, error) { + i := strings.Index(prompt, sampleMarker) + if i < 0 { + return "", nil + } + rest := prompt[i+len(sampleMarker):] + end := strings.Index(rest, "\n\n") + if end >= 0 { + rest = rest[:end] + } + var arr []any + if json.Unmarshal([]byte(rest), &arr) == nil && len(arr) > 0 { + b, _ := json.Marshal(arr[:1]) + return string(b), nil + } + return rest, nil +} + +// paraphraseModel always returns an invented value (fails containment). +type paraphraseModel struct{} + +func (paraphraseModel) Complete(_ context.Context, _ string) (string, error) { + return `[{"id":999,"name":"INVENTED"}]`, nil +} + +func TestRunExtractionSingleSelects(t *testing.T) { + body := `[{"id":1,"name":"alpha"},{"id":2,"name":"beta"},{"id":3,"name":"gamma"}]` + cfg := DefaultCfg() + cfg.Mode = "single" + cfg.AllowDeterministic = false + out, strat := RunExtraction(context.Background(), body, "find alpha", []string{"alpha"}, 100, cfg, firstElemModel{}) + if strat != "single" { + t.Fatalf("strategy = %q, want single", strat) + } + if !IsContained(parseBody(out), parseBody(body)) { + t.Fatalf("accepted result not contained: %s", out) + } + if len(out) >= len(body) { + t.Fatalf("result not smaller: %d vs %d", len(out), len(body)) + } +} + +func TestRunExtractionRejectsParaphraseFallsToDeterministic(t *testing.T) { + // Records carry a bulky non-important "detail" field that deterministic drops. + blob := strings.Repeat("noise ", 30) + body := `[{"id":1,"detail":"` + blob + `"},{"id":2,"detail":"` + blob + `"}]` + cfg := DefaultCfg() + cfg.Mode = "single" // single fails containment, then deterministic runs + out, strat := RunExtraction(context.Background(), body, "find alpha", []string{"alpha"}, 100, cfg, paraphraseModel{}) + if strat != "deterministic" { + t.Fatalf("strategy = %q, want deterministic (paraphrase must be rejected)", strat) + } + if !IsContained(parseBody(out), parseBody(body)) { + t.Fatalf("deterministic fallback not contained: %s", out) + } +} + +func TestRunExtractionRLMBatchedMergesChunks(t *testing.T) { + // 50 uniform records -> chunked (size 20) -> 3 chunks -> first of each merged. + var recs []string + for i := 0; i < 50; i++ { + recs = append(recs, `{"id":`+itoa(i)+`,"v":"rec`+itoa(i)+`"}`) + } + body := "[" + strings.Join(recs, ",") + "]" + cfg := DefaultCfg() + cfg.Mode = "rlm" + cfg.AllowDeterministic = false + out, strat := RunExtraction(context.Background(), body, "anything", nil, 100, cfg, firstElemModel{}) + if strat != "rlm" { + t.Fatalf("strategy = %q, want rlm", strat) + } + if !IsContained(parseBody(out), parseBody(body)) { + t.Fatalf("merged RLM result not contained: %s", out) + } + var merged []any + if err := json.Unmarshal([]byte(out), &merged); err != nil { + t.Fatalf("rlm output not a JSON array: %v", err) + } + if len(merged) != 3 { // one per chunk + t.Fatalf("expected 3 merged records, got %d", len(merged)) + } +} + +func TestTruncateValueDoesNotSplitRunes(t *testing.T) { + s := strings.Repeat("é", 10) // 2 bytes each + out := truncateValue(s, 5).(string) + if !utf8.ValidString(out) { + t.Fatalf("truncation split a rune: %q", out) + } +} + +func itoa(n int) string { + b, _ := json.Marshal(n) + return string(b) +} diff --git a/internal/extract/prompt.go b/internal/extract/prompt.go new file mode 100644 index 0000000..5c40157 --- /dev/null +++ b/internal/extract/prompt.go @@ -0,0 +1,147 @@ +package extract + +import ( + "encoding/json" + "strings" +) + +// Prompt building. Because the model returns the filtered VALUE (which containment +// then verifies), it must SEE the values — so the prompt shows the actual JSON/text +// (truncated). For very large lists the RLM strategy chunks the body so each chunk is +// shown in full. The rule set is winnow's "select, never summarize, recall-first" +// contract, retargeted from "write a function" to "return the JSON". + +// sampleMarker precedes the body in the prompt; tests and the (future) model both +// locate the payload after it. +const sampleMarker = "INPUT (return a smaller value of this same shape):\n" + +const rules = `Return ONLY a JSON value (or, for raw text input, the kept text): a SMALLER value +of the SAME shape, selecting only what the agent needs next. Rules, in priority order: +1. RECALL FIRST. When unsure whether a record/field is relevant, KEEP IT. +2. SELECT, NEVER SUMMARIZE. Return whole records/objects/values byte-for-byte. Never + paraphrase, truncate, round, reformat, or invent values. +3. PRESERVE EXACTLY: ids, numbers, names, paths, timestamps, error messages, stack + traces — and anything matching the KEEP list. +4. Only drop records that are CLEARLY irrelevant boilerplate, duplicates, or noise. +5. If you cannot identify clearly-irrelevant content, RETURN THE INPUT UNCHANGED. +6. Keep the natural shape and types. Output ONLY the value — no prose, no markdown.` + +const example = `EXAMPLE +Goal: "Fix failing test test_auth_expiry; find the relevant hit." +KEEP: ["test_auth_expiry","auth/session.py"] +INPUT: [{"path":"auth/session.py","snippet":"def test_auth_expiry()..."},{"path":"README.md","snippet":"intro"}] +OUTPUT: [{"path":"auth/session.py","snippet":"def test_auth_expiry()..."}]` + +func buildPrompt(bodyText, goal string, keepIDs []string) string { + g := strings.TrimSpace(goal) + if g == "" { + g = "(no explicit goal stated)" + } + if len(g) > 2000 { + g = g[:2000] + } + keep := keepIDs + if len(keep) > 60 { + keep = keep[:60] + } + keepBlock := "" + if len(keep) > 0 { + kb, _ := json.Marshal(keep) + keepBlock = "IDENTIFIERS THE AGENT REFERENCED RECENTLY — keep every record or field\n" + + "whose value matches any of these, verbatim:\n" + string(kb) + "\n\n" + } + + // Show the actual value (pretty-printed JSON if it parses), truncated. + sample := bodyText + if v := parseBody(bodyText); !isRawString(v) { + if b, err := json.MarshalIndent(v, "", " "); err == nil { + sample = string(b) + } + } + sample = truncate(sample, sampleChars) + + return "You filter ONE tool output down to only what the agent needs next.\n\n" + + "WHAT THE AGENT IS DOING NOW (filter toward this):\n" + g + "\n\n" + + keepBlock + sampleMarker + sample + "\n\n" + rules + "\n\n" + example +} + +// codeRules is the Starlark code-writing contract: the model writes a program that +// runs over the real INPUT (it is NOT shown the full body), so the prompt stays cheap. +// Same "select, never summarize, recall-first" discipline as buildPrompt, retargeted +// from "return the value" to "write the filter". +const codeRules = `Write a Starlark program (a safe Python subset) that filters this ONE tool output +down to only what the agent needs next. Contract: +- The global string INPUT holds the FULL tool output. The module ` + "`json`" + ` is available. +- Start: data = json.decode(INPUT) +- Select a SMALLER value of the SAME shape (e.g. drop irrelevant list records / fields). +- End: OUTPUT = json.encode(result) # OUTPUT must be a string +Rules, in priority order: +1. RECALL FIRST. When unsure whether a record/field is relevant, KEEP IT. +2. SELECT, NEVER SUMMARIZE. Keep whole records/values byte-for-byte. Never paraphrase, + truncate, round, reformat, or invent values — only DROP clearly-irrelevant ones. +3. PRESERVE EXACTLY: ids, numbers, names, paths, timestamps, errors, stack traces — + and anything matching the KEEP list. +4. NO imports (no load()), NO I/O, NO network. Use only json.decode/json.encode and + plain Starlark (list comprehensions, dict/list ops, string ops). +5. If you cannot identify clearly-irrelevant content, set OUTPUT = INPUT. +Output ONLY the Starlark program — no prose, no markdown fences.` + +const codeExample = `EXAMPLE +Goal: "Fix failing test test_auth_expiry; find the relevant hit." +KEEP: ["test_auth_expiry","auth/session.py"] +INPUT schema: list of {"path": str, "snippet": str} +PROGRAM: +data = json.decode(INPUT) +result = [r for r in data if "auth" in r["path"] or "test_auth_expiry" in r["snippet"]] +OUTPUT = json.encode(result)` + +// buildCodePrompt builds the prompt for the Starlark code-writing strategy. Unlike +// buildPrompt it does NOT inline the full body — the program runs over the real INPUT. +// It shows the parsed shape and a small sample so the model knows the schema cheaply. +func buildCodePrompt(bodyText, goal string, keepIDs []string) string { + g := strings.TrimSpace(goal) + if g == "" { + g = "(no explicit goal stated)" + } + if len(g) > 2000 { + g = g[:2000] + } + keep := keepIDs + if len(keep) > 60 { + keep = keep[:60] + } + keepBlock := "" + if len(keep) > 0 { + kb, _ := json.Marshal(keep) + keepBlock = "IDENTIFIERS THE AGENT REFERENCED RECENTLY — keep every record or field\n" + + "whose value matches any of these, verbatim:\n" + string(kb) + "\n\n" + } + + // A small sample of the parsed shape — enough to infer the schema, not the full body. + sample := bodyText + if v := parseBody(bodyText); !isRawString(v) { + if b, err := json.MarshalIndent(v, "", " "); err == nil { + sample = string(b) + } + } + sample = truncate(sample, codeSampleChars) + + return "You write a Starlark filter for ONE tool output, keeping only what the agent needs next.\n\n" + + "WHAT THE AGENT IS DOING NOW (filter toward this):\n" + g + "\n\n" + + keepBlock + "INPUT SAMPLE (the real INPUT at runtime is the FULL output of this same shape):\n" + + sample + "\n\n" + codeRules + "\n\n" + codeExample +} + +const codeSampleChars = 1500 + +func isRawString(v any) bool { + _, ok := v.(string) + return ok +} + +func truncate(s string, n int) string { + if len(s) <= n { + return s + } + return s[:n] +} diff --git a/internal/extract/starlark.go b/internal/extract/starlark.go new file mode 100644 index 0000000..96709ae --- /dev/null +++ b/internal/extract/starlark.go @@ -0,0 +1,63 @@ +package extract + +import ( + "context" + "time" + + starjson "go.starlark.net/lib/json" + "go.starlark.net/starlark" +) + +const ( + starlarkMaxSteps = 50_000_000 + starlarkTimeout = 2 * time.Second +) + +// runStarlark asks the model for a Starlark program whose contract is: read the +// global string INPUT (the full tool output), assign a string global OUTPUT (the +// filtered value). It runs sandboxed over the FULL body — no imports, no I/O, step + +// time limits — and returns OUTPUT, or "" on any failure (fail-open). Containment is +// verified by the caller (RunExtraction). +func runStarlark(ctx context.Context, body, goal string, keepIDs []string, model Model) (out string) { + defer func() { + if recover() != nil { + out = "" + } + }() + if model == nil { + return "" + } + src, err := model.Complete(ctx, buildCodePrompt(body, goal, keepIDs)) + if err != nil { + return "" + } + src = stripFences(src) + + ctx, cancel := context.WithTimeout(ctx, starlarkTimeout) + defer cancel() + thread := &starlark.Thread{Name: "extract"} // Load==nil => load() disabled + thread.SetMaxExecutionSteps(starlarkMaxSteps) + done := make(chan struct{}) + go func() { + select { + case <-ctx.Done(): + thread.Cancel(ctx.Err().Error()) + case <-done: + } + }() + defer close(done) + + predeclared := starlark.StringDict{ + "json": starjson.Module, + "INPUT": starlark.String(body), + } + globals, err := starlark.ExecFile(thread, "extract.star", src, predeclared) + if err != nil { + return "" + } + res, ok := globals["OUTPUT"].(starlark.String) + if !ok { + return "" + } + return string(res) +} diff --git a/internal/extract/starlark_test.go b/internal/extract/starlark_test.go new file mode 100644 index 0000000..6ae2f0f --- /dev/null +++ b/internal/extract/starlark_test.go @@ -0,0 +1,57 @@ +package extract + +import ( + "context" + "strings" + "testing" +) + +// starlarkModel returns a fixed Starlark program that keeps records whose name +// contains "keep" — exercises real code execution over the full input. +type starlarkModel struct{} + +func (starlarkModel) Complete(_ context.Context, _ string) (string, error) { + return ` +data = json.decode(INPUT) +kept = [r for r in data if "keep" in r["name"]] +OUTPUT = json.encode(kept) +`, nil +} + +func TestRunStarlarkFiltersFullBody(t *testing.T) { + var recs []string + for i := 0; i < 100; i++ { + name := "drop" + if i%10 == 0 { + name = "keep" + } + recs = append(recs, `{"id":`+itoa(i)+`,"name":"`+name+`"}`) + } + body := "[" + strings.Join(recs, ",") + "]" + out := runStarlark(context.Background(), body, "find keep", nil, starlarkModel{}) + if out == "" { + t.Fatal("expected a Starlark result") + } + if !IsContained(parseBody(out), parseBody(body)) { + t.Fatalf("Starlark output must be a contained subset: %s", out) + } + if strings.Contains(out, "drop") { + t.Fatal("filter should have dropped non-keep records") + } + if !strings.Contains(out, "keep") { + t.Fatal("filter should have kept the keep records (recall, not truncation)") + } +} + +// malicious program must fail-open (no panic, returns ""). +type evilModel struct{} + +func (evilModel) Complete(_ context.Context, _ string) (string, error) { + return `load("os", "x")`, nil // imports disabled +} + +func TestRunStarlarkFailsOpenOnDisallowed(t *testing.T) { + if out := runStarlark(context.Background(), `[{"a":1}]`, "", nil, evilModel{}); out != "" { + t.Fatalf("disallowed program must fail open to \"\", got %q", out) + } +} diff --git a/internal/markers/markers.go b/internal/markers/markers.go new file mode 100644 index 0000000..083ed21 --- /dev/null +++ b/internal/markers/markers.go @@ -0,0 +1,43 @@ +// Package markers handles reversible, namespaced content markers. winnow's marker +// is «winnow:HEXID»; foreign markers from other reducers (headroom, claw) are left +// alone so winnow stacks cleanly on top of them. Ported from winnow's markers.py. +package markers + +import ( + "fmt" + "regexp" +) + +var ( + winnowRe = regexp.MustCompile(`«winnow:([0-9a-f]{4,64})»`) + foreignRe = regexp.MustCompile(`<>|\[rewind:[0-9a-zA-Z]{4,64}\]`) +) + +// Make returns the marker for a rewind id. +func Make(rewindID string) string { return fmt.Sprintf("«winnow:%s»", rewindID) } + +// RecoveryNote is a self-advertising, model-readable note appended to a reduced +// block so the omission is a known unknown the model can recover, not a silent drop. +func RecoveryNote(label, what, rewindID string) string { + return fmt.Sprintf("[winnow: %s %s; call winnow_expand(%q) to restore] %s", + label, what, rewindID, Make(rewindID)) +} + +// FindIDs returns all winnow rewind ids referenced in text. +func FindIDs(text string) []string { + m := winnowRe.FindAllStringSubmatch(text, -1) + out := make([]string, 0, len(m)) + for _, g := range m { + out = append(out, g[1]) + } + return out +} + +// HasForeign reports whether text carries another reducer's marker. +func HasForeign(text string) bool { return foreignRe.MatchString(text) } + +// Strip removes winnow and foreign markers from text (used for a marker-insensitive +// content key). +func Strip(text string) string { + return foreignRe.ReplaceAllString(winnowRe.ReplaceAllString(text, ""), "") +} diff --git a/internal/proxyhttp/proxy.go b/internal/proxyhttp/proxy.go new file mode 100644 index 0000000..fb85159 --- /dev/null +++ b/internal/proxyhttp/proxy.go @@ -0,0 +1,278 @@ +// Package proxyhttp is the HTTP shell around the engine: a transparent proxy that +// reduces /v1/messages and /v1/chat/completions requests and forwards everything +// (including streaming responses and non-model endpoints) upstream unchanged. This +// is the "any agent" and eval-containers integration — agents point their +// *_BASE_URL at it. Fail-open: any error forwards the original request. +package proxyhttp + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "net/http" + "strings" + "time" + + "github.com/kagenti/lab-context-engineering/engine" + "github.com/kagenti/lab-context-engineering/observability" + "github.com/kagenti/lab-context-engineering/surfaces" +) + +// Config configures the proxy handler. +type Config struct { + Engine *engine.Engine + // Upstream, if set, is the base URL every request is forwarded to (the + // eval-containers gateway case). If empty, requests route to the provider + // default by path (api.anthropic.com / api.openai.com). + Upstream string + AnthropicDefault string // default "https://api.anthropic.com" + OpenAIDefault string // default "https://api.openai.com" + Client *http.Client + Emitter observability.Emitter // default observability.Nop + // Aggregator, if set, is served at GET /stats (Snapshot as JSON). It is + // independent of Emitter: a host may stream via Emitter and also expose + // process-wide stats here, or pass the same *observability.Aggregator as both. + Aggregator *observability.Aggregator + // MaxBodyBytes caps the request body read from clients; 0 means no cap. A body + // exceeding the cap yields HTTP 413. + MaxBodyBytes int64 + // UpstreamTimeout bounds each upstream request when Client is not set; 0 means + // no timeout (http.DefaultClient). + UpstreamTimeout time.Duration +} + +type proxy struct { + cfg Config +} + +// New returns the proxy http.Handler. Routing is by path SUFFIX so it works both for +// direct agents (POST /v1/messages) and for eval-containers, which prefix the wire +// path (POST /anthropic/v1/messages, /openai/v1/chat/completions). The full original +// path is preserved when forwarding upstream. +func New(cfg Config) http.Handler { + if cfg.Client == nil { + if cfg.UpstreamTimeout > 0 { + cfg.Client = &http.Client{Timeout: cfg.UpstreamTimeout} + } else { + cfg.Client = http.DefaultClient + } + } + if cfg.AnthropicDefault == "" { + cfg.AnthropicDefault = "https://api.anthropic.com" + } + if cfg.OpenAIDefault == "" { + cfg.OpenAIDefault = "https://api.openai.com" + } + if cfg.Emitter == nil { + cfg.Emitter = observability.Nop{} + } + return &proxy{cfg: cfg} +} + +func (p *proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { + path := r.URL.Path + switch { + case path == "/health" || path == "/ready": + p.ok(w, r) + case path == "/winnow/expand": + p.expand(w, r) + case path == "/stats": + p.stats(w, r) + case strings.HasSuffix(path, "/v1/messages"): + p.model(surfaces.Anthropic{}, p.cfg.AnthropicDefault)(w, r) + case strings.HasSuffix(path, "/chat/completions"): + p.model(surfaces.OpenAI{}, p.cfg.OpenAIDefault)(w, r) + default: + p.passthrough(p.cfg.AnthropicDefault)(w, r) + } +} + +func (p *proxy) ok(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) } + +// stats serves the process-wide reduction Snapshot as JSON. Returns 404 when no +// Aggregator is configured. +func (p *proxy) stats(w http.ResponseWriter, _ *http.Request) { + if p.cfg.Aggregator == nil { + http.Error(w, "stats not enabled", http.StatusNotFound) + return + } + w.Header().Set("Content-Type", "application/json") + _ = p.cfg.Aggregator.WriteJSON(w) +} + +func (p *proxy) expand(w http.ResponseWriter, r *http.Request) { + id := r.URL.Query().Get("id") + original, ok := p.cfg.Engine.Expand(id) + if !ok { + http.Error(w, "not found", http.StatusNotFound) + return + } + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + _, _ = io.WriteString(w, original) +} + +// upstreamBase returns the base URL to forward to: the configured single upstream, +// or the provider default. +func (p *proxy) upstreamBase(providerDefault string) string { + if p.cfg.Upstream != "" { + return p.cfg.Upstream + } + return providerDefault +} + +func (p *proxy) model(surface surfaces.Surface, providerDefault string) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + base := p.upstreamBase(providerDefault) + body, err := p.readBody(w, r) + if err != nil { + return // readBody already wrote the response + } + // Reduce unless bypassed or the surface can't map this format. + outBody := body + if !bypassed(r) { + start := time.Now() + reduced, rep, ok := p.reduce(r.Context(), surface, body) + latencyMs := int(time.Since(start).Milliseconds()) + if ok { + outBody = reduced + p.cfg.Emitter.Emit(r.Context(), observability.Event{ + System: surface.Name(), Surface: surface.Name(), + RequestModel: modelOf(body), + TokensBefore: rep.Reduce.TokensBefore, TokensAfter: rep.Reduce.TokensAfter, + TokensSaved: rep.Reduce.TokensSaved, Ratio: rep.Reduce.Ratio, + CacheInject: rep.CacheInjected, Extracted: len(rep.Candidates) > 0, + StageErrors: rep.StageErrors, + LatencyMillis: latencyMs, + }) + } + } + p.forward(w, r, base, outBody) + } +} + +// reduce maps → transforms → renders, returning the reduced body and report. ok=false +// means forward the original (unsupported surface or any failure — fail-open). +func (p *proxy) reduce(ctx context.Context, surface surfaces.Surface, body []byte) (out []byte, rep engine.Report, ok bool) { + defer func() { + if recover() != nil { + out, ok = nil, false + } + }() + req, token, err := surface.ToInternal(body) + if err != nil { + return nil, engine.Report{}, false + } + transformed, report := p.cfg.Engine.Transform(ctx, req) + rendered, err := surface.Render(transformed, token) + if err != nil { + return nil, engine.Report{}, false + } + return rendered, report, true +} + +// modelOf best-effort reads the "model" field from a request body for telemetry. +func modelOf(body []byte) string { + var m struct { + Model string `json:"model"` + } + _ = json.Unmarshal(body, &m) + return m.Model +} + +func (p *proxy) passthrough(providerDefault string) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + body, err := p.readBody(w, r) + if err != nil { + return // readBody already wrote the response + } + p.forward(w, r, p.upstreamBase(providerDefault), body) + } +} + +// readBody reads the full request body, enforcing cfg.MaxBodyBytes when set. On a +// cap overflow it writes HTTP 413; on any other read error it writes HTTP 400. In +// both error cases the response is already written and the caller must just return. +func (p *proxy) readBody(w http.ResponseWriter, r *http.Request) ([]byte, error) { + if p.cfg.MaxBodyBytes > 0 { + r.Body = http.MaxBytesReader(w, r.Body, p.cfg.MaxBodyBytes) + } + body, err := io.ReadAll(r.Body) + if err != nil { + var maxErr *http.MaxBytesError + if errors.As(err, &maxErr) { + http.Error(w, "request body too large", http.StatusRequestEntityTooLarge) + } else { + http.Error(w, "read body", http.StatusBadRequest) + } + return nil, err + } + return body, nil +} + +// hop-by-hop headers that must not be forwarded. +var hopHeaders = map[string]struct{}{ + "Connection": {}, "Keep-Alive": {}, "Proxy-Authenticate": {}, "Proxy-Authorization": {}, + "Te": {}, "Trailer": {}, "Transfer-Encoding": {}, "Upgrade": {}, +} + +// forward proxies the request to base+path with the given body and streams the +// response back (flushing per chunk so SSE streams pass through live). +func (p *proxy) forward(w http.ResponseWriter, r *http.Request, base string, body []byte) { + url := strings.TrimRight(base, "/") + r.URL.Path + if r.URL.RawQuery != "" { + url += "?" + r.URL.RawQuery + } + req, err := http.NewRequestWithContext(r.Context(), r.Method, url, bytes.NewReader(body)) + if err != nil { + http.Error(w, "bad upstream request", http.StatusBadGateway) + return + } + for k, vs := range r.Header { + if _, hop := hopHeaders[http.CanonicalHeaderKey(k)]; hop || k == "Host" { + continue + } + for _, v := range vs { + req.Header.Add(k, v) + } + } + req.ContentLength = int64(len(body)) + + resp, err := p.cfg.Client.Do(req) + if err != nil { + http.Error(w, "upstream error: "+err.Error(), http.StatusBadGateway) + return + } + defer resp.Body.Close() + + for k, vs := range resp.Header { + if _, hop := hopHeaders[http.CanonicalHeaderKey(k)]; hop { + continue + } + for _, v := range vs { + w.Header().Add(k, v) + } + } + w.WriteHeader(resp.StatusCode) + flusher, _ := w.(http.Flusher) + buf := make([]byte, 16*1024) + for { + n, rerr := resp.Body.Read(buf) + if n > 0 { + if _, werr := w.Write(buf[:n]); werr != nil { + return + } + if flusher != nil { + flusher.Flush() + } + } + if rerr != nil { + return + } + } +} + +func bypassed(r *http.Request) bool { + return strings.EqualFold(r.Header.Get("x-winnow-bypass"), "true") +} diff --git a/internal/proxyhttp/proxy_test.go b/internal/proxyhttp/proxy_test.go new file mode 100644 index 0000000..910243c --- /dev/null +++ b/internal/proxyhttp/proxy_test.go @@ -0,0 +1,232 @@ +package proxyhttp + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/kagenti/lab-context-engineering/config" + "github.com/kagenti/lab-context-engineering/engine" + "github.com/kagenti/lab-context-engineering/internal/markers" + "github.com/kagenti/lab-context-engineering/observability" +) + +// captureUpstream records the last body it received and returns a fixed response. +func captureUpstream(t *testing.T, got *string) *httptest.Server { + t.Helper() + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + b, _ := io.ReadAll(r.Body) + *got = string(b) + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"ok":true}`) + })) +} + +func newProxy(t *testing.T, upstream string) http.Handler { + t.Helper() + s := config.Default() + s.ProtectRecent = 1 + return New(Config{Engine: engine.New(s, nil, nil), Upstream: upstream}) +} + +func TestProxyReducesBeforeForwarding(t *testing.T) { + var upstreamGot string + up := captureUpstream(t, &upstreamGot) + defer up.Close() + h := newProxy(t, up.URL) + + big := strings.Repeat("a.go long file content line for the read result here\\n", 12) + body := `{"messages":[ + {"role":"assistant","content":[{"type":"tool_use","id":"u1","name":"read","input":{"file_path":"a.go"}}]}, + {"role":"user","content":[{"type":"tool_result","tool_use_id":"u1","content":"` + big + `"}]}, + {"role":"assistant","content":[{"type":"tool_use","id":"u2","name":"edit","input":{"file_path":"a.go"}}]}, + {"role":"user","content":[{"type":"tool_result","tool_use_id":"u2","content":"success: edited"}]}, + {"role":"user","content":[{"type":"text","text":"thanks"}]} + ]}` + + req := httptest.NewRequest(http.MethodPost, "/v1/messages", strings.NewReader(body)) + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d", rec.Code) + } + if len(upstreamGot) >= len(body) { + t.Fatalf("upstream body not reduced: got %d, original %d", len(upstreamGot), len(body)) + } + if len(markers.FindIDs(upstreamGot)) == 0 { + t.Fatalf("expected a winnow marker in the forwarded body") + } +} + +func TestBypassForwardsOriginal(t *testing.T) { + var upstreamGot string + up := captureUpstream(t, &upstreamGot) + defer up.Close() + h := newProxy(t, up.URL) + + body := `{"messages":[{"role":"user","content":[{"type":"text","text":"hi"}]}]}` + req := httptest.NewRequest(http.MethodPost, "/v1/messages", strings.NewReader(body)) + req.Header.Set("x-winnow-bypass", "true") + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + + if upstreamGot != body { + t.Fatalf("bypass should forward the original body verbatim:\n got: %s\nwant: %s", upstreamGot, body) + } +} + +func TestPassthroughForwardsUnknownPaths(t *testing.T) { + var upstreamGot string + up := captureUpstream(t, &upstreamGot) + defer up.Close() + h := newProxy(t, up.URL) + + req := httptest.NewRequest(http.MethodGet, "/v1/models", nil) + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("passthrough status = %d", rec.Code) + } +} + +// TestEvalContainersPrefixedPath: eval-containers posts to /anthropic/v1/messages; +// the proxy must still reduce it and forward the full prefixed path to the gateway. +func TestEvalContainersPrefixedPath(t *testing.T) { + var gotPath, gotBody string + up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + b, _ := io.ReadAll(r.Body) + gotBody = string(b) + _, _ = io.WriteString(w, `{"ok":true}`) + })) + defer up.Close() + h := newProxy(t, up.URL) + + big := strings.Repeat("a.go long file content line for the read result here\\n", 12) + body := `{"messages":[ + {"role":"assistant","content":[{"type":"tool_use","id":"u1","name":"read","input":{"file_path":"a.go"}}]}, + {"role":"user","content":[{"type":"tool_result","tool_use_id":"u1","content":"` + big + `"}]}, + {"role":"assistant","content":[{"type":"tool_use","id":"u2","name":"edit","input":{"file_path":"a.go"}}]}, + {"role":"user","content":[{"type":"tool_result","tool_use_id":"u2","content":"success: edited"}]}, + {"role":"user","content":[{"type":"text","text":"thanks"}]} + ]}` + req := httptest.NewRequest(http.MethodPost, "/anthropic/v1/messages", strings.NewReader(body)) + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + + if gotPath != "/anthropic/v1/messages" { + t.Fatalf("upstream path = %q, want the prefix preserved", gotPath) + } + if len(gotBody) >= len(body) { + t.Fatalf("prefixed request was not reduced") + } +} + +type captureEmitter struct{ events []observability.Event } + +func (c *captureEmitter) Emit(_ context.Context, e observability.Event) { + c.events = append(c.events, e) +} + +func TestProxyEmitsSavingsEvent(t *testing.T) { + var upstreamGot string + up := captureUpstream(t, &upstreamGot) + defer up.Close() + cap := &captureEmitter{} + s := config.Default() + s.ProtectRecent = 1 + h := New(Config{Engine: engine.New(s, nil, nil), Upstream: up.URL, Emitter: cap}) + + big := strings.Repeat("a.go long file content line for the read result here\\n", 12) + body := `{"model":"claude-x","messages":[ + {"role":"assistant","content":[{"type":"tool_use","id":"u1","name":"read","input":{"file_path":"a.go"}}]}, + {"role":"user","content":[{"type":"tool_result","tool_use_id":"u1","content":"` + big + `"}]}, + {"role":"assistant","content":[{"type":"tool_use","id":"u2","name":"edit","input":{"file_path":"a.go"}}]}, + {"role":"user","content":[{"type":"tool_result","tool_use_id":"u2","content":"success: edited"}]}, + {"role":"user","content":[{"type":"text","text":"thanks"}]} + ]}` + req := httptest.NewRequest(http.MethodPost, "/v1/messages", strings.NewReader(body)) + h.ServeHTTP(httptest.NewRecorder(), req) + + if len(cap.events) != 1 { + t.Fatalf("expected 1 emitted event, got %d", len(cap.events)) + } + e := cap.events[0] + if e.System != "anthropic" || e.RequestModel != "claude-x" || e.TokensSaved <= 0 { + t.Fatalf("unexpected event: %+v", e) + } +} + +func TestRequestBodyCap(t *testing.T) { + h := New(Config{Engine: engine.New(config.Default(), nil, nil), Upstream: "http://unused", MaxBodyBytes: 10}) + req := httptest.NewRequest(http.MethodPost, "/v1/messages", strings.NewReader(strings.Repeat("x", 100))) + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + if rec.Code != http.StatusRequestEntityTooLarge { + t.Fatalf("status=%d want 413", rec.Code) + } +} + +func TestStatsExposesSavings(t *testing.T) { + var upstreamGot string + up := captureUpstream(t, &upstreamGot) + defer up.Close() + + s := config.Default() + s.ProtectRecent = 1 + agg := observability.NewAggregator(map[string]observability.CostRate{ + observability.DefaultCostKey: {InputPerMTok: 1.0}, + }) + h := New(Config{Engine: engine.New(s, nil, nil), Upstream: up.URL, Emitter: agg, Aggregator: agg}) + + big := strings.Repeat("a.go long file content line for the read result here\\n", 12) + body := `{"model":"claude-x","messages":[ + {"role":"assistant","content":[{"type":"tool_use","id":"u1","name":"read","input":{"file_path":"a.go"}}]}, + {"role":"user","content":[{"type":"tool_result","tool_use_id":"u1","content":"` + big + `"}]}, + {"role":"assistant","content":[{"type":"tool_use","id":"u2","name":"edit","input":{"file_path":"a.go"}}]}, + {"role":"user","content":[{"type":"tool_result","tool_use_id":"u2","content":"success: edited"}]}, + {"role":"user","content":[{"type":"text","text":"thanks"}]} + ]}` + req := httptest.NewRequest(http.MethodPost, "/v1/messages", strings.NewReader(body)) + h.ServeHTTP(httptest.NewRecorder(), req) + + srec := httptest.NewRecorder() + h.ServeHTTP(srec, httptest.NewRequest(http.MethodGet, "/stats", nil)) + if srec.Code != http.StatusOK { + t.Fatalf("/stats status = %d", srec.Code) + } + var snap observability.Snapshot + if err := json.Unmarshal(srec.Body.Bytes(), &snap); err != nil { + t.Fatalf("unmarshal /stats: %v", err) + } + if snap.Requests != 1 { + t.Fatalf("Requests = %d, want 1", snap.Requests) + } + if snap.TokensSaved <= 0 { + t.Fatalf("TokensSaved = %d, want > 0", snap.TokensSaved) + } +} + +func TestStatsDisabledWhenNoAggregator(t *testing.T) { + h := newProxy(t, "http://unused") + rec := httptest.NewRecorder() + h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/stats", nil)) + if rec.Code != http.StatusNotFound { + t.Fatalf("/stats without aggregator = %d, want 404", rec.Code) + } +} + +func TestHealth(t *testing.T) { + h := newProxy(t, "http://unused") + req := httptest.NewRequest(http.MethodGet, "/health", nil) + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("health = %d", rec.Code) + } +} diff --git a/internal/reduce/actions.go b/internal/reduce/actions.go new file mode 100644 index 0000000..d759adf --- /dev/null +++ b/internal/reduce/actions.go @@ -0,0 +1,560 @@ +package reduce + +import ( + "bytes" + "encoding/csv" + "encoding/json" + "fmt" + "regexp" + "sort" + "strings" + + "github.com/kagenti/lab-context-engineering/internal/markers" + "github.com/kagenti/lab-context-engineering/internal/store" + "github.com/kagenti/lab-context-engineering/internal/tokens" + "github.com/kagenti/lab-context-engineering/internal/treesitter" + toon "github.com/toon-format/toon-go" + sitter "github.com/tree-sitter/go-tree-sitter" +) + +// ---------- collapse ---------- + +// collapse replaces text with a reversible marker, storing the original. +func collapse(text, filePath, reason string, st store.Rewind, terse bool) (string, string) { + rid := st.Put(text) + marker := markers.Make(rid) + label := filePath + if label == "" { + label = "tool output" + } + if terse { + return fmt.Sprintf("[winnow %s: %s] %s", reason, label, marker), rid + } + return fmt.Sprintf("[winnow: %s omitted (%s); call winnow_expand(%q) to restore] %s", + label, reason, rid, marker), rid +} + +// ---------- failed_run ---------- + +var ( + failRe = regexp.MustCompile(`(?i)\b(fail(ed|ures?)?|error|exception|traceback|assert|panic|FAILED|✗|✖)\b`) + passRe = regexp.MustCompile(`(?i)\b(pass(ed)?|ok|success(ful)?|0 failed|all tests passed|✓|✔)\b`) +) + +func isFailure(text string) bool { return failRe.MatchString(text) && !passRe.MatchString(text) } +func isSuccess(text string) bool { return passRe.MatchString(text) && !failRe.MatchString(text) } + +// supersededFailedRuns returns indices of failed runs followed by a later success. +func supersededFailedRuns(texts []string) []int { + lastSuccess := -1 + for i := len(texts) - 1; i >= 0; i-- { + if isSuccess(texts[i]) { + lastSuccess = i + break + } + } + if lastSuccess < 0 { + return nil + } + var out []int + for i := 0; i < lastSuccess; i++ { + if isFailure(texts[i]) { + out = append(out, i) + } + } + return out +} + +// ---------- dedup ---------- + +const shingleK = 5 + +func shingles(text string) map[string]struct{} { + toks := strings.Fields(text) + out := map[string]struct{}{} + if len(toks) < shingleK { + out[text] = struct{}{} + return out + } + for i := 0; i+shingleK <= len(toks); i++ { + out[strings.Join(toks[i:i+shingleK], " ")] = struct{}{} + } + return out +} + +func jaccard(a, b map[string]struct{}) float64 { + if len(a) == 0 && len(b) == 0 { + return 1 + } + inter := 0 + for s := range a { + if _, ok := b[s]; ok { + inter++ + } + } + union := len(a) + len(b) - inter + if union == 0 { + return 0 + } + return float64(inter) / float64(union) +} + +// nearDuplicateEarlier returns indices of earlier outputs that a later one +// near-duplicates (Jaccard >= threshold over 5-gram shingles). +// +// ponytail: exact O(n²) Jaccard instead of MinHash — candidate sets are small +// (sizeable tool outputs only); add MinHash if a transcript ever makes this hot. +func nearDuplicateEarlier(texts []string, threshold float64) []int { + sh := make([]map[string]struct{}, len(texts)) + for i, t := range texts { + sh[i] = shingles(t) + } + drop := map[int]struct{}{} + for i := 0; i < len(texts); i++ { + for j := i + 1; j < len(texts); j++ { + if jaccard(sh[i], sh[j]) >= threshold { + drop[i] = struct{}{} + break + } + } + } + out := make([]int, 0, len(drop)) + for i := range drop { + out = append(out, i) + } + sort.Ints(out) + return out +} + +// ---------- format re-encoding ---------- + +// encoder is one named, ranked format re-encoder. The rank is the tie-break priority +// when two encoders produce the same token count (lower wins). +type encoder struct { + name string + rank int + fn func(any) (string, bool) +} + +// allEncoders is the built-in re-encoder table, keyed by name so config can enable, +// disable, and reorder them purely by NAME. To ADD an encoder: append an entry here +// (give it a unique name and rank), then list that name under reduce.encoders in the +// config file — no other code change is needed. +var allEncoders = []encoder{ + {"json_compact", 0, encCompact}, + {"toon", 1, encTOON}, + {"jsonl", 2, encJSONL}, + {"markdown_kv", 3, encMarkdownKV}, + {"tsv", 4, func(d any) (string, bool) { return encDelimited(d, '\t') }}, + {"csv", 5, func(d any) (string, bool) { return encDelimited(d, ',') }}, +} + +// selectEncoders returns the encoders allowed by the named set, in the order the names +// were given (a config-controlled priority). An empty/nil set means "all built-ins" in +// their default order, preserving prior behavior. Unknown names are ignored. +func selectEncoders(enabled []string) []encoder { + if len(enabled) == 0 { + return allEncoders + } + byName := make(map[string]encoder, len(allEncoders)) + for _, e := range allEncoders { + byName[e.name] = e + } + out := make([]encoder, 0, len(enabled)) + for i, name := range enabled { + if e, ok := byName[name]; ok { + // Honor the config-given order as the tie-break rank: the first listed + // encoder wins ties, regardless of its built-in rank. + e.rank = i + out = append(out, e) + } + } + return out +} + +// bestEncoding returns the smallest faithful re-encoding strictly smaller than text, +// plus its format name, or ("","") if none helps / text isn't structured. enabled is an +// allowed-encoder set referenced by name; empty/nil means all built-in encoders. +// +// ponytail: object keys re-encode in alphabetical order (Go maps don't preserve +// source order). Lossless to the DATA — the original is stored for exact recovery. +func bestEncoding(text string, enabled []string) (string, string) { + var data any + dec := json.NewDecoder(strings.NewReader(text)) + dec.UseNumber() + if err := dec.Decode(&data); err != nil { + recs := parseNDJSON(text) + if recs == nil { + return "", "" + } + data = recs + } + orig := tokens.Count(text) + type cand struct { + enc, name string + rank, tok int + } + var best *cand + for _, e := range selectEncoders(enabled) { + enc, ok := e.fn(data) + if !ok { + continue + } + t := tokens.Count(enc) + if t < orig && (best == nil || t < best.tok || (t == best.tok && e.rank < best.rank)) { + best = &cand{enc, e.name, e.rank, t} + } + } + if best == nil { + return "", "" + } + return best.enc, best.name +} + +func parseNDJSON(text string) []any { + var lines []string + for _, ln := range strings.Split(text, "\n") { + if strings.TrimSpace(ln) != "" { + lines = append(lines, ln) + } + } + if len(lines) < 2 { + return nil + } + var recs []any + for _, ln := range lines { + var v any + dec := json.NewDecoder(strings.NewReader(ln)) + dec.UseNumber() + if err := dec.Decode(&v); err != nil { + return nil + } + switch v.(type) { + case map[string]any, []any: + recs = append(recs, v) + default: + return nil + } + } + return recs +} + +func isScalar(v any) bool { + switch v.(type) { + case nil, string, bool, float64, int, json.Number: + return true + } + return false +} + +// uniformFlat returns sorted column names if data is a non-empty list of objects +// sharing the same key set with only scalar values; else nil. +func uniformFlat(data any) []string { + list, ok := data.([]any) + if !ok || len(list) == 0 { + return nil + } + first, ok := list[0].(map[string]any) + if !ok { + return nil + } + cols := make([]string, 0, len(first)) + for k := range first { + cols = append(cols, k) + } + sort.Strings(cols) + colset := map[string]struct{}{} + for _, c := range cols { + colset[c] = struct{}{} + } + for _, r := range list { + row, ok := r.(map[string]any) + if !ok || len(row) != len(colset) { + return nil + } + for k, v := range row { + if _, ok := colset[k]; !ok || !isScalar(v) { + return nil + } + } + } + return cols +} + +func scalarStr(v any) string { + if v == nil { + return "" + } + return fmt.Sprint(v) +} + +func encDelimited(data any, delim rune) (string, bool) { + cols := uniformFlat(data) + if cols == nil { + return "", false + } + var buf bytes.Buffer + w := csv.NewWriter(&buf) + w.Comma = delim + _ = w.Write(cols) + for _, r := range data.([]any) { + row := r.(map[string]any) + rec := make([]string, len(cols)) + for i, c := range cols { + rec[i] = scalarStr(row[c]) + } + _ = w.Write(rec) + } + w.Flush() + return strings.TrimRight(buf.String(), "\n"), true +} + +func encJSONL(data any) (string, bool) { + list, ok := data.([]any) + if !ok || len(list) == 0 { + return "", false + } + for _, x := range list { + if _, ok := x.(map[string]any); !ok { + return "", false + } + } + var lines []string + for _, x := range list { + b, err := json.Marshal(x) + if err != nil { + return "", false + } + lines = append(lines, string(b)) + } + return strings.Join(lines, "\n"), true +} + +func encMarkdownKV(data any) (string, bool) { + m, ok := data.(map[string]any) + if !ok || len(m) == 0 { + return "", false + } + keys := make([]string, 0, len(m)) + for k, v := range m { + if !isScalar(v) { + return "", false + } + keys = append(keys, k) + } + sort.Strings(keys) + var lines []string + for _, k := range keys { + lines = append(lines, fmt.Sprintf("%s: %s", k, scalarStr(m[k]))) + } + return strings.Join(lines, "\n"), true +} + +// encTOON re-encodes data as Token-Oriented Object Notation. The decoded-JSON +// value may carry json.Number; toon-go renders those natively. Returns false on +// error or empty output so the candidate is simply skipped. +func encTOON(data any) (string, bool) { + out, err := toon.MarshalString(data, toon.WithLengthMarkers(true)) + if err != nil || out == "" { + return "", false + } + return out, true +} + +func encCompact(data any) (string, bool) { + b, err := json.Marshal(data) + if err != nil { + return "", false + } + return string(b), true +} + +// ---------- skeleton ---------- + +var bodyDefKinds = map[string]bool{ + "function_declaration": true, "function_definition": true, "function_item": true, + "method_declaration": true, "method_definition": true, "method": true, + "constructor_declaration": true, +} + +// skeletonize keeps signatures and drops function/method BODIES (the "body" field), +// language-agnostic via tree-sitter. Returns ok=false for non-code, parse failure, +// no bodies, or no token savings. +func skeletonize(source, filePath string) (string, bool) { + lang := treesitter.LangForExt(filePath) + if lang == "" { + return "", false + } + src := []byte(source) + tree, _, ok := treesitter.Parse(lang, src) + if !ok { + return "", false + } + defer tree.Close() + type span struct{ start, end uint } + var bodies []span + var walk func(n *sitter.Node) + walk = func(n *sitter.Node) { + if bodyDefKinds[n.Kind()] { + if b := n.ChildByFieldName("body"); b != nil { + bodies = append(bodies, span{b.StartByte(), b.EndByte()}) + return // don't recurse into a body we're dropping (nested fns) + } + } + for i := uint(0); i < n.NamedChildCount(); i++ { + walk(n.NamedChild(i)) + } + } + walk(tree.RootNode()) + if len(bodies) == 0 { + return "", false + } + sort.Slice(bodies, func(i, j int) bool { return bodies[i].start > bodies[j].start }) + out := append([]byte(nil), src...) + for _, b := range bodies { + out = append(out[:b.start], append([]byte("{ ... }"), out[b.end:]...)...) + } + result := string(out) + if tokens.Count(result) >= tokens.Count(source) { + return "", false + } + return result, true +} + +// ---------- is_structured ---------- + +// IsStructured reports whether text is machine-structured (a JSON object, a JSON +// array of length >= 2, or NDJSON) — the safe target for filter-style extraction. +func IsStructured(text string) bool { + var v any + if err := json.Unmarshal([]byte(text), &v); err == nil { + switch t := v.(type) { + case map[string]any: + return len(t) > 0 + case []any: + return len(t) >= 2 + } + } + return parseNDJSON(text) != nil +} + +// ---------- router ---------- + +var collapseReasons = set("stale", "superseded_dup", "empty", "duplicate", "failed_run", "unused") + +// Reducer is a pluggable reduction strategy. +type Reducer struct { + Name string + Applies func(ContextItem, Verdict) bool + Reduce func(ContextItem, Verdict, store.Rewind) *Reduced +} + +func keepItem(item ContextItem) *Reduced { return &Reduced{ItemID: item.ID, Action: "keep"} } + +// reversible makes a lossy reduction reversible and self-advertising; falls back to +// keep if the marker overhead eats the savings. +func reversible(item ContextItem, reduced, action, what string, st store.Rewind) *Reduced { + rid := st.Put(item.Text) + label := item.FilePath + if label == "" { + label = "tool output" + } + newText := strings.TrimRight(reduced, "\n") + "\n" + markers.RecoveryNote(label, what, rid) + if tokens.Count(newText) < tokens.Count(item.Text) { + return &Reduced{ItemID: item.ID, Action: action, NewText: &newText, RewindID: rid} + } + return keepItem(item) +} + +func collapseReduce(item ContextItem, v Verdict, st store.Rewind) *Reduced { + newText, rid := collapse(item.Text, item.FilePath, v.Reason, st, false) + if tokens.Count(newText) < tokens.Count(item.Text) { + return &Reduced{ItemID: item.ID, Action: "collapse", NewText: &newText, RewindID: rid} + } + return keepItem(item) +} + +func skeletonReduce(item ContextItem, v Verdict, st store.Rewind) *Reduced { + if !isCodePath(item.FilePath) { + return nil + } + sk, ok := skeletonize(item.Text, item.FilePath) + if !ok { + return nil + } + return reversible(item, sk, "skeleton", "code body skeletonized", st) +} + +func formatReduce(item ContextItem, v Verdict, st store.Rewind, enabledEncoders []string) *Reduced { + fr, fmtName := bestEncoding(item.Text, enabledEncoders) + if fr == "" { + return keepItem(item) + } + return reversible(item, fr, "format", "reformatted as "+fmtName, st) +} + +// formatReducerName is the built-in name of the lossless format re-encoder. route +// applies it specially (it threads the config-selected encoder set into bestEncoding), +// so the table entry below is a marker whose Reduce is never invoked directly. +const formatReducerName = "format" + +var reducers = []Reducer{ + {"collapse", func(i ContextItem, v Verdict) bool { _, ok := collapseReasons[v.Reason]; return ok }, collapseReduce}, + {"skeleton", func(i ContextItem, v Verdict) bool { return isCodePath(i.FilePath) }, skeletonReduce}, + {formatReducerName, func(i ContextItem, v Verdict) bool { return true }, + func(i ContextItem, v Verdict, st store.Rewind) *Reduced { return formatReduce(i, v, st, nil) }}, +} + +// RegisterReducer inserts a custom strategy just before the format fallback. A custom +// reducer is selectable by config purely by its Reducer.Name — list it under +// reduce.reducers and it is honored; omit it and it is filtered out. +func RegisterReducer(r Reducer) { + idx := len(reducers) - 1 + if idx < 0 { + idx = 0 + } + reducers = append(reducers[:idx], append([]Reducer{r}, reducers[idx:]...)...) +} + +// reducerAllowed reports whether a reducer name is enabled by the named set. An +// empty/nil set means "all built-ins", preserving prior behavior. +func reducerAllowed(name string, enabled []string) bool { + if len(enabled) == 0 { + return true + } + for _, n := range enabled { + if n == name { + return true + } + } + return false +} + +// route picks the cheapest faithful action for an item. enabledReducers/enabledEncoders +// are config-selected allow-lists referenced by name; empty means "all built-ins". +func route(item ContextItem, v Verdict, st store.Rewind, enabledReducers, enabledEncoders []string) *Reduced { + if v.Protected { + // Recent content keeps full fidelity for lossy ops, but a lossless format + // re-encode is safe even here — when the format reducer is enabled. + if reducerAllowed(formatReducerName, enabledReducers) { + return formatReduce(item, v, st, enabledEncoders) + } + return keepItem(item) + } + for _, r := range reducers { + if !reducerAllowed(r.Name, enabledReducers) || !r.Applies(item, v) { + continue + } + if r.Name == formatReducerName { + // Thread the config-selected encoder set into the format re-encoder. + if res := formatReduce(item, v, st, enabledEncoders); res != nil { + return res + } + continue + } + if res := r.Reduce(item, v, st); res != nil { + return res + } + } + return keepItem(item) +} diff --git a/internal/reduce/cmdfilter.go b/internal/reduce/cmdfilter.go new file mode 100644 index 0000000..e698dae --- /dev/null +++ b/internal/reduce/cmdfilter.go @@ -0,0 +1,130 @@ +package reduce + +import ( + "fmt" + "regexp" + "strings" +) + +// Deterministic, LLM-free compaction of KNOWN command outputs (rtk-style): keep the +// signal (failures, errors, summary) and drop routine noise. Lossy but reversible — +// the caller stores the original and applies only when strictly smaller. Ported from +// winnow's cmdfilter.py. + +type commandRule struct { + name string + command *regexp.Regexp + mode string // "keep" | "strip" + keep []*regexp.Regexp + drop []*regexp.Regexp + head int + tail int +} + +func ci(pats ...string) []*regexp.Regexp { + out := make([]*regexp.Regexp, len(pats)) + for i, p := range pats { + out[i] = regexp.MustCompile("(?i)" + p) + } + return out +} + +var ( + failPats = []string{`\berror\b`, `\bfailed\b`, `\bfailure`, `\bpanic`, `traceback`, + `\bexception\b`, `assert`, `\bFAIL\b`, `^E\s`, `error\[`, `:\d+:\d+`} + summaryPats = []string{`test result:`, `^\s*=+.*(passed|failed|error)`, `^tests?:\s`, + `^ok\s`, `^---\s*(FAIL|PASS)`, `in \d+\.\d+\s*s`} +) + +func failAndSummary(extra ...string) []*regexp.Regexp { + return ci(append(append(append([]string{}, failPats...), summaryPats...), extra...)...) +} + +var commandRules = []commandRule{ + {"pytest", regexp.MustCompile(`pytest|python -m pytest`), "keep", + failAndSummary(`warnings? summary`), nil, 1, 6}, + {"cargo", regexp.MustCompile(`\bcargo\s+(test|build|check|clippy|run)`), "keep", + failAndSummary(), + ci(`^\s*Compiling\b`, `^\s*Finished\b`, `^\s*Running\b`, `^\s*Downloading\b`, + `^\s*Updating\b`, `test .* \.\.\. ok`), 1, 6}, + {"gotest", regexp.MustCompile(`\bgo\s+(test|build|vet)`), "keep", + failAndSummary(`no test files`), nil, 1, 6}, + {"jsnode", regexp.MustCompile(`\b(npm|pnpm|yarn)\s+(run\s+)?test|jest|vitest|mocha`), "keep", + failAndSummary(`✕|✗|×|✘`), ci(`✓|√|PASS\b|passing`), 1, 6}, + {"tsc_lint", regexp.MustCompile(`\btsc\b|eslint|ruff|mypy|flake8`), "keep", + ci(append(append([]string{}, failPats...), `warning`, `\d+\s+problems?`)...), nil, 1, 6}, + {"install", regexp.MustCompile(`\b(npm|pnpm|yarn)\s+(install|i|ci|add)\b|pip3?\s+install|apt-get|brew\s+install`), "strip", + ci(failPats...), + ci(`already satisfied`, `requirement already`, `^\s*Downloading`, `^\s*Collecting`, + `added \d+ packages`, `^npm warn`, `^\s*\|`, `^\s*[-\\|/]\s*$`, + `packages are looking for funding`), 1, 6}, + {"gitstatus", regexp.MustCompile(`\bgit\s+status`), "keep", + ci(`modified|new file|deleted|renamed|untracked|both modified`, `branch|ahead|behind|up to date`), + ci(`use "git`, `^\s*$`), 1, 6}, +} + +const minCmdFilterLines = 8 + +func matchesAny(line string, pats []*regexp.Regexp) bool { + for _, p := range pats { + if p.MatchString(line) { + return true + } + } + return false +} + +func ruleFor(command string) *commandRule { + if command == "" { + return nil + } + for i := range commandRules { + if commandRules[i].command.MatchString(command) { + return &commandRules[i] + } + } + return nil +} + +// compactCommandOutput filters output per the rule matching command; returns ("", +// false) if no rule matches, the output is too small, or nothing was removed. +func compactCommandOutput(command, output string) (string, bool) { + rule := ruleFor(command) + if rule == nil || output == "" { + return "", false + } + lines := strings.Split(output, "\n") + n := len(lines) + if n < minCmdFilterLines { + return "", false + } + var kept []string + for i, line := range lines { + if i < rule.head || i >= n-rule.tail { + kept = append(kept, line) + continue + } + isKeep := matchesAny(line, rule.keep) + if rule.mode == "keep" { + if isKeep { + kept = append(kept, line) + } + } else { // strip + if matchesAny(line, rule.drop) && !isKeep { + continue + } + kept = append(kept, line) + } + } + if len(kept) >= n { + return "", false + } + dropped := n - len(kept) + result := strings.Join(kept, "\n") + cmd := command + if len(cmd) > 60 { + cmd = cmd[:60] + } + result += fmt.Sprintf("\n[winnow: %d routine line(s) filtered from `%s`]", dropped, cmd) + return result, true +} diff --git a/internal/reduce/compaction.go b/internal/reduce/compaction.go new file mode 100644 index 0000000..42c5a11 --- /dev/null +++ b/internal/reduce/compaction.go @@ -0,0 +1,102 @@ +package reduce + +import ( + "strings" + + "github.com/kagenti/lab-context-engineering/canon" + "github.com/kagenti/lab-context-engineering/internal/markers" + "github.com/kagenti/lab-context-engineering/internal/store" +) + +// Phrases characterizing a known agent compaction prompt (Claude Code / Codex / +// Gemini CLI). A false positive only forgoes one turn's savings. Ported from +// winnow's compaction.py. +var compactionPhrases = []string{ + "this session is being continued from a previous conversation", + "create a detailed summary of the conversation", + "context checkpoint compaction", + "your task is to create a summary", + "create a handoff summary", + "summarize the conversation", + "summary of the conversation so far", + "continue from where we left off", +} + +func textOf(content any) string { + switch c := content.(type) { + case string: + return c + case []any: + var parts []string + for _, b := range c { + if bb, ok := b.(map[string]any); ok { + if t, ok := bb["text"].(string); ok { + parts = append(parts, t) + } else if cs, ok := bb["content"].(string); ok { + parts = append(parts, cs) + } + } + } + return strings.Join(parts, "\n") + } + return "" +} + +// IsCompactionRequest reports whether the request looks like the agent asking the +// model to summarize the conversation (so reduction should pass it through). +func IsCompactionRequest(req canon.Request) bool { + haystack := strings.ToLower(textOf(req.Root["system"])) + msgs := req.Messages() + for i := len(msgs) - 1; i >= 0; i-- { + if msgs[i]["role"] == "user" { + haystack += "\n" + strings.ToLower(textOf(msgs[i]["content"])) + break + } + } + for _, p := range compactionPhrases { + if strings.Contains(haystack, p) { + return true + } + } + return false +} + +// RehydrateMarkers replaces any winnow-collapsed block with its stored original, in +// place. Returns the number of blocks restored. +func RehydrateMarkers(req canon.Request, st store.Rewind) int { + restored := 0 + for _, m := range req.Messages() { + content, ok := m["content"].([]any) + if !ok { + continue + } + for _, bRaw := range content { + blk, ok := bRaw.(map[string]any) + if !ok { + continue + } + var key string + switch blk["type"] { + case "tool_result": + key = "content" + case "text": + key = "text" + default: + continue + } + s, ok := blk[key].(string) + if !ok { + continue + } + ids := markers.FindIDs(s) + if len(ids) == 0 { + continue + } + if original, ok := st.Get(ids[0]); ok { + blk[key] = original + restored++ + } + } + } + return restored +} diff --git a/internal/reduce/encoderfilter_test.go b/internal/reduce/encoderfilter_test.go new file mode 100644 index 0000000..9278dcb --- /dev/null +++ b/internal/reduce/encoderfilter_test.go @@ -0,0 +1,52 @@ +package reduce + +import ( + "strconv" + "strings" + "testing" +) + +// TestBestEncodingHonorsAllowedSet verifies named encoder selection: when only "toon" +// is allowed, bestEncoding must return either a toon result or "" — never csv/tsv/jsonl, +// even on a flat uniform array where a delimited encoder would otherwise win on tokens. +func TestBestEncodingHonorsAllowedSet(t *testing.T) { + // A wide flat uniform array: csv/tsv normally beat toon here by dropping repeated + // keys entirely, so this fixture is exactly where a delimited encoder "would win". + var recs []string + for i := 0; i < 50; i++ { + recs = append(recs, `{"id":`+strconv.Itoa(i)+`,"name":"item","status":"active","kind":"row"}`) + } + body := "[" + strings.Join(recs, ",") + "]" + + // Sanity: with no filter, a delimited encoder is allowed to win. + encAll, _ := bestEncoding(body, nil) + if encAll == "" { + t.Fatal("unfiltered bestEncoding returned nothing on a flat array") + } + + enc, name := bestEncoding(body, []string{"toon"}) + if enc == "" { + // Allowed: toon may not beat the original; "" is a valid result. + return + } + if name != "toon" { + t.Fatalf("Encoders=[toon] but bestEncoding returned %q", name) + } + if name == "csv" || name == "tsv" || name == "jsonl" { + t.Fatalf("disallowed encoder %q leaked through filter", name) + } +} + +// TestBestEncodingEmptyFilterPreservesBehavior: an empty/nil allowed set behaves +// exactly like before (no filtering). +func TestBestEncodingEmptyFilterPreservesBehavior(t *testing.T) { + var recs []string + for i := 0; i < 40; i++ { + recs = append(recs, `{"id":`+strconv.Itoa(i)+`,"name":"item","status":"active"}`) + } + body := "[" + strings.Join(recs, ",") + "]" + enc, _ := bestEncoding(body, nil) + if enc == "" || len(enc) >= len(body) { + t.Fatalf("empty-filter bestEncoding should still reduce: enc=%q", enc) + } +} diff --git a/internal/reduce/extract.go b/internal/reduce/extract.go new file mode 100644 index 0000000..854c671 --- /dev/null +++ b/internal/reduce/extract.go @@ -0,0 +1,118 @@ +package reduce + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "strings" + + "github.com/kagenti/lab-context-engineering/canon" + "github.com/kagenti/lab-context-engineering/internal/tokens" +) + +func hashParts(parts ...any) string { + h := sha256.New() + for _, p := range parts { + h.Write([]byte(fmt.Sprint(p))) + h.Write([]byte{0}) + } + return hex.EncodeToString(h.Sum(nil))[:24] +} + +func toolUseText(input any) string { + if b, err := json.Marshal(input); err == nil { + return string(b) + } + return fmt.Sprint(input) +} + +// resultText flattens tool_result content (string, or list of text blocks/strings). +func resultText(content any) string { + switch c := content.(type) { + case string: + return c + case []any: + var out []string + for _, b := range c { + switch bb := b.(type) { + case map[string]any: + if bb["type"] == "text" { + if t, ok := bb["text"].(string); ok { + out = append(out, t) + } + } + case string: + out = append(out, bb) + } + } + return strings.Join(out, "\n") + } + return "" +} + +func asString(v any) string { s, _ := v.(string); return s } + +// ExtractItems parses a canonical request into a flat list of ContextItem. +func ExtractItems(req canon.Request) []ContextItem { + var items []ContextItem + tooluseFile := map[string]string{} + tooluseRange := map[string][2]*int{} + + for mi, msg := range req.Messages() { + raw, isList := msg["content"].([]any) + if !isList { + if s, ok := msg["content"].(string); ok && s != "" { + items = append(items, ContextItem{ + ID: hashParts(mi, 0, s), MsgIndex: mi, BlockIndex: 0, + Kind: "text", Text: s, TokenEst: tokens.Count(s), + }) + } + continue + } + for bi, bRaw := range raw { + blk, ok := bRaw.(map[string]any) + if !ok { + continue + } + switch blk["type"] { + case "tool_use": + input, _ := blk["input"].(map[string]any) + if input == nil { + input = map[string]any{} + } + fp := fileArg(input) + off, lim := readRange(input) + tuID := asString(blk["id"]) + if tuID != "" { + tooluseFile[tuID] = fp + tooluseRange[tuID] = [2]*int{off, lim} + } + text := toolUseText(blk["input"]) + items = append(items, ContextItem{ + ID: hashParts(mi, bi, blk["name"], text), MsgIndex: mi, BlockIndex: bi, + Kind: "tool_use", ToolName: asString(blk["name"]), ToolUseID: tuID, + FilePath: fp, Text: text, TokenEst: tokens.Count(text), + ReadOffset: off, ReadLimit: lim, + }) + case "tool_result": + tuID := asString(blk["tool_use_id"]) + text := resultText(blk["content"]) + rng := tooluseRange[tuID] + items = append(items, ContextItem{ + ID: hashParts(mi, bi, tuID, text), MsgIndex: mi, BlockIndex: bi, + Kind: "tool_result", ToolUseID: tuID, FilePath: tooluseFile[tuID], + Text: text, TokenEst: tokens.Count(text), + ReadOffset: rng[0], ReadLimit: rng[1], + }) + case "text": + text := asString(blk["text"]) + items = append(items, ContextItem{ + ID: hashParts(mi, bi, text), MsgIndex: mi, BlockIndex: bi, + Kind: "text", Text: text, TokenEst: tokens.Count(text), + }) + } + } + } + return items +} diff --git a/internal/reduce/format_test.go b/internal/reduce/format_test.go new file mode 100644 index 0000000..7501a7b --- /dev/null +++ b/internal/reduce/format_test.go @@ -0,0 +1,42 @@ +package reduce + +import ( + "strconv" + "strings" + "testing" +) + +func itoaTest(n int) string { return strconv.Itoa(n) } + +func TestBestEncodingConsidersTOON(t *testing.T) { + // A flat uniform array is where TOON's tabular header shines; bestEncoding + // must return a non-empty, strictly-smaller encoding. We don't hard-assert + // the winning format name (TOON, tsv, or csv may win on token count). + var recs []string + for i := 0; i < 40; i++ { + recs = append(recs, `{"id":`+itoaTest(i)+`,"name":"item","status":"active"}`) + } + body := "[" + strings.Join(recs, ",") + "]" + enc, name := bestEncoding(body, nil) + if enc == "" { + t.Fatal("expected a smaller encoding") + } + if len(enc) >= len(body) { + t.Fatal("encoding not smaller") + } + _ = name // may be toon, tsv, or csv depending on token counts; just assert it ran +} + +func TestEncTOON(t *testing.T) { + data := []any{ + map[string]any{"id": "1", "name": "a"}, + map[string]any{"id": "2", "name": "b"}, + } + out, ok := encTOON(data) + if !ok || out == "" { + t.Fatalf("encTOON failed: ok=%v out=%q", ok, out) + } + if !strings.Contains(out, "name") { + t.Fatalf("TOON output missing field: %q", out) + } +} diff --git a/internal/reduce/pipeline.go b/internal/reduce/pipeline.go new file mode 100644 index 0000000..949c776 --- /dev/null +++ b/internal/reduce/pipeline.go @@ -0,0 +1,371 @@ +package reduce + +import ( + "encoding/json" + "math" + "sort" + + "github.com/kagenti/lab-context-engineering/canon" + "github.com/kagenti/lab-context-engineering/internal/markers" + "github.com/kagenti/lab-context-engineering/internal/store" + "github.com/kagenti/lab-context-engineering/internal/tokens" +) + +// Candidate is a large output left verbatim for the async Extract stage. +type Candidate struct { + ID string + MsgIndex int + BlockIndex int + Text string + FilePath string + ToolName string + TokenEst int +} + +// Report summarizes a reduction pass. +// +// ponytail: per-item "where did tokens go" breakdown omitted — counts + reduced ids +// are enough for metrics and cross-turn stickiness. Add breakdown if a dashboard needs it. +type Report struct { + SessionID string + AtCompaction bool + FrozenCount int + TokensBefore int + TokensAfter int + TokensSaved int + Ratio float64 + ReducerErrors int + ToolDefTokens int + ToolsTotal int + ReducedIDs []string + LLMCandidates []Candidate + CompactionPassthrough bool + Rehydrated int +} + +// Opts configures a reduction pass. Use DefaultOpts and adjust. +type Opts struct { + ProtectRecent int + ContextLimit int + StickyIDs map[string]struct{} + CollapseOutputs bool + CacheFloor int + LLMCompact bool + LLMCompactFloor int + LLMCompactStructuredOnly bool + RehydrateOnCompaction bool + ProtectRecentToolUses int + ProvableOnly bool + ReduceCachedPrefix bool + CmdFilter bool + + // EnabledReducers / EnabledEncoders are config-selected allow-lists referenced by + // NAME (Reducer.Name / encoder name). Empty means "all built-ins" — prior behavior. + // They let config enable, disable, and (for encoders) reorder components without a + // core edit; see config.Settings.Reducers / .Encoders. + EnabledReducers []string + EnabledEncoders []string +} + +// DefaultOpts mirrors winnow's reduce_request defaults. +func DefaultOpts() Opts { + return Opts{ + CollapseOutputs: true, CacheFloor: -1, LLMCompactFloor: 3000, + LLMCompactStructuredOnly: true, CmdFilter: true, + } +} + +func measure(before, after string) (int, int, int, float64) { + b, a := tokens.Count(before), tokens.Count(after) + saved := b - a + ratio := 0.0 + if b > 0 { + ratio = math.Round(float64(saved)/float64(b)*10000) / 10000 + } + return b, a, saved, ratio +} + +func firstUserText(msgs []map[string]any) string { + for _, m := range msgs { + if m["role"] != "user" { + continue + } + switch c := m["content"].(type) { + case string: + return c + case []any: + for _, b := range c { + if bb, ok := b.(map[string]any); ok && bb["type"] == "text" { + if t, ok := bb["text"].(string); ok { + return t + } + } + } + } + } + return "" +} + +func serialize(msgs []map[string]any) string { + b, _ := json.Marshal(msgs) + return string(b) +} + +func setBlockText(block map[string]any, newText string) { + switch block["type"] { + case "tool_result": + block["content"] = newText + case "text": + block["text"] = newText + } +} + +func blockAt(msgs []map[string]any, mi, bi int) map[string]any { + if mi < 0 || mi >= len(msgs) { + return nil + } + list, ok := msgs[mi]["content"].([]any) + if !ok || bi < 0 || bi >= len(list) { + return nil + } + blk, _ := list[bi].(map[string]any) + return blk +} + +// ReduceRequest reduces req in place and returns a Report. Fail-open: a per-item +// reducer error is counted and leaves that item verbatim. Ported from winnow's +// reduce_request. +func ReduceRequest(req canon.Request, st store.Rewind, ev *store.Eviction, opts Opts) Report { + msgs := req.Messages() + beforeText := serialize(msgs) + systemStr := textOf(req.Root["system"]) + sid := store.SessionID(systemStr, firstUserText(msgs)) + + toolsList, _ := req.Root["tools"].([]any) + toolDefTokens := 0 + if len(toolsList) > 0 { + if b, err := json.Marshal(toolsList); err == nil { + toolDefTokens = tokens.Count(string(b)) + } + } + + if opts.RehydrateOnCompaction && IsCompactionRequest(req) { + restored := RehydrateMarkers(req, st) + b, a, saved, ratio := measure(beforeText, serialize(req.Messages())) + return Report{ + SessionID: sid, AtCompaction: true, TokensBefore: b, TokensAfter: a, + TokensSaved: saved, Ratio: ratio, ToolDefTokens: toolDefTokens, + ToolsTotal: len(toolsList), CompactionPassthrough: true, Rehydrated: restored, + } + } + + items := ExtractItems(req) + inputTokens := 0 + for _, it := range items { + inputTokens += it.TokenEst + } + zones := ComputeZones(len(msgs), inputTokens, opts.ContextLimit, 0) + frozen := zones.FrozenCount + if !opts.ReduceCachedPrefix && opts.CacheFloor+1 > frozen { + frozen = opts.CacheFloor + 1 + } + + scoreOpts := DefaultScoreOpts(opts.ProtectRecent) + scoreOpts.CollapseOutputs = opts.CollapseOutputs + scoreOpts.ProtectRecentToolUses = opts.ProtectRecentToolUses + scoreOpts.ProvableOnly = opts.ProvableOnly + verdicts := map[string]Verdict{} + for _, v := range ScoreRelevance(items, scoreOpts) { + verdicts[v.ItemID] = v + } + + byID := map[string]ContextItem{} + for _, it := range items { + byID[it.ID] = it + } + + handled := map[string]struct{}{} + reducedIDs := map[string]struct{}{} + reducerErrors := 0 + + // Batch collapse pre-pass. + batchPass := func(selectFn func(ContextItem) bool, detect func([]string) []int, reason string, minTokens int) { + var cands []ContextItem + for _, it := range items { + if it.Kind != "tool_result" { + continue + } + if _, done := handled[it.ID]; done { + continue + } + if it.MsgIndex < frozen || markers.HasForeign(it.Text) { + continue + } + if v, ok := verdicts[it.ID]; ok && v.Protected { + continue + } + if it.TokenEst < minTokens || !selectFn(it) { + continue + } + cands = append(cands, it) + } + texts := make([]string, len(cands)) + for i, c := range cands { + texts[i] = c.Text + } + for _, idx := range detect(texts) { + it := cands[idx] + block := blockAt(msgs, it.MsgIndex, it.BlockIndex) + if block == nil { + continue + } + newText, _ := collapse(it.Text, it.FilePath, reason, st, false) + if tokens.Count(newText) < tokens.Count(it.Text) { + setBlockText(block, newText) + handled[it.ID] = struct{}{} + reducedIDs[it.ID] = struct{}{} + } + } + } + + batchPass(func(it ContextItem) bool { return isFailure(it.Text) || isSuccess(it.Text) }, + supersededFailedRuns, "failed_run", 0) + batchPass(func(ContextItem) bool { return true }, + func(t []string) []int { return nearDuplicateEarlier(t, 0.85) }, "duplicate", 80) + + // Command-filter pre-pass. + if opts.CmdFilter { + cmdByID := map[string]string{} + for _, m := range msgs { + if list, ok := m["content"].([]any); ok { + for _, bRaw := range list { + b, ok := bRaw.(map[string]any) + if !ok || b["type"] != "tool_use" { + continue + } + input, _ := b["input"].(map[string]any) + id, _ := b["id"].(string) + if input != nil && id != "" { + if cmd, ok := input["command"].(string); ok { + cmdByID[id] = cmd + } + } + } + } + } + for _, it := range items { + if it.Kind != "tool_result" { + continue + } + if _, done := handled[it.ID]; done { + continue + } + if it.MsgIndex < frozen || markers.HasForeign(it.Text) { + continue + } + if v, ok := verdicts[it.ID]; ok && v.Protected { + continue + } + cmd := cmdByID[it.ToolUseID] + if cmd == "" { + continue + } + filtered, ok := compactCommandOutput(cmd, it.Text) + if !ok || tokens.Count(filtered) >= tokens.Count(it.Text) { + continue + } + block := blockAt(msgs, it.MsgIndex, it.BlockIndex) + if block == nil { + continue + } + rid := st.Put(it.Text) + label := cmd + if len(label) > 48 { + label = label[:48] + } + newText := filtered + "\n" + markers.RecoveryNote(label, "command output filtered", rid) + if tokens.Count(newText) < tokens.Count(it.Text) { + setBlockText(block, newText) + handled[it.ID] = struct{}{} + reducedIDs[it.ID] = struct{}{} + } + } + } + + var llmCandidates []Candidate + + for _, item := range items { + if _, done := handled[item.ID]; done { + continue + } + if item.MsgIndex < frozen || markers.HasForeign(item.Text) { + continue + } + block := blockAt(msgs, item.MsgIndex, item.BlockIndex) + if block == nil { + continue + } + + evictKey := item.ToolUseID + if evictKey == "" { + evictKey = item.FilePath + } + if evictKey != "" && ev != nil && ev.IsEvicted(sid, evictKey) { + newText, _ := collapse(item.Text, item.FilePath, "pruned", st, true) + setBlockText(block, newText) + continue + } + + v, ok := verdicts[item.ID] + if !ok { + continue + } + + // LLM-extraction candidate selection. + structured := IsStructured(item.Text) + reasonOK := false + switch { + case opts.LLMCompactStructuredOnly && !structured: + reasonOK = false + case structured: + reasonOK = v.Reason == "unused" || v.Reason == "lossy_candidate" || v.Reason == "kept_default" + default: + reasonOK = v.Reason == "unused" || v.Reason == "lossy_candidate" + } + if opts.LLMCompact && reasonOK && item.TokenEst >= opts.LLMCompactFloor && len(markers.FindIDs(item.Text)) == 0 { + llmCandidates = append(llmCandidates, Candidate{ + ID: item.ID, MsgIndex: item.MsgIndex, BlockIndex: item.BlockIndex, + Text: item.Text, FilePath: item.FilePath, ToolName: item.ToolName, + TokenEst: item.TokenEst, + }) + continue + } + + reduced := route(byID[item.ID], v, st, opts.EnabledReducers, opts.EnabledEncoders) + if reduced.NewText != nil { + setBlockText(block, *reduced.NewText) + reducedIDs[item.ID] = struct{}{} + } else if opts.StickyIDs != nil { + if _, sticky := opts.StickyIDs[item.ID]; sticky { + newText, _ := collapse(item.Text, item.FilePath, "sticky", st, false) + if tokens.Count(newText) < tokens.Count(item.Text) { + setBlockText(block, newText) + reducedIDs[item.ID] = struct{}{} + } + } + } + } + + b, a, saved, ratio := measure(beforeText, serialize(req.Messages())) + ids := make([]string, 0, len(reducedIDs)) + for id := range reducedIDs { + ids = append(ids, id) + } + sort.Strings(ids) + return Report{ + SessionID: sid, AtCompaction: zones.AtCompaction, FrozenCount: frozen, + TokensBefore: b, TokensAfter: a, TokensSaved: saved, Ratio: ratio, + ReducerErrors: reducerErrors, ToolDefTokens: toolDefTokens, ToolsTotal: len(toolsList), + ReducedIDs: ids, LLMCandidates: llmCandidates, + } +} diff --git a/internal/reduce/pipeline_test.go b/internal/reduce/pipeline_test.go new file mode 100644 index 0000000..a98b646 --- /dev/null +++ b/internal/reduce/pipeline_test.go @@ -0,0 +1,146 @@ +package reduce + +import ( + "encoding/json" + "fmt" + "strings" + "testing" + + "github.com/kagenti/lab-context-engineering/canon" + "github.com/kagenti/lab-context-engineering/internal/markers" + "github.com/kagenti/lab-context-engineering/internal/store" +) + +// jsonStr returns text as a JSON string literal for embedding in a request body. +func jsonStr(text string) string { + b, _ := json.Marshal(text) + return string(b) +} + +// bigUniformArray builds a uniform flat JSON array large enough to clear the +// reducer's min-output gate and where CSV is strictly smaller than JSON. +func bigUniformArray(n int) string { + var recs []string + for i := 0; i < n; i++ { + recs = append(recs, fmt.Sprintf( + `{"id":%d,"name":"item_%d","status":"active","note":"a short note here"}`, 1001+i, i)) + } + return "[" + strings.Join(recs, ",") + "]" +} + +func bigText(prefix string, n int) string { + var b strings.Builder + for i := 0; i < n; i++ { + b.WriteString(prefix) + b.WriteString(" line of content that is reasonably long to exceed the marker\n") + } + return b.String() +} + +func reduceFixture(t *testing.T, body string, opts Opts) (canon.Request, Report, store.Rewind) { + t.Helper() + req, err := canon.Decode([]byte(body)) + if err != nil { + t.Fatalf("decode: %v", err) + } + st := store.NewMemory(0) + rep := ReduceRequest(req, st, store.NewEviction(), opts) + return req, rep, st +} + +// markerOriginal returns the stored original for the first winnow marker found in text. +func markerOriginal(t *testing.T, st store.Rewind, text string) (string, bool) { + ids := markers.FindIDs(text) + if len(ids) == 0 { + return "", false + } + return st.Get(ids[0]) +} + +func TestStaleReadCollapsesAndIsReversible(t *testing.T) { + original := bigText("a.go", 12) + body := `{"messages":[ + {"role":"user","content":[{"type":"text","text":"fix the bug"}]}, + {"role":"assistant","content":[{"type":"tool_use","id":"u1","name":"read","input":{"file_path":"a.go"}}]}, + {"role":"user","content":[{"type":"tool_result","tool_use_id":"u1","content":` + jsonStr(original) + `}]}, + {"role":"assistant","content":[{"type":"tool_use","id":"u2","name":"edit","input":{"file_path":"a.go"}}]}, + {"role":"user","content":[{"type":"tool_result","tool_use_id":"u2","content":"success: file edited"}]}, + {"role":"user","content":[{"type":"text","text":"thanks"}]} + ]}` + opts := DefaultOpts() + opts.ProtectRecent = 1 + req, rep, st := reduceFixture(t, body, opts) + + if rep.TokensSaved <= 0 { + t.Fatalf("expected token savings, got %d (before=%d after=%d)", rep.TokensSaved, rep.TokensBefore, rep.TokensAfter) + } + // The stale read block should now hold a marker recoverable to the original. + block := blockAt(req.Messages(), 2, 0) + got, ok := markerOriginal(t, st, block["content"].(string)) + if !ok { + t.Fatalf("stale read was not collapsed to a recoverable marker: %v", block["content"]) + } + if got != original { + t.Fatalf("recovered content mismatch") + } +} + +func TestEmptyResultCollapsed(t *testing.T) { + body := `{"messages":[ + {"role":"assistant","content":[{"type":"tool_use","id":"u1","name":"grep","input":{}}]}, + {"role":"user","content":[{"type":"tool_result","tool_use_id":"u1","content":"[]"}]}, + {"role":"user","content":[{"type":"text","text":"next"}]} + ]}` + opts := DefaultOpts() + opts.ProtectRecent = 1 + req, _, _ := reduceFixture(t, body, opts) + block := blockAt(req.Messages(), 1, 0) + // "[]" is tiny so the marker can't beat it (never-inflate); verdict is still + // "empty" — assert the scorer agrees rather than forcing an inflating collapse. + items := ExtractItems(req) + var emptyVerdict bool + for _, v := range ScoreRelevance(items, DefaultScoreOpts(1)) { + if v.Reason == "empty" { + emptyVerdict = true + } + } + if !emptyVerdict { + t.Fatalf("expected an 'empty' verdict for the [] result; block=%v", block["content"]) + } +} + +func TestRecentReadProtected(t *testing.T) { + original := bigText("hot.go", 12) + body := `{"messages":[ + {"role":"assistant","content":[{"type":"tool_use","id":"u1","name":"read","input":{"file_path":"hot.go"}}]}, + {"role":"user","content":[{"type":"tool_result","tool_use_id":"u1","content":` + jsonStr(original) + `}]} + ]}` + opts := DefaultOpts() + opts.ProtectRecent = 5 // protects everything + req, _, _ := reduceFixture(t, body, opts) + block := blockAt(req.Messages(), 1, 0) + if block["content"].(string) != original { + t.Fatalf("recent read should be untouched") + } +} + +func TestProvableFormatReencode(t *testing.T) { + // A uniform JSON array as generic (non-file) output, never referenced later. + arr := bigUniformArray(30) + body := `{"messages":[ + {"role":"assistant","content":[{"type":"tool_use","id":"u1","name":"list_items","input":{}}]}, + {"role":"user","content":[{"type":"tool_result","tool_use_id":"u1","content":` + jsonStr(arr) + `}]}, + {"role":"user","content":[{"type":"text","text":"ok"}]} + ]}` + opts := DefaultOpts() + opts.ProtectRecent = 1 + opts.ProvableOnly = true // predicted-unused becomes a lossless re-encode, not a drop + req, rep, st := reduceFixture(t, body, opts) + if rep.TokensSaved <= 0 { + t.Fatalf("expected savings from re-encoding; before=%d after=%d", rep.TokensBefore, rep.TokensAfter) + } + block := blockAt(req.Messages(), 1, 0) + if _, ok := markerOriginal(t, st, block["content"].(string)); !ok { + t.Fatalf("re-encoded block should carry a recoverable marker: %v", block["content"]) + } +} diff --git a/internal/reduce/relevance.go b/internal/reduce/relevance.go new file mode 100644 index 0000000..8ac7206 --- /dev/null +++ b/internal/reduce/relevance.go @@ -0,0 +1,248 @@ +package reduce + +import ( + "regexp" + "strings" + "unicode" +) + +// Relevance scoring — the core contribution. For each reducible read/tool_result +// item, decide whether it is still relevant using deterministic transcript signals. +// Ported from winnow's relevance.py. + +var ( + salientRe = regexp.MustCompile(`[A-Za-z_][A-Za-z0-9_./-]{7,}`) + idTokenRe = regexp.MustCompile(`[A-Za-z0-9_][A-Za-z0-9_.\-/]*`) +) + +// ScoreOpts configures relevance scoring. Zero value is not the default; callers +// pass DefaultScoreOpts() and adjust. +type ScoreOpts struct { + ProtectRecent int + CollapseOutputs bool + MinOutputTokens int + ProtectRecentToolUses int + ProvableOnly bool + LiteralSignal bool +} + +// DefaultScoreOpts mirrors winnow's score_relevance defaults. +func DefaultScoreOpts(protectRecent int) ScoreOpts { + return ScoreOpts{ + ProtectRecent: protectRecent, CollapseOutputs: true, + MinOutputTokens: 200, LiteralSignal: true, + } +} + +func salientTokens(text string) map[string]struct{} { + return tokenSet(salientRe, text) +} + +func shortIDTokens(text string) map[string]struct{} { + out := map[string]struct{}{} + for _, t := range idTokenRe.FindAllString(text, -1) { + if len(t) < 2 || len(t) > 7 { + continue + } + hasDigit := strings.ContainsFunc(t, unicode.IsDigit) + if hasDigit || t == strings.ToUpper(t) { + out[t] = struct{}{} + } + } + return out +} + +func outputReferencedLater(item ContextItem, items []ContextItem) bool { + salient := salientTokens(item.Text) + ids := shortIDTokens(item.Text) + if len(salient) == 0 && len(ids) == 0 { + return true // nothing distinctive -> don't risk collapsing + } + for _, t := range items { + if t.MsgIndex <= item.MsgIndex { + continue + } + for s := range salient { + if strings.Contains(t.Text, s) { + return true + } + } + if len(ids) > 0 { + for tok := range shortIDTokens(t.Text) { + if _, ok := ids[tok]; ok { + return true + } + } + } + } + return false +} + +func isEmptyResult(text string) bool { + t := strings.ToLower(strings.TrimSpace(text)) + switch t { + case "", "none", "null", "[]", "{}", `""`, "{ }", "[ ]": + return true + } + return false +} + +// ScoreRelevance returns one verdict per reducible item. +func ScoreRelevance(items []ContextItem, opts ScoreOpts) []Verdict { + if len(items) == 0 { + return nil + } + maxMI := 0 + for _, i := range items { + if i.MsgIndex > maxMI { + maxMI = i.MsgIndex + } + } + protectFrom := maxMI - opts.ProtectRecent + 1 + if opts.ProtectRecentToolUses > 0 { + var tuMIs []int + seen := map[int]struct{}{} + for _, i := range items { + if i.Kind == "tool_use" { + if _, ok := seen[i.MsgIndex]; !ok { + seen[i.MsgIndex] = struct{}{} + tuMIs = append(tuMIs, i.MsgIndex) + } + } + } + sortInts(tuMIs) + if len(tuMIs) >= opts.ProtectRecentToolUses { + if cand := tuMIs[len(tuMIs)-opts.ProtectRecentToolUses]; cand < protectFrom { + protectFrom = cand + } + } + } + + // Hot path: the file the agent is touching right now is never predictively dropped. + hotPath := "" + lastMI := -1 + for _, i := range items { + if i.Kind == "tool_use" && i.FilePath != "" && i.MsgIndex >= lastMI { + lastMI = i.MsgIndex + hotPath = i.FilePath + } + } + unusedReason := "unused" + if opts.ProvableOnly { + unusedReason = "lossy_candidate" + } + + resultText := map[string]string{} + for _, it := range items { + if it.Kind == "tool_result" && it.ToolUseID != "" { + resultText[it.ToolUseID] = it.Text + } + } + + fullReadMIs := map[string][]int{} + mutateMIs := map[string][]int{} + for _, it := range items { + if it.FilePath != "" && isMutateTool(it.ToolName) { + if !isFailure(resultText[it.ToolUseID]) { + mutateMIs[it.FilePath] = append(mutateMIs[it.FilePath], it.MsgIndex) + } + } + isReadLike := it.FilePath != "" && (isReadTool(it.ToolName) || it.Kind == "tool_result") + if isReadLike && it.ReadOffset == nil && it.ReadLimit == nil { + fullReadMIs[it.FilePath] = append(fullReadMIs[it.FilePath], it.MsgIndex) + } + } + + referencedLater := func(item ContextItem) bool { + fp := item.FilePath + if fp == "" { + return false + } + for _, mi := range mutateMIs[fp] { + if mi > item.MsgIndex { + return true + } + } + syms := definedSymbols(item.Text, fp) + var lits map[string]struct{} + if opts.LiteralSignal { + lits = salientLiterals(item.Text) + } + for _, t := range items { + if t.MsgIndex <= item.MsgIndex { + continue + } + if pathReferenced(fp, t.Text) || symbolsUsed(syms, t.Text) { + return true + } + if len(lits) > 0 && literalsUsed(lits, t.Text) { + return true + } + } + return false + } + + var verdicts []Verdict + for _, it := range items { + protected := it.MsgIndex >= protectFrom + onHotPath := opts.ProvableOnly && hotPath != "" && it.FilePath == hotPath + unusedHere := unusedReason + if onHotPath { + unusedHere = "kept_default" + } + isFileRead := it.FilePath != "" && (isReadTool(it.ToolName) || it.Kind == "tool_result") + + if protected { + verdicts = append(verdicts, Verdict{it.ID, 1.0, "protected_recent", true}) + continue + } + if opts.CollapseOutputs && it.Kind == "tool_result" && isEmptyResult(it.Text) { + verdicts = append(verdicts, Verdict{it.ID, 0.1, "empty", false}) + continue + } + if !isFileRead { + if opts.CollapseOutputs && it.Kind == "tool_result" && + it.TokenEst >= opts.MinOutputTokens && !outputReferencedLater(it, items) { + verdicts = append(verdicts, Verdict{it.ID, 0.2, unusedHere, false}) + } else { + verdicts = append(verdicts, Verdict{it.ID, 0.7, "kept_default", false}) + } + continue + } + + fp := it.FilePath + laterMutate := false + for _, mi := range mutateMIs[fp] { + if mi > it.MsgIndex { + laterMutate = true + break + } + } + laterFullRead := false + for _, mi := range fullReadMIs[fp] { + if mi > it.MsgIndex { + laterFullRead = true + break + } + } + switch { + case laterMutate: + verdicts = append(verdicts, Verdict{it.ID, 0.1, "stale", false}) + case laterFullRead: + verdicts = append(verdicts, Verdict{it.ID, 0.15, "superseded_dup", false}) + case referencedLater(it): + verdicts = append(verdicts, Verdict{it.ID, 0.9, "referenced", false}) + default: + verdicts = append(verdicts, Verdict{it.ID, 0.2, unusedHere, false}) + } + } + return verdicts +} + +func sortInts(a []int) { + for i := 1; i < len(a); i++ { + for j := i; j > 0 && a[j-1] > a[j]; j-- { + a[j-1], a[j] = a[j], a[j-1] + } + } +} diff --git a/internal/reduce/signals.go b/internal/reduce/signals.go new file mode 100644 index 0000000..227234e --- /dev/null +++ b/internal/reduce/signals.go @@ -0,0 +1,172 @@ +package reduce + +import ( + "path" + "regexp" + "strings" + "sync" + + "github.com/kagenti/lab-context-engineering/internal/treesitter" + sitter "github.com/tree-sitter/go-tree-sitter" +) + +// Code-reference signals. The question is not "was the basename restated" but: was +// the read file's PATH referenced later, or a SYMBOL it DEFINES used later, or a +// distinctive LITERAL it contains reused later? Every signal biases toward KEEP (the +// safe direction; reductions are reversible). Ported from winnow's signals/refs.py. +// +// definedSymbols uses tree-sitter (go-tree-sitter + per-grammar forest), which catches +// methods the old regex missed and ignores commented-out definitions. It still biases +// toward KEEP and fails open (empty on non-code paths or parse failure). + +var ( + identRe = regexp.MustCompile(`[A-Za-z_][A-Za-z0-9_]*`) + constRe = regexp.MustCompile(`\b[A-Z][A-Z0-9_]{2,}\b`) + numRe = regexp.MustCompile(`(?:^|[^\w.])(\d{3,})(?:[^\w.]|$)`) + quotedRe = regexp.MustCompile(`['"]([^'"\n]{3,40})['"]`) +) + +var stopLiterals = set( + "todo", "fixme", "note", "xxx", "hack", "true", "false", "null", "none", + "get", "post", "put", "delete", "patch", "and", "or", "not", "the", "for", + "error", "warning", "info", "debug", "string", "number", "object", "array", +) + +func isCodePath(fp string) bool { return treesitter.LangForExt(fp) != "" } + +// defNodeKinds: definition node kinds whose "name" field (or identifier child) +// names a symbol, across grammars. +var defNodeKinds = map[string]bool{ + "function_declaration": true, "function_definition": true, "function_item": true, + "method_declaration": true, "method_definition": true, "method": true, + "class_declaration": true, "class_definition": true, "class": true, + "struct_item": true, "enum_item": true, "trait_item": true, + "type_spec": true, "interface_declaration": true, "module": true, +} + +// definedSymbols returns names of functions/classes/methods/types defined in code +// text. Empty for non-code paths or on any parse failure (fail-open). Drops names +// shorter than 3 chars. +func definedSymbols(text, filePath string) map[string]struct{} { + lang := treesitter.LangForExt(filePath) + if lang == "" { + return nil + } + src := []byte(text) + tree, _, ok := treesitter.Parse(lang, src) + if !ok { + return nil + } + defer tree.Close() + out := map[string]struct{}{} + var walk func(n *sitter.Node) + walk = func(n *sitter.Node) { + if defNodeKinds[n.Kind()] { + if name := n.ChildByFieldName("name"); name != nil { + if s := name.Utf8Text(src); len(s) >= 3 { + out[s] = struct{}{} + } + } + } + for i := uint(0); i < n.NamedChildCount(); i++ { + walk(n.NamedChild(i)) + } + } + walk(tree.RootNode()) + return out +} + +// salientLiterals returns distinctive literal values (ALL_CAPS, 3+ digit numbers, +// short quoted strings) whose later standalone reuse implies the content was used. +func salientLiterals(text string) map[string]struct{} { + out := map[string]struct{}{} + add := func(s string) { + if len(s) >= 3 { + if _, stop := stopLiterals[strings.ToLower(s)]; !stop { + out[s] = struct{}{} + } + } + } + for _, m := range constRe.FindAllString(text, -1) { + add(m) + } + for _, m := range numRe.FindAllStringSubmatch(text, -1) { + add(m[1]) + } + for _, m := range quotedRe.FindAllStringSubmatch(text, -1) { + add(strings.TrimSpace(m[1])) + } + return out +} + +func isIdentifier(s string) bool { + return s != "" && identRe.FindString(s) == s +} + +// literalsUsed reports whether any salient literal reappears in text. +func literalsUsed(literals map[string]struct{}, text string) bool { + if len(literals) == 0 || text == "" { + return false + } + idents := tokenSet(identRe, text) + for lit := range literals { + if _, ok := idents[lit]; ok { + return true + } + if !isIdentifier(lit) && strings.Contains(text, lit) { + return true + } + } + return false +} + +// pathReferenced reports whether the full path or basename appears as a standalone +// path token (boundary-aware, not a naive substring). +func pathReferenced(filePath, text string) bool { + if filePath == "" || text == "" { + return false + } + // Compile the boundary patterns once per call (the caller scans this in an + // O(items²) loop, so building the regex per token inside the loop was hot). + for _, tok := range []string{filePath, path.Base(filePath)} { + re := pathTokenRe(tok) + if re.MatchString(text) { + return true + } + } + return false +} + +// pathTokenCache memoizes the boundary-aware regex for each path token across calls. +var pathTokenCache sync.Map // string -> *regexp.Regexp + +func pathTokenRe(tok string) *regexp.Regexp { + if v, ok := pathTokenCache.Load(tok); ok { + return v.(*regexp.Regexp) + } + re := regexp.MustCompile(`(?:^|[^\w./-])` + regexp.QuoteMeta(tok) + `(?:[^\w]|$)`) + pathTokenCache.Store(tok, re) + return re +} + +// symbolsUsed reports whether any defined symbol appears as a standalone identifier. +func symbolsUsed(symbols map[string]struct{}, text string) bool { + if len(symbols) == 0 || text == "" { + return false + } + idents := tokenSet(identRe, text) + for s := range symbols { + if _, ok := idents[s]; ok { + return true + } + } + return false +} + +func tokenSet(re *regexp.Regexp, text string) map[string]struct{} { + out := map[string]struct{}{} + for _, t := range re.FindAllString(text, -1) { + out[t] = struct{}{} + } + return out +} diff --git a/internal/reduce/signals_test.go b/internal/reduce/signals_test.go new file mode 100644 index 0000000..b633714 --- /dev/null +++ b/internal/reduce/signals_test.go @@ -0,0 +1,27 @@ +package reduce + +import "testing" + +func TestDefinedSymbolsCatchesGoMethod(t *testing.T) { + src := "package p\ntype T struct{}\nfunc (r *T) DoThing() {}\nfunc Helper() {}\n" + syms := definedSymbols(src, "x.go") + for _, want := range []string{"DoThing", "Helper"} { + if _, ok := syms[want]; !ok { + t.Fatalf("missing symbol %q in %v", want, syms) + } + } +} + +func TestDefinedSymbolsIgnoresComments(t *testing.T) { + src := "package p\n// func Ghost() {}\nfunc Real() {}\n" + syms := definedSymbols(src, "x.go") + if _, ok := syms["Ghost"]; ok { + t.Fatal("commented-out func must not be a defined symbol") + } +} + +func TestDefinedSymbolsNonCodeEmpty(t *testing.T) { + if len(definedSymbols("hello", "notes.txt")) != 0 { + t.Fatal("non-code path must yield no symbols") + } +} diff --git a/internal/reduce/skeleton_test.go b/internal/reduce/skeleton_test.go new file mode 100644 index 0000000..a9c981e --- /dev/null +++ b/internal/reduce/skeleton_test.go @@ -0,0 +1,26 @@ +package reduce + +import ( + "strings" + "testing" +) + +func TestSkeletonizeDropsGoBodies(t *testing.T) { + src := "package p\nfunc Big() int {\n\t" + strings.Repeat("x := 1\n\t", 50) + "return x\n}\n" + sk, ok := skeletonize(src, "big.go") + if !ok { + t.Fatal("expected skeletonization to apply") + } + if !strings.Contains(sk, "func Big()") { + t.Fatal("signature must be kept") + } + if strings.Contains(sk, "x := 1\n\tx := 1") { + t.Fatal("body should have been elided") + } +} + +func TestSkeletonizeNonCodeFails(t *testing.T) { + if _, ok := skeletonize("hello", "n.txt"); ok { + t.Fatal("non-code must return ok=false") + } +} diff --git a/internal/reduce/taxonomy.go b/internal/reduce/taxonomy.go new file mode 100644 index 0000000..6f9eaec --- /dev/null +++ b/internal/reduce/taxonomy.go @@ -0,0 +1,86 @@ +package reduce + +import ( + "encoding/json" + "strings" +) + +// Tool-name taxonomy covering common coding agents (Claude Code, Cursor, Aider, +// Cline, Continue, Codex). Ported from winnow's taxonomy.py. +// +// ponytail: defaults only; WINNOW_TOOLS-style env overrides land with the config +// package rather than being re-parsed here. +var ( + readTools = set( + "read", "cat", "view", "open", "notebookread", "fetch_file", + "read_file", "readfile", "open_file", "view_file", "get_file", "cat_file", + ) + mutateTools = set( + "edit", "write", "create", "apply_patch", "str_replace", "notebookedit", + "edit_file", "write_file", "create_file", "str_replace_editor", "apply_diff", + "replace_in_file", "insert", "search_replace", + ) + fileKeys = []string{"file_path", "path", "filename", "file", "target_file", "filepath"} + offsetKeys = []string{"offset", "start_line", "line_start", "from_line"} + limitKeys = []string{"limit", "end_line", "line_end", "to_line"} +) + +func set(xs ...string) map[string]struct{} { + m := make(map[string]struct{}, len(xs)) + for _, x := range xs { + m[x] = struct{}{} + } + return m +} + +func isReadTool(name string) bool { + if name == "" { + return false + } + _, ok := readTools[strings.ToLower(name)] + return ok +} + +func isMutateTool(name string) bool { + if name == "" { + return false + } + _, ok := mutateTools[strings.ToLower(name)] + return ok +} + +// fileArg pulls the file path out of a tool input. +func fileArg(input map[string]any) string { + for _, k := range fileKeys { + if v, ok := input[k].(string); ok && v != "" { + return v + } + } + return "" +} + +// readRange extracts an (offset, limit) read range; nil pointers mean a full read. +func readRange(input map[string]any) (*int, *int) { + return firstInt(input, offsetKeys), firstInt(input, limitKeys) +} + +func firstInt(input map[string]any, keys []string) *int { + for _, k := range keys { + switch v := input[k].(type) { + case bool: + continue // bool excluded + case float64: + n := int(v) + return &n + case int: + n := v + return &n + case json.Number: + if i, err := v.Int64(); err == nil { + n := int(i) + return &n + } + } + } + return nil +} diff --git a/internal/reduce/types.go b/internal/reduce/types.go new file mode 100644 index 0000000..be958fe --- /dev/null +++ b/internal/reduce/types.go @@ -0,0 +1,43 @@ +// Package reduce is the deterministic, lossless-first reduction core ported from +// winnow: parse a canonical request into items, score each item's relevance from +// transcript signals, and apply the cheapest faithful, reversible action. No I/O, +// clock, or randomness beyond the injected rewind store. +package reduce + +// ContextItem is one reducible block of the request. Ids are content hashes so they +// are stable turn-to-turn (required for cache stability). +type ContextItem struct { + ID string + MsgIndex int + BlockIndex int + Kind string // "tool_result" | "text" | "tool_use" + ToolName string + ToolUseID string + FilePath string + Text string + TokenEst int + ReadOffset *int + ReadLimit *int +} + +// Verdict is the relevance decision for an item. +type Verdict struct { + ItemID string + Score float64 + Reason string // referenced|stale|unused|lossy_candidate|superseded_dup|empty|protected_recent|kept_default + Protected bool +} + +// Reduced is the outcome of routing an item to an action. +type Reduced struct { + ItemID string + Action string // keep|collapse|skeleton|format + NewText *string // nil means keep verbatim + RewindID string +} + +// Zones is the frozen-prefix / live-zone split. +type Zones struct { + FrozenCount int + AtCompaction bool +} diff --git a/internal/reduce/zones.go b/internal/reduce/zones.go new file mode 100644 index 0000000..a97a308 --- /dev/null +++ b/internal/reduce/zones.go @@ -0,0 +1,19 @@ +package reduce + +const compactionRatio = 0.8 + +// ComputeZones derives the frozen-prefix / live-zone split. At compaction (input +// near the limit) frozen_count is forced to 0 since the cache is rebuilt regardless. +func ComputeZones(numMessages, inputTokens, contextLimit, frozenCount int) Zones { + atCompaction := contextLimit > 0 && float64(inputTokens) >= compactionRatio*float64(contextLimit) + if atCompaction { + frozenCount = 0 + } + if frozenCount < 0 { + frozenCount = 0 + } + if frozenCount > numMessages { + frozenCount = numMessages + } + return Zones{FrozenCount: frozenCount, AtCompaction: atCompaction} +} diff --git a/internal/store/store.go b/internal/store/store.go new file mode 100644 index 0000000..921bcce --- /dev/null +++ b/internal/store/store.go @@ -0,0 +1,108 @@ +// Package store holds reversible reduction state: the content-addressed rewind +// store (original text keyed by content hash, so a collapsed/reduced block can be +// expanded back), the per-session eviction set, and session identity. +// +// The default Rewind implementation is in-memory — correct for a single proxy or +// sidecar process. A Redis/SQLite-backed implementation can satisfy the same +// interface later for multi-replica recovery (the plan's deferred option). +// Ported from winnow's rewind.py / session.py. +package store + +import ( + "crypto/sha256" + "encoding/hex" + "sync" + "time" +) + +// ContentID is the deterministic id for a piece of content (first 24 hex chars of +// its SHA-256). Exported because item ids elsewhere use the same scheme. +func ContentID(content string) string { + sum := sha256.Sum256([]byte(content)) + return hex.EncodeToString(sum[:])[:24] +} + +// Rewind stores originals so reductions are reversible. +type Rewind interface { + // Put stores original and returns its content id. + Put(original string) string + // Get returns the original for id, or ("", false) if absent/expired. + Get(id string) (string, bool) +} + +type entry struct { + original string + createdAt time.Time +} + +// Memory is an in-memory Rewind with TTL and touch-on-sight (a marker still being +// recovered must not expire mid-session). +type Memory struct { + mu sync.Mutex + m map[string]entry + ttl time.Duration + now func() time.Time // injectable for tests +} + +// NewMemory returns an in-memory store. ttl <= 0 means no expiry. +func NewMemory(ttl time.Duration) *Memory { + return &Memory{m: map[string]entry{}, ttl: ttl, now: time.Now} +} + +func (s *Memory) Put(original string) string { + id := ContentID(original) + s.mu.Lock() + s.m[id] = entry{original: original, createdAt: s.now()} + s.mu.Unlock() + return id +} + +func (s *Memory) Get(id string) (string, bool) { + s.mu.Lock() + defer s.mu.Unlock() + e, ok := s.m[id] + if !ok { + return "", false + } + if s.ttl > 0 && s.now().Sub(e.createdAt) > s.ttl { + delete(s.m, id) + return "", false + } + e.createdAt = s.now() // touch-on-sight + s.m[id] = e + return e.original, true +} + +// Eviction tracks user-directed prune targets per session. +type Eviction struct { + mu sync.Mutex + sets map[string]map[string]struct{} +} + +// NewEviction returns an empty eviction store. +func NewEviction() *Eviction { return &Eviction{sets: map[string]map[string]struct{}{}} } + +func (e *Eviction) Evict(sid, target string) { + e.mu.Lock() + defer e.mu.Unlock() + if e.sets[sid] == nil { + e.sets[sid] = map[string]struct{}{} + } + e.sets[sid][target] = struct{}{} +} + +func (e *Eviction) IsEvicted(sid, target string) bool { + e.mu.Lock() + defer e.mu.Unlock() + _, ok := e.sets[sid][target] + return ok +} + +// SessionID is a stable hash of (system, first user text). +func SessionID(system, firstUserText string) string { + h := sha256.New() + h.Write([]byte(system)) + h.Write([]byte{0}) + h.Write([]byte(firstUserText)) + return hex.EncodeToString(h.Sum(nil))[:16] +} diff --git a/internal/tokens/tokens.go b/internal/tokens/tokens.go new file mode 100644 index 0000000..e767fdb --- /dev/null +++ b/internal/tokens/tokens.go @@ -0,0 +1,40 @@ +// Package tokens estimates token counts using a real BPE tokenizer (o200k_base, +// the modern GPT family encoding) — an accurate offline proxy. The provider's +// usage remains authoritative; this drives reduction gating and never-inflate guards. +package tokens + +import ( + "sync" + + "github.com/tiktoken-go/tokenizer" +) + +var ( + encOnce sync.Once + enc tokenizer.Codec +) + +func codec() tokenizer.Codec { + encOnce.Do(func() { + // o200k_base is embedded in the binary (pure-Go, offline, no CGO). + enc, _ = tokenizer.Get(tokenizer.O200kBase) + }) + return enc +} + +// Count returns the BPE token count of text (0 for empty). Falls back to a +// chars/4 estimate only if the tokenizer failed to initialize. +func Count(text string) int { + if text == "" { + return 0 + } + c := codec() + if c == nil { + return (len(text) + 3) / 4 + } + ids, _, err := c.Encode(text) + if err != nil { + return (len(text) + 3) / 4 + } + return len(ids) +} diff --git a/internal/tokens/tokens_test.go b/internal/tokens/tokens_test.go new file mode 100644 index 0000000..9f7da60 --- /dev/null +++ b/internal/tokens/tokens_test.go @@ -0,0 +1,28 @@ +package tokens + +import "testing" + +func TestCountIsBPENotCharQuarter(t *testing.T) { + // "hello world" is 2 BPE tokens in cl100k/o200k, not len/4 == 2..3 by luck; + // use a case where chars/4 is clearly wrong: repeated punctuation. + got := Count("!!!!!!!!!!!!!!!!") + if got == len("!!!!!!!!!!!!!!!!")/4 { + t.Fatalf("Count still looks like chars/4: %d", got) + } + if got <= 0 { + t.Fatalf("Count returned %d", got) + } +} + +func TestCountEmpty(t *testing.T) { + if Count("") != 0 { + t.Fatal("empty must be 0") + } +} + +func TestCountStableAcrossCalls(t *testing.T) { + a, b := Count("the quick brown fox"), Count("the quick brown fox") + if a != b { + t.Fatalf("non-deterministic: %d vs %d", a, b) + } +} diff --git a/internal/treesitter/treesitter.go b/internal/treesitter/treesitter.go new file mode 100644 index 0000000..f065599 --- /dev/null +++ b/internal/treesitter/treesitter.go @@ -0,0 +1,95 @@ +// Package treesitter wraps the tree-sitter binding for the reduce signals and +// skeletonizer. CGO-backed; fail-open (unknown language / parse error => ok=false). +// +// It pairs the official github.com/tree-sitter/go-tree-sitter binding (which +// owns the *Tree/*Language types and the explicit Close lifecycle the callers +// rely on) with the per-language grammar subpackages of +// github.com/alexaandru/go-sitter-forest. Each subpackage exposes +// GetLanguage() unsafe.Pointer, which sitter.NewLanguage adopts directly. +// +// Only the grammars referenced by extToLang are linked, instead of importing +// the forest root package (which links all ~490 grammars into the binary). +package treesitter + +import ( + "path" + "strings" + "unsafe" + + sitterc "github.com/alexaandru/go-sitter-forest/c" + csharp "github.com/alexaandru/go-sitter-forest/c_sharp" + cpp "github.com/alexaandru/go-sitter-forest/cpp" + golang "github.com/alexaandru/go-sitter-forest/go" + java "github.com/alexaandru/go-sitter-forest/java" + javascript "github.com/alexaandru/go-sitter-forest/javascript" + kotlin "github.com/alexaandru/go-sitter-forest/kotlin" + php "github.com/alexaandru/go-sitter-forest/php" + python "github.com/alexaandru/go-sitter-forest/python" + ruby "github.com/alexaandru/go-sitter-forest/ruby" + rust "github.com/alexaandru/go-sitter-forest/rust" + scala "github.com/alexaandru/go-sitter-forest/scala" + swift "github.com/alexaandru/go-sitter-forest/swift" + tsx "github.com/alexaandru/go-sitter-forest/tsx" + typescript "github.com/alexaandru/go-sitter-forest/typescript" + sitter "github.com/tree-sitter/go-tree-sitter" +) + +var extToLang = map[string]string{ + ".go": "go", ".py": "python", ".js": "javascript", ".jsx": "javascript", + ".ts": "typescript", ".tsx": "tsx", ".rs": "rust", ".java": "java", + ".c": "c", ".h": "c", ".cc": "cpp", ".cpp": "cpp", ".hpp": "cpp", + ".rb": "ruby", ".php": "php", ".cs": "c_sharp", ".kt": "kotlin", + ".swift": "swift", ".scala": "scala", +} + +// langPtr maps a grammar name to its raw tree-sitter language pointer accessor. +// Keyed by the grammar names that extToLang can return. +var langPtr = map[string]func() unsafe.Pointer{ + "go": golang.GetLanguage, + "python": python.GetLanguage, + "javascript": javascript.GetLanguage, + "typescript": typescript.GetLanguage, + "tsx": tsx.GetLanguage, + "rust": rust.GetLanguage, + "java": java.GetLanguage, + "c": sitterc.GetLanguage, + "cpp": cpp.GetLanguage, + "ruby": ruby.GetLanguage, + "php": php.GetLanguage, + "c_sharp": csharp.GetLanguage, + "kotlin": kotlin.GetLanguage, + "swift": swift.GetLanguage, + "scala": scala.GetLanguage, +} + +// LangForExt maps a file path to a tree-sitter grammar name, or "" if unsupported. +func LangForExt(p string) string { + return extToLang[strings.ToLower(path.Ext(p))] +} + +// Parse parses src under the named grammar. ok=false on unknown grammar or a nil +// language; the caller treats that as "skip" (fail-open). Caller must Close the tree. +func Parse(lang string, src []byte) (*sitter.Tree, *sitter.Language, bool) { + getPtr := langPtr[lang] + if getPtr == nil { + return nil, nil, false + } + raw := getPtr() + if raw == nil { + return nil, nil, false + } + language := sitter.NewLanguage(raw) + if language == nil { + return nil, nil, false + } + parser := sitter.NewParser() + defer parser.Close() + if err := parser.SetLanguage(language); err != nil { + return nil, nil, false + } + tree := parser.Parse(src, nil) + if tree == nil { + return nil, nil, false + } + return tree, language, true +} diff --git a/internal/treesitter/treesitter_test.go b/internal/treesitter/treesitter_test.go new file mode 100644 index 0000000..a978453 --- /dev/null +++ b/internal/treesitter/treesitter_test.go @@ -0,0 +1,29 @@ +package treesitter + +import "testing" + +func TestLangForExt(t *testing.T) { + cases := map[string]string{"a.go": "go", "b.py": "python", "c.ts": "typescript", "d.txt": ""} + for path, want := range cases { + if got := LangForExt(path); got != want { + t.Fatalf("LangForExt(%q)=%q want %q", path, got, want) + } + } +} + +func TestParseGo(t *testing.T) { + tree, lang, ok := Parse("go", []byte("package p\nfunc Foo() int { return 1 }\n")) + if !ok { + t.Fatal("expected parse ok for valid Go") + } + defer tree.Close() + if lang == nil || tree.RootNode().HasError() { + t.Fatal("unexpected nil language or parse error") + } +} + +func TestParseUnknownLangFailsOpen(t *testing.T) { + if _, _, ok := Parse("klingon", []byte("x")); ok { + t.Fatal("unknown language must return ok=false") + } +} diff --git a/observability/aggregator.go b/observability/aggregator.go new file mode 100644 index 0000000..f119ee9 --- /dev/null +++ b/observability/aggregator.go @@ -0,0 +1,177 @@ +package observability + +import ( + "context" + "encoding/json" + "fmt" + "io" + "sort" + "sync" +) + +// DefaultCostKey is the CostRates key used to price a model whose own rate is not +// listed. Lets a host configure one generic fallback (e.g. an average input rate) +// while still naming specific models. +const DefaultCostKey = "default" + +// CostRate is the per-million-token price for a model, split by direction. Only the +// input rate is used to estimate savings, since reduction removes input tokens. +type CostRate struct { + InputPerMTok float64 `json:"input_per_mtok"` + OutputPerMTok float64 `json:"output_per_mtok"` +} + +// maxLatencySamples bounds the retained latency samples so memory stays flat over a +// long-running process. The most recent maxLatencySamples are kept (ring buffer). +const maxLatencySamples = 1024 + +// Aggregator is an Emitter that accumulates Events into process-wide reduction stats, +// safe for concurrent use. A host installs it as the proxy Emitter and serves its +// Snapshot/WriteJSON on a /stats endpoint. It does not replace a streaming Emitter — +// a host can fan out to both. +type Aggregator struct { + rates map[string]CostRate + + mu sync.Mutex + requests int64 + tokensBefore int64 + tokensAfter int64 + tokensSaved int64 + cacheInjected int64 + extracted int64 + stageErrors int64 + costSavedUSD float64 + + // latency holds a bounded ring of recent added-latency samples (ms). next is the + // next write index; filled tracks how many slots are valid. + latency [maxLatencySamples]int + next int + filled int +} + +// NewAggregator returns an Aggregator pricing savings with rates (keyed by model, plus +// an optional DefaultCostKey fallback). A nil rates map disables cost estimation. +func NewAggregator(rates map[string]CostRate) *Aggregator { + return &Aggregator{rates: rates} +} + +// Emit folds one Event into the running totals. +func (a *Aggregator) Emit(_ context.Context, e Event) { + a.mu.Lock() + defer a.mu.Unlock() + + a.requests++ + a.tokensBefore += int64(e.TokensBefore) + a.tokensAfter += int64(e.TokensAfter) + a.tokensSaved += int64(e.TokensSaved) + if e.CacheInject { + a.cacheInjected++ + } + if e.Extracted { + a.extracted++ + } + a.stageErrors += int64(e.StageErrors) + a.costSavedUSD += float64(e.TokensSaved) / 1e6 * a.inputRate(e.RequestModel) + + a.latency[a.next] = e.LatencyMillis + a.next = (a.next + 1) % maxLatencySamples + if a.filled < maxLatencySamples { + a.filled++ + } +} + +// inputRate resolves the input price for a model, falling back to DefaultCostKey, then 0. +func (a *Aggregator) inputRate(model string) float64 { + if a.rates == nil { + return 0 + } + if r, ok := a.rates[model]; ok { + return r.InputPerMTok + } + if r, ok := a.rates[DefaultCostKey]; ok { + return r.InputPerMTok + } + return 0 +} + +// Snapshot is a point-in-time, JSON-serializable view of the aggregated stats. +type Snapshot struct { + Requests int64 `json:"requests"` + TokensBefore int64 `json:"tokens_before"` + TokensAfter int64 `json:"tokens_after"` + TokensSaved int64 `json:"tokens_saved"` + Ratio float64 `json:"reduction_ratio"` // tokens_saved / tokens_before (higher is more reduction; 0 = no savings) + CacheInjected int64 `json:"cache_injected"` + Extracted int64 `json:"extracted"` + StageErrors int64 `json:"stage_errors"` + CostSavedUSD float64 `json:"cost_saved_usd"` + AddedLatencyP50Millis int `json:"added_latency_p50_ms"` + AddedLatencyP95Millis int `json:"added_latency_p95_ms"` +} + +// Snapshot returns the current totals. +func (a *Aggregator) Snapshot() Snapshot { + a.mu.Lock() + defer a.mu.Unlock() + + s := Snapshot{ + Requests: a.requests, + TokensBefore: a.tokensBefore, + TokensAfter: a.tokensAfter, + TokensSaved: a.tokensSaved, + CacheInjected: a.cacheInjected, + Extracted: a.extracted, + StageErrors: a.stageErrors, + CostSavedUSD: a.costSavedUSD, + } + if a.tokensBefore > 0 { + // Fraction of input tokens removed: 0 = no savings, 0.9 = 90% reduced. + s.Ratio = float64(a.tokensSaved) / float64(a.tokensBefore) + } + s.AddedLatencyP50Millis = a.percentileLocked(0.50) + s.AddedLatencyP95Millis = a.percentileLocked(0.95) + return s +} + +// percentileLocked computes the p-quantile of the retained latency samples. Caller +// holds a.mu. Returns 0 when no samples have been recorded. +func (a *Aggregator) percentileLocked(p float64) int { + if a.filled == 0 { + return 0 + } + xs := make([]int, a.filled) + copy(xs, a.latency[:a.filled]) + sort.Ints(xs) + // Nearest-rank: index = ceil(p*N)-1, clamped. + idx := int(p*float64(a.filled)+0.999999) - 1 + if idx < 0 { + idx = 0 + } + if idx >= a.filled { + idx = a.filled - 1 + } + return xs[idx] +} + +// WriteJSON writes the current Snapshot as indented JSON. +func (a *Aggregator) WriteJSON(w io.Writer) error { + enc := json.NewEncoder(w) + enc.SetIndent("", " ") + return enc.Encode(a.Snapshot()) +} + +// Summary returns a one-line human-readable digest of the current stats. +func (a *Aggregator) Summary() string { + return SummaryOf(a.Snapshot()) +} + +// SummaryOf renders a Snapshot as a one-line human-readable digest. Useful for a +// client (e.g. `lab-cx stats`) that holds only a decoded Snapshot. +func SummaryOf(s Snapshot) string { + return fmt.Sprintf( + "requests=%d tokens %d→%d saved=%d (reduction=%.1f%%) cost_saved=$%.4f cache=%d extract=%d errors=%d latency p50=%dms p95=%dms", + s.Requests, s.TokensBefore, s.TokensAfter, s.TokensSaved, s.Ratio*100, + s.CostSavedUSD, s.CacheInjected, s.Extracted, s.StageErrors, + s.AddedLatencyP50Millis, s.AddedLatencyP95Millis, + ) +} diff --git a/observability/aggregator_test.go b/observability/aggregator_test.go new file mode 100644 index 0000000..57c19bc --- /dev/null +++ b/observability/aggregator_test.go @@ -0,0 +1,108 @@ +package observability + +import ( + "bytes" + "context" + "encoding/json" + "testing" +) + +func TestAggregatorSnapshot(t *testing.T) { + // A cost model with a known input rate for the model under test: $1 per MTok. + a := NewAggregator(map[string]CostRate{ + "claude-haiku-4-5": {InputPerMTok: 1.0, OutputPerMTok: 5.0}, + }) + + ctx := context.Background() + a.Emit(ctx, Event{ + RequestModel: "claude-haiku-4-5", + TokensBefore: 1000, TokensAfter: 600, TokensSaved: 400, Ratio: 0.6, + CacheInject: true, Extracted: true, StageErrors: 0, LatencyMillis: 10, + }) + a.Emit(ctx, Event{ + RequestModel: "claude-haiku-4-5", + TokensBefore: 2000, TokensAfter: 1400, TokensSaved: 600, Ratio: 0.7, + CacheInject: false, Extracted: true, StageErrors: 1, LatencyMillis: 30, + }) + a.Emit(ctx, Event{ + RequestModel: "claude-haiku-4-5", + TokensBefore: 1000, TokensAfter: 500, TokensSaved: 500, Ratio: 0.5, + CacheInject: true, Extracted: false, StageErrors: 0, LatencyMillis: 20, + }) + + s := a.Snapshot() + + if s.Requests != 3 { + t.Fatalf("Requests = %d, want 3", s.Requests) + } + if s.TokensBefore != 4000 { + t.Fatalf("TokensBefore = %d, want 4000", s.TokensBefore) + } + if s.TokensAfter != 2500 { + t.Fatalf("TokensAfter = %d, want 2500", s.TokensAfter) + } + if s.TokensSaved != 1500 { + t.Fatalf("TokensSaved = %d, want 1500", s.TokensSaved) + } + // Reduction ratio = saved/before = 1500/4000 = 0.375 (fraction of input removed). + if s.Ratio < 0.374 || s.Ratio > 0.376 { + t.Fatalf("Ratio = %v, want ~0.375 (saved/before)", s.Ratio) + } + if s.CacheInjected != 2 { + t.Fatalf("CacheInjected = %d, want 2", s.CacheInjected) + } + if s.Extracted != 2 { + t.Fatalf("Extracted = %d, want 2", s.Extracted) + } + if s.StageErrors != 1 { + t.Fatalf("StageErrors = %d, want 1", s.StageErrors) + } + // cost_saved = tokens_saved/1e6 * inputRate = 1500/1e6 * 1.0 = 0.0015 + if s.CostSavedUSD < 0.00149 || s.CostSavedUSD > 0.00151 { + t.Fatalf("CostSavedUSD = %v, want ~0.0015", s.CostSavedUSD) + } + // p50 of {10,20,30} is 20; p95 should be >= p50 and <= max. + if s.AddedLatencyP50Millis < 10 || s.AddedLatencyP50Millis > 30 { + t.Fatalf("AddedLatencyP50Millis = %d, want in [10,30]", s.AddedLatencyP50Millis) + } + if s.AddedLatencyP95Millis < s.AddedLatencyP50Millis { + t.Fatalf("p95 (%d) < p50 (%d)", s.AddedLatencyP95Millis, s.AddedLatencyP50Millis) + } +} + +func TestAggregatorWriteJSONAndSummary(t *testing.T) { + a := NewAggregator(nil) + a.Emit(context.Background(), Event{ + RequestModel: "x", TokensBefore: 100, TokensAfter: 60, TokensSaved: 40, Ratio: 0.6, LatencyMillis: 5, + }) + + var buf bytes.Buffer + if err := a.WriteJSON(&buf); err != nil { + t.Fatalf("WriteJSON: %v", err) + } + var s Snapshot + if err := json.Unmarshal(buf.Bytes(), &s); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if s.TokensSaved != 40 { + t.Fatalf("round-trip TokensSaved = %d, want 40", s.TokensSaved) + } + if a.Summary() == "" { + t.Fatal("Summary() empty") + } +} + +// Snapshot uses the default cost rate when a model is unknown. +func TestAggregatorDefaultCostRate(t *testing.T) { + a := NewAggregator(map[string]CostRate{ + DefaultCostKey: {InputPerMTok: 2.0}, + }) + a.Emit(context.Background(), Event{ + RequestModel: "unknown-model", TokensBefore: 1_000_000, TokensAfter: 0, TokensSaved: 1_000_000, Ratio: 0, + }) + s := a.Snapshot() + // 1e6/1e6 * 2.0 = 2.0 + if s.CostSavedUSD < 1.99 || s.CostSavedUSD > 2.01 { + t.Fatalf("CostSavedUSD = %v, want ~2.0", s.CostSavedUSD) + } +} diff --git a/observability/observability.go b/observability/observability.go new file mode 100644 index 0000000..8c14bb1 --- /dev/null +++ b/observability/observability.go @@ -0,0 +1,76 @@ +// Package observability emits per-request reduction telemetry in OpenTelemetry GenAI +// vocabulary (gen_ai.*) plus this component's savings attributes. The Emitter +// interface lets a host plug its own sink; the default emits structured logs via +// stdlib slog so the core stays dependency-free. +// +// ponytail: slog in gen_ai.* field names, not the OTel SDK — keeps zero deps. A real +// OTLP span exporter is a drop-in Emitter implementation (the named upgrade); Kagenti +// hosts typically already run a collector and can map these fields, or supply an +// Emitter that opens spans. +package observability + +import ( + "context" + "log/slog" +) + +// Event is one reduction's telemetry. Field comments give the OTel attribute key the +// default emitter uses. +type Event struct { + System string // gen_ai.system (e.g. "anthropic", "openai") + RequestModel string // gen_ai.request.model + Surface string // context_engineering.surface + TokensBefore int // context_engineering.tokens.before + TokensAfter int // context_engineering.tokens.after + TokensSaved int // context_engineering.tokens.saved + Ratio float64 // context_engineering.tokens.ratio + CacheInject bool // context_engineering.cache_injected + Extracted bool // context_engineering.extracted + StageErrors int // context_engineering.stage_errors + LatencyMillis int // context_engineering.added_latency_ms (time the reduce path added) +} + +// Emitter records reduction events. Implementations must be safe for concurrent use. +type Emitter interface { + Emit(ctx context.Context, e Event) +} + +// Nop discards events. +type Nop struct{} + +func (Nop) Emit(context.Context, Event) {} + +// Tee fans an event out to several Emitters in order (e.g. stream via slog AND +// accumulate in an Aggregator). nil entries are skipped. +type Tee []Emitter + +func (t Tee) Emit(ctx context.Context, e Event) { + for _, em := range t { + if em != nil { + em.Emit(ctx, e) + } + } +} + +// SlogEmitter logs events as structured records in gen_ai.* vocabulary. +type SlogEmitter struct{ Logger *slog.Logger } + +func (s SlogEmitter) Emit(ctx context.Context, e Event) { + l := s.Logger + if l == nil { + l = slog.Default() + } + l.LogAttrs(ctx, slog.LevelInfo, "context.reduction", + slog.String("gen_ai.system", e.System), + slog.String("gen_ai.request.model", e.RequestModel), + slog.String("context_engineering.surface", e.Surface), + slog.Int("context_engineering.tokens.before", e.TokensBefore), + slog.Int("context_engineering.tokens.after", e.TokensAfter), + slog.Int("context_engineering.tokens.saved", e.TokensSaved), + slog.Float64("context_engineering.tokens.ratio", e.Ratio), + slog.Bool("context_engineering.cache_injected", e.CacheInject), + slog.Bool("context_engineering.extracted", e.Extracted), + slog.Int("context_engineering.stage_errors", e.StageErrors), + slog.Int("context_engineering.added_latency_ms", e.LatencyMillis), + ) +} diff --git a/pyproject.toml b/pyproject.toml deleted file mode 100644 index 68bb60d..0000000 --- a/pyproject.toml +++ /dev/null @@ -1,23 +0,0 @@ -[project] -name = "kagenti-adk-agents" -version = "0.0.0" -description = "Agent built with Kagenti ADK" -readme = "README.md" -authors = [{ name = "BeeAI a Series of LF Projects, LLC" }] -requires-python = ">=3.11,<4.0" -dependencies = [ - "kagenti-adk>=0.6.1", - "pyyaml>=6.0.2", -] - -[project.scripts] -server = "kagenti_adk_agents.agent:run" - -[build-system] -requires = ["uv_build>=0.9.0,<0.10.0"] -build-backend = "uv_build" - -[dependency-groups] -dev = [ - "watchfiles>=1.1.0", -] diff --git a/scripts/cc-demo.sh b/scripts/cc-demo.sh new file mode 100755 index 0000000..1a39b4f --- /dev/null +++ b/scripts/cc-demo.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Real Claude Code integration demo: route `claude` through lab-cx and read /stats. +# Requires: lab-cx built (make build), and ANTHROPIC_BASE_URL + ANTHROPIC_AUTH_TOKEN in +# the env (an Anthropic-compatible endpoint). Uses claude-haiku-4-5 as the agent model. +# +# Usage: ANTHROPIC_BASE_URL=... ANTHROPIC_AUTH_TOKEN=... scripts/cc-demo.sh +set -euo pipefail + +PORT="${PORT:-8090}" +MODEL="${LCX_MODEL:-claude-haiku-4-5}" +BIN="${BIN:-./bin/lab-cx}" +: "${ANTHROPIC_BASE_URL:?set ANTHROPIC_BASE_URL to the upstream model endpoint}" +: "${ANTHROPIC_AUTH_TOKEN:?set ANTHROPIC_AUTH_TOKEN}" + +demo=$(mktemp -d) +printf 'package main\nimport "fmt"\nfunc main(){ fmt.Println(greet("world")) }\nfunc greet(n string) string { return "hello "+n }\n' > "$demo/main.go" +printf '# cc-demo\nTiny repo for the lab-cx Claude Code integration demo.\n' > "$demo/README.md" + +"$BIN" proxy --addr "127.0.0.1:$PORT" --preset balanced \ + --upstream "$ANTHROPIC_BASE_URL" \ + --extract-model "$MODEL" --extract-provider anthropic --extract-auth bearer \ + --extract-base "$ANTHROPIC_BASE_URL" >/tmp/cc_proxy.log 2>&1 & +proxy=$! +trap 'kill $proxy 2>/dev/null' EXIT +sleep 2 + +settings=$(mktemp) +cat > "$settings" < user text + aMsgs = append(aMsgs, map[string]any{ + "role": "user", + "content": []any{map[string]any{"type": "text", "text": contentString(m["content"])}}, + }) + } + } + + var nonEmpty []string + for _, p := range systemParts { + if p != "" { + nonEmpty = append(nonEmpty, p) + } + } + internal := map[string]any{ + "system": strings.Join(nonEmpty, "\n"), + "messages": aMsgs, + "model": req["model"], + } + return internal, toolmap +} + +// openaiApplyBack copies reduced tool_result content from the canonical request back +// into the original OpenAI message list. +func openaiApplyBack(out map[string]any, reduced map[string]any, toolmap map[string]int) { + outMsgs, _ := out["messages"].([]any) + redMsgs, _ := reduced["messages"].([]any) + for _, mRaw := range redMsgs { + m, ok := mRaw.(map[string]any) + if !ok { + continue + } + content, ok := m["content"].([]any) + if !ok { + continue + } + for _, bRaw := range content { + blk, ok := bRaw.(map[string]any) + if !ok || blk["type"] != "tool_result" { + continue + } + tcid, _ := blk["tool_use_id"].(string) + j, ok := toolmap[tcid] + if !ok || j < 0 || j >= len(outMsgs) { + continue + } + if om, ok := outMsgs[j].(map[string]any); ok { + om["content"] = blk["content"] + } + } + } +} diff --git a/surfaces/surfaces.go b/surfaces/surfaces.go new file mode 100644 index 0000000..29a32af --- /dev/null +++ b/surfaces/surfaces.go @@ -0,0 +1,63 @@ +// Package surfaces maps a provider's wire format onto the canonical request model +// (canon.Request) and back. Adding support for an agent/provider API is a new +// Surface, not a new copy of the engine — the same stage pipeline runs for every +// surface. +package surfaces + +import ( + "errors" + + "github.com/kagenti/lab-context-engineering/canon" +) + +// ErrUnsupported is returned by a surface that cannot map its wire format to the +// canonical model yet. Callers MUST treat it as "forward the original request +// untouched" (fail-open), never as a hard failure. +var ErrUnsupported = errors.New("surfaces: wire format not supported for reduction") + +// RenderToken is opaque per-request state a surface needs to reconstruct the wire +// request after the canonical one has been reduced. It is nil for the identity +// (Anthropic) surface. +type RenderToken any + +// Surface is the wire ⇄ canonical seam. +type Surface interface { + // Name identifies the wire format ("anthropic", "openai", "gemini"). + Name() string + // ToInternal parses a wire request body into the canonical model plus a render + // token. It returns ErrUnsupported when the format cannot be reduced. + ToInternal(body []byte) (canon.Request, RenderToken, error) + // Render serializes a (possibly reduced) canonical request back to the wire + // format using the token from ToInternal. + Render(req canon.Request, token RenderToken) ([]byte, error) +} + +// Anthropic is the identity surface: the canonical model IS the Anthropic wire +// shape, so both directions are decode/encode of the same object. +type Anthropic struct{} + +func (Anthropic) Name() string { return "anthropic" } + +func (Anthropic) ToInternal(body []byte) (canon.Request, RenderToken, error) { + req, err := canon.Decode(body) + return req, nil, err +} + +func (Anthropic) Render(req canon.Request, _ RenderToken) ([]byte, error) { + return req.Encode() +} + +// Gemini (generateContent) is not yet mapped to the canonical model. It returns +// ErrUnsupported so the proxy forwards Gemini traffic untransformed — correct and +// safe — until a faithful mapping lands. +type Gemini struct{} + +func (Gemini) Name() string { return "gemini" } + +func (Gemini) ToInternal(body []byte) (canon.Request, RenderToken, error) { + return canon.Request{}, nil, ErrUnsupported +} + +func (Gemini) Render(req canon.Request, _ RenderToken) ([]byte, error) { + return req.Encode() +} diff --git a/surfaces/surfaces_test.go b/surfaces/surfaces_test.go new file mode 100644 index 0000000..803b9a8 --- /dev/null +++ b/surfaces/surfaces_test.go @@ -0,0 +1,145 @@ +package surfaces + +import ( + "encoding/json" + "reflect" + "testing" + + "github.com/kagenti/lab-context-engineering/canon" +) + +// jsonEqual compares two JSON byte slices for semantic (not byte) equality. +func jsonEqual(t *testing.T, a, b []byte) bool { + t.Helper() + var av, bv any + if err := json.Unmarshal(a, &av); err != nil { + t.Fatalf("unmarshal a: %v", err) + } + if err := json.Unmarshal(b, &bv); err != nil { + t.Fatalf("unmarshal b: %v", err) + } + return reflect.DeepEqual(av, bv) +} + +func TestAnthropicRoundTrip(t *testing.T) { + body := []byte(`{"model":"claude-x","max_tokens":1024,"system":"be terse","messages":[{"role":"user","content":[{"type":"text","text":"hi"}]}]}`) + s := Anthropic{} + req, token, err := s.ToInternal(body) + if err != nil { + t.Fatalf("ToInternal: %v", err) + } + if token != nil { + t.Fatalf("anthropic token should be nil, got %v", token) + } + out, err := s.Render(req, token) + if err != nil { + t.Fatalf("Render: %v", err) + } + if !jsonEqual(t, body, out) { + t.Fatalf("round-trip changed the request:\n in: %s\nout: %s", body, out) + } +} + +func TestAnthropicPreservesUnknownFields(t *testing.T) { + body := []byte(`{"model":"m","temperature":0.2,"stream":true,"metadata":{"user_id":"u1"},"messages":[]}`) + req, _, err := Anthropic{}.ToInternal(body) + if err != nil { + t.Fatalf("ToInternal: %v", err) + } + out, _ := req.Encode() + if !jsonEqual(t, body, out) { + t.Fatalf("unknown fields lost:\n in: %s\nout: %s", body, out) + } +} + +func TestOpenAIToInternalShape(t *testing.T) { + body := []byte(`{ + "model":"gpt-x", + "messages":[ + {"role":"system","content":"sys"}, + {"role":"user","content":"hello"}, + {"role":"assistant","content":"", "tool_calls":[{"id":"call_1","function":{"name":"read","arguments":"{\"path\":\"a.go\"}"}}]}, + {"role":"tool","tool_call_id":"call_1","content":"FILE CONTENTS"} + ] + }`) + req, token, err := OpenAI{}.ToInternal(body) + if err != nil { + t.Fatalf("ToInternal: %v", err) + } + if req.Model() != "gpt-x" { + t.Fatalf("model = %q", req.Model()) + } + if s, _ := req.Root["system"].(string); s != "sys" { + t.Fatalf("system = %q", s) + } + msgs := req.Messages() + if len(msgs) != 3 { // user, assistant(tool_use), user(tool_result) + t.Fatalf("got %d canonical messages, want 3", len(msgs)) + } + // assistant message should carry a tool_use block with parsed input. + asst := msgs[1] + blocks := canon.Blocks(asst) + var foundToolUse bool + for _, b := range blocks { + if canon.BlockType(b) == "tool_use" { + foundToolUse = true + input, _ := b["input"].(map[string]any) + if input["path"] != "a.go" { + t.Fatalf("tool_use input = %v", b["input"]) + } + } + } + if !foundToolUse { + t.Fatalf("no tool_use block in assistant message: %v", blocks) + } + if token == nil { + t.Fatalf("openai token must not be nil") + } +} + +func TestOpenAIRenderWritesBackToolResult(t *testing.T) { + body := []byte(`{ + "model":"gpt-x", + "messages":[ + {"role":"assistant","content":"","tool_calls":[{"id":"call_1","function":{"name":"read","arguments":"{}"}}]}, + {"role":"tool","tool_call_id":"call_1","content":"ORIGINAL LONG OUTPUT"} + ] + }`) + req, token, err := OpenAI{}.ToInternal(body) + if err != nil { + t.Fatalf("ToInternal: %v", err) + } + // Simulate a stage reducing the tool_result content. + for _, m := range req.Messages() { + for _, b := range canon.Blocks(m) { + if canon.BlockType(b) == "tool_result" { + b["content"] = "REDUCED" + } + } + } + out, err := OpenAI{}.Render(req, token) + if err != nil { + t.Fatalf("Render: %v", err) + } + var rendered map[string]any + if err := json.Unmarshal(out, &rendered); err != nil { + t.Fatalf("unmarshal rendered: %v", err) + } + msgs := rendered["messages"].([]any) + toolMsg := msgs[1].(map[string]any) + if toolMsg["content"] != "REDUCED" { + t.Fatalf("tool result not written back: %v", toolMsg["content"]) + } + // The assistant message (not a reduction target) must be untouched. + asst := msgs[0].(map[string]any) + if _, ok := asst["tool_calls"]; !ok { + t.Fatalf("assistant tool_calls were dropped") + } +} + +func TestGeminiUnsupportedIsFailOpen(t *testing.T) { + _, _, err := Gemini{}.ToInternal([]byte(`{"contents":[]}`)) + if err != ErrUnsupported { + t.Fatalf("want ErrUnsupported, got %v", err) + } +} diff --git a/testdata/fixtures/README.md b/testdata/fixtures/README.md new file mode 100644 index 0000000..a041820 --- /dev/null +++ b/testdata/fixtures/README.md @@ -0,0 +1,83 @@ +# Measurement fixtures + +Real-world tool-output fixtures used by `cmd/labcx-bench` to measure, on REAL data, +how much each deterministic reduction component reduces tokens/bytes. Every fixture is +sourced from a real project's command output or test corpus; provenance is listed below. + +The harness wraps each fixture as a canonical Anthropic-shaped request (a `tool_result` +block, paired with the matching `tool_use` so command-aware passes can see the command), +runs the engine with one component enabled at a time, and reports token/byte reduction +plus reversibility. See `docs/RESULTS-offline.md` for the measured numbers. + +## `cmd_outputs/` — exercises the command-output filter (`internal/reduce/cmdfilter.go`) + +These are real CLI command outputs. The cmdfilter keeps the signal (failures, errors, +test-result summary) and drops routine noise (compile chatter, passing-line spam), +storing the original so the reduction is reversible. + +| file | source | what it exercises | +| --- | --- | --- | +| `pytest_failures.txt` | rtk `src/cmds/python/pytest_cmd.rs` (inline `-q`-mode test fixture, `test_filter_pytest_quiet_mode_failures`) — https://github.com/rtk-ai/rtk | `pytest` rule: keep FAILURES + summary, verbatim | +| `cargo_test_failure.txt` | rtk `src/cmds/rust/cargo_cmd.rs` (inline `filter_cargo_test` fixture) — https://github.com/rtk-ai/rtk | `cargo test` rule: drop passing `... ok` lines, keep failures + `test result:` | +| `cargo_build.txt` | reconstructed from rtk's real `Cargo.lock` (one `Compiling v` line per locked dependency, 203 crates, + a `Finished` line) — https://github.com/rtk-ai/rtk | `cargo build` rule: drop `Compiling` noise, keep `Finished` | + +Note on `cargo_build.txt`: this is the genuine output shape `cargo build --release` emits +for rtk — every crate name and version is taken verbatim from rtk's committed +`Cargo.lock` (`awk '/^name = /{n=$3} /^version = /{print n,$3}' Cargo.lock`). It is a +faithful reconstruction of a real build's stdout, used because rtk's inline unit-test +fixture (3 crates) sits below the 8-line cmdfilter floor. The other two cmd fixtures are +verbatim from rtk's Rust unit tests. + +## `structured_json/` — exercises the format re-encoder (`toon`, `tsv`/`csv`, `json_compact`) + +Large structured tool/MCP outputs. The format reducer re-encodes them in the smallest +faithful representation (Token-Oriented Object Notation, delimited tables, or compact +JSON), keeping the data identical (the original is stored for exact recovery). + +| file | source | what it exercises | +| --- | --- | --- | +| `flights_search.json` | winnow `benchmarks/data/tool_outputs.jsonl` record id-7 `context` (flight-search tool output) — `../winnow` | flat array of uniform scalar rows → `tsv`/`csv`/`toon` | +| `users_directory.json` | winnow `benchmarks/data/tool_outputs.jsonl` (user-directory lookup `context`) — `../winnow` | flat array of uniform scalar rows → `tsv`/`csv`/`toon` | +| `products_inventory.json` | winnow `benchmarks/data/tool_outputs.jsonl` (product-inventory `context`) — `../winnow` | flat array of uniform scalar rows → `tsv`/`csv`/`toon` | +| `oc_pods_slice.json` | rtk `tests/fixtures/oc_pods.json` (`oc get pods -o json`, representative 6-item slice of `items[]`) — https://github.com/rtk-ai/rtk | nested k8s object → `toon`/`json_compact` | + +The winnow JSON values were stored in the JSONL as already-parsed JSON arrays; they are +written here as canonical pretty-printed JSON (no content changed). + +## `search_results/` — exercises the format re-encoder on nested record arrays + +Real list-style command results (arrays of objects with nested fields). + +| file | source | what it exercises | +| --- | --- | --- | +| `glab_issue_list.json` | rtk `tests/fixtures/glab_issue_list_raw.json` (`glab issue list -F json`) — https://github.com/rtk-ai/rtk | nested array of records → `toon`/`json_compact` | +| `glab_mr_list.json` | rtk `tests/fixtures/glab_mr_list_raw.json` (`glab mr list -F json`) — https://github.com/rtk-ai/rtk | nested array of records → `toon`/`json_compact` | + +## `file_reads/` — exercises the code skeletonizer (`internal/reduce/actions.go`) + +A real source file read into context. The skeleton reducer keeps function/method +signatures and drops their bodies (tree-sitter, language-agnostic), reversibly. + +| file | source | what it exercises | +| --- | --- | --- | +| `runner.py` | rtk `scripts/benchmark-sessions/lib/runner.py` (verbatim) — https://github.com/rtk-ai/rtk | Python function bodies dropped to `{ ... }` | + +## A note on "headroom" fixtures + +The task referenced a possible "headroom" fixture source (see winnow's +`benchmarks/bench_headroom.py` / `compare_real_headroom.py`). Inspecting those, headroom +is the *real `headroom-ai` CLI* run over the same trace requests for a head-to-head +comparison — it is not a separate fixture corpus; the inputs it compresses are the same +structured tool outputs winnow targets. Those structured tool outputs are exactly what +`structured_json/` already represents (from winnow's `tool_outputs.jsonl`). So the +"headroom-style" workload (large structured outputs scored by token reduction with +lossless recovery) is represented here via the winnow data, and no separate headroom +package was vendored. This is the honest provenance. + +## Reproducing the fixture collection + +- rtk: `git clone --depth 1 https://github.com/rtk-ai/rtk /tmp/rtk` then copy the paths + listed above. The two cmd fixtures verbatim-from-source live inside rtk's Rust unit + tests (`r#"..."#` literals); they were lifted unchanged. +- winnow: `../winnow/benchmarks/data/tool_outputs.jsonl` — the `context` field of the + named records, written out as canonical JSON. diff --git a/testdata/fixtures/cmd_outputs/cargo_build.txt b/testdata/fixtures/cmd_outputs/cargo_build.txt new file mode 100644 index 0000000..1c070f3 --- /dev/null +++ b/testdata/fixtures/cmd_outputs/cargo_build.txt @@ -0,0 +1,204 @@ + Compiling adler2 v2.0.1 + Compiling ahash v0.8.12 + Compiling aho-corasick v1.1.4 + Compiling android_system_properties v0.1.5 + Compiling anstream v0.6.21 + Compiling anstyle v1.0.13 + Compiling anstyle-parse v0.2.7 + Compiling anstyle-query v1.1.5 + Compiling anstyle-wincon v3.0.11 + Compiling anyhow v1.0.102 + Compiling autocfg v1.5.0 + Compiling automod v1.0.16 + Compiling base64 v0.22.1 + Compiling bitflags v2.11.0 + Compiling block-buffer v0.10.4 + Compiling bstr v1.12.1 + Compiling bumpalo v3.20.2 + Compiling cc v1.2.56 + Compiling cfg-if v1.0.4 + Compiling chrono v0.4.44 + Compiling clap v4.5.60 + Compiling clap_builder v4.5.60 + Compiling clap_derive v4.5.55 + Compiling clap_lex v1.0.0 + Compiling colorchoice v1.0.4 + Compiling colored v2.2.0 + Compiling core-foundation-sys v0.8.7 + Compiling cpufeatures v0.2.17 + Compiling crc32fast v1.5.0 + Compiling crossbeam-deque v0.8.6 + Compiling crossbeam-epoch v0.9.18 + Compiling crossbeam-utils v0.8.21 + Compiling crypto-common v0.1.7 + Compiling digest v0.10.7 + Compiling dirs v5.0.1 + Compiling dirs-sys v0.4.1 + Compiling displaydoc v0.2.5 + Compiling env_home v0.1.0 + Compiling equivalent v1.0.2 + Compiling errno v0.3.14 + Compiling fallible-iterator v0.3.0 + Compiling fallible-streaming-iterator v0.1.9 + Compiling fastrand v2.3.0 + Compiling find-msvc-tools v0.1.9 + Compiling flate2 v1.1.9 + Compiling foldhash v0.1.5 + Compiling form_urlencoded v1.2.2 + Compiling generic-array v0.14.7 + Compiling getrandom v0.2.17 + Compiling getrandom v0.4.2 + Compiling globset v0.4.18 + Compiling hashbrown v0.14.5 + Compiling hashbrown v0.15.5 + Compiling hashbrown v0.16.1 + Compiling hashlink v0.9.1 + Compiling heck v0.5.0 + Compiling iana-time-zone v0.1.65 + Compiling iana-time-zone-haiku v0.1.2 + Compiling icu_collections v2.1.1 + Compiling icu_locale_core v2.1.1 + Compiling icu_normalizer v2.1.1 + Compiling icu_normalizer_data v2.1.1 + Compiling icu_properties v2.1.2 + Compiling icu_properties_data v2.1.2 + Compiling icu_provider v2.1.1 + Compiling id-arena v2.3.0 + Compiling idna v1.1.0 + Compiling idna_adapter v1.2.1 + Compiling ignore v0.4.25 + Compiling indexmap v2.13.0 + Compiling is_terminal_polyfill v1.70.2 + Compiling itoa v1.0.17 + Compiling js-sys v0.3.91 + Compiling lazy_static v1.5.0 + Compiling leb128fmt v0.1.0 + Compiling libc v0.2.182 + Compiling libredox v0.1.14 + Compiling libsqlite3-sys v0.28.0 + Compiling linux-raw-sys v0.12.1 + Compiling litemap v0.8.1 + Compiling log v0.4.29 + Compiling memchr v2.8.0 + Compiling miniz_oxide v0.8.9 + Compiling num-traits v0.2.19 + Compiling once_cell v1.21.3 + Compiling once_cell_polyfill v1.70.2 + Compiling option-ext v0.2.0 + Compiling percent-encoding v2.3.2 + Compiling pkg-config v0.3.32 + Compiling potential_utf v0.1.4 + Compiling prettyplease v0.2.37 + Compiling proc-macro2 v1.0.106 + Compiling quick-xml v0.37.5 + Compiling quote v1.0.45 + Compiling r-efi v6.0.0 + Compiling redox_users v0.4.6 + Compiling regex v1.12.3 + Compiling regex-automata v0.4.14 + Compiling regex-syntax v0.8.10 + Compiling ring v0.17.14 + Compiling rtk v0.42.4 + Compiling rusqlite v0.31.0 + Compiling rustix v1.1.4 + Compiling rustls v0.23.37 + Compiling rustls-pki-types v1.14.0 + Compiling rustls-webpki v0.103.13 + Compiling rustversion v1.0.22 + Compiling same-file v1.0.6 + Compiling semver v1.0.27 + Compiling serde v1.0.228 + Compiling serde_core v1.0.228 + Compiling serde_derive v1.0.228 + Compiling serde_json v1.0.149 + Compiling serde_spanned v0.6.9 + Compiling sha2 v0.10.9 + Compiling shlex v1.3.0 + Compiling simd-adler32 v0.3.8 + Compiling smallvec v1.15.1 + Compiling stable_deref_trait v1.2.1 + Compiling strsim v0.11.1 + Compiling subtle v2.6.1 + Compiling syn v2.0.117 + Compiling synstructure v0.13.2 + Compiling tempfile v3.26.0 + Compiling thiserror v1.0.69 + Compiling thiserror-impl v1.0.69 + Compiling tinystr v0.8.2 + Compiling toml v0.8.23 + Compiling toml_datetime v0.6.11 + Compiling toml_edit v0.22.27 + Compiling toml_write v0.1.2 + Compiling typenum v1.19.0 + Compiling unicode-ident v1.0.24 + Compiling unicode-xid v0.2.6 + Compiling untrusted v0.9.0 + Compiling ureq v2.12.1 + Compiling url v2.5.8 + Compiling utf8_iter v1.0.4 + Compiling utf8parse v0.2.2 + Compiling vcpkg v0.2.15 + Compiling version_check v0.9.5 + Compiling walkdir v2.5.0 + Compiling wasi v0.11.1+wasi-snapshot-preview1 + Compiling wasip2 v1.0.2+wasi-0.2.9 + Compiling wasip3 v0.4.0+wasi-0.3.0-rc-2026-01-06 + Compiling wasm-bindgen v0.2.114 + Compiling wasm-bindgen-macro v0.2.114 + Compiling wasm-bindgen-macro-support v0.2.114 + Compiling wasm-bindgen-shared v0.2.114 + Compiling wasm-encoder v0.244.0 + Compiling wasm-metadata v0.244.0 + Compiling wasmparser v0.244.0 + Compiling webpki-roots v0.26.11 + Compiling webpki-roots v1.0.6 + Compiling which v8.0.1 + Compiling winapi-util v0.1.11 + Compiling windows-core v0.62.2 + Compiling windows-implement v0.60.2 + Compiling windows-interface v0.59.3 + Compiling windows-link v0.2.1 + Compiling windows-result v0.4.1 + Compiling windows-strings v0.5.1 + Compiling windows-sys v0.48.0 + Compiling windows-sys v0.52.0 + Compiling windows-sys v0.59.0 + Compiling windows-sys v0.61.2 + Compiling windows-targets v0.48.5 + Compiling windows-targets v0.52.6 + Compiling windows_aarch64_gnullvm v0.48.5 + Compiling windows_aarch64_gnullvm v0.52.6 + Compiling windows_aarch64_msvc v0.48.5 + Compiling windows_aarch64_msvc v0.52.6 + Compiling windows_i686_gnu v0.48.5 + Compiling windows_i686_gnu v0.52.6 + Compiling windows_i686_gnullvm v0.52.6 + Compiling windows_i686_msvc v0.48.5 + Compiling windows_i686_msvc v0.52.6 + Compiling windows_x86_64_gnu v0.48.5 + Compiling windows_x86_64_gnu v0.52.6 + Compiling windows_x86_64_gnullvm v0.48.5 + Compiling windows_x86_64_gnullvm v0.52.6 + Compiling windows_x86_64_msvc v0.48.5 + Compiling windows_x86_64_msvc v0.52.6 + Compiling winnow v0.7.15 + Compiling winsafe v0.0.19 + Compiling wit-bindgen v0.51.0 + Compiling wit-bindgen-core v0.51.0 + Compiling wit-bindgen-rust v0.51.0 + Compiling wit-bindgen-rust-macro v0.51.0 + Compiling wit-component v0.244.0 + Compiling wit-parser v0.244.0 + Compiling writeable v0.6.2 + Compiling yoke v0.8.1 + Compiling yoke-derive v0.8.1 + Compiling zerocopy v0.8.40 + Compiling zerocopy-derive v0.8.40 + Compiling zerofrom v0.1.6 + Compiling zerofrom-derive v0.1.6 + Compiling zeroize v1.8.2 + Compiling zerotrie v0.2.3 + Compiling zerovec v0.11.5 + Compiling zerovec-derive v0.11.2 + Compiling zmij v1.0.21 + Finished `release` profile [optimized] target(s) in 1m 12s diff --git a/testdata/fixtures/cmd_outputs/cargo_test_failure.txt b/testdata/fixtures/cmd_outputs/cargo_test_failure.txt new file mode 100644 index 0000000..6df82b4 --- /dev/null +++ b/testdata/fixtures/cmd_outputs/cargo_test_failure.txt @@ -0,0 +1,14 @@ +running 5 tests +test foo::test_a ... ok +test foo::test_b ... FAILED +test foo::test_c ... ok + +failures: + +---- foo::test_b stdout ---- +thread 'foo::test_b' panicked at 'assert_eq!(1, 2)' + +failures: + foo::test_b + +test result: FAILED. 4 passed; 1 failed; 0 ignored; 0 measured; 0 filtered out diff --git a/testdata/fixtures/cmd_outputs/pytest_failures.txt b/testdata/fixtures/cmd_outputs/pytest_failures.txt new file mode 100644 index 0000000..e35ad9d --- /dev/null +++ b/testdata/fixtures/cmd_outputs/pytest_failures.txt @@ -0,0 +1,14 @@ +=== test session starts === +platform linux -- Python 3.12.11, pytest-8.1.0 +collected 1705 items + +.......F....... + +=== FAILURES === +___ test_something ___ + +E AssertionError: expected True + +=== short test summary info === +FAILED tests/test_foo.py::test_something - AssertionError +5 failed, 1698 passed, 2 skipped in 108.89s diff --git a/testdata/fixtures/file_reads/runner.py b/testdata/fixtures/file_reads/runner.py new file mode 100644 index 0000000..bd02dc1 --- /dev/null +++ b/testdata/fixtures/file_reads/runner.py @@ -0,0 +1,163 @@ +from __future__ import annotations + +import asyncio +import os +import subprocess +import tempfile +from pathlib import Path + +from .config import TaskConfig +from .manifest import ( + RunManifest, + SessionEntry, + TbEntry, + TbTaskEntry, + write_manifest, +) +from .session import run_all_sessions, setup_codebase, setup_rtk +from .terminal_bench import run_terminal_bench +from .vm import create_vm_pool, destroy_vm_pool + +ROOT_DIR = Path(__file__).resolve().parent.parent + + +def _create_tarball(source_dir: Path) -> str: + fd, tarball = tempfile.mkstemp(suffix=".tar.gz") + os.close(fd) + try: + subprocess.run( + ["tar", "czf", tarball, "-C", str(source_dir), "."], + check=True, + ) + except Exception: + Path(tarball).unlink(missing_ok=True) + raise + return tarball + + +def _print_step(step: int, total: int, msg: str): + print(f"\n[{step}/{total}] {msg}") + + +def _session_to_entry(r) -> SessionEntry: + return SessionEntry( + vm_name=r.vm_name, + group=r.group, + stdout_json=f"{r.vm_name}-stdout.json", + otel_log=f"{r.vm_name}-otel.log", + rtk_db=f"{r.vm_name}-tracking.db" if r.rtk_db_path else None, + exit_code=r.exit_code, + error=r.error or None, + ) + + +def _tb_to_entry(r) -> TbEntry: + return TbEntry( + vm_name=r.vm_name, + group=r.group, + total=r.total, + passed=r.passed, + failed=r.failed, + tasks=[TbTaskEntry(name=t.name, passed=t.passed, duration_s=t.duration_s) for t in r.tasks], + error=r.error, + ) + + +async def run_benchmark( + task: TaskConfig, + vms: int, + api_key: str, + output_dir: Path, + cloud_init: Path | None = None, + terminal_bench: bool = False, + keep_vms: bool = False, +) -> RunManifest: + if cloud_init is None: + cloud_init = ROOT_DIR / "cloud-init-base.yaml" + + output_dir.mkdir(parents=True, exist_ok=True) + + total_steps = 5 if terminal_bench else 4 + vm_names: list[str] = [] + local_tarball: str | None = None + + manifest = RunManifest( + task_name=task.name, + model=task.model, + vm_count=vms, + ) + + try: + _print_step(1, total_steps, f"Creating {vms * 2} VMs ({vms} RTK ON + {vms} RTK OFF)") + vm_names = await create_vm_pool(vms, cloud_init) + print(f" VMs ready: {', '.join(vm_names)}") + + _print_step(2, total_steps, "Setting up codebases") + if not task.codebase.is_github: + local_tarball = _create_tarball(task.codebase.local_path()) + + await asyncio.gather(*( + setup_codebase(name, task.codebase, local_tarball) + for name in vm_names + )) + print(" Codebases deployed") + + _print_step(3, total_steps, "Configuring RTK on ON VMs") + setup_script = ROOT_DIR / "setup-rtk.sh" + on_vms = [n for n in vm_names if "-on-" in n] + off_vms = [n for n in vm_names if "-off-" in n] + await asyncio.gather(*(setup_rtk(vm, setup_script) for vm in on_vms)) + print(f" RTK configured on {len(on_vms)} VMs") + + _print_step(4, total_steps, f"Running Claude sessions (timeout: {task.timeout_minutes}min)") + results = await run_all_sessions(vm_names, task, api_key, output_dir) + + on_ok = [r for r in results if r.group == "on" and not r.error] + off_ok = [r for r in results if r.group == "off" and not r.error] + errors = [r for r in results if r.error] + print(f" Completed: {len(on_ok)} ON, {len(off_ok)} OFF, {len(errors)} errors") + for r in errors: + print(f" {r.vm_name}: {r.error}") + + manifest.sessions = [_session_to_entry(r) for r in results] + + if terminal_bench: + _print_step(5, total_steps, "Running terminal-bench precision tests") + tb_on = await asyncio.gather(*( + run_terminal_bench(vm, "on", task.model, api_key) + for vm in on_vms + )) + tb_off = await asyncio.gather(*( + run_terminal_bench(vm, "off", task.model, api_key) + for vm in off_vms + )) + + manifest.terminal_bench = [_tb_to_entry(r) for r in list(tb_on) + list(tb_off)] + + ok_on = [r for r in tb_on if not r.error] + ok_off = [r for r in tb_off if not r.error] + if ok_on and ok_off: + on_total = sum(r.total for r in ok_on) + on_passed = sum(r.passed for r in ok_on) + off_total = sum(r.total for r in ok_off) + off_passed = sum(r.passed for r in ok_off) + on_rate = on_passed / on_total if on_total else 0 + off_rate = off_passed / off_total if off_total else 0 + print(f" terminal-bench: ON pass rate={on_rate:.0%}, OFF pass rate={off_rate:.0%}, delta={on_rate - off_rate:+.0%}") + + tb_errors = [r for r in list(tb_on) + list(tb_off) if r.error] + for r in tb_errors: + print(f" {r.vm_name}: {r.error}") + + write_manifest(manifest, output_dir) + print(f"\n Manifest written to {output_dir / 'manifest.json'}") + + finally: + if local_tarball: + Path(local_tarball).unlink(missing_ok=True) + if not keep_vms and vm_names: + print("\nCleaning up VMs...") + await destroy_vm_pool(vm_names) + print(" VMs destroyed") + + return manifest diff --git a/testdata/fixtures/search_results/glab_issue_list.json b/testdata/fixtures/search_results/glab_issue_list.json new file mode 100644 index 0000000..9ca28a1 --- /dev/null +++ b/testdata/fixtures/search_results/glab_issue_list.json @@ -0,0 +1,122 @@ +[ + { + "iid": 156, + "title": "Support glab CI pipeline filtering", + "state": "opened", + "author": {"username": "alice_dev", "name": "Alice Developer", "id": 42}, + "web_url": "https://gitlab.example.com/acme/toolkit/-/issues/156", + "created_at": "2026-03-01T10:00:00Z", + "updated_at": "2026-03-05T14:30:00Z", + "labels": ["enhancement", "glab"], + "assignees": [{"username": "alice_dev"}], + "description": "## Request\n\nAdd support for `glab ci` pipeline filtering.\n\n\n\n### Acceptance Criteria\n- [ ] `rtk glab ci list` shows compact pipeline summary\n- [ ] `rtk glab ci status` shows current pipeline status\n- [ ] Token savings >= 80%\n\n---\n\n[![status](https://img.shields.io/badge/status-in_progress-yellow)](https://example.com)\n" + }, + { + "iid": 150, + "title": "rtk cargo test shows full output when no failures", + "state": "opened", + "author": {"username": "bob_report", "name": "Bob Reporter", "id": 100}, + "web_url": "https://gitlab.example.com/acme/toolkit/-/issues/150", + "created_at": "2026-02-28T08:00:00Z", + "updated_at": "2026-03-02T16:00:00Z", + "labels": ["bug", "cargo"], + "assignees": [{"username": "dave_fix"}], + "description": "When all tests pass, `rtk cargo test` still shows verbose compilation output instead of just the summary line.\n\n### Steps to Reproduce\n1. Run `rtk cargo test` in a project with all passing tests\n2. Observe that compiler output is included\n\n### Expected\nOnly show test summary when all tests pass.\n\n### Actual\nFull compiler warnings and test output shown." + }, + { + "iid": 145, + "title": "Add Helm CLI support", + "state": "opened", + "author": {"username": "carol_infra", "name": "Carol Infra", "id": 200}, + "web_url": "https://gitlab.example.com/acme/toolkit/-/issues/145", + "created_at": "2026-02-25T12:00:00Z", + "updated_at": "2026-03-04T09:00:00Z", + "labels": ["enhancement", "infra"], + "assignees": [], + "description": "Helm CLI outputs are verbose. Would be great to have RTK support for:\n- `helm list` (compact table)\n- `helm status` (summary only)\n- `helm install/upgrade` (ok confirmation)\n\nSimilar to how `rtk kubectl` works." + }, + { + "iid": 140, + "title": "Binary size increased 30% after Python/Go modules", + "state": "opened", + "author": {"username": "eve_perf", "name": "Eve Performance", "id": 300}, + "web_url": "https://gitlab.example.com/acme/toolkit/-/issues/140", + "created_at": "2026-02-20T15:00:00Z", + "updated_at": "2026-02-22T10:00:00Z", + "labels": ["performance", "build"], + "assignees": [{"username": "frank_contrib"}], + "description": "After merging Python and Go support, stripped release binary went from 3.2MB to 4.1MB.\n\nInvestigate if we can:\n- Use feature flags to make modules optional\n- Reduce regex count (share patterns across modules)\n- Review serde usage (maybe avoid full JSON parsing for simple cases)" + }, + { + "iid": 135, + "title": "rtk gain --history shows wrong dates on macOS", + "state": "closed", + "author": {"username": "george_mac", "name": "George Mac", "id": 400}, + "web_url": "https://gitlab.example.com/acme/toolkit/-/issues/135", + "created_at": "2026-02-15T09:00:00Z", + "updated_at": "2026-02-18T11:00:00Z", + "labels": ["bug", "macos"], + "assignees": [{"username": "alice_dev"}], + "description": "On macOS, `rtk gain --history` shows dates in UTC instead of local timezone.\n\nFixed in v0.23.1." + }, + { + "iid": 130, + "title": "Support TOML-based filter DSL", + "state": "opened", + "author": {"username": "heidi_arch", "name": "Heidi Architect", "id": 500}, + "web_url": "https://gitlab.example.com/acme/toolkit/-/issues/130", + "created_at": "2026-02-10T08:00:00Z", + "updated_at": "2026-02-12T16:00:00Z", + "labels": ["enhancement", "architecture"], + "assignees": [], + "description": "Instead of writing Rust code for each new filter, allow users to define filters in TOML.\n\n```toml\n[[filter]]\ncommand = \"terraform plan\"\npattern = \"^(Plan|Apply|Error):\"\nformat = \"compact\"\n```\n\nThis would make RTK extensible without recompilation." + }, + { + "iid": 125, + "title": "Improve error messages for missing commands", + "state": "closed", + "author": {"username": "ivan_docs", "name": "Ivan Writer", "id": 600}, + "web_url": "https://gitlab.example.com/acme/toolkit/-/issues/125", + "created_at": "2026-02-05T14:00:00Z", + "updated_at": "2026-02-06T09:00:00Z", + "labels": ["enhancement", "ux"], + "assignees": [{"username": "ivan_docs"}], + "description": "When the underlying command is not installed (e.g., `rtk glab mr list` without glab), the error message is confusing:\n\n```\nError: Failed to run glab mr list\n```\n\nShould say something like:\n```\nError: glab not found. Install it: https://gitlab.com/gitlab-org/cli\n```" + }, + { + "iid": 120, + "title": "Add rtk completion command for shell completions", + "state": "opened", + "author": {"username": "judy_shell", "name": "Judy Shell", "id": 700}, + "web_url": "https://gitlab.example.com/acme/toolkit/-/issues/120", + "created_at": "2026-02-01T11:00:00Z", + "updated_at": "2026-02-03T15:00:00Z", + "labels": ["enhancement", "shell"], + "assignees": [], + "description": "Clap supports generating shell completions via `clap_complete`. Add a `rtk completion bash/zsh/fish` command.\n\nThis would help discoverability of available commands." + }, + { + "iid": 115, + "title": "rtk read crashes on binary files", + "state": "closed", + "author": {"username": "karl_refactor", "name": "Karl Refactorer", "id": 800}, + "web_url": "https://gitlab.example.com/acme/toolkit/-/issues/115", + "created_at": "2026-01-28T10:00:00Z", + "updated_at": "2026-01-30T12:00:00Z", + "labels": ["bug", "crash"], + "assignees": [{"username": "dave_fix"}], + "description": "Running `rtk read /path/to/binary.exe` panics with:\n```\nthread 'main' panicked at 'called `Result::unwrap()` on an `Err` value: Utf8Error'\n```\n\nShould detect binary files and skip filtering." + }, + { + "iid": 110, + "title": "Track savings per project directory", + "state": "opened", + "author": {"username": "lisa_feat", "name": "Lisa Feature", "id": 900}, + "web_url": "https://gitlab.example.com/acme/toolkit/-/issues/110", + "created_at": "2026-01-25T09:00:00Z", + "updated_at": "2026-01-27T14:00:00Z", + "labels": ["enhancement", "analytics"], + "assignees": [], + "description": "Currently `rtk gain` shows global stats. It would be useful to see savings broken down by project directory.\n\nProposal: store `cwd` in the tracking database and add `rtk gain --by-project` flag." + } +] diff --git a/testdata/fixtures/search_results/glab_mr_list.json b/testdata/fixtures/search_results/glab_mr_list.json new file mode 100644 index 0000000..c502b62 --- /dev/null +++ b/testdata/fixtures/search_results/glab_mr_list.json @@ -0,0 +1,182 @@ +[ + { + "iid": 314, + "title": "feat(glab): add GitLab CLI (glab) command support", + "state": "opened", + "author": {"username": "alice_dev", "name": "Alice Developer", "id": 42}, + "web_url": "https://gitlab.example.com/acme/toolkit/-/merge_requests/314", + "created_at": "2026-03-01T10:00:00Z", + "updated_at": "2026-03-05T14:30:00Z", + "source_branch": "feat/glab-support", + "target_branch": "master", + "merge_status": "can_be_merged", + "draft": false, + "labels": ["enhancement", "cli"], + "assignees": [{"username": "alice_dev", "name": "Alice Developer"}], + "reviewers": [{"username": "bob_review"}, {"username": "carol_review"}], + "description": "## Summary\n\nAdd GitLab CLI support.\n\n\n\n## Changes\n- New module\n- MR/issue/CI filtering\n- Token savings 80-87%\n\n---\n\n[![CI](https://img.shields.io/badge/CI-passing-green)](https://ci.example.com)\n", + "head_pipeline": {"id": 98765, "status": "success", "ref": "feat/glab-support"} + }, + { + "iid": 310, + "title": "fix(git): handle merge commits in compact diff", + "state": "merged", + "author": {"username": "dave_fix", "name": "Dave Fixer", "id": 100}, + "web_url": "https://gitlab.example.com/acme/toolkit/-/merge_requests/310", + "created_at": "2026-02-28T08:00:00Z", + "updated_at": "2026-03-02T16:00:00Z", + "source_branch": "fix/merge-commits", + "target_branch": "master", + "merge_status": "can_be_merged", + "draft": false, + "labels": ["bug", "git"], + "assignees": [{"username": "dave_fix"}], + "reviewers": [{"username": "eve_review"}], + "description": "Fix handling of merge commits in `compact_diff`. Previously, merge commits were being skipped entirely which lost context.\n\n### Test Plan\n- [x] Unit tests added\n- [x] Manual verification with merge-heavy repos\n", + "head_pipeline": {"id": 98700, "status": "success", "ref": "fix/merge-commits"} + }, + { + "iid": 305, + "title": "feat(aws): add AWS CLI module with token-optimized output", + "state": "opened", + "author": {"username": "frank_contrib", "name": "Frank Contributor", "id": 200}, + "web_url": "https://gitlab.example.com/acme/toolkit/-/merge_requests/305", + "created_at": "2026-02-25T12:00:00Z", + "updated_at": "2026-03-04T09:00:00Z", + "source_branch": "feat/aws-cli", + "target_branch": "master", + "merge_status": "cannot_be_merged", + "draft": true, + "labels": ["enhancement", "infra"], + "assignees": [], + "reviewers": [{"username": "grace_review"}, {"username": "heidi_review"}], + "description": "Add AWS CLI support.\n\n![architecture](https://example.com/arch.png)\n\n## Commands\n- `rtk aws s3 ls`\n- `rtk aws ec2 describe-instances`\n- `rtk aws ecs list-services`\n\n## Token Savings\n| Command | Savings |\n|---------|--------|\n| s3 ls | 75% |\n| ec2 describe | 85% |\n| ecs list | 80% |\n", + "head_pipeline": {"id": 98650, "status": "failed", "ref": "feat/aws-cli"} + }, + { + "iid": 302, + "title": "chore(master): release 0.24.0", + "state": "merged", + "author": {"username": "release-bot", "name": "Release Bot", "id": 1}, + "web_url": "https://gitlab.example.com/acme/toolkit/-/merge_requests/302", + "created_at": "2026-02-20T00:00:00Z", + "updated_at": "2026-02-20T01:00:00Z", + "source_branch": "release-please--branches--master", + "target_branch": "master", + "merge_status": "can_be_merged", + "draft": false, + "labels": ["release"], + "assignees": [], + "reviewers": [], + "description": "## [0.24.0](https://example.com/compare/v0.23.0...v0.24.0)\n\n### Features\n* feat(aws): add AWS CLI module\n* feat(psql): add PostgreSQL module\n\n### Bug Fixes\n* fix(playwright): fix JSON parser\n", + "head_pipeline": {"id": 98600, "status": "success", "ref": "release-please--branches--master"} + }, + { + "iid": 298, + "title": "docs: update README with Python and Go command examples", + "state": "merged", + "author": {"username": "ivan_docs", "name": "Ivan Writer", "id": 300}, + "web_url": "https://gitlab.example.com/acme/toolkit/-/merge_requests/298", + "created_at": "2026-02-18T15:00:00Z", + "updated_at": "2026-02-19T10:00:00Z", + "source_branch": "docs/python-go-examples", + "target_branch": "master", + "merge_status": "can_be_merged", + "draft": false, + "labels": ["documentation"], + "assignees": [{"username": "ivan_docs"}], + "reviewers": [{"username": "judy_review"}], + "description": "Update README.md with comprehensive examples for:\n- Python commands (ruff, pytest, pip)\n- Go commands (go test, go build, golangci-lint)\n\nAll examples tested manually.", + "head_pipeline": null + }, + { + "iid": 295, + "title": "refactor: extract parser module from runner.rs", + "state": "closed", + "author": {"username": "karl_refactor", "name": "Karl Refactorer", "id": 400}, + "web_url": "https://gitlab.example.com/acme/toolkit/-/merge_requests/295", + "created_at": "2026-02-15T09:00:00Z", + "updated_at": "2026-02-16T11:00:00Z", + "source_branch": "refactor/parser-module", + "target_branch": "master", + "merge_status": "can_be_merged", + "draft": false, + "labels": ["refactor"], + "assignees": [{"username": "karl_refactor"}], + "reviewers": [], + "description": "Extract parser logic from runner.rs into dedicated parser/ module.\n\n---\n\nThis was superseded by #300 which took a different approach.\n\n***\n", + "head_pipeline": {"id": 98500, "status": "canceled", "ref": "refactor/parser-module"} + }, + { + "iid": 290, + "title": "feat(tee): save raw output on failure for LLM re-read", + "state": "merged", + "author": {"username": "lisa_feat", "name": "Lisa Feature", "id": 500}, + "web_url": "https://gitlab.example.com/acme/toolkit/-/merge_requests/290", + "created_at": "2026-02-10T08:00:00Z", + "updated_at": "2026-02-12T16:00:00Z", + "source_branch": "feat/tee-output", + "target_branch": "master", + "merge_status": "can_be_merged", + "draft": false, + "labels": ["enhancement"], + "assignees": [{"username": "lisa_feat"}], + "reviewers": [{"username": "mike_review"}], + "description": "## Tee Output Recovery\n\nSave raw unfiltered output on command failure.\nPrint one-line hint so LLMs can re-read instead of re-run.\n\n### Configuration\n```toml\n[tee]\nenabled = true\ndir = \"~/.local/share/rtk/tee\"\nmax_files = 20\nmax_size = 1048576\n```\n", + "head_pipeline": {"id": 98400, "status": "success", "ref": "feat/tee-output"} + }, + { + "iid": 285, + "title": "ci: add ARM64 Linux build to release workflow", + "state": "merged", + "author": {"username": "nancy_ci", "name": "Nancy CI", "id": 600}, + "web_url": "https://gitlab.example.com/acme/toolkit/-/merge_requests/285", + "created_at": "2026-02-05T14:00:00Z", + "updated_at": "2026-02-06T09:00:00Z", + "source_branch": "ci/arm64-build", + "target_branch": "master", + "merge_status": "can_be_merged", + "draft": false, + "labels": ["ci"], + "assignees": [{"username": "nancy_ci"}], + "reviewers": [{"username": "oscar_review"}], + "description": "Add ARM64 Linux target to the release workflow.\n\n- Uses `cross` for cross-compilation\n- Generates `.deb` and `.rpm` packages\n- Tested on Raspberry Pi 4 and AWS Graviton", + "head_pipeline": {"id": 98300, "status": "success", "ref": "ci/arm64-build"} + }, + { + "iid": 280, + "title": "fix(vitest): handle watch mode output gracefully", + "state": "opened", + "author": {"username": "peter_bugfix", "name": "Peter Bugfix", "id": 700}, + "web_url": "https://gitlab.example.com/acme/toolkit/-/merge_requests/280", + "created_at": "2026-02-01T11:00:00Z", + "updated_at": "2026-02-03T15:00:00Z", + "source_branch": "fix/vitest-watch", + "target_branch": "master", + "merge_status": "unchecked", + "draft": false, + "labels": ["bug", "vitest"], + "assignees": [{"username": "peter_bugfix"}], + "reviewers": [], + "description": "When vitest runs in watch mode, output is continuous and doesn't have a clear end marker. This fix detects watch mode and falls back to passthrough.\n\n\n", + "head_pipeline": {"id": 98200, "status": "running", "ref": "fix/vitest-watch"} + }, + { + "iid": 275, + "title": "feat(discover): add rtk discover command for missed savings analysis", + "state": "merged", + "author": {"username": "quinn_dev", "name": "Quinn Developer", "id": 800}, + "web_url": "https://gitlab.example.com/acme/toolkit/-/merge_requests/275", + "created_at": "2026-01-28T10:00:00Z", + "updated_at": "2026-01-30T12:00:00Z", + "source_branch": "feat/discover", + "target_branch": "master", + "merge_status": "can_be_merged", + "draft": false, + "labels": ["enhancement", "analytics"], + "assignees": [{"username": "quinn_dev"}], + "reviewers": [{"username": "rachel_review"}, {"username": "sam_review"}], + "description": "Add `rtk discover` command that scans Claude Code JSONL sessions and reports missed savings opportunities.\n\n## Features\n- Classifies commands as Supported/Unsupported/Ignored\n- Groups by category with estimated token savings\n- Reports top missed commands\n\n## Example\n```\n$ rtk discover\nAnalyzed 1,234 commands across 45 sessions\n\nMissed savings by category:\n Git: 234 commands, ~16,800 tokens\n Cargo: 89 commands, ~7,120 tokens\n```\n", + "head_pipeline": {"id": 98100, "status": "success", "ref": "feat/discover"} + } +] diff --git a/testdata/fixtures/structured_json/flights_search.json b/testdata/fixtures/structured_json/flights_search.json new file mode 100644 index 0000000..193a92f --- /dev/null +++ b/testdata/fixtures/structured_json/flights_search.json @@ -0,0 +1,90 @@ +[ + { + "id": "FL001", + "from": "SFO", + "to": "JFK", + "date": "2026-07-15", + "stops": 1, + "price": 320 + }, + { + "id": "FL002", + "from": "LAX", + "to": "JFK", + "date": "2026-07-16", + "stops": 0, + "price": 410 + }, + { + "id": "FL003", + "from": "SFO", + "to": "JFK", + "date": "2026-07-15", + "stops": 1, + "price": 290 + }, + { + "id": "FL004", + "from": "SFO", + "to": "JFK", + "date": "2026-07-16", + "stops": 0, + "price": 380 + }, + { + "id": "FL005", + "from": "LAX", + "to": "JFK", + "date": "2026-07-15", + "stops": 1, + "price": 260 + }, + { + "id": "FL006", + "from": "SFO", + "to": "JFK", + "date": "2026-07-15", + "stops": 1, + "price": 305 + }, + { + "id": "FL007", + "from": "SFO", + "to": "JFK", + "date": "2026-07-16", + "stops": 1, + "price": 275 + }, + { + "id": "FL008", + "from": "LAX", + "to": "JFK", + "date": "2026-07-15", + "stops": 0, + "price": 430 + }, + { + "id": "FL009", + "from": "SFO", + "to": "JFK", + "date": "2026-07-15", + "stops": 1, + "price": 333 + }, + { + "id": "FL010", + "from": "SFO", + "to": "JFK", + "date": "2026-07-16", + "stops": 0, + "price": 360 + }, + { + "id": "FL999", + "from": "SFO", + "to": "JFK", + "date": "2026-07-15", + "stops": 0, + "price": 149 + } +] diff --git a/testdata/fixtures/structured_json/oc_pods_slice.json b/testdata/fixtures/structured_json/oc_pods_slice.json new file mode 100644 index 0000000..f8ba424 --- /dev/null +++ b/testdata/fixtures/structured_json/oc_pods_slice.json @@ -0,0 +1,147 @@ +{ + "apiVersion": "v1", + "kind": "List", + "metadata": { + "resourceVersion": "" + }, + "items": [ + { + "metadata": { + "name": "alertmanager-main-0", + "namespace": "openshift-monitoring" + }, + "status": { + "phase": "Running", + "containerStatuses": [ + { + "name": "alertmanager", + "restartCount": 0 + }, + { + "name": "config-reloader", + "restartCount": 0 + }, + { + "name": "kube-rbac-proxy", + "restartCount": 0 + }, + { + "name": "kube-rbac-proxy-metric", + "restartCount": 0 + }, + { + "name": "kube-rbac-proxy-web", + "restartCount": 0 + }, + { + "name": "prom-label-proxy", + "restartCount": 0 + } + ] + } + }, + { + "metadata": { + "name": "alertmanager-main-1", + "namespace": "openshift-monitoring" + }, + "status": { + "phase": "Running", + "containerStatuses": [ + { + "name": "alertmanager", + "restartCount": 0 + }, + { + "name": "config-reloader", + "restartCount": 0 + }, + { + "name": "kube-rbac-proxy", + "restartCount": 0 + }, + { + "name": "kube-rbac-proxy-metric", + "restartCount": 0 + }, + { + "name": "kube-rbac-proxy-web", + "restartCount": 0 + }, + { + "name": "prom-label-proxy", + "restartCount": 0 + } + ] + } + }, + { + "metadata": { + "name": "cluster-monitoring-operator-7fb9fd4c5d-wm2k6", + "namespace": "openshift-monitoring" + }, + "status": { + "phase": "Running", + "containerStatuses": [ + { + "name": "cluster-monitoring-operator", + "restartCount": 0 + } + ] + } + }, + { + "metadata": { + "name": "kube-state-metrics-75586fdddb-5zw6b", + "namespace": "openshift-monitoring" + }, + "status": { + "phase": "Running", + "containerStatuses": [ + { + "name": "kube-rbac-proxy-main", + "restartCount": 0 + }, + { + "name": "kube-rbac-proxy-self", + "restartCount": 0 + }, + { + "name": "kube-state-metrics", + "restartCount": 0 + } + ] + } + }, + { + "metadata": { + "name": "metrics-server-fdfb77995-jmqnm", + "namespace": "openshift-monitoring" + }, + "status": { + "phase": "Running", + "containerStatuses": [ + { + "name": "metrics-server", + "restartCount": 0 + } + ] + } + }, + { + "metadata": { + "name": "metrics-server-fdfb77995-w4r8p", + "namespace": "openshift-monitoring" + }, + "status": { + "phase": "Running", + "containerStatuses": [ + { + "name": "metrics-server", + "restartCount": 0 + } + ] + } + } + ] +} \ No newline at end of file diff --git a/testdata/fixtures/structured_json/products_inventory.json b/testdata/fixtures/structured_json/products_inventory.json new file mode 100644 index 0000000..876c5f3 --- /dev/null +++ b/testdata/fixtures/structured_json/products_inventory.json @@ -0,0 +1,44 @@ +[ + { + "sku": "SKU0000", + "name": "item 0", + "price": 0, + "in_stock": true + }, + { + "sku": "SKU0001", + "name": "item 1", + "price": 13, + "in_stock": false + }, + { + "sku": "SKU0002", + "name": "item 2", + "price": 26, + "in_stock": false + }, + { + "sku": "SKU0003", + "name": "item 3", + "price": 39, + "in_stock": true + }, + { + "sku": "SKU0004", + "name": "item 4", + "price": 52, + "in_stock": false + }, + { + "sku": "SKU0006", + "name": "item 6", + "price": 78, + "in_stock": true + }, + { + "sku": "SKU0009", + "name": "item 9", + "price": 117, + "in_stock": true + } +] diff --git a/testdata/fixtures/structured_json/users_directory.json b/testdata/fixtures/structured_json/users_directory.json new file mode 100644 index 0000000..4e7b2b7 --- /dev/null +++ b/testdata/fixtures/structured_json/users_directory.json @@ -0,0 +1,32 @@ +[ + { + "uid": "user_0001", + "email": "person1@corp.com", + "role": "member" + }, + { + "uid": "user_0015", + "email": "person15@corp.com", + "role": "member" + }, + { + "uid": "user_0022", + "email": "person22@corp.com", + "role": "member" + }, + { + "uid": "user_0037", + "email": "alice.target@corp.com", + "role": "admin" + }, + { + "uid": "user_0041", + "email": "person41@corp.com", + "role": "member" + }, + { + "uid": "user_0058", + "email": "person58@corp.com", + "role": "member" + } +]