diff --git a/CHANGELOG.md b/CHANGELOG.md index a5eada9a..67c34f84 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,48 @@ All notable changes to pxpipe are documented here. This project adheres to [Semantic Versioning](https://semver.org/) (pre-1.0: minor = features / behavioral changes, patch = fixes). +## Unreleased — 2026-07-19 + +### Added +- **Adaptive chars-per-token: the profitability gate can learn text density from + telemetry instead of using hand-tuned constants.** Opt in with + `PXPIPE_ADAPTIVE_CPT=1`; unset, gate behavior is byte-identical to before. The + gate prices text as `chars / CHARS_PER_TOKEN` — a constant of 4 for + reminders/tool_results and 2.0 for slab/history — while telemetry + ([plan](docs/ADAPTIVE_CPT_PLAN.md)) measured real marginal density near ~1.5 on + dense content, so the gate under-counted text cost and passed up profitable + compressions. A dependency-free 6-bucket least-squares fit + (`src/core/cpt-fit.ts`, pure/Workers-safe) over the `bucket_chars`, + `baseline_tokens` and `image_pixels` fields already logged to + `~/.pxpipe/events.jsonl` produces per-bucket rates; `src/cpt-store.ts` + (Node-only) reads the log, persists `~/.pxpipe/cpt-state.jsonl`, and serves the + gate a resolver. Image cost is subtracted on the 28×28 patch grid from + `anthropic-vision.ts` (exact for pxpipe's 1568×728 pages, both whole multiples + of 28). Learns both a per-`system_sha8` table and a pooled cross-project + fallback. Precedence: explicit host `charsPerToken` → learned → baked constant. + Every guard (min 20 samples, min 8 bucket appearances, plausible band 0.8–6.0, + condition-number check, non-positive slope, throwing resolver) falls back to + the constant **per bucket** — a wrong learned rate costs money, a missing one + is just today's behavior. Refits are throttled and run in the background, never + blocking a request. New `cpt_source` / `cpt_used` event fields record which + price each gate bucket ran with. + +### Known limitations / evidence +- Reproduce without fixtures or screenshots: + `npx tsx eval/adaptive-cpt/demo-cpt.mts` seeds a synthetic event log at a known + density, recovers it, and prices one request three ways — showing the constant + declining a compression that a learned rate correctly takes. +- Ships **verified correct but not measured for savings.** The fit recovers CPTs + it was generated from within 5% (including under noise), every guard fails + closed, and a learned rate demonstrably changes a real gate verdict in both + directions. How much this saves on production traffic is **not** established — + that needs a shadow-mode A/B. No savings figure is claimed until that run + exists, which is why the feature is opt-in rather than default. +- Workers keeps the baked constants (no filesystem for the sidecar); only the + Node path learns. +- Cold start: baked constants apply until a project has ≥20 events and the first + background refit completes. + ## 0.9.0 — 2026-07-14 ### Changed diff --git a/eval/adaptive-cpt/demo-cpt.mts b/eval/adaptive-cpt/demo-cpt.mts new file mode 100644 index 00000000..6f75d92a --- /dev/null +++ b/eval/adaptive-cpt/demo-cpt.mts @@ -0,0 +1,71 @@ +/** + * Demo B — adaptive CPT: the gate learns, and the learned rate changes decisions. + * Self-contained: seeds a synthetic events log, fits it, and A/Bs the real gate. + * Run: npx tsx eval/adaptive-cpt/demo-cpt.mts + */ +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { buildCptState, resolverFor, writeCptState } from '../../src/cpt-store.js'; +import { transformRequest } from '../../src/core/transform.js'; +import { ANTHROPIC_PATCH_PX } from '../../src/core/anthropic-vision.js'; +const PIXELS_PER_VISUAL_TOKEN = ANTHROPIC_PATCH_PX * ANTHROPIC_PATCH_PX; + +const TRUE_CPT = 1.5; // the density we secretly encode +const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'pxpipe-demo-')); +const log = path.join(dir, 'events.jsonl'); + +// ── 1. Seed a log that behaves as if this project's slab text is 1.5 chars/token +let s = 7; const rnd = () => ((s = (s * 1664525 + 1013904223) >>> 0) / 0x100000000); +const rows = Array.from({ length: 60 }, () => { + const chars = Math.floor(20000 + rnd() * 30000); + const px = Math.floor(rnd() * 400000); + return JSON.stringify({ + ts: new Date().toISOString(), system_sha8: 'demo1234', + bucket_chars: { static_slab: chars }, image_pixels: px, + baseline_tokens: chars / TRUE_CPT + px / PIXELS_PER_VISUAL_TOKEN, + }); +}); +fs.writeFileSync(log, rows.join('\n') + '\n'); +console.log(`1. Seeded ${rows.length} events encoding a TRUE slab CPT of ${TRUE_CPT}`); +console.log(` ${log}`); + +// ── 2. Learn from it +const state = await buildCptState(log); +writeCptState(state, path.join(dir, 'cpt-state.jsonl')); +const fit = state.fits.get('demo1234')!; +console.log(`\n2. Learned from the log:`); +console.log(` static_slab CPT = ${fit.cpt.static_slab?.toFixed(4)} (truth ${TRUE_CPT}) ` + + `${Math.abs((fit.cpt.static_slab ?? 0) - TRUE_CPT) < 0.05 ? '✅ recovered' : '❌'}`); +console.log(` n_events = ${fit.nSamples}`); +console.log(` buckets with no data were rejected, with reasons:`); +for (const [b, why] of Object.entries(fit.rejected)) console.log(` - ${b}: ${why}`); + +// ── 3. Show the learned rate changing a real gate decision +const enc = new TextEncoder(); +const slabReq = (lines: number, width: number) => enc.encode(JSON.stringify({ + model: 'claude-3-5-sonnet', + system: Array.from({ length: lines }, () => 'a'.repeat(width)).join('\n'), + messages: [{ role: 'user', content: 'hi' }], +})); + +console.log(`\n3. The same request, priced three ways (sparse slab: 900 lines × 32 chars):`); +for (const [label, opts] of [ + ['baked constant (2.0)', { reflow: false as const }], + ['learned dense (1.0)', { reflow: false as const, cptFor: () => 1.0 }], + ['learned sparse (6.0)', { reflow: false as const, cptFor: () => 6.0 }], +] as const) { + const { info } = await transformRequest(slabReq(900, 32), opts as any); + console.log(` ${label.padEnd(22)} → imaged=${String(info.compressed).padEnd(5)} ` + + `cpt=${info.cptUsed?.static_slab} (${info.cptSource?.static_slab})` + + `${info.compressed ? '' : ` reason=${info.reason}`}`); +} +console.log(`\n ↑ The decision FLIPS purely from the price of text — that is the feature.`); + +// ── 4. Per-project beats pooled +const resolve = resolverFor(state); +console.log(`\n4. Resolver: known project → ${resolve('static_slab', 'demo1234')?.toFixed(4)}` + + `, unknown project → ${resolve('static_slab', 'ffffffff')?.toFixed(4)} (pooled fallback)`); +console.log(` history bucket (no data) → ${resolve('history', 'demo1234') ?? 'undefined → baked constant 2.0'}`); + +fs.rmSync(dir, { recursive: true, force: true }); diff --git a/src/core/cpt-fit.ts b/src/core/cpt-fit.ts new file mode 100644 index 00000000..70f46865 --- /dev/null +++ b/src/core/cpt-fit.ts @@ -0,0 +1,218 @@ +/** + * Adaptive chars-per-token (CPT) — the fit. + * + * The profitability gate compares `imageTokens` (exact, from pixel area) against + * `textTokens = chars / CPT`. CPT was a hand-tuned constant per call site (4 for + * reminders/tool_results, 2.0 for slab/history). Production telemetry showed the + * real marginal density is ~1.5 for dense content, so the constant under-counted + * text cost and biased the gate toward passthrough — leaving savings on the table. + * + * This module learns CPT from the events pxpipe already logs. Model: a request's + * TEXT token cost decomposes into a per-bucket marginal rate times that bucket's + * char count, + * + * textTokens ≈ Σ_b α_b · chars_b with CPT_b = 1 / α_b + * + * where `textTokens` is the observed `baseline_tokens` (a free count_tokens probe + * on the ORIGINAL uncompressed body) minus the image cost, priced off Anthropic's + * 28×28-patch grid (see `PIXELS_PER_VISUAL_TOKEN` in `src/cpt-store.ts`, which + * builds the samples). Solved by ordinary least squares + * via the normal equations `α = (XᵀX)⁻¹ Xᵀy`. XᵀX is at most 6×6, so a hand-rolled + * Gauss-Jordan inverse is microseconds and needs no dependency. + * + * Pure math — no `fs`, no `process`, no `Buffer`. Safe to import from the Workers + * build. Reading the event log and persisting state is `src/cpt-store.ts` (Node). + * + * The fit is deliberately conservative: every guard below fails a bucket CLOSED + * (back to the baked constant) rather than open. A wrong learned CPT would make + * the gate image unprofitably, which costs real money; a missing one just returns + * today's behavior. + */ + +import type { BucketName } from './transform.js'; + +/** Column order for the design matrix. Stable + explicit so a fit is reproducible. */ +export const CPT_BUCKETS: readonly BucketName[] = [ + 'static_slab', + 'reminder', + 'tool_result_json', + 'tool_result_log', + 'tool_result_prose', + 'history', +]; + +/** Minimum events before any fit is trusted. Below this, a slope is noise. */ +export const MIN_SAMPLES = 20; +/** A bucket must actually appear this many times to get its own column; + * an all-but-empty column makes XᵀX singular and the slope meaningless. */ +export const MIN_BUCKET_PRESENCE = 8; +/** Plausible CPT band. Real content runs ~1.2 (dense JSON) to ~4 (prose); + * anything outside this is a fit artifact, not a measurement. */ +export const CPT_PLAUSIBLE_MIN = 0.8; +export const CPT_PLAUSIBLE_MAX = 6.0; +/** Reject the whole fit above this pivot ratio — the buckets are too collinear + * to separate (e.g. reminders that always grow with tool_results). */ +export const MAX_CONDITION = 1e8; + +/** One request's regressors + target. */ +export interface CptSample { + /** Pre-compression TEXT chars per bucket (the logged `bucket_chars`). */ + bucketChars: Partial>; + /** Observed text tokens: `baseline_tokens − imagePixels / 750`. */ + textTokens: number; +} + +export interface CptFitResult { + /** Learned chars-per-token, per bucket. Absent = use the baked default. */ + cpt: Partial>; + nSamples: number; + /** Why each bucket was NOT learned. Surfaced so a null result is explainable. */ + rejected: Partial>; + /** Pivot ratio of XᵀX. Higher = less trustworthy; > MAX_CONDITION rejects. */ + conditionEstimate: number; + /** Buckets that got a column in this fit. */ + active: BucketName[]; +} + +/** + * Invert a square matrix by Gauss-Jordan elimination with partial pivoting. + * Returns `null` when the matrix is singular. `condition` is the ratio of the + * largest to smallest pivot — a cheap stand-in for the true condition number, + * good enough to detect "these columns are not independent". + */ +function invertMatrix(src: readonly number[][]): { inv: number[][]; condition: number } | null { + const n = src.length; + if (n === 0) return null; + // Augment [A | I] and reduce the left half to identity. + const a: number[][] = src.map((row, i) => { + const aug = new Array(2 * n).fill(0); + for (let j = 0; j < n; j++) aug[j] = row[j] ?? 0; + aug[n + i] = 1; + return aug; + }); + + let minPivot = Infinity; + let maxPivot = 0; + + for (let col = 0; col < n; col++) { + let best = col; + for (let r = col + 1; r < n; r++) { + if (Math.abs(a[r]![col]!) > Math.abs(a[best]![col]!)) best = r; + } + const pivot = Math.abs(a[best]![col]!); + if (!Number.isFinite(pivot) || pivot === 0) return null; + minPivot = Math.min(minPivot, pivot); + maxPivot = Math.max(maxPivot, pivot); + + if (best !== col) { + const tmp = a[best]!; + a[best] = a[col]!; + a[col] = tmp; + } + + const d = a[col]![col]!; + for (let j = 0; j < 2 * n; j++) a[col]![j] = a[col]![j]! / d; + + for (let r = 0; r < n; r++) { + if (r === col) continue; + const f = a[r]![col]!; + if (f === 0) continue; + for (let j = 0; j < 2 * n; j++) a[r]![j] = a[r]![j]! - f * a[col]![j]!; + } + } + + const inv = a.map((row) => row.slice(n)); + return { inv, condition: minPivot > 0 ? maxPivot / minPivot : Infinity }; +} + +/** Reject every bucket with one reason. Used for the whole-fit bailouts. */ +function rejectAll(reason: string): Partial> { + const out: Partial> = {}; + for (const b of CPT_BUCKETS) out[b] = reason; + return out; +} + +/** + * Fit per-bucket CPT from samples. Never throws; a fit it cannot trust comes back + * with an empty `cpt` and a populated `rejected` explaining why. + */ +export function fitCpt(samples: readonly CptSample[]): CptFitResult { + const n = samples.length; + if (n < MIN_SAMPLES) { + return { + cpt: {}, + nSamples: n, + rejected: rejectAll(`n=${n} < MIN_SAMPLES=${MIN_SAMPLES}`), + conditionEstimate: Infinity, + active: [], + }; + } + + const rejected: Partial> = {}; + + // Only give a column to buckets that actually show up. An all-zero (or + // nearly-all-zero) column is exactly collinear with nothing and makes the + // normal equations singular. + const active: BucketName[] = []; + for (const b of CPT_BUCKETS) { + let present = 0; + for (const s of samples) if ((s.bucketChars[b] ?? 0) > 0) present++; + if (present < MIN_BUCKET_PRESENCE) { + rejected[b] = `present in ${present}/${n} samples < ${MIN_BUCKET_PRESENCE}`; + } else { + active.push(b); + } + } + if (active.length === 0) { + return { cpt: {}, nSamples: n, rejected, conditionEstimate: Infinity, active }; + } + + // Accumulate XᵀX and Xᵀy in one pass. + const k = active.length; + const xtx: number[][] = Array.from({ length: k }, () => new Array(k).fill(0)); + const xty = new Array(k).fill(0); + for (const s of samples) { + if (!Number.isFinite(s.textTokens)) continue; + const row = new Array(k); + for (let i = 0; i < k; i++) row[i] = s.bucketChars[active[i]!] ?? 0; + for (let i = 0; i < k; i++) { + const ri = row[i]!; + xty[i] = xty[i]! + ri * s.textTokens; + for (let j = 0; j < k; j++) xtx[i]![j] = xtx[i]![j]! + ri * row[j]!; + } + } + + const inverted = invertMatrix(xtx); + if (!inverted) { + for (const b of active) rejected[b] = 'singular XᵀX (collinear buckets)'; + return { cpt: {}, nSamples: n, rejected, conditionEstimate: Infinity, active }; + } + if (inverted.condition > MAX_CONDITION) { + for (const b of active) { + rejected[b] = `ill-conditioned (${inverted.condition.toExponential(1)} > ${MAX_CONDITION.toExponential(0)})`; + } + return { cpt: {}, nSamples: n, rejected, conditionEstimate: inverted.condition, active }; + } + + // α = (XᵀX)⁻¹ Xᵀy, then CPT = 1/α with a plausibility band. + const cpt: Partial> = {}; + for (let i = 0; i < k; i++) { + const bucket = active[i]!; + let alpha = 0; + for (let j = 0; j < k; j++) alpha += inverted.inv[i]![j]! * xty[j]!; + + if (!Number.isFinite(alpha) || alpha <= 0) { + rejected[bucket] = `non-positive slope (${Number.isFinite(alpha) ? alpha.toExponential(2) : 'NaN'})`; + continue; + } + const value = 1 / alpha; + if (value < CPT_PLAUSIBLE_MIN || value > CPT_PLAUSIBLE_MAX) { + rejected[bucket] = + `cpt ${value.toFixed(2)} outside [${CPT_PLAUSIBLE_MIN}, ${CPT_PLAUSIBLE_MAX}]`; + continue; + } + cpt[bucket] = value; + } + + return { cpt, nSamples: n, rejected, conditionEstimate: inverted.condition, active }; +} diff --git a/src/core/tracker.ts b/src/core/tracker.ts index e3f047c9..03187dca 100644 --- a/src/core/tracker.ts +++ b/src/core/tracker.ts @@ -80,6 +80,12 @@ export interface TrackEvent { 'history', number >>; + /** Adaptive CPT: which chars-per-token source each gate bucket used this request + * (`host` = caller-pinned, `learned` = fitted from telemetry, `default` = baked + * constant), and the value it ran with. Lets a shadow-mode/eval run attribute a + * decision change to the learned rate. Absent when no gate bucket fired. */ + cpt_source?: Partial>; + cpt_used?: Partial>; /** TEXT chars that fed the history-image renderer; separate from bucket_chars because it credits a synthetic message. */ history_text_chars?: number; /** sha8 of the collapsed history image. Unchanged across turns proves the prompt cache is hitting (cache_read). @@ -262,6 +268,12 @@ export function toTrackEvent(ev: ProxyEvent): TrackEvent { // Omit empty object so noop-pass requests stay lean; presence means at least one gate fired. out.bucket_chars = info.bucketChars; } + // Adaptive CPT: only emitted when a learned rate actually reached a gate, so + // default-constant traffic keeps byte-identical rows. + if (info.cptSource && Object.values(info.cptSource).some((s) => s === 'learned')) { + out.cpt_source = info.cptSource; + if (info.cptUsed) out.cpt_used = info.cptUsed; + } if (info.historyTextChars !== undefined && info.historyTextChars > 0) { out.history_text_chars = info.historyTextChars; } diff --git a/src/core/transform.ts b/src/core/transform.ts index 8ce9c8f8..90ce9830 100644 --- a/src/core/transform.ts +++ b/src/core/transform.ts @@ -123,6 +123,11 @@ export interface TransformOptions { * for every block rendered to images. Off by default (entries inflate `info`; * only a stateful harness can use them). */ emitRecoverable?: boolean; + /** Adaptive CPT: learned chars-per-token for a gate bucket, or `undefined` to use + * the baked constant. Supplied by `src/cpt-store.ts` on the Node path; the Workers + * build passes nothing and keeps today's constants. An explicit `charsPerToken` + * always wins over a learned value. See src/core/cpt-fit.ts. */ + cptFor?: (bucket: BucketName, systemSha8?: string) => number | undefined; } const DEFAULTS: Required = { @@ -148,6 +153,9 @@ const DEFAULTS: Required = { reflow: true, keepSharp: () => false, emitRecoverable: false, + // No learned CPT by default — every gate uses its baked constant unless the + // Node host injects a resolver. + cptFor: () => undefined, // GPT-only knobs; the Anthropic transform ignores them but Required<> needs them. collapseHistory: true, gptHistory: {}, @@ -477,6 +485,46 @@ function bumpBucket(info: TransformInfo, bucket: BucketName, chars: number): voi info.bucketChars[bucket] = (info.bucketChars[bucket] ?? 0) + chars; } +/** + * Resolve the chars-per-token the gate should use for one bucket. + * + * Precedence, highest first: + * 1. an explicit host `charsPerToken` (a caller pinning the gate wins outright); + * 2. a learned CPT for this (system, bucket) from `options.cptFor` — adaptive CPT; + * 3. the baked constant for that call site (today's behavior). + * + * A resolver that throws or returns a non-finite/≤0 number is ignored, so a broken + * learned table can never be worse than the constant. Records which source won on + * `info.cptSource` for the dashboard. + */ +function resolveCpt( + opts: TransformOptions, + o: Required, + info: TransformInfo, + bucket: BucketName, + fallback: number, +): number { + if (opts.charsPerToken !== undefined) { + (info.cptSource ??= {})[bucket] = 'host'; + (info.cptUsed ??= {})[bucket] = o.charsPerToken; + return o.charsPerToken; + } + let learned: number | undefined; + try { + learned = o.cptFor(bucket, info.systemSha8); + } catch { + learned = undefined; + } + if (typeof learned === 'number' && Number.isFinite(learned) && learned > 0) { + (info.cptSource ??= {})[bucket] = 'learned'; + (info.cptUsed ??= {})[bucket] = learned; + return learned; + } + (info.cptSource ??= {})[bucket] = 'default'; + (info.cptUsed ??= {})[bucket] = fallback; + return fallback; +} + /** Map `classifyContent` shape to a tool_result bucket name. */ function toolResultBucket(shape: 'structured' | 'log' | 'other'): BucketName { if (shape === 'structured') return 'tool_result_json'; @@ -612,6 +660,12 @@ export interface TransformInfo { }; /** Pre-compaction TEXT char totals per gate-call bucket. Rolling-cpt regression denominator. */ bucketChars?: BucketChars; + /** Adaptive CPT: which CPT source each gate bucket actually used this request. + * `host` = caller pinned `charsPerToken`; `learned` = fitted from telemetry; + * `default` = the baked constant. */ + cptSource?: Partial>; + /** The chars-per-token value each bucket's gate ran with (pairs with cptSource). */ + cptUsed?: Partial>; /** Chars fed into the history-image renderer. Folded into `bucketChars.history` too. */ historyTextChars?: number; /** Blocks pinned as text by the caller's `keepSharp` predicate this request. */ @@ -1631,9 +1685,7 @@ async function runHistoryCollapseAndFinalize( ): Promise<{ body: Uint8Array; info: TransformInfo; collapsed: boolean }> { let collapsedFlag = false; if (Array.isArray(req.messages) && req.messages.length > 0) { - const historyCpt = opts.charsPerToken !== undefined - ? o.charsPerToken - : HISTORY_CHARS_PER_TOKEN; + const historyCpt = resolveCpt(opts, o, info, 'history', HISTORY_CHARS_PER_TOKEN); const horizon = Math.max(1, Math.floor(o.historyAmortizationHorizon)); // Pass the symmetric warm-cache burn through to the history-collapse // gate as well. The slab gate alone got the symmetric treatment, which @@ -1921,10 +1973,9 @@ export async function transformRequest( ); // Gate geometry for dense single-col (tool_result/reminder) paths — 384-col/240-row. const denseGeo = denseGateGeometry(o.cols, numCols); - // Use slab cpt (2.0) unless host pinned charsPerToken explicitly. - const slabCpt = opts.charsPerToken !== undefined - ? o.charsPerToken - : SLAB_CHARS_PER_TOKEN; + // Use slab cpt (2.0) unless host pinned charsPerToken explicitly or adaptive + // CPT learned a rate for this project's slab. + const slabCpt = resolveCpt(opts, o, info, 'static_slab', SLAB_CHARS_PER_TOKEN); // Shrink canvas to longest actual line — pure function of (text, cols) so the // cache prefix stays byte-identical across turns. The banner sets a natural width floor. const reflowNoteImg = o.reflow @@ -2107,7 +2158,8 @@ export async function transformRequest( // model reads. const reminderRaw = (blk as TextBlock).text; const reminderText = maybeReflow(compactSlabWhitespace(reminderRaw), o.reflow); - if (!isCompressionProfitable(reminderText, denseGeo.cols, undefined, numCols, o.charsPerToken, 0, 0, true, denseGeo.maxChars)) { + const reminderCpt = resolveCpt(opts, o, info, 'reminder', o.charsPerToken); + if (!isCompressionProfitable(reminderText, denseGeo.cols, undefined, numCols, reminderCpt, 0, 0, true, denseGeo.maxChars)) { bumpPassthrough(info, 'not_profitable'); processedExisting.push(blk); continue; @@ -2182,10 +2234,13 @@ export async function transformRequest( const inner = compactSlabWhitespace(innerRaw); // classifyContent sees pre-reflow `inner` so shape bucketing reflects real structure. const innerR = maybeReflow(inner, o.reflow); + // Resolved once: the same bucket drives the adaptive-CPT gate below + // and the telemetry attribution after a successful render. + const trBucket = toolResultBucket(classifyContent(inner)); if (innerR.length < o.minToolResultChars) { bumpPassthrough(info, 'below_threshold'); rewritten.push(blk); - } else if (!isCompressionProfitable(innerR, denseGeo.cols, o.maxImagesPerToolResult, numCols, o.charsPerToken, 0, 0, true, denseGeo.maxChars)) { + } else if (!isCompressionProfitable(innerR, denseGeo.cols, o.maxImagesPerToolResult, numCols, resolveCpt(opts, o, info, trBucket, o.charsPerToken), 0, 0, true, denseGeo.maxChars)) { bumpPassthrough(info, 'not_profitable'); rewritten.push(blk); } else { @@ -2220,7 +2275,7 @@ export async function transformRequest( content: trFactSheet ? [...imgs, { type: 'text' as const, text: trFactSheet }] : imgs, }); changed = true; - bumpBucket(info, toolResultBucket(classifyContent(inner)), innerRaw.length); + bumpBucket(info, trBucket, innerRaw.length); } } else if (Array.isArray(innerRaw)) { const newInner: Array = []; @@ -2246,12 +2301,14 @@ export async function transformRequest( const innerText = compactSlabWhitespace(innerTextRaw); // R3: gate/page/render on reflowed text; classify pre-reflow. const innerTextR = maybeReflow(innerText, o.reflow); + // One bucket for both the adaptive-CPT gate and the telemetry below. + const trTextBucket = toolResultBucket(classifyContent(innerText)); if (innerTextR.length < o.minToolResultChars) { bumpPassthrough(info, 'below_threshold'); newInner.push(ib as TextBlock | ImageBlock); continue; } - if (!isCompressionProfitable(innerTextR, denseGeo.cols, o.maxImagesPerToolResult, numCols, o.charsPerToken, 0, 0, true, denseGeo.maxChars)) { + if (!isCompressionProfitable(innerTextR, denseGeo.cols, o.maxImagesPerToolResult, numCols, resolveCpt(opts, o, info, trTextBucket, o.charsPerToken), 0, 0, true, denseGeo.maxChars)) { bumpPassthrough(info, 'not_profitable'); newInner.push(ib as TextBlock | ImageBlock); continue; @@ -2291,7 +2348,7 @@ export async function transformRequest( for (const [cp, n] of dcp) { droppedCodepoints.set(cp, (droppedCodepoints.get(cp) ?? 0) + n); } - bumpBucket(info, toolResultBucket(classifyContent(innerText)), innerTextRaw.length); + bumpBucket(info, trTextBucket, innerTextRaw.length); innerChanged = true; } if (innerChanged) { @@ -2319,9 +2376,7 @@ export async function transformRequest( // protectedPrefix excludes the slab-bearing first user message — collapsing it // would reduce slab images to [image] placeholders and destroy the cache anchor. if (Array.isArray(req.messages) && req.messages.length > 0) { - const historyCpt = opts.charsPerToken !== undefined - ? o.charsPerToken - : HISTORY_CHARS_PER_TOKEN; + const historyCpt = resolveCpt(opts, o, info, 'history', HISTORY_CHARS_PER_TOKEN); const horizon = Math.max(1, Math.floor(o.historyAmortizationHorizon)); const historyProfitable = (text: string, cols: number): boolean => { // Gate at dense 384-col/240-row geometry (matches history.ts renderer). diff --git a/src/cpt-store.ts b/src/cpt-store.ts new file mode 100644 index 00000000..87b00386 --- /dev/null +++ b/src/cpt-store.ts @@ -0,0 +1,222 @@ +/** + * Adaptive chars-per-token (CPT) — the store. NODE ONLY. + * + * Reads the events pxpipe already logs (`~/.pxpipe/events.jsonl`), turns them + * into regression samples, runs the fit (`src/core/cpt-fit.ts`), and hands the + * proxy a resolver the profitability gate can consult. + * + * Split rationale: `core/` must stay Workers-safe (no fs), so all disk access + * lives here. The Workers build never imports this module and therefore always + * uses the baked constants — documented, intentional divergence. + * + * Two tables are learned: + * - per `system_sha8` (a project fingerprint), used when that project has + * enough events of its own; + * - a pooled GLOBAL table over every event, used as the fallback so a brand-new + * project still beats the hand-tuned constant on its first request. + * Resolution order at the gate: explicit host override → per-project → global → + * baked constant. + */ + +import fs from 'node:fs'; +import path from 'node:path'; +import { defaultPaths, readEvents } from './sessions.js'; +import { fitCpt, type CptSample, type CptFitResult } from './core/cpt-fit.js'; +import { ANTHROPIC_PATCH_PX } from './core/anthropic-vision.js'; +import type { BucketName } from './core/transform.js'; + +/** + * Pixels per visual token, used to price the image half of a logged request. + * + * Anthropic bills `⌈w/28⌉ × ⌈h/28⌉` patches, but the event log records only the + * SUM of `w×h` across a request's images, not their individual dimensions — so + * the exact per-image ceiling cannot be reconstructed. Dividing total pixels by + * the patch AREA (28² = 784) is the aggregate form of the same grid. + * + * This is exact, not an approximation, for pxpipe's own output: pages are + * 1568×728, and both edges are whole multiples of 28 (56×26 = 1456 patches; + * 1568×728/784 = 1456). Residual error only appears for height-shrunk pages + * whose edges aren't multiples of 28, and is bounded by one patch row/column. + * (The retired `/750` constant was a ~4% continuous fit to this same grid and + * over-stated image cost, which biased every learned rate.) + */ +const PIXELS_PER_VISUAL_TOKEN = ANTHROPIC_PATCH_PX * ANTHROPIC_PATCH_PX; + +/** Key used for the pooled cross-project table in the state file. */ +export const GLOBAL_KEY = '*'; + +export interface CptState { + /** Fit results keyed by `system_sha8`, plus GLOBAL_KEY for the pooled table. */ + fits: Map; + /** mtimeMs of the events file this was built from (cache key). */ + sourceMtimeMs: number; + builtAt: string; +} + +/** `(bucket, systemSha8?) => learned CPT | undefined`. Undefined = use the default. */ +export type CptResolver = (bucket: BucketName, systemSha8?: string) => number | undefined; + +/** A resolver that never learns anything — the Workers/default behavior. */ +export const NO_CPT: CptResolver = () => undefined; + +/** + * Convert one logged event into a regression sample. + * Returns null when the row lacks the fields the fit needs. + * + * `textTokens` removes the image cost we already know exactly, leaving only the + * text cost whose density we are trying to learn. + */ +export function sampleFromEvent(ev: { + bucket_chars?: Partial>; + baseline_tokens?: number; + image_pixels?: number; +}): CptSample | null { + const buckets = ev.bucket_chars; + const baseline = ev.baseline_tokens; + if (!buckets || typeof baseline !== 'number' || !Number.isFinite(baseline)) return null; + + let totalChars = 0; + for (const v of Object.values(buckets)) totalChars += typeof v === 'number' ? v : 0; + if (totalChars <= 0) return null; + + const pixels = typeof ev.image_pixels === 'number' ? ev.image_pixels : 0; + const textTokens = baseline - pixels / PIXELS_PER_VISUAL_TOKEN; + // A request whose text cost prices out at ≤0 is a broken/partial row. + if (!Number.isFinite(textTokens) || textTokens <= 0) return null; + + return { bucketChars: buckets, textTokens }; +} + +/** + * Stream the event log and fit every project plus the pooled global table. + * Never throws on a missing/unreadable log — returns an empty state. + */ +export async function buildCptState(eventsFile?: string): Promise { + const file = eventsFile ?? defaultPaths().eventsFile; + const fits = new Map(); + let sourceMtimeMs = 0; + try { + sourceMtimeMs = fs.statSync(file).mtimeMs; + } catch { + return { fits, sourceMtimeMs: 0, builtAt: new Date().toISOString() }; + } + + const bySystem = new Map(); + const pooled: CptSample[] = []; + try { + for await (const { ev } of readEvents(file)) { + const sample = sampleFromEvent(ev as Parameters[0]); + if (!sample) continue; + pooled.push(sample); + const key = (ev as { system_sha8?: string }).system_sha8; + if (key) { + const list = bySystem.get(key); + if (list) list.push(sample); + else bySystem.set(key, [sample]); + } + } + } catch { + // Partial read still yields a usable (smaller) fit. + } + + for (const [key, samples] of bySystem) { + const fit = fitCpt(samples); + if (Object.keys(fit.cpt).length > 0) fits.set(key, fit); + } + const globalFit = fitCpt(pooled); + if (Object.keys(globalFit.cpt).length > 0) fits.set(GLOBAL_KEY, globalFit); + + return { fits, sourceMtimeMs, builtAt: new Date().toISOString() }; +} + +/** Persist the learned tables next to the events log, for inspection + the dashboard. */ +export function writeCptState(state: CptState, stateFile?: string): void { + const file = stateFile ?? path.join(path.dirname(defaultPaths().eventsFile), 'cpt-state.jsonl'); + const lines: string[] = []; + for (const [key, fit] of state.fits) { + lines.push( + JSON.stringify({ + system_sha8: key, + updated_at: state.builtAt, + n_events: fit.nSamples, + condition: Number.isFinite(fit.conditionEstimate) + ? Number(fit.conditionEstimate.toFixed(1)) + : null, + cpt: fit.cpt, + rejected: fit.rejected, + }), + ); + } + try { + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, lines.length ? lines.join('\n') + '\n' : '', 'utf8'); + } catch { + // Persisting is a convenience, never a hard requirement. + } +} + +/** Build a resolver over an already-computed state. Per-project first, then pooled. */ +export function resolverFor(state: CptState): CptResolver { + if (state.fits.size === 0) return NO_CPT; + return (bucket, systemSha8) => { + if (systemSha8) { + const own = state.fits.get(systemSha8)?.cpt[bucket]; + if (typeof own === 'number') return own; + } + const pooledCpt = state.fits.get(GLOBAL_KEY)?.cpt[bucket]; + return typeof pooledCpt === 'number' ? pooledCpt : undefined; + }; +} + +/** + * Process-lifetime cached resolver. Refits only when the events file has grown + * (mtime change) and at most every `minRefitMs`, so a busy proxy never pays the + * scan cost per request. + * + * ponytail: full re-fit over the whole log on refresh, no streaming/incremental + * update. Logs are ≤ ~1e5 rows and refits are throttled to minutes, so this is + * milliseconds; switch to an incremental XᵀX accumulator if the log outgrows RAM. + */ +export function createCachedCptResolver(opts?: { + eventsFile?: string; + minRefitMs?: number; + persist?: boolean; +}): { resolver: CptResolver; refresh: () => Promise } { + const minRefitMs = opts?.minRefitMs ?? 5 * 60_000; + let state: CptState | null = null; + let resolver: CptResolver = NO_CPT; + let lastAttempt = 0; + let inFlight: Promise | null = null; + + const refresh = async (): Promise => { + if (inFlight) return inFlight; + const now = Date.now(); + if (now - lastAttempt < minRefitMs) return state; + lastAttempt = now; + inFlight = (async () => { + try { + const file = opts?.eventsFile ?? defaultPaths().eventsFile; + let mtime = 0; + try { + mtime = fs.statSync(file).mtimeMs; + } catch { + return state; + } + if (state && mtime === state.sourceMtimeMs) return state; + const next = await buildCptState(file); + state = next; + resolver = resolverFor(next); + if (opts?.persist !== false) writeCptState(next); + return next; + } catch { + return state; + } finally { + inFlight = null; + } + })(); + return inFlight; + }; + + // Indirect so callers keep one stable function identity across refreshes. + return { resolver: (bucket, sha) => resolver(bucket, sha), refresh }; +} diff --git a/src/node.ts b/src/node.ts index ff200cde..8e76d4b7 100644 --- a/src/node.ts +++ b/src/node.ts @@ -23,6 +23,7 @@ import { type ExportResult, } from './core/export.js'; import { readExportTextFile } from './export-collect.js'; +import { createCachedCptResolver } from './cpt-store.js'; import { toTrackEvent, TRACK_BODY_INLINE_MAX, @@ -198,6 +199,10 @@ Environment: PXPIPE_CONFIG JSON config path (default ~/.config/pxpipe/config.json) supports {"models": [...]} or {"models": "off"} PXPIPE_LOG JSONL events path (default ~/.pxpipe/events.jsonl) + PXPIPE_ADAPTIVE_CPT set to 1 to price the profitability gate with + chars-per-token LEARNED from your own events.jsonl + instead of the baked constants. Falls back per bucket + until there is enough data. Writes ~/.pxpipe/cpt-state.jsonl PXPIPE_DUMP_DIR debug: write every rendered PNG here (what the model sees); off unless set. Compress arm only. PXPIPE_DEBUG_CAPTURE_4XX debug: set to 1 to persist full 4xx request bodies @@ -943,6 +948,21 @@ async function main(): Promise { if (forcePassthrough) { console.log('[pxpipe] PXPIPE_DISABLE set — passthrough mode (compress=false), still logging usage + baselines'); } + // Adaptive CPT (opt-in, PXPIPE_ADAPTIVE_CPT=1): learn chars-per-token per + // content bucket from this machine's own events.jsonl instead of using the + // baked constants, so the profitability gate stops mispricing text on + // workloads that differ from the hand-tuned defaults. Off by default — it + // changes gate decisions, so it ships behind a flag until a shadow-mode run + // has measured the delta. See src/core/cpt-fit.ts and src/cpt-store.ts. + const adaptiveCpt = /^(1|true|yes|on)$/i.test(process.env.PXPIPE_ADAPTIVE_CPT ?? '') + ? createCachedCptResolver() + : null; + if (adaptiveCpt) { + console.log( + '[pxpipe] PXPIPE_ADAPTIVE_CPT set — profitability gate will use chars-per-token ' + + 'learned from events.jsonl (falls back to baked constants until enough data)', + ); + } // Debug aid: when PXPIPE_DUMP_DIR is set, persist every rendered PNG this // process emits, so you can eyeball exactly what the model received (OCR / // legibility audits, demo inspection). Best-effort — never affects requests. @@ -1016,6 +1036,13 @@ async function main(): Promise { // still logging real usage + count_tokens baselines to its own PXPIPE_LOG. // (The dashboard kill switch does the same thing at runtime.) if (forcePassthrough || !dashboard.getCompressionEnabled()) return { compress: false }; + // Adaptive CPT (opt-in): kick a throttled background refit and hand the + // gate the learned chars-per-token table. Never blocks the request — the + // first turns after start use the baked constants until the fit lands. + if (adaptiveCpt) { + void adaptiveCpt.refresh(); + return { cptFor: adaptiveCpt.resolver }; + } // Active path: use DEFAULTS in transform.ts for break-even gating. return {}; }, diff --git a/tests/cpt-adaptive.test.ts b/tests/cpt-adaptive.test.ts new file mode 100644 index 00000000..733087d1 --- /dev/null +++ b/tests/cpt-adaptive.test.ts @@ -0,0 +1,151 @@ +/** + * Adaptive CPT — gate wiring. + * + * `cpt-fit.test.ts` proves the regression recovers the right numbers. This file + * proves the numbers are actually *consumed*: that a learned chars-per-token + * reaches the profitability gate, changes real decisions, and can never make the + * gate worse than the baked constant when the learned table is absent or broken. + * + * Contract: + * - No resolver → every bucket reports `default` and the baked constant (today's + * behavior, byte-for-byte). + * - A learned CPT is reported on `info.cptSource` / `info.cptUsed`. + * - A learned CPT changes the gate verdict (the whole point of the feature). + * - Precedence: explicit `charsPerToken` > learned > baked constant. + * - A throwing or nonsensical resolver is ignored, never fatal. + * - The resolver is told which project (`system_sha8`) is asking. + */ + +import { describe, expect, it } from 'vitest'; +import { + transformRequest, + SLAB_CHARS_PER_TOKEN, + type BucketName, +} from '../src/core/transform.js'; + +const enc = new TextEncoder(); + +/** + * A system slab of `nLines` lines each `width` chars wide. With `reflow: false` + * the rendered page keeps one row per line, so a narrow width makes a tall, + * mostly-empty (expensive) image — the regime where the gate is genuinely + * cpt-sensitive rather than trivially profitable. + */ +function slabReq(nLines: number, width: number): Uint8Array { + const line = 'a'.repeat(width); + return enc.encode( + JSON.stringify({ + model: 'claude-3-5-sonnet', + system: Array.from({ length: nLines }, () => line).join('\n'), + messages: [{ role: 'user', content: 'hi' }], + }), + ); +} + +/** Slab shape the constant (2.0) images but a sparse learned rate (6.0) refuses. */ +const SPARSE = { lines: 900, width: 48 } as const; +/** Slab shape the constant passes up but a dense learned rate (1.0) captures — + * the case that motivates the whole feature: savings the constant leaves behind. */ +const MISSED = { lines: 900, width: 32 } as const; + +describe('adaptive CPT — gate wiring', () => { + it('uses the baked constant and reports "default" when no resolver is supplied', async () => { + const { info } = await transformRequest(slabReq(400, 120), { reflow: false }); + expect(info.cptSource?.static_slab).toBe('default'); + expect(info.cptUsed?.static_slab).toBe(SLAB_CHARS_PER_TOKEN); + }); + + it('consumes a learned CPT and reports it as "learned"', async () => { + const { info } = await transformRequest(slabReq(400, 120), { + reflow: false, + cptFor: () => 1.37, + }); + expect(info.cptSource?.static_slab).toBe('learned'); + expect(info.cptUsed?.static_slab).toBeCloseTo(1.37, 5); + }); + + it('changes the gate verdict — a high learned CPT refuses an image the constant would have made', async () => { + const req = () => slabReq(SPARSE.lines, SPARSE.width); + const base = { reflow: false as const }; + + // Baked constant (2.0): dense assumption → images this sparse slab. + const withDefault = await transformRequest(req(), base); + // Learned 6.0 (genuinely sparse prose): text is cheaper than the image → skip. + const withLearned = await transformRequest(req(), { ...base, cptFor: () => 6.0 }); + + expect(withDefault.info.compressed).toBe(true); + expect(withLearned.info.compressed).toBe(false); + expect(withLearned.info.reason).toContain('not_profitable'); + }); + + it('captures a compression the baked constant passes up (the motivating case)', async () => { + const req = () => slabReq(MISSED.lines, MISSED.width); + const base = { reflow: false as const }; + + // Constant 2.0 under-counts this slab's true token cost → declines to image. + const withDefault = await transformRequest(req(), base); + // Learned 1.0 prices the same text honestly → the image now wins. + const withLearned = await transformRequest(req(), { ...base, cptFor: () => 1.0 }); + + expect(withDefault.info.compressed).toBe(false); + expect(withLearned.info.compressed).toBe(true); + }); + + it('keeps imaging when the learned CPT says the content is dense', async () => { + const { info } = await transformRequest(slabReq(SPARSE.lines, SPARSE.width), { + reflow: false, + cptFor: () => 1.0, + }); + expect(info.compressed).toBe(true); + }); + + it('lets an explicit charsPerToken override the learned value', async () => { + const { info } = await transformRequest(slabReq(SPARSE.lines, SPARSE.width), { + reflow: false, + charsPerToken: 6.0, // host pins the gate … + cptFor: () => 1.0, // … and the learned table disagrees; the host must win. + }); + expect(info.cptSource?.static_slab).toBe('host'); + expect(info.cptUsed?.static_slab).toBe(6.0); + expect(info.compressed).toBe(false); + }); + + it('ignores a throwing resolver instead of failing the request', async () => { + const { info } = await transformRequest(slabReq(400, 120), { + reflow: false, + cptFor: () => { + throw new Error('corrupt cpt-state.jsonl'); + }, + }); + expect(info.cptSource?.static_slab).toBe('default'); + expect(info.cptUsed?.static_slab).toBe(SLAB_CHARS_PER_TOKEN); + }); + + it('ignores nonsensical learned values (NaN, zero, negative)', async () => { + for (const bad of [Number.NaN, 0, -2, Number.POSITIVE_INFINITY]) { + const { info } = await transformRequest(slabReq(400, 120), { + reflow: false, + cptFor: () => bad, + }); + expect(info.cptSource?.static_slab, `value ${bad}`).toBe('default'); + expect(info.cptUsed?.static_slab).toBe(SLAB_CHARS_PER_TOKEN); + } + }); + + it('tells the resolver which bucket and which project is asking', async () => { + const seen: Array<{ bucket: BucketName; sha?: string }> = []; + const { info } = await transformRequest(slabReq(400, 120), { + reflow: false, + cptFor: (bucket, systemSha8) => { + seen.push({ bucket, sha: systemSha8 }); + return undefined; + }, + }); + const slab = seen.find((s) => s.bucket === 'static_slab'); + expect(slab).toBeDefined(); + // The slab gate runs after the system fingerprint is computed, so the + // resolver can serve a per-project rate. + expect(slab!.sha).toBe(info.systemSha8); + expect(typeof slab!.sha).toBe('string'); + }); +}); diff --git a/tests/cpt-fit.test.ts b/tests/cpt-fit.test.ts new file mode 100644 index 00000000..814b8d44 --- /dev/null +++ b/tests/cpt-fit.test.ts @@ -0,0 +1,157 @@ +import { describe, expect, it } from 'vitest'; +import { + fitCpt, + CPT_BUCKETS, + MIN_SAMPLES, + CPT_PLAUSIBLE_MAX, + type CptSample, +} from '../src/core/cpt-fit.js'; +import type { BucketName } from '../src/core/transform.js'; + +/** Build a sample whose textTokens is exactly Σ chars_b / cpt_b (noise-free). */ +function synth( + chars: Partial>, + trueCpt: Partial>, +): CptSample { + let tokens = 0; + for (const b of CPT_BUCKETS) { + const c = chars[b] ?? 0; + const cpt = trueCpt[b]; + if (c > 0 && cpt) tokens += c / cpt; + } + return { bucketChars: chars, textTokens: tokens }; +} + +/** Deterministic pseudo-random so failures reproduce. */ +function rng(seed: number): () => number { + let s = seed >>> 0; + return () => { + s = (s * 1664525 + 1013904223) >>> 0; + return s / 0x100000000; + }; +} + +describe('fitCpt', () => { + it('recovers known CPTs from noise-free synthetic data', () => { + const truth: Partial> = { + static_slab: 1.5, + tool_result_json: 1.3, + tool_result_prose: 3.6, + }; + const r = rng(42); + const samples: CptSample[] = []; + for (let i = 0; i < 60; i++) { + samples.push( + synth( + { + static_slab: Math.floor(20000 + r() * 40000), + tool_result_json: Math.floor(5000 + r() * 30000), + tool_result_prose: Math.floor(2000 + r() * 20000), + }, + truth, + ), + ); + } + + const fit = fitCpt(samples); + expect(fit.nSamples).toBe(60); + for (const [bucket, expected] of Object.entries(truth) as Array<[BucketName, number]>) { + const got = fit.cpt[bucket]; + expect(got, `${bucket} rejected: ${fit.rejected[bucket]}`).toBeDefined(); + // ±5% — the fit is exact here, so this is a generous correctness bound. + expect(Math.abs(got! - expected) / expected).toBeLessThan(0.05); + } + }); + + it('still recovers CPTs within tolerance under mild noise', () => { + const truth: Partial> = { static_slab: 1.5, history: 1.4 }; + const r = rng(7); + const samples: CptSample[] = []; + for (let i = 0; i < 120; i++) { + const s = synth( + { + static_slab: Math.floor(20000 + r() * 40000), + history: Math.floor(10000 + r() * 50000), + }, + truth, + ); + // ±2% multiplicative noise on the observed token count. + s.textTokens *= 1 + (r() - 0.5) * 0.04; + samples.push(s); + } + const fit = fitCpt(samples); + expect(fit.cpt.static_slab).toBeDefined(); + expect(Math.abs(fit.cpt.static_slab! - 1.5) / 1.5).toBeLessThan(0.1); + expect(Math.abs(fit.cpt.history! - 1.4) / 1.4).toBeLessThan(0.1); + }); + + it('refuses to fit below MIN_SAMPLES and says why', () => { + const truth = { static_slab: 1.5 }; + const samples = Array.from({ length: MIN_SAMPLES - 1 }, (_, i) => + synth({ static_slab: 10000 + i * 100 }, truth), + ); + const fit = fitCpt(samples); + expect(fit.cpt).toEqual({}); + expect(fit.rejected.static_slab).toContain('MIN_SAMPLES'); + }); + + it('rejects a bucket that barely appears instead of fitting noise', () => { + const truth: Partial> = { static_slab: 1.5, reminder: 2.0 }; + const r = rng(11); + const samples: CptSample[] = []; + for (let i = 0; i < 40; i++) { + // `reminder` present in only 3 of 40 samples → below MIN_BUCKET_PRESENCE. + const chars: Partial> = { + static_slab: Math.floor(20000 + r() * 20000), + }; + if (i < 3) chars.reminder = 7000; + samples.push(synth(chars, truth)); + } + const fit = fitCpt(samples); + expect(fit.cpt.static_slab).toBeDefined(); + expect(fit.cpt.reminder).toBeUndefined(); + expect(fit.rejected.reminder).toContain('present in 3/40'); + }); + + it('rejects implausible CPTs rather than handing them to the gate', () => { + // Truth far outside the plausible band (20 chars/token) must not be adopted. + const truth: Partial> = { static_slab: 20 }; + const r = rng(3); + const samples = Array.from({ length: 40 }, () => + synth({ static_slab: Math.floor(20000 + r() * 20000) }, truth), + ); + const fit = fitCpt(samples); + expect(fit.cpt.static_slab).toBeUndefined(); + expect(fit.rejected.static_slab).toContain('outside'); + expect(CPT_PLAUSIBLE_MAX).toBeLessThan(20); + }); + + it('rejects perfectly collinear buckets instead of inventing slopes', () => { + // history is always exactly 2× slab → the two columns are not separable. + const r = rng(5); + const samples: CptSample[] = []; + for (let i = 0; i < 40; i++) { + const slab = Math.floor(20000 + r() * 20000); + samples.push({ + bucketChars: { static_slab: slab, history: slab * 2 }, + textTokens: slab / 1.5 + (slab * 2) / 1.4, + }); + } + const fit = fitCpt(samples); + // Either singular or ill-conditioned — both must fail closed, never emit a value. + expect(fit.cpt.static_slab).toBeUndefined(); + expect(fit.cpt.history).toBeUndefined(); + expect(fit.rejected.static_slab).toBeTruthy(); + }); + + it('never throws on degenerate input', () => { + expect(() => fitCpt([])).not.toThrow(); + expect(fitCpt([]).cpt).toEqual({}); + const junk: CptSample[] = Array.from({ length: 30 }, () => ({ + bucketChars: { static_slab: 0 }, + textTokens: Number.NaN, + })); + expect(() => fitCpt(junk)).not.toThrow(); + expect(fitCpt(junk).cpt).toEqual({}); + }); +}); diff --git a/tests/cpt-store.test.ts b/tests/cpt-store.test.ts new file mode 100644 index 00000000..85f15db3 --- /dev/null +++ b/tests/cpt-store.test.ts @@ -0,0 +1,192 @@ +/** + * Adaptive CPT — the store (Node path). + * + * End-to-end for the learning loop: a real events.jsonl on disk → samples → + * fit → the resolver the gate consults. Uses synthetic rows whose token counts + * are generated from CPTs we choose, so "did it learn the right number?" has a + * ground truth. + */ + +import { describe, expect, it, afterAll } from 'vitest'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { + buildCptState, + resolverFor, + sampleFromEvent, + writeCptState, + GLOBAL_KEY, +} from '../src/cpt-store.js'; +import { ANTHROPIC_PATCH_PX } from '../src/core/anthropic-vision.js'; +const PIXELS_PER_VISUAL_TOKEN = ANTHROPIC_PATCH_PX * ANTHROPIC_PATCH_PX; + +const tmpDirs: string[] = []; +function tmpdir(): string { + const d = fs.mkdtempSync(path.join(os.tmpdir(), 'pxpipe-cpt-')); + tmpDirs.push(d); + return d; +} +afterAll(() => { + for (const d of tmpDirs) fs.rmSync(d, { recursive: true, force: true }); +}); + +function rng(seed: number): () => number { + let s = seed >>> 0; + return () => { + s = (s * 1664525 + 1013904223) >>> 0; + return s / 0x100000000; + }; +} + +/** + * Write an events.jsonl whose rows encode the given per-bucket CPTs exactly: + * baseline_tokens = Σ chars_b/cpt_b + image_pixels/750. + */ +function writeEvents( + file: string, + rows: Array<{ sha: string; chars: Record; cpt: Record }>, +): void { + const r = rng(99); + const lines = rows.map(({ sha, chars, cpt }) => { + let textTokens = 0; + for (const [b, c] of Object.entries(chars)) textTokens += c / cpt[b]!; + const image_pixels = Math.floor(r() * 500_000); + return JSON.stringify({ + ts: new Date().toISOString(), + system_sha8: sha, + bucket_chars: chars, + image_pixels, + baseline_tokens: textTokens + image_pixels / PIXELS_PER_VISUAL_TOKEN, + }); + }); + fs.writeFileSync(file, lines.join('\n') + '\n', 'utf8'); +} + +describe('sampleFromEvent', () => { + it('subtracts the image cost on the 28px patch grid to isolate text tokens', () => { + // One full pxpipe page: 1568×728. Both edges are whole multiples of 28, so + // the patch count is exact — 56×26 = 1456 visual tokens — and the aggregate + // pixels/784 form agrees with it exactly. + const pagePixels = 1568 * 728; + expect(pagePixels / (ANTHROPIC_PATCH_PX * ANTHROPIC_PATCH_PX)).toBe(1456); + expect(Math.ceil(1568 / ANTHROPIC_PATCH_PX) * Math.ceil(728 / ANTHROPIC_PATCH_PX)).toBe(1456); + + const s = sampleFromEvent({ + bucket_chars: { static_slab: 30_000 }, + image_pixels: pagePixels, + baseline_tokens: 20_000 + 1456, + }); + expect(s).not.toBeNull(); + expect(s!.textTokens).toBeCloseTo(20_000, 6); + }); + + it('rejects rows missing the fields the fit needs', () => { + expect(sampleFromEvent({})).toBeNull(); + expect(sampleFromEvent({ bucket_chars: { static_slab: 10 } })).toBeNull(); + expect(sampleFromEvent({ baseline_tokens: 100 })).toBeNull(); + // zero chars → no regressor + expect(sampleFromEvent({ bucket_chars: { static_slab: 0 }, baseline_tokens: 100 })).toBeNull(); + // image cost exceeds the whole bill → broken row, not a negative text cost + expect( + sampleFromEvent({ + bucket_chars: { static_slab: 1000 }, + image_pixels: 7_500_000, + baseline_tokens: 10, + }), + ).toBeNull(); + }); +}); + +describe('buildCptState', () => { + it('recovers the CPTs used to generate a real events file', async () => { + const dir = tmpdir(); + const file = path.join(dir, 'events.jsonl'); + const cpt = { static_slab: 1.5, tool_result_json: 1.3 }; + const r = rng(5); + writeEvents( + file, + Array.from({ length: 50 }, () => ({ + sha: 'aaaa1111', + chars: { + static_slab: Math.floor(20_000 + r() * 30_000), + tool_result_json: Math.floor(5_000 + r() * 25_000), + }, + cpt, + })), + ); + + const state = await buildCptState(file); + const fit = state.fits.get('aaaa1111'); + expect(fit, 'per-project fit missing').toBeDefined(); + expect(fit!.cpt.static_slab!).toBeCloseTo(1.5, 1); + expect(fit!.cpt.tool_result_json!).toBeCloseTo(1.3, 1); + // The pooled table is fitted too, so a fresh project has a prior. + expect(state.fits.get(GLOBAL_KEY)).toBeDefined(); + }); + + it('returns an empty state for a missing log instead of throwing', async () => { + const state = await buildCptState(path.join(tmpdir(), 'nope.jsonl')); + expect(state.fits.size).toBe(0); + expect(resolverFor(state)('static_slab')).toBeUndefined(); + }); + + it('prefers the project rate over the pooled one, and pools for unknown projects', async () => { + const dir = tmpdir(); + const file = path.join(dir, 'events.jsonl'); + const r = rng(17); + // Project A is dense (1.2); project B is loose (3.5). Both are well-sampled. + const rows = [ + ...Array.from({ length: 40 }, () => ({ + sha: 'aaaaaaaa', + chars: { static_slab: Math.floor(20_000 + r() * 20_000) }, + cpt: { static_slab: 1.2 }, + })), + ...Array.from({ length: 40 }, () => ({ + sha: 'bbbbbbbb', + chars: { static_slab: Math.floor(20_000 + r() * 20_000) }, + cpt: { static_slab: 3.5 }, + })), + ]; + writeEvents(file, rows); + + const state = await buildCptState(file); + const resolve = resolverFor(state); + + expect(resolve('static_slab', 'aaaaaaaa')!).toBeCloseTo(1.2, 1); + expect(resolve('static_slab', 'bbbbbbbb')!).toBeCloseTo(3.5, 1); + // An unseen project falls back to the pooled table — a real number, and + // strictly better informed than a hand-picked constant. + const pooled = resolve('static_slab', 'ffffffff'); + expect(pooled).toBeDefined(); + expect(pooled!).toBeGreaterThan(1.0); + expect(pooled!).toBeLessThan(6.0); + }); + + it('persists a readable state file', async () => { + const dir = tmpdir(); + const file = path.join(dir, 'events.jsonl'); + const r = rng(23); + writeEvents( + file, + Array.from({ length: 40 }, () => ({ + sha: 'cccccccc', + chars: { static_slab: Math.floor(20_000 + r() * 20_000) }, + cpt: { static_slab: 1.5 }, + })), + ); + const state = await buildCptState(file); + const out = path.join(dir, 'cpt-state.jsonl'); + writeCptState(state, out); + + const parsed = fs + .readFileSync(out, 'utf8') + .trim() + .split('\n') + .map((l) => JSON.parse(l) as { system_sha8: string; cpt: Record; n_events: number }); + const proj = parsed.find((p) => p.system_sha8 === 'cccccccc'); + expect(proj).toBeDefined(); + expect(proj!.n_events).toBe(40); + expect(proj!.cpt.static_slab!).toBeCloseTo(1.5, 1); + }); +});