diff --git a/README.md b/README.md index b30bfd2..210fff1 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,6 @@ core.status(compressed, tokenCount, config); // context-usage report |--------|---------| | `truncateLargeToolOutputs` | Emergency context-threshold-gated truncation of large visible tool outputs (last-resort safety valve; summaries are never touched) | | `hideConsumedCompressCalls` | Hide historical compress tool-calls whose block is inactive | -| `resolveKeepMarkers` | Expand `[[KEEP:mNNNNN]]` / rewrite `[[REF:mNNNNN\|desc]]` | | `buildStatusReport` / `buildRecap` | Context-usage report + block recap | | `mergeMarkedBlocks` / `collectOldGenBlocks` | Batch merge old-gen blocks into one summary | | `rebuildCompressionState` | Fork-recovery: replay historical compress calls | diff --git a/src/compression-rules.ts b/src/compression-rules.ts index 4bc749b..767e6b0 100644 --- a/src/compression-rules.ts +++ b/src/compression-rules.ts @@ -9,8 +9,7 @@ export const COMPRESS_PHILOSOPHY = `Compression Philosophy: - All compression serves the primary task, but be frugal. - Context capacity is precious. Save context by compressing consumed outputs, not by avoiding tools. - Compress by need, not by percentage. -- Work from summaries, not raw tool outputs. All listed ranges (user prompts, tool outputs, code, logs, exploration, intermediate steps) should be compressed to summary format — the ONLY exceptions are protected content, content the current step is actively using, or critical content you cannot reconstruct. -- Curate summaries like a well-structured document. User prompts, compressed tool outputs, code, logs, or skill-call intermediate results that are critically important should be preserved — not by exempting them from compression, but by embedding them in the summary via [[KEEP:mNNNNN]] (auto-expanded verbatim) and [[REF:mNNNNN|description]] (compact link).`; +- Work from summaries, not raw tool outputs. All listed ranges (user prompts, tool outputs, code, logs, exploration, intermediate steps) should be compressed to summary format — the ONLY exceptions are protected content, content the current step is actively using, or critical content you cannot reconstruct.`; export const HOW_TO_COMPRESS_RULES = `HOW TO COMPRESS @@ -40,8 +39,6 @@ DROP — extract the signal, discard the vessel: For each significant item you DROP (scripts, reports, large analyses, long tool outputs), add a one-line CONTENT description of what it covers — not where it lives. Bad: "probe script at /path/probe_kvnet.py". Good: "probe_kvnet.py: tests n-gram baseline, generation quality, long-range dependency, position sensitivity, op pipeline, QUERY attention." This lets a later decompress target the right block by relevance, not by guessing locations. -KEEP MARKERS: \`[[KEEP:mNNNNN]]\` expands original message content into the summary (truncated to a max length). Do NOT use KEEP for verbose command output, diagnostic scripts, log dumps, or any content whose value is in the conclusion rather than the raw output — summarize these or use \`[[REF:mNNNNN|desc]\` instead. - PRIORITY — when the summary must be compact, preserve in this order: 1. User's overall goal, goal evolution, intent, and hard constraints (losing these changes the task). 2. Decisions and rationale. diff --git a/src/index.ts b/src/index.ts index 1f15fc4..4fa7a6e 100644 --- a/src/index.ts +++ b/src/index.ts @@ -37,8 +37,6 @@ export type { NudgeVoice, RenderedNudge } from "./nudge-text.js"; export { COMPRESS_PHILOSOPHY, HOW_TO_COMPRESS_RULES, TIER2_DISTILL_RULES, TIER3_CONDENSE_RULES } from "./compression-rules.js"; export { truncateLargeToolOutputs } from "./truncate-tools.js"; export type { TruncateOptions, TruncateResult } from "./truncate-tools.js"; -export { resolveKeepMarkers } from "./keep-markers.js"; -export type { KeepMarkerResult } from "./keep-markers.js"; export { parseBlockIdArg, findBlocksOverlappingMessages, diff --git a/src/keep-markers.ts b/src/keep-markers.ts deleted file mode 100644 index 2f25f81..0000000 --- a/src/keep-markers.ts +++ /dev/null @@ -1,77 +0,0 @@ -import { refForRaw } from "./refs.js"; -import type { CompressionState, CoreMessage } from "./types.js"; - -const KEEP_REGEX = /\[\[KEEP:(m\d+)\]\]/g; -const REF_REGEX = /\[\[REF:(m\d+)\|([^\]]+)\]\]/g; - -export interface KeepMarkerResult { - summary: string; - expandedCount: number; - refCount: number; - unresolvedRefs: string[]; -} - -export function resolveKeepMarkers( - summary: string, - messages: CoreMessage[], - state: CompressionState, - maxChars = 2000, -): KeepMarkerResult { - const messageByRef = new Map(); - for (const message of messages) { - const ref = refForRaw(state.messageRefs, message.id); - if (ref) messageByRef.set(ref, message); - } - - let expandedCount = 0; - let refCount = 0; - const unresolvedRefs: string[] = []; - - const expanded = summary - .replace(KEEP_REGEX, (match, ref: string) => { - const normalized = normalizeRef(ref); - const message = normalized ? messageByRef.get(normalized) : undefined; - if (!message) { - unresolvedRefs.push(ref); - return match; - } - expandedCount++; - return formatKeptMessage(message, normalized!, maxChars); - }) - .replace(REF_REGEX, (_match, ref: string, desc: string) => { - const normalized = normalizeRef(ref); - const message = normalized ? messageByRef.get(normalized) : undefined; - if (!message) { - unresolvedRefs.push(ref); - return _match; - } - refCount++; - return `[→ ${normalized}: ${desc.trim()}]`; - }); - - return { summary: expanded, expandedCount, refCount, unresolvedRefs }; -} - -function normalizeRef(ref: string): string | null { - const match = /^m0*(\d{1,5})$/.exec(ref.trim().toLowerCase()); - if (!match || match[1] === undefined) return null; - return `m${match[1].padStart(5, "0")}`; -} - -function formatKeptMessage(message: CoreMessage, ref: string, maxChars: number): string { - const label = labelFor(message); - const body = truncate(message.text ?? "[empty message]", maxChars); - return `\n--- [${ref}: ${label}] ---\n${body}\n--- end ---\n`; -} - -function labelFor(message: CoreMessage): string { - if (message.contentType === "tool-call" || message.contentType === "tool-result") { - return message.toolName ?? "tool"; - } - return message.role; -} - -function truncate(text: string, maxChars: number): string { - if (text.length <= maxChars) return text; - return text.slice(0, maxChars) + `\n... [truncated, ${text.length} chars total]`; -} diff --git a/tests/truncate-keep-filter.test.ts b/tests/truncate-keep-filter.test.ts index e956923..16964c6 100644 --- a/tests/truncate-keep-filter.test.ts +++ b/tests/truncate-keep-filter.test.ts @@ -1,14 +1,11 @@ import { test } from "node:test"; import assert from "node:assert/strict"; import { truncateLargeToolOutputs } from "../src/truncate-tools.js"; -import { resolveKeepMarkers } from "../src/keep-markers.js"; import { applyMessageFilters, clearMessageFilters, registerMessageFilter, } from "../src/filter/index.js"; -import { createInitialState } from "../src/state.js"; -import { assignRefs } from "../src/refs.js"; import { defaultConfig } from "../src/config.js"; import type { CoreMessage } from "../src/types.js"; @@ -57,34 +54,6 @@ test("truncateLargeToolOutputs protects recent messages", () => { assert.equal(result.truncatedCount, 0); }); -test("resolveKeepMarkers expands [[KEEP:mNNNNN]] with message content", () => { - const state = createInitialState(); - const messages = [msg("a", "the kept content"), msg("b", "other")]; - state.messageRefs = assignRefs(messages, { existing: state.messageRefs, nextIndex: 1 }).map; - - const result = resolveKeepMarkers("see [[KEEP:m00001]] here", messages, state); - assert.equal(result.expandedCount, 1); - assert.ok(result.summary.includes("the kept content")); - assert.deepEqual(result.unresolvedRefs, []); -}); - -test("resolveKeepMarkers rewrites [[REF:mNNNNN|desc]] into a pointer", () => { - const state = createInitialState(); - const messages = [msg("a", "content")]; - state.messageRefs = assignRefs(messages, { existing: state.messageRefs, nextIndex: 1 }).map; - - const result = resolveKeepMarkers("ref [[REF:m00001|the finding]] end", messages, state); - assert.equal(result.refCount, 1); - assert.ok(result.summary.includes("[→ m00001: the finding]")); -}); - -test("resolveKeepMarkers reports unresolved refs", () => { - const state = createInitialState(); - const result = resolveKeepMarkers("[[KEEP:m00099]]", [], state); - assert.equal(result.expandedCount, 0); - assert.deepEqual(result.unresolvedRefs, ["m00099"]); -}); - test("applyMessageFilters is a no-op when disabled", () => { clearMessageFilters(); const messages = [msg("a", "hello")];