feat: content-specific extract-code + marker_mode + live examples

components/all/markermode_test.go

@@ -0,0 +1,113 @@
+package all_test
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/kagenti/context-guru/apply"
+	"github.com/kagenti/context-guru/config"
+	"github.com/kagenti/context-guru/expand"
+	"github.com/kagenti/context-guru/schema"
+	"github.com/kagenti/context-guru/store"
+	"github.com/maximhq/bifrost/core/schemas"
+	"github.com/tidwall/gjson"
+)
+
+// bigTool builds a many-line tool output collapse will reduce (well over its
+// head+tail window and token floor).
+func bigTool() *schemas.BifrostChatRequest {
+	var b strings.Builder
+	for i := 0; i < 120; i++ {
+		fmt.Fprintf(&b, "line %d: some tool output content to pad this out\n", i)
+	}
+	return &schemas.BifrostChatRequest{Input: []schemas.ChatMessage{toolMsg(b.String())}}
+}
+
+// TestMarkerModeSummarySurvivesWire drives the summary sentinel through the real
+// wire path (apply.Body → JSON splice, Anthropic tool_result shape) to confirm the
+// non-ASCII ⟪cg⟫ survives JSON round-trip and HasPlaceholder still matches it — the
+// cross-turn skip-detection depends on this (HANDOVER review item 5).
+func TestMarkerModeSummarySurvivesWire(t *testing.T) {
+	cfg, err := config.LoadBytes([]byte("pipeline: [collapse]\ncomponents:\n  collapse: {max_tokens: 10, head_lines: 2, tail_lines: 2, marker_mode: summary}\n"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	p, _ := cfg.Build(nil)
+	big := strings.Repeat("a fairly long line of tool output content to reduce\n", 60)
+	body, _ := json.Marshal(map[string]any{
+		"model": "claude-sonnet-4-6",
+		"messages": []any{
+			map[string]any{"role": "user", "content": "go"},
+			map[string]any{"role": "user", "content": []any{
+				map[string]any{"type": "tool_result", "tool_use_id": "t1", "content": big},
+			}},
+		},
+	})
+	out, changed := apply.Body(context.Background(), p, store.NewMemory(store.Options{}), schemas.Anthropic, body, "", false)
+	if !changed {
+		t.Fatal("summary-mode collapse should have reduced the tool_result")
+	}
+	got := gjson.GetBytes(out, "messages.1.content.0.content").String()
+	if !strings.Contains(got, expand.SummaryMarker) || !expand.HasPlaceholder(got) {
+		t.Fatalf("⟪cg⟫ sentinel did not survive the wire round-trip: %q", got)
+	}
+	if len(expand.ParseMarkers(got)) != 0 {
+		t.Fatalf("summary mode must not leave a resolvable marker: %q", got)
+	}
+}
+
+// TestMarkerModeFull is the default: content is dropped, a resolvable <<cg:HASH>>
+// marker is left, and the original is recoverable from the Store.
+func TestMarkerModeFull(t *testing.T) {
+	req := bigTool()
+	before := schema.MessagesTokens(req)
+	_, st := run(t, "pipeline: [collapse]\ncomponents:\n  collapse: {max_tokens: 10, head_lines: 2, tail_lines: 2}\n", req)
+	got := schema.MessageText(req.Input[0])
+	if schema.MessagesTokens(req) >= before {
+		t.Fatalf("full: expected reduction, before=%d after=%d", before, schema.MessagesTokens(req))
+	}
+	keys := expand.ParseMarkers(got)
+	if len(keys) != 1 {
+		t.Fatalf("full: expected one resolvable marker, got %q", got)
+	}
+	if orig, ok := expand.Resolve(st, keys[0]); !ok || !strings.Contains(orig, "line 50:") {
+		t.Fatal("full: original must be recoverable from the Store")
+	}
+}
+
+// TestMarkerModeSummary: content is dropped, a non-resolvable ⟪cg⟫ sentinel is
+// left (recognized by HasPlaceholder for cross-turn skip), and nothing is stashed
+// — the reduction must survive the pipeline's dropped-without-stash guard.
+func TestMarkerModeSummary(t *testing.T) {
+	req := bigTool()
+	before := schema.MessagesTokens(req)
+	run(t, "pipeline: [collapse]\ncomponents:\n  collapse: {max_tokens: 10, head_lines: 2, tail_lines: 2, marker_mode: summary}\n", req)
+	got := schema.MessageText(req.Input[0])
+	if schema.MessagesTokens(req) >= before {
+		t.Fatalf("summary: expected reduction (guard must not revert), before=%d after=%d", before, schema.MessagesTokens(req))
+	}
+	if len(expand.ParseMarkers(got)) != 0 {
+		t.Fatalf("summary: must NOT leave a resolvable <<cg:HASH>> marker, got %q", got)
+	}
+	if !strings.Contains(got, expand.SummaryMarker) || !expand.HasPlaceholder(got) {
+		t.Fatalf("summary: expected a ⟪cg⟫ sentinel, got %q", got)
+	}
+}
+
+// TestMarkerModeOff: content is dropped with no marker at all; the reduction must
+// still survive the guard.
+func TestMarkerModeOff(t *testing.T) {
+	req := bigTool()
+	before := schema.MessagesTokens(req)
+	run(t, "pipeline: [collapse]\ncomponents:\n  collapse: {max_tokens: 10, head_lines: 2, tail_lines: 2, marker_mode: off}\n", req)
+	got := schema.MessageText(req.Input[0])
+	if schema.MessagesTokens(req) >= before {
+		t.Fatalf("off: expected reduction (guard must not revert), before=%d after=%d", before, schema.MessagesTokens(req))
+	}
+	if expand.HasPlaceholder(got) {
+		t.Fatalf("off: expected no marker or sentinel, got %q", got)
+	}
+}

components/all/more_test.go

@@ -54,30 +54,7 @@ func TestFormatCompactsJSON(t *testing.T) {
 	}
 }
 
-func TestSkeletonElidesBodies(t *testing.T) {
-	code := "```go\n" +
-		"package main\n\n" +
-		"func Add(a, b int) int {\n" +
-		"\tsum := 0\n\tsum += a\n\tsum += b\n\tfor i := 0; i < 10; i++ { sum += i }\n\treturn sum\n}\n\n" +
-		"func Mul(a, b int) int {\n\tp := 0\n\tfor i := 0; i < b; i++ { p += a }\n\treturn p\n}\n" +
-		"```"
-	req := &schemas.BifrostChatRequest{Input: []schemas.ChatMessage{toolMsg(code)}}
-	_, st := run(t, "pipeline: [skeleton]\ncomponents:\n  skeleton: {min_tokens: 5}\n", req)
-	got := schema.MessageText(req.Input[0])
-	if !strings.Contains(got, "func Add(a, b int) int") || !strings.Contains(got, "{ … }") {
-		t.Fatalf("skeleton should keep signatures and elide bodies: %q", got)
-	}
-	if strings.Contains(got, "sum += a") {
-		t.Fatalf("skeleton should have removed body statements: %q", got)
-	}
-	keys := expand.ParseMarkers(got)
-	if len(keys) != 1 {
-		t.Fatalf("expected expand marker, got %q", got)
-	}
-	if orig, ok := expand.Resolve(st, keys[0]); !ok || !strings.Contains(orig, "sum += a") {
-		t.Fatal("expand did not recover original source")
-	}
-}
+// TestSkeletonElidesBodies moved to skeleton_test.go (cg_skeleton build tag).
 
 func TestFailedRunSupersedes(t *testing.T) {
 	run1 := "=== test session starts ===\n" + strings.Repeat("detail line about the failing run\n", 20) + "3 failed, 2 passed in 1.2s\n"

components/all/p4_test.go

@@ -128,17 +128,7 @@ func TestExtractProjectsRelevantLines(t *testing.T) {
 	}
 }
 
-// TestSkeletonSkipsNonToolMessages locks the role-scope fix: skeleton must
-// leave a user/assistant message's own code untouched (only tool outputs are
-// offloaded), otherwise it would mangle the caller's code with no live recovery.
-func TestSkeletonSkipsNonToolMessages(t *testing.T) {
-	code := "```go\nfunc Add(a, b int) int {\n\tsum := 0\n\tsum += a\n\tsum += b\n\tfor i := 0; i < 10; i++ { sum += i }\n\treturn sum\n}\n```"
-	req := &bschemas.BifrostChatRequest{Input: []bschemas.ChatMessage{userMsg(code)}}
-	run(t, "pipeline: [skeleton]\ncomponents:\n  skeleton: {min_tokens: 5}\n", req)
-	if got := schema.MessageText(req.Input[0]); got != code {
-		t.Fatalf("skeleton must not rewrite a non-tool message; got %q", got)
-	}
-}
+// TestSkeletonSkipsNonToolMessages moved to skeleton_test.go (cg_skeleton tag).
 
 func min(a, b int) int {
 	if a < b {

components/all/skeleton_test.go

@@ -0,0 +1,54 @@
+//go:build cg_skeleton
+
+// Skeleton tests live here (behind the cg_skeleton tag) because the skeleton
+// component is only registered when built with that tag. The shared helpers
+// (run, toolMsg, userMsg) come from the untagged test files in this package,
+// which also compile under the tag.
+
+package all_test
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/kagenti/context-guru/expand"
+	"github.com/kagenti/context-guru/schema"
+	"github.com/maximhq/bifrost/core/schemas"
+)
+
+func TestSkeletonElidesBodies(t *testing.T) {
+	code := "```go\n" +
+		"package main\n\n" +
+		"func Add(a, b int) int {\n" +
+		"\tsum := 0\n\tsum += a\n\tsum += b\n\tfor i := 0; i < 10; i++ { sum += i }\n\treturn sum\n}\n\n" +
+		"func Mul(a, b int) int {\n\tp := 0\n\tfor i := 0; i < b; i++ { p += a }\n\treturn p\n}\n" +
+		"```"
+	req := &schemas.BifrostChatRequest{Input: []schemas.ChatMessage{toolMsg(code)}}
+	_, st := run(t, "pipeline: [skeleton]\ncomponents:\n  skeleton: {min_tokens: 5}\n", req)
+	got := schema.MessageText(req.Input[0])
+	if !strings.Contains(got, "func Add(a, b int) int") || !strings.Contains(got, "{ … }") {
+		t.Fatalf("skeleton should keep signatures and elide bodies: %q", got)
+	}
+	if strings.Contains(got, "sum += a") {
+		t.Fatalf("skeleton should have removed body statements: %q", got)
+	}
+	keys := expand.ParseMarkers(got)
+	if len(keys) != 1 {
+		t.Fatalf("expected expand marker, got %q", got)
+	}
+	if orig, ok := expand.Resolve(st, keys[0]); !ok || !strings.Contains(orig, "sum += a") {
+		t.Fatal("expand did not recover original source")
+	}
+}
+
+// TestSkeletonSkipsNonToolMessages locks the role-scope fix: skeleton must
+// leave a user/assistant message's own code untouched (only tool outputs are
+// offloaded), otherwise it would mangle the caller's code with no live recovery.
+func TestSkeletonSkipsNonToolMessages(t *testing.T) {
+	code := "```go\nfunc Add(a, b int) int {\n\tsum := 0\n\tsum += a\n\tsum += b\n\tfor i := 0; i < 10; i++ { sum += i }\n\treturn sum\n}\n```"
+	req := &schemas.BifrostChatRequest{Input: []schemas.ChatMessage{userMsg(code)}}
+	run(t, "pipeline: [skeleton]\ncomponents:\n  skeleton: {min_tokens: 5}\n", req)
+	if got := schema.MessageText(req.Input[0]); got != code {
+		t.Fatalf("skeleton must not rewrite a non-tool message; got %q", got)
+	}
+}

components/component.go

@@ -43,9 +43,10 @@ type Reformat interface {
 // returns the cache_keys under which it stashed the originals (via c.Store) —
 // one per offloaded item. If it shrinks the request but returns no keys, the
 // pipeline treats it as a failed offload and reverts (you cannot silently lose
-// data). Returning no keys AND leaving the request unchanged is a legitimate
-// no-op (set rep.Skipped). Examples: collapse, dedup, cmdfilter, extract,
-// smartcrush.
+// data) — UNLESS it set rep.Irreversible, the deliberate lossy drop a non-`full`
+// marker_mode makes (summary/off: no stash, no restoration). Returning no keys
+// AND leaving the request unchanged is a legitimate no-op (set rep.Skipped).
+// Examples: collapse, dedup, cmdfilter, extract, smartcrush.
 type Offload interface {
 	Component
 	Offload(req *schemas.BifrostChatRequest, rep *Report, c *Ctx) (cacheKeys []string, err error)
@@ -119,6 +120,7 @@ type Report struct {
 	CacheKeys    []string // set by Offload components (one per stashed original)
 	Skipped      bool     // component ran but chose not to act
 	Reverted     bool     // pipeline reverted it (error/panic/never-worse)
+	Irreversible bool     // Offload dropped content on purpose without stashing (marker_mode summary/off)
 	Err          error
 }
 

components/offload/cmdfilter.go

@@ -22,11 +22,15 @@ func init() { components.Register("cmdfilter", newCmdfilter) }
 // Offload: it stashes the original before filtering so the expand tool can
 // recover it. Filters match on the tool output's first non-empty line (the
 // proxy-world stand-in for rtk's shell command).
-type Cmdfilter struct{ reg *dsl.Registry }
+type Cmdfilter struct {
+	reg  *dsl.Registry
+	mode markerMode
+}
 
 type cmdfilterConfig struct {
 	Filters         []string `yaml:"filters"`          // inline filter YAML documents
 	DisableBuiltins bool     `yaml:"disable_builtins"` // skip the bundled starter filters
+	MarkerMode      string   `yaml:"marker_mode"`      // full (default) | summary | off
 }
 
 func newCmdfilter(raw []byte) (components.Component, error) {
@@ -47,7 +51,7 @@ func newCmdfilter(raw []byte) (components.Component, error) {
 			return nil, err
 		}
 	}
-	return &Cmdfilter{reg: reg}, nil
+	return &Cmdfilter{reg: reg, mode: parseMarkerMode(cfg.MarkerMode)}, nil
 }
 
 func (Cmdfilter) Name() string { return "cmdfilter" }
@@ -56,6 +60,7 @@ func (f *Cmdfilter) Enabled(*components.Ctx) bool { return f.reg.Len() > 0 }
 
 func (f *Cmdfilter) Offload(req *schemas.BifrostChatRequest, rep *components.Report, c *components.Ctx) ([]string, error) {
 	var keys []string
+	changed := 0
 	for i := range req.Input {
 		m := &req.Input[i]
 		if m.Role != schemas.ChatMessageRoleTool {
@@ -76,19 +81,36 @@ func (f *Cmdfilter) Offload(req *schemas.BifrostChatRequest, rep *components.Rep
 		if out == content {
 			continue
 		}
-		// Compare the FULL rewritten text (marker + recovery hint included) against
-		// the original — the marker costs tokens too, so filtering that barely wins
-		// can still make the message larger (rtk never_worse, at the message level).
+		// Build the token that goes where the restoration marker would (per
+		// marker_mode) WITHOUT stashing yet, so the never-worse check below can
+		// still bail. Compare the FULL rewritten text (token included) against the
+		// original — the marker costs tokens too, so filtering that barely wins can
+		// still make the message larger (rtk never_worse, at the message level).
 		key := hashKey(content)
-		newText := out + "\n" + expand.Marker(key) + recoveryHint(loss)
+		var token string
+		switch f.mode {
+		case markerFull:
+			token = expand.Marker(key) + recoveryHint(loss)
+		case markerSummary:
+			token = expand.SummaryMarker
+		} // off: no token
+		newText := out
+		if token != "" {
+			newText += "\n" + token
+		}
 		if schema.TextTokens(newText) >= schema.TextTokens(content) {
 			continue
 		}
-		c.Store.Put(key, []byte(content))
+		if f.mode == markerFull {
+			c.Store.Put(key, []byte(content))
+			keys = append(keys, key)
+		} else {
+			rep.Irreversible = true
+		}
 		schema.SetMessageText(m, newText)
-		keys = append(keys, key)
+		changed++
 	}
-	if len(keys) == 0 {
+	if changed == 0 {
 		rep.Skipped = true
 	}
 	return keys, nil

components/offload/collapse.go

@@ -22,12 +22,14 @@ type Collapse struct {
 	maxTokens int
 	headLines int
 	tailLines int
+	mode      markerMode
 }
 
 type collapseConfig struct {
-	MaxTokens int `yaml:"max_tokens"`
-	HeadLines int `yaml:"head_lines"`
-	TailLines int `yaml:"tail_lines"`
+	MaxTokens  int    `yaml:"max_tokens"`
+	HeadLines  int    `yaml:"head_lines"`
+	TailLines  int    `yaml:"tail_lines"`
+	MarkerMode string `yaml:"marker_mode"` // full (default) | summary | off
 }
 
 func newCollapse(raw []byte) (components.Component, error) {
@@ -37,14 +39,15 @@ func newCollapse(raw []byte) (components.Component, error) {
 			return nil, err
 		}
 	}
-	return &Collapse{maxTokens: cfg.MaxTokens, headLines: cfg.HeadLines, tailLines: cfg.TailLines}, nil
+	return &Collapse{maxTokens: cfg.MaxTokens, headLines: cfg.HeadLines, tailLines: cfg.TailLines, mode: parseMarkerMode(cfg.MarkerMode)}, nil
 }
 
 func (Collapse) Name() string                 { return "collapse" }
 func (Collapse) Enabled(*components.Ctx) bool { return true }
 
 func (cl *Collapse) Offload(req *schemas.BifrostChatRequest, rep *components.Report, c *components.Ctx) ([]string, error) {
 	var keys []string
+	changed := 0
 	for i := range req.Input {
 		m := &req.Input[i]
 		if m.Role != schemas.ChatMessageRoleTool {
@@ -57,25 +60,27 @@ func (cl *Collapse) Offload(req *schemas.BifrostChatRequest, rep *components.Rep
 		if content == "" || schema.TextTokens(content) <= cl.maxTokens {
 			continue
 		}
-		if len(expand.ParseMarkers(content)) > 0 {
+		if expand.HasPlaceholder(content) {
 			continue // already offloaded by an earlier component
 		}
 		lines := strings.Split(content, "\n")
 		if len(lines) <= cl.headLines+cl.tailLines {
 			continue // few long lines; head/tail wouldn't help
 		}
 		omitted := len(lines) - cl.headLines - cl.tailLines
+		tok, key := mark(c, rep, cl.mode, content, " [full output: call "+expand.ToolName+"]")
 		var b strings.Builder
 		b.WriteString(strings.Join(lines[:cl.headLines], "\n"))
 		fmt.Fprintf(&b, "\n... (%d lines omitted) ", omitted)
-		key := hashKey(content)
-		b.WriteString(expand.Marker(key) + " [full output: call " + expand.ToolName + "]\n")
+		b.WriteString(tok + "\n")
 		b.WriteString(strings.Join(lines[len(lines)-cl.tailLines:], "\n"))
-		c.Store.Put(key, []byte(content))
 		schema.SetMessageText(m, b.String())
-		keys = append(keys, key)
+		changed++
+		if key != "" {
+			keys = append(keys, key)
+		}
 	}
-	if len(keys) == 0 {
+	if changed == 0 {
 		rep.Skipped = true
 	}
 	return keys, nil