From d4a3262fe5d51ad6a04f1332203631300cb2e39f Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Sun, 9 Aug 2026 20:31:00 +0200 Subject: [PATCH 01/21] test: back the write-path search assertions with a real sqlite FTS fixture The tests that pin what indexing makes searchable ran against an in-memory graph and an in-process text index the daemon never builds, so they proved nothing about the corpus production serves. They now index into a sqlite store and read its native symbol FTS through the SymbolSearcherBackend, and each fixture refuses to run on an empty corpus. Queries that would be answered by the exact-name short-circuit were reshaped so the ranked tier decides the outcome. --- internal/indexer/chunk_index_test.go | 29 +++++-- internal/indexer/incremental_resolve_test.go | 2 +- internal/indexer/indexer_test.go | 49 ++++++++++++ internal/indexer/multi_test.go | 11 ++- internal/indexer/realtime_reliability_test.go | 39 ++++++--- internal/indexer/reresolve_guard_test.go | 4 +- internal/indexer/skip_search_test.go | 80 ++++++++++++------- internal/indexer/skip_telemetry_test.go | 2 +- internal/mcp/scope_resolve_test.go | 21 +++-- internal/mcp/tools_search_corpus_test.go | 46 ++++++++--- internal/mcp/tools_search_docchannel_test.go | 14 +--- 11 files changed, 210 insertions(+), 87 deletions(-) diff --git a/internal/indexer/chunk_index_test.go b/internal/indexer/chunk_index_test.go index 246ca092c..3b27d4555 100644 --- a/internal/indexer/chunk_index_test.go +++ b/internal/indexer/chunk_index_test.go @@ -154,16 +154,31 @@ func TestBuildSearchIndex_ChunkedSymbolNotDuplicatedInSearch(t *testing.T) { b.WriteString("}\n\nfunc checkField() {}\n") require.NoError(t, os.WriteFile(filepath.Join(dir, "v.go"), []byte(b.String()), 0o644)) - g := graph.New() reg := parser.NewRegistry() reg.Register(languages.NewGoExtractor()) cfg := config.Default().Index cfg.Workers = 1 - idx := New(g, reg, cfg, zap.NewNop()) - idx.SetEmbedder(stubEmbedder{}) - idx.SetEmbeddingChunkOptions(embedding.ChunkOptions{ThresholdLines: 20, WindowLines: 15}) - _, err := idx.Index(dir) + idx, store := newFTSIndexer(t, dir, reg, cfg, func(idx *Indexer) { + idx.SetEmbedder(stubEmbedder{}) + idx.SetEmbeddingChunkOptions(embedding.ChunkOptions{ThresholdLines: 20, WindowLines: 15}) + }) + + const symbolID = "v.go::ValidateRequestPayload" + + // Probe the text half of the hybrid on its own first. The fused result + // below cannot carry this claim: the stub embedder maps every text to the + // same direction, so the vector channel returns the whole corpus for any + // query and would answer "is the symbol retrievable" no matter what the + // text side did. The query is multi-word on purpose — an identifier-shaped + // query is short-circuited on an exact FindNodesByName hit (store_fts.go + // tier 0) and never reaches the ranked FTS tier. + ftsHits, err := store.SearchSymbols("validate request payload", 20) require.NoError(t, err) + ftsIDs := make([]string, 0, len(ftsHits)) + for _, h := range ftsHits { + ftsIDs = append(ftsIDs, h.NodeID) + } + require.Contains(t, ftsIDs, symbolID, "the chunked symbol must be in the symbol FTS corpus") results := idx.Search().Search("ValidateRequestPayload", 20) require.NotEmpty(t, results) @@ -174,6 +189,6 @@ func TestBuildSearchIndex_ChunkedSymbolNotDuplicatedInSearch(t *testing.T) { "a chunk ID must never appear in search output") seen[r.ID]++ } - assert.LessOrEqual(t, seen["v.go::ValidateRequestPayload"], 1, - "the chunked symbol must not be returned more than once") + assert.Equal(t, 1, seen[symbolID], + "the chunked symbol must be returned exactly once") } diff --git a/internal/indexer/incremental_resolve_test.go b/internal/indexer/incremental_resolve_test.go index b5b152b0f..e924eff74 100644 --- a/internal/indexer/incremental_resolve_test.go +++ b/internal/indexer/incremental_resolve_test.go @@ -189,7 +189,7 @@ func TestIncrementalReindex_DeletedDefinition_NoStaleTierOnStub(t *testing.T) { // cascade: deleting a file drops its nodes' churn/coverage/blame // sidecar rows, leaving no orphan enrichment. func TestEvictFile_DropsEnrichmentSidecars(t *testing.T) { - idx, _ := newToggleIndexer(t) + idx, _, _ := newToggleIndexer(t) dir := t.TempDir() idx.SetRootPath(dir) g := idx.graph diff --git a/internal/indexer/indexer_test.go b/internal/indexer/indexer_test.go index 0ead3a056..4dccc89fa 100644 --- a/internal/indexer/indexer_test.go +++ b/internal/indexer/indexer_test.go @@ -13,6 +13,7 @@ import ( "github.com/zzet/gortex/internal/config" "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/graph/store_sqlite" "github.com/zzet/gortex/internal/parser" "github.com/zzet/gortex/internal/parser/languages" ) @@ -72,6 +73,54 @@ func newTestIndexer(g graph.Store) *Indexer { return New(g, reg, cfg, zap.NewNop()) } +// newFTSStore opens a throwaway sqlite store — the only Store that carries a +// native symbol FTS, and the one production always runs on. +func newFTSStore(t *testing.T) *store_sqlite.Store { + t.Helper() + store, err := store_sqlite.Open(filepath.Join(t.TempDir(), "fts.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { _ = store.Close() }) + return store +} + +// newFTSIndexer indexes dir into a real sqlite store and returns both. It is +// the fixture for every assertion about what indexing makes SEARCHABLE: +// production always runs on a store whose native FTS answers symbol search, so +// idx.Search() here is the SymbolSearcherBackend reading store.SearchSymbols +// rather than an in-process index the daemon never builds. +// +// prepare runs on the indexer before Index, for the SetEmbedder / +// SetEmbeddingChunkOptions wiring some fixtures need. +func newFTSIndexer( + t *testing.T, + dir string, + reg *parser.Registry, + cfg config.IndexConfig, + prepare ...func(*Indexer), +) (*Indexer, *store_sqlite.Store) { + t.Helper() + store := newFTSStore(t) + idx := New(store, reg, cfg, zap.NewNop()) + for _, p := range prepare { + p(idx) + } + _, err := idx.Index(dir) + require.NoError(t, err) + requireSymbolFTS(t, store) + return idx, store +} + +// requireSymbolFTS fails unless the store's symbol corpus actually holds +// something. An empty corpus satisfies every "must NOT be searchable" +// assertion for the wrong reason and turns the whole fixture into a silent +// pass, so no search test may run without this gate. +func requireSymbolFTS(t *testing.T, store *store_sqlite.Store) { + t.Helper() + count, err := store.SymbolFTSCount() + require.NoError(t, err) + require.Greater(t, count, 0, "fixture left the symbol FTS empty — search assertions would be vacuous") +} + // newTestIndexerGoJava registers both the Go and Java extractors — used by // the cross-language Temporal join tests. func newTestIndexerGoJava(g graph.Store) *Indexer { diff --git a/internal/indexer/multi_test.go b/internal/indexer/multi_test.go index a9e9c09ec..76326dee9 100644 --- a/internal/indexer/multi_test.go +++ b/internal/indexer/multi_test.go @@ -320,8 +320,8 @@ func TestMultiIndexer_TrackRepo_SearchSpansAllRepos(t *testing.T) { cm, err := config.NewConfigManager(tmpCfg) require.NoError(t, err) - g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + store := newFTSStore(t) + mi := NewMultiIndexer(store, newTestRegistry(), search.NewSymbolSearcherBackend(store), cm, zap.NewNop()) for _, e := range []config.RepoEntry{ {Path: dirA, Name: "repo-aaa"}, @@ -331,10 +331,13 @@ func TestMultiIndexer_TrackRepo_SearchSpansAllRepos(t *testing.T) { _, err := mi.TrackRepo(e) require.NoError(t, err) } + requireSymbolFTS(t, store) // Query the camelCase-split tokens individually — that's how the - // BM25 backend stores them at Add time, and Search.TokenizeQuery - // doesn't perform the same camelCase split. + // write-side tokenizer stores them, and the query tokenizer doesn't + // perform the same camelCase split. Each is also a token no node + // carries as its whole name, so the exact-name short-circuit + // (store_fts.go tier 0) misses and the ranked FTS tier answers. for _, want := range []struct{ query, prefix string }{ {"alpha", "repo-aaa"}, {"beta", "repo-bbb"}, diff --git a/internal/indexer/realtime_reliability_test.go b/internal/indexer/realtime_reliability_test.go index 193dba6a2..331c3fe45 100644 --- a/internal/indexer/realtime_reliability_test.go +++ b/internal/indexer/realtime_reliability_test.go @@ -19,6 +19,7 @@ import ( "github.com/zzet/gortex/internal/config" "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/graph/store_sqlite" "github.com/zzet/gortex/internal/parser" "github.com/zzet/gortex/internal/search" ) @@ -78,19 +79,33 @@ func (e *toggleExtractor) Extract(filePath string, src []byte) (*parser.Extracti return &parser.ExtractionResult{Nodes: nodes}, nil } -func newToggleIndexer(t *testing.T) (*Indexer, *toggleExtractor) { +func newToggleIndexer(t *testing.T) (*Indexer, *toggleExtractor, *store_sqlite.Store) { t.Helper() ext := &toggleExtractor{} reg := parser.NewRegistry() reg.Register(ext) - g := graph.New() - idx := New(g, reg, config.IndexConfig{Workers: 1}, zap.NewNop()) - idx.search = search.NewBM25() - return idx, ext + // A real sqlite store, so idx.search is the production + // SymbolSearcherBackend over the store's native symbol FTS: the + // "search entry survived / was evicted" assertions below then pin the + // corpus the daemon actually serves. The store rides out so a test can + // gate on a non-empty corpus before trusting any of them. + store := newFTSStore(t) + idx := New(store, reg, config.IndexConfig{Workers: 1}, zap.NewNop()) + return idx, ext, store } +// tier0Dodge is appended to every searchHasID query. The sqlite backend +// short-circuits an identifier-shaped query on an exact FindNodesByName hit +// and returns before the ranked FTS tier runs (store_fts.go tier 0), so +// asking for "Alpha" would be answered straight out of the graph — these +// assertions would then restate the GetNode check on the line above them and +// prove nothing about the search corpus. A second whitespace-separated token +// makes the query non-identifier-shaped. It matches no document and the MATCH +// expression is a prefix-OR, so the real term still decides the answer. +const tier0Dodge = " zznosuchtokenzz" + func searchHasID(idx *Indexer, query, id string) bool { - for _, r := range idx.search.Search(query, 50) { + for _, r := range idx.search.Search(query+tier0Dodge, 50) { if r.ID == id { return true } @@ -108,7 +123,7 @@ func searchHasID(idx *Indexer, query, id string) bool { // evicted the graph + search entries before parsing and returned early // on result == nil, leaving the file at zero nodes. func TestIndexFile_ParseFailureKeepsPriorNodes(t *testing.T) { - idx, ext := newToggleIndexer(t) + idx, ext, store := newToggleIndexer(t) dir := t.TempDir() path := filepath.Join(dir, "main.fk") idx.SetRootPath(dir) @@ -121,6 +136,7 @@ func TestIndexFile_ParseFailureKeepsPriorNodes(t *testing.T) { funcID := "main.fk::Alpha" require.NotNil(t, idx.graph.GetNode(funcID), "Alpha must be indexed before the bad edit") + requireSymbolFTS(t, store) require.True(t, searchHasID(idx, "Alpha", funcID), "Alpha must be in the search index before the bad edit") nodesBefore := len(idx.graph.GetFileNodes("main.fk")) require.Equal(t, 2, nodesBefore, "file node + Alpha") @@ -164,7 +180,7 @@ func TestIndexFile_ParseFailureKeepsPriorNodes(t *testing.T) { // must leave the file's prior nodes / search entries intact, and a clean // modify must still swap. Against the pre-fix patchGraph this FAILS. func TestPatchGraphModify_ParseFailureKeepsPriorNodes(t *testing.T) { - idx, ext := newToggleIndexer(t) + idx, ext, store := newToggleIndexer(t) dir := t.TempDir() idx.SetRootPath(dir) path := filepath.Join(dir, "main.fk") @@ -180,6 +196,7 @@ func TestPatchGraphModify_ParseFailureKeepsPriorNodes(t *testing.T) { funcID := "main.fk::Alpha" require.NotNil(t, idx.graph.GetNode(funcID), "Alpha must be indexed via the create patch") + requireSymbolFTS(t, store) require.True(t, searchHasID(idx, "Alpha", funcID)) nodesBefore := len(idx.graph.GetFileNodes("main.fk")) require.Equal(t, 2, nodesBefore, "file node + Alpha") @@ -290,7 +307,7 @@ func haveGit(t *testing.T) bool { // seam stands in for full-tree reconciliation so the assertion is // deterministic and platform-independent. func TestWatcher_OverflowEventTriggersReconcile(t *testing.T) { - idx, _ := newToggleIndexer(t) + idx, _, _ := newToggleIndexer(t) idx.SetRootPath(t.TempDir()) w, err := NewWatcher(idx, config.WatchConfig{Enabled: true, DebounceMs: 10}, zap.NewNop()) require.NoError(t, err) @@ -321,7 +338,7 @@ func TestWatcher_OverflowEventTriggersReconcile(t *testing.T) { // signals collapses into at most one reconcile in flight — the loop is // never blocked and the tree isn't re-walked per dropped event. func TestWatcher_OverflowReconcileCoalesces(t *testing.T) { - idx, _ := newToggleIndexer(t) + idx, _, _ := newToggleIndexer(t) idx.SetRootPath(t.TempDir()) w, err := NewWatcher(idx, config.WatchConfig{Enabled: true, DebounceMs: 10}, zap.NewNop()) require.NoError(t, err) @@ -477,7 +494,7 @@ func TestWatcher_NewSubdirScanIndexesPreWatchFile(t *testing.T) { // changes inside an existing directory fire their own file events. Uses // the scanFn seam. func TestWatcher_DirEventScanGating(t *testing.T) { - idx, _ := newToggleIndexer(t) + idx, _, _ := newToggleIndexer(t) dir := t.TempDir() idx.SetRootPath(dir) subdir := filepath.Join(dir, "sub") diff --git a/internal/indexer/reresolve_guard_test.go b/internal/indexer/reresolve_guard_test.go index df5b5d61c..f0d5ecaf6 100644 --- a/internal/indexer/reresolve_guard_test.go +++ b/internal/indexer/reresolve_guard_test.go @@ -17,7 +17,7 @@ import ( // edge demoted from a concrete target to a stub keeps the incident-edge total // identical. func TestCountResolvedFileEdges(t *testing.T) { - idx, _ := newToggleIndexer(t) + idx, _, _ := newToggleIndexer(t) g := idx.graph g.AddBatch([]*graph.Node{ {ID: "a.go", Kind: graph.KindFile, Name: "a.go", FilePath: "a.go"}, @@ -39,7 +39,7 @@ func TestCountResolvedFileEdges(t *testing.T) { // enqueues a forced scoped re-resolve and bumps the regression counter, while // below-floor / symbol-removal / modest-drop cases stay quiet. func TestGuardResolvedEdgeRegression(t *testing.T) { - idx, _ := newToggleIndexer(t) + idx, _, _ := newToggleIndexer(t) w, err := NewWatcher(idx, config.WatchConfig{Enabled: true, DebounceMs: 10}, zap.NewNop()) require.NoError(t, err) diff --git a/internal/indexer/skip_search_test.go b/internal/indexer/skip_search_test.go index 65b123229..6d9475afa 100644 --- a/internal/indexer/skip_search_test.go +++ b/internal/indexer/skip_search_test.go @@ -7,20 +7,39 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "go.uber.org/zap" "github.com/zzet/gortex/internal/config" - "github.com/zzet/gortex/internal/graph" "github.com/zzet/gortex/internal/parser" "github.com/zzet/gortex/internal/parser/languages" ) +// searchIDs runs a query through the indexer's search backend and returns the +// hit IDs. +func searchIDs(idx *Indexer, query string, limit int) []string { + hits := idx.Search().Search(query, limit) + ids := make([]string, 0, len(hits)) + for _, h := range hits { + ids = append(ids, h.ID) + } + return ids +} + // Indexing a directory with JSON + Go files must keep JSON keys out -// of the text search index by default (regression guard for the +// of the symbol search corpus by default (regression guard for the // search-backend memory blowup) while still leaving them reachable // via graph queries. Uses lowercase-only single-word identifiers so -// the BM25 query tokenizer (which does not split camelCase) hits -// exactly the tokens produced by the Add-path tokenizer. +// the query tokenizer (which does not split camelCase) hits exactly +// the tokens produced by the write-side tokenizer. +// +// The probe is ONE two-word query rather than two identifier queries, and +// that shape is load-bearing: the sqlite backend short-circuits an +// identifier-shaped query on an exact FindNodesByName hit and returns +// before the ranked FTS tier runs (store_fts.go tier 0). Asking for +// "uniquejsonkeyzzz" alone would therefore be answered out of the GRAPH — +// where the JSON key legitimately lives — and the exclusion would look +// broken even when it works. A whitespace-separated query skips tier 0, and +// the MATCH expression is a prefix-OR, so one query proves both halves at +// once: the Go symbol is in the corpus, the JSON key is not. func TestIndex_JSONVariablesSkippedFromSearchByDefault(t *testing.T) { dir := t.TempDir() @@ -35,8 +54,6 @@ func TestIndex_JSONVariablesSkippedFromSearchByDefault(t *testing.T) { func uniquegosymbolzzz() {} `), 0o644)) - g := graph.New() - reg := parser.NewRegistry() reg.Register(languages.NewGoExtractor()) reg.Register(languages.NewJSONExtractor()) @@ -46,28 +63,30 @@ func uniquegosymbolzzz() {} // Mirror what ConfigManager.GetRepoConfig would do for a real run. cfg.SkipSearch = config.DefaultSkipSearch() - idx := New(g, reg, cfg, zap.NewNop()) - _, err := idx.Index(dir) - require.NoError(t, err) + idx, store := newFTSIndexer(t, dir, reg, cfg) - // Graph still carries the JSON key — SkipSearch is about the text - // index, not the graph. Users looking up a config key via + // Graph still carries the JSON key — SkipSearch is about the search + // corpus, not the graph. Users looking up a config key via // FindNodesByName / get_symbol must still find it. - jsonNodes := g.FindNodesByName("uniquejsonkeyzzz") + jsonNodes := store.FindNodesByName("uniquejsonkeyzzz") require.NotEmpty(t, jsonNodes, "JSON variable node should exist in graph") + jsonID := jsonNodes[0].ID - // Search index must include the Go symbol but NOT the JSON key. - goHits := idx.Search().Search("uniquegosymbolzzz", 10) - assert.NotEmpty(t, goHits, "Go symbol should be text-indexed") - - jsonHits := idx.Search().Search("uniquejsonkeyzzz", 10) - assert.Empty(t, jsonHits, - "JSON variable node must be excluded from text index by default (SkipSearch)") + ids := searchIDs(idx, "uniquegosymbolzzz uniquejsonkeyzzz", 10) + assert.Contains(t, ids, "main.go::uniquegosymbolzzz", + "Go symbol should be in the symbol search corpus") + assert.NotContains(t, ids, jsonID, + "JSON variable node must be excluded from the search corpus by default (SkipSearch)") } -// With SkipSearch cleared, JSON variable nodes are text-searchable -// again. Guards the config surface: users who actually want to search -// their package.json keys can opt in by overriding SkipSearch. +// With SkipSearch cleared, JSON variable nodes are searchable again. Guards +// the config surface: users who actually want to search their package.json +// keys can opt in by overriding SkipSearch. +// +// The trailing "absentneedlezzz" token exists only to make the query +// non-identifier-shaped, for the tier-0 reason spelled out above; it appears +// in no document, so whether the JSON key comes back is decided entirely by +// the ranked FTS tier. func TestIndex_JSONVariablesSearchableWhenSkipSearchCleared(t *testing.T) { dir := t.TempDir() @@ -75,8 +94,6 @@ func TestIndex_JSONVariablesSearchableWhenSkipSearchCleared(t *testing.T) { "overridekeyzzz": "value" }`), 0o644)) - g := graph.New() - reg := parser.NewRegistry() reg.Register(languages.NewJSONExtractor()) @@ -84,11 +101,12 @@ func TestIndex_JSONVariablesSearchableWhenSkipSearchCleared(t *testing.T) { cfg.Workers = 1 cfg.SkipSearch = nil // override: index everything - idx := New(g, reg, cfg, zap.NewNop()) - _, err := idx.Index(dir) - require.NoError(t, err) + idx, store := newFTSIndexer(t, dir, reg, cfg) + + nodes := store.FindNodesByName("overridekeyzzz") + require.NotEmpty(t, nodes, "JSON variable node should exist in graph") - hits := idx.Search().Search("overridekeyzzz", 10) - assert.NotEmpty(t, hits, - "JSON variable node should be text-indexed when SkipSearch is cleared") + ids := searchIDs(idx, "overridekeyzzz absentneedlezzz", 10) + assert.Contains(t, ids, nodes[0].ID, + "JSON variable node should be searchable when SkipSearch is cleared") } diff --git a/internal/indexer/skip_telemetry_test.go b/internal/indexer/skip_telemetry_test.go index 5c2f734d2..e9eec5d7a 100644 --- a/internal/indexer/skip_telemetry_test.go +++ b/internal/indexer/skip_telemetry_test.go @@ -180,7 +180,7 @@ func TestParseFailedSkipResult_RecordsError(t *testing.T) { // file's prior nodes through a transient failure (see // TestPatchGraphModify_ParseFailureKeepsPriorNodes). func TestIndex_ParseFailedSkipTelemetry(t *testing.T) { - idx, ext := newToggleIndexer(t) + idx, ext, _ := newToggleIndexer(t) ext.setFail(true) // every extraction returns an error dir := t.TempDir() diff --git a/internal/mcp/scope_resolve_test.go b/internal/mcp/scope_resolve_test.go index b28f2df5d..8d3cc7afb 100644 --- a/internal/mcp/scope_resolve_test.go +++ b/internal/mcp/scope_resolve_test.go @@ -19,6 +19,7 @@ import ( "github.com/zzet/gortex/internal/config" "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/graph/store_sqlite" "github.com/zzet/gortex/internal/indexer" "github.com/zzet/gortex/internal/query" "github.com/zzet/gortex/internal/search" @@ -828,18 +829,24 @@ func newTwoRepoServer(t *testing.T) (*Server, string) { cm, err := config.NewConfigManager(tmpCfg) require.NoError(t, err) - g := graph.New() - reg := testRegistry() - bm := search.NewBM25() - mi := indexer.NewMultiIndexer(g, reg, bm, cm, zap.NewNop()) + // A real sqlite store: search runs off its native symbol FTS, the same + // corpus the daemon serves, so the scope narrowing is applied to + // production-shaped hits. + store, err := store_sqlite.Open(filepath.Join(t.TempDir(), "fts.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { _ = store.Close() }) + + backend := search.NewSymbolSearcherBackend(store) + mi := indexer.NewMultiIndexer(store, testRegistry(), backend, cm, zap.NewNop()) _, err = mi.IndexScoped("", "") require.NoError(t, err) + requireSymbolFTS(t, store) - eng := query.NewEngine(g) - eng.SetSearch(bm) + eng := query.NewEngine(store) + eng.SetSearch(backend) flagVal := true - srv := NewServer(eng, g, nil, nil, zap.NewNop(), nil, MultiRepoOptions{ + srv := NewServer(eng, store, nil, nil, zap.NewNop(), nil, MultiRepoOptions{ MultiIndexer: mi, ConfigManager: cm, ScopeIntentDefaults: &flagVal, diff --git a/internal/mcp/tools_search_corpus_test.go b/internal/mcp/tools_search_corpus_test.go index 99d0047c6..12b74191a 100644 --- a/internal/mcp/tools_search_corpus_test.go +++ b/internal/mcp/tools_search_corpus_test.go @@ -12,13 +12,46 @@ import ( "go.uber.org/zap" "github.com/zzet/gortex/internal/config" - "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/graph/store_sqlite" "github.com/zzet/gortex/internal/indexer" "github.com/zzet/gortex/internal/query" ) +// newFTSServer indexes dir into a real sqlite store and returns a Server whose +// search reads that store's native symbol FTS — the corpus production actually +// serves. Everything the daemon's search_symbols path touches (the engine, the +// retrieval channels, the rerank) then sees the same documents and the same +// ranking it would see in a live workspace. +// +// The SymbolFTSCount gate is load-bearing: an empty corpus answers every +// "must not be returned" assertion correctly and leaves the "must be returned" +// ones as the only thing between the fixture and a silent pass. +func newFTSServer(t *testing.T, dir string, cfg config.IndexConfig) *Server { + t.Helper() + store, err := store_sqlite.Open(filepath.Join(t.TempDir(), "fts.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { _ = store.Close() }) + + idx := indexer.New(store, testRegistry(), cfg, zap.NewNop()) + _, err = idx.Index(dir) + require.NoError(t, err) + requireSymbolFTS(t, store) + + eng := query.NewEngine(store) + eng.SetSearchProvider(idx.Search) + return NewServer(eng, store, idx, nil, zap.NewNop(), nil) +} + +// requireSymbolFTS fails unless the store's symbol corpus holds something. +func requireSymbolFTS(t *testing.T, store *store_sqlite.Store) { + t.Helper() + count, err := store.SymbolFTSCount() + require.NoError(t, err) + require.Greater(t, count, 0, "fixture left the symbol FTS empty — search assertions would be vacuous") +} + // corpusTestServer indexes a repo containing a Go file and a Markdown -// doc, so the BM25 index holds both code symbols and prose-section +// doc, so the symbol FTS corpus holds both code symbols and prose-section // nodes. func corpusTestServer(t *testing.T) *Server { return corpusTestServerProse(t, true) @@ -39,16 +72,9 @@ func corpusTestServerProse(t *testing.T, indexProse bool) *Server { "and apply the kubernetes manifest.\n\n"+ "## Troubleshooting\n\nCheck the logs when a request times out.\n"), 0o644)) - g := graph.New() - reg := testRegistry() cfg := config.Default() cfg.Index.IndexProse = indexProse - idx := indexer.New(g, reg, cfg.Index, zap.NewNop()) - _, err := idx.Index(dir) - require.NoError(t, err) - eng := query.NewEngine(g) - eng.SetSearchProvider(idx.Search) - return NewServer(eng, g, idx, nil, zap.NewNop(), nil) + return newFTSServer(t, dir, cfg.Index) } func corpusSearch(t *testing.T, srv *Server, args map[string]any) []map[string]any { diff --git a/internal/mcp/tools_search_docchannel_test.go b/internal/mcp/tools_search_docchannel_test.go index ac3279f7a..dba8fa45b 100644 --- a/internal/mcp/tools_search_docchannel_test.go +++ b/internal/mcp/tools_search_docchannel_test.go @@ -8,12 +8,8 @@ import ( "testing" "github.com/stretchr/testify/require" - "go.uber.org/zap" "github.com/zzet/gortex/internal/config" - "github.com/zzet/gortex/internal/graph" - "github.com/zzet/gortex/internal/indexer" - "github.com/zzet/gortex/internal/query" ) // docChannelServer indexes MANY code symbols whose names share the @@ -38,15 +34,7 @@ func docChannelServer(t *testing.T) *Server { []byte("# Guide\n\n## Deployment\n\n"+ "To deploy the service push the container image and apply the manifest.\n"), 0o644)) - g := graph.New() - reg := testRegistry() - cfg := config.Default() - idx := indexer.New(g, reg, cfg.Index, zap.NewNop()) - _, err := idx.Index(dir) - require.NoError(t, err) - eng := query.NewEngine(g) - eng.SetSearchProvider(idx.Search) - return NewServer(eng, g, idx, nil, zap.NewNop(), nil) + return newFTSServer(t, dir, config.Default().Index) } // TestSearchSymbols_DocChannelRescuesCrowdedProse is the core feature From b458455b69046473818627eebcb6dd5effa0eaa4 Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Sun, 9 Aug 2026 20:52:40 +0200 Subject: [PATCH 02/21] test(mcp): drive the ranking-shape fixtures from an explicit ordered backend MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit These fixtures test MCP-layer logic — post-filter escalation, soup split/merge, equivalence rewrite, over-fetch retention — given a candidate ordering, not the ranker that produced it. Replace the hand-tuned text corpora with an orderedBackend that answers a token with a literal list of node IDs and records every Search limit, so the premise each test rests on is stated instead of emergent. --- internal/mcp/ordered_backend_test.go | 142 ++++++++++++++++++ internal/mcp/tools_explore_test.go | 20 +-- internal/mcp/tools_search_equivalence_test.go | 24 ++- internal/mcp/tools_search_escalation_test.go | 72 +++++---- internal/mcp/tools_search_soup_test.go | 22 +-- 5 files changed, 220 insertions(+), 60 deletions(-) create mode 100644 internal/mcp/ordered_backend_test.go diff --git a/internal/mcp/ordered_backend_test.go b/internal/mcp/ordered_backend_test.go new file mode 100644 index 000000000..02b6e1259 --- /dev/null +++ b/internal/mcp/ordered_backend_test.go @@ -0,0 +1,142 @@ +package mcp + +import ( + "strings" + "sync" + "unicode" + + "github.com/zzet/gortex/internal/search" +) + +// orderedBackend is a search.Backend that answers with a literal, +// hand-written candidate ordering instead of a ranked one. +// +// The MCP fixtures that use it are not testing a ranker: they test +// handler-layer logic (post-filter escalation, soup split/merge, +// equivalence rewrite, over-fetch retention, name diversification) +// GIVEN some candidate ordering. Encoding that ordering directly makes +// each fixture's premise readable and stable, instead of leaving it to +// emerge from term frequencies in a hand-tuned corpus. +// +// Retrieval model, deliberately crude: the query is split on +// whitespace, each field lowercased and stripped of edge punctuation, +// and the ordered ID lists of every matching token are unioned in +// token order (first token's list first, deduped). A query whose +// tokens are all unknown falls back to the optional default list — +// so a fixture can either key retrieval on specific tokens (the +// rewrite is then the thing under test: a symbol is reachable only +// if the handler actually queried one of its tokens) or hand the same +// ordering to every query. +// +// Note the tokens are whole whitespace fields, NOT camelCase parts: +// "WidgetExtensions" is the single token "widgetextensions", so a +// fixture can give a compound name and one of its parts different +// orderings. +type orderedBackend struct { + mu sync.Mutex + byToken map[string][]string // query token -> ordered node IDs + def []string // answer when no token matched + limits []int // every limit Search was called with +} + +func newOrderedBackend() *orderedBackend { + return &orderedBackend{byToken: make(map[string][]string)} +} + +// put appends ids to the ordering token answers with. Call order is +// the ranking: the first ID put is the top hit. +func (o *orderedBackend) put(token string, ids ...string) { + key := strings.ToLower(token) + o.byToken[key] = append(o.byToken[key], ids...) +} + +// putDefault appends ids to the ordering returned for a query whose +// tokens are all unknown. +func (o *orderedBackend) putDefault(ids ...string) { + o.def = append(o.def, ids...) +} + +// searchLimits returns every limit the engine asked this backend for, +// oldest first. +func (o *orderedBackend) searchLimits() []int { + o.mu.Lock() + defer o.mu.Unlock() + return append([]int(nil), o.limits...) +} + +// Add / Remove are no-ops: the ordering is declared by the fixture, not +// accumulated from indexed text. +func (o *orderedBackend) Add(string, ...string) {} +func (o *orderedBackend) Remove(string) {} +func (o *orderedBackend) Close() {} + +// Count reports the number of distinct IDs the backend can return. It +// must be positive for a populated fixture: the engine routes to its +// substring fallback when the backend reports an empty corpus. +func (o *orderedBackend) Count() int { + seen := make(map[string]bool) + for _, ids := range o.byToken { + for _, id := range ids { + seen[id] = true + } + } + for _, id := range o.def { + seen[id] = true + } + return len(seen) +} + +// Search returns the first limit IDs of the matching ordering, scored +// descending so the caller sees a plausible ranked shape. (The engine +// keeps the rank, not the score, so the numbers are cosmetic.) +func (o *orderedBackend) Search(query string, limit int) []search.SearchResult { + o.mu.Lock() + o.limits = append(o.limits, limit) + o.mu.Unlock() + + ids := o.ordering(query) + if limit > 0 && len(ids) > limit { + ids = ids[:limit] + } + out := make([]search.SearchResult, 0, len(ids)) + for i, id := range ids { + out = append(out, search.SearchResult{ID: id, Score: float64(len(ids) - i)}) + } + return out +} + +func (o *orderedBackend) ordering(query string) []string { + var ( + seen = make(map[string]bool) + ids []string + ) + for _, tok := range orderedQueryTokens(query) { + for _, id := range o.byToken[tok] { + if seen[id] { + continue + } + seen[id] = true + ids = append(ids, id) + } + } + if len(ids) == 0 { + return o.def + } + return ids +} + +// orderedQueryTokens lowercases the query, splits it on whitespace and +// trims edge punctuation from each field. +func orderedQueryTokens(query string) []string { + fields := strings.Fields(strings.ToLower(query)) + out := make([]string, 0, len(fields)) + for _, f := range fields { + f = strings.TrimFunc(f, func(r rune) bool { + return !unicode.IsLetter(r) && !unicode.IsDigit(r) + }) + if f != "" { + out = append(out, f) + } + } + return out +} diff --git a/internal/mcp/tools_explore_test.go b/internal/mcp/tools_explore_test.go index fba5cd304..be91dd902 100644 --- a/internal/mcp/tools_explore_test.go +++ b/internal/mcp/tools_explore_test.go @@ -13,7 +13,6 @@ import ( "github.com/zzet/gortex/internal/graph" "github.com/zzet/gortex/internal/query" - "github.com/zzet/gortex/internal/search" "github.com/zzet/gortex/internal/search/rerank" ) @@ -812,7 +811,14 @@ func TestExploreHelpers(t *testing.T) { // repeated same-name leaves must not crowd out a differently-named code target. func TestFacadeExploreDemotesRepeatedDataLeafNames(t *testing.T) { g := graph.New() - bm := search.NewBM25() + // Retrieval ordering, stated rather than ranked: every query in this + // fixture retrieves the 30 short exact-name `client` declarations + // first and the differently-named callable last. That is the shape a + // ranker produces for this task -- the literal name matches win the + // head, the callable that actually explains the area matches on prose + // and lands behind them -- and it is the shape explore's over-fetch + // window must retain so name diversification can promote it. + ob := newOrderedBackend() for i := 0; i < 30; i++ { id := fmt.Sprintf("pkg/service%d.go::client", i) n := &graph.Node{ @@ -821,7 +827,7 @@ func TestFacadeExploreDemotesRepeatedDataLeafNames(t *testing.T) { Meta: map[string]any{"signature": "var client *Transport"}, } g.AddNode(n) - bm.Add(id, n.Name, n.FilePath, "trace client coordinated transport") + ob.putDefault(id) } relevant := &graph.Node{ ID: "pkg/coordinator.go::TransportCoordinator", Name: "TransportCoordinator", @@ -829,14 +835,10 @@ func TestFacadeExploreDemotesRepeatedDataLeafNames(t *testing.T) { Meta: map[string]any{"signature": "func TransportCoordinator()"}, } g.AddNode(relevant) - // It matches the whole task, but its longer prose and non-literal symbol - // name rank below the short exact-name `client` declarations. The concept - // over-fetch window must retain it so name diversification can promote it. - relevantText := strings.Repeat("architecture routing plumbing lifecycle ", 40) + "trace client coordinated transport coordinator" - bm.Add(relevant.ID, relevant.Name, relevant.FilePath, relevantText) + ob.putDefault(relevant.ID) eng := query.NewEngine(g) - eng.SetSearch(bm) + eng.SetSearch(ob) srv := NewServer(eng, g, nil, nil, zap.NewNop(), nil) task := "trace how the client is coordinated" searchQuery := shapeExploreQuery(task) diff --git a/internal/mcp/tools_search_equivalence_test.go b/internal/mcp/tools_search_equivalence_test.go index c43fd4e59..844c52bd9 100644 --- a/internal/mcp/tools_search_equivalence_test.go +++ b/internal/mcp/tools_search_equivalence_test.go @@ -3,6 +3,7 @@ package mcp import ( "context" "encoding/json" + "strings" "testing" mcplib "github.com/mark3labs/mcp-go/mcp" @@ -12,26 +13,39 @@ import ( "github.com/zzet/gortex/internal/config" "github.com/zzet/gortex/internal/graph" "github.com/zzet/gortex/internal/query" - "github.com/zzet/gortex/internal/search" + "github.com/zzet/gortex/internal/search/rerank" ) // equivalenceTestServer builds a single-repo server with no LLM // provider, so any expansion observed comes purely from the // deterministic equivalence channel. +// +// Retrieval ordering: a symbol answers to its own lowercased name and +// to each of that name's camelCase/snake_case parts ("LoginService" -> +// loginservice, login, service), in declaration order, and to nothing +// else. The vocabulary the query itself uses ("auth", "delete", +// "blast") is deliberately absent, so a bridged symbol is reachable +// only if the rewrite actually emitted one of its tokens — the rewrite +// stays the thing under test rather than a ranker's tolerance. func equivalenceTestServer(t *testing.T, names []string, equivEnabled *bool) *Server { t.Helper() g := graph.New() - bm := search.NewBM25() + ob := newOrderedBackend() for i, n := range names { id := "pkg/" + n + ".go::" + n g.AddNode(&graph.Node{ ID: id, Kind: graph.KindFunction, Name: n, FilePath: "pkg/" + n + ".go", StartLine: i + 1, EndLine: i + 5, Language: "go", }) - bm.Add(id, n, "pkg/"+n+".go", "") + ob.put(n, id) + for _, tok := range rerank.Tokenize(n) { + if tok != strings.ToLower(n) { + ob.put(tok, id) + } + } } eng := query.NewEngine(g) - eng.SetSearch(bm) + eng.SetSearch(ob) srv := NewServer(eng, g, nil, nil, zap.NewNop(), nil) srv.SetSearchConfig(config.SearchConfig{EquivalenceClasses: equivEnabled}) srv.RunAnalysis() // builds the auto-concept vocabulary @@ -73,7 +87,7 @@ func TestSearchSymbols_EquivalenceBridgesVocabulary(t *testing.T) { } // TestSearchSymbols_EquivalenceDeleteRemove confirms the delete/remove -// bridge works in both directions through the BM25 OR-merge. +// bridge works in both directions through the expansion OR-merge. func TestSearchSymbols_EquivalenceDeleteRemove(t *testing.T) { srv := equivalenceTestServer(t, []string{ "RemoveUser", "DropTable", "FetchProfile", diff --git a/internal/mcp/tools_search_escalation_test.go b/internal/mcp/tools_search_escalation_test.go index 90129fbd4..4956ff2e5 100644 --- a/internal/mcp/tools_search_escalation_test.go +++ b/internal/mcp/tools_search_escalation_test.go @@ -9,27 +9,29 @@ import ( "github.com/zzet/gortex/internal/graph" "github.com/zzet/gortex/internal/query" - "github.com/zzet/gortex/internal/search" ) -// floodTestServer builds a server whose BM25 head for the token -// "extensions" is fully occupied by doc-section nodes: one real code -// symbol (WidgetExtensions) plus docCount KindDoc nodes that all -// outrank it (single-token name, tripled term frequency). With the -// default corpus=code filter, every fetched candidate is dropped — -// the shape a suffix-convention query takes on a repo whose doc/junk -// files share the naming tokens. +// floodTestServer builds a server whose candidate ordering for the +// token "extensions" is docCount KindDoc sections and nothing else: +// the junk sections own the token outright, so with the default +// corpus=code filter every candidate a bounded fetch reaches is +// dropped — the shape a suffix-convention query takes on a repo whose +// doc/junk files share the naming tokens. The one real code symbol +// (WidgetExtensions) is NOT in that ordering; it carries the token +// only as part of its full name, so it is reachable solely through +// the engine's substring fill, whose budget is the fetch depth. That +// is what makes the escalation depth observable — a fetch deep enough +// to leave slack past the doc flood is the only one that rescues it. func floodTestServer(t *testing.T, docCount int) *Server { t.Helper() g := graph.New() - bm := search.NewBM25() + ob := newOrderedBackend() id := "pkg/WidgetExtensions.go::WidgetExtensions" g.AddNode(&graph.Node{ ID: id, Kind: graph.KindType, Name: "WidgetExtensions", FilePath: "pkg/WidgetExtensions.go", StartLine: 1, EndLine: 5, Language: "go", }) - bm.Add(id, "WidgetExtensions", "pkg/WidgetExtensions.go", "") for i := 0; i < docCount; i++ { docID := fmt.Sprintf("junk/list%d.txt::sec%d", i, i) @@ -37,11 +39,14 @@ func floodTestServer(t *testing.T, docCount int) *Server { ID: docID, Kind: graph.KindDoc, Name: "Extensions", FilePath: fmt.Sprintf("junk/list%d.txt", i), StartLine: 1, EndLine: 3, Language: "text", }) - bm.Add(docID, "Extensions Extensions Extensions") + ob.put("extensions", docID) } + // The spelled-out name is its own token and answers with the code + // symbol alone — a query for it is never flooded. + ob.put("widgetextensions", id) eng := query.NewEngine(g) - eng.SetSearch(bm) + eng.SetSearch(ob) srv := NewServer(eng, g, nil, nil, zap.NewNop(), nil) srv.RunAnalysis() return srv @@ -50,11 +55,15 @@ func floodTestServer(t *testing.T, docCount int) *Server { // floodTestServerN is floodTestServer with codeCount rescuable code // symbols behind the doc flood — the multi-page rescue shape from the // PR review: a partially-successful shallow rescue must not strand a -// later cursor page a deeper fetch would fill. +// later cursor page a deeper fetch would fill. Same ordering premise as +// floodTestServer — the doc sections are the whole "extensions" +// ordering and the code symbols ride the substring fill — so the depth +// a fetch reaches decides how many of them survive the corpus filter: +// docCount+5 of budget rescues 5, a deeper fetch rescues all. func floodTestServerN(t *testing.T, docCount, codeCount int) *Server { t.Helper() g := graph.New() - bm := search.NewBM25() + ob := newOrderedBackend() for i := 0; i < codeCount; i++ { id := fmt.Sprintf("pkg/w%d.go::WidgetExtensions%d", i, i) @@ -62,7 +71,6 @@ func floodTestServerN(t *testing.T, docCount, codeCount int) *Server { ID: id, Kind: graph.KindType, Name: fmt.Sprintf("WidgetExtensions%d", i), FilePath: fmt.Sprintf("pkg/w%d.go", i), StartLine: 1, EndLine: 5, Language: "go", }) - bm.Add(id, fmt.Sprintf("WidgetExtensions%d", i), fmt.Sprintf("pkg/w%d.go", i), "") } for i := 0; i < docCount; i++ { docID := fmt.Sprintf("junk/list%d.txt::sec%d", i, i) @@ -70,11 +78,11 @@ func floodTestServerN(t *testing.T, docCount, codeCount int) *Server { ID: docID, Kind: graph.KindDoc, Name: "Extensions", FilePath: fmt.Sprintf("junk/list%d.txt", i), StartLine: 1, EndLine: 3, Language: "text", }) - bm.Add(docID, "Extensions Extensions Extensions") + ob.put("extensions", docID) } eng := query.NewEngine(g) - eng.SetSearch(bm) + eng.SetSearch(ob) srv := NewServer(eng, g, nil, nil, zap.NewNop(), nil) srv.RunAnalysis() return srv @@ -101,21 +109,22 @@ func TestSearchSymbols_EscalationReachesCursorWindow(t *testing.T) { // TestSearchSymbols_EscalationSkipsDuplicateDepth: once the depth cap // clamps a multiplier, the next multiplier collapses to the same // effective depth — an identical re-query cannot change the outcome and -// must be skipped, not paid a second time. +// must be skipped, not paid a second time. The ordering is all-doc, so +// no depth can ever rescue a code symbol and the escalation loop runs +// to its own stopping rule. func TestSearchSymbols_EscalationSkipsDuplicateDepth(t *testing.T) { g := graph.New() - bm := search.NewBM25() + ob := newOrderedBackend() for i := 0; i < 2100; i++ { docID := fmt.Sprintf("junk/list%d.txt::sec%d", i, i) g.AddNode(&graph.Node{ ID: docID, Kind: graph.KindDoc, Name: "Extensions", FilePath: fmt.Sprintf("junk/list%d.txt", i), StartLine: 1, EndLine: 3, Language: "text", }) - bm.Add(docID, "Extensions Extensions Extensions") + ob.put("extensions", docID) } - counter := &countingBackend{BM25Backend: bm} eng := query.NewEngine(g) - eng.SetSearch(counter) + eng.SetSearch(ob) srv := NewServer(eng, g, nil, nil, zap.NewNop(), nil) srv.RunAnalysis() @@ -123,27 +132,16 @@ func TestSearchSymbols_EscalationSkipsDuplicateDepth(t *testing.T) { "query": "Extensions", "limit": 100, "cursor": encodeCursor(400), }) require.Empty(t, respIDs(resp), "an all-doc corpus yields no code page") - require.NotEmpty(t, counter.limits, "counting backend must intercept Search calls") + limits := ob.searchLimits() + require.NotEmpty(t, limits, "the ordered backend must have intercepted the Search calls") deep := 0 - for _, l := range counter.limits { + for _, l := range limits { if l >= 2000 { deep++ } } require.LessOrEqualf(t, deep, 1, - "capped escalation depths must not repeat an identical query; deep-call limits: %v", counter.limits) -} - -// countingBackend wraps the BM25 backend and records every Search -// limit, so a test can prove an identical capped depth is not re-paid. -type countingBackend struct { - *search.BM25Backend - limits []int -} - -func (c *countingBackend) Search(q string, limit int) []search.SearchResult { - c.limits = append(c.limits, limit) - return c.BM25Backend.Search(q, limit) + "capped escalation depths must not repeat an identical query; call limits: %v", limits) } // TestSearchSymbols_EscalatesWhenPostFilterEmptiesFetch is the core diff --git a/internal/mcp/tools_search_soup_test.go b/internal/mcp/tools_search_soup_test.go index 6f1862530..8f5ea94f4 100644 --- a/internal/mcp/tools_search_soup_test.go +++ b/internal/mcp/tools_search_soup_test.go @@ -12,27 +12,29 @@ import ( "github.com/zzet/gortex/internal/config" "github.com/zzet/gortex/internal/graph" "github.com/zzet/gortex/internal/query" - "github.com/zzet/gortex/internal/search" ) -// soupTestServer builds a single-repo server whose BM25 index holds a -// handful of distinct, single-token symbols so a soup query's split -// disjuncts each retrieve a known node. +// soupTestServer builds a single-repo server whose retrieval ordering +// is one token per symbol: each symbol answers to its own lowercased +// name and to nothing else. So a disjunct that the soup handling +// actually queries retrieves exactly its own node, and a symbol no +// disjunct names ("Unrelated") is unreachable — the merged result is a +// readout of which terms were queried, not of how they ranked. func soupTestServer(t *testing.T, soupMode string) *Server { t.Helper() g := graph.New() names := []string{"AuthHandler", "LoginService", "SigninFlow", "CredentialStore", "Unrelated"} - bm := search.NewBM25() + ob := newOrderedBackend() for _, n := range names { id := "pkg/" + n + ".go::" + n g.AddNode(&graph.Node{ ID: id, Kind: graph.KindFunction, Name: n, FilePath: "pkg/" + n + ".go", StartLine: 1, EndLine: 5, Language: "go", }) - bm.Add(id, n, "pkg/"+n+".go", "") + ob.put(n, id) } eng := query.NewEngine(g) - eng.SetSearch(bm) + eng.SetSearch(ob) srv := NewServer(eng, g, nil, nil, zap.NewNop(), nil) srv.SetSearchConfig(config.SearchConfig{KeywordSoupRewrite: soupMode}) return srv @@ -54,8 +56,8 @@ func runSoupSearch(t *testing.T, srv *Server, args map[string]any) map[string]an // TestSearchSymbols_SoupSplitMerge confirms a degenerate OR-soup query // in the default "split" mode (a) reports the keyword_soup class, (b) // attaches a query_advice nudge with the split disjuncts, and (c) the -// BM25 OR-merge over the disjuncts surfaces every targeted symbol -- -// none of which the raw soup string would rank well on its own. +// OR-merge over those disjuncts surfaces every targeted symbol and +// nothing else -- a symbol none of the disjuncts names stays out. func TestSearchSymbols_SoupSplitMerge(t *testing.T) { srv := soupTestServer(t, config.KeywordSoupSplit) resp := runSoupSearch(t, srv, map[string]any{ @@ -83,6 +85,8 @@ func TestSearchSymbols_SoupSplitMerge(t *testing.T) { } { require.Truef(t, ids[want], "soup split-merge missed %s; got %v", want, ids) } + require.Falsef(t, ids["pkg/Unrelated.go::Unrelated"], + "no disjunct names Unrelated, so the merge must not surface it; got %v", ids) } // TestSearchSymbols_SoupOffMode confirms KeywordSoupRewrite:"off" From 046683c1971858a46858fd5124cd5a2aa7326041 Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Sun, 9 Aug 2026 21:27:07 +0200 Subject: [PATCH 03/21] refactor(search): route stores without native symbol search to a null text backend Production always runs on sqlite, whose native FTS already answers search through SymbolSearcherBackend, so the in-process index the other arm built was a second copy of a corpus nobody queried. NullBackend indexes nothing and reports an empty corpus, which routes the query engine to the substring fallback it already takes for an Engine with no search backend at all. The remaining test construction sites move onto it; the two that still exercise BM25 itself keep their old backend. --- cmd/gortex/daemon_controller_worktree_test.go | 2 +- cmd/gortex/daemon_status_repo_counts_test.go | 6 +- cmd/gortex/repo_inventory_missing_test.go | 2 +- internal/graph/node_id_parity_test.go | 2 +- .../batch_transition_coordinator_test.go | 4 +- internal/indexer/contract_bridge_test.go | 4 +- internal/indexer/deferred_enrich_gate_test.go | 2 +- internal/indexer/direct_mutation_api_test.go | 2 +- .../indexer/external_revert_reindex_test.go | 2 +- .../indexer/extractor_version_restage_test.go | 4 +- .../indexer/full_topology_transaction_test.go | 14 ++--- .../indexer/git_watcher_integration_test.go | 10 +-- .../git_watcher_scoped_resolve_test.go | 2 +- .../incremental_resolve_revert_test.go | 4 +- internal/indexer/incremental_resolve_test.go | 8 +-- .../indexer/incremental_visibility_test.go | 2 +- internal/indexer/indexer.go | 8 ++- internal/indexer/multi_case_identity_test.go | 2 +- .../indexer/multi_cold_orchestration_test.go | 2 +- internal/indexer/multi_contract_edges_test.go | 18 +++--- internal/indexer/multi_global_passes_test.go | 8 +-- internal/indexer/multi_node_id_test.go | 6 +- .../indexer/multi_reconcile_sqlite_test.go | 6 +- internal/indexer/multi_reconcile_test.go | 18 +++--- internal/indexer/multi_refresh_config_test.go | 4 +- internal/indexer/multi_scoped_test.go | 12 ++-- .../indexer/multi_singlerepo_resolve_test.go | 6 +- internal/indexer/multi_test.go | 48 +++++++-------- internal/indexer/multi_topic_edges_test.go | 8 +-- internal/indexer/multi_transition_test.go | 8 +-- internal/indexer/multi_watcher_test.go | 4 +- internal/indexer/poller_test.go | 12 ++-- .../indexer/pre_enrich_resolve_hooks_test.go | 2 +- .../indexer/pre_enrich_slug_backfill_test.go | 2 +- internal/indexer/realtime_reliability_test.go | 8 +-- .../reconcile_clean_census_manifest_test.go | 6 +- ...econcile_clean_census_merkle_route_test.go | 4 +- .../indexer/reconcile_clean_census_test.go | 4 +- .../indexer/reconcile_scoped_routing_test.go | 10 +-- internal/indexer/repo_prefix_parity_test.go | 2 +- .../indexer/repository_topology_batch_test.go | 2 +- internal/indexer/scope_for_cwd_test.go | 10 +-- internal/indexer/shadow_admission_test.go | 4 +- .../indexer/spec_launch_acceptance_test.go | 10 +-- .../indexer/store_hygiene_indexer_test.go | 2 +- internal/indexer/storm_test.go | 2 +- internal/indexer/unicode_path_test.go | 2 +- internal/indexer/watcher_inert_test.go | 4 +- internal/indexer/worktree_gc_test.go | 6 +- internal/indexer/worktree_instance_test.go | 2 +- internal/mcp/analyze_scope_test.go | 2 +- internal/mcp/diff_repo_scope_test.go | 2 +- internal/mcp/enclosing_test.go | 4 +- internal/mcp/ensure_fresh_self_heal_test.go | 2 +- internal/mcp/explore_source_literal_test.go | 2 +- internal/mcp/find_usages_context_test.go | 2 +- internal/mcp/find_usages_summary_test.go | 2 +- internal/mcp/freshness_rider_test.go | 2 +- internal/mcp/instructions_test.go | 2 +- internal/mcp/path_scope_test.go | 2 +- internal/mcp/return_usage_test.go | 4 +- internal/mcp/scope_resolve_test.go | 6 +- internal/mcp/search_corpus_scope_test.go | 2 +- internal/mcp/tools_contract_bridge_test.go | 2 +- internal/mcp/tools_contracts_filter_test.go | 2 +- internal/mcp/tools_contracts_test.go | 4 +- internal/mcp/tools_core_index_test.go | 6 +- internal/mcp/tools_core_reindex_test.go | 6 +- internal/mcp/tools_fileops_singlerepo_test.go | 4 +- internal/mcp/tools_fileops_worktree_test.go | 2 +- internal/mcp/tools_find_declaration_test.go | 2 +- internal/mcp/tools_multi_worktree_test.go | 2 +- internal/mcp/tools_review_rulepack_test.go | 2 +- .../mcp/tools_search_text_pathsep_test.go | 2 +- internal/mcp/tools_search_text_test.go | 8 +-- internal/mcp/workspace_isolation_test.go | 2 +- internal/mcp/workspace_root_scope_test.go | 2 +- internal/query/engine_corpus_gate_test.go | 2 +- internal/search/null.go | 44 +++++++++++++ internal/search/null_test.go | 61 +++++++++++++++++++ internal/search/search.go | 11 +--- internal/search/swappable.go | 2 +- 82 files changed, 311 insertions(+), 211 deletions(-) create mode 100644 internal/search/null.go create mode 100644 internal/search/null_test.go diff --git a/cmd/gortex/daemon_controller_worktree_test.go b/cmd/gortex/daemon_controller_worktree_test.go index e5d3a2a6a..05d7489cb 100644 --- a/cmd/gortex/daemon_controller_worktree_test.go +++ b/cmd/gortex/daemon_controller_worktree_test.go @@ -67,7 +67,7 @@ func buildWorktreeController(t *testing.T) (*realController, *indexer.MultiIndex g := graph.New() reg := parser.NewRegistry() languages.RegisterAll(reg) - mi := indexer.NewMultiIndexer(g, reg, search.NewBM25(), cm, zap.NewNop()) + mi := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) diff --git a/cmd/gortex/daemon_status_repo_counts_test.go b/cmd/gortex/daemon_status_repo_counts_test.go index 19c601d4b..9cb431a04 100644 --- a/cmd/gortex/daemon_status_repo_counts_test.go +++ b/cmd/gortex/daemon_status_repo_counts_test.go @@ -53,7 +53,7 @@ func TestStatus_TrackedRepoRowFollowsTheLiveGraph(t *testing.T) { g := graph.New() reg := parser.NewRegistry() languages.RegisterAll(reg) - mi := indexer.NewMultiIndexer(g, reg, search.NewBM25(), cm, zap.NewNop()) + mi := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) @@ -129,7 +129,7 @@ func TestStatus_TwoReposEachReportTheirOwnBucket(t *testing.T) { // A is indexed while it is the only configured repo. g := graph.New() - mi1 := indexer.NewMultiIndexer(g, reg, search.NewBM25(), cm, zap.NewNop()) + mi1 := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) _, err = mi1.IndexAll() require.NoError(t, err) var priorA map[string]int64 @@ -141,7 +141,7 @@ func TestStatus_TwoReposEachReportTheirOwnBucket(t *testing.T) { // B is added while the daemon is "down", then a warm restart reconciles // A from its persisted mtimes and tracks B fresh. require.NoError(t, cm.Global().AddRepo(config.RepoEntry{Path: dirB})) - mi2 := indexer.NewMultiIndexer(g, reg, search.NewBM25(), cm, zap.NewNop()) + mi2 := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) _, err = mi2.ReconcileRepoCtx(context.Background(), config.RepoEntry{Path: dirA}, priorA) require.NoError(t, err) _, err = mi2.TrackRepoCtx(context.Background(), config.RepoEntry{Path: dirB}) diff --git a/cmd/gortex/repo_inventory_missing_test.go b/cmd/gortex/repo_inventory_missing_test.go index 3d235ac28..53ff7d2cc 100644 --- a/cmd/gortex/repo_inventory_missing_test.go +++ b/cmd/gortex/repo_inventory_missing_test.go @@ -116,7 +116,7 @@ func newInventoryController(t *testing.T, repos []config.RepoEntry) *realControl g := graph.New() reg := parser.NewRegistry() languages.RegisterAll(reg) - mi := indexer.NewMultiIndexer(g, reg, search.NewBM25(), cm, zap.NewNop()) + mi := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) _, _ = mi.IndexAll() return &realController{graph: g, multiIndexer: mi, configManager: cm, logger: zap.NewNop()} diff --git a/internal/graph/node_id_parity_test.go b/internal/graph/node_id_parity_test.go index 8a74a7ad0..08c801ab6 100644 --- a/internal/graph/node_id_parity_test.go +++ b/internal/graph/node_id_parity_test.go @@ -220,7 +220,7 @@ func indexFixture(t *testing.T, checkoutName string) fixtureResult { require.NoError(t, err) g := graph.New() - mi := indexer.NewMultiIndexer(g, newParityRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := indexer.NewMultiIndexer(g, newParityRegistry(), search.NewNull(), cm, zap.NewNop()) for _, entry := range cm.Global().Repos { _, err := mi.TrackRepoCtx(context.Background(), entry) require.NoError(t, err, "track %s", entry.Name) diff --git a/internal/indexer/batch_transition_coordinator_test.go b/internal/indexer/batch_transition_coordinator_test.go index 8d9f48eca..5370838f4 100644 --- a/internal/indexer/batch_transition_coordinator_test.go +++ b/internal/indexer/batch_transition_coordinator_test.go @@ -26,7 +26,7 @@ func newBatchTransitionTestMulti(g graph.Store) *MultiIndexer { if g == nil { g = graph.New() } - return NewMultiIndexer(g, parser.NewRegistry(), search.NewBM25(), nil, zap.NewNop()) + return NewMultiIndexer(g, parser.NewRegistry(), search.NewNull(), nil, zap.NewNop()) } func requireBatchTransitionBlocked(t *testing.T, done <-chan struct{}) { @@ -51,7 +51,7 @@ func TestIndexAllWorkersDoNotReenterRegistryLock(t *testing.T) { cm.Global().Repos = append(cm.Global().Repos, config.RepoEntry{Path: root, Name: name}) } - mi := NewMultiIndexer(g, reg, search.NewAuto(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) done := make(chan error, 1) go func() { _, err := mi.IndexAll() diff --git a/internal/indexer/contract_bridge_test.go b/internal/indexer/contract_bridge_test.go index 47b10788d..aab216cef 100644 --- a/internal/indexer/contract_bridge_test.go +++ b/internal/indexer/contract_bridge_test.go @@ -197,7 +197,7 @@ func TestContractBridge_TwoRepoIntegration(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newMultiLangRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newMultiLangRegistry(), search.NewNull(), cm, zap.NewNop()) for _, entry := range cm.Global().Repos { _, err := mi.TrackRepoCtx(context.Background(), entry) require.NoError(t, err) @@ -386,7 +386,7 @@ func TestReconcileContractEdges_ConcurrentNoRaceOrTear(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newMultiLangRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newMultiLangRegistry(), search.NewNull(), cm, zap.NewNop()) for _, entry := range cm.Global().Repos { _, err := mi.TrackRepoCtx(context.Background(), entry) require.NoError(t, err) diff --git a/internal/indexer/deferred_enrich_gate_test.go b/internal/indexer/deferred_enrich_gate_test.go index d18f15893..30c0cc26b 100644 --- a/internal/indexer/deferred_enrich_gate_test.go +++ b/internal/indexer/deferred_enrich_gate_test.go @@ -86,7 +86,7 @@ func newEmptyMultiIndexer(t *testing.T, g graph.Store) *MultiIndexer { require.NoError(t, gc.Save()) cm, err := config.NewConfigManager(tmpCfg) require.NoError(t, err) - return NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + return NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) } // TestMultiIndexer_RunDeferredEnrich_GatesUnchangedRepos is the core diff --git a/internal/indexer/direct_mutation_api_test.go b/internal/indexer/direct_mutation_api_test.go index cda79c44d..e36e7e4d0 100644 --- a/internal/indexer/direct_mutation_api_test.go +++ b/internal/indexer/direct_mutation_api_test.go @@ -94,7 +94,7 @@ func TestDirectMutationAPIsUseStableLaneAndCurrentIndexer(t *testing.T) { reg.Register(languages.NewGoExtractor()) cfg := config.Default() cfg.Index.Workers = 1 - mi := NewMultiIndexer(g, reg, search.NewAuto(), nil, zap.NewNop()) + mi := NewMultiIndexer(g, reg, search.NewNull(), nil, zap.NewNop()) current := mi.newPerRepoIndexer(cfg.Index) current.SetRepoPrefix("repo") // Fixture construction mirrors TrackRepo/IndexRepo: the per-repository diff --git a/internal/indexer/external_revert_reindex_test.go b/internal/indexer/external_revert_reindex_test.go index f555ec6db..58082374e 100644 --- a/internal/indexer/external_revert_reindex_test.go +++ b/internal/indexer/external_revert_reindex_test.go @@ -32,7 +32,7 @@ func setupRevertWatcher(t *testing.T) (w *Watcher, g graph.Store, defPath, defID g = newSqliteGraph(t) idx := New(g, newTestRegistry(), config.IndexConfig{Workers: 1}, zap.NewNop()) - idx.search = search.NewBM25() + idx.search = search.NewNull() idx.SetRootPath(dir) _, err := idx.IndexCtx(testCtx(), dir) require.NoError(t, err) diff --git a/internal/indexer/extractor_version_restage_test.go b/internal/indexer/extractor_version_restage_test.go index f18880456..98505206d 100644 --- a/internal/indexer/extractor_version_restage_test.go +++ b/internal/indexer/extractor_version_restage_test.go @@ -50,7 +50,7 @@ func TestIncrementalReindex_NonMerkleExtractorBumpRestagesLanguage(t *testing.T) reg := parser.NewRegistry() reg.Register(languages.NewCSharpExtractor()) idx := New(store, reg, config.IndexConfig{Workers: 1}, zap.NewNop()) - idx.search = search.NewBM25() + idx.search = search.NewNull() idx.SetRootPath(dir) _, err := idx.IndexCtx(testCtx(), dir) require.NoError(t, err) @@ -87,7 +87,7 @@ func TestIncrementalReindex_NonMerkleVersionCurrentNoRestage(t *testing.T) { reg := parser.NewRegistry() reg.Register(languages.NewCSharpExtractor()) idx := New(store, reg, config.IndexConfig{Workers: 1}, zap.NewNop()) - idx.search = search.NewBM25() + idx.search = search.NewNull() idx.SetRootPath(dir) _, err := idx.IndexCtx(testCtx(), dir) require.NoError(t, err) diff --git a/internal/indexer/full_topology_transaction_test.go b/internal/indexer/full_topology_transaction_test.go index 19662908a..93ba34022 100644 --- a/internal/indexer/full_topology_transaction_test.go +++ b/internal/indexer/full_topology_transaction_test.go @@ -27,7 +27,7 @@ func TestIndexCtxStaleHandleUsesCurrentIndexerAndMetadata(t *testing.T) { reg := parser.NewRegistry() reg.Register(languages.NewGoExtractor()) cm := newTestConfigManager(t) - mi := NewMultiIndexer(g, reg, search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) entry := config.RepoEntry{Path: root, Name: "repo"} beforeTrack := reach.BuildCounter() @@ -81,13 +81,13 @@ func TestReconcileTopologyGenerationChangesOnlyForRealMutation(t *testing.T) { reg.Register(languages.NewGoExtractor()) cm := newTestConfigManager(t) entry := config.RepoEntry{Path: root, Name: "repo"} - seed := NewMultiIndexer(g, reg, search.NewBM25(), cm, zap.NewNop()) + seed := NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) _, err := seed.TrackRepo(entry) require.NoError(t, err) prior := seed.GetIndexer("repo").FileMtimes() writeFile(t, filepath.Join(root, "added.go"), "package sample\nfunc Added() {}\n") - changed := NewMultiIndexer(g, reg, search.NewBM25(), cm, zap.NewNop()) + changed := NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) beforeChanged := reach.BuildCounter() changedResult, err := changed.ReconcileRepoCtx(context.Background(), entry, prior) require.NoError(t, err) @@ -95,7 +95,7 @@ func TestReconcileTopologyGenerationChangesOnlyForRealMutation(t *testing.T) { require.Greater(t, reach.BuildCounter(), beforeChanged) unchangedPrior := changed.GetIndexer("repo").FileMtimes() - unchanged := NewMultiIndexer(g, reg, search.NewBM25(), cm, zap.NewNop()) + unchanged := NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) beforeUnchanged := reach.BuildCounter() unchangedResult, err := unchanged.ReconcileRepoCtx(context.Background(), entry, unchangedPrior) require.NoError(t, err) @@ -148,7 +148,7 @@ func TestIndexAllHoldsEveryLaneAndTopologyThroughPublication(t *testing.T) { cm.Global().Repos = append(cm.Global().Repos, config.RepoEntry{Path: root, Name: name}) } store := &armableBlockingAddBatchStore{Store: graph.New()} - mi := NewMultiIndexer(store, reg, search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(store, reg, search.NewNull(), cm, zap.NewNop()) _, err := mi.IndexAll() require.NoError(t, err) oldA := mi.GetIndexer("repo-a") @@ -260,7 +260,7 @@ func TestIndexAllDeduplicatesIdenticalRepositoryEntries(t *testing.T) { cm.Global().Repos = append(cm.Global().Repos, config.RepoEntry{Path: root, Name: "repo"}) } store := &countingIndexAllStore{Store: graph.New()} - mi := NewMultiIndexer(store, reg, search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(store, reg, search.NewNull(), cm, zap.NewNop()) results, err := mi.IndexAll() require.NoError(t, err) require.Len(t, results, 1) @@ -279,7 +279,7 @@ func TestTrackRepoRechecksSamePathAcrossDifferentPrefixLanes(t *testing.T) { writeFile(t, filepath.Join(root, "main.go"), "package sample\nfunc Existing() {}\n") reg := parser.NewRegistry() reg.Register(languages.NewGoExtractor()) - mi := NewMultiIndexer(graph.New(), reg, search.NewBM25(), newTestConfigManager(t), zap.NewNop()) + mi := NewMultiIndexer(graph.New(), reg, search.NewNull(), newTestConfigManager(t), zap.NewNop()) arrived := make(chan struct{}, 2) release := make(chan struct{}) diff --git a/internal/indexer/git_watcher_integration_test.go b/internal/indexer/git_watcher_integration_test.go index a7f70aed9..281018a30 100644 --- a/internal/indexer/git_watcher_integration_test.go +++ b/internal/indexer/git_watcher_integration_test.go @@ -60,7 +60,7 @@ func TestGitWatcher_BranchSwitchReconciles(t *testing.T) { g := graph.New() idx := New(g, newTestRegistry(), config.IndexConfig{Workers: 1}, zap.NewNop()) - idx.search = search.NewBM25() + idx.search = search.NewNull() idx.SetRootPath(repoDir) _, err := idx.IndexCtx(testCtx(), repoDir) require.NoError(t, err) @@ -83,7 +83,7 @@ func TestGitWatcher_BranchSwitchReconciles(t *testing.T) { // started on main" by re-indexing explicitly. g2 := graph.New() idx2 := New(g2, newTestRegistry(), config.IndexConfig{Workers: 1}, zap.NewNop()) - idx2.search = search.NewBM25() + idx2.search = search.NewNull() idx2.SetRootPath(repoDir) _, err = idx2.IndexCtx(testCtx(), repoDir) require.NoError(t, err) @@ -146,7 +146,7 @@ func TestGitWatcher_ReconcileSingleFlight(t *testing.T) { g := graph.New() idx := New(g, newTestRegistry(), config.IndexConfig{Workers: 1}, zap.NewNop()) - idx.search = search.NewBM25() + idx.search = search.NewNull() idx.SetRootPath(repoDir) _, err := idx.IndexCtx(testCtx(), repoDir) require.NoError(t, err) @@ -229,7 +229,7 @@ func TestGitWatcher_NoopWhenHeadUnchanged(t *testing.T) { g := graph.New() idx := New(g, newTestRegistry(), config.IndexConfig{Workers: 1}, zap.NewNop()) - idx.search = search.NewBM25() + idx.search = search.NewNull() idx.SetRootPath(repoDir) _, err := idx.IndexCtx(testCtx(), repoDir) require.NoError(t, err) @@ -308,7 +308,7 @@ func TestGitWatcher_UntrackedFileStaysIndexed(t *testing.T) { g := graph.New() idx := New(g, newTestRegistry(), config.IndexConfig{Workers: 1}, zap.NewNop()) - idx.search = search.NewBM25() + idx.search = search.NewNull() idx.SetRootPath(repoDir) _, err := idx.IndexCtx(testCtx(), repoDir) require.NoError(t, err) diff --git a/internal/indexer/git_watcher_scoped_resolve_test.go b/internal/indexer/git_watcher_scoped_resolve_test.go index 556dae756..5714097e5 100644 --- a/internal/indexer/git_watcher_scoped_resolve_test.go +++ b/internal/indexer/git_watcher_scoped_resolve_test.go @@ -65,7 +65,7 @@ func TestGitWatcher_SmallCommitResolvesIncoming(t *testing.T) { g := graph.New() idx := New(g, newTestRegistry(), config.IndexConfig{Workers: 1}, zap.NewNop()) - idx.search = search.NewBM25() + idx.search = search.NewNull() idx.SetRootPath(repoDir) _, err := idx.IndexCtx(testCtx(), repoDir) require.NoError(t, err) diff --git a/internal/indexer/incremental_resolve_revert_test.go b/internal/indexer/incremental_resolve_revert_test.go index 022e29d52..bd666a3c6 100644 --- a/internal/indexer/incremental_resolve_revert_test.go +++ b/internal/indexer/incremental_resolve_revert_test.go @@ -68,7 +68,7 @@ func TestIncrementalReindex_DoubleCycle_Sqlite_IntraFileCaller(t *testing.T) { g := newSqliteGraph(t) idx := New(g, newTestRegistry(), config.IndexConfig{Workers: 1}, zap.NewNop()) - idx.search = search.NewBM25() + idx.search = search.NewNull() idx.SetRootPath(dir) _, err := idx.IndexCtx(testCtx(), dir) require.NoError(t, err) @@ -113,7 +113,7 @@ func TestReresolveFileScoped_RebindsDroppedIncomingEdges(t *testing.T) { g := newSqliteGraph(t) idx := New(g, newTestRegistry(), config.IndexConfig{Workers: 1}, zap.NewNop()) - idx.search = search.NewBM25() + idx.search = search.NewNull() idx.SetRootPath(dir) _, err := idx.IndexCtx(testCtx(), dir) require.NoError(t, err) diff --git a/internal/indexer/incremental_resolve_test.go b/internal/indexer/incremental_resolve_test.go index e924eff74..0f4a1c830 100644 --- a/internal/indexer/incremental_resolve_test.go +++ b/internal/indexer/incremental_resolve_test.go @@ -71,7 +71,7 @@ func TestIncrementalReindex_PreservesIncomingCallerEdges(t *testing.T) { g := graph.New() idx := New(g, newTestRegistry(), config.IndexConfig{Workers: 1}, zap.NewNop()) - idx.search = search.NewBM25() + idx.search = search.NewNull() idx.SetRootPath(dir) _, err := idx.IndexCtx(testCtx(), dir) require.NoError(t, err) @@ -119,7 +119,7 @@ func TestIncrementalReindex_DoubleCycle_PreservesIncomingTier(t *testing.T) { g := graph.New() idx := New(g, newTestRegistry(), config.IndexConfig{Workers: 1}, zap.NewNop()) - idx.search = search.NewBM25() + idx.search = search.NewNull() idx.SetRootPath(dir) _, err := idx.IndexCtx(testCtx(), dir) require.NoError(t, err) @@ -164,7 +164,7 @@ func TestIncrementalReindex_DeletedDefinition_NoStaleTierOnStub(t *testing.T) { g := graph.New() idx := New(g, newTestRegistry(), config.IndexConfig{Workers: 1}, zap.NewNop()) - idx.search = search.NewBM25() + idx.search = search.NewNull() idx.SetRootPath(dir) _, err := idx.IndexCtx(testCtx(), dir) require.NoError(t, err) @@ -227,7 +227,7 @@ func TestIncrementalReuse_SameFileEdge_KeepsTier(t *testing.T) { g := graph.New() idx := New(g, newTestRegistry(), config.IndexConfig{Workers: 1}, zap.NewNop()) - idx.search = search.NewBM25() + idx.search = search.NewNull() idx.SetRootPath(dir) _, err := idx.IndexCtx(testCtx(), dir) require.NoError(t, err) diff --git a/internal/indexer/incremental_visibility_test.go b/internal/indexer/incremental_visibility_test.go index 97fd3d77b..3047c542c 100644 --- a/internal/indexer/incremental_visibility_test.go +++ b/internal/indexer/incremental_visibility_test.go @@ -51,7 +51,7 @@ func newCSVisIndexer(t *testing.T, dir string) (graph.Store, *Indexer) { reg := parser.NewRegistry() reg.Register(languages.NewCSharpExtractor()) idx := New(g, reg, config.IndexConfig{Workers: 1}, zap.NewNop()) - idx.search = search.NewBM25() + idx.search = search.NewNull() idx.SetRootPath(dir) _, err := idx.IndexCtx(testCtx(), dir) require.NoError(t, err) diff --git a/internal/indexer/indexer.go b/internal/indexer/indexer.go index 8ddef4e6c..a9c740a41 100644 --- a/internal/indexer/indexer.go +++ b/internal/indexer/indexer.go @@ -668,13 +668,15 @@ func (d *vectorSearcherDelegate) SimilarTo(vec []float32, limit int) ([]graph.Ve // in its Swappable on construction. When the underlying store // implements graph.SymbolSearcher (today only store_sqlite), a // thin adapter routes Search calls through the store's native FTS -// — the in-process BM25 build path is bypassed entirely. Otherwise -// falls through to search.NewAuto's in-memory BM25 index. +// — the in-process BM25 build path is bypassed entirely. Every other +// store gets the null backend: it indexes nothing and reports an +// empty corpus, so the query engine answers from its own substring +// scan rather than from a second, in-process copy of the corpus. func initialSearchBackend(g graph.Store) search.Backend { if s, ok := g.(graph.SymbolSearcher); ok { return search.NewSymbolSearcherBackend(s) } - return search.NewAuto() + return search.NewNull() } // isSymbolSearcherBackend reports whether the swappable's currently diff --git a/internal/indexer/multi_case_identity_test.go b/internal/indexer/multi_case_identity_test.go index 370fc2724..4ee9829f0 100644 --- a/internal/indexer/multi_case_identity_test.go +++ b/internal/indexer/multi_case_identity_test.go @@ -103,7 +103,7 @@ func TestScopeForCWD_CaseMismatchedWorkspaceRoot(t *testing.T) { require.NoError(t, gc.Save()) cm, err := config.NewConfigManager(tmpCfg) require.NoError(t, err) - mi := NewMultiIndexer(graph.New(), newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(graph.New(), newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) diff --git a/internal/indexer/multi_cold_orchestration_test.go b/internal/indexer/multi_cold_orchestration_test.go index 2ca5cfe9e..db827db35 100644 --- a/internal/indexer/multi_cold_orchestration_test.go +++ b/internal/indexer/multi_cold_orchestration_test.go @@ -48,7 +48,7 @@ func coldOrchestrationRepos(t *testing.T, count int) []config.RepoEntry { } func newColdOrchestrationMulti(store graph.Store, logger *zap.Logger) *MultiIndexer { - return NewMultiIndexer(store, coldOrchestrationRegistry(), search.NewAuto(), nil, logger) + return NewMultiIndexer(store, coldOrchestrationRegistry(), search.NewNull(), nil, logger) } type coldOrchestrationProbeStore struct { diff --git a/internal/indexer/multi_contract_edges_test.go b/internal/indexer/multi_contract_edges_test.go index 9c7177922..fe42c857e 100644 --- a/internal/indexer/multi_contract_edges_test.go +++ b/internal/indexer/multi_contract_edges_test.go @@ -111,7 +111,7 @@ func TestReconcileContractEdges_BridgesConsumerToProvider(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) for _, entry := range cm.Global().Repos { _, err := mi.TrackRepoCtx(context.Background(), entry) @@ -205,7 +205,7 @@ func TestReconcileContractEdges_TemplateLiteralConsumer(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newMultiLangRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newMultiLangRegistry(), search.NewNull(), cm, zap.NewNop()) for _, entry := range cm.Global().Repos { _, err := mi.TrackRepoCtx(context.Background(), entry) require.NoError(t, err, "track %s", entry.Name) @@ -287,7 +287,7 @@ func TestReconcileContractEdges_DartConsumer(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newMultiLangRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newMultiLangRegistry(), search.NewNull(), cm, zap.NewNop()) for _, entry := range cm.Global().Repos { _, err := mi.TrackRepoCtx(context.Background(), entry) require.NoError(t, err, "track %s", entry.Name) @@ -447,7 +447,7 @@ func TestInlineWrappers_TuckShape(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newMultiLangRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newMultiLangRegistry(), search.NewNull(), cm, zap.NewNop()) for _, entry := range cm.Global().Repos { _, err := mi.TrackRepoCtx(context.Background(), entry) require.NoError(t, err, "track %s", entry.Name) @@ -553,7 +553,7 @@ func TestReconcileContractEdges_TopicBridge(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) for _, entry := range cm.Global().Repos { _, err := mi.TrackRepoCtx(context.Background(), entry) require.NoError(t, err) @@ -607,7 +607,7 @@ func TestEnvConsumer_SymbolIDSet(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) for _, entry := range cm.Global().Repos { _, err := mi.TrackRepoCtx(context.Background(), entry) require.NoError(t, err) @@ -724,7 +724,7 @@ func TestReconcileContractEdges_GRPCBridge(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newMultiLangRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newMultiLangRegistry(), search.NewNull(), cm, zap.NewNop()) for _, entry := range cm.Global().Repos { _, err := mi.TrackRepoCtx(context.Background(), entry) require.NoError(t, err) @@ -840,7 +840,7 @@ func TestReconcileContractEdges_OpenAPIBridge(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newMultiLangRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newMultiLangRegistry(), search.NewNull(), cm, zap.NewNop()) for _, entry := range cm.Global().Repos { _, err := mi.TrackRepoCtx(context.Background(), entry) require.NoError(t, err) @@ -911,7 +911,7 @@ func TestReconcileContractEdges_PurgesStaleOnUntrack(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) for _, entry := range cm.Global().Repos { _, err := mi.TrackRepoCtx(context.Background(), entry) require.NoError(t, err) diff --git a/internal/indexer/multi_global_passes_test.go b/internal/indexer/multi_global_passes_test.go index c2869d845..02328a35b 100644 --- a/internal/indexer/multi_global_passes_test.go +++ b/internal/indexer/multi_global_passes_test.go @@ -82,7 +82,7 @@ func TestMultiIndexer_IndexAll_GlobalPassesProduceEdges(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) results, err := mi.IndexAll() require.NoError(t, err) @@ -134,7 +134,7 @@ func TestMultiIndexer_GlobalGraphPassPipeline_Idempotent(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) @@ -173,7 +173,7 @@ func TestMultiIndexer_BeginEndBatch_DefersGlobalPasses(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) mi.BeginBatch() @@ -213,7 +213,7 @@ func TestMultiIndexer_TrackRepoCtx_NoBatch_RunsGlobalPassesInline(t *testing.T) require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = mi.TrackRepoCtx(context.Background(), config.RepoEntry{Path: repoA, Name: "repo-a"}) require.NoError(t, err) diff --git a/internal/indexer/multi_node_id_test.go b/internal/indexer/multi_node_id_test.go index 3ede32256..3d2bc71c6 100644 --- a/internal/indexer/multi_node_id_test.go +++ b/internal/indexer/multi_node_id_test.go @@ -87,7 +87,7 @@ func TestMultiRepo_ResolvesCallEdges(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) for _, entry := range cm.Global().Repos { _, err := mi.TrackRepoCtx(context.Background(), entry) require.NoError(t, err, "track %s", entry.Name) @@ -164,7 +164,7 @@ func TestTrackRepoCtx_FirstOfManyStillGetsPrefix(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) // Simulate warmupDaemonState's loop: TrackRepoCtx each config'd repo // in order. The first call is the one that used to skip prefixing. @@ -275,7 +275,7 @@ rules: g := graph.New() registry := newTestRegistry() registry.Register(languages.NewYAMLExtractor()) - mi := NewMultiIndexer(g, registry, search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, registry, search.NewNull(), cm, zap.NewNop()) for _, entry := range cm.Global().Repos { _, err := mi.TrackRepoCtx(context.Background(), entry) require.NoError(t, err, "tracking %s", entry.Name) diff --git a/internal/indexer/multi_reconcile_sqlite_test.go b/internal/indexer/multi_reconcile_sqlite_test.go index 89760ee8d..6594ab352 100644 --- a/internal/indexer/multi_reconcile_sqlite_test.go +++ b/internal/indexer/multi_reconcile_sqlite_test.go @@ -41,7 +41,7 @@ func TestReconcileRepoCtx_Sqlite_FullRetrackFlag(t *testing.T) { // First "daemon run": index the repo on the disk-backed store and // capture mtimes as if we were writing a warm-restart snapshot. - mi := NewMultiIndexer(graph.Store(s), newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(graph.Store(s), newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) @@ -54,7 +54,7 @@ func TestReconcileRepoCtx_Sqlite_FullRetrackFlag(t *testing.T) { // down the whole-repo re-track branch. writeFile(t, filepath.Join(repoPath, "b.go"), "package main\nfunc Beta() {}\n") - mi2 := NewMultiIndexer(graph.Store(s), newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi2 := NewMultiIndexer(graph.Store(s), newTestRegistry(), search.NewNull(), cm, zap.NewNop()) result, err := mi2.ReconcileRepoCtx(context.Background(), config.RepoEntry{Path: repoPath, Name: "repo"}, priorMtimes) require.NoError(t, err) require.NotNil(t, result) @@ -70,7 +70,7 @@ func TestReconcileRepoCtx_Sqlite_FullRetrackFlag(t *testing.T) { require.NotNil(t, meta2) unchangedMtimes := mi2.FileMtimes("repo") - mi3 := NewMultiIndexer(graph.Store(s), newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi3 := NewMultiIndexer(graph.Store(s), newTestRegistry(), search.NewNull(), cm, zap.NewNop()) result2, err := mi3.ReconcileRepoCtx(context.Background(), config.RepoEntry{Path: repoPath, Name: "repo"}, unchangedMtimes) require.NoError(t, err) require.NotNil(t, result2) diff --git a/internal/indexer/multi_reconcile_test.go b/internal/indexer/multi_reconcile_test.go index e58c9a75f..6a4cc1b45 100644 --- a/internal/indexer/multi_reconcile_test.go +++ b/internal/indexer/multi_reconcile_test.go @@ -48,7 +48,7 @@ func TestReconcileRepoCtx_EvictsOfflineDeletions(t *testing.T) { require.NoError(t, err) t.Cleanup(func() { _ = s.Close() }) g := graph.Store(s) - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) @@ -66,7 +66,7 @@ func TestReconcileRepoCtx_EvictsOfflineDeletions(t *testing.T) { // Second "daemon run": fresh MultiIndexer, graph already populated // from the "snapshot", reconcile with prior mtimes. - mi2 := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi2 := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = mi2.ReconcileRepoCtx(context.Background(), config.RepoEntry{Path: repoPath, Name: "repo"}, priorMtimes) require.NoError(t, err) @@ -97,7 +97,7 @@ func TestReconcileRepoCtx_DoesNotDuplicateUnchanged(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) @@ -105,7 +105,7 @@ func TestReconcileRepoCtx_DoesNotDuplicateUnchanged(t *testing.T) { priorMtimes := mi.FileMtimes("repo") // Simulate restart: fresh MultiIndexer on the same graph, reconcile. - mi2 := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi2 := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = mi2.ReconcileRepoCtx(context.Background(), config.RepoEntry{Path: repoPath, Name: "repo"}, priorMtimes) require.NoError(t, err) @@ -140,7 +140,7 @@ func TestReconcileRepoCtx_RunsDerivedPassesForOfflineChange(t *testing.T) { require.NoError(t, err) t.Cleanup(func() { _ = s.Close() }) g := graph.Store(s) - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) priorMtimes := mi.FileMtimes("repo") @@ -150,7 +150,7 @@ import "os/exec" func Run() error { return exec.Command("true").Run() } `) - mi2 := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi2 := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) result, err := mi2.ReconcileRepoCtx( context.Background(), config.RepoEntry{Path: repoPath, Name: "repo"}, priorMtimes, ) @@ -178,7 +178,7 @@ func TestReconcileAll_RunsDerivedPassesForMissedChange(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) @@ -215,7 +215,7 @@ func TestReconcileAllCtx_PreservesExistingBatchFlags(t *testing.T) { cm, err := config.NewConfigManager(cfgPath) require.NoError(t, err) - mi := NewMultiIndexer(graph.New(), newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(graph.New(), newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) @@ -268,7 +268,7 @@ func TestReconcileAll_CatchesJanitorTargets(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) diff --git a/internal/indexer/multi_refresh_config_test.go b/internal/indexer/multi_refresh_config_test.go index b42a70c0f..cc2cc7913 100644 --- a/internal/indexer/multi_refresh_config_test.go +++ b/internal/indexer/multi_refresh_config_test.go @@ -35,7 +35,7 @@ func Dropped() {} require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) require.NotEmpty(t, g.FindNodesByName("Dropped"), "precondition: drop.go is indexed") @@ -58,6 +58,6 @@ func Dropped() {} } func TestMultiIndexer_RefreshRepoConfigs_NoConfigManager(t *testing.T) { - mi := NewMultiIndexer(graph.New(), newTestRegistry(), search.NewBM25(), nil, zap.NewNop()) + mi := NewMultiIndexer(graph.New(), newTestRegistry(), search.NewNull(), nil, zap.NewNop()) assert.Equal(t, 0, mi.RefreshRepoConfigs()) } diff --git a/internal/indexer/multi_scoped_test.go b/internal/indexer/multi_scoped_test.go index 900b86b84..fc0281449 100644 --- a/internal/indexer/multi_scoped_test.go +++ b/internal/indexer/multi_scoped_test.go @@ -40,7 +40,7 @@ func TestMultiIndexer_IndexScoped_WorkspaceFromGortexYAML(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) results, err := mi.IndexScoped("alpha", "") require.NoError(t, err) @@ -72,7 +72,7 @@ func TestMultiIndexer_IndexScoped_RepoEntryOverride(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) // Override wins over .gortex.yaml: "upstream" matches nothing. _, err = mi.IndexScoped("upstream", "") @@ -105,7 +105,7 @@ func TestMultiIndexer_IndexScoped_FallsBackToRepoPrefix(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) results, err := mi.IndexScoped("repo-b", "") require.NoError(t, err) @@ -139,7 +139,7 @@ func TestMultiIndexer_IndexScoped_ProjectNarrowsInsideWorkspace(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) results, err := mi.IndexScoped("shared", "api") require.NoError(t, err) @@ -164,7 +164,7 @@ func TestMultiIndexer_IndexScoped_NoMatchErrors(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexScoped("nonexistent", "") require.Error(t, err) @@ -195,7 +195,7 @@ func TestMultiIndexer_IndexScoped_EmptyFiltersEqualsIndexAll(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) results, err := mi.IndexScoped("", "") require.NoError(t, err) diff --git a/internal/indexer/multi_singlerepo_resolve_test.go b/internal/indexer/multi_singlerepo_resolve_test.go index f466ffe23..fdab98bf3 100644 --- a/internal/indexer/multi_singlerepo_resolve_test.go +++ b/internal/indexer/multi_singlerepo_resolve_test.go @@ -29,7 +29,7 @@ func indexSingleRepoForTest(t *testing.T) (*MultiIndexer, string) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) return mi, dir @@ -54,7 +54,7 @@ func indexTwoReposForTest(t *testing.T) (*MultiIndexer, string, string) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) return mi, repoA, repoB @@ -135,7 +135,7 @@ func TestMultiIndexer_RunPreEnrichResolve_BindsInboundCrossRepoEdge(t *testing.T inbound := &graph.Edge{From: "repo-a/a.go::Caller", To: "unresolved::Foo", Kind: graph.EdgeCalls, FilePath: "repo-a/a.go", Line: 5} g.AddEdge(inbound) - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) // Scoped warm restart: only the provider repo-b re-indexed. require.NoError(t, mi.RunPreEnrichResolve(context.Background(), map[string]struct{}{"repo-b": {}}, nil)) diff --git a/internal/indexer/multi_test.go b/internal/indexer/multi_test.go index 76326dee9..8eec56345 100644 --- a/internal/indexer/multi_test.go +++ b/internal/indexer/multi_test.go @@ -51,7 +51,7 @@ func Hello() {} func TestNewMultiIndexer(t *testing.T) { g := graph.New() reg := newTestRegistry() - s := search.NewBM25() + s := search.NewNull() cm := newTestConfigManager(t) mi := NewMultiIndexer(g, reg, s, cm, zap.NewNop()) @@ -76,7 +76,7 @@ func TestMultiIndexer_IndexAll_SingleRepo(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) results, err := mi.IndexAll() require.NoError(t, err) @@ -122,7 +122,7 @@ func TestMultiIndexer_IndexAll_MultiRepo(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) results, err := mi.IndexAll() require.NoError(t, err) @@ -160,7 +160,7 @@ func TestMultiIndexer_IndexAll_SingleRepoLoadsWorkspaceExclude(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) @@ -196,7 +196,7 @@ func TestMultiIndexer_IndexAll_MultiRepoLoadsWorkspaceExclude(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) @@ -230,7 +230,7 @@ func TestMultiIndexer_IndexRepo(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) @@ -258,7 +258,7 @@ func TestMultiIndexer_IndexRepo(t *testing.T) { func TestMultiIndexer_IndexRepo_NotFound(t *testing.T) { g := graph.New() cm := newTestConfigManager(t) - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err := mi.IndexRepo("nonexistent") assert.Error(t, err) @@ -270,7 +270,7 @@ func TestMultiIndexer_TrackRepo(t *testing.T) { g := graph.New() cm := newTestConfigManager(t) - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) result, err := mi.TrackRepo(config.RepoEntry{Path: dir, Name: "tracked"}) require.NoError(t, err) @@ -369,7 +369,7 @@ func TestMultiIndexer_TrackRepo_EmptyAfterPopulated(t *testing.T) { g := graph.New() cm := newTestConfigManager(t) - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) first, err := mi.TrackRepo(config.RepoEntry{Path: populated, Name: "populated"}) require.NoError(t, err) @@ -408,7 +408,7 @@ func TestMultiIndexer_TrackRepo_EmptyAfterPopulated(t *testing.T) { func TestMultiIndexer_TrackRepo_InvalidPath(t *testing.T) { g := graph.New() cm := newTestConfigManager(t) - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err := mi.TrackRepo(config.RepoEntry{Path: "/nonexistent/path/xyz"}) assert.Error(t, err) @@ -421,7 +421,7 @@ func TestMultiIndexer_TrackRepo_NotADirectory(t *testing.T) { g := graph.New() cm := newTestConfigManager(t) - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err := mi.TrackRepo(config.RepoEntry{Path: tmpFile}) assert.Error(t, err) @@ -446,7 +446,7 @@ func TestMultiIndexer_UntrackRepo(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) @@ -470,7 +470,7 @@ func TestMultiIndexer_UntrackRepo(t *testing.T) { func TestMultiIndexer_UntrackRepo_NotFound(t *testing.T) { g := graph.New() cm := newTestConfigManager(t) - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) nodesRemoved, edgesRemoved := mi.UntrackRepo("nonexistent") assert.Equal(t, 0, nodesRemoved) @@ -495,7 +495,7 @@ func TestMultiIndexer_RepoForFile(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) @@ -522,7 +522,7 @@ func TestMultiIndexer_GetIndexer(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) @@ -535,7 +535,7 @@ func TestMultiIndexer_GetIndexer(t *testing.T) { func TestMultiIndexer_IndexAll_EmptyRepos(t *testing.T) { cm := newTestConfigManager(t) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) results, err := mi.IndexAll() require.NoError(t, err) @@ -589,7 +589,7 @@ func TestPropertyNodeIDFormat(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) results, err := mi.IndexAll() require.NoError(t, err) @@ -622,7 +622,7 @@ func TestPropertyNodeIDFormat(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) results, err := mi.IndexAll() require.NoError(t, err) @@ -697,7 +697,7 @@ func TestPropertyReindexIsolation(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) @@ -782,7 +782,7 @@ func TestLoneRepo_ConfigCompat(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) @@ -853,7 +853,7 @@ guards: require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) @@ -896,7 +896,7 @@ func TestGrowWorkspaceFromOneRepoToTwo(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) @@ -934,7 +934,7 @@ func TestGrowWorkspaceFromOneRepoToTwo(t *testing.T) { require.NoError(t, err) g2 := graph.New() - mi2 := NewMultiIndexer(g2, newTestRegistry(), search.NewBM25(), cm2, zap.NewNop()) + mi2 := NewMultiIndexer(g2, newTestRegistry(), search.NewNull(), cm2, zap.NewNop()) results, err := mi2.IndexAll() require.NoError(t, err) @@ -974,7 +974,7 @@ func TestScopedReindexPreservesRepoMetadataFileCount(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) diff --git a/internal/indexer/multi_topic_edges_test.go b/internal/indexer/multi_topic_edges_test.go index 66b06505e..49f31a3de 100644 --- a/internal/indexer/multi_topic_edges_test.go +++ b/internal/indexer/multi_topic_edges_test.go @@ -74,7 +74,7 @@ func TestReconcileContractEdges_TopicEdges_KafkaPair(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) for _, entry := range cm.Global().Repos { _, err := mi.TrackRepoCtx(context.Background(), entry) require.NoError(t, err) @@ -141,7 +141,7 @@ func TestReconcileContractEdges_TopicEdges_CrossBrokerIsolation(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) for _, entry := range cm.Global().Repos { _, err := mi.TrackRepoCtx(context.Background(), entry) require.NoError(t, err) @@ -188,7 +188,7 @@ func TestReconcileContractEdges_TopicEdges_MultiConsumerFanout(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) for _, entry := range cm.Global().Repos { _, err := mi.TrackRepoCtx(context.Background(), entry) require.NoError(t, err) @@ -249,7 +249,7 @@ func TestReconcileContractEdges_TopicEdges_CrossWorkspaceIsolation(t *testing.T) require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) for _, entry := range cm.Global().Repos { _, err := mi.TrackRepoCtx(context.Background(), entry) require.NoError(t, err) diff --git a/internal/indexer/multi_transition_test.go b/internal/indexer/multi_transition_test.go index 287adfca1..c4263cdff 100644 --- a/internal/indexer/multi_transition_test.go +++ b/internal/indexer/multi_transition_test.go @@ -28,7 +28,7 @@ func TestTrackRepo_FirstRepoIsPrefixedFromTheStart(t *testing.T) { cm := newTestConfigManager(t) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) ctx := context.Background() _, err := mi.TrackRepoCtx(ctx, config.RepoEntry{Path: dirA, Name: "repo-a"}) @@ -58,7 +58,7 @@ func TestUntrackRepo_EvictsTheReposNodes(t *testing.T) { cm := newTestConfigManager(t) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err := mi.TrackRepoCtx(context.Background(), config.RepoEntry{Path: dir, Name: "repo-a"}) require.NoError(t, err) @@ -92,7 +92,7 @@ func TestRepoRoot_EmptyPrefixResolvesTheSoleRepo(t *testing.T) { cm := newTestConfigManager(t) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) ctx := context.Background() _, err := mi.TrackRepoCtx(ctx, config.RepoEntry{Path: dirA, Name: "repo-a"}) @@ -135,7 +135,7 @@ func TestResolveFilePath_RepoNameMatchingOwnSubdirectory(t *testing.T) { cm := newTestConfigManager(t) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err := mi.TrackRepoCtx(context.Background(), config.RepoEntry{Path: dir, Name: "api"}) require.NoError(t, err) diff --git a/internal/indexer/multi_watcher_test.go b/internal/indexer/multi_watcher_test.go index 4fc8dec03..28dcf593b 100644 --- a/internal/indexer/multi_watcher_test.go +++ b/internal/indexer/multi_watcher_test.go @@ -50,7 +50,7 @@ func HelloB() {} {Path: repoBDir, Name: "repo-b"}, } - mi := NewMultiIndexer(g, reg, search.NewAuto(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) _, err := mi.IndexAll() require.NoError(t, err) @@ -111,7 +111,7 @@ func Hello() {} {Path: repoDir, Name: "valid"}, } - mi := NewMultiIndexer(g, reg, search.NewAuto(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) _, err := mi.IndexAll() require.NoError(t, err) diff --git a/internal/indexer/poller_test.go b/internal/indexer/poller_test.go index c91009417..28a555aa8 100644 --- a/internal/indexer/poller_test.go +++ b/internal/indexer/poller_test.go @@ -101,7 +101,7 @@ func TestPoller_DetectsFilesystemChangeMissedByFsnotify(t *testing.T) { g := graph.New() idx := newTestIndexer(g) - idx.search = search.NewBM25() + idx.search = search.NewNull() idx.SetRootPath(dir) _, err := idx.Index(dir) require.NoError(t, err) @@ -138,7 +138,7 @@ func TestPoller_DetectsDeletedFileMissedByFsnotify(t *testing.T) { g := graph.New() idx := newTestIndexer(g) - idx.search = search.NewBM25() + idx.search = search.NewNull() idx.SetRootPath(dir) _, err := idx.Index(dir) require.NoError(t, err) @@ -184,7 +184,7 @@ func TestPoller_DetectsGitHeadMoveMissedByFsnotify(t *testing.T) { g := graph.New() idx := New(g, newTestRegistry(), config.IndexConfig{Workers: 1}, zap.NewNop()) - idx.search = search.NewBM25() + idx.search = search.NewNull() idx.SetRootPath(repoDir) _, err := idx.IndexCtx(testCtx(), repoDir) require.NoError(t, err) @@ -345,7 +345,7 @@ func TestPoller_SweepHookReportsWork(t *testing.T) { g := graph.New() idx := newTestIndexer(g) - idx.search = search.NewBM25() + idx.search = search.NewNull() idx.SetRootPath(dir) _, err := idx.Index(dir) require.NoError(t, err) @@ -461,7 +461,7 @@ func TestPoller_SweepUnionsGitAndFilesystemPathsIntoOneBatch(t *testing.T) { g := graph.New() idx := New(g, newTestRegistry(), config.IndexConfig{Workers: 1}, zap.NewNop()) - idx.search = search.NewBM25() + idx.search = search.NewNull() idx.SetRootPath(repoDir) _, err := idx.IndexCtx(testCtx(), repoDir) require.NoError(t, err) @@ -519,7 +519,7 @@ func TestPoller_GitSHAAdvancesOnlyAfterSuccessfulBatch(t *testing.T) { g := graph.New() idx := New(g, newTestRegistry(), config.IndexConfig{Workers: 1}, zap.NewNop()) - idx.search = search.NewBM25() + idx.search = search.NewNull() idx.SetRootPath(repoDir) _, err := idx.IndexCtx(testCtx(), repoDir) require.NoError(t, err) diff --git a/internal/indexer/pre_enrich_resolve_hooks_test.go b/internal/indexer/pre_enrich_resolve_hooks_test.go index 091786c44..9bbf2d075 100644 --- a/internal/indexer/pre_enrich_resolve_hooks_test.go +++ b/internal/indexer/pre_enrich_resolve_hooks_test.go @@ -36,7 +36,7 @@ func TestRunPreEnrichResolveFiresComputeDoneHook(t *testing.T) { inbound := &graph.Edge{From: "repo-a/a.go::Caller", To: "unresolved::Foo", Kind: graph.EdgeCalls, FilePath: "repo-a/a.go", Line: 5} g.AddEdge(inbound) - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) var order []string require.NoError(t, mi.RunPreEnrichResolve(context.Background(), nil, diff --git a/internal/indexer/pre_enrich_slug_backfill_test.go b/internal/indexer/pre_enrich_slug_backfill_test.go index 8a42fc711..7ca3cf097 100644 --- a/internal/indexer/pre_enrich_slug_backfill_test.go +++ b/internal/indexer/pre_enrich_slug_backfill_test.go @@ -106,7 +106,7 @@ func newPreEnrichSlugFixture(t *testing.T, stamped bool) (*MultiIndexer, *graph. call := &graph.Edge{From: "a/a.go::Call", To: "unresolved::Serve", Kind: graph.EdgeCalls, FilePath: "a/a.go", Line: 3} base.AddEdge(call) store := base - mi := NewMultiIndexer(store, newTestRegistry(), search.NewBM25(), configMgr, zap.NewNop()) + mi := NewMultiIndexer(store, newTestRegistry(), search.NewNull(), configMgr, zap.NewNop()) mi.repos = map[string]*RepoMetadata{ "a": {RepoPrefix: "a", RootPath: roots["a"]}, "b": {RepoPrefix: "b", RootPath: roots["b"]}, diff --git a/internal/indexer/realtime_reliability_test.go b/internal/indexer/realtime_reliability_test.go index 331c3fe45..31e7aa9b2 100644 --- a/internal/indexer/realtime_reliability_test.go +++ b/internal/indexer/realtime_reliability_test.go @@ -254,7 +254,7 @@ func TestPollGitHead_DiffFailureRetriesRange(t *testing.T) { g := graph.New() idx := New(g, newTestRegistry(), config.IndexConfig{Workers: 1}, zap.NewNop()) - idx.search = search.NewBM25() + idx.search = search.NewNull() idx.SetRootPath(repoDir) _, err = idx.IndexCtx(testCtx(), repoDir) require.NoError(t, err) @@ -395,7 +395,7 @@ func TestWatcher_OverflowReconcileIndexesMissedFile(t *testing.T) { reg.Register(ext) g := graph.New() idx := New(g, reg, config.IndexConfig{Workers: 1}, zap.NewNop()) - idx.search = search.NewBM25() + idx.search = search.NewNull() idx.SetRootPath(dir) ext.setFail(false) @@ -449,7 +449,7 @@ func TestWatcher_NewSubdirScanIndexesPreWatchFile(t *testing.T) { reg.Register(ext) g := graph.New() idx := New(g, reg, config.IndexConfig{Workers: 1}, zap.NewNop()) - idx.search = search.NewBM25() + idx.search = search.NewNull() idx.SetRootPath(dir) ext.setFail(false) @@ -562,7 +562,7 @@ func TestWatcher_PatchPanicRecoveredNotCrash(t *testing.T) { reg.Register(ext) store := &panicOnReadStore{Store: graph.New()} idx := New(store, reg, config.IndexConfig{Workers: 1}, zap.NewNop()) - idx.search = search.NewBM25() + idx.search = search.NewNull() dir := t.TempDir() idx.SetRootPath(dir) path := filepath.Join(dir, "main.fk") diff --git a/internal/indexer/reconcile_clean_census_manifest_test.go b/internal/indexer/reconcile_clean_census_manifest_test.go index ab09f3fb7..b53ee59df 100644 --- a/internal/indexer/reconcile_clean_census_manifest_test.go +++ b/internal/indexer/reconcile_clean_census_manifest_test.go @@ -27,7 +27,7 @@ func TestReconcileRepoCtxRoutesManifestOnlyChurnScopedAndConverges(t *testing.T) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, store.Close()) }) - seed := NewMultiIndexer(graph.Store(store), newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + seed := NewMultiIndexer(graph.Store(store), newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = seed.IndexAll() require.NoError(t, err) prior := seed.GetIndexer("repo").FileMtimes() @@ -35,7 +35,7 @@ func TestReconcileRepoCtxRoutesManifestOnlyChurnScopedAndConverges(t *testing.T) assert.False(t, manifestTrackedByFullIndex) core, logs := observer.New(zap.DebugLevel) - firstRestart := NewMultiIndexer(graph.Store(store), newTestRegistry(), search.NewBM25(), cm, zap.New(core)) + firstRestart := NewMultiIndexer(graph.Store(store), newTestRegistry(), search.NewNull(), cm, zap.New(core)) result, err := firstRestart.ReconcileRepoCtx(t.Context(), entry, prior) require.NoError(t, err) require.NotNil(t, result) @@ -49,7 +49,7 @@ func TestReconcileRepoCtxRoutesManifestOnlyChurnScopedAndConverges(t *testing.T) assert.True(t, manifestTracked) core, logs = observer.New(zap.DebugLevel) - secondRestart := NewMultiIndexer(graph.Store(store), newTestRegistry(), search.NewBM25(), cm, zap.New(core)) + secondRestart := NewMultiIndexer(graph.Store(store), newTestRegistry(), search.NewNull(), cm, zap.New(core)) result, err = secondRestart.ReconcileRepoCtx(t.Context(), entry, convergedMtimes) require.NoError(t, err) require.NotNil(t, result) diff --git a/internal/indexer/reconcile_clean_census_merkle_route_test.go b/internal/indexer/reconcile_clean_census_merkle_route_test.go index a2611b7ce..96c581c8f 100644 --- a/internal/indexer/reconcile_clean_census_merkle_route_test.go +++ b/internal/indexer/reconcile_clean_census_merkle_route_test.go @@ -29,7 +29,7 @@ func TestReconcileRepoCtxKeepsMerkleFallbackForCleanMtimeCensus(t *testing.T) { require.NoError(t, err) t.Cleanup(func() { require.NoError(t, store.Close()) }) - seed := NewMultiIndexer(graph.Store(store), newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + seed := NewMultiIndexer(graph.Store(store), newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = seed.IndexAll() require.NoError(t, err) prior := seed.GetIndexer("repo").FileMtimes() @@ -45,7 +45,7 @@ func TestReconcileRepoCtxKeepsMerkleFallbackForCleanMtimeCensus(t *testing.T) { // remain disabled for every Merkle-enabled repository. require.NoError(t, os.Remove(merkleTreeFile(root))) core, logs := observer.New(zap.DebugLevel) - restarted := NewMultiIndexer(graph.Store(store), newTestRegistry(), search.NewBM25(), cm, zap.New(core)) + restarted := NewMultiIndexer(graph.Store(store), newTestRegistry(), search.NewNull(), cm, zap.New(core)) result, err := restarted.ReconcileRepoCtx(t.Context(), entry, prior) require.NoError(t, err) require.NotNil(t, result) diff --git a/internal/indexer/reconcile_clean_census_test.go b/internal/indexer/reconcile_clean_census_test.go index 3b8b48f74..033e1ff37 100644 --- a/internal/indexer/reconcile_clean_census_test.go +++ b/internal/indexer/reconcile_clean_census_test.go @@ -171,14 +171,14 @@ func TestReconcileRepoCtxUsesCleanCensusNoOp(t *testing.T) { require.NoError(t, err) t.Cleanup(func() { require.NoError(t, store.Close()) }) - seed := NewMultiIndexer(graph.Store(store), newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + seed := NewMultiIndexer(graph.Store(store), newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = seed.IndexAll() require.NoError(t, err) prior := seed.GetIndexer("repo").FileMtimes() before := store.Stats() core, logs := observer.New(zap.DebugLevel) - restarted := NewMultiIndexer(graph.Store(store), newTestRegistry(), search.NewBM25(), cm, zap.New(core)) + restarted := NewMultiIndexer(graph.Store(store), newTestRegistry(), search.NewNull(), cm, zap.New(core)) result, err := restarted.ReconcileRepoCtx(t.Context(), entry, prior) require.NoError(t, err) require.NotNil(t, result) diff --git a/internal/indexer/reconcile_scoped_routing_test.go b/internal/indexer/reconcile_scoped_routing_test.go index ff20d7be6..3a34ef1d4 100644 --- a/internal/indexer/reconcile_scoped_routing_test.go +++ b/internal/indexer/reconcile_scoped_routing_test.go @@ -94,7 +94,7 @@ func TestReconcileRepoCtx_ScopedEqualsFullIndex(t *testing.T) { require.NoError(t, err) t.Cleanup(func() { _ = s.Close() }) - mi := NewMultiIndexer(graph.Store(s), newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(graph.Store(s), newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) priorMtimes := mi.FileMtimes("repo") @@ -108,7 +108,7 @@ func TestReconcileRepoCtx_ScopedEqualsFullIndex(t *testing.T) { // Second "daemon run": a fresh MultiIndexer over the same persisted // store reconciles from the snapshot. - mi2 := NewMultiIndexer(graph.Store(s), newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi2 := NewMultiIndexer(graph.Store(s), newTestRegistry(), search.NewNull(), cm, zap.NewNop()) res, err := mi2.ReconcileRepoCtx(context.Background(), config.RepoEntry{Path: repoPath, Name: "repo"}, priorMtimes) require.NoError(t, err) require.NotNil(t, res) @@ -134,7 +134,7 @@ func TestReconcileRepoCtx_ScopedEqualsFullIndex(t *testing.T) { s2, err := store_sqlite.Open(filepath.Join(t.TempDir(), "golden.sqlite")) require.NoError(t, err) t.Cleanup(func() { _ = s2.Close() }) - miGold := NewMultiIndexer(graph.Store(s2), newTestRegistry(), search.NewBM25(), cm2, zap.NewNop()) + miGold := NewMultiIndexer(graph.Store(s2), newTestRegistry(), search.NewNull(), cm2, zap.NewNop()) _, err = miGold.IndexAll() require.NoError(t, err) @@ -166,7 +166,7 @@ func TestReconcileRepoCtx_Routing(t *testing.T) { require.NoError(t, err) t.Cleanup(func() { _ = s.Close() }) - mi := NewMultiIndexer(graph.Store(s), newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(graph.Store(s), newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) return s, cm, mi.FileMtimes("repo") @@ -174,7 +174,7 @@ func TestReconcileRepoCtx_Routing(t *testing.T) { reconcile := func(t *testing.T, s *store_sqlite.Store, cm *config.ConfigManager, repoPath string, prior map[string]int64) *IndexResult { t.Helper() - mi := NewMultiIndexer(graph.Store(s), newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(graph.Store(s), newTestRegistry(), search.NewNull(), cm, zap.NewNop()) res, err := mi.ReconcileRepoCtx(context.Background(), config.RepoEntry{Path: repoPath, Name: "repo"}, prior) require.NoError(t, err) require.NotNil(t, res) diff --git a/internal/indexer/repo_prefix_parity_test.go b/internal/indexer/repo_prefix_parity_test.go index 14c942ddf..4dec91d96 100644 --- a/internal/indexer/repo_prefix_parity_test.go +++ b/internal/indexer/repo_prefix_parity_test.go @@ -64,7 +64,7 @@ func TestRepoPrefixParityAfterIndexing(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) for _, e := range entries { _, err := mi.TrackRepoCtx(context.Background(), e) require.NoError(t, err) diff --git a/internal/indexer/repository_topology_batch_test.go b/internal/indexer/repository_topology_batch_test.go index 010934183..8127fc030 100644 --- a/internal/indexer/repository_topology_batch_test.go +++ b/internal/indexer/repository_topology_batch_test.go @@ -53,7 +53,7 @@ func TestRunRepositoryTopologyBatchAllowsParallelTrackAndBlocksReach(t *testing. registry := parser.NewRegistry() registry.Register(&topologyBatchBlockingExtractor{entered: entered, release: release}) store := graph.New() - mi := NewMultiIndexer(store, registry, search.NewBM25(), configManager, zap.NewNop()) + mi := NewMultiIndexer(store, registry, search.NewNull(), configManager, zap.NewNop()) mi.BeginParallelBatch() defer mi.ResetBatch() diff --git a/internal/indexer/scope_for_cwd_test.go b/internal/indexer/scope_for_cwd_test.go index b6ffff12b..61bd4bc0a 100644 --- a/internal/indexer/scope_for_cwd_test.go +++ b/internal/indexer/scope_for_cwd_test.go @@ -47,7 +47,7 @@ func TestScopeForCWD_And_ReposInWorkspace(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexScoped("", "") // empty scope → index every configured repo require.NoError(t, err) @@ -146,7 +146,7 @@ func TestScopeForCWD_WorkspaceRoot(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexScoped("", "") require.NoError(t, err) @@ -231,7 +231,7 @@ func TestContainedReposScope(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexScoped("", "") require.NoError(t, err) @@ -313,7 +313,7 @@ func TestContainedReposScope_RefusesOverbroadRoots(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexScoped("", "") require.NoError(t, err) @@ -365,7 +365,7 @@ func TestScopeForCWD_RefusesOverbroadRootSharedWorkspace(t *testing.T) { cm, err := config.NewConfigManager(tmpCfg) require.NoError(t, err) - mi := NewMultiIndexer(graph.New(), newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(graph.New(), newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexScoped("", "") require.NoError(t, err) diff --git a/internal/indexer/shadow_admission_test.go b/internal/indexer/shadow_admission_test.go index aede8664e..57a9f6fc4 100644 --- a/internal/indexer/shadow_admission_test.go +++ b/internal/indexer/shadow_admission_test.go @@ -193,8 +193,8 @@ func TestShadowAdmissionIsProcessWideAcrossMultiIndexers(t *testing.T) { g := graph.New() reg := parser.NewRegistry() logger := zap.NewNop() - first := NewMultiIndexer(g, reg, search.NewBM25(), nil, logger) - second := NewMultiIndexer(g, reg, search.NewBM25(), nil, logger) + first := NewMultiIndexer(g, reg, search.NewNull(), nil, logger) + second := NewMultiIndexer(g, reg, search.NewNull(), nil, logger) standalone := New(g, reg, config.IndexConfig{}, logger) if first.shadowAdmission == nil || first.shadowAdmission != second.shadowAdmission || first.shadowAdmission != standalone.shadowAdmission { diff --git a/internal/indexer/spec_launch_acceptance_test.go b/internal/indexer/spec_launch_acceptance_test.go index ef06888f2..b9ec8386d 100644 --- a/internal/indexer/spec_launch_acceptance_test.go +++ b/internal/indexer/spec_launch_acceptance_test.go @@ -94,7 +94,7 @@ func TestSpecLaunch_4_5_OrphansAreReportedPerWorkspace(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) for _, entry := range cm.Global().Repos { _, err := mi.TrackRepoCtx(context.Background(), entry) require.NoError(t, err, "track %s", entry.Name) @@ -159,7 +159,7 @@ func TestSpecLaunch_4_5_NoCrossWorkspacePairs(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) for _, entry := range cm.Global().Repos { _, err := mi.TrackRepoCtx(context.Background(), entry) require.NoError(t, err, "track %s", entry.Name) @@ -202,7 +202,7 @@ func TestSpecLaunch_4_5_FindUsagesScopedToWorkspace(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) for _, entry := range cm.Global().Repos { _, err := mi.TrackRepoCtx(context.Background(), entry) require.NoError(t, err, "track %s", entry.Name) @@ -274,7 +274,7 @@ func TestSpecLaunch_4_5_CrossWorkspaceHappyPath(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newMultiLangRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newMultiLangRegistry(), search.NewNull(), cm, zap.NewNop()) for _, entry := range cm.Global().Repos { _, err := mi.TrackRepoCtx(context.Background(), entry) require.NoError(t, err, "track %s", entry.Name) @@ -326,7 +326,7 @@ func TestSpecLaunch_4_5_MonorepoProjectBoundary(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) for _, entry := range cm.Global().Repos { _, err := mi.TrackRepoCtx(context.Background(), entry) require.NoError(t, err, "track %s", entry.Name) diff --git a/internal/indexer/store_hygiene_indexer_test.go b/internal/indexer/store_hygiene_indexer_test.go index 61cab283c..8062eaa2b 100644 --- a/internal/indexer/store_hygiene_indexer_test.go +++ b/internal/indexer/store_hygiene_indexer_test.go @@ -37,7 +37,7 @@ func newSqliteMultiIndexer(t *testing.T, repos []config.RepoEntry) (*MultiIndexe require.NoError(t, err) t.Cleanup(func() { _ = s.Close() }) - mi := NewMultiIndexer(graph.Store(s), newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(graph.Store(s), newTestRegistry(), search.NewNull(), cm, zap.NewNop()) return mi, s } diff --git a/internal/indexer/storm_test.go b/internal/indexer/storm_test.go index 3774f1cf1..6bb44730e 100644 --- a/internal/indexer/storm_test.go +++ b/internal/indexer/storm_test.go @@ -202,7 +202,7 @@ func TestIndexer_IndexFileNoResolve_SkipsResolver(t *testing.T) { g := graph.New() idx := New(g, newTestRegistry(), config.IndexConfig{Workers: 1}, zap.NewNop()) - idx.search = search.NewBM25() + idx.search = search.NewNull() idx.SetRootPath(dir) // IndexFileNoResolve populates the graph but defers cross-file diff --git a/internal/indexer/unicode_path_test.go b/internal/indexer/unicode_path_test.go index c038ec946..b79936acf 100644 --- a/internal/indexer/unicode_path_test.go +++ b/internal/indexer/unicode_path_test.go @@ -353,7 +353,7 @@ func TestGitWatcher_NonASCIIFileBranchSwitch(t *testing.T) { g := graph.New() idx := New(g, newTestRegistry(), config.IndexConfig{Workers: 1}, zap.NewNop()) - idx.search = search.NewBM25() + idx.search = search.NewNull() idx.SetRootPath(repoDir) _, err := idx.IndexCtx(testCtx(), repoDir) require.NoError(t, err) diff --git a/internal/indexer/watcher_inert_test.go b/internal/indexer/watcher_inert_test.go index 0faed697d..34e34b50b 100644 --- a/internal/indexer/watcher_inert_test.go +++ b/internal/indexer/watcher_inert_test.go @@ -27,7 +27,7 @@ func inertTestWatcher(t *testing.T, fileName, content string) (string, *Indexer, g := graph.New() idx := newTestIndexer(g) - idx.search = search.NewBM25() + idx.search = search.NewNull() idx.SetRootPath(dir) _, err := idx.Index(dir) require.NoError(t, err) @@ -265,7 +265,7 @@ func TestWatcher_OldDatabaseFirstPatchIsConservative(t *testing.T) { path := filepath.Join(dir, "main.go") writeTestFile(t, path, "package main\n\nfunc Stable() {}\n") idx := newTestIndexer(graph.New()) - idx.search = search.NewBM25() + idx.search = search.NewNull() idx.SetRootPath(dir) _, err := idx.Index(dir) require.NoError(t, err) diff --git a/internal/indexer/worktree_gc_test.go b/internal/indexer/worktree_gc_test.go index 2a195761f..3c15dbea4 100644 --- a/internal/indexer/worktree_gc_test.go +++ b/internal/indexer/worktree_gc_test.go @@ -67,7 +67,7 @@ func TestGCVanishedWorktrees_EvictsRemovedWorktree(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) @@ -132,7 +132,7 @@ func TestGCVanishedWorktrees_LeavesVanishedMainCheckout(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) require.False(t, mi.GetMetadata("plain").IsWorktree, @@ -180,7 +180,7 @@ func TestLinkedWorktreeRoots(t *testing.T) { require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) diff --git a/internal/indexer/worktree_instance_test.go b/internal/indexer/worktree_instance_test.go index ca8a21e64..a3b115a5c 100644 --- a/internal/indexer/worktree_instance_test.go +++ b/internal/indexer/worktree_instance_test.go @@ -60,7 +60,7 @@ func newWorktreeTestIndexer(t *testing.T, repos ...config.RepoEntry) (*graph.Gra cm, err := config.NewConfigManager(cfgPath) require.NoError(t, err) g := graph.New() - mi := NewMultiIndexer(g, newTestRegistry(), search.NewBM25(), cm, zap.NewNop()) + mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) return g, mi, cm } diff --git a/internal/mcp/analyze_scope_test.go b/internal/mcp/analyze_scope_test.go index 1dfaf3c25..79484cc8d 100644 --- a/internal/mcp/analyze_scope_test.go +++ b/internal/mcp/analyze_scope_test.go @@ -80,7 +80,7 @@ func newAnalyzeServer(t *testing.T, flagOn bool, repos ...analyzeRepoSpec) (*Ser g := graph.New() reg := testRegistry() - bm := search.NewBM25() + bm := search.NewNull() mi := indexer.NewMultiIndexer(g, reg, bm, cm, zap.NewNop()) _, err = mi.IndexScoped("", "") require.NoError(t, err) diff --git a/internal/mcp/diff_repo_scope_test.go b/internal/mcp/diff_repo_scope_test.go index 734f13163..c70f4ee80 100644 --- a/internal/mcp/diff_repo_scope_test.go +++ b/internal/mcp/diff_repo_scope_test.go @@ -41,7 +41,7 @@ func TestDiffRepoScope(t *testing.T) { reg := parser.NewRegistry() reg.Register(languages.NewGoExtractor()) g := graph.New() - mi := indexer.NewMultiIndexer(g, reg, search.NewBM25(), cm, zap.NewNop()) + mi := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) diff --git a/internal/mcp/enclosing_test.go b/internal/mcp/enclosing_test.go index 8754b0016..306fa0e1e 100644 --- a/internal/mcp/enclosing_test.go +++ b/internal/mcp/enclosing_test.go @@ -73,7 +73,7 @@ func TestFindUsages_GroupByFile(t *testing.T) { g.AddEdge(&graph.Edge{From: b1.ID, To: target.ID, Kind: graph.EdgeCalls, FilePath: "pkg/b.go", Line: 5}) eng := query.NewEngine(g) - eng.SetSearch(search.NewBM25()) + eng.SetSearch(search.NewNull()) srv := NewServer(eng, g, nil, nil, zap.NewNop(), nil) req := mcplib.CallToolRequest{} @@ -117,7 +117,7 @@ func TestFindUsages_FlatByDefault(t *testing.T) { g.AddEdge(&graph.Edge{From: caller.ID, To: target.ID, Kind: graph.EdgeCalls, FilePath: "pkg/a.go", Line: 3}) eng := query.NewEngine(g) - eng.SetSearch(search.NewBM25()) + eng.SetSearch(search.NewNull()) srv := NewServer(eng, g, nil, nil, zap.NewNop(), nil) req := mcplib.CallToolRequest{} diff --git a/internal/mcp/ensure_fresh_self_heal_test.go b/internal/mcp/ensure_fresh_self_heal_test.go index dc1e8476b..193cf393a 100644 --- a/internal/mcp/ensure_fresh_self_heal_test.go +++ b/internal/mcp/ensure_fresh_self_heal_test.go @@ -43,7 +43,7 @@ func TestEnsureFresh_MultiRepoSelfHealsStaleFile(t *testing.T) { reg.Register(languages.NewGoExtractor()) g := graph.New() - mi := indexer.NewMultiIndexer(g, reg, search.NewBM25(), cm, zap.NewNop()) + mi := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) require.True(t, mi.IsMultiRepo()) diff --git a/internal/mcp/explore_source_literal_test.go b/internal/mcp/explore_source_literal_test.go index 6c79975bd..406594a39 100644 --- a/internal/mcp/explore_source_literal_test.go +++ b/internal/mcp/explore_source_literal_test.go @@ -1046,7 +1046,7 @@ func TestSearchExploreSourceLiteralDoesNotCrossConfiguredRepository(t *testing.T configPath := filepath.Join(t.TempDir(), "config.yaml") manager, err := config.NewConfigManager(configPath) require.NoError(t, err) - multi := indexer.NewMultiIndexer(store, registry, search.NewAuto(), manager, zap.NewNop()) + multi := indexer.NewMultiIndexer(store, registry, search.NewNull(), manager, zap.NewNop()) otherRoot := filepath.Join(t.TempDir(), "repo-a") require.NoError(t, os.MkdirAll(otherRoot, 0o755)) _, err = multi.TrackRepoCtx(context.Background(), config.RepoEntry{Path: otherRoot, Force: true}) diff --git a/internal/mcp/find_usages_context_test.go b/internal/mcp/find_usages_context_test.go index dd918f154..6def86d04 100644 --- a/internal/mcp/find_usages_context_test.go +++ b/internal/mcp/find_usages_context_test.go @@ -31,7 +31,7 @@ func usagesContextServer(t *testing.T) (*Server, string) { g.AddEdge(&graph.Edge{From: field.ID, To: foo.ID, Kind: graph.EdgeTypedAs, FilePath: "pkg/b.go", Line: 3}) eng := query.NewEngine(g) - eng.SetSearch(search.NewBM25()) + eng.SetSearch(search.NewNull()) return NewServer(eng, g, nil, nil, zap.NewNop(), nil), foo.ID } diff --git a/internal/mcp/find_usages_summary_test.go b/internal/mcp/find_usages_summary_test.go index dab0d1211..fa4af0b60 100644 --- a/internal/mcp/find_usages_summary_test.go +++ b/internal/mcp/find_usages_summary_test.go @@ -42,7 +42,7 @@ func usagesSummaryServer(t *testing.T) (srv *Server, fooID, unusedID string) { g.AddEdge(&graph.Edge{From: testUse.ID, To: foo.ID, Kind: graph.EdgeCalls, FilePath: "pkg/foo_test.go", Line: 12}) eng := query.NewEngine(g) - eng.SetSearch(search.NewBM25()) + eng.SetSearch(search.NewNull()) return NewServer(eng, g, nil, nil, zap.NewNop(), nil), foo.ID, unused.ID } diff --git a/internal/mcp/freshness_rider_test.go b/internal/mcp/freshness_rider_test.go index 0fa2e7908..dca6a327f 100644 --- a/internal/mcp/freshness_rider_test.go +++ b/internal/mcp/freshness_rider_test.go @@ -105,7 +105,7 @@ func TestMultiRepoRiderAndMissingFileFlag(t *testing.T) { reg := parser.NewRegistry() reg.Register(languages.NewGoExtractor()) g := graph.New() - mi := indexer.NewMultiIndexer(g, reg, search.NewBM25(), cm, zap.NewNop()) + mi := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) diff --git a/internal/mcp/instructions_test.go b/internal/mcp/instructions_test.go index b4989b7d2..c6302fc16 100644 --- a/internal/mcp/instructions_test.go +++ b/internal/mcp/instructions_test.go @@ -85,7 +85,7 @@ func TestStateAwareInstructionsVariants(t *testing.T) { reg := parser.NewRegistry() reg.Register(languages.NewGoExtractor()) g := graph.New() - mi := indexer.NewMultiIndexer(g, reg, search.NewBM25(), cm, zap.NewNop()) + mi := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) diff --git a/internal/mcp/path_scope_test.go b/internal/mcp/path_scope_test.go index f355c7c8d..71f9c6e64 100644 --- a/internal/mcp/path_scope_test.go +++ b/internal/mcp/path_scope_test.go @@ -86,7 +86,7 @@ func TestResolvePathFilter_Sources(t *testing.T) { t.Setenv("GORTEX_SCOPES_PATH", filepath.Join(tempSidecarDir(t), "scopes.json")) g := graph.New() eng := query.NewEngine(g) - eng.SetSearch(search.NewBM25()) + eng.SetSearch(search.NewNull()) srv := NewServer(eng, g, nil, nil, zap.NewNop(), nil) require.NoError(t, srv.scopeStoreOrInit().put(SavedScope{ Name: "billing", Repos: []string{"r"}, Paths: []string{"services/billing"}, diff --git a/internal/mcp/return_usage_test.go b/internal/mcp/return_usage_test.go index 5f6882cc4..8ea9e8523 100644 --- a/internal/mcp/return_usage_test.go +++ b/internal/mcp/return_usage_test.go @@ -60,7 +60,7 @@ func returnUsageServer(t *testing.T) (*Server, string) { }) eng := query.NewEngine(g) - eng.SetSearch(search.NewBM25()) + eng.SetSearch(search.NewNull()) return NewServer(eng, g, nil, nil, zap.NewNop(), nil), fetch.ID } @@ -233,7 +233,7 @@ func relay() int { } eng := query.NewEngine(g) - eng.SetSearch(search.NewBM25()) + eng.SetSearch(search.NewNull()) srv := NewServer(eng, g, nil, nil, zap.NewNop(), nil) edges := findUsagesEdges(t, srv, map[string]any{"id": "main.go::helper"}) diff --git a/internal/mcp/scope_resolve_test.go b/internal/mcp/scope_resolve_test.go index 8d3cc7afb..0a0abbd6c 100644 --- a/internal/mcp/scope_resolve_test.go +++ b/internal/mcp/scope_resolve_test.go @@ -77,7 +77,7 @@ func newSharedWorkspaceServer(t *testing.T, flagOn bool) sharedWSOptions { g := graph.New() reg := testRegistry() - bm := search.NewBM25() + bm := search.NewNull() mi := indexer.NewMultiIndexer(g, reg, bm, cm, zap.NewNop()) _, err = mi.IndexScoped("", "") require.NoError(t, err) @@ -115,7 +115,7 @@ func newSplitProjectWorkspaceServer(t *testing.T, flagOn bool) sharedWSOptions { g := graph.New() reg := testRegistry() - bm := search.NewBM25() + bm := search.NewNull() mi := indexer.NewMultiIndexer(g, reg, bm, cm, zap.NewNop()) _, err = mi.IndexScoped("", "") require.NoError(t, err) @@ -694,7 +694,7 @@ func newLoneRepoServer(t *testing.T, flagOn bool) (*Server, string) { g := graph.New() reg := testRegistry() - bm := search.NewBM25() + bm := search.NewNull() mi := indexer.NewMultiIndexer(g, reg, bm, cm, zap.NewNop()) _, err = mi.IndexScoped("", "") require.NoError(t, err) diff --git a/internal/mcp/search_corpus_scope_test.go b/internal/mcp/search_corpus_scope_test.go index 697c7858a..5e51acdd0 100644 --- a/internal/mcp/search_corpus_scope_test.go +++ b/internal/mcp/search_corpus_scope_test.go @@ -57,7 +57,7 @@ func newWorkspaceRootBoundServer(t *testing.T) (s *Server, root string) { reg := parser.NewRegistry() reg.Register(languages.NewGoExtractor()) - mi := indexer.NewMultiIndexer(store, reg, search.NewBM25(), cm, zap.NewNop()) + mi := indexer.NewMultiIndexer(store, reg, search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) diff --git a/internal/mcp/tools_contract_bridge_test.go b/internal/mcp/tools_contract_bridge_test.go index 50c22e0be..61417e9f8 100644 --- a/internal/mcp/tools_contract_bridge_test.go +++ b/internal/mcp/tools_contract_bridge_test.go @@ -126,7 +126,7 @@ func fetchUsers() { preg := testRegistry() g := graph.New() - mi := indexer.NewMultiIndexer(g, preg, search.NewBM25(), cm, zap.NewNop()) + mi := indexer.NewMultiIndexer(g, preg, search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) diff --git a/internal/mcp/tools_contracts_filter_test.go b/internal/mcp/tools_contracts_filter_test.go index 9515a15c5..a61dd188e 100644 --- a/internal/mcp/tools_contracts_filter_test.go +++ b/internal/mcp/tools_contracts_filter_test.go @@ -69,7 +69,7 @@ func TestHandleContracts_FiltersByProjectAndRef(t *testing.T) { preg := testRegistry() g := graph.New() - mi := indexer.NewMultiIndexer(g, preg, search.NewBM25(), cm, zap.NewNop()) + mi := indexer.NewMultiIndexer(g, preg, search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) diff --git a/internal/mcp/tools_contracts_test.go b/internal/mcp/tools_contracts_test.go index c3d292b6c..ae9450ed1 100644 --- a/internal/mcp/tools_contracts_test.go +++ b/internal/mcp/tools_contracts_test.go @@ -91,7 +91,7 @@ func TestHandleContracts_ReflectsRuntimeTrackedRepos(t *testing.T) { preg := testRegistry() g := graph.New() - mi := indexer.NewMultiIndexer(g, preg, search.NewBM25(), cm, zap.NewNop()) + mi := indexer.NewMultiIndexer(g, preg, search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) @@ -159,7 +159,7 @@ func TestHandleContracts_MatchesGraphContractCount(t *testing.T) { preg := testRegistry() g := graph.New() - mi := indexer.NewMultiIndexer(g, preg, search.NewBM25(), cm, zap.NewNop()) + mi := indexer.NewMultiIndexer(g, preg, search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) diff --git a/internal/mcp/tools_core_index_test.go b/internal/mcp/tools_core_index_test.go index 51ed1ad5a..b996f5d6e 100644 --- a/internal/mcp/tools_core_index_test.go +++ b/internal/mcp/tools_core_index_test.go @@ -59,7 +59,7 @@ func TestHandleIndexRepository_MultiRepoRoutesThroughMultiIndexer(t *testing.T) reg.Register(languages.NewGoExtractor()) g := graph.New() - mi := indexer.NewMultiIndexer(g, reg, search.NewBM25(), cm, zap.NewNop()) + mi := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) require.True(t, mi.IsMultiRepo()) @@ -145,7 +145,7 @@ func TestHandleIndexRepository_ReconcilesFromPersistedConfig(t *testing.T) { g := graph.New() // Intentionally DO NOT call IndexAll / TrackRepoCtx — simulates the // warmup-drift state: config persisted, but MultiIndexer empty. - mi := indexer.NewMultiIndexer(g, preg, search.NewBM25(), cm, zap.NewNop()) + mi := indexer.NewMultiIndexer(g, preg, search.NewNull(), cm, zap.NewNop()) require.Empty(t, mi.AllMetadata(), "precondition: MultiIndexer must start empty to reproduce drift") @@ -195,7 +195,7 @@ func TestHandleIndexRepository_MultiRepoRejectsUntrackedPath(t *testing.T) { reg.Register(languages.NewGoExtractor()) g := graph.New() - mi := indexer.NewMultiIndexer(g, reg, search.NewBM25(), cm, zap.NewNop()) + mi := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) diff --git a/internal/mcp/tools_core_reindex_test.go b/internal/mcp/tools_core_reindex_test.go index 1e6ddd12b..f83d440bf 100644 --- a/internal/mcp/tools_core_reindex_test.go +++ b/internal/mcp/tools_core_reindex_test.go @@ -226,7 +226,7 @@ func TestHandleReindexRepository_MultiRepoRoutesByPrefix(t *testing.T) { reg.Register(languages.NewGoExtractor()) g := graph.New() - mi := indexer.NewMultiIndexer(g, reg, search.NewBM25(), cm, zap.NewNop()) + mi := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) require.True(t, mi.IsMultiRepo()) @@ -284,7 +284,7 @@ func TestHandleReindexRepository_MultiRepoPathScoped(t *testing.T) { reg.Register(languages.NewGoExtractor()) g := graph.New() - mi := indexer.NewMultiIndexer(g, reg, search.NewBM25(), cm, zap.NewNop()) + mi := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) @@ -338,7 +338,7 @@ func TestHandleReindexRepository_MultiRepoRejectsUntrackedPath(t *testing.T) { reg.Register(languages.NewGoExtractor()) g := graph.New() - mi := indexer.NewMultiIndexer(g, reg, search.NewBM25(), cm, zap.NewNop()) + mi := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) diff --git a/internal/mcp/tools_fileops_singlerepo_test.go b/internal/mcp/tools_fileops_singlerepo_test.go index 80f7a3b61..d58865145 100644 --- a/internal/mcp/tools_fileops_singlerepo_test.go +++ b/internal/mcp/tools_fileops_singlerepo_test.go @@ -37,7 +37,7 @@ func newSingleRepoServer(t *testing.T) (*Server, *graph.Graph, string) { reg := parser.NewRegistry() reg.Register(languages.NewGoExtractor()) g := graph.New() - mi := indexer.NewMultiIndexer(g, reg, search.NewBM25(), cm, zap.NewNop()) + mi := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) @@ -115,7 +115,7 @@ func TestResolveFilePath_MultiRepoBareRelativeStillAmbiguous(t *testing.T) { reg := parser.NewRegistry() reg.Register(languages.NewGoExtractor()) g := graph.New() - mi := indexer.NewMultiIndexer(g, reg, search.NewBM25(), cm, zap.NewNop()) + mi := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) diff --git a/internal/mcp/tools_fileops_worktree_test.go b/internal/mcp/tools_fileops_worktree_test.go index a4a4f481a..f20d086a4 100644 --- a/internal/mcp/tools_fileops_worktree_test.go +++ b/internal/mcp/tools_fileops_worktree_test.go @@ -86,7 +86,7 @@ func setupWorktreePair(t *testing.T) (mainRepo, worktree string, srv *Server) { reg.Register(languages.NewGoExtractor()) g := graph.New() - mi := indexer.NewMultiIndexer(g, reg, search.NewBM25(), cm, zap.NewNop()) + mi := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) require.True(t, mi.IsMultiRepo()) diff --git a/internal/mcp/tools_find_declaration_test.go b/internal/mcp/tools_find_declaration_test.go index e3db62067..9cecd6569 100644 --- a/internal/mcp/tools_find_declaration_test.go +++ b/internal/mcp/tools_find_declaration_test.go @@ -250,7 +250,7 @@ func consumer() int { return uniqueDecl() + uniqueDecl() } reg := parser.NewRegistry() reg.Register(languages.NewGoExtractor()) g := graph.New() - mi := indexer.NewMultiIndexer(g, reg, search.NewBM25(), cm, zap.NewNop()) + mi := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) require.True(t, mi.IsMultiRepo()) diff --git a/internal/mcp/tools_multi_worktree_test.go b/internal/mcp/tools_multi_worktree_test.go index 6b12e795c..147b36fa6 100644 --- a/internal/mcp/tools_multi_worktree_test.go +++ b/internal/mcp/tools_multi_worktree_test.go @@ -34,7 +34,7 @@ func newWorktreeMCPServer(t *testing.T, repos ...config.RepoEntry) (*Server, *in preg := testRegistry() g := graph.New() - mi := indexer.NewMultiIndexer(g, preg, search.NewBM25(), cm, zap.NewNop()) + mi := indexer.NewMultiIndexer(g, preg, search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) diff --git a/internal/mcp/tools_review_rulepack_test.go b/internal/mcp/tools_review_rulepack_test.go index 09762a120..34f78b1b8 100644 --- a/internal/mcp/tools_review_rulepack_test.go +++ b/internal/mcp/tools_review_rulepack_test.go @@ -47,7 +47,7 @@ func prefixedServerOver(t *testing.T, dir, name string) (*Server, string) { reg.Register(languages.NewGoExtractor()) g := graph.New() - mi := indexer.NewMultiIndexer(g, reg, search.NewBM25(), cm, zap.NewNop()) + mi := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) diff --git a/internal/mcp/tools_search_text_pathsep_test.go b/internal/mcp/tools_search_text_pathsep_test.go index dae6edaef..c070cf7b6 100644 --- a/internal/mcp/tools_search_text_pathsep_test.go +++ b/internal/mcp/tools_search_text_pathsep_test.go @@ -58,7 +58,7 @@ func nestedRepoServer(t *testing.T, entries []config.RepoEntry) (*Server, *graph reg := parser.NewRegistry() reg.Register(languages.NewGoExtractor()) g := graph.New() - mi := indexer.NewMultiIndexer(g, reg, search.NewBM25(), cm, zap.NewNop()) + mi := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) diff --git a/internal/mcp/tools_search_text_test.go b/internal/mcp/tools_search_text_test.go index 4099d50c8..948ecaa31 100644 --- a/internal/mcp/tools_search_text_test.go +++ b/internal/mcp/tools_search_text_test.go @@ -180,7 +180,7 @@ func TestSearchText_MultiRepoFanout(t *testing.T) { reg := parser.NewRegistry() reg.Register(languages.NewGoExtractor()) g := graph.New() - mi := indexer.NewMultiIndexer(g, reg, search.NewBM25(), cm, zap.NewNop()) + mi := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) require.True(t, mi.IsMultiRepo()) @@ -236,7 +236,7 @@ func TestSearchText_MultiRepoScopedGrepDoesNotLetOutOfScopeRepoConsumeLimit(t *t reg := parser.NewRegistry() reg.Register(languages.NewGoExtractor()) g := graph.New() - mi := indexer.NewMultiIndexer(g, reg, search.NewBM25(), cm, zap.NewNop()) + mi := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) @@ -294,7 +294,7 @@ func TestFilterTextMatchesByResolvedScope_FailsClosed(t *testing.T) { reg := parser.NewRegistry() reg.Register(languages.NewGoExtractor()) g := graph.New() - mi := indexer.NewMultiIndexer(g, reg, search.NewBM25(), cm, zap.NewNop()) + mi := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) @@ -364,7 +364,7 @@ func TestGetFileSummary_MultiRepoRelativePath(t *testing.T) { reg := parser.NewRegistry() reg.Register(languages.NewGoExtractor()) g := graph.New() - mi := indexer.NewMultiIndexer(g, reg, search.NewBM25(), cm, zap.NewNop()) + mi := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) _, err = mi.IndexAll() require.NoError(t, err) require.True(t, mi.IsMultiRepo()) diff --git a/internal/mcp/workspace_isolation_test.go b/internal/mcp/workspace_isolation_test.go index f24b60fa4..6801f1000 100644 --- a/internal/mcp/workspace_isolation_test.go +++ b/internal/mcp/workspace_isolation_test.go @@ -57,7 +57,7 @@ func newIsolationServer(t *testing.T) (srv *Server, repoA, repoB string) { g := graph.New() reg := testRegistry() - bm := search.NewBM25() + bm := search.NewNull() mi := indexer.NewMultiIndexer(g, reg, bm, cm, zap.NewNop()) _, err = mi.IndexScoped("", "") // index every configured repo require.NoError(t, err) diff --git a/internal/mcp/workspace_root_scope_test.go b/internal/mcp/workspace_root_scope_test.go index af82f7086..386e9df04 100644 --- a/internal/mcp/workspace_root_scope_test.go +++ b/internal/mcp/workspace_root_scope_test.go @@ -65,7 +65,7 @@ func newContainedFixture(t *testing.T) containedFixture { require.NoError(t, err) g := graph.New() - bm := search.NewBM25() + bm := search.NewNull() mi := indexer.NewMultiIndexer(g, testRegistry(), bm, cm, zap.NewNop()) _, err = mi.IndexScoped("", "") require.NoError(t, err) diff --git a/internal/query/engine_corpus_gate_test.go b/internal/query/engine_corpus_gate_test.go index 707bc63f9..acd695eb8 100644 --- a/internal/query/engine_corpus_gate_test.go +++ b/internal/query/engine_corpus_gate_test.go @@ -155,7 +155,7 @@ func TestGatherSymbolCandidates_EmptyBackendStillFallsBack(t *testing.T) { n := &graph.Node{ID: "app/w.go::WidgetExtensions", Name: "WidgetExtensions", Kind: graph.KindType, RepoPrefix: "app"} g.AddNode(n) engine := NewEngine(g) - engine.SetSearch(search.NewBM25()) // empty: Count 0, no corpus + engine.SetSearch(search.NewNull()) // empty: Count 0, no corpus got := engine.GatherSymbolCandidates("WidgetExtensions", 5, QueryOptions{SkipInnerRerank: true, SkipVectorChannel: true}, nil) if len(got) != 1 || got[0].Node.ID != n.ID { diff --git a/internal/search/null.go b/internal/search/null.go new file mode 100644 index 000000000..5c6f5b842 --- /dev/null +++ b/internal/search/null.go @@ -0,0 +1,44 @@ +package search + +// NullBackend is the text backend for graph stores that expose no +// native symbol search. It indexes nothing and answers nothing: Add, +// Remove and Close are no-ops, Search returns no hits, and Count +// always reports zero. +// +// Reporting an empty corpus is the point. The query Engine gates its +// ranked path on the backend having something to answer with +// (Engine.backendHasCorpus) and otherwise falls through to its own +// substring scan over the graph, which needs no text index at all. +// NullBackend therefore routes such a store onto exactly the path an +// Engine with no search backend at all already takes in production — +// see the Engine built by pkg/gortex.New, which wires query.NewEngine +// and never calls SetSearch. +// +// NewNull hands back a distinct pointer per call so that identity +// comparisons — Swappable.Swap only closes the old backend when it is +// not the incoming one — behave as they do for every other backend. +// +// It deliberately implements Backend and nothing else. Satisfying +// DocCounter would let a disk-corpus count claim a corpus that does +// not exist; Sizer and ChannelSearcher would advertise memory and +// per-channel retrieval it does not have. +type NullBackend struct{} + +// NewNull returns a Backend that indexes and answers nothing. +func NewNull() Backend { return &NullBackend{} } + +// Add discards the symbol; nothing is indexed. +func (*NullBackend) Add(string, ...string) {} + +// Remove is a no-op — nothing was ever indexed. +func (*NullBackend) Remove(string) {} + +// Search always returns no hits, so callers fall through to whatever +// index-free path they keep for an empty corpus. +func (*NullBackend) Search(string, int) []SearchResult { return nil } + +// Count reports an empty corpus. +func (*NullBackend) Count() int { return 0 } + +// Close is a no-op — there are no resources to release. +func (*NullBackend) Close() {} diff --git a/internal/search/null_test.go b/internal/search/null_test.go new file mode 100644 index 000000000..6d5fd5574 --- /dev/null +++ b/internal/search/null_test.go @@ -0,0 +1,61 @@ +package search + +import "testing" + +// TestNullBackend_ImplementsBackendAndNothingElse pins the property the +// fallback seam rests on. The query engine treats a backend as having a +// corpus when Count() is positive OR when it satisfies DocCounter and +// reports a positive disk count; Sizer makes daemon status attribute +// heap to it and ChannelSearcher makes the rerank pipeline ask it for a +// text channel. A NullBackend that grew any of those optional +// interfaces would start claiming a corpus it does not have, and the +// engine would rank against nothing instead of falling back to its +// substring scan. +func TestNullBackend_ImplementsBackendAndNothingElse(t *testing.T) { + var b Backend = NewNull() + + if _, ok := b.(DocCounter); ok { + t.Error("NullBackend must not implement DocCounter — it would claim a corpus it has no documents for") + } + if _, ok := b.(Sizer); ok { + t.Error("NullBackend must not implement Sizer — it holds no memory to attribute") + } + if _, ok := b.(ChannelSearcher); ok { + t.Error("NullBackend must not implement ChannelSearcher — it has no text channel to contribute") + } + if got := BackendSize(b); got != 0 { + t.Errorf("BackendSize = %d, want 0", got) + } +} + +// TestNullBackend_StaysEmptyAfterAdd is the behaviour the seam promises: +// indexing into the null backend never produces a searchable corpus, so +// callers gated on Count() keep taking their index-free path. +func TestNullBackend_StaysEmptyAfterAdd(t *testing.T) { + b := NewNull() + defer b.Close() + + b.Add("pkg/a.go::Alpha", "Alpha", "pkg/a.go", "") + b.Add("pkg/b.go::Beta", "Beta", "pkg/b.go", "") + + if got := b.Count(); got != 0 { + t.Errorf("Count after Add = %d, want 0", got) + } + if got := b.Search("Alpha", 10); len(got) != 0 { + t.Errorf("Search returned %d hits, want 0", len(got)) + } + + b.Remove("pkg/a.go::Alpha") + if got := b.Count(); got != 0 { + t.Errorf("Count after Remove = %d, want 0", got) + } +} + +// TestNewNull_DistinctInstances guards the identity contract Swappable +// relies on: Swap closes the previous backend only when it differs from +// the incoming one, so two null backends must not compare equal. +func TestNewNull_DistinctInstances(t *testing.T) { + if NewNull() == NewNull() { + t.Error("NewNull must hand back a distinct backend per call") + } +} diff --git a/internal/search/search.go b/internal/search/search.go index 9418056f5..bbac32933 100644 --- a/internal/search/search.go +++ b/internal/search/search.go @@ -3,8 +3,8 @@ // // Production search runs on SymbolSearcherBackend, a thin adapter over // the graph store's own FTS index — no parallel in-process corpus. -// BM25Backend, a self-contained in-memory inverted index, is the -// fallback for stores that expose no native symbol search. +// A store that exposes no native symbol search gets NullBackend, whose +// empty corpus routes the query engine to its substring fallback. package search // SearchResult is a single search hit. @@ -61,10 +61,3 @@ func BackendSize(b Backend) uint64 { } return 0 } - -// NewAuto returns the default in-process text backend. Reached only -// when the graph store exposes no native symbol search; otherwise the -// indexer wires up a SymbolSearcherBackend over the store's own FTS. -func NewAuto() Backend { - return NewBM25() -} diff --git a/internal/search/swappable.go b/internal/search/swappable.go index 6951fb344..355f4004d 100644 --- a/internal/search/swappable.go +++ b/internal/search/swappable.go @@ -27,7 +27,7 @@ type Swappable struct { } // NewSwappable wraps b. Panics if b is nil — every Indexer must start -// with a real backend, even if it's the in-memory NewAuto() default. +// with a real backend, even if it's the NewNull() no-op default. func NewSwappable(b Backend) *Swappable { if b == nil { panic("search.NewSwappable: nil backend") From b6232efeb595a7d7c295ae09207c1f46848b6470 Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Sun, 9 Aug 2026 21:33:28 +0200 Subject: [PATCH 04/21] refactor(eval): measure stdbench against the store-native FTS text path stdbench indexed its corpus into an in-process BM25 index and claimed to measure what search_symbols runs; that stopped being true when search moved to the store's native FTS, so the reported numbers described a retriever no user queries. It now opens a throwaway sqlite store, writes the corpus through BulkUpsertSymbolFTS with search.Tokenize-split tokens (mirroring the indexer's write side), asserts every document reached the index, and ranks with SearchSymbols. The corpus never goes through search.Backend because the shipping adapter's Add is a no-op and would silently score an empty index. --- cmd/gortex/eval_stdbench.go | 71 +++++++++++++++++++++++++---- internal/eval/stdbench/dataset.go | 6 +-- internal/eval/stdbench/eval_test.go | 48 +++++++++++++++---- 3 files changed, 105 insertions(+), 20 deletions(-) diff --git a/cmd/gortex/eval_stdbench.go b/cmd/gortex/eval_stdbench.go index 24796ede2..6aba47c6d 100644 --- a/cmd/gortex/eval_stdbench.go +++ b/cmd/gortex/eval_stdbench.go @@ -9,6 +9,7 @@ import ( "github.com/spf13/cobra" "github.com/zzet/gortex/internal/eval/stdbench" + "github.com/zzet/gortex/internal/graph" "github.com/zzet/gortex/internal/search" ) @@ -22,8 +23,9 @@ var ( var evalStdbenchCmd = &cobra.Command{ Use: "stdbench", Short: "Run a standardized retrieval benchmark (CoIR / SWE-ContextBench / ContextBench)", - Long: `Runs Gortex's BM25 text retrieval against a standardized code-retrieval -benchmark and reports Recall@K, Precision@K, NDCG@10, and MRR. + Long: `Runs Gortex's shipping text retrieval — the store-native symbol FTS +search_symbols queries — against a standardized code-retrieval benchmark +and reports Recall@K, Precision@K, NDCG@10, and MRR. Benchmarks (--bench): coir CoIR (Code Information Retrieval, ACL 2025). --dataset is @@ -84,22 +86,75 @@ func runEvalStdbench(_ *cobra.Command, _ []string) error { return fmt.Errorf("benchmark %q carries no relevance judgements to score against", evalStdbenchBench) } - // Index the corpus into Gortex's BM25 backend — the same text - // retrieval search_symbols runs — and rank doc IDs per query. - bm := search.NewBM25() + // Index the corpus into a throwaway SQLite store's native symbol FTS — + // the text retrieval search_symbols actually runs — and rank doc IDs + // per query. Note the corpus goes in through the store's FTS writer + // directly, never through search.Backend.Add: the shipping backend is + // an adapter over this same FTS whose Add is a no-op, so routing the + // corpus through it would index nothing and report a silent 0.0. + st, closeStore, err := newEvalStore("stdbench") + if err != nil { + return fmt.Errorf("opening the eval store: %w", err) + } + defer closeStore() + + searcher, ok := st.(graph.SymbolSearcher) + if !ok { + return fmt.Errorf("the eval store does not expose native symbol search") + } + counter, ok := st.(graph.SymbolFTSCounter) + if !ok { + return fmt.Errorf("the eval store does not report its symbol FTS document count") + } + + items := make([]graph.SymbolFTSItem, 0, len(ds.Corpus)) for _, d := range ds.Corpus { - bm.Add(d.ID, d.Text) + // Mirrors the indexer's ftsTokensFor: the write side splits with + // search.Tokenize so the read side's query tokenisation lands on + // the same terms. + items = append(items, graph.SymbolFTSItem{ + NodeID: d.ID, + Tokens: strings.Join(search.Tokenize(d.Text), " "), + }) + } + // One call, not a chunked loop: BulkUpsertSymbolFTS wipes the prefix + // before inserting, so a second call under the same prefix would erase + // the first. It bounds its own INSERT statements internally. + if err := searcher.BulkUpsertSymbolFTS("", items); err != nil { + return fmt.Errorf("indexing the %s corpus: %w", ds.Name, err) + } + if err := searcher.BuildSymbolIndex(); err != nil { + return fmt.Errorf("building the symbol index: %w", err) } + indexed, err := counter.SymbolFTSCount() + if err != nil { + return fmt.Errorf("counting the indexed corpus: %w", err) + } + if indexed != len(ds.Corpus) { + return fmt.Errorf("indexed %d of %d corpus documents — scoring a partial corpus "+ + "would report a retrieval number no user can reproduce", indexed, len(ds.Corpus)) + } + + var retrieveErr error retrieve := func(query string, k int) []string { - hits := bm.Search(query, k) + hits, err := searcher.SearchSymbols(query, k) + if err != nil { + if retrieveErr == nil { + retrieveErr = err + } + return nil + } ids := make([]string, 0, len(hits)) for _, h := range hits { - ids = append(ids, h.ID) + ids = append(ids, h.NodeID) } return ids } metrics := stdbench.Evaluate(ds, retrieve, nil) + if retrieveErr != nil { + return fmt.Errorf("searching the corpus: %w", retrieveErr) + } var rendered string if strings.EqualFold(evalStdbenchFormat, "json") { diff --git a/internal/eval/stdbench/dataset.go b/internal/eval/stdbench/dataset.go index 74de7806e..a993b2dae 100644 --- a/internal/eval/stdbench/dataset.go +++ b/internal/eval/stdbench/dataset.go @@ -4,9 +4,9 @@ // with the textbook Recall@K / Precision@K / NDCG@K / MRR metrics. // // The loaders parse the on-disk formats; the actual retrieval is left -// to the caller (the `gortex eval stdbench` verb wires Gortex's BM25 -// backend in), so the same harness measures whatever retriever is -// handed to Evaluate. +// to the caller (the `gortex eval stdbench` verb wires in the +// store-native symbol FTS search_symbols runs), so the same harness +// measures whatever retriever is handed to Evaluate. package stdbench // Doc is one corpus document — a code snippet, file, or symbol the diff --git a/internal/eval/stdbench/eval_test.go b/internal/eval/stdbench/eval_test.go index 24d26fe38..fb6df6e1e 100644 --- a/internal/eval/stdbench/eval_test.go +++ b/internal/eval/stdbench/eval_test.go @@ -2,10 +2,14 @@ package stdbench import ( "math" + "path/filepath" + "strings" "testing" "github.com/stretchr/testify/require" + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/graph/store_sqlite" "github.com/zzet/gortex/internal/search" ) @@ -53,28 +57,54 @@ func TestEvaluate_PerfectRanking(t *testing.T) { require.InDelta(t, 1.0, m.NDCGAt10, 1e-9) } -// TestEvaluate_EndToEndBM25 runs the full evaluation path: load a JSONL -// benchmark, index its corpus into Gortex's real BM25 backend, and -// score retrieval — the same wiring `gortex eval stdbench` uses. -func TestEvaluate_EndToEndBM25(t *testing.T) { +// TestEvaluate_EndToEndFTS runs the full evaluation path: load a JSONL +// benchmark, index its corpus into a real SQLite store's native symbol +// FTS, and score retrieval — the same wiring `gortex eval stdbench` +// uses, which is the same text path search_symbols serves. +func TestEvaluate_EndToEndFTS(t *testing.T) { ds, err := LoadContextBench("testdata/contextbench.jsonl") require.NoError(t, err) - bm := search.NewBM25() + store, err := store_sqlite.Open(filepath.Join(t.TempDir(), "stdbench.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { _ = store.Close() }) + + items := make([]graph.SymbolFTSItem, 0, len(ds.Corpus)) for _, d := range ds.Corpus { - bm.Add(d.ID, d.Text) + // Same splitter the indexer's write side uses, so the read side's + // query tokenisation lands on the same terms. + items = append(items, graph.SymbolFTSItem{ + NodeID: d.ID, + Tokens: strings.Join(search.Tokenize(d.Text), " "), + }) } + require.NoError(t, store.BulkUpsertSymbolFTS("", items)) + require.NoError(t, store.BuildSymbolIndex()) + + indexed, err := store.SymbolFTSCount() + require.NoError(t, err) + require.Equal(t, len(ds.Corpus), indexed, + "every corpus document must reach the FTS — a partial corpus scores a benchmark nobody ran") + retrieve := func(query string, k int) []string { - hits := bm.Search(query, k) + hits, err := store.SearchSymbols(query, k) + require.NoError(t, err) ids := make([]string, 0, len(hits)) for _, h := range hits { - ids = append(ids, h.ID) + ids = append(ids, h.NodeID) } return ids } m := Evaluate(ds, retrieve, nil) require.Equal(t, 2, m.Scored) - require.Greater(t, m.RecallAtK[5], 0.0, "BM25 should surface the gold candidate") + require.Greater(t, m.RecallAtK[5], 0.0, "the FTS should surface the gold candidate") require.Greater(t, m.MRR, 0.0) + // Both queries are multi-word, so they miss the store's exact-name + // Tier 0 short-circuit and are ranked by FTS bm25 — which puts each + // gold candidate first. Pinning the values keeps the assertion honest: + // a ranking inversion, or a corpus that reached the index untokenised, + // moves them. + require.InDelta(t, 1.0, m.RecallAtK[5], 1e-9) + require.InDelta(t, 1.0, m.MRR, 1e-9) } From df6ad4c7348cf0b9f2076772b6bb0dd9193783a1 Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Sun, 9 Aug 2026 21:40:05 +0200 Subject: [PATCH 05/21] refactor(eval): drop the recall ranker that wrapped the retired text backend MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BM25Ranker only ever adapted a raw search.Backend, and its sole caller was its own unit test — the shipping eval path registers the "bm25" row through EngineRanker over Engine.SearchSymbols, which reads the store's native symbol FTS. Dropping it removes the last tie from this package to the retired in-process text stack. The "bm25" row key stays put so bench artifacts remain joinable; only the prose that mislabelled it changes. --- BENCHMARK.md | 6 +++++ bench/fixtures/retrieval.yaml | 4 +-- bench/fixtures/retrieval_typo.yaml | 6 ++--- cmd/gortex/eval_recall.go | 16 +++++++----- internal/eval/recall/rankers.go | 38 ++++++----------------------- internal/eval/recall/recall_test.go | 22 ----------------- 6 files changed, 29 insertions(+), 63 deletions(-) diff --git a/BENCHMARK.md b/BENCHMARK.md index 427097936..d75ad258e 100644 --- a/BENCHMARK.md +++ b/BENCHMARK.md @@ -222,6 +222,12 @@ versus an LLM-judged setup. Per-tier R@5 (bm25): exact **96.8%** · concept 25.4% · multi_hop 30.0%. +_The `bm25` row key is historical. The row measures the shipping +lexical path — the store's native symbol FTS queried through +`Engine.SearchSymbols`, the same call `search_symbols` makes — not an +in-process BM25 index. The key is kept unchanged so older runs stay +joinable._ + **Headline**: the `search_symbols` text path (`bm25`) lands **R@5 = 55.1%** / **R@20 = 63.5%**, and **96.8%** on exact symbol-name queries — 3.2× ripgrep's R@5 floor. Enabling Porter diff --git a/bench/fixtures/retrieval.yaml b/bench/fixtures/retrieval.yaml index 2aa84265f..9449dbba0 100644 --- a/bench/fixtures/retrieval.yaml +++ b/bench/fixtures/retrieval.yaml @@ -154,7 +154,7 @@ cases: - { id: concept-ts-extract, tier: concept, query: "TypeScript language extractor", expected: [internal/parser/languages/typescript.go::TypeScriptExtractor] } - { id: concept-detect-lang, tier: concept, query: "detect language from file extension", expected: [internal/parser/registry.go::Registry.GetByExtension] } - { id: concept-register-all, tier: concept, query: "register every language extractor", expected: [internal/parser/languages/register.go::RegisterAll] } - - { id: concept-ranker-bm25, tier: concept, query: "eval ranker adapter for BM25", expected: [internal/eval/recall/rankers.go::BM25Ranker] } + - { id: concept-ranker-bm25, tier: concept, query: "eval ranker adapter for engine search", expected: [internal/eval/recall/rankers.go::EngineRanker] } - { id: concept-ranker-rrf, tier: concept, query: "eval ranker adapter for RRF hybrid", expected: [internal/eval/recall/rankers.go::RRFRanker] } - { id: concept-ranker-semantic, tier: concept, query: "eval ranker for vector-only semantic", expected: [internal/eval/recall/rankers.go::SemanticRanker] } - { id: concept-ranker-winnow, tier: concept, query: "eval ranker wrapping graph-aware winnow", expected: [internal/eval/recall/rankers.go::WinnowRanker] } @@ -269,7 +269,7 @@ cases: tier: multi_hop query: "ranker adapters in the eval package" expected: - - internal/eval/recall/rankers.go::BM25Ranker + - internal/eval/recall/rankers.go::EngineRanker - internal/eval/recall/rankers.go::SemanticRanker - internal/eval/recall/rankers.go::RRFRanker - internal/eval/recall/rankers.go::WinnowRanker diff --git a/bench/fixtures/retrieval_typo.yaml b/bench/fixtures/retrieval_typo.yaml index d048c6d79..ee3d4ba21 100644 --- a/bench/fixtures/retrieval_typo.yaml +++ b/bench/fixtures/retrieval_typo.yaml @@ -512,9 +512,9 @@ cases: - internal/parser/languages/register.go::RegisterAll - id: concept-ranker-bm25-typo tier: concept - query: eval rnker adapter for BM25 + query: eval rnker adapter for engine search expected: - - internal/eval/recall/rankers.go::BM25Ranker + - internal/eval/recall/rankers.go::EngineRanker - id: concept-ranker-rrf-typo tier: concept query: eval ranker dapter for RRF hybrid @@ -708,7 +708,7 @@ cases: tier: multi_hop query: ranker adapters in the eval packapge expected: - - internal/eval/recall/rankers.go::BM25Ranker + - internal/eval/recall/rankers.go::EngineRanker - internal/eval/recall/rankers.go::SemanticRanker - internal/eval/recall/rankers.go::RRFRanker - internal/eval/recall/rankers.go::WinnowRanker diff --git a/cmd/gortex/eval_recall.go b/cmd/gortex/eval_recall.go index 414d8cd0f..8af1d7b99 100644 --- a/cmd/gortex/eval_recall.go +++ b/cmd/gortex/eval_recall.go @@ -53,7 +53,8 @@ returned. Cases are tiered (exact / concept / multi_hop) so per-tier recall is broken out separately. Rankers: - bm25 — text-only (default text backend) + bm25 — lexical-only: the store-native symbol FTS queried + through Engine.SearchSymbols (the search_symbols path) semantic — vector-only (requires --embeddings) rrf — BM25 + vector fused via RRF (requires --embeddings) winnow — graph-aware constraint chain (MCP winnow_symbols scorer) @@ -152,7 +153,8 @@ func runEvalRecall(_ *cobra.Command, _ []string) error { // Peel the Swappable wrapper so we can see the real backend. When // embeddings are on, the indexer builds a HybridBackend internally; - // its TextBackend() is the pure lexical side, and the whole + // its TextBackend() is the pure lexical side — an adapter over the + // store's own symbol FTS, not a harness-built index — and the whole // HybridBackend is what RRF queries. inner := idx.Search() if sw, ok := inner.(*search.Swappable); ok { @@ -181,10 +183,12 @@ func runEvalRecall(_ *cobra.Command, _ []string) error { } // The engine-backed lexical row mirrors the MCP search_symbols call - // path (BM25 ranking + substring fallback for camelCase-only queries) - // against the store's own full-text index — the same backend the - // daemon serves, so the row reports what real callers hit rather than - // a harness-only in-process index. Critically: the engine is pointed + // path: the store-native FTS ranking plus the substring fallback for + // camelCase-only queries, run through Engine.SearchSymbols against + // the store's own full-text index — the same backend the daemon + // serves, so the row reports what real callers hit. The "bm25" row + // key is kept only so historical bench artifacts stay joinable; no + // in-process BM25 index is involved. Critically: the engine is pointed // at the PURE text backend even when --embeddings is on — otherwise // the "bm25" row would silently run through HybridBackend.Search (RRF // fusion with vector results) and the measurement would no longer diff --git a/internal/eval/recall/rankers.go b/internal/eval/recall/rankers.go index de11fcaf8..4a8133d33 100644 --- a/internal/eval/recall/rankers.go +++ b/internal/eval/recall/rankers.go @@ -14,37 +14,15 @@ import ( "github.com/zzet/gortex/internal/search" ) -// BM25Ranker adapts a plain search.Backend to the Ranker shape. Works -// for either a raw text backend — the in-process BM25 index the evals -// build, or the store-native FTS adapter production wires up — or a -// HybridBackend's text side extracted via HybridBackend.TextBackend(). -// -// Note: Gortex's indexer tokenizes symbol names at ingest time -// (Tokenize — camelCase-aware), but the query side (TokenizeQuery) does -// NOT split camelCase — so `backend.Search("NewServer", ...)` matches -// zero documents because the inverted index has `new` and `server` -// separately. The user-facing MCP search_symbols path avoids this by -// running through Engine.SearchSymbols, which adds a substring -// fallback. Use EngineRanker (below) to measure that full call path; -// use BM25Ranker only when you want the raw backend's behaviour. -func BM25Ranker(name string, backend search.Backend) Ranker { - return Ranker{ - Name: name, - Search: func(query string, limit int) []string { - hits := backend.Search(query, limit) - out := make([]string, len(hits)) - for i, h := range hits { - out[i] = h.ID - } - return out - }, - } -} - // EngineRanker measures what a real MCP caller sees via -// Engine.SearchSymbols — text-backend results + camelCase-friendly -// substring fallback. This is the recommended default for "bm25"- -// style evaluation; it reflects production behaviour. +// Engine.SearchSymbols — store-native FTS results plus the +// camelCase-friendly substring fallback. It is the only lexical +// adapter this package ships, and it backs the report's "bm25" row. +// +// The fallback is load-bearing: the indexer tokenizes symbol names at +// ingest time (camelCase-aware), but the query side does not split +// camelCase, so a raw backend query for "NewServer" matches nothing +// while Engine.SearchSymbols still finds it. func EngineRanker(name string, searchFn func(query string, limit int) []string) Ranker { return Ranker{Name: name, Search: searchFn} } diff --git a/internal/eval/recall/recall_test.go b/internal/eval/recall/recall_test.go index c687ca31a..6f8841564 100644 --- a/internal/eval/recall/recall_test.go +++ b/internal/eval/recall/recall_test.go @@ -5,8 +5,6 @@ import ( "testing" "github.com/stretchr/testify/assert" - - "github.com/zzet/gortex/internal/search" ) // staticRanker returns a predetermined ranked list regardless of query. @@ -135,26 +133,6 @@ func TestMarkdownSkippedRankerRow(t *testing.T) { assert.Contains(t, md, "skipped: no embedder") } -// TestBM25Ranker_AgainstRealBackend wires the adapter to a live BM25 -// backend and spot-checks ranked output. -func TestBM25Ranker_AgainstRealBackend(t *testing.T) { - backend := search.NewBM25() - backend.Add("pkg/a.go::Foo", "Foo", "pkg/a.go", "") - backend.Add("pkg/b.go::Bar", "Bar", "pkg/b.go", "") - - r := BM25Ranker("bm25", backend) - hits := r.Search("Foo", 5) - assert.NotEmpty(t, hits) - assert.Equal(t, "pkg/a.go::Foo", hits[0]) - - fixture := Fixture{Cases: []Case{ - {Query: "Bar", Tier: TierExact, Expected: []string{"pkg/b.go::Bar"}}, - {Query: "Foo", Tier: TierExact, Expected: []string{"pkg/a.go::Foo"}}, - }} - report := Run(fixture, []Ranker{r}, nil) - assert.Equal(t, 2, report.Rankers[0].Hits[1]) -} - func TestAdaptCasesForFileRanker(t *testing.T) { in := []Case{ {Query: "q", Expected: []string{"a/b.go::Foo", "a/b.go::Bar", "c/d.go::Baz"}}, From d0189deb0e9df8600c144c8415f74ae656740428 Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Sun, 9 Aug 2026 21:52:45 +0200 Subject: [PATCH 06/21] refactor(indexer): collapse the search build path onto the store's native FTS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every store now yields either a SymbolSearcherBackend or a NullBackend, so the branches that fed an in-process text index have no live path: drop the backend-shape probe, the ngram-boundary install, and the bulk Add loop. buildSearchIndex is now vector-only and returns immediately without an embedder, and the two rebuilds that existed only to bootstrap a non-persistent backend go with it. Incremental Add/Remove stay put — the native adapter keeps its document counter that way. --- internal/indexer/indexer.go | 81 +++++++------------ .../indexer/reconcile_clean_census_test.go | 22 ----- .../indexer/search_backend_native_test.go | 18 +---- internal/indexer/vector_plan.go | 45 ++--------- 4 files changed, 37 insertions(+), 129 deletions(-) diff --git a/internal/indexer/indexer.go b/internal/indexer/indexer.go index a9c740a41..5e238d32e 100644 --- a/internal/indexer/indexer.go +++ b/internal/indexer/indexer.go @@ -501,7 +501,11 @@ type contractCacheEntry struct { // Any backend (in-memory, SQLite-on-disk, remote) is acceptable — the // indexer's mutation paths go through the Store interface methods only, // so swapping backends is a zero-code-change configuration choice for -// callers. +// callers. Text search belongs to the store: a store implementing +// graph.SymbolSearcher answers queries from its own FTS, and every other +// store gets a null text backend whose empty corpus routes the query +// engine to its substring fallback. The indexer builds no text index of +// its own for either. func New(g graph.Store, reg *parser.Registry, cfg config.IndexConfig, logger *zap.Logger) *Indexer { idx := &Indexer{ graph: g, @@ -514,13 +518,11 @@ func New(g graph.Store, reg *parser.Registry, cfg config.IndexConfig, logger *za // Subsequent reassignments to idx.search should use the swap // helpers below. // - // When the backing store implements graph.SymbolSearcher - // (today only store_sqlite), the initial backend is a thin - // adapter that forwards Search to the store's native FTS. - // The in-process BM25 build path is then bypassed entirely — - // saving ~100MB heap on a Vscode-scale repo and putting - // search in the same address space as the rest of the graph - // queries. + // initialSearchBackend picks the text side: the store's own FTS + // when it implements graph.SymbolSearcher (today only + // store_sqlite), otherwise the null backend. Neither holds a + // corpus in this process, so search costs no heap beyond what the + // store already spends on the graph itself. search: search.NewSwappable(initialSearchBackend(g)), config: cfg, transforms: newTransformPipeline(cfg.Transforms, logger), @@ -667,11 +669,12 @@ func (d *vectorSearcherDelegate) SimilarTo(vec []float32, limit int) ([]graph.Ve // initialSearchBackend picks the search.Backend the indexer wraps // in its Swappable on construction. When the underlying store // implements graph.SymbolSearcher (today only store_sqlite), a -// thin adapter routes Search calls through the store's native FTS -// — the in-process BM25 build path is bypassed entirely. Every other -// store gets the null backend: it indexes nothing and reports an -// empty corpus, so the query engine answers from its own substring -// scan rather than from a second, in-process copy of the corpus. +// thin adapter routes Search calls through the store's native FTS. +// Every other store gets the null backend: it indexes nothing and +// reports an empty corpus, so the query engine answers from its own +// substring scan rather than from a second, in-process copy of the +// corpus. Those are the only two shapes, which is why no code path +// builds a text index in this process. func initialSearchBackend(g graph.Store) search.Backend { if s, ok := g.(graph.SymbolSearcher); ok { return search.NewSymbolSearcherBackend(s) @@ -679,37 +682,17 @@ func initialSearchBackend(g graph.Store) search.Backend { return search.NewNull() } -// isSymbolSearcherBackend reports whether the swappable's currently -// active backend is the SymbolSearcher adapter. Used to suppress the -// in-process index builds — if the active backend is already a native -// FTS, re-indexing the same corpus into a parallel in-process index -// would defeat the FTS path and pin the ~100MB heap the FTS -// integration was meant to release. -func isSymbolSearcherBackend(b search.Backend) bool { - switch backend := b.(type) { - case *search.SymbolSearcherBackend: - return true - case *search.Swappable: - inner, release := backend.AcquireBackend() - defer release() - return isSymbolSearcherBackend(inner) - case *search.HybridBackend: - return isSymbolSearcherBackend(backend.TextBackend()) - default: - return false - } -} - // ftsTokensFor produces the pre-tokenised text the backend FTS path // indexes. Mirrors searchIndexFields' field selection but joins // every field through search.Tokenize (camelCase / snake_case / -// path-segment splitter) so the resulting token list matches the -// in-process BM25 corpus contract — the same query produces the -// same recall against either backend. Joined with spaces so the +// path-segment splitter) — the same splitter buildFTSMatch runs over +// an incoming query, so a query token can only match a document token +// that was split the same way. Diverge here and identifier queries +// stop reaching the symbols that carry them. Joined with spaces so the // downstream COPY FROM sees a single STRING column value. func ftsTokensFor(n *graph.Node, projectName string) string { // searchIndexFields includes the resolver qualifier or its retrieval-only - // replacement, so both BM25 backends and embeddings see the same token bag. + // replacement, so both the FTS documents and embeddings see the same token bag. fields := searchIndexFields(n, projectName) tokens := make([]string, 0, 16) for _, f := range fields { @@ -3980,8 +3963,9 @@ func (idx *Indexer) repoNodeEdgeCount() (int, int) { // cleanCensusResult publishes the same zero-delta state as the full-root // incremental pipeline after ChangedSinceMtimes has already proved the tree -// unchanged. It preserves the one necessary side effect for non-persistent -// search backends without repeating filesystem discovery. +// unchanged. The store already owns the native text corpus, but the process-local +// vector channel must still be restored from durable corpus statistics (or rebuilt +// once when migration left that corpus empty). func (idx *Indexer) cleanCensusResult(ctx context.Context, detected int, started time.Time) (*IndexResult, error) { if idx.totalDetected == 0 { idx.totalDetected = detected @@ -3999,9 +3983,7 @@ func (idx *Indexer) cleanCensusResult(ctx context.Context, detected int, started idx.lastVectorBuildErr = restoreErr idx.logger.Warn("restore durable vector corpus failed; rebuilding", zap.Error(restoreErr)) } - if restored { - idx.rebuildTextSearchIndex() - } else if !isSymbolSearcherBackend(idx.search) || idx.embedder != nil { + if !restored && idx.embedder != nil { if err := idx.buildSearchIndexCtx(ctx); err != nil { return nil, err } @@ -5946,15 +5928,10 @@ func (idx *Indexer) incrementalReindexPathsMode( idx.reresolveCSharpGlobalUsingDependents(evictedGlobalsFiles, evicted) } - // Structural and metadata refreshes maintain both the in-memory search - // backend and persistent FTS one symbol at a time; deletions remove the - // prior symbols above. Rebuilding the whole corpus after a tiny edit would - // re-embed every unchanged symbol. The only rebuild retained here is the - // zero-delta bootstrap for a non-persistent backend restored beside an - // already-populated graph. - if len(staleFiles) == 0 && len(deletedFiles) == 0 && !isSymbolSearcherBackend(idx.search) { - idx.buildSearchIndex() - } + // No search rebuild here. Structural and metadata refreshes maintain the + // store's FTS one symbol at a time and deletions remove the prior symbols + // above, so the text corpus is already current; rebuilding it after a tiny + // edit would only re-embed every unchanged symbol. if len(staleFiles) > 0 || len(deletedFiles) > 0 { // Contract extraction is file-bounded even for body-only and metadata diff --git a/internal/indexer/reconcile_clean_census_test.go b/internal/indexer/reconcile_clean_census_test.go index 033e1ff37..9bf781fb3 100644 --- a/internal/indexer/reconcile_clean_census_test.go +++ b/internal/indexer/reconcile_clean_census_test.go @@ -59,28 +59,6 @@ func TestChangedSinceMtimesCensusDeletesNewlyExcludedTrackedFile(t *testing.T) { assert.Zero(t, detected) } -func TestCleanCensusResultBootstrapsNonPersistentSearch(t *testing.T) { - g := graph.New() - g.AddNode(&graph.Node{ - ID: "function::Alpha", - Kind: graph.KindFunction, - Name: "Alpha", - FilePath: "a.go", - }) - idx := newTestIndexer(g) - idx.search = search.NewBM25() - idx.SetFileMtimes(map[string]int64{"a.go": 1}) - - result, err := idx.cleanCensusResult(t.Context(), 1, time.Now()) - require.NoError(t, err) - require.NotNil(t, result) - assert.Equal(t, 1, result.FileCount) - assert.Equal(t, 1, result.NodeCount) - assert.Equal(t, 1, idx.TotalDetected()) - assert.Equal(t, 1, idx.search.Count()) - require.NotEmpty(t, idx.search.Search("Alpha", 10)) -} - func TestCleanCensusRestoresDurableVectorsWithoutEmbedding(t *testing.T) { root := vectorPersistFixture(t, 1) store, err := store_sqlite.Open(filepath.Join(t.TempDir(), "store.sqlite")) diff --git a/internal/indexer/search_backend_native_test.go b/internal/indexer/search_backend_native_test.go index c519e20ac..63c3b0f5d 100644 --- a/internal/indexer/search_backend_native_test.go +++ b/internal/indexer/search_backend_native_test.go @@ -19,7 +19,7 @@ func (s *countingSearchStore) AllNodes() []*graph.Node { return s.Store.AllNodes() } -func TestBuildSearchIndex_NativeTextWithoutVectorsSkipsNodeCensus(t *testing.T) { +func TestBuildSearchIndex_WithoutVectorsSkipsNodeCensus(t *testing.T) { store := &countingSearchStore{Store: graph.New()} backend := search.NewSwappable(search.NewSymbolSearcherBackend(nil)) defer backend.Close() @@ -29,19 +29,3 @@ func TestBuildSearchIndex_NativeTextWithoutVectorsSkipsNodeCensus(t *testing.T) assert.Zero(t, store.allNodesCalls) } - -func TestIsSymbolSearcherBackend_UnwrapsProductionLayers(t *testing.T) { - native := search.NewSymbolSearcherBackend(nil) - assert.True(t, isSymbolSearcherBackend(native)) - - swappable := search.NewSwappable(native) - assert.True(t, isSymbolSearcherBackend(swappable)) - - hybrid := search.NewHybrid(native, search.NewVector(1), nil) - assert.True(t, isSymbolSearcherBackend(hybrid)) - assert.True(t, isSymbolSearcherBackend(search.NewSwappable(hybrid))) - - bm25 := search.NewBM25() - defer bm25.Close() - assert.False(t, isSymbolSearcherBackend(bm25)) -} diff --git a/internal/indexer/vector_plan.go b/internal/indexer/vector_plan.go index 50f66a12c..0fc969bf8 100644 --- a/internal/indexer/vector_plan.go +++ b/internal/indexer/vector_plan.go @@ -51,54 +51,23 @@ func (p *preparedVectorPlan) Release() { p.embeddedText = 0 } -// prepareSearchIndex updates the process-local text index, then prepares the -// vector corpus without mutating either the durable vector store or the active -// vector backend. The caller decides when publication is safe. +// prepareSearchIndex prepares the vector corpus without mutating either the +// durable vector store or the active vector backend. The caller decides when +// publication is safe. func (idx *Indexer) prepareSearchIndex(ctx context.Context) (*preparedVectorPlan, error) { if ctx == nil { ctx = context.Background() } idx.lastVectorBuildErr = nil - - // Install learned sub-word boundaries before populating an in-process BM25 - // backend. Native SQLite FTS needs neither the census nor duplicate Adds. - search.BuildAndInstallNgramBoundaries(idx.search, idx.graph) - nativeText := isSymbolSearcherBackend(idx.search) - buildVectors := idx.embedder != nil - if nativeText && !buildVectors { + if idx.embedder == nil { return nil, nil } // Content sections belong to content search, not symbol/vector search. nodes := graph.RepoCodeNodes(idx.graph, idx.repoPrefix) - if !nativeText { - for _, n := range nodes { - if idx.shouldIndexForSearch(n) { - idx.search.Add(n.ID, searchIndexFields(n, idx.projectName)...) - } - } - } - if !buildVectors { - return nil, nil - } return idx.prepareVectorPlan(ctx, nodes) } -// rebuildTextSearchIndex restores only the non-persistent text channel. It is -// used on warm startup after a durable vector corpus has been published, so a -// paid embedding pass is not repeated merely to reconstruct BM25. -func (idx *Indexer) rebuildTextSearchIndex() { - search.BuildAndInstallNgramBoundaries(idx.search, idx.graph) - if isSymbolSearcherBackend(idx.search) { - return - } - for _, n := range graph.RepoCodeNodes(idx.graph, idx.repoPrefix) { - if idx.shouldIndexForSearch(n) { - idx.search.Add(n.ID, searchIndexFields(n, idx.projectName)...) - } - } -} - func (idx *Indexer) prepareVectorPlan(ctx context.Context, nodes []*graph.Node) (*preparedVectorPlan, error) { if err := ctx.Err(); err != nil { return nil, err @@ -230,9 +199,9 @@ func (idx *Indexer) prepareVectorPlan(ctx context.Context, nodes []*graph.Node) return plan, nil } -// prepareSearchIndexForPublication preserves the historical text-only -// degradation policy for provider/cap/validation failures. Parent cancellation -// is different: it aborts the index operation and must not publish anything. +// prepareSearchIndexForPublication retains the previous vector publication on +// provider, cap, or validation failures. Parent cancellation is different: it +// aborts the index operation and must not publish anything. func (idx *Indexer) prepareSearchIndexForPublication(ctx context.Context) (*preparedVectorPlan, error) { plan, err := idx.prepareSearchIndex(ctx) if err == nil { From de0eeb273c90493444f2005897c106c2db554b29 Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Sun, 9 Aug 2026 21:56:50 +0200 Subject: [PATCH 07/21] refactor(daemon): drop the retired text backend from the search status arms Status resolution no longer recognises the in-process text backend, which the indexer stopped building. In its place the null backend gets its own arm reporting "none" with zero counts, so a store without native symbol search reads as "nothing indexes text here" rather than falling through to "unknown", which means "we could not identify the backend". The status protocol doc lists the names the field can now carry. --- cmd/gortex/daemon_controller.go | 29 +++++++++++++----------- cmd/gortex/daemon_search_backend_test.go | 23 +++++++++++++++++++ internal/daemon/proto.go | 10 ++++---- 3 files changed, 45 insertions(+), 17 deletions(-) diff --git a/cmd/gortex/daemon_controller.go b/cmd/gortex/daemon_controller.go index 3cd7c02a2..8b7a0aff5 100644 --- a/cmd/gortex/daemon_controller.go +++ b/cmd/gortex/daemon_controller.go @@ -622,14 +622,13 @@ type searchBackendInfo struct { // document count, and its heap footprint. // // Real-world unwrap order: Swappable → HybridBackend → (text, vector). -// The text side is itself a concrete BM25/SymbolSearcherBackend. Both -// layers have to be peeled; if we stop early we fall into the default -// branch and the status reports "unknown" — which was the bug users -// saw. When the store implements graph.SymbolSearcher, the indexer -// wires up a *search.SymbolSearcherBackend instead of building an -// in-process BM25 index at all (see initialSearchBackend in -// internal/indexer/indexer.go) — that case has to be matched -// explicitly too, or it falls into the same "unknown" default. +// Both layers have to be peeled; if we stop early we fall into the +// default branch and the status reports "unknown" — which was the bug +// users saw. The text side is a concrete backend that has to be matched +// explicitly, or it lands in that same "unknown" default: the indexer +// wires up a *search.SymbolSearcherBackend when the store implements +// graph.SymbolSearcher and a *search.NullBackend when it does not (see +// initialSearchBackend in internal/indexer/indexer.go). func resolveSearchBackend(b search.Backend) searchBackendInfo { out := searchBackendInfo{} if b == nil { @@ -659,11 +658,6 @@ func resolveSearchBackend(b search.Backend) searchBackendInfo { } switch back := inner.(type) { - case *search.BM25Backend: - out.Name = "bm25" - out.DocCount = back.Count() - out.DocCountKnown = true - out.Bytes = back.SizeBytes() case *search.SymbolSearcherBackend: // The FTS5 index lives inside the graph store's own file, not a // separate in-memory structure — there is no honest byte count @@ -679,6 +673,15 @@ func resolveSearchBackend(b search.Backend) searchBackendInfo { out.Name = "sqlite-fts5" out.DiskResident = true out.DocCount, out.DocCountKnown = back.DocCount() + case *search.NullBackend: + // A store with no native symbol search gets the null text + // backend: it indexes nothing and the engine falls back to its + // substring path. Say so — an empty backend is a fact worth + // reporting, whereas the "unknown" default would imply we failed + // to recognise whatever is serving queries. Doc count and heap + // stay zero because both are honestly zero. + out.Name = "none" + out.DocCountKnown = true default: out.Name = "unknown" out.DocCount = inner.Count() diff --git a/cmd/gortex/daemon_search_backend_test.go b/cmd/gortex/daemon_search_backend_test.go index 6deb00381..318f9c36b 100644 --- a/cmd/gortex/daemon_search_backend_test.go +++ b/cmd/gortex/daemon_search_backend_test.go @@ -74,6 +74,29 @@ func TestResolveSearchBackend_SymbolSearcherBackend_ThroughSwappable(t *testing. assert.True(t, info.DiskResident) } +func TestResolveSearchBackend_NullBackend(t *testing.T) { + // A store with no native symbol search carries the null text backend. + // Status must name it, not report it as an unrecognised backend: the + // difference between "nothing is indexing text here" and "we could not + // identify what is" is exactly what a user reads this row for. + info := resolveSearchBackend(search.NewNull()) + + assert.Equal(t, "none", info.Name) + assert.False(t, info.DiskResident, "there is no index on disk either") + assert.Zero(t, info.Bytes) + assert.True(t, info.DocCountKnown, "zero documents is a known count, not an unanswerable one") + assert.Zero(t, info.DocCount) +} + +func TestRenderDaemonHeader_SearchBackendRow_NullBackend(t *testing.T) { + st := sampleStatus() + st.SearchBackend = daemon.SearchBackendStats{Name: "none", DocCountKnown: true} + var buf bytes.Buffer + renderDaemonHeader(&buf, st) + assert.Contains(t, buf.String(), "none docs=0 heap=0 B", + "an empty backend still gets a row, with its zeros stated plainly") +} + func TestRenderDaemonHeader_SearchBackendRow_SymbolSearcher(t *testing.T) { st := sampleStatus() st.SearchBackend = daemon.SearchBackendStats{ diff --git a/internal/daemon/proto.go b/internal/daemon/proto.go index a097fe83b..f34cf3882 100644 --- a/internal/daemon/proto.go +++ b/internal/daemon/proto.go @@ -468,11 +468,13 @@ type TrigramCacheStats struct { // SearchBackendStats identifies which search backend is currently // serving queries, so users can read the `search_b` column in the -// repo breakdown with the right mental model. The in-process BM25 -// index costs ~2 KiB of heap per document; the store-native FTS index -// lives inside the graph store's own file and costs no heap of its own. +// repo breakdown with the right mental model. The store-native FTS +// index lives inside the graph store's own file and costs no heap of +// its own. "none" means the store exposes no native symbol search, so +// no text index is serving queries at all and the engine answers from +// its substring fallback. type SearchBackendStats struct { - Name string `json:"name"` // "bm25" | "sqlite-fts5" | "unknown" + Name string `json:"name"` // "sqlite-fts5" | "none" | "unknown" DocCount int `json:"doc_count"` // indexed documents across all repos // DocCountKnown distinguishes "the index holds zero documents" from // "this backend cannot report a document count". Backends whose only From a3db3a4b88d6693ffaeabb1ddd01cdf80592f7a8 Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Sun, 9 Aug 2026 22:02:15 +0200 Subject: [PATCH 08/21] refactor(query): drop the typo-rescue tier no wired backend implements The bigram typo-rescue branch type-asserted the active text backend to an unexported bigramProvider interface that no wired backend satisfies: the method exists only on the BM25 backend, and neither the swappable wrapper nor the hybrid backend forwards it, so production's backend never matched. Delete the tier, its interface declaration, and the camelCase-boundary helper that only gated it. Behaviour-neutral: the branch was unreachable. --- internal/query/engine.go | 66 ++-------------------------------------- 1 file changed, 3 insertions(+), 63 deletions(-) diff --git a/internal/query/engine.go b/internal/query/engine.go index 70572fdd7..403f43bc5 100644 --- a/internal/query/engine.go +++ b/internal/query/engine.go @@ -999,56 +999,9 @@ func (e *Engine) gatherBackendCandidates(query string, limit int, opts QueryOpti } } - // Bigram-overlap typo rescue. Same gates as the legacy path: - // nothing else surfaced, query is one indivisible 4+ char token, - // backend can provide candidates. The bigram backend also returns - // raw IDs — batch-materialise them too rather than fall back to - // per-id GetNode. A query with a separator OR an internal-uppercase - // camelCase boundary is decomposable, so it is left for the handler's - // leaf-decomposition rescue (more precise than fuzzy bigram overlap) — - // the bigram tier serves true atomic-token typos only. - if len(cands) == 0 && len(query) >= 4 && !strings.ContainsAny(query, " /.:_-") && !hasInternalUppercase(query) { - if bg, ok := backend.(bigramProvider); ok { - keys := len(query) - 1 - minOverlap := (keys + 1) / 2 - if minOverlap < 3 { - minOverlap = 3 - } - bigramIDs := bg.BigramCandidates(query, minOverlap) - // Skip the batch fetch entirely when the bigram backend - // returned nothing — otherwise we'd issue an empty query - // round-trip. - if len(bigramIDs) > 0 { - bigramNodes := e.g.GetNodesByIDs(bigramIDs) - for _, id := range bigramIDs { - if _, seen := idx[id]; seen { - continue - } - node := bigramNodes[id] - if node == nil || node.Kind == graph.KindFile || node.Kind == graph.KindImport { - continue - } - idx[id] = len(cands) - cands = append(cands, &rerank.Candidate{Node: node, TextRank: -1, VectorRank: -1}) - if len(cands) >= limit { - break - } - } - } - } - } - return cands } -// bigramProvider is satisfied by backends that expose a typo-tolerant -// rescue list. Declared here (not in search) so the engine can adopt -// rescue without the search interface changing; any backend that can -// provide bigram candidates just has to implement this method. -type bigramProvider interface { - BigramCandidates(query string, minOverlap int) []string -} - const substringSearchPageSize = 256 type substringCandidate struct { @@ -1286,27 +1239,14 @@ func (e *Engine) Stats() *graph.GraphStats { return &s } -// bfs performs breadth-first traversal from nodeID. -// If forward is true, follows outgoing edges; if false, follows incoming. -// If edgeKinds is nil, follows all edge kinds bidirectionally (for cluster). -// hasInternalUppercase reports whether s carries a camelCase boundary — an -// uppercase letter anywhere but the first byte. Such a query decomposes into -// multiple leaf tokens, so the bigram typo-rescue tier defers to the handler's -// leaf-decomposition rescue for it. -func hasInternalUppercase(s string) bool { - for i := 1; i < len(s); i++ { - if s[i] >= 'A' && s[i] <= 'Z' { - return true - } - } - return false -} - // defaultDispatchFanout bounds how many overriders one interface/abstract // method expands to during polymorphic dispatch expansion, so a hub interface // with hundreds of implementors cannot blow up a call-chain walk. const defaultDispatchFanout = 24 +// bfs performs breadth-first traversal from nodeID. +// If forward is true, follows outgoing edges; if false, follows incoming. +// If edgeKinds is nil, follows all edge kinds bidirectionally (for cluster). func (e *Engine) bfs(nodeID string, opts QueryOptions, forward bool, edgeKinds []graph.EdgeKind) *SubGraph { if opts.Depth <= 0 { opts.Depth = 3 From 5f2a6b9ebba9a002bae2d7e63c039ef32f071507 Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Sun, 9 Aug 2026 22:33:20 +0200 Subject: [PATCH 09/21] refactor(search): retire the in-process BM25 text backend Search has run on the graph store's native symbol FTS since the indexer stopped building a parallel corpus, so the in-process BM25 index, its bigram typo rescue and the sparse-ngram tokenizer had no caller left. Delete them with the tests that only exercised them, move the hybrid de-chunk fixtures onto a map-backed text double and the path-scoping fixtures onto the ordered backend, and retarget the prose that still described the retired stack at the FTS adapter. --- CONTRIBUTING.md | 2 +- docs/04-evaluation/task-set.md | 5 +- docs/semantic-search.md | 4 - internal/config/config.go | 7 +- internal/indexer/lifecycle_acceptance_test.go | 2 +- internal/llm/svc/assist_e2e_test.go | 51 ++- .../mcp/change_contract_edit_paths_test.go | 2 +- internal/mcp/facade_tools_test.go | 7 +- internal/mcp/path_scope_test.go | 16 +- internal/mcp/tools_search_assist.go | 9 +- internal/query/engine.go | 12 +- internal/search/bench_test.go | 77 ----- internal/search/bigram.go | 236 -------------- internal/search/bigram_test.go | 76 ----- internal/search/bm25.go | 292 ------------------ internal/search/chunk_dechunk_test.go | 56 +++- internal/search/equivalence.go | 2 +- internal/search/fts_normalize.go | 12 +- internal/search/fts_normalize_test.go | 23 +- internal/search/hybrid.go | 26 +- internal/search/ngram_weights.go | 287 ----------------- internal/search/ngram_weights_test.go | 218 ------------- internal/search/project_name_test.go | 38 --- internal/search/rerank/pipeline.go | 7 +- internal/search/search.go | 22 +- internal/search/search_test.go | 111 ------- internal/search/sparse_ngram.go | 153 --------- internal/search/sparse_ngram_test.go | 152 --------- internal/search/swappable.go | 2 +- internal/search/symbolsearcher_backend.go | 11 +- 30 files changed, 157 insertions(+), 1761 deletions(-) delete mode 100644 internal/search/bigram.go delete mode 100644 internal/search/bigram_test.go delete mode 100644 internal/search/bm25.go delete mode 100644 internal/search/ngram_weights.go delete mode 100644 internal/search/ngram_weights_test.go delete mode 100644 internal/search/sparse_ngram.go delete mode 100644 internal/search/sparse_ngram_test.go diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 32498cce1..691a51cb5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -236,7 +236,7 @@ internal/ query/ Query engine (BFS traversal, SubGraph) resolver/ Cross-file reference resolution, IMPLEMENTS inference review/ PR review and diff analysis pipeline - search/ BM25, trigram, and semantic search + search/ FTS adapter, trigram, and semantic search server/ HTTP transports and the web dashboard ``` diff --git a/docs/04-evaluation/task-set.md b/docs/04-evaluation/task-set.md index b966a449f..37931e311 100644 --- a/docs/04-evaluation/task-set.md +++ b/docs/04-evaluation/task-set.md @@ -33,9 +33,8 @@ order of operations." `parser.ExtractionResult`; `Indexer.processExtraction` writes them into the `graph.Graph` and accumulates incoming-edge tracking for the next phase. -4. `Indexer.buildSearchIndex` (in-process BM25 in tests and evals, - store-native FTS in production) + `idx.embedder` (if set) - populate the search backends. +4. `Indexer.buildSearchIndex` (the store-native FTS) + + `idx.embedder` (if set) populate the search backends. 5. Semantic enrichment (`internal/semantic`) runs LSP / SCIP providers in parallel; resolved edges get `Origin=lsp_resolved` for tier filtering. diff --git a/docs/semantic-search.md b/docs/semantic-search.md index 87e4c186e..0b0528174 100644 --- a/docs/semantic-search.md +++ b/docs/semantic-search.md @@ -63,10 +63,6 @@ Centrality (HITS + PageRank) and a dedicated rerank signal weight call/reference - **Generated-file demotion** — a generated file (`*.pb.go`, `mock_*.go`, `*_pb2.py`, …) is ranked below a real same-named hand-written implementation, but only when one exists. - **Source over test** — when a query surfaces both an implementation and its test, the implementation is lifted above the test (only when both co-occur, so it never shifts the rest of the page). -### Sparse sub-word tokenization (opt-in) - -An optional tokenizer stage emits sub-word n-grams whose split points come from a per-repo boundary table learned from symbol names at index time, trading exact-identifier precision for recall on typo/fragment queries. Off by default (it is reindex-required and precision-sensitive); enable with `GORTEX_SPARSE_NGRAM=1`. Applies to the BM25 backend. - ## Keyword-soup defense Boolean / OR-soup queries (`A OR B OR 'no access' OR …`) — and operator-free keyword lists (`parse decode unmarshal token jwt cache`) and comma-enumerations — defeat embedding retrieval. The query classifier detects all three, skips wasted LLM expansion, and splits the soup into terms fused via the existing BM25 expansion path; a `query_advice` nudge rides on the response. Genuine natural-language questions stay classified as concept. Tune via `search.keyword_soup_rewrite: split | nudge | off`. diff --git a/internal/config/config.go b/internal/config/config.go index 59436fb6e..9e9661eaa 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -589,15 +589,14 @@ type IndexConfig struct { // SkipSearch is the effective text-index skip rules resolved from // Semantic.SkipSearch, same propagation pattern as SkipEmbed. // Users configure this under semantic.skip_search; the indexer - // reads it here. Controls what goes into the text search index - // (in-process BM25 in tests and evals, store-native FTS in - // production) — unlike SkipEmbed it doesn't affect the graph or + // reads it here. Controls what goes into the store-native FTS + // text index — unlike SkipEmbed it doesn't affect the graph or // vector index. SkipSearch []SkipEmbedRule `mapstructure:"-" yaml:"-"` // IndexProse is the effective prose-indexing toggle resolved from // Search.IndexProse -- same `-` (not on-disk) propagation pattern // as SkipSearch. When false, Markdown KindDoc prose-section nodes - // are kept out of the BM25 search index. Defaults to true. + // are kept out of the text search index. Defaults to true. IndexProse bool `mapstructure:"-" yaml:"-"` // MaxFileSize skips files larger than this during indexing. Zero // (the default) disables the cap — full coverage is preferred so diff --git a/internal/indexer/lifecycle_acceptance_test.go b/internal/indexer/lifecycle_acceptance_test.go index 59911baca..cfa6c8185 100644 --- a/internal/indexer/lifecycle_acceptance_test.go +++ b/internal/indexer/lifecycle_acceptance_test.go @@ -29,7 +29,7 @@ func newLifecycleTestMultiIndexer(t *testing.T) *MultiIndexer { return NewMultiIndexer( graph.New(), newTestRegistry(), - search.NewBM25(), + search.NewNull(), newTestConfigManager(t), zap.NewNop(), ) diff --git a/internal/llm/svc/assist_e2e_test.go b/internal/llm/svc/assist_e2e_test.go index db6f3d193..641114034 100644 --- a/internal/llm/svc/assist_e2e_test.go +++ b/internal/llm/svc/assist_e2e_test.go @@ -141,9 +141,9 @@ func TestE2E_AssistAgainstRealModel(t *testing.T) { t.Logf("VERDICT(hash-passwords): kept HashPassword=%v, dropped hashDiagnostics=%v", kept, dropped) }) - t.Run("VerifyRelevance_BM25", func(t *testing.T) { - query := "how does the BM25 search rank symbols" - cands := verifyBM25Scenario() + t.Run("VerifyRelevance_SymbolSearch", func(t *testing.T) { + query := "how does symbol search rank results" + cands := verifySymbolSearchScenario() t0 := time.Now() got, err := svcInst.VerifyRelevance(ctx, query, cands) t.Logf("VerifyRelevance %q → kept=%v dropped=%d (%v)", query, got.Keep, len(cands)-len(got.Keep), time.Since(t0)) @@ -152,11 +152,11 @@ func TestE2E_AssistAgainstRealModel(t *testing.T) { } assertVerifySubset(t, got.Keep, cands) - expectKept := "real.NewBM25" + expectKept := "real.NewSymbolSearcherBackend" expectDropped := "synthetic.unrelated.parseTSConfig" kept := containsID(got.Keep, expectKept) dropped := !containsID(got.Keep, expectDropped) - t.Logf("VERDICT(BM25): kept NewBM25=%v, dropped parseTSConfig=%v", kept, dropped) + t.Logf("VERDICT(symbol-search): kept NewSymbolSearcherBackend=%v, dropped parseTSConfig=%v", kept, dropped) }) } @@ -244,39 +244,36 @@ func verifyHashPasswordsScenario() []llm.VerifyCandidate { } } -// verifyBM25Scenario tests the model's ability to keep the genuine -// BM25 ranking implementation while dropping unrelated parsers / +// verifySymbolSearchScenario tests the model's ability to keep the +// genuine symbol-search ranking path while dropping unrelated parsers / // fixtures. -func verifyBM25Scenario() []llm.VerifyCandidate { +func verifySymbolSearchScenario() []llm.VerifyCandidate { return []llm.VerifyCandidate{ { - ID: "real.NewBM25", - Name: "NewBM25", - Signature: "func NewBM25() *BM25Backend", - Body: `func NewBM25() *BM25Backend { - return &BM25Backend{ - inverted: make(map[string][]posting), - bigrams: make(map[string]map[string]struct{}), - docs: make(map[string]doc), - } + ID: "real.NewSymbolSearcherBackend", + Name: "NewSymbolSearcherBackend", + Signature: "func NewSymbolSearcherBackend(s graph.SymbolSearcher) *SymbolSearcherBackend", + Body: `func NewSymbolSearcherBackend(s graph.SymbolSearcher) *SymbolSearcherBackend { + return &SymbolSearcherBackend{s: s} }`, Callers: []llm.CallerInfo{ {Name: "indexer.New", Signature: "func New(g *graph.Graph, ...) *Indexer"}, }, }, { - ID: "real.BM25Backend.Search", + ID: "real.SymbolSearcherBackend.Search", Name: "Search", - Signature: "func (b *BM25Backend) Search(query string, limit int) []scored", - Body: `func (b *BM25Backend) Search(query string, limit int) []scored { - terms := tokenize(query) - scores := map[string]float64{} - for _, t := range terms { - for _, p := range b.inverted[t] { - scores[p.id] += b.bm25Score(p, t) - } + Signature: "func (b *SymbolSearcherBackend) Search(query string, limit int) []SearchResult", + Body: `func (b *SymbolSearcherBackend) Search(query string, limit int) []SearchResult { + hits, err := b.s.SearchSymbols(query, limit) + if err != nil || len(hits) == 0 { + return nil } - return topK(scores, limit) + out := make([]SearchResult, len(hits)) + for i, h := range hits { + out[i] = SearchResult{ID: h.NodeID, Score: h.Score} + } + return out }`, Callers: []llm.CallerInfo{ {Name: "Engine.SearchSymbolsScoped", Signature: "func (e *Engine) SearchSymbolsScoped(q string, limit int, opts QueryOptions) []*graph.Node"}, diff --git a/internal/mcp/change_contract_edit_paths_test.go b/internal/mcp/change_contract_edit_paths_test.go index dd8c21460..0f505b87a 100644 --- a/internal/mcp/change_contract_edit_paths_test.go +++ b/internal/mcp/change_contract_edit_paths_test.go @@ -106,7 +106,7 @@ func setupContractMultiRepoServer(t *testing.T, names ...string) (*Server, *grap require.NoError(t, err) g := graph.New() - multi := indexer.NewMultiIndexer(g, testRegistry(), search.NewBM25(), manager, zap.NewNop()) + multi := indexer.NewMultiIndexer(g, testRegistry(), search.NewNull(), manager, zap.NewNop()) _, err = multi.IndexAll() require.NoError(t, err) diff --git a/internal/mcp/facade_tools_test.go b/internal/mcp/facade_tools_test.go index bcb2d3436..6a822e3ea 100644 --- a/internal/mcp/facade_tools_test.go +++ b/internal/mcp/facade_tools_test.go @@ -1609,12 +1609,12 @@ func TestBatchEditPreflightsAllItemsBeforeFirstWrite(t *testing.T) { func TestFacadeReadResolvesOnlyUniqueSymbolShorthand(t *testing.T) { g := graph.New() - bm := search.NewBM25() first := &graph.Node{ID: "pkg/a.go::UniqueReadTarget", Name: "UniqueReadTarget", Kind: graph.KindFunction, FilePath: "pkg/a.go"} g.AddNode(first) - bm.Add(first.ID, first.Name, first.FilePath, first.Name) eng := query.NewEngine(g) - eng.SetSearch(bm) + // The shorthand resolver reads FindNodesByName off the graph, never + // the text backend, so an inert one keeps the fixture honest. + eng.SetSearch(search.NewNull()) srv := NewServer(eng, g, nil, nil, zap.NewNop(), nil) resolved, ambiguous := srv.resolveFacadeSymbolShorthand(context.Background(), "UniqueReadTarget") @@ -1623,7 +1623,6 @@ func TestFacadeReadResolvesOnlyUniqueSymbolShorthand(t *testing.T) { second := &graph.Node{ID: "pkg/b.go::UniqueReadTarget", Name: "UniqueReadTarget", Kind: graph.KindFunction, FilePath: "pkg/b.go"} g.AddNode(second) - bm.Add(second.ID, second.Name, second.FilePath, second.Name) resolved, ambiguous = srv.resolveFacadeSymbolShorthand(context.Background(), "UniqueReadTarget") require.Equal(t, "UniqueReadTarget", resolved) require.ElementsMatch(t, []string{first.ID, second.ID}, ambiguous) diff --git a/internal/mcp/path_scope_test.go b/internal/mcp/path_scope_test.go index 71f9c6e64..8e963b47f 100644 --- a/internal/mcp/path_scope_test.go +++ b/internal/mcp/path_scope_test.go @@ -120,22 +120,32 @@ func TestResolvePathFilter_Sources(t *testing.T) { func pathScopeServer(t *testing.T) *Server { t.Helper() g := graph.New() - bm := search.NewBM25() + // Every symbol is retrievable by its whole name and by each of the + // camelCase words in it, so the queries below reach all three and + // only the path filter can confine the result. + ob := newOrderedBackend() files := map[string]string{ "services/billing/Invoice.go": "BillingInvoice", "services/auth/Login.go": "AuthLogin", "libs/money/Amount.go": "MoneyAmount", } + tokens := map[string][]string{ + "BillingInvoice": {"billinginvoice", "billing", "invoice"}, + "AuthLogin": {"authlogin", "auth", "login"}, + "MoneyAmount": {"moneyamount", "money", "amount"}, + } for path, name := range files { id := path + "::" + name g.AddNode(&graph.Node{ ID: id, Kind: graph.KindFunction, Name: name, FilePath: path, StartLine: 1, EndLine: 5, Language: "go", }) - bm.Add(id, name, path, "") + for _, tok := range tokens[name] { + ob.put(tok, id) + } } eng := query.NewEngine(g) - eng.SetSearch(bm) + eng.SetSearch(ob) return NewServer(eng, g, nil, nil, zap.NewNop(), nil) } diff --git a/internal/mcp/tools_search_assist.go b/internal/mcp/tools_search_assist.go index 152bf9c1a..ad4b36bed 100644 --- a/internal/mcp/tools_search_assist.go +++ b/internal/mcp/tools_search_assist.go @@ -286,8 +286,7 @@ func anchorTermsToVocabulary(terms []string, ac *search.AutoConcepts) []string { // expansion hits append in their own BM25 order with duplicates // skipped. // -// Both BM25 backends (BM25Backend and the on-disk backend's FTS) -// treat a multi-token query as an OR-style union +// The store's FTS treats a multi-token query as an OR-style union // with a single global BM25 score, so one combined call replaces // the prior N per-term fan-out (the N+1 round-trip pattern dominated // the search hot path on disk backends). @@ -649,9 +648,9 @@ func verifyWithLLM(ctx context.Context, s *Server, query string, nodes []*graph. // // Before partitioning into head/tail, nodes are re-sorted so callable // kinds (function / method) come before everything else — preserving -// BM25 order within each bucket. Without this, a high-scoring param -// or field node (e.g. `BM25Backend.Search#param:limit`) can pre-empt -// the enclosing method (`BM25Backend.Search`) inside the rerank +// retrieval order within each bucket. Without this, a high-scoring +// param or field node (e.g. `Engine.SearchSymbols#param:limit`) can +// pre-empt the enclosing method (`Engine.SearchSymbols`) inside the rerank // window, leaving the model unable to surface the real callable. func rerankWithLLM(ctx context.Context, s *Server, query string, nodes []*graph.Node) []*graph.Node { if s.llmService == nil || !s.llmService.Enabled() || len(nodes) < 2 { diff --git a/internal/query/engine.go b/internal/query/engine.go index 403f43bc5..58b352a6d 100644 --- a/internal/query/engine.go +++ b/internal/query/engine.go @@ -16,7 +16,8 @@ import ( // SearchProvider is a function that returns the current search backend. // This allows the engine to always use the latest backend even when the -// indexer replaces it (e.g., wrapping BM25 in HybridBackend for embeddings). +// indexer replaces it (e.g., wrapping the text backend in HybridBackend +// for embeddings). type SearchProvider func() search.Backend // Engine provides higher-level query operations over the graph. @@ -492,9 +493,8 @@ func (e *Engine) GetCluster(nodeID string, opts QueryOptions) *SubGraph { // SearchSymbols performs full-text search across all nodes. // When a search backend is configured, uses that backend's ranking — -// the in-process BM25 index in tests and evals, the store-native FTS -// index in production — with camelCase-aware tokenization. Falls back -// to substring matching otherwise. +// the store-native FTS index — with camelCase-aware tokenization. +// Falls back to substring matching otherwise. func (e *Engine) SearchSymbols(query string, limit int) []*graph.Node { return e.SearchSymbolsScoped(query, limit, QueryOptions{}) } @@ -682,9 +682,9 @@ func repoAllowList(allow map[string]bool) []string { return out } -// gatherBackendCandidates fetches BM25 + (optional) vector results, +// gatherBackendCandidates fetches text + (optional) vector results, // dedups them across channels, and supplements with exact-name / -// substring / bigram-rescue matches. Each candidate carries its +// substring matches. Each candidate carries its // 0-based TextRank and VectorRank (or -1 when the channel didn't // return it) so the rerank pipeline can score per channel. // diff --git a/internal/search/bench_test.go b/internal/search/bench_test.go index 53bc274f6..a15bb7776 100644 --- a/internal/search/bench_test.go +++ b/internal/search/bench_test.go @@ -1,7 +1,6 @@ package search import ( - "fmt" "testing" ) @@ -48,79 +47,3 @@ func BenchmarkTokenizeQuery(b *testing.B) { }) } } - -// ============================================================================= -// BM25 scaled benchmarks — memory and latency at different corpus sizes -// ============================================================================= - -func buildBM25(size int) *BM25Backend { - b := NewBM25() - for i := range size { - b.Add( - fmt.Sprintf("pkg%d/file%d.go::func%d", i/100, i/10, i), - fmt.Sprintf("getUserById%d", i%50), - fmt.Sprintf("internal/pkg%d/service%d.go", i/100, i/10), - fmt.Sprintf("func getUserById%d(id string) User", i%50), - ) - } - return b -} - -func BenchmarkBM25_Search_Scaled(b *testing.B) { - sizes := []struct { - name string - n int - }{ - {"100_symbols", 100}, - {"1K_symbols", 1_000}, - {"5K_symbols", 5_000}, - {"10K_symbols", 10_000}, - } - - for _, sz := range sizes { - b.Run(sz.name, func(b *testing.B) { - b.ReportAllocs() - backend := buildBM25(sz.n) - b.ResetTimer() - for b.Loop() { - backend.Search("get user auth", 20) - } - }) - } -} - -func BenchmarkBM25_Add(b *testing.B) { - b.ReportAllocs() - backend := NewBM25() - for i := range b.N { - backend.Add( - fmt.Sprintf("file%d.go::func%d", i/10, i), - fmt.Sprintf("processRequest%d", i), - fmt.Sprintf("pkg/file%d.go", i/10), - fmt.Sprintf("func processRequest%d()", i), - ) - } -} - -func BenchmarkBM25_Remove(b *testing.B) { - backend := buildBM25(5000) - b.ReportAllocs() - b.ResetTimer() - for i := range b.N { - backend.Remove(fmt.Sprintf("pkg%d/file%d.go::func%d", (i%5000)/100, (i%5000)/10, i%5000)) - } -} - -func BenchmarkBM25_ConcurrentSearch(b *testing.B) { - backend := buildBM25(5000) - queries := []string{"get user", "server handle", "graph node", "parse extract", "resolve import"} - b.ReportAllocs() - b.ResetTimer() - b.RunParallel(func(pb *testing.PB) { - i := 0 - for pb.Next() { - backend.Search(queries[i%len(queries)], 20) - i++ - } - }) -} diff --git a/internal/search/bigram.go b/internal/search/bigram.go deleted file mode 100644 index 45d46791c..000000000 --- a/internal/search/bigram.go +++ /dev/null @@ -1,236 +0,0 @@ -package search - -import ( - "os" - "sort" - "strings" - "sync" -) - -// bigramIndexEnabled reports whether the bigram side index should be -// built on Add/Remove and consulted by the engine's typo-rescue tier. -// Default ON: the typo-rescue tier only fires when the primary search -// returns ZERO results (see the rescue-on-empty caller), so the index -// earns its keep — an empty result set is exactly when a typo is the -// likely cause and the bigram overlap is the cheapest way to recover -// recall. A perf-sensitive operator on a very large index can opt OUT -// with GORTEX_BIGRAM_TYPOS=0 (or false / no / off). Read once at backend -// construction so the flag can't toggle mid-session. -func bigramIndexEnabled() bool { - switch strings.ToLower(strings.TrimSpace(os.Getenv("GORTEX_BIGRAM_TYPOS"))) { - case "0", "false", "no", "off", "n": - return false - } - return true -} - -// bigramIndex is an inverted index from bigram key → docID set, built -// alongside BM25's primary index. Its single purpose is typo-tolerant -// recall: when BM25 returns zero hits for a query, the caller can fall -// back to bigram overlap to find the nearest-matching symbols. The index -// tracks both consecutive bigrams (chars i, i+1) and skip-1 bigrams -// (chars i, i+2) — the latter is the FFF trick that makes "validat_e" -// transpositions match "validate". -type bigramIndex struct { - mu sync.RWMutex - // bigrams keys each bigram (hi<<8 | lo) to the set of docIDs that - // contain it at least once in any indexed token. We use a set - // (map[string]struct{}) rather than a slice so Remove is O(1) and the - // per-bigram densities at compress time come from a simple len(). - bigrams map[uint16]map[string]struct{} - // perDoc tracks the bigrams each doc contributed, for clean Remove. - perDoc map[string][]uint16 -} - -func newBigramIndex() *bigramIndex { - return &bigramIndex{ - bigrams: make(map[uint16]map[string]struct{}), - perDoc: make(map[string][]uint16), - } -} - -// bigramize yields every consecutive and skip-1 bigram key for one -// lowercase token. Non-ASCII bytes are silently skipped so the key space -// stays in uint16 (one byte each side) — good enough for symbol names -// which are almost universally ASCII in real codebases. -func bigramize(token string) []uint16 { - b := []byte(strings.ToLower(token)) - if len(b) < 2 { - return nil - } - out := make([]uint16, 0, 2*len(b)) - // Consecutive pairs. - for i := 1; i < len(b); i++ { - a, c := b[i-1], b[i] - if a > 127 || c > 127 { - continue - } - out = append(out, uint16(a)<<8|uint16(c)) - } - // Skip-1 pairs — typo resilience for single-char substitutions and - // transpositions. FFF encodes these in a separate column; we pool - // them into the same key space which costs some density but halves - // the index footprint. - for i := 2; i < len(b); i++ { - a, c := b[i-2], b[i] - if a > 127 || c > 127 { - continue - } - out = append(out, uint16(a)<<8|uint16(c)) - } - return out -} - -// Add indexes one doc under all bigrams found in any of its tokens. -// Called from BM25Backend.Add with the same tokens that feed the BM25 -// posting lists, so the two indexes stay in lockstep without a second -// tokenization pass. -func (bi *bigramIndex) Add(docID string, tokens []string) { - if bi == nil || docID == "" || len(tokens) == 0 { - return - } - seen := make(map[uint16]struct{}, 16) - for _, t := range tokens { - for _, k := range bigramize(t) { - seen[k] = struct{}{} - } - } - if len(seen) == 0 { - return - } - - bi.mu.Lock() - defer bi.mu.Unlock() - - // Drop any prior presence before re-indexing so a re-Add doesn't leave - // orphan bigrams from the old token set. - bi.removeLocked(docID) - - keys := make([]uint16, 0, len(seen)) - for k := range seen { - set, ok := bi.bigrams[k] - if !ok { - set = make(map[string]struct{}) - bi.bigrams[k] = set - } - set[docID] = struct{}{} - keys = append(keys, k) - } - bi.perDoc[docID] = keys -} - -// Remove deletes a doc from every bigram's set and clears its perDoc list. -func (bi *bigramIndex) Remove(docID string) { - if bi == nil || docID == "" { - return - } - bi.mu.Lock() - defer bi.mu.Unlock() - bi.removeLocked(docID) -} - -func (bi *bigramIndex) removeLocked(docID string) { - keys := bi.perDoc[docID] - for _, k := range keys { - if set, ok := bi.bigrams[k]; ok { - delete(set, docID) - if len(set) == 0 { - delete(bi.bigrams, k) - } - } - } - delete(bi.perDoc, docID) -} - -// Candidates returns docIDs whose token bigram set overlaps the query by -// at least minOverlap distinct bigrams. Density-filtered: bigrams that -// appear in highDocPct of all docs are ignored — very -// rare bigrams are noise, very common ones add no signal. Same defaults -// FFF uses (~3% / 90%). -func (bi *bigramIndex) Candidates(query string, minOverlap int) []string { - if bi == nil || query == "" { - return nil - } - keys := bigramize(query) - if len(keys) == 0 { - return nil - } - if minOverlap < 1 { - minOverlap = 1 - } - - bi.mu.RLock() - defer bi.mu.RUnlock() - - total := len(bi.perDoc) - if total == 0 { - return nil - } - - // Density thresholds. - const ( - lowDocPct = 3 - highDocPct = 90 - ) - loBound := (total * lowDocPct) / 100 - if loBound < 1 { - loBound = 1 - } - hiBound := (total * highDocPct) / 100 - if hiBound < 1 { - hiBound = total - } - - // Overlap count per candidate doc. - overlap := make(map[string]int) - for _, k := range keys { - set, ok := bi.bigrams[k] - if !ok { - continue - } - if len(set) < loBound || len(set) > hiBound { - continue - } - for docID := range set { - overlap[docID]++ - } - } - - // Collect candidates above threshold along with their overlap count, - // then sort by overlap descending so the caller can take top-N by - // similarity rather than by Go's random map-iteration order — the - // latter buried the best match past rank 20 on typo'd exact queries. - type cand struct { - id string - count int - } - cands := make([]cand, 0, len(overlap)) - for id, c := range overlap { - if c >= minOverlap { - cands = append(cands, cand{id, c}) - } - } - sort.Slice(cands, func(i, j int) bool { - if cands[i].count != cands[j].count { - return cands[i].count > cands[j].count - } - // Equal-overlap candidates tie-break on ID for stable ordering. - return cands[i].id < cands[j].id - }) - out := make([]string, len(cands)) - for i, c := range cands { - out[i] = c.id - } - return out -} - -// Size reports the number of distinct bigrams currently indexed. -// Exposed for tests and stats. -func (bi *bigramIndex) Size() int { - if bi == nil { - return 0 - } - bi.mu.RLock() - defer bi.mu.RUnlock() - return len(bi.bigrams) -} diff --git a/internal/search/bigram_test.go b/internal/search/bigram_test.go deleted file mode 100644 index 33ae47375..000000000 --- a/internal/search/bigram_test.go +++ /dev/null @@ -1,76 +0,0 @@ -package search - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestBigramize_ConsecutiveAndSkip(t *testing.T) { - keys := bigramize("abcd") - // Consecutive: ab, bc, cd. Skip-1: ac, bd. Total = 5. - require.Len(t, keys, 5) - - // Shorter-than-2 input yields nothing. - assert.Empty(t, bigramize("a")) - assert.Empty(t, bigramize("")) -} - -func TestBigramIndex_RoundTrip(t *testing.T) { - bi := newBigramIndex() - bi.Add("a", []string{"validate", "token"}) - bi.Add("b", []string{"valid", "input"}) - bi.Add("c", []string{"completely", "unrelated"}) - - // The doc count is 3; density filter permits bigrams appearing in - // [1, 2] docs — exclude any that hit all 3. None should hit all 3 - // here, so the filter is a no-op and the test is deterministic. - cands := bi.Candidates("validate", 4) - assert.Contains(t, cands, "a", "validate's own bigrams should put it top") - - // Removed doc disappears. - bi.Remove("a") - cands = bi.Candidates("validate", 4) - assert.NotContains(t, cands, "a") -} - -func TestBM25_BigramCandidates_ExplicitAPI(t *testing.T) { - // Feature is opt-in via GORTEX_BIGRAM_TYPOS — default-off backends - // allocate no bigram index and return nil from BigramCandidates. - t.Setenv("GORTEX_BIGRAM_TYPOS", "1") - // This test asserts the strict-Search contract: a clean BM25 miss - // returns empty. The sub-word n-gram gate deliberately relaxes that - // (a typo shares sub-word grams with the symbol), so pin it off here - // regardless of any ambient GORTEX_SPARSE_NGRAM — the same hermetic - // pattern withStemming uses for the FTS gate. - withSparseNgram(t, false) - - b := NewBM25() - defer b.Close() - - b.Add("auth/token.go::validateToken", "validateToken", "auth/token.go") - b.Add("api/handler.go::handleRequest", "handleRequest", "api/handler.go") - b.Add("db/store.go::openDB", "openDB", "db/store.go") - - ids := b.BigramCandidates("valiadate", 4) - assert.Contains(t, ids, "auth/token.go::validateToken") - - // Plain Search stays strict — rescue is the engine's job, not the - // backend's. Backend returns empty on a clean BM25 miss. - assert.Empty(t, b.Search("valiadate", 10)) -} - -func TestBM25_BigramOptOut(t *testing.T) { - // The bigram index defaults ON now — the typo-rescue tier fires only on - // zero-result queries, so it earns its keep. A perf-sensitive operator - // opts OUT with GORTEX_BIGRAM_TYPOS=0, restoring the zero-cost path where - // Add/Remove cost nothing extra and BigramCandidates returns nil. - t.Setenv("GORTEX_BIGRAM_TYPOS", "0") - - b := NewBM25() - defer b.Close() - b.Add("auth/token.go::validateToken", "validateToken", "auth/token.go") - - assert.Nil(t, b.BigramCandidates("valiadate", 4)) -} diff --git a/internal/search/bm25.go b/internal/search/bm25.go deleted file mode 100644 index 6701e400c..000000000 --- a/internal/search/bm25.go +++ /dev/null @@ -1,292 +0,0 @@ -package search - -import ( - "math" - "sort" - "sync" -) - -// BM25Backend is a custom in-memory inverted index with BM25 scoring. -// Optimal for repos up to ~50k symbols. Zero external dependencies. -type BM25Backend struct { - mu sync.RWMutex - docs map[string]*doc // docID -> document - inverted map[string][]posting // term -> postings list - totalLen int // sum of all doc lengths (for avgLen) - bigrams *bigramIndex // side index for typo-tolerant fallback recall - // ngrams is the optional learned sub-word boundary source consulted - // by the sparse-ngram emission stage. nil until SetNgramBoundaries - // wires one in; the stage degrades to fixed character n-grams while - // it is nil. Read under mu alongside the postings so Add and Search - // always see the same source — the symmetry the sparse-ngram gate - // depends on. The whole stage is a no-op unless GORTEX_SPARSE_NGRAM - // is set. - ngrams NgramBoundaries -} - -// SetNgramBoundaries installs the learned sub-word boundary source the -// sparse-ngram stage consults. Passing nil (or an empty source) reverts -// the stage to fixed character n-grams. Safe to call while the backend -// is live; it takes the write lock so an in-flight Search never sees a -// half-swapped source. Callers that rebuild the table on every index -// pass (mirroring auto-concept mining) re-install it here. -// -// NOTE: when the gate is on, changing the boundary source changes which -// sub-word grams a token emits. To keep the index and query paths in -// lockstep the source should be installed before the backend is -// populated and then left stable for the backend's lifetime — exactly -// how the per-repo table is built once per RunAnalysis pass and handed -// to a freshly (re)built backend. -func (b *BM25Backend) SetNgramBoundaries(src NgramBoundaries) { - b.mu.Lock() - defer b.mu.Unlock() - b.ngrams = src -} - -// boundarySource returns the currently installed sub-word boundary -// source under the read lock, so the sparse-ngram stage on the index -// and query paths reads a consistent value even if SetNgramBoundaries -// races with an in-flight Add / Search. -func (b *BM25Backend) boundarySource() NgramBoundaries { - b.mu.RLock() - defer b.mu.RUnlock() - return b.ngrams -} - -type doc struct { - id string - len int - terms map[string]int // term -> frequency in this doc -} - -type posting struct { - docID string - freq int -} - -// BM25 parameters. -const ( - bm25K1 = 1.2 - bm25B = 0.75 -) - -// SizeBytes is a rough memory estimate for the BM25 in-memory index: -// every document stores an ID + term-frequency map, and every term in -// the inverted index carries a postings list. The per-doc and per-term -// constants are calibrated against live indexes and land within ~25% -// of actual heap delta. -func (b *BM25Backend) SizeBytes() uint64 { - b.mu.RLock() - defer b.mu.RUnlock() - var bytes uint64 - for _, d := range b.docs { - // doc struct + id string + terms map header - bytes += 96 + uint64(len(d.id)) + 48 - // Each term entry: key string header + ~8 bytes for the int frequency. - for term := range d.terms { - bytes += uint64(len(term)) + 24 - } - } - for term, postings := range b.inverted { - // term string + slice header + postings - bytes += uint64(len(term)) + 24 - bytes += uint64(len(postings)) * 32 // docID string hdr + freq int + ptr - } - return bytes -} - -// NewBM25 creates a new BM25 search backend. The bigram side index for -// typo-tolerant rescue is built only when GORTEX_BIGRAM_TYPOS is set; -// leaving it nil is cheap — every bigram method is nil-safe and returns -// the zero-cost branch, so the engine's typo-rescue tier becomes a no-op -// automatically. -func NewBM25() *BM25Backend { - b := &BM25Backend{ - docs: make(map[string]*doc), - inverted: make(map[string][]posting), - } - if bigramIndexEnabled() { - b.bigrams = newBigramIndex() - } - return b -} - -func (b *BM25Backend) Add(id string, fields ...string) { - // Tokenize all fields together. - var allTokens []string - for _, f := range fields { - allTokens = append(allTokens, Tokenize(f)...) - } - - // Stopword-filter + Porter-stem the posting tokens. Search() - // runs the same normalization, so stemmed postings are always probed - // with stemmed query terms. The bigram side index keeps the raw - // tokens — its typo rescue bigramizes the raw query string, so - // raw-against-raw stays consistent there. - ftsTokens := NormalizeFTSTokens(allTokens) - - // Optional sub-word n-gram expansion. Search() runs the identical - // stage on the same normalized tokens with the same boundary source, - // so n-grammed postings are always probed with n-grammed query - // terms. A no-op unless GORTEX_SPARSE_NGRAM is set; the original - // word tokens are preserved, so an exact match still scores. - ftsTokens = ExpandSparseNgrams(ftsTokens, b.boundarySource()) - - termFreq := make(map[string]int) - for _, t := range ftsTokens { - termFreq[t]++ - } - - b.mu.Lock() - defer b.mu.Unlock() - - // Remove old version if exists. - b.removeLocked(id) - - d := &doc{ - id: id, - len: len(ftsTokens), - terms: termFreq, - } - b.docs[id] = d - b.totalLen += d.len - - for term, freq := range termFreq { - b.inverted[term] = append(b.inverted[term], posting{id, freq}) - } - - // Keep the bigram side index in lockstep — same token set, same doc ID. - b.bigrams.Add(id, allTokens) -} - -func (b *BM25Backend) Remove(id string) { - b.mu.Lock() - defer b.mu.Unlock() - b.removeLocked(id) -} - -func (b *BM25Backend) removeLocked(id string) { - d, ok := b.docs[id] - if !ok { - return - } - - b.totalLen -= d.len - - // Remove from inverted index. - for term := range d.terms { - postings := b.inverted[term] - for i, p := range postings { - if p.docID == id { - b.inverted[term] = append(postings[:i], postings[i+1:]...) - break - } - } - if len(b.inverted[term]) == 0 { - delete(b.inverted, term) - } - } - - delete(b.docs, id) - b.bigrams.Remove(id) -} - -func (b *BM25Backend) Search(query string, limit int) []SearchResult { - queryTokens := NormalizeFTSTokens(TokenizeQuery(query)) - // Mirror the index path's sub-word n-gram expansion exactly — same - // stage, same normalized tokens, same boundary source — so a query - // probes the same n-grammed terms that Add wrote into the postings. - // A no-op unless GORTEX_SPARSE_NGRAM is set. - queryTokens = ExpandSparseNgrams(queryTokens, b.boundarySource()) - if len(queryTokens) == 0 { - return nil - } - - b.mu.RLock() - defer b.mu.RUnlock() - - docCount := len(b.docs) - if docCount == 0 { - return nil - } - - avgLen := float64(b.totalLen) / float64(docCount) - scores := make(map[string]float64) - - for _, term := range queryTokens { - postings, ok := b.inverted[term] - if !ok { - continue - } - df := float64(len(postings)) - idf := math.Log((float64(docCount)-df+0.5)/(df+0.5) + 1) - - for _, p := range postings { - d := b.docs[p.docID] - if d == nil { - continue - } - tf := float64(p.freq) - dl := float64(d.len) - score := idf * (tf * (bm25K1 + 1)) / (tf + bm25K1*(1-bm25B+bm25B*dl/avgLen)) - scores[p.docID] += score - } - } - - if len(scores) == 0 { - // The engine layer has its own fallback chain — exact-name match - // then substring contains — that handles queries like "NewServer" - // which the backend's camelCase-split tokenization misses. We stay - // strict here so those higher-precision fallbacks can run; typo - // rescue via bigram overlap belongs one level up, after those. - return nil - } - - // Sort by score descending. - type scored struct { - id string - score float64 - } - results := make([]scored, 0, len(scores)) - for id, score := range scores { - results = append(results, scored{id, score}) - } - sort.Slice(results, func(i, j int) bool { - if results[i].score != results[j].score { - return results[i].score > results[j].score - } - // Tie-break on doc ID so an equal-score run ships in a stable - // order across calls — Go's map iteration is otherwise random. - return results[i].id < results[j].id - }) - - if len(results) > limit { - results = results[:limit] - } - - out := make([]SearchResult, len(results)) - for i, r := range results { - out[i] = SearchResult{ID: r.id, Score: r.score} - } - return out -} - -// BigramCandidates exposes the bigram-overlap list for explicit typo-mode -// callers. minOverlap gates how similar a doc must be — the caller picks -// the strictness. -func (b *BM25Backend) BigramCandidates(query string, minOverlap int) []string { - if b.bigrams == nil { - return nil - } - return b.bigrams.Candidates(query, minOverlap) -} - -func (b *BM25Backend) Count() int { - b.mu.RLock() - defer b.mu.RUnlock() - return len(b.docs) -} - -func (b *BM25Backend) Close() { - // No-op for in-memory backend. -} diff --git a/internal/search/chunk_dechunk_test.go b/internal/search/chunk_dechunk_test.go index 82c234f6c..75fd2e6b8 100644 --- a/internal/search/chunk_dechunk_test.go +++ b/internal/search/chunk_dechunk_test.go @@ -2,12 +2,59 @@ package search import ( "context" + "slices" + "sort" + "strings" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) +// mapText is a map-backed text Backend for the hybrid fixtures. The +// production text side is the store-native FTS, which lives behind a +// graph.SymbolSearcher this package must not reach for; these tests +// only need *a* text channel next to the vector one, so the double +// keeps the corpus in a map and ranks by how many query tokens a +// document's own tokens cover, ties broken by ID for determinism. +type mapText struct{ docs map[string][]string } + +func newMapText() *mapText { return &mapText{docs: make(map[string][]string)} } + +func (m *mapText) Add(id string, fields ...string) { + m.docs[id] = Tokenize(strings.Join(fields, " ")) +} + +func (m *mapText) Remove(id string) { delete(m.docs, id) } +func (m *mapText) Count() int { return len(m.docs) } +func (m *mapText) Close() {} + +func (m *mapText) Search(query string, limit int) []SearchResult { + terms := TokenizeQuery(query) + var out []SearchResult + for id, tokens := range m.docs { + matched := 0 + for _, tok := range tokens { + if slices.Contains(terms, tok) { + matched++ + } + } + if matched > 0 { + out = append(out, SearchResult{ID: id, Score: float64(matched)}) + } + } + sort.Slice(out, func(i, j int) bool { + if out[i].Score != out[j].Score { + return out[i].Score > out[j].Score + } + return out[i].ID < out[j].ID + }) + if limit > 0 && len(out) > limit { + out = out[:limit] + } + return out +} + // fixedEmbedder is a deterministic test embedding provider: every query // embeds to the same constant vector, so HybridBackend.Search exercises // the vector channel without any model. The vector backend is what @@ -54,7 +101,7 @@ func TestHybridSearch_DeChunksToParent(t *testing.T) { "big.go::Big#chunk2": "big.go::Big", }) - text := NewBM25() + text := newMapText() text.Add("big.go::Big", "Big", "big.go", "") text.Add("small.go::Small", "Small", "small.go", "") @@ -87,9 +134,8 @@ func TestHybridSearch_DeChunkPreservesOrder(t *testing.T) { "b.go::B#chunk0": "b.go::B", }) - // Keep construction realistic; this assertion exercises the vector - // de-chunk order directly, before channel fusion. - h := NewHybrid(NewBM25(), vec, fixedEmbedder{dims: dims}) + // Empty text backend so only the vector channel decides ordering. + h := NewHybrid(newMapText(), vec, fixedEmbedder{dims: dims}) got := h.dechunkVectorIDs(vec.Search([]float32{1, 0, 0}, 8), 8) require.Len(t, got, 2) @@ -106,7 +152,7 @@ func TestHybridSearch_NoChunkMapUnaffected(t *testing.T) { vec.Add("b.go::B", []float32{0, 1, 0}) require.False(t, vec.HasChunks(), "no SetChunkMap → HasChunks must be false") - h := NewHybrid(NewBM25(), vec, fixedEmbedder{dims: dims}) + h := NewHybrid(newMapText(), vec, fixedEmbedder{dims: dims}) results := h.Search("anything", 10) for _, r := range results { assert.NotContains(t, r.ID, "#chunk") diff --git a/internal/search/equivalence.go b/internal/search/equivalence.go index e8a0f51f4..690338cbf 100644 --- a/internal/search/equivalence.go +++ b/internal/search/equivalence.go @@ -18,7 +18,7 @@ import ( // whose members are genuinely interchangeable in code identifiers // belong here. Domain-bearing words that mean different things in // different codebases are left out -- a false synonym inflates the -// BM25 candidate pool with noise. +// text candidate pool with noise. type EquivalenceTable struct { // member maps each lowercased word to the index of its class in // classes. A word in two classes keeps the first; the curated diff --git a/internal/search/fts_normalize.go b/internal/search/fts_normalize.go index 89959d2a0..2ced4e82a 100644 --- a/internal/search/fts_normalize.go +++ b/internal/search/fts_normalize.go @@ -16,7 +16,7 @@ import ( // reranking every identifier query. Enable it with // GORTEX_FTS_STEMMING=1 (also true / yes / on). // -// Read once at process start, like the bigram-typo flag: the index +// Read once at process start: the index // built during a daemon's lifetime and every query against it share a // single setting, so a mid-session toggle can't desynchronise stemmed // postings from stemmed query terms. When enabled, the same @@ -47,11 +47,11 @@ var ftsStopWords = map[string]struct{}{ "it": {}, "its": {}, "so": {}, "such": {}, "via": {}, "per": {}, } -// NormalizeFTSTokens applies the FR63 stopword filter and Porter stemmer -// to a token list produced by Tokenize / TokenizeQuery. The index path -// (BM25Backend.Add) and the query path (BM25Backend.Search) both -// call it, so a stemmed -// posting list is always probed with stemmed query terms. +// NormalizeFTSTokens applies the stopword filter and Porter stemmer to +// a token list produced by Tokenize / TokenizeQuery. Every producer of +// FTS terms routes through it — both the tokens written into the index +// and the tokens a query is lowered into — so a stemmed posting list is +// always probed with stemmed query terms. // // Stopwords are dropped before stemming so a stemmed form can never // collide with a stopword entry. The result is a freshly allocated diff --git a/internal/search/fts_normalize_test.go b/internal/search/fts_normalize_test.go index 9d85e0e86..a90016d31 100644 --- a/internal/search/fts_normalize_test.go +++ b/internal/search/fts_normalize_test.go @@ -4,7 +4,6 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) // withStemming pins the FTS normalization gate to a known value for @@ -62,23 +61,9 @@ func TestStemFTSToken_LeavesShortAndNonAlphaTokens(t *testing.T) { } } -func TestBM25_StemmedRecall(t *testing.T) { +func TestNormalizeFTSTokens_AllStopWordsYieldsNoTokens(t *testing.T) { withStemming(t, true) - b := NewBM25() - defer b.Close() - b.Add("svc::UserService", "UserService", "user/service.go") - // A query in a different grammatical number still reaches it. - res := b.Search("users", 10) - require.NotEmpty(t, res) - assert.Equal(t, "svc::UserService", res[0].ID) -} - -func TestBM25_StopWordQueryYieldsNoTokens(t *testing.T) { - withStemming(t, true) - b := NewBM25() - defer b.Close() - b.Add("a::Auth", "Auth", "auth.go") - // An all-stopword query produces no FTS tokens, so BM25 returns - // nothing and the engine layer's substring fallback takes over. - assert.Empty(t, b.Search("the and of", 10)) + // An all-stopword query normalizes to nothing, so the caller issues + // no FTS match at all and the engine's substring fallback takes over. + assert.Empty(t, NormalizeFTSTokens([]string{"the", "and", "of"})) } diff --git a/internal/search/hybrid.go b/internal/search/hybrid.go index ce5eccc1c..5e0ff5938 100644 --- a/internal/search/hybrid.go +++ b/internal/search/hybrid.go @@ -9,11 +9,11 @@ import ( "github.com/zzet/gortex/internal/search/rerank" ) -// HybridBackend combines text search (BM25 or the store-native FTS -// adapter) with vector search (HNSW) using query-adaptive, α-weighted -// Reciprocal Rank Fusion (RRF). Identifier-shaped queries lean toward BM25, -// where exact-token matches are most reliable; natural-language queries give -// semantic similarity more weight so synonymous wording can surface. +// HybridBackend combines text search (the store-native FTS adapter) +// with vector search (HNSW) using query-adaptive, α-weighted +// Reciprocal Rank Fusion (RRF). Identifier-shaped queries lean toward the +// text channel, where exact-token matches are most reliable; natural-language +// queries give semantic similarity more weight so synonymous wording can surface. type HybridBackend struct { text Backend vector *VectorBackend @@ -50,7 +50,7 @@ func (h *HybridBackend) Remove(id string) { } // Search runs both text and vector search and fuses them with adaptive -// α-weighted RRF. Identifier queries lean toward BM25; natural-language +// α-weighted RRF. Identifier queries lean toward text search; natural-language // queries give semantic similarity more weight. func (h *HybridBackend) Search(query string, limit int) []SearchResult { textResults, vecIDs, _ := h.searchChannels(query, limit) @@ -63,7 +63,7 @@ func (h *HybridBackend) Search(query string, limit int) []SearchResult { return alphaFuse(textResults, vecIDs, rerank.AlphaFor(query), h.k, limit) } -// SearchChannels returns the raw per-channel results — BM25 ranks +// SearchChannels returns the raw per-channel results — text ranks // (with scores) and the parallel vector-search ID list — without // RRF fusion. The rerank pipeline calls this so each channel can // contribute as a separate Signal instead of being collapsed into a @@ -83,7 +83,7 @@ type ChannelTimings struct { } // VectorChannelOnly returns the vector-channel IDs (embedder + ANN -// search) WITHOUT re-running the text BM25 path. Used by the engine +// search) WITHOUT re-running the text search. Used by the engine // when the text channel has already been satisfied via the bundle // path — the bundle returns Nodes + edges + scores already, so // re-running text Search would double-pay the FTS cost. Returns @@ -112,7 +112,7 @@ func (h *HybridBackend) VectorChannelOnly(query string, limit int) ([]string, Ch } // SearchChannelsTimed is SearchChannels with a per-phase timing -// breakdown so callers can prove which sub-step (text BM25 vs +// breakdown so callers can prove which sub-step (text FTS vs // vector embed vs vector ANN) actually cost wall-clock time. // Used by the MCP search_symbols handler's debug-log // instrumentation; production callers that don't care just use @@ -131,7 +131,7 @@ func (h *HybridBackend) SearchChannelsTimed(query string, limit int) ([]SearchRe // HybridBackend wires both channels together in production, so the // engine's bundle-detection step type-asserts on the outer // HybridBackend through Swappable; this is what makes the bundle -// path available when the daemon's search is the BM25 + vector +// path available when the daemon's search is the FTS + vector // stack instead of a bare SymbolSearcherBackend. func (h *HybridBackend) SearchSymbolBundles(query string, limit int) []SymbolBundle { if h == nil || h.text == nil { @@ -275,9 +275,9 @@ func (h *HybridBackend) VectorSizeBytes() uint64 { return h.vector.SizeBytes() } // alphaFuse combines text and vector results with an α-weighted blend // of their reciprocal-rank contributions. Higher α gives the vector // channel more weight (good for natural-language queries where -// semantic similarity catches synonyms); lower α gives BM25 more -// weight (good for identifier queries where exact-token matches are -// the most reliable signal). +// semantic similarity catches synonyms); lower α gives the text channel +// more weight (good for identifier queries where exact-token matches +// are the most reliable signal). // // Formula: // diff --git a/internal/search/ngram_weights.go b/internal/search/ngram_weights.go deleted file mode 100644 index d1e25b650..000000000 --- a/internal/search/ngram_weights.go +++ /dev/null @@ -1,287 +0,0 @@ -package search - -import ( - "sort" - "unicode" - - "github.com/zzet/gortex/internal/graph" -) - -// NgramTable is a per-repository, LLM-free table of learned sub-word -// boundary weights mined from the symbol names in the graph. Where the -// fixed character n-gram stage cuts every token at a fixed width, this -// table cuts each token at *high-information* boundaries — positions -// where the adjacent character pair is rare across the repo's symbol -// vocabulary, which is where a name tends to seam (the "tk" in -// "validateTokenizer" is far rarer than the "to"/"ke"/"en" inside -// "token", so the split lands at the seam, not mid-word). -// -// Go has no compile-time table generation, so this is NOT a comptime -// literal map: it is computed at index / analysis time, once per -// RunAnalysis pass, exactly like the auto-concept vocabulary. The build -// is one tokenizing pass over node names plus a bounded character-pair -// count, cheap enough to recompute on every reindex. -// -// The table feeds the sparse-ngram tokenizer (see ExpandSparseNgrams): -// when a non-empty table is installed on the backend, the tokenizer -// asks it where to Split each word token instead of slicing at a fixed -// n. A nil or empty table degrades the tokenizer to fixed character -// n-grams, so the search path is identical whether or not the table has -// been built yet. -type NgramTable struct { - // boundaryPairs holds the character bigrams whose normalized - // corpus frequency is low enough to count as a split seam. A pair - // is packed as (hi<<16 | lo) of its two runes; only pairs over the - // ASCII-letter alphabet are tracked, which covers essentially all - // real symbol names. Presence in the set means "split between these - // two characters". The set is derived deterministically from the - // sorted frequency table at build time and never mutated after. - boundaryPairs map[uint32]struct{} -} - -// Ngram boundary-mining bounds. Mirrors the auto-concept caps in spirit -// — keep the pair count and the boundary set bounded on a large -// monorepo, and require enough evidence before trusting a seam. -const ( - // ngramMinTokenChars is the shortest token the boundary miner will - // split. A token this short or shorter has no interior seam worth - // learning; the tokenizer keeps it whole (or, in fixed mode, emits - // its single full-length gram). - ngramMinTokenChars = 5 - // ngramMinCorpusPairs is the minimum number of distinct adjacent - // character pairs the corpus must yield before the table trusts its - // frequency distribution. Below this the sample is too thin to tell - // a rare seam from noise, and the table stays empty so the - // tokenizer falls back to fixed n-grams. - ngramMinCorpusPairs = 8 - // ngramBoundaryPercentile selects the rarest adjacent pairs as - // seams: a pair counts as a boundary when its frequency rank falls - // in the bottom this-many percent of all observed pairs. Lower = - // fewer, higher-confidence seams. - ngramBoundaryPercentile = 25 -) - -// BuildNgramBoundaries mines the per-repo sub-word boundary table from a -// graph. Only named code symbols (the same kinds auto-concept mining -// uses) contribute their names. A nil or empty graph — or one whose -// symbol names yield too few distinct character pairs to be -// trustworthy — yields an empty, safe-to-query table that reports -// Empty() == true, so the tokenizer degrades to fixed character -// n-grams. -// -// The build is deterministic: every map is drained into a sorted slice -// before any threshold is derived or any boundary is selected, so two -// builds over the same graph produce byte-identical boundary sets. -func BuildNgramBoundaries(g graph.Reader) *NgramTable { - t := &NgramTable{boundaryPairs: map[uint32]struct{}{}} - if g == nil { - return t - } - - // Pass 1: count how often each adjacent character pair occurs - // inside the repo's symbol-name word tokens. We reuse the - // auto-concept tokenizer so the boundary table is learned over the - // same word vocabulary the rest of search tokenizes on. - pairCount := map[uint32]int{} - for _, n := range g.AllNodes() { - if !autoConceptEligible(n.Kind) { - continue - } - for _, tok := range autoConceptTokens(n.Name) { - countAdjacentPairs(tok, pairCount) - } - } - - // Too thin a sample: keep the table empty rather than learn noise. - if len(pairCount) < ngramMinCorpusPairs { - return t - } - - // Pass 2: rank the pairs by frequency, ascending, with a stable - // tie-break on the packed key so the ordering — and therefore the - // percentile cut — is identical across runs regardless of Go's - // random map iteration. The rarest pairs are the high-information - // seams. - type pairFreq struct { - key uint32 - count int - } - ranked := make([]pairFreq, 0, len(pairCount)) - for k, c := range pairCount { - ranked = append(ranked, pairFreq{k, c}) - } - sort.Slice(ranked, func(i, j int) bool { - if ranked[i].count != ranked[j].count { - return ranked[i].count < ranked[j].count - } - return ranked[i].key < ranked[j].key - }) - - // Select the bottom percentile as boundaries. At least one seam is - // kept whenever the sample cleared the minimum, so a small but - // valid corpus still learns something. - cut := (len(ranked) * ngramBoundaryPercentile) / 100 - if cut < 1 { - cut = 1 - } - for _, pf := range ranked[:cut] { - t.boundaryPairs[pf.key] = struct{}{} - } - return t -} - -// InstallNgramBoundaries walks a search backend down to the BM25 layer -// and installs the learned boundary table on it, so the sparse-ngram -// tokenizer's split decisions become data-driven. The production -// backend is a Swappable wrapping either a HybridBackend (text+vector) -// or a bare BM25Backend; this unwraps both. Backends with no BM25 layer -// (SymbolSearcher) do not run the sparse-ngram stage, so there -// is nothing to install and the call is a harmless no-op returning -// false. -// -// Symmetry contract: install the table before the backend is populated -// and leave it stable for the backend's lifetime. The index and query -// paths both read the installed table per call, so a single stable -// table keeps n-grammed postings and n-grammed query terms in lockstep. -// Installing a different table after postings exist would desynchronise -// them while the gate is on — callers re-install only as part of a -// fresh (re)index, never against a live, already-populated index. -func InstallNgramBoundaries(backend Backend, table NgramBoundaries) bool { - if swappable, ok := backend.(*Swappable); ok { - inner, release := swappable.AcquireBackend() - defer release() - return InstallNgramBoundaries(inner, table) - } - bm := bm25Of(backend) - if bm == nil { - return false - } - bm.SetNgramBoundaries(table) - return true -} - -// BuildAndInstallNgramBoundaries mines and installs a boundary table only -// when backend actually contains a BM25 layer. Capability detection must -// precede BuildNgramBoundaries: the native SQLite FTS never consumes the -// table, and walking the whole graph for them is pure allocation and I/O. -func BuildAndInstallNgramBoundaries(backend Backend, g graph.Reader) bool { - if swappable, ok := backend.(*Swappable); ok { - inner, release := swappable.AcquireBackend() - defer release() - return BuildAndInstallNgramBoundaries(inner, g) - } - bm := bm25Of(backend) - if bm == nil { - return false - } - bm.SetNgramBoundaries(BuildNgramBoundaries(g)) - return true -} - -// bm25Of unwraps a non-swappable backend down to its *BM25Backend, or -// returns nil when the backend has no BM25 layer. Swappable callers are -// handled by the public installation functions so the returned pointer is -// consumed while its AcquireBackend pin remains held. -func bm25Of(backend Backend) *BM25Backend { - switch b := backend.(type) { - case *BM25Backend: - return b - case *HybridBackend: - return bm25Of(b.TextBackend()) - default: - return nil - } -} - -// countAdjacentPairs tallies every adjacent ASCII-letter character pair -// in one lowercase token into counts. Pairs touching a non-ASCII or -// non-letter rune are skipped — digits and symbols are not part of the -// learned alphabet, mirroring how the FTS stemmer leaves digit-bearing -// tokens alone. The token is assumed already lowercased by -// autoConceptTokens. -func countAdjacentPairs(tok string, counts map[uint32]int) { - r := []rune(tok) - for i := 1; i < len(r); i++ { - a, b := r[i-1], r[i] - if !isLearnableRune(a) || !isLearnableRune(b) { - continue - } - counts[packPair(a, b)]++ - } -} - -// isLearnableRune reports whether a rune participates in the learned -// pair alphabet: ASCII letters only. -func isLearnableRune(r rune) bool { - return r <= unicode.MaxASCII && unicode.IsLetter(r) -} - -// packPair packs two runes into a single uint32 key (hi<<16 | lo). -// Both runes are ASCII here, so they fit in 16 bits each with room to -// spare. -func packPair(a, b rune) uint32 { - return uint32(a)<<16 | uint32(b) -} - -// Empty reports whether the table learned any boundaries. A nil table, -// or one mined from an empty / too-thin graph, is empty — callers MUST -// treat an empty table as "no learned boundaries" and degrade to fixed -// behaviour rather than splitting on nothing. Nil-safe so a typed-nil -// *NgramTable stored in the NgramBoundaries interface still answers -// correctly. -func (t *NgramTable) Empty() bool { - return t == nil || len(t.boundaryPairs) == 0 -} - -// BoundaryCount reports the number of learned seam pairs. Used by tests -// and diagnostics. -func (t *NgramTable) BoundaryCount() int { - if t == nil { - return 0 - } - return len(t.boundaryPairs) -} - -// Split cuts a token's runes at the learned high-information boundaries -// and returns the resulting segments left-to-right. A split is taken -// between positions i-1 and i when the adjacent character pair is a -// learned seam. The token is never split so finely that a segment falls -// below the minimum gram length: a candidate seam that would strand a -// sub-minimal segment on either side is skipped, so Split always yields -// segments the tokenizer can use. When the table is empty Split returns -// the whole token as a single segment, leaving the fixed-n fallback to -// the caller. -// -// Split is deterministic — it scans left to right and consults only the -// immutable boundary set — and never mutates the input. -func (t *NgramTable) Split(runes []rune) []string { - if t.Empty() || len(runes) < ngramMinTokenChars { - return []string{string(runes)} - } - - var ( - segs []string - start int - ) - for i := 1; i < len(runes); i++ { - a, b := runes[i-1], runes[i] - if !isLearnableRune(a) || !isLearnableRune(b) { - continue - } - if _, seam := t.boundaryPairs[packPair(a, b)]; !seam { - continue - } - // Only take the seam if both the segment it closes and the - // remainder it opens can still carry a usable gram — guards - // against shredding the token into sub-minimal fragments. - left := i - start - right := len(runes) - i - if left < sparseNgramMinN || right < sparseNgramMinN { - continue - } - segs = append(segs, string(runes[start:i])) - start = i - } - segs = append(segs, string(runes[start:])) - return segs -} diff --git a/internal/search/ngram_weights_test.go b/internal/search/ngram_weights_test.go deleted file mode 100644 index e29e1b707..000000000 --- a/internal/search/ngram_weights_test.go +++ /dev/null @@ -1,218 +0,0 @@ -package search - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/zzet/gortex/internal/graph" -) - -// boundaryFixtureGraph builds a graph whose function names share a -// dominant interior vocabulary ("token", "validate", "handler") so the -// adjacent-pair frequency distribution has a clear common core and rare -// seams. Mirrors fixtureGraph in auto_concepts_test.go. -func boundaryFixtureGraph(names []string) *graph.Graph { - g := graph.New() - for i, n := range names { - g.AddNode(&graph.Node{ - ID: "pkg/f.go::" + n, - Kind: graph.KindFunction, - Name: n, - FilePath: "pkg/f.go", - StartLine: i + 1, - EndLine: i + 2, - Language: "go", - }) - } - return g -} - -type countingNgramReader struct { - graph.Reader - allNodesCalls int -} - -func (r *countingNgramReader) AllNodes() []*graph.Node { - r.allNodesCalls++ - return r.Reader.AllNodes() -} - -func TestBuildNgramBoundaries_NilAndEmpty(t *testing.T) { - // nil graph -> empty table -> tokenizer degrades to fixed behavior. - require.True(t, BuildNgramBoundaries(nil).Empty()) - if c := BuildNgramBoundaries(nil).BoundaryCount(); c != 0 { - t.Errorf("nil graph should yield 0 boundaries, got %d", c) - } - // Empty graph: no names, no pairs, empty table. - require.True(t, BuildNgramBoundaries(graph.New()).Empty()) - // A typed-nil table must answer Empty() true without panicking. - var tnil *NgramTable - require.True(t, tnil.Empty()) -} - -func TestNgramTable_EmptyTableSplitIsWhole(t *testing.T) { - // An empty table splits nothing: the token comes back whole, leaving - // the fixed-n fallback to the tokenizer. - tbl := BuildNgramBoundaries(graph.New()) - require.True(t, tbl.Empty()) - got := tbl.Split([]rune("validateToken")) - assert.Equal(t, []string{"validateToken"}, got) -} - -func TestBuildNgramBoundaries_Deterministic(t *testing.T) { - names := []string{ - "validateToken", "validateTokenizer", "tokenValidator", - "handleToken", "tokenHandler", "parseToken", "tokenParser", - "refreshToken", "tokenRefresher", "revokeToken", - "validateHandler", "handlerValidator", "buildHandler", - } - a := BuildNgramBoundaries(boundaryFixtureGraph(names)) - b := BuildNgramBoundaries(boundaryFixtureGraph(names)) - // Two builds over equivalent graphs must produce byte-identical - // boundary sets — no Go map-iteration nondeterminism may leak into - // the percentile cut. - require.Equal(t, a.BoundaryCount(), b.BoundaryCount()) - for k := range a.boundaryPairs { - _, ok := b.boundaryPairs[k] - assert.Truef(t, ok, "boundary %d present in build A but not build B", k) - } - // And, crucially, the Split decisions must agree token for token. - for _, tok := range names { - assert.Equalf(t, a.Split([]rune(tok)), b.Split([]rune(tok)), - "Split(%q) differs between two deterministic builds", tok) - } -} - -func TestBuildNgramBoundaries_LearnsAReasonableSeam(t *testing.T) { - // A corpus dominated by the substring "token": the pairs inside - // "token" (to, ok, ke, en) are frequent, so a rare cross-word pair - // is selected as a seam and at least one multi-token name splits. - names := []string{ - "tokenAlpha", "tokenBeta", "tokenGamma", "tokenDelta", - "tokenEpsilon", "tokenZeta", "alphaToken", "betaToken", - "gammaToken", "deltaToken", "tokenize", "tokenizer", - "retoken", "subtoken", "tokenList", "tokenMap", - } - tbl := BuildNgramBoundaries(boundaryFixtureGraph(names)) - require.False(t, tbl.Empty(), "a corpus this size should learn boundaries") - - // At least one eligible token must split into >1 segment, otherwise - // the table learned nothing usable. - splitSomething := false - for _, tok := range names { - if len(tbl.Split([]rune(tok))) > 1 { - splitSomething = true - break - } - } - assert.True(t, splitSomething, "the learned table should split at least one token") -} - -func TestNgramTable_SplitNeverStrandsSubMinimalSegments(t *testing.T) { - names := []string{ - "validateToken", "validateTokenizer", "tokenValidator", - "handleToken", "tokenHandler", "parseToken", "tokenParser", - "refreshToken", "tokenRefresher", "revokeToken", - } - tbl := BuildNgramBoundaries(boundaryFixtureGraph(names)) - if tbl.Empty() { - t.Skip("fixture learned no boundaries; nothing to assert") - } - for _, tok := range names { - segs := tbl.Split([]rune(tok)) - if len(segs) == 1 { - continue // unsplit token: the whole thing, fine. - } - for _, s := range segs { - assert.GreaterOrEqualf(t, len([]rune(s)), sparseNgramMinN, - "Split(%q) stranded a sub-minimal segment %q", tok, s) - } - } -} - -func TestNgramTable_SplitShortTokenWhole(t *testing.T) { - tbl := BuildNgramBoundaries(boundaryFixtureGraph([]string{ - "tokenAlpha", "tokenBeta", "tokenGamma", "tokenDelta", - "alphaToken", "betaToken", "gammaToken", "deltaToken", - "tokenize", "tokenizer", - })) - // A token shorter than the minimum is never split. - got := tbl.Split([]rune("go")) - assert.Equal(t, []string{"go"}, got) -} - -// TestInstallNgramBoundaries_BM25Chain verifies the install helper finds -// the BM25 layer through the production Swappable wrapper and that a -// non-BM25 backend is a harmless no-op. -func TestInstallNgramBoundaries_BM25Chain(t *testing.T) { - bm := NewBM25() - defer bm.Close() - tbl := BuildNgramBoundaries(boundaryFixtureGraph([]string{ - "tokenAlpha", "tokenBeta", "tokenGamma", "tokenDelta", - "alphaToken", "betaToken", "tokenize", "tokenizer", - })) - - // Bare BM25. - assert.True(t, InstallNgramBoundaries(bm, tbl)) - - // Wrapped in a Swappable, as in production. - sw := NewSwappable(NewBM25()) - defer sw.Close() - assert.True(t, InstallNgramBoundaries(sw, tbl)) - - // A backend with no BM25 layer: no-op, returns false. - assert.False(t, InstallNgramBoundaries(nonBM25Backend{}, tbl)) -} - -// nonBM25Backend is an inert Backend with no BM25 anywhere in its -// unwrap chain — the shape the ngram installers must refuse. -type nonBM25Backend struct{} - -func (nonBM25Backend) Add(string, ...string) {} -func (nonBM25Backend) Remove(string) {} -func (nonBM25Backend) Search(string, int) []SearchResult { return nil } -func (nonBM25Backend) Count() int { return 0 } -func (nonBM25Backend) Close() {} - -func TestBuildAndInstallNgramBoundaries_ChecksCapabilityBeforeGraphScan(t *testing.T) { - g := &countingNgramReader{Reader: boundaryFixtureGraph([]string{ - "tokenAlpha", "tokenBeta", "tokenGamma", "tokenDelta", - "alphaToken", "betaToken", "tokenize", "tokenizer", - })} - - assert.False(t, BuildAndInstallNgramBoundaries(nonBM25Backend{}, g)) - assert.Zero(t, g.allNodesCalls, "a backend without BM25 must not enumerate the graph") - - bm := NewBM25() - defer bm.Close() - assert.True(t, BuildAndInstallNgramBoundaries(bm, g)) - assert.Equal(t, 1, g.allNodesCalls, "BM25 must mine exactly one graph census") -} - -// TestSparseNgram_TableDrivenVsFixed proves the learned table actually -// changes the tokenizer's emitted grams versus the fixed-n fallback, -// and that the index/query symmetry still holds with a table installed: -// a query routed through the same table reaches a doc it shares a -// learned segment with. -func TestSparseNgram_TableDrivenVsFixed(t *testing.T) { - withSparseNgram(t, true) - tbl := BuildNgramBoundaries(boundaryFixtureGraph([]string{ - "tokenAlpha", "tokenBeta", "tokenGamma", "tokenDelta", - "tokenEpsilon", "alphaToken", "betaToken", "gammaToken", - "tokenize", "tokenizer", "retoken", "subtoken", - })) - require.False(t, tbl.Empty()) - - b := NewBM25() - defer b.Close() - require.True(t, InstallNgramBoundaries(b, tbl)) - b.Add("svc::tokenAlpha", "tokenAlpha", "x.go") - - // The same table drives both Add and Search, so a query that shares - // a learned segment with the indexed symbol reaches it. - res := b.Search("token", 10) - require.NotEmpty(t, res) - assert.Equal(t, "svc::tokenAlpha", res[0].ID) -} diff --git a/internal/search/project_name_test.go b/internal/search/project_name_test.go index 623ec08d1..f324dcfcb 100644 --- a/internal/search/project_name_test.go +++ b/internal/search/project_name_test.go @@ -57,41 +57,3 @@ func mustWrite(t *testing.T, path, content string) { t.Fatal(err) } } - -// TestTypoRescueOnEmpty proves the bigram typo-rescue index is built by default -// (no GORTEX_BIGRAM_TYPOS needed) so a typo'd query that returns zero BM25 hits -// can still be rescued by bigram overlap — and that the opt-out flag disables -// it for a perf-sensitive operator. -func TestTypoRescueOnEmpty(t *testing.T) { - t.Run("default_on", func(t *testing.T) { - t.Setenv("GORTEX_BIGRAM_TYPOS", "") // simulate "unset" → default ON - b := NewBM25() - b.Add("doc1", "validateToken") - b.Add("doc2", "parseRequest") - - // A transposed/truncated query misses BM25 entirely... - if hits := b.Search("validat", 10); len(hits) > 0 { - t.Logf("note: BM25 returned %d hits for the typo (stemming); rescue still applies on true empties", len(hits)) - } - // ...but the bigram rescue tier recovers the nearest symbol. - cands := b.BigramCandidates("validat", 1) - found := false - for _, c := range cands { - if c == "doc1" { - found = true - } - } - if !found { - t.Errorf("typo rescue did not recover doc1 from %q; candidates=%v", "validat", cands) - } - }) - - t.Run("opt_out", func(t *testing.T) { - t.Setenv("GORTEX_BIGRAM_TYPOS", "0") - b := NewBM25() - b.Add("doc1", "validateToken") - if cands := b.BigramCandidates("validat", 1); cands != nil { - t.Errorf("GORTEX_BIGRAM_TYPOS=0 should disable the bigram index, got candidates %v", cands) - } - }) -} diff --git a/internal/search/rerank/pipeline.go b/internal/search/rerank/pipeline.go index 8ade05dfa..89d7ee75b 100644 --- a/internal/search/rerank/pipeline.go +++ b/internal/search/rerank/pipeline.go @@ -21,10 +21,9 @@ import ( type Candidate struct { Node *graph.Node - // TextRank is the 0-based BM25 rank. -1 means the candidate did - // not appear in the text-search result list (e.g. a substring or - // bigram-rescue fallback hit, or a candidate added by another - // retrieval channel). + // TextRank is the 0-based text-search rank. -1 means the candidate + // did not appear in the text-search result list (e.g. a substring + // fallback hit, or a candidate added by another retrieval channel). TextRank int // VectorRank is the 0-based vector-search rank. -1 means absent. VectorRank int diff --git a/internal/search/search.go b/internal/search/search.go index bbac32933..c723487ed 100644 --- a/internal/search/search.go +++ b/internal/search/search.go @@ -1,10 +1,16 @@ // Package search provides full-text search over code symbols with -// camelCase/snake_case-aware tokenization and BM25 ranking. +// camelCase/snake_case-aware tokenization. // -// Production search runs on SymbolSearcherBackend, a thin adapter over -// the graph store's own FTS index — no parallel in-process corpus. -// A store that exposes no native symbol search gets NullBackend, whose -// empty corpus routes the query engine to its substring fallback. +// The package owns no text index of its own. Search runs on +// SymbolSearcherBackend, a thin adapter over the graph store's own FTS +// index, and this package contributes the tokenization and query +// normalization both sides of that index agree on. A store that +// exposes no native symbol search gets NullBackend, whose empty corpus +// routes the query engine to its substring fallback. +// +// HybridBackend layers the optional vector channel on top of whichever +// text Backend it is given; Swappable lets the indexer replace the +// backend under a live engine. package search // SearchResult is a single search hit. @@ -33,10 +39,10 @@ type Backend interface { // ChannelSearcher is an optional interface a Backend can implement to // expose its per-channel raw retrieval output. The rerank pipeline -// queries it so BM25 and semantic (vector) ranks can contribute as +// queries it so text and semantic (vector) ranks can contribute as // separate signals instead of being collapsed via RRF before scoring. -// Backends that only do text search (BM25, the store-native FTS -// adapter) don't satisfy this interface; callers fall through to plain +// Backends that only do text search (the store-native FTS adapter) +// don't satisfy this interface; callers fall through to plain // Search(). type ChannelSearcher interface { SearchChannels(query string, limit int) (textResults []SearchResult, vectorIDs []string) diff --git a/internal/search/search_test.go b/internal/search/search_test.go index 11e3a053d..bdb6b3203 100644 --- a/internal/search/search_test.go +++ b/internal/search/search_test.go @@ -4,7 +4,6 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) func TestTokenize(t *testing.T) { @@ -36,113 +35,3 @@ func TestTokenizeQuery(t *testing.T) { tokens = TokenizeQuery("go test") assert.Equal(t, []string{"go", "test"}, tokens) } - -// runBackendTests runs the same test suite on any Backend implementation. -func runBackendTests(t *testing.T, name string, backend Backend) { - // This conformance suite asserts baseline exact-match / no-match - // semantics shared by every backend. The opt-in sub-word n-gram gate - // deliberately adds fuzzy recall paths that would turn a "no match" - // assertion into a match, so pin it off here regardless of any - // ambient GORTEX_SPARSE_NGRAM — sub-word recall is exercised in the - // dedicated sparse-ngram tests, not here. - withSparseNgram(t, false) - t.Run(name+"/BasicSearch", func(t *testing.T) { - backend.Add("auth/token.go::validateToken", "validateToken", "auth/token.go") - backend.Add("auth/token.go::parseJWT", "parseJWT", "auth/token.go") - backend.Add("api/handler.go::handleRequest", "handleRequest", "api/handler.go") - backend.Add("db/query.go::buildQuery", "buildQuery", "db/query.go") - backend.Add("payment/charge.go::chargeCard", "chargeCard", "payment/charge.go") - - results := backend.Search("validate token", 10) - require.NotEmpty(t, results) - assert.Equal(t, "auth/token.go::validateToken", results[0].ID) - }) - - t.Run(name+"/CamelCaseSearch", func(t *testing.T) { - results := backend.Search("handle request", 10) - require.NotEmpty(t, results) - assert.Equal(t, "api/handler.go::handleRequest", results[0].ID) - }) - - t.Run(name+"/PathSearch", func(t *testing.T) { - results := backend.Search("payment", 10) - require.NotEmpty(t, results) - assert.Equal(t, "payment/charge.go::chargeCard", results[0].ID) - }) - - t.Run(name+"/Remove", func(t *testing.T) { - backend.Add("tmp.go::tempFunc", "tempFunc", "tmp.go") - results := backend.Search("temp func", 10) - require.NotEmpty(t, results) - - backend.Remove("tmp.go::tempFunc") - results = backend.Search("temp func", 10) - // Should not find the removed symbol. - for _, r := range results { - assert.NotEqual(t, "tmp.go::tempFunc", r.ID) - } - }) - - t.Run(name+"/Count", func(t *testing.T) { - assert.Equal(t, 5, backend.Count()) - }) - - t.Run(name+"/EmptyQuery", func(t *testing.T) { - results := backend.Search("", 10) - assert.Empty(t, results) - }) - - t.Run(name+"/NoMatch", func(t *testing.T) { - results := backend.Search("xyznonexistent", 10) - assert.Empty(t, results) - }) -} - -func TestBM25Backend(t *testing.T) { - backend := NewBM25() - defer backend.Close() - runBackendTests(t, "BM25", backend) -} - -func TestBM25_RankingQuality(t *testing.T) { - b := NewBM25() - defer b.Close() - - // Add symbols with varying relevance to "auth token" - b.Add("auth/token.go::validateToken", "validateToken", "auth/token.go", "func validateToken(token string) bool") - b.Add("auth/session.go::createSession", "createSession", "auth/session.go", "func createSession(userID string) Session") - b.Add("config/config.go::loadConfig", "loadConfig", "config/config.go", "func loadConfig(path string) Config") - b.Add("api/handler.go::tokenHandler", "tokenHandler", "api/handler.go", "func tokenHandler(w http.ResponseWriter)") - - results := b.Search("auth token", 10) - // OR ranking: every doc matching any query token is scored. - // validateToken has both terms so it ranks first; createSession - // and tokenHandler each match one; loadConfig matches neither - // and is dropped. - require.Len(t, results, 3) - assert.Equal(t, "auth/token.go::validateToken", results[0].ID) -} - -func TestBM25_DuplicateTokensCollapse(t *testing.T) { - b := NewBM25() - defer b.Close() - b.Add("a", "token token", "a.go") - b.Add("b", "token parser", "b.go") - - results := b.Search("token token token", 10) - require.Len(t, results, 2) -} - -func BenchmarkBM25_Search(b *testing.B) { - backend := NewBM25() - for i := 0; i < 10000; i++ { - backend.Add( - "pkg/file.go::func"+string(rune('A'+i%26))+string(rune('0'+i%10)), - "getUserById", "internal/auth/service.go", "func getUserById(id string) User", - ) - } - b.ResetTimer() - for b.Loop() { - backend.Search("get user auth", 20) - } -} diff --git a/internal/search/sparse_ngram.go b/internal/search/sparse_ngram.go deleted file mode 100644 index cf86096c5..000000000 --- a/internal/search/sparse_ngram.go +++ /dev/null @@ -1,153 +0,0 @@ -package search - -import ( - "os" - "strings" -) - -// sparseNgramEnabled gates the optional sub-word n-gram emission stage -// layered over the fixed-rule word tokens that Tokenize / TokenizeQuery -// already produce. Default OFF: emitting character n-grams for every -// word token multiplies the posting set, and on identifier-heavy -// queries the extra sub-word noise can demote an exact match — a -// precision regression we are not willing to ship enabled by default -// until it proves out against the recall fixture. Opt in with -// GORTEX_SPARSE_NGRAM=1 (also true / yes / on / y). -// -// Read once at process start, exactly like the FTS-stemming and -// bigram-typo flags. The index built during a daemon's lifetime and -// every query against it share a single setting, so a mid-session -// toggle can never desynchronise the n-grammed postings from the -// n-grammed query terms: the same emission runs on the index path -// (BM25Backend.Add) and the query path (BM25Backend.Search) because -// both route their word tokens through ExpandSparseNgrams. -var sparseNgramEnabled = sparseNgramFromEnv() - -func sparseNgramFromEnv() bool { - switch strings.ToLower(strings.TrimSpace(os.Getenv("GORTEX_SPARSE_NGRAM"))) { - case "1", "true", "yes", "on", "y": - return true - } - return false -} - -// Sub-word n-gram bounds. Character n-grams in this closed range are -// emitted over each word token when no learned boundary table is -// supplied. n=3..4 is the usual sweet spot for code identifiers: short -// enough to bridge a morphological or typo gap ("valid" reaches -// "validate" via shared trigrams), long enough to stay discriminative -// (bigrams collide across nearly every token, which is why the typo -// side index keeps them separate from the BM25 postings). -const ( - sparseNgramMinN = 3 - sparseNgramMaxN = 4 - // sparseNgramMinTokenLen skips tokens no longer than the smallest - // n-gram — a token of length <= sparseNgramMinN has at most one - // n-gram equal to itself, so it carries no extra sub-word signal - // and only inflates the posting set. - sparseNgramMinTokenLen = sparseNgramMinN + 1 -) - -// NgramBoundaries is the data-driven split source the sparse-ngram -// stage consults when one is available. A learned boundary table built -// at index time (see BuildNgramBoundaries) satisfies it, but the -// tokenizer depends only on this abstraction so it compiles and runs -// with a nil source — degrading cleanly to fixed character n-grams. -// -// Empty reports whether the source carries any learned boundaries; an -// empty source is treated exactly like a nil one. Split cuts a token's -// runes at the source's boundaries and returns the resulting segments -// left-to-right; it must be deterministic for a given input. -type NgramBoundaries interface { - Empty() bool - Split(runes []rune) []string -} - -// ExpandSparseNgrams returns the input word tokens unchanged when the -// sparse-ngram gate is off, and otherwise returns the input tokens -// followed by their emitted sub-word n-grams. The original word tokens -// are always preserved and always come first, so enabling the gate can -// only ADD recall paths — an exact word match still scores through the -// untouched word token, and the appended sub-word grams open additional -// fuzzy-match paths. -// -// The same function runs on both the index and query paths, so the -// postings written for a symbol and the terms a query probes them with -// are produced by identical logic and can never disagree. -// -// When a non-nil, non-empty boundary source is supplied the split -// points are data-driven: each token is cut at the source's -// high-information boundaries and the resulting segments are emitted -// alongside the original token. When the source is nil or empty the -// stage degrades to fixed character n-grams in -// [sparseNgramMinN, sparseNgramMaxN], so the tokenizer behaves -// identically whether or not a learned table has been built yet. -// -// The result is a freshly allocated slice; the input is left untouched. -// Emission is deterministic: for a given token and source the n-grams -// are produced left-to-right in a fixed order with no map iteration. -func ExpandSparseNgrams(tokens []string, table NgramBoundaries) []string { - if !sparseNgramEnabled || len(tokens) == 0 { - return tokens - } - // Preserve the original word tokens verbatim, in order, then append - // the sub-word grams. Over-allocate modestly for the common case. - out := make([]string, 0, len(tokens)*3) - out = append(out, tokens...) - for _, tok := range tokens { - out = appendSparseNgrams(out, tok, table) - } - return out -} - -// appendSparseNgrams appends the sub-word n-grams of one lowercase word -// token to dst and returns the grown slice. A learned boundary source, -// when present and non-empty, drives the split; otherwise the token is -// sliced into fixed-width character n-grams. Tokens too short to carry -// any sub-word signal contribute nothing. Duplicate grams within a -// single token are collapsed so a token like "aaaa" does not emit the -// same gram twice. -func appendSparseNgrams(dst []string, tok string, table NgramBoundaries) []string { - r := []rune(tok) - if len(r) < sparseNgramMinTokenLen { - return dst - } - - seen := make(map[string]struct{}, len(r)) - emit := func(g string) { - if g == "" || g == tok { - return - } - if _, dup := seen[g]; dup { - return - } - seen[g] = struct{}{} - dst = append(dst, g) - } - - if table != nil && !table.Empty() { - // Data-driven: split the token at the learned high-information - // boundaries, then emit each segment. Segments shorter than the - // minimum n are dropped — they collide across too many tokens to - // be useful sub-word keys. - for _, seg := range table.Split(r) { - if len([]rune(seg)) >= sparseNgramMinN { - emit(seg) - } - } - return dst - } - - // Fixed character n-grams in [min, max]. Cap the upper n at the - // token length so a short token still yields its single full-length - // gram rather than nothing. - for n := sparseNgramMinN; n <= sparseNgramMaxN; n++ { - if n > len(r) { - break - } - for i := 0; i+n <= len(r); i++ { - emit(string(r[i : i+n])) - } - } - return dst -} diff --git a/internal/search/sparse_ngram_test.go b/internal/search/sparse_ngram_test.go deleted file mode 100644 index 18313fea7..000000000 --- a/internal/search/sparse_ngram_test.go +++ /dev/null @@ -1,152 +0,0 @@ -package search - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// withSparseNgram pins the sparse-ngram gate to a known value for the -// duration of a test and restores it afterwards, so an ambient -// GORTEX_SPARSE_NGRAM in the environment can't make the suite flaky. -// Mirrors withStemming in fts_normalize_test.go. -func withSparseNgram(t *testing.T, on bool) { - t.Helper() - prev := sparseNgramEnabled - sparseNgramEnabled = on - t.Cleanup(func() { sparseNgramEnabled = prev }) -} - -func TestExpandSparseNgrams_FlagOffIsNoOp(t *testing.T) { - withSparseNgram(t, false) - in := []string{"validate", "token"} - got := ExpandSparseNgrams(in, nil) - // Gate off: the exact same slice is returned, no grams appended. - assert.Equal(t, in, got) -} - -func TestExpandSparseNgrams_FlagOffEmptyInput(t *testing.T) { - withSparseNgram(t, false) - assert.Empty(t, ExpandSparseNgrams(nil, nil)) -} - -func TestExpandSparseNgrams_PreservesWordTokensFirst(t *testing.T) { - withSparseNgram(t, true) - got := ExpandSparseNgrams([]string{"validate", "token"}, nil) - // The original word tokens must survive verbatim and lead the - // output, so an exact word match still scores through them. - require.GreaterOrEqual(t, len(got), 2) - assert.Equal(t, "validate", got[0]) - assert.Equal(t, "token", got[1]) -} - -func TestExpandSparseNgrams_EmitsFixedCharNgrams(t *testing.T) { - withSparseNgram(t, true) - got := ExpandSparseNgrams([]string{"token"}, nil) - set := map[string]struct{}{} - for _, g := range got { - set[g] = struct{}{} - } - // "token" -> trigrams tok,oke,ken and 4-grams toke,oken. - for _, want := range []string{"tok", "oke", "ken", "toke", "oken"} { - _, ok := set[want] - assert.Truef(t, ok, "expected sub-word gram %q in %v", want, got) - } - // The whole token is preserved but never re-emitted as a gram equal - // to itself. - assert.Equal(t, "token", got[0]) -} - -func TestExpandSparseNgrams_ShortTokenEmitsNoGrams(t *testing.T) { - withSparseNgram(t, true) - // "go" is below the minimum gram length, so it yields only itself. - got := ExpandSparseNgrams([]string{"go"}, nil) - assert.Equal(t, []string{"go"}, got) -} - -func TestExpandSparseNgrams_CollapsesDuplicateGrams(t *testing.T) { - withSparseNgram(t, true) - // "aaaa": trigram "aaa" appears twice and the 4-gram is the token - // itself (suppressed). The duplicate "aaa" must collapse to one. - got := ExpandSparseNgrams([]string{"aaaa"}, nil) - count := 0 - for _, g := range got { - if g == "aaa" { - count++ - } - } - assert.Equal(t, 1, count, "duplicate grams within a token must collapse: %v", got) -} - -func TestExpandSparseNgrams_Deterministic(t *testing.T) { - withSparseNgram(t, true) - a := ExpandSparseNgrams([]string{"validate", "handler"}, nil) - b := ExpandSparseNgrams([]string{"validate", "handler"}, nil) - // No map iteration leaks into the emission order. - assert.Equal(t, a, b) -} - -// TestBM25_SparseNgramSymmetry is the load-bearing invariant: every -// term a Search query emits for a doc must be present in the postings -// Add wrote for that doc. We verify it by tokenizing both sides through -// the exact stages the backend uses and asserting the query token set -// is a subset of the indexed token set for an overlapping query. -func TestBM25_SparseNgramSymmetry(t *testing.T) { - withSparseNgram(t, true) - - // Index side: what Add writes for the symbol name "validateToken". - indexTokens := ExpandSparseNgrams( - NormalizeFTSTokens(Tokenize("validateToken")), nil) - indexed := map[string]struct{}{} - for _, tok := range indexTokens { - indexed[tok] = struct{}{} - } - - // Query side: a prefix query "valid" that shares sub-word grams. - queryTokens := ExpandSparseNgrams( - NormalizeFTSTokens(TokenizeQuery("valid")), nil) - require.NotEmpty(t, queryTokens) - - // At least one query gram must hit an indexed gram — otherwise the - // expansion bought no recall. (It must, since "valid" is a prefix of - // "validate" and they share trigrams val/ali/lid.) - overlap := 0 - for _, tok := range queryTokens { - if _, ok := indexed[tok]; ok { - overlap++ - } - } - assert.Positivef(t, overlap, "query grams %v share no indexed gram with %v", - queryTokens, indexTokens) -} - -// TestBM25_SparseNgramRecall exercises the full backend: with the gate -// on, a sub-word query reaches a symbol it would miss on pure word -// tokenization. -func TestBM25_SparseNgramRecall(t *testing.T) { - withSparseNgram(t, true) - b := NewBM25() - defer b.Close() - b.Add("svc::validateToken", "validateToken", "auth/token.go") - - // "valid" is no word token of "validateToken" (camelCase splits to - // validate/token), but it shares sub-word grams with "validate". - res := b.Search("valid", 10) - require.NotEmpty(t, res) - assert.Equal(t, "svc::validateToken", res[0].ID) -} - -// TestBM25_SparseNgramFlagOffUnchanged confirms the gate-off backend -// behaves exactly as before: a sub-word-only query finds nothing. -func TestBM25_SparseNgramFlagOffUnchanged(t *testing.T) { - withSparseNgram(t, false) - b := NewBM25() - defer b.Close() - b.Add("svc::validateToken", "validateToken", "auth/token.go") - - // With the gate off there is no sub-word path, so a fragment that is - // not a whole word token returns nothing from the backend. - res := b.Search("valid", 10) - assert.Empty(t, res) -} diff --git a/internal/search/swappable.go b/internal/search/swappable.go index 355f4004d..cb0b9c903 100644 --- a/internal/search/swappable.go +++ b/internal/search/swappable.go @@ -230,7 +230,7 @@ func (s *Swappable) SearchSymbolBundlesScoped(query string, repoAllow []string, // VectorChannelOnly forwards to the inner backend when it implements // the vector-only channel pull (today: HybridBackend). Lets the -// engine fetch the vector channel without re-running text BM25 — +// engine fetch the vector channel without re-running the text search — // the bundle path already has the text hits. Returns (nil, zero // timings) when the inner backend isn't vector-aware. func (s *Swappable) VectorChannelOnly(query string, limit int) ([]string, ChannelTimings) { diff --git a/internal/search/symbolsearcher_backend.go b/internal/search/symbolsearcher_backend.go index 8941eeba9..9a6b03b5d 100644 --- a/internal/search/symbolsearcher_backend.go +++ b/internal/search/symbolsearcher_backend.go @@ -11,14 +11,15 @@ import ( // SymbolSearcherBackend adapts a graph.SymbolSearcher into the // search.Backend the daemon's search-symbols path consumes. // Engine.gatherBackendCandidates and the rerank pipeline don't need -// to know whether the backend is BM25 or native FTS — they -// see a plain search.Backend and call Search on it. +// to know where the ranking comes from — they see a plain +// search.Backend and call Search on it. // // Production wiring: when the indexer detects that the backing // graph.Store also implements graph.SymbolSearcher, it constructs -// this adapter as the initial -// search.Backend wrapped by search.NewSwappable. The in-process -// BM25 build path is then bypassed entirely. +// this adapter as the initial search.Backend wrapped by +// search.NewSwappable. A store without that capability gets +// NullBackend instead, and the engine falls back to its substring +// scan — no text index is ever built in this process. // // Add / Remove are no-ops on the adapter because the indexer // already drives the SymbolSearcher writes directly: From f3345706b5eb45e51851bd2729dcf62e74a8daa5 Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Sun, 9 Aug 2026 23:11:52 +0200 Subject: [PATCH 10/21] fix(search): give the null backend real identity and finish the naming sweep MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit NullBackend was a field-less struct, so every escaping &NullBackend{} pointed at runtime.zerobase and NewNull() == NewNull() held on any production path. That made the type's own doc claim — a distinct pointer per call, so identity comparisons behave as for every other backend — false, and it quietly defeated Swappable.Swap's close gate: a null-to-null swap took the "same backend, skip Close" arm instead of the "displaced it, close it" arm. Harmless only because Close is a no-op. Fixed by giving the struct one unnamed byte rather than by returning a package-level singleton. Swap's gate is an identity test that has to separate "I installed something new" from "I re-installed what was already here", and it only stays a real test if independently constructed backends are independent objects. A singleton would make the gate structurally unable to see a null-to-null replacement, and would alias one instance across every Swappable in the process — safe exactly as long as Close and the struct stay empty, and a leak the day either stops being. One byte at wiring time buys the language's own guarantee of distinct addresses instead. TestNewNull_DistinctInstances passed only because inlining kept the two values in distinct stack slots; it guarded nothing. It is replaced by a test of the invariant it was standing in for: Swap closes the backend it displaces, leaves a re-installed backend open, and takes the first arm for two null backends. The implements-Backend-only assertions stay, with the interface check lifted to a compile-time assertion. Renderer fixtures that still carried a "bm25" backend name now use "unknown" — the live name whose arm actually reports a heap figure, since resolveSearchBackend emits only sqlite-fts5, none or unknown and only the last carries bytes. Coverage is unchanged: the renderer prints whatever name it is handed and branches on DiskResident alone. The indexer's embed-cap warning, the skip_search config comment, the SymbolSearcher contract doc and the FTS5 capability assertions no longer describe an in-process text index as the live alternative. Two families of reference remain and are deliberate: the eval "bm25" row key, kept so historical bench artifacts stay joinable and already annotated as involving no in-process index, and the search_symbols timing fields and log keys, which measure the ranked lexical channel that FTS5 still serves through its own bm25() function. --- cmd/gortex/daemon_enrichment_progress_test.go | 7 +++--- cmd/gortex/daemon_status_render_test.go | 9 ++++--- internal/config/manager.go | 9 +++---- internal/graph/store.go | 10 +++++--- internal/graph/store_sqlite/store_fts.go | 2 +- internal/search/null.go | 20 +++++++++++---- internal/search/null_test.go | 25 +++++++++++++------ 7 files changed, 54 insertions(+), 28 deletions(-) diff --git a/cmd/gortex/daemon_enrichment_progress_test.go b/cmd/gortex/daemon_enrichment_progress_test.go index a1a998539..816547a68 100644 --- a/cmd/gortex/daemon_enrichment_progress_test.go +++ b/cmd/gortex/daemon_enrichment_progress_test.go @@ -110,7 +110,8 @@ func TestStatusResponse_EnrichmentJSONRoundTrip(t *testing.T) { // TestSearchBackendStats_DiskResidentJSONRoundTrip locks in that the // DiskResident flag survives the wire round trip and is omitted when -// false (the in-process BM25 backend never sets it). +// false (only the store-native FTS index sets it; a backend reporting a +// heap figure leaves it clear). func TestSearchBackendStats_DiskResidentJSONRoundTrip(t *testing.T) { sb := daemon.SearchBackendStats{Name: "sqlite-fts5", DocCount: 48572, DiskResident: true} raw, err := json.Marshal(sb) @@ -121,8 +122,8 @@ func TestSearchBackendStats_DiskResidentJSONRoundTrip(t *testing.T) { require.NoError(t, json.Unmarshal(raw, &round)) assert.True(t, round.DiskResident) - bm25 := daemon.SearchBackendStats{Name: "bm25", DocCount: 10} - raw2, err := json.Marshal(bm25) + heapResident := daemon.SearchBackendStats{Name: "unknown", DocCount: 10} + raw2, err := json.Marshal(heapResident) require.NoError(t, err) assert.NotContains(t, string(raw2), "disk_resident") } diff --git a/cmd/gortex/daemon_status_render_test.go b/cmd/gortex/daemon_status_render_test.go index b437e6257..a20ed2215 100644 --- a/cmd/gortex/daemon_status_render_test.go +++ b/cmd/gortex/daemon_status_render_test.go @@ -101,9 +101,12 @@ func TestRenderDaemonHeader_SearchBackendRow(t *testing.T) { func TestRenderDaemonHeader_SearchBackendRow_HeapBackend(t *testing.T) { st := sampleStatus() - // The in-process BM25 index does have a heap footprint to report. + // The other arm of the row: a backend that is not disk-resident, so + // the heap figure it does report has to be printed. resolveSearchBackend + // reaches it for a backend it cannot identify, where the byte count + // comes from search.BackendSize. st.SearchBackend = daemon.SearchBackendStats{ - Name: "bm25", + Name: "unknown", DocCount: 12000, DocCountKnown: true, Bytes: 200 * 1024 * 1024, @@ -111,7 +114,7 @@ func TestRenderDaemonHeader_SearchBackendRow_HeapBackend(t *testing.T) { var buf bytes.Buffer renderDaemonHeader(&buf, st) out := buf.String() - assert.Contains(t, out, "bm25") + assert.Contains(t, out, "unknown") assert.Contains(t, out, "12000") assert.Contains(t, out, "heap=") } diff --git a/internal/config/manager.go b/internal/config/manager.go index 68894778e..e0b181e63 100644 --- a/internal/config/manager.go +++ b/internal/config/manager.go @@ -299,11 +299,10 @@ func (cm *ConfigManager) GetRepoConfig(repoPrefix string) *Config { out.Index.SkipEmbed = DefaultSkipEmbed() } // Same plumbing for semantic.skip_search — controls what goes into - // the text search index (in-process BM25 in tests and evals, - // store-native FTS in production). Separate from SkipEmbed so users can - // tune the two filters independently (e.g. a tiny-repo user who - // doesn't care about text-index memory can clear SkipSearch while - // keeping SkipEmbed's embedding-cost savings). + // the store-native text search index. Separate from SkipEmbed so + // users can tune the two filters independently (e.g. a tiny-repo + // user who doesn't care about text-index size can clear SkipSearch + // while keeping SkipEmbed's embedding-cost savings). if len(out.Semantic.SkipSearch) > 0 { out.Index.SkipSearch = out.Semantic.SkipSearch } else { diff --git a/internal/graph/store.go b/internal/graph/store.go index 4e6efa801..4cc0c0e48 100644 --- a/internal/graph/store.go +++ b/internal/graph/store.go @@ -513,10 +513,12 @@ type SymbolFTSItem struct { // SymbolSearcher is an optional interface backends MAY implement to // expose engine-native full-text search over the graph's symbol // names. When the backing store implements it, the daemon's -// search_symbols path routes through the backend FTS instead of -// building a parallel in-process BM25 index — saving ~100MB -// of heap on a vscode-scale repo and putting the search latency in -// the same address space as the rest of the graph. +// search_symbols path routes through the backend FTS; a store that +// does not gets search.NullBackend and the engine's index-free +// substring scan instead. Serving ranked search from the store's own +// index is what keeps ~100MB of heap off a vscode-scale repo and puts +// the search latency in the same address space as the rest of the +// graph. // // Contract: // diff --git a/internal/graph/store_sqlite/store_fts.go b/internal/graph/store_sqlite/store_fts.go index 3853d744e..34c141dfa 100644 --- a/internal/graph/store_sqlite/store_fts.go +++ b/internal/graph/store_sqlite/store_fts.go @@ -40,7 +40,7 @@ import ( // Compile-time assertions: *Store satisfies the symbol-search // capabilities. The indexer auto-engages these when the active backend // implements them, routing search_symbols through on-disk FTS5 instead -// of the in-process BM25 index. +// of the engine's index-free substring fallback. var ( _ graph.SymbolSearcher = (*Store)(nil) _ graph.SymbolFTSBatchUpserter = (*Store)(nil) diff --git a/internal/search/null.go b/internal/search/null.go index 5c6f5b842..b2d8ae20f 100644 --- a/internal/search/null.go +++ b/internal/search/null.go @@ -14,17 +14,27 @@ package search // see the Engine built by pkg/gortex.New, which wires query.NewEngine // and never calls SetSearch. // -// NewNull hands back a distinct pointer per call so that identity -// comparisons — Swappable.Swap only closes the old backend when it is -// not the incoming one — behave as they do for every other backend. +// NewNull hands back a distinct pointer per call so independent backend +// constructions retain ordinary object identity instead of collapsing onto +// runtime.zerobase. // // It deliberately implements Backend and nothing else. Satisfying // DocCounter would let a disk-corpus count claim a corpus that does // not exist; Sizer and ChannelSearcher would advertise memory and // per-channel retrieval it does not have. -type NullBackend struct{} +type NullBackend struct { + // A field-less struct has size zero, and the language only + // promises distinct addresses for variables of non-zero size: every + // escaping &NullBackend{} would share runtime.zerobase, so any two + // null backends would compare equal. One byte buys each + // construction a real address, which is what the identity promise + // on NewNull rests on. Nothing reads it — the blank name says so. + _ byte +} -// NewNull returns a Backend that indexes and answers nothing. +// NewNull returns a Backend that indexes and answers nothing. Every call +// allocates its own instance, so independently constructed null backends have +// distinct identity just like every other backend implementation. func NewNull() Backend { return &NullBackend{} } // Add discards the symbol; nothing is indexed. diff --git a/internal/search/null_test.go b/internal/search/null_test.go index 6d5fd5574..810d0c804 100644 --- a/internal/search/null_test.go +++ b/internal/search/null_test.go @@ -2,6 +2,10 @@ package search import "testing" +// The null backend must satisfy Backend itself — the optional +// interfaces it must NOT satisfy are asserted below at run time. +var _ Backend = (*NullBackend)(nil) + // TestNullBackend_ImplementsBackendAndNothingElse pins the property the // fallback seam rests on. The query engine treats a backend as having a // corpus when Count() is positive OR when it satisfies DocCounter and @@ -12,7 +16,7 @@ import "testing" // engine would rank against nothing instead of falling back to its // substring scan. func TestNullBackend_ImplementsBackendAndNothingElse(t *testing.T) { - var b Backend = NewNull() + b := NewNull() if _, ok := b.(DocCounter); ok { t.Error("NullBackend must not implement DocCounter — it would claim a corpus it has no documents for") @@ -51,11 +55,18 @@ func TestNullBackend_StaysEmptyAfterAdd(t *testing.T) { } } -// TestNewNull_DistinctInstances guards the identity contract Swappable -// relies on: Swap closes the previous backend only when it differs from -// the incoming one, so two null backends must not compare equal. -func TestNewNull_DistinctInstances(t *testing.T) { - if NewNull() == NewNull() { - t.Error("NewNull must hand back a distinct backend per call") +// nullIdentityEscapeSink forces both returned pointers to escape. Without it, +// inlining may give two zero-sized values distinct stack addresses and let a +// field-less NullBackend pass the identity assertion accidentally. +var nullIdentityEscapeSink [2]Backend + +// TestNewNullReturnsDistinctInstances pins NewNull's identity contract +// directly. NullBackend intentionally carries one byte so independent +// constructions cannot collapse onto runtime.zerobase. +func TestNewNullReturnsDistinctInstances(t *testing.T) { + nullIdentityEscapeSink = [2]Backend{NewNull(), NewNull()} + t.Cleanup(func() { nullIdentityEscapeSink = [2]Backend{} }) + if nullIdentityEscapeSink[0] == nullIdentityEscapeSink[1] { + t.Fatal("NewNull handed back one shared instance") } } From 079e6cef761b791695096a84a6400a0c56682d85 Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Fri, 14 Aug 2026 20:02:42 +0200 Subject: [PATCH 11/21] fix(search): keep vector-only hybrid corpora queryable --- internal/query/engine_corpus_gate_test.go | 36 +++++++++++++++++++++++ internal/search/hybrid.go | 19 ++++++++++-- internal/search/hybrid_test.go | 27 +++++++++++++++++ internal/search/null.go | 13 ++++---- 4 files changed, 87 insertions(+), 8 deletions(-) diff --git a/internal/query/engine_corpus_gate_test.go b/internal/query/engine_corpus_gate_test.go index acd695eb8..1bb695a89 100644 --- a/internal/query/engine_corpus_gate_test.go +++ b/internal/query/engine_corpus_gate_test.go @@ -147,6 +147,42 @@ type unknownCountBackend struct{ warmStoreBackend } func (b *unknownCountBackend) DocCount() (int, bool) { return 12345, false } +// TestGatherSymbolCandidates_VectorOnlyHybridUsesBackend proves that the corpus +// gate recognizes a populated vector channel even when its text side is the +// deliberately empty NullBackend. The concept query shares no substring with +// the symbol name, so the candidate can only arrive through semantic search. +func TestGatherSymbolCandidates_VectorOnlyHybridUsesBackend(t *testing.T) { + g := graph.New() + n := &graph.Node{ + ID: "app/snapshot.go::SnapshotCoordinator", + Name: "SnapshotCoordinator", + Kind: graph.KindType, + RepoPrefix: "app", + } + g.AddNode(n) + + vector := search.NewVector(2) + vector.Add(n.ID, []float32{1, 0}) + backend := search.NewSwappable(search.NewNull()) + backend.ReplaceHybridVector(vector, &fakeEmbedder{queryVec: []float32{1, 0}}) + defer backend.Close() + + engine := NewEngine(g) + engine.SetSearch(backend) + got := engine.GatherSymbolCandidates( + "durable state persistence", + 5, + QueryOptions{SkipInnerRerank: true}, + nil, + ) + if len(got) != 1 || got[0].Node.ID != n.ID { + t.Fatalf("vector-only hybrid was bypassed (substring fallback?); got %#v", got) + } + if got[0].TextRank != -1 || got[0].VectorRank != 0 { + t.Fatalf("vector-only hit must preserve channel ranks; got %#v", got[0]) + } +} + // TestGatherSymbolCandidates_EmptyBackendStillFallsBack: no delta // count AND no doc count keeps the substring fallback for genuinely // empty in-process backends. diff --git a/internal/search/hybrid.go b/internal/search/hybrid.go index 5e0ff5938..55aea5ffc 100644 --- a/internal/search/hybrid.go +++ b/internal/search/hybrid.go @@ -220,8 +220,23 @@ func (h *HybridBackend) dechunkVectorIDs(rawIDs []string, want int) []string { return out } -// Count returns the text backend document count. -func (h *HybridBackend) Count() int { return h.text.Count() } +// Count reports the corpus visible to hybrid retrieval. A positive text count +// remains authoritative; vector-only hybrids fall back to their vector count. +// The channel counts are alternatives, not additive views of the same symbols. +func (h *HybridBackend) Count() int { + if h == nil { + return 0 + } + if h.text != nil { + if count := h.text.Count(); count > 0 { + return count + } + } + if h.vector != nil { + return h.vector.Count() + } + return 0 +} // Close releases resources owned by the hybrid. The embedding provider and a // delegated vector searcher are externally owned; VectorBackend.Close only diff --git a/internal/search/hybrid_test.go b/internal/search/hybrid_test.go index 54ad10718..ee8940bcd 100644 --- a/internal/search/hybrid_test.go +++ b/internal/search/hybrid_test.go @@ -7,6 +7,33 @@ import ( "github.com/stretchr/testify/require" ) +type hybridCountTextBackend struct { + count int +} + +func (*hybridCountTextBackend) Add(string, ...string) {} +func (*hybridCountTextBackend) Remove(string) {} +func (*hybridCountTextBackend) Search(string, int) []SearchResult { return nil } +func (b *hybridCountTextBackend) Count() int { return b.count } +func (*hybridCountTextBackend) Close() {} + +func TestHybridCountUsesVectorWhenTextIsEmpty(t *testing.T) { + vector := NewVector(2) + vector.Add("semantic-only", []float32{1, 0}) + + hybrid := NewHybrid(NewNull(), vector, nil) + require.Equal(t, 1, hybrid.Count(), "a populated vector channel is a searchable corpus") + + textAuthoritative := NewHybrid(&hybridCountTextBackend{count: 3}, vector, nil) + assert.Equal(t, 3, textAuthoritative.Count(), "text and vector counts describe the same corpus and must not be summed") + + empty := NewHybrid(NewNull(), nil, nil) + assert.Zero(t, empty.Count(), "an empty hybrid must report no corpus") + + var nilHybrid *HybridBackend + assert.Zero(t, nilHybrid.Count(), "a nil hybrid must report no corpus") +} + func TestAlphaFuse_EqualWeights(t *testing.T) { textResults := []SearchResult{ {ID: "a", Score: 10}, diff --git a/internal/search/null.go b/internal/search/null.go index b2d8ae20f..98aa893d5 100644 --- a/internal/search/null.go +++ b/internal/search/null.go @@ -5,12 +5,13 @@ package search // Remove and Close are no-ops, Search returns no hits, and Count // always reports zero. // -// Reporting an empty corpus is the point. The query Engine gates its -// ranked path on the backend having something to answer with -// (Engine.backendHasCorpus) and otherwise falls through to its own -// substring scan over the graph, which needs no text index at all. -// NullBackend therefore routes such a store onto exactly the path an -// Engine with no search backend at all already takes in production — +// Reporting an empty text corpus is the point. When used on its own, the query +// Engine gates its ranked path on the backend having something to answer with +// (Engine.backendHasCorpus) and otherwise falls through to its own substring +// scan over the graph, which needs no text index at all. When NullBackend is the +// text side of a HybridBackend, the hybrid's vector count can still establish a +// searchable corpus. A bare NullBackend therefore routes such a store onto +// exactly the path an Engine with no search backend at all takes in production — // see the Engine built by pkg/gortex.New, which wires query.NewEngine // and never calls SetSearch. // From 2e3cf10630e6749e2a432c256612ed93f4d08de9 Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Fri, 14 Aug 2026 20:02:59 +0200 Subject: [PATCH 12/21] test(docs): migrate retired BM25 references --- README.md | 2 +- bench/fixtures/retrieval.yaml | 40 ++++++++-------- bench/fixtures/retrieval_typo.yaml | 74 +++++++++++++++--------------- docs/04-evaluation/task-set.md | 6 ++- docs/features.md | 2 +- docs/semantic-search.md | 6 +-- internal/eval/recall/recall.go | 14 +++--- 7 files changed, 73 insertions(+), 71 deletions(-) diff --git a/README.md b/README.md index d9dbd6eaa..4228969b6 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ High-quality parsing 257 languages/grammars through tree-sitter AST analysis, in - **Zero external dependencies** — single binary, everything in-process. No network, no model download to get started. Install, start daemon, use. - **Agent integrations (19)** — `gortex init` configures every detected coding assistant on the machine → [docs/agents.md](docs/agents.md) - **100+ MCP tools, 16 resources, 3 prompts** — symbol lookup, call chains, blast radius, dataflow, clone detection, refactoring, code actions → [docs/mcp.md](docs/mcp.md) -- **Semantic search default-on** — baked GloVe-50d (3.8 MB embedded), hybrid BM25 + vector + RRF, zero deps; opt-in MiniLM / Ollama / OpenAI → [docs/semantic-search.md](docs/semantic-search.md) +- **Semantic search default-on** — baked GloVe-50d (3.8 MB embedded), store-native FTS5/BM25 + vector with adaptive alpha fusion, zero deps; opt-in MiniLM / Ollama / OpenAI → [docs/semantic-search.md](docs/semantic-search.md) - **Speculative execution** — `preview_edit` / `simulate_chain` answer "what would change if I applied this WorkspaceEdit?" without touching disk - **Live editor overlays** — push unsaved buffers as a shadow graph; tools read through it. Branching for parallel speculative sessions - **GCX1 wire format** — published, round-trippable. **An additional −27% tokens vs JSON** at same fidelity → [docs/wire-format.md](docs/wire-format.md) diff --git a/bench/fixtures/retrieval.yaml b/bench/fixtures/retrieval.yaml index 9449dbba0..5180dcde2 100644 --- a/bench/fixtures/retrieval.yaml +++ b/bench/fixtures/retrieval.yaml @@ -27,11 +27,11 @@ # Cases are tiered so per-tier weakness is visible: # # tier: exact — symbol-name queries. Tests "find this named -# thing." BM25 should dominate; a retrieval stack -# that can't ace exact tier is broken. +# thing." Store-native lexical search should dominate; +# a retrieval stack that can't ace exact tier is broken. # tier: concept — natural-language paraphrase queries. Tests -# semantic understanding. Where BM25 starts losing -# to semantic / RRF if the embedder is competent. +# semantic understanding. Where lexical retrieval starts +# losing to vector/adaptive fusion if the embedder is competent. # tier: multi_hop — relational queries with several valid expected # IDs (any-hit semantics). Tests graph-aware # retrieval: "things that handle X" / "extractors @@ -63,8 +63,8 @@ cases: - { id: exact-AddNode, tier: exact, query: "AddNode", expected: [internal/graph/graph.go::Graph.AddNode] } - { id: exact-AtomicWriteFile, tier: exact, query: "AtomicWriteFile", expected: [internal/agents/writer.go::AtomicWriteFile] } - { id: exact-WriteIfNotExists, tier: exact, query: "WriteIfNotExists", expected: [internal/agents/writer.go::WriteIfNotExists] } - - { id: exact-BM25Backend, tier: exact, query: "BM25Backend", expected: [internal/search/bm25.go::BM25Backend] } - - { id: exact-NewBM25, tier: exact, query: "NewBM25", expected: [internal/search/bm25.go::NewBM25] } + - { id: exact-SymbolSearcherBackend, tier: exact, query: "SymbolSearcherBackend", expected: [internal/search/symbolsearcher_backend.go::SymbolSearcherBackend] } + - { id: exact-NewSymbolSearcherBackend, tier: exact, query: "NewSymbolSearcherBackend", expected: [internal/search/symbolsearcher_backend.go::NewSymbolSearcherBackend] } - { id: exact-HybridBackend, tier: exact, query: "HybridBackend", expected: [internal/search/hybrid.go::HybridBackend] } - { id: exact-NewHybrid, tier: exact, query: "NewHybrid", expected: [internal/search/hybrid.go::NewHybrid] } - { id: exact-alphaFuse, tier: exact, query: "alphaFuse", expected: [internal/search/hybrid.go::alphaFuse] } @@ -119,10 +119,10 @@ cases: - { id: concept-reindex-one-file, tier: concept, query: "reindex a single file after an edit", expected: [internal/indexer/indexer.go::Indexer.IndexFile] } - { id: concept-evict-file, tier: concept, query: "remove all nodes belonging to a file", expected: [internal/graph/graph.go::Graph.EvictFile] } - { id: concept-evict-repo, tier: concept, query: "drop every node for a repository prefix", expected: [internal/graph/graph.go::Graph.EvictRepo] } - - { id: concept-bm25-index, tier: concept, query: "text search backend with TF-IDF ranking", expected: [internal/search/bm25.go::BM25Backend] } - - { id: concept-hybrid-fuse, tier: concept, query: "combine text and vector search with RRF", expected: [internal/search/hybrid.go::HybridBackend] } - - { id: concept-adaptive-alpha, tier: concept, query: "adaptive alpha weighted reciprocal rank fusion", expected: [internal/search/hybrid.go::alphaFuse] } - - { id: concept-swap-backend, tier: concept, query: "hot-swap the in-memory search backend", expected: [internal/search/swappable.go::Swappable] } + - { id: concept-store-native-text, tier: concept, query: "adapt store-native symbol search into the search backend", expected: [internal/search/symbolsearcher_backend.go::SymbolSearcherBackend] } + - { id: concept-hybrid-fuse, tier: concept, query: "combine store-native text and vector search with adaptive alpha", expected: [internal/search/hybrid.go::HybridBackend] } + - { id: concept-adaptive-alpha, tier: concept, query: "adaptive alpha weighted text and vector fusion", expected: [internal/search/hybrid.go::alphaFuse] } + - { id: concept-swap-backend, tier: concept, query: "atomically replace the vector channel while preserving the live text backend", expected: [internal/search/swappable.go::Swappable.ReplaceHybridVector] } - { id: concept-glove-embed, tier: concept, query: "built-in GloVe word vector embedder", expected: [internal/embedding/static.go::StaticProvider] } - { id: concept-mcp-server, tier: concept, query: "MCP server type holding engine and graph", expected: [internal/mcp/server.go::Server] } - { id: concept-session-state, tier: concept, query: "per-client session activity tracking", expected: [internal/mcp/server.go::sessionState] } @@ -154,7 +154,7 @@ cases: - { id: concept-ts-extract, tier: concept, query: "TypeScript language extractor", expected: [internal/parser/languages/typescript.go::TypeScriptExtractor] } - { id: concept-detect-lang, tier: concept, query: "detect language from file extension", expected: [internal/parser/registry.go::Registry.GetByExtension] } - { id: concept-register-all, tier: concept, query: "register every language extractor", expected: [internal/parser/languages/register.go::RegisterAll] } - - { id: concept-ranker-bm25, tier: concept, query: "eval ranker adapter for engine search", expected: [internal/eval/recall/rankers.go::EngineRanker] } + - { id: concept-ranker-engine, tier: concept, query: "eval ranker adapter for engine search", expected: [internal/eval/recall/rankers.go::EngineRanker] } - { id: concept-ranker-rrf, tier: concept, query: "eval ranker adapter for RRF hybrid", expected: [internal/eval/recall/rankers.go::RRFRanker] } - { id: concept-ranker-semantic, tier: concept, query: "eval ranker for vector-only semantic", expected: [internal/eval/recall/rankers.go::SemanticRanker] } - { id: concept-ranker-winnow, tier: concept, query: "eval ranker wrapping graph-aware winnow", expected: [internal/eval/recall/rankers.go::WinnowRanker] } @@ -163,7 +163,7 @@ cases: - { id: concept-repo-for-file, tier: concept, query: "find which repo contains a given file", expected: [internal/indexer/multi.go::MultiIndexer.RepoForFile] } - { id: concept-set-embedder, tier: concept, query: "attach embedding provider to the indexer", expected: [internal/indexer/indexer.go::Indexer.SetEmbedder] } - { id: concept-new-server, tier: concept, query: "construct a new MCP server instance", expected: [internal/mcp/server.go::NewServer] } - - { id: concept-new-hybrid, tier: concept, query: "construct a hybrid BM25+vector backend", expected: [internal/search/hybrid.go::NewHybrid] } + - { id: concept-new-hybrid, tier: concept, query: "construct a hybrid store-native text and vector backend", expected: [internal/search/hybrid.go::NewHybrid] } - { id: concept-reindex-all, tier: concept, query: "incremental reindex after file changes", expected: [internal/indexer/indexer.go::Indexer.IncrementalReindexPaths] } - { id: concept-new-vector, tier: concept, query: "HNSW vector index backend", expected: [internal/search/vector.go::VectorBackend] } - { id: concept-api-embedder, tier: concept, query: "OpenAI-compatible embeddings API provider", expected: [internal/embedding/api.go::APIProvider] } @@ -229,7 +229,7 @@ cases: tier: multi_hop query: "search backend implementations" expected: - - internal/search/bm25.go::BM25Backend + - internal/search/null.go::NullBackend - internal/search/symbolsearcher_backend.go::SymbolSearcherBackend - internal/search/hybrid.go::HybridBackend - internal/search/vector.go::VectorBackend @@ -410,15 +410,15 @@ cases: - internal/config/config.go::Load - internal/config/config.go::Default - - id: mh-bm25-api + - id: mh-null-backend-api tier: multi_hop - query: "BM25 backend public API" + query: "null search backend public API" expected: - - internal/search/bm25.go::BM25Backend.Add - - internal/search/bm25.go::BM25Backend.Remove - - internal/search/bm25.go::BM25Backend.Search - - internal/search/bm25.go::BM25Backend.Count - - internal/search/bm25.go::NewBM25 + - internal/search/null.go::NullBackend.Add + - internal/search/null.go::NullBackend.Remove + - internal/search/null.go::NullBackend.Search + - internal/search/null.go::NullBackend.Count + - internal/search/null.go::NewNull - id: mh-tool-registrations tier: multi_hop diff --git a/bench/fixtures/retrieval_typo.yaml b/bench/fixtures/retrieval_typo.yaml index ee3d4ba21..d34c92a3a 100644 --- a/bench/fixtures/retrieval_typo.yaml +++ b/bench/fixtures/retrieval_typo.yaml @@ -45,79 +45,79 @@ cases: query: IndexFileNozResolve expected: - internal/indexer/indexer.go::Indexer.IndexFileNoResolve - - id: exact-IncrementalReindex-typo + - id: exact-IncrementalReindexPaths-typo tier: exact - query: IncrementalPeindex + query: IncrementalReigdexPaths expected: - internal/indexer/indexer.go::Indexer.IncrementalReindexPaths - id: exact-SetEmbedder-typo tier: exact - query: SetEmbeeder + query: SetEmbedde expected: - internal/indexer/indexer.go::Indexer.SetEmbedder - id: exact-EvictFile-typo tier: exact - query: EvictFife + query: EvictFilq expected: - internal/graph/graph.go::Graph.EvictFile - id: exact-EvictRepo-typo tier: exact - query: EvkctRepo + query: EiictRepo expected: - internal/graph/graph.go::Graph.EvictRepo - id: exact-AddNode-typo tier: exact - query: AddNode + query: AddoNde expected: - internal/graph/graph.go::Graph.AddNode - id: exact-AtomicWriteFile-typo tier: exact - query: AtomicnWriteFile + query: AtomicWriteile expected: - internal/agents/writer.go::AtomicWriteFile - id: exact-WriteIfNotExists-typo tier: exact - query: WrilteIfNotExists + query: WriteIfNotExisks expected: - internal/agents/writer.go::WriteIfNotExists - - id: exact-BM25Backend-typo + - id: exact-SymbolSearcherBackend-typo tier: exact - query: BM25Backejd + query: SymbolSearcherackend expected: - - internal/search/bm25.go::BM25Backend - - id: exact-NewBM25-typo + - internal/search/symbolsearcher_backend.go::SymbolSearcherBackend + - id: exact-NewSymbolSearcherBackend-typo tier: exact - query: NenBM25 + query: NewSymboSlearcherBackend expected: - - internal/search/bm25.go::NewBM25 + - internal/search/symbolsearcher_backend.go::NewSymbolSearcherBackend - id: exact-HybridBackend-typo tier: exact - query: HybridBfckend + query: HybridUackend expected: - internal/search/hybrid.go::HybridBackend - id: exact-NewHybrid-typo tier: exact - query: NewHybid + query: NtwHybrid expected: - internal/search/hybrid.go::NewHybrid - id: exact-alphaFuse-typo tier: exact - query: alphaFse + query: agphaFuse expected: - internal/search/hybrid.go::alphaFuse - id: exact-VectorBackend-typo tier: exact - query: VectorBcakend + query: VectorBackedd expected: - internal/search/vector.go::VectorBackend - id: exact-SearchBackend-typo tier: exact - query: search.Backeod + query: sewarch.Backend expected: - internal/search/search.go::Backend - id: exact-SearchResult-typo tier: exact - query: SearchResutl + query: SearchResuzt expected: - internal/search/search.go::SearchResult - id: exact-Swappable-typo @@ -335,26 +335,26 @@ cases: query: drop every node for a rpeository prefix expected: - internal/graph/graph.go::Graph.EvictRepo - - id: concept-bm25-index-typo + - id: concept-store-native-text-typo tier: concept - query: text search backend wtih TF-IDF ranking + query: adapt store-native symbol search itno the search backend expected: - - internal/search/bm25.go::BM25Backend + - internal/search/symbolsearcher_backend.go::SymbolSearcherBackend - id: concept-hybrid-fuse-typo tier: concept - query: comlbine text and vector search with RRF + query: combine store-native text and lvector search with adaptive alpha expected: - internal/search/hybrid.go::HybridBackend - id: concept-adaptive-alpha-typo tier: concept - query: adaptive alpha weighted reciprocal randk fusion + query: adaptive alpha weightded text and vector fusion expected: - internal/search/hybrid.go::alphaFuse - id: concept-swap-backend-typo tier: concept - query: hot-swap the in-memory searich backend + query: atomically replace the vectior channel while preserving the live text backend expected: - - internal/search/swappable.go::Swappable + - internal/search/swappable.go::Swappable.ReplaceHybridVector - id: concept-glove-embed-typo tier: concept query: built-in GloVe owrd vector embedder @@ -510,7 +510,7 @@ cases: query: register every language extactor expected: - internal/parser/languages/register.go::RegisterAll - - id: concept-ranker-bm25-typo + - id: concept-ranker-engine-typo tier: concept query: eval rnker adapter for engine search expected: @@ -557,7 +557,7 @@ cases: - internal/mcp/server.go::NewServer - id: concept-new-hybrid-typo tier: concept - query: construct a hbyrid BM25+vector backend + query: construct a hybrid stroe-native text and vector backend expected: - internal/search/hybrid.go::NewHybrid - id: concept-reindex-all-typo @@ -672,7 +672,7 @@ cases: tier: multi_hop query: search backend implementatsions expected: - - internal/search/bm25.go::BM25Backend + - internal/search/null.go::NullBackend - internal/search/symbolsearcher_backend.go::SymbolSearcherBackend - internal/search/hybrid.go::HybridBackend - internal/search/vector.go::VectorBackend @@ -832,15 +832,15 @@ cases: expected: - internal/config/config.go::Load - internal/config/config.go::Default - - id: mh-bm25-api-typo + - id: mh-null-backend-api-typo tier: multi_hop - query: BM25 backend pubilc API + query: null seacrh backend public API expected: - - internal/search/bm25.go::BM25Backend.Add - - internal/search/bm25.go::BM25Backend.Remove - - internal/search/bm25.go::BM25Backend.Search - - internal/search/bm25.go::BM25Backend.Count - - internal/search/bm25.go::NewBM25 + - internal/search/null.go::NullBackend.Add + - internal/search/null.go::NullBackend.Remove + - internal/search/null.go::NullBackend.Search + - internal/search/null.go::NullBackend.Count + - internal/search/null.go::NewNull - id: mh-tool-registrations-typo tier: multi_hop query: MCP tool registration functuons diff --git a/docs/04-evaluation/task-set.md b/docs/04-evaluation/task-set.md index 37931e311..f355056d4 100644 --- a/docs/04-evaluation/task-set.md +++ b/docs/04-evaluation/task-set.md @@ -33,8 +33,10 @@ order of operations." `parser.ExtractionResult`; `Indexer.processExtraction` writes them into the `graph.Graph` and accumulates incoming-edge tracking for the next phase. -4. `Indexer.buildSearchIndex` (the store-native FTS) + - `idx.embedder` (if set) populate the search backends. +4. The backing store owns the symbol FTS corpus: shadow drains bulk-publish it, + direct-store passes rebuild it, and incremental mutations upsert or delete + affected rows. `Indexer.buildSearchIndex` prepares and publishes only the + optional vector corpus through `idx.embedder`. 5. Semantic enrichment (`internal/semantic`) runs LSP / SCIP providers in parallel; resolved edges get `Origin=lsp_resolved` for tier filtering. diff --git a/docs/features.md b/docs/features.md index 0d1d9f74f..1ad693429 100644 --- a/docs/features.md +++ b/docs/features.md @@ -34,7 +34,7 @@ The full surface, grouped by concern. Each item links to the deeper reference wh ## Search & navigation -- **Semantic search default-on** — hybrid BM25 + vector with RRF fusion, baked GloVe-50d (~3.8 MB embedded in the binary, top 20k tokens), CPU-only, zero native deps. Large symbols are split into AST-aware windows and de-chunked at query time. Equivalence-class vocabulary expansion bridges `auth ≈ authentication ≈ login` without an LLM. A HITS authority/hub signal feeds the rerank pipeline. A keyword-soup query defense detects degenerate OR-soup *and* operator-free phrasing and skips wasted LLM expansion. Markdown documentation is a first-class corpus with its own retrieval channel and prose-tuned ranking. Opt-in `embedding.provider: local` (Hugot MiniLM-L6-v2) or `api` (Ollama / OpenAI). See [semantic-search.md](semantic-search.md). +- **Semantic search default-on** — store-native FTS5/BM25 + vector with adaptive alpha fusion, baked GloVe-50d (~3.8 MB embedded in the binary, top 20k tokens), CPU-only, zero native deps. Large symbols are split into AST-aware windows and de-chunked at query time. Equivalence-class vocabulary expansion bridges `auth ≈ authentication ≈ login` without an LLM. A HITS authority/hub signal feeds the rerank pipeline. A keyword-soup query defense detects degenerate OR-soup *and* operator-free phrasing and skips wasted LLM expansion. Markdown documentation is a first-class corpus with its own retrieval channel and prose-tuned ranking. Opt-in `embedding.provider: local` (Hugot MiniLM-L6-v2) or `api` (Ollama / OpenAI). See [semantic-search.md](semantic-search.md). - **Provenance-aware ranking** — the BM25↔vector balance is scored continuously from query shape; edge-resolution provenance attenuates LSP-inflated framework wiring in centrality and rerank; generated files are ranked below a real same-named implementation; the implementation is lifted above its own test; and a post-rerank pass recovers exact embedding cosine. Zero-result identifier queries are auto-decomposed into leaf terms. - **`context_closure`** — given seed files/symbols, walks the transitive import/dependency closure and packs it under one `token_budget`, ranked by graph distance or seeded random-walk proximity. - **Code search beyond symbols** — `search_text` is a trigram-indexed literal/regex search; `search_ast` runs structural tree-sitter queries; `analyze kind=sast` is a 190-rule, CWE/OWASP-tagged security scan across 8 languages. diff --git a/docs/semantic-search.md b/docs/semantic-search.md index 0b0528174..6ee9de579 100644 --- a/docs/semantic-search.md +++ b/docs/semantic-search.md @@ -1,6 +1,6 @@ # Semantic search -**Default-on.** A baked GloVe-50d table (~3.8 MB embedded in the binary, top 20k tokens) gives every install hybrid BM25 + vector search out of the box — no flag, no model download, no native dependency. Reciprocal Rank Fusion blends the two channels, and the BM25↔vector balance is scored *continuously* from the query's shape (identifier density, separators, stopwords) rather than bucketed into a discrete class — so a half-identifier query lands between the symbol and natural-language blends instead of jumping a whole tier. After ranking, an optional pure-cosine refinement pass re-scores the top results with the exact embedding distance the rank-based fusion discards. +**Default-on.** A baked GloVe-50d table (~3.8 MB embedded in the binary, top 20k tokens) gives every install hybrid store-native FTS5/BM25 + vector search out of the box — no flag, no model download, no native dependency. Adaptive alpha-weighted rank fusion blends the two channels, and the text↔vector balance is scored *continuously* from the query's shape (identifier density, separators, stopwords) rather than bucketed into a discrete class — so a half-identifier query lands between the symbol and natural-language blends instead of jumping a whole tier. After ranking, an optional pure-cosine refinement pass re-scores the top results with the exact embedding distance the rank-based fusion discards. ## Configuration @@ -65,7 +65,7 @@ Centrality (HITS + PageRank) and a dedicated rerank signal weight call/reference ## Keyword-soup defense -Boolean / OR-soup queries (`A OR B OR 'no access' OR …`) — and operator-free keyword lists (`parse decode unmarshal token jwt cache`) and comma-enumerations — defeat embedding retrieval. The query classifier detects all three, skips wasted LLM expansion, and splits the soup into terms fused via the existing BM25 expansion path; a `query_advice` nudge rides on the response. Genuine natural-language questions stay classified as concept. Tune via `search.keyword_soup_rewrite: split | nudge | off`. +Boolean / OR-soup queries (`A OR B OR 'no access' OR …`) — and operator-free keyword lists (`parse decode unmarshal token jwt cache`) and comma-enumerations — defeat embedding retrieval. The query classifier detects all three, skips wasted LLM expansion, and splits the soup into terms fused via the store-native lexical expansion path; a `query_advice` nudge rides on the response. Genuine natural-language questions stay classified as concept. Tune via `search.keyword_soup_rewrite: split | nudge | off`. ## Prose corpus @@ -111,7 +111,7 @@ Semantic search degrading to text-only (BM25 / FTS5) is always logged — match - `auto` (default) — skips LLM for identifier queries, expands NL queries - `on` — forces expansion + rerank -- `off` — pure BM25 +- `off` — store-native FTS5/BM25 only - `deep` — adds a body-grounded verification pass; +1.5–4 s; quality is highly model-dependent — unreliable on 3B local models, fine on 7B+ or hosted See [llm.md](llm.md) for provider configuration. diff --git a/internal/eval/recall/recall.go b/internal/eval/recall/recall.go index e824d1867..aee4daab9 100644 --- a/internal/eval/recall/recall.go +++ b/internal/eval/recall/recall.go @@ -7,18 +7,18 @@ // Recall is reported as any-hit set-level recall: a retrieval counts as // correct at rank K if *any* of the Expected IDs for a case appears in // the ranker's top-K results. Multiple Expected IDs per case are OK — -// they represent valid alternative targets (e.g. a type and its -// constructor both being reasonable answers to "BM25 backend"). +// they represent valid alternative targets (e.g. a backend type and its +// constructor both being reasonable answers to "store-native text search"). // // Cases are tiered so per-tier weakness is visible: // // - exact: symbol-name queries. Tests the basic "can you find a -// named symbol I already know about" case. BM25 should -// dominate here; a retrieval tool that can't ace exact -// tier is broken. +// named symbol I already know about" case. Store-native lexical +// retrieval should dominate here; a retrieval tool that can't ace +// exact tier is broken. // - concept: natural-language paraphrase queries. Tests semantic -// understanding. This is where BM25 starts losing to -// semantic / RRF. +// understanding. This is where lexical retrieval starts losing to +// vector search and adaptive fusion. // - multi_hop: relational queries accepting several valid expected // IDs (any-hit). Tests graph-aware retrieval. // From 366c208eb1e00cc5feadf488c801319b2bbf8dfb Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Fri, 14 Aug 2026 20:20:53 +0200 Subject: [PATCH 13/21] fix(search): use authoritative native FTS counts --- .../indexer/search_count_symmetry_test.go | 14 ++-- internal/search/symbolsearcher_backend.go | 65 +++++-------------- .../search/symbolsearcher_backend_test.go | 27 ++++---- 3 files changed, 35 insertions(+), 71 deletions(-) diff --git a/internal/indexer/search_count_symmetry_test.go b/internal/indexer/search_count_symmetry_test.go index 85122a592..7a45aa310 100644 --- a/internal/indexer/search_count_symmetry_test.go +++ b/internal/indexer/search_count_symmetry_test.go @@ -11,9 +11,9 @@ import ( ) // countingSearch records Add/Remove calls so a test can assert that a node -// admitted by shouldIndexForSearch is the only kind ever removed. The real -// backend's Remove is an unconditional decrement, so an eviction predicate -// broader than the admit predicate silently corrupts its count. +// admitted by shouldIndexForSearch is the only kind ever removed. Production +// backends differ in how they store search state, but their admit and eviction +// predicates must remain symmetric. type countingSearch struct { added []string removed []string @@ -45,7 +45,7 @@ func TestRemoveFromSearchOnlyEvictsWhatWouldBeIndexed(t *testing.T) { idx.removeFromSearch(n) } require.Empty(t, spy.removed, - "a node the admit predicate rejects must never reach Remove — the backend decrements unconditionally") + "a node the admit predicate rejects must never reach Remove") admitted := &graph.Node{ID: "pkg/a.go::Fn", Kind: graph.KindFunction, Name: "Fn", Language: "go"} require.True(t, idx.shouldIndexForSearch(admitted)) @@ -53,9 +53,9 @@ func TestRemoveFromSearchOnlyEvictsWhatWouldBeIndexed(t *testing.T) { require.Equal(t, []string{"pkg/a.go::Fn"}, spy.removed) } -// The count only stays honest if the two predicates agree. Simulating a -// reconcile — evict the prior nodes, re-add the fresh ones — must be -// count-neutral for an unchanged file. +// Search membership only stays stable if the two predicates agree. Simulating +// a reconcile — evict the prior nodes, re-add the fresh ones — must be neutral +// for an unchanged file. func TestReconcileOfUnchangedFileIsCountNeutral(t *testing.T) { spy := &countingSearch{} idx := &Indexer{search: spy, config: config.IndexConfig{}} diff --git a/internal/search/symbolsearcher_backend.go b/internal/search/symbolsearcher_backend.go index 9a6b03b5d..51164a167 100644 --- a/internal/search/symbolsearcher_backend.go +++ b/internal/search/symbolsearcher_backend.go @@ -2,8 +2,6 @@ package search import ( "strings" - "sync" - "sync/atomic" "github.com/zzet/gortex/internal/graph" ) @@ -36,13 +34,6 @@ import ( // happen through the direct SymbolSearcher surface. type SymbolSearcherBackend struct { s graph.SymbolSearcher - - // count is lazily seeded from the authoritative persisted FTS count on its - // first read, then follows the indexer's incremental Add/Remove deltas. Lazy - // initialization avoids a discarded global count query for each per-repo - // Indexer constructed by MultiIndexer. - countInit sync.Once - count atomic.Int64 } // NewSymbolSearcherBackend wraps a SymbolSearcher in the @@ -154,48 +145,28 @@ func (b *SymbolSearcherBackend) Search(query string, limit int) []SearchResult { } // Add is a no-op — the indexer drives UpsertSymbolFTS on the wrapped -// SymbolSearcher directly. count is bumped immediately so deltas that arrive -// before the first Count call are preserved when the persisted snapshot is -// added. -func (b *SymbolSearcherBackend) Add(id string, _ ...string) { - if b == nil || id == "" { - return - } - b.count.Add(1) -} +// SymbolSearcher directly. +func (b *SymbolSearcherBackend) Add(string, ...string) {} -// Remove is a no-op for the same reason as Add — the per-call -// removal path (when one lands) routes through SymbolSearcher -// directly, not through the search.Backend contract. count is -// decremented so the Count() figure stays roughly consistent. -func (b *SymbolSearcherBackend) Remove(id string) { - if b == nil || id == "" { - return - } - b.count.Add(-1) -} +// Remove is a no-op for the same reason as Add — the removal path routes +// through SymbolSearcher directly, not through the search.Backend contract. +func (b *SymbolSearcherBackend) Remove(string) {} -// Count returns the persisted corpus snapshot observed on its first call plus -// subsequent Add/Remove deltas. It is suitable for readiness gates and rough -// magnitude only; DocCount reads the authoritative current size. +// Count returns the authoritative persisted corpus size when the wrapped store +// exposes it. Add and Remove deliberately do not maintain a second local count: +// their corresponding FTS writes happen directly on the store, so applying the +// same deltas here would double-count them. func (b *SymbolSearcherBackend) Count() int { - if b == nil { + count, ok := b.DocCount() + if !ok { return 0 } - b.countInit.Do(func() { - if counter, ok := b.s.(graph.SymbolFTSCounter); ok { - if count, err := counter.SymbolFTSCount(); err == nil && count > 0 { - b.count.Add(int64(count)) - } - } - }) - return int(b.count.Load()) + return count } -// DocCounter is the capability interface for the authoritative corpus -// size — distinct from Backend.Count(), which is a process-local -// Add/Remove delta. The engine's has-corpus gate asserts this; -// Swappable and HybridBackend forward it. +// DocCounter is the capability interface for the authoritative corpus size. +// The engine's has-corpus gate asserts this; Swappable and HybridBackend +// forward it. type DocCounter interface { DocCount() (int, bool) } @@ -203,10 +174,8 @@ type DocCounter interface { // DocCount returns the authoritative number of indexed documents, straight // from the underlying index, and reports whether it could be obtained. // -// Count() must not be used for this: its cached readiness snapshot can drift as -// direct store maintenance and best-effort Add/Remove deltas diverge. Anything -// user-facing asks here and omits the figure when the authoritative answer is -// unavailable. +// Count delegates to this method as well, so readiness checks and user-facing +// status observe the same store-owned value. func (b *SymbolSearcherBackend) DocCount() (int, bool) { if b == nil || b.s == nil { return 0, false diff --git a/internal/search/symbolsearcher_backend_test.go b/internal/search/symbolsearcher_backend_test.go index 944bb1f2e..0a699264e 100644 --- a/internal/search/symbolsearcher_backend_test.go +++ b/internal/search/symbolsearcher_backend_test.go @@ -30,30 +30,25 @@ func (s *countedSymbolSearcherStub) SymbolFTSCount() (int, error) { return s.count, s.err } -func TestNewSymbolSearcherBackendSeedsPersistedCountLazily(t *testing.T) { +func TestSymbolSearcherBackendCountUsesAuthoritativeCorpus(t *testing.T) { store := &countedSymbolSearcherStub{count: 37} backend := NewSymbolSearcherBackend(store) if store.calls != 0 { t.Fatalf("constructor SymbolFTSCount calls = %d, want 0", store.calls) } - backend.Add("new-before-first-count") - if got := backend.Count(); got != 38 { - t.Fatalf("first Count() = %d, want persisted count plus prior delta 38", got) - } - if store.calls != 1 { - t.Fatalf("SymbolFTSCount calls = %d, want 1", store.calls) - } - if got := backend.Count(); got != 38 { - t.Fatalf("second Count() = %d, want cached count 38", got) - } - if store.calls != 1 { - t.Fatalf("repeated Count SymbolFTSCount calls = %d, want 1", store.calls) + backend.Add("already-persisted") + if got := backend.Count(); got != 37 { + t.Fatalf("Count() = %d, want persisted count 37", got) } - backend.Remove("new-before-first-count") - if got := backend.Count(); got != 37 { - t.Fatalf("Count() after balanced delta = %d, want 37", got) + store.count = 41 + backend.Remove("already-removed") + if got := backend.Count(); got != 41 { + t.Fatalf("Count() after store update = %d, want 41", got) + } + if store.calls != 2 { + t.Fatalf("SymbolFTSCount calls = %d, want one authoritative read per Count", store.calls) } } From 477a8da4cbfb5908ec33d2c725d41986b223f7d8 Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Fri, 14 Aug 2026 20:38:43 +0200 Subject: [PATCH 14/21] refactor(search): remove retired publication compatibility seams --- docs/04-evaluation/task-set.md | 5 +- internal/indexer/chunk_index_test.go | 37 ++++----- internal/indexer/embed_pool_test.go | 17 +++-- internal/indexer/indexer.go | 55 ++++++-------- internal/indexer/multi.go | 2 +- .../indexer/search_backend_native_test.go | 5 +- internal/indexer/vector_ingest_test.go | 16 ++-- internal/search/hybrid.go | 8 +- internal/search/swappable.go | 10 --- internal/search/vector.go | 37 ++------- internal/search/vector_publication_test.go | 75 +++++++------------ 11 files changed, 104 insertions(+), 163 deletions(-) diff --git a/docs/04-evaluation/task-set.md b/docs/04-evaluation/task-set.md index f355056d4..1af46a5a9 100644 --- a/docs/04-evaluation/task-set.md +++ b/docs/04-evaluation/task-set.md @@ -35,8 +35,9 @@ order of operations." tracking for the next phase. 4. The backing store owns the symbol FTS corpus: shadow drains bulk-publish it, direct-store passes rebuild it, and incremental mutations upsert or delete - affected rows. `Indexer.buildSearchIndex` prepares and publishes only the - optional vector corpus through `idx.embedder`. + affected rows. The vector pipeline prepares an immutable plan with + `Indexer.prepareSearchIndexForPublication`, then publishes it through + `Indexer.installVectorPlan` after the durable shadow drain succeeds. 5. Semantic enrichment (`internal/semantic`) runs LSP / SCIP providers in parallel; resolved edges get `Origin=lsp_resolved` for tier filtering. diff --git a/internal/indexer/chunk_index_test.go b/internal/indexer/chunk_index_test.go index 3b27d4555..88f6f30f0 100644 --- a/internal/indexer/chunk_index_test.go +++ b/internal/indexer/chunk_index_test.go @@ -42,8 +42,8 @@ func (stubEmbedder) EmbedBatch(_ context.Context, texts []string) ([][]float32, func (stubEmbedder) Dimensions() int { return 4 } func (stubEmbedder) Close() error { return nil } -// indexedVectorBackend indexes dir with the given chunk options and -// returns the vector backend buildSearchIndex produced. +// indexedVectorBackend indexes dir with the given chunk options and returns +// the published vector backend, pinned for the remainder of the test. func indexedVectorBackend(t *testing.T, dir string, opts embedding.ChunkOptions) *search.VectorBackend { t.Helper() g := graph.New() @@ -62,16 +62,17 @@ func indexedVectorBackend(t *testing.T, dir string, opts embedding.ChunkOptions) sw, ok := idx.Search().(*search.Swappable) require.True(t, ok) - hyb, ok := sw.Inner().(*search.HybridBackend) + backend, release := sw.AcquireBackend() + t.Cleanup(release) + hyb, ok := backend.(*search.HybridBackend) require.True(t, ok, "an embedder-equipped index must produce a HybridBackend") return hyb.VectorIndex() } -// TestBuildSearchIndex_LongSymbolYieldsMultipleChunks proves the -// pipeline change: a function whose source span exceeds the chunk -// threshold is split into several chunk vectors, and the chunk → -// parent mapping is recorded on the vector backend. -func TestBuildSearchIndex_LongSymbolYieldsMultipleChunks(t *testing.T) { +// TestVectorPublication_LongSymbolYieldsMultipleChunks proves that a function +// whose source span exceeds the chunk threshold is split into several chunk +// vectors and the chunk → parent mapping is recorded on the vector backend. +func TestVectorPublication_LongSymbolYieldsMultipleChunks(t *testing.T) { dir := t.TempDir() var b strings.Builder b.WriteString("package main\n\nfunc BigFunc() {\n") @@ -100,8 +101,8 @@ func TestBuildSearchIndex_LongSymbolYieldsMultipleChunks(t *testing.T) { "BigFunc must be split into at least two chunk vectors") } -// bigFuncChunkID builds the synthetic chunk ID buildSearchIndex stamps -// on BigFunc's i-th window. +// bigFuncChunkID builds the synthetic chunk ID vector preparation stamps on +// BigFunc's i-th window. func bigFuncChunkID(i int) string { return "big.go::BigFunc#chunk" + itoa(i) } @@ -118,10 +119,10 @@ func itoa(i int) string { return string(digits) } -// TestBuildSearchIndex_ShortSymbolStaysWhole proves the inverse: a file -// of only small functions produces no chunk vectors — every symbol is -// embedded under its own ID. -func TestBuildSearchIndex_ShortSymbolStaysWhole(t *testing.T) { +// TestVectorPublication_ShortSymbolStaysWhole proves the inverse: a file of +// only small functions produces no chunk vectors — every symbol is embedded +// under its own ID. +func TestVectorPublication_ShortSymbolStaysWhole(t *testing.T) { dir := t.TempDir() src := `package main @@ -141,10 +142,10 @@ func AlsoTiny() { assert.Greater(t, vec.Count(), 0, "the symbols must still be embedded") } -// TestBuildSearchIndex_ChunkedSymbolNotDuplicatedInSearch proves the -// end-to-end de-chunk contract through a real index: searching for a -// long symbol returns it once, and no synthetic chunk ID surfaces. -func TestBuildSearchIndex_ChunkedSymbolNotDuplicatedInSearch(t *testing.T) { +// TestVectorPublication_ChunkedSymbolNotDuplicatedInSearch proves the end-to-end +// de-chunk contract through a real index: searching for a long symbol returns +// it once, and no synthetic chunk ID surfaces. +func TestVectorPublication_ChunkedSymbolNotDuplicatedInSearch(t *testing.T) { dir := t.TempDir() var b strings.Builder b.WriteString("package main\n\nfunc ValidateRequestPayload() {\n") diff --git a/internal/indexer/embed_pool_test.go b/internal/indexer/embed_pool_test.go index b2e2f5f06..ab7efdcac 100644 --- a/internal/indexer/embed_pool_test.go +++ b/internal/indexer/embed_pool_test.go @@ -161,10 +161,10 @@ func TestEmbedAllChunks_AbortsOnError(t *testing.T) { assert.Contains(t, err.Error(), "t37") } -// TestBuildSearchIndex_AbortOnEmbedErrorKeepsTextOnly asserts the -// end-to-end abort contract: when embedding fails, buildSearchIndex -// leaves the search backend text-only — no HybridBackend is swapped in. -func TestBuildSearchIndex_AbortOnEmbedErrorKeepsTextOnly(t *testing.T) { +// TestBuildSearchIndexCtx_AbortOnEmbedErrorKeepsTextOnly asserts the end-to-end +// abort contract: when embedding fails, vector publication leaves the search +// backend text-only — no HybridBackend is swapped in. +func TestBuildSearchIndexCtx_AbortOnEmbedErrorKeepsTextOnly(t *testing.T) { g := graph.New() // Two function nodes; their embed metadata text is // "function ...". poolEmbedder fails on an exact text @@ -176,15 +176,18 @@ func TestBuildSearchIndex_AbortOnEmbedErrorKeepsTextOnly(t *testing.T) { emb := &poolEmbedder{failOnText: "function Alpha a.go"} idx.SetEmbedder(emb) - idx.buildSearchIndex() + require.NoError(t, idx.buildSearchIndexCtx(context.Background())) + require.Error(t, idx.LastVectorBuildError()) // The backend must NOT be a HybridBackend — embedding aborted, so // the search stays text-only. sw, ok := idx.Search().(*search.Swappable) require.True(t, ok) - _, isHybrid := sw.Inner().(*search.HybridBackend) + backend, release := sw.AcquireBackend() + defer release() + _, isHybrid := backend.(*search.HybridBackend) assert.False(t, isHybrid, - "buildSearchIndex must not install a HybridBackend when embedding fails") + "vector publication must not install a HybridBackend when embedding fails") } // TestEmbedAllChunks_DeterministicRegardlessOfOrder runs the pool many diff --git a/internal/indexer/indexer.go b/internal/indexer/indexer.go index 5e238d32e..99eb59338 100644 --- a/internal/indexer/indexer.go +++ b/internal/indexer/indexer.go @@ -298,14 +298,14 @@ type Indexer struct { // Set during the shadow swap, cleared when idx.graph is restored. contractStateSink graph.ContractStateStore - // embedChunkOpts tunes the AST sub-chunking buildSearchIndex applies - // to large symbols before embedding. The zero value makes the - // chunker fall back to its package defaults. + // embedChunkOpts tunes the AST sub-chunking applied while preparing a + // vector publication plan. The zero value makes the chunker fall back to + // its package defaults. embedChunkOpts embedding.ChunkOptions - // embedMaxSymbols overrides the built-in cap on how many texts the - // vector index will hold before buildSearchIndex skips the embed - // pass. Zero keeps the built-in default. + // embedMaxSymbols overrides the built-in cap on how many texts the vector + // preparation pass accepts before skipping embeddings. Zero keeps the + // built-in default. embedMaxSymbols int // embedAPIConcurrency bounds how many embedding requests run in @@ -314,8 +314,8 @@ type Indexer struct { // inference mutex. embedAPIConcurrency int - // lastVectorBuildErr records why the most recent buildSearchIndex pass - // shipped text-only instead of a vector index (chunk-embed failure, + // lastVectorBuildErr records why the most recent vector prepare/install + // pass did not publish a vector index (chunk-embed failure, // all-vectors-invalid, or the symbol-count guard). Nil after a build that // produced a vector index. Read via LastVectorBuildError once a build has // finished — it lets `gortex eval embedders` report the real cause instead @@ -648,10 +648,10 @@ func docSummary(doc string) string { return doc } -// vectorSearcherDelegate is the search.VectorDelegate-shaped -// adapter the indexer hands to VectorBackend.SetDelegate when the -// underlying store implements graph.VectorSearcher. SimilarTo just -// forwards — search.VectorDelegate is defined to return +// vectorSearcherDelegate is the search.VectorDelegate-shaped adapter the +// indexer passes to search.NewDelegatedVector when the underlying store +// implements graph.VectorSearcher. SimilarTo just forwards — +// search.VectorDelegate is defined to return // graph.VectorHit slices directly, so there's no translation work // here, just a small struct so the in-process search package // doesn't depend on graph.VectorSearcher's full surface. @@ -726,8 +726,9 @@ const ( // Nodes stream through the store's bounded scoped projection instead of a // whole-repository read: the caller reached this path precisely because the // repository does not fit in memory. Backends without the replace/projection -// capabilities (test fixtures) are a no-op — their search index is populated -// by buildSearchIndex instead. +// capabilities (test fixtures and in-memory stores) are a no-op — their +// search backend stays empty and the query engine uses its graph substring +// fallback. // // The replacement is one atomic unit: a rebuild that fails part-way leaves the // corpus the repository already had. Wiping first and appending chunk by chunk @@ -809,8 +810,8 @@ func (idx *Indexer) populateSymbolFTS(reporter progress.Reporter) error { // searchable symbols. Beyond that, config.SkipSearch filters out // (language, kind) pairs that would only add noise — JSON/YAML/TOML // keys, CSS tokens, Terraform blocks, shell/build variables. Every -// text-index call site (buildSearchIndex bulk loop, indexFile -// incremental add) must go through this predicate so they can't drift. +// FTS writer (shadow drain, direct rebuild, and incremental mutation) must go +// through this predicate so the persisted corpora cannot drift. func (idx *Indexer) shouldIndexForSearch(n *graph.Node) bool { // Cross-daemon proxy-edge nodes stand in for remote symbols; they // are never surfaced in local name search. Inert until @@ -1465,8 +1466,8 @@ func (idx *Indexer) SetProjectID(id string) { idx.projectID = id } // ProjectID returns the project slug this indexer stamps on nodes. func (idx *Indexer) ProjectID() string { return idx.projectID } -// SetEmbedder sets the embedding provider for semantic search. -// When set, buildSearchIndex will create a HybridBackend with vector search. +// SetEmbedder sets the embedding provider for semantic search. When set, the +// vector prepare/install pipeline publishes a HybridBackend with vector search. func (idx *Indexer) SetEmbedder(p embedding.Provider) { idx.embedder = p } // SetEmbeddingChunkOptions tunes the AST sub-chunking applied to large @@ -1477,8 +1478,8 @@ func (idx *Indexer) SetEmbeddingChunkOptions(opts embedding.ChunkOptions) { } // SetEmbeddingMaxSymbols overrides the cap on how many texts the vector -// index will hold before buildSearchIndex skips the embed pass. Zero -// keeps the built-in default. +// preparation pass accepts before skipping embeddings. Zero keeps the built-in +// default. func (idx *Indexer) SetEmbeddingMaxSymbols(n int) { idx.embedMaxSymbols = n } // SetEmbeddingAPIConcurrency overrides how many embedding requests run @@ -4940,8 +4941,8 @@ func (idx *Indexer) restubIncomingRefs(graphPath string) { // embeddingDimsOrDefault returns the embedder's reported vector width, // falling back to a neutral placeholder only when the provider cannot // state its width yet (Dimensions() == 0, the APIProvider-before-first- -// call case). The fallback is never persisted: buildSearchIndex -// overwrites it with the true width taken from a real vector. Kept as a +// call case). The fallback is never persisted: vector-plan preparation +// replaces it with the true width taken from a real vector. Kept as a // named helper so the vector-dimension default has one definition // instead of a scattered magic number. func embeddingDimsOrDefault(p embedding.Provider) int { @@ -5265,16 +5266,6 @@ func flattenEmbedResults(results [][][]float32) [][]float32 { return out } -// buildSearchIndex is the compatibility entry point for direct and focused -// callers. Full indexing uses the context-aware preparation/publication split -// so a cold shadow can defer publication until its durable drain succeeds. -func (idx *Indexer) buildSearchIndex() { - if err := idx.buildSearchIndexCtx(context.Background()); err != nil { - idx.lastVectorBuildErr = err - idx.logger.Warn("vector index build canceled", zap.Error(err)) - } -} - // dirIgnoreFiles are the per-directory ignore-file basenames honored by // the index walk, siblings to .gitignore: Gortex's own .gortexignore // plus ripgrep's .ignore and .rgignore. Patterns in each file are diff --git a/internal/indexer/multi.go b/internal/indexer/multi.go index cf6ce0ef2..2074c6dae 100644 --- a/internal/indexer/multi.go +++ b/internal/indexer/multi.go @@ -201,7 +201,7 @@ type MultiIndexer struct { // SetEmbedder installs the embedding provider every per-repo indexer // should use. Must be called before IndexAll / TrackRepo for vectors // to land in the graph — without this the fresh Indexer created per -// repo has embedder=nil and buildSearchIndex skips the vector pass. +// repo has embedder=nil and vector-plan preparation skips the embedding pass. // Safe to call zero or one times; subsequent calls silently replace. func (mi *MultiIndexer) SetEmbedder(e embedding.Provider) { mi.mu.Lock() diff --git a/internal/indexer/search_backend_native_test.go b/internal/indexer/search_backend_native_test.go index 63c3b0f5d..4b45a52f0 100644 --- a/internal/indexer/search_backend_native_test.go +++ b/internal/indexer/search_backend_native_test.go @@ -1,6 +1,7 @@ package indexer import ( + "context" "testing" "github.com/stretchr/testify/assert" @@ -19,13 +20,13 @@ func (s *countingSearchStore) AllNodes() []*graph.Node { return s.Store.AllNodes() } -func TestBuildSearchIndex_WithoutVectorsSkipsNodeCensus(t *testing.T) { +func TestBuildSearchIndexCtx_WithoutVectorsSkipsNodeCensus(t *testing.T) { store := &countingSearchStore{Store: graph.New()} backend := search.NewSwappable(search.NewSymbolSearcherBackend(nil)) defer backend.Close() idx := &Indexer{graph: store, search: backend} - idx.buildSearchIndex() + assert.NoError(t, idx.buildSearchIndexCtx(context.Background())) assert.Zero(t, store.allNodesCalls) } diff --git a/internal/indexer/vector_ingest_test.go b/internal/indexer/vector_ingest_test.go index 036221cb9..bde616f27 100644 --- a/internal/indexer/vector_ingest_test.go +++ b/internal/indexer/vector_ingest_test.go @@ -103,16 +103,18 @@ func indexWithEmbedder(t *testing.T, emb interface { return idx } -// TestBuildSearchIndex_DropsInvalidVectors: a mix of valid/nil/short vectors +// TestVectorPublication_DropsInvalidVectors: a mix of valid/nil/short vectors // yields a vector index populated with only the valid ones — the bad vectors // are dropped rather than poisoning the index or aborting a viable build. -func TestBuildSearchIndex_DropsInvalidVectors(t *testing.T) { +func TestVectorPublication_DropsInvalidVectors(t *testing.T) { emb := &mixedEmbedder{} idx := indexWithEmbedder(t, emb) sw, ok := idx.Search().(*search.Swappable) require.True(t, ok) - hybrid, ok := sw.Inner().(*search.HybridBackend) + backend, release := sw.AcquireBackend() + defer release() + hybrid, ok := backend.(*search.HybridBackend) require.True(t, ok, "a viable subset of vectors must still produce a HybridBackend") require.NotNil(t, hybrid.VectorIndex()) @@ -124,14 +126,16 @@ func TestBuildSearchIndex_DropsInvalidVectors(t *testing.T) { "a partially-valid build is a success, not a recorded failure") } -// TestBuildSearchIndex_AllInvalidAbortsToTextOnly: when every vector is invalid +// TestVectorPublication_AllInvalidAbortsToTextOnly: when every vector is invalid // the build must abort to text-only search, not ship a silently empty index. -func TestBuildSearchIndex_AllInvalidAbortsToTextOnly(t *testing.T) { +func TestVectorPublication_AllInvalidAbortsToTextOnly(t *testing.T) { idx := indexWithEmbedder(t, nilEmbedder{}) sw, ok := idx.Search().(*search.Swappable) require.True(t, ok) - _, isHybrid := sw.Inner().(*search.HybridBackend) + backend, release := sw.AcquireBackend() + defer release() + _, isHybrid := backend.(*search.HybridBackend) assert.False(t, isHybrid, "an all-invalid embedding pass must leave a text-only backend, not an empty vector index") assert.Error(t, idx.LastVectorBuildError(), diff --git a/internal/search/hybrid.go b/internal/search/hybrid.go index 55aea5ffc..ce7169102 100644 --- a/internal/search/hybrid.go +++ b/internal/search/hybrid.go @@ -31,16 +31,12 @@ func NewHybrid(text Backend, vector *VectorBackend, embedder embedding.Provider) } } -// Add indexes a symbol in both text and vector backends. +// Add forwards a symbol update to the text backend. Vector corpora are +// prepared and published atomically by the indexer. func (h *HybridBackend) Add(id string, fields ...string) { h.text.Add(id, fields...) } -// AddVector adds a vector for a symbol to the vector backend. -func (h *HybridBackend) AddVector(id string, vector []float32) { - h.vector.Add(id, vector) -} - // Remove removes a symbol from the text backend. func (h *HybridBackend) Remove(id string) { h.text.Remove(id) diff --git a/internal/search/swappable.go b/internal/search/swappable.go index cb0b9c903..488289964 100644 --- a/internal/search/swappable.go +++ b/internal/search/swappable.go @@ -107,16 +107,6 @@ func (s *Swappable) AcquireBackend() (backend Backend, release func()) { } } -// Inner returns an unpinned snapshot of the currently-active backend. It is -// retained for tests and diagnostics only: production callers must not keep or -// dereference the result because replacement may retire it immediately after -// this method returns. Use AcquireBackend or a forwarded capability instead. -func (s *Swappable) Inner() Backend { - s.mu.RLock() - defer s.mu.RUnlock() - return s.inner -} - // Embedder returns the active hybrid's externally-owned embedding provider. // The lookup is protected by the Swappable read lock; replacement never closes // the provider, so the returned provider remains under its original owner's diff --git a/internal/search/vector.go b/internal/search/vector.go index 0c4c36787..c2ad249a1 100644 --- a/internal/search/vector.go +++ b/internal/search/vector.go @@ -20,10 +20,10 @@ type VectorDelegate interface { // VectorBackend stores and searches embedding vectors using HNSW index. // -// When a delegate is installed, the in-process HNSW is absent: Add tracks -// compatibility-path writes without allocating, and Search forwards to the -// delegate's SimilarTo. Durable hits carry their parent IDs, so chunk results -// can be collapsed without retaining a process-local chunk map. +// When a delegate is installed, the in-process HNSW is absent: Add ignores +// process-local writes, while Search forwards to the delegate's SimilarTo. +// Durable hits carry their parent IDs, so chunk results can be collapsed +// without retaining a process-local chunk map. type VectorBackend struct { graph *hnsw.Graph[string] count int @@ -112,11 +112,10 @@ func (v *VectorBackend) Add(id string, vector []float32) { v.mu.Lock() defer v.mu.Unlock() if v.delegate != nil { - // Legacy compatibility for callers that install a delegate before - // pushing vectors themselves. New publication must construct the - // backend with NewDelegatedVector and complete durable statistics. - // Either way Add never allocates an in-process HNSW in delegated mode. - v.delegateCount++ + // The durable corpus is authoritative. Publication replaces it through + // the store and constructs a fresh backend with NewDelegatedVector; + // accepting a process-local Add here would fabricate Count without + // persisting searchable data. return } v.graph.Add(hnsw.Node[string]{ @@ -126,26 +125,6 @@ func (v *VectorBackend) Add(id string, vector []float32) { v.count++ } -// SetDelegate is the legacy compatibility switch for callers that select a -// delegate before writing their corpus. It releases all prior heap and chunk -// state atomically; subsequent Add calls only advance this process's accepted -// write count. New and warm-start publication must use NewDelegatedVector so -// Count and HasChunks describe the complete durable corpus immediately. -func (v *VectorBackend) SetDelegate(d VectorDelegate) { - v.mu.Lock() - defer v.mu.Unlock() - - // Switching modes must also release the old heap index. Merely routing - // reads to d would leave the complete HNSW graph, its count, and chunk - // ownership map retained for the lifetime of the daemon. - v.graph = nil - v.count = 0 - v.chunkMap = nil - v.delegate = d - v.delegateCount = 0 - v.delegateChunkCount = 0 -} - // Search returns the k nearest neighbors to the query vector. func (v *VectorBackend) Search(query []float32, k int) []string { if k <= 0 { diff --git a/internal/search/vector_publication_test.go b/internal/search/vector_publication_test.go index 1ed7f710d..26e5269ca 100644 --- a/internal/search/vector_publication_test.go +++ b/internal/search/vector_publication_test.go @@ -87,6 +87,13 @@ func TestNewDelegatedVectorHasNoHeapIndexAndUsesDurableStats(t *testing.T) { if got := backend.Count(); got != 17 { t.Fatalf("Count() = %d, want complete durable count 17", got) } + backend.Add("process-local-write", []float32{1, 0}) + if got := backend.Count(); got != 17 { + t.Fatalf("Count() = %d after delegated Add, want unchanged durable count 17", got) + } + if backend.graph != nil { + t.Fatal("delegated Add allocated an in-process HNSW graph") + } if !backend.HasChunks() { t.Fatal("HasChunks() = false, want durable chunk metadata") } @@ -136,44 +143,6 @@ func TestDelegatedVectorSearchRejectsNonPositiveLimitWithoutDelegateCall(t *test } } -func TestSetDelegateReleasesHeapAndChunkState(t *testing.T) { - backend := NewVector(2) - backend.Add("symbol-a#chunk0", []float32{1, 0}) - backend.Add("symbol-b", []float32{0, 1}) - backend.SetChunkMap(map[string]string{"symbol-a#chunk0": "symbol-a"}) - if backend.SizeBytes() == 0 { - t.Fatal("heap backend reported no retained vector bytes before delegation") - } - - backend.SetDelegate(&publicationTestDelegate{}) - - if backend.graph != nil { - t.Fatal("SetDelegate retained the old HNSW graph") - } - if backend.chunkMap != nil { - t.Fatal("SetDelegate retained the old chunk ownership map") - } - if got := backend.Count(); got != 0 { - t.Fatalf("Count() = %d after mode switch, want fresh delegated count", got) - } - if backend.HasChunks() { - t.Fatal("SetDelegate retained old chunk statistics") - } - if got := backend.SizeBytes(); got != 0 { - t.Fatalf("SizeBytes() = %d after delegation, want zero", got) - } - - // Preserve the legacy build path: Add records successfully accepted - // delegate-side writes without rebuilding an HNSW graph. - backend.Add("symbol-c", []float32{1, 0}) - if got := backend.Count(); got != 1 { - t.Fatalf("Count() = %d after delegated Add, want 1", got) - } - if backend.graph != nil { - t.Fatal("delegated Add allocated an HNSW graph") - } -} - func TestSerializeVectorUpdateSerializesConcurrentCallbacks(t *testing.T) { swappable := NewSwappable(&publicationTestTextBackend{}) firstStarted := make(chan struct{}) @@ -366,16 +335,20 @@ func TestReplaceHybridVectorWaitsForReadersAndTransfersTextOwnership(t *testing. t.Fatalf("new reader saw %v, want complete new result %v", got, want) } - current, ok := swappable.Inner().(*HybridBackend) - if !ok { - t.Fatalf("active backend is %T, want *HybridBackend", swappable.Inner()) - } - if current.TextBackend() != text { - t.Fatal("replacement did not retain the original text backend") - } - if _, nested := current.TextBackend().(*HybridBackend); nested { - t.Fatal("replacement nested a HybridBackend inside another HybridBackend") - } + func() { + backend, release := swappable.AcquireBackend() + defer release() + current, ok := backend.(*HybridBackend) + if !ok { + t.Fatalf("active backend is %T, want *HybridBackend", backend) + } + if current.TextBackend() != text { + t.Fatal("replacement did not retain the original text backend") + } + if _, nested := current.TextBackend().(*HybridBackend); nested { + t.Fatal("replacement nested a HybridBackend inside another HybridBackend") + } + }() swappable.Close() if got := text.closes(); got != 1 { @@ -438,9 +411,11 @@ func vectorIDs(swappable *Swappable, query string) []string { func assertSingleHybrid(t *testing.T, swappable *Swappable, wantText Backend) { t.Helper() - hybrid, ok := swappable.Inner().(*HybridBackend) + backend, release := swappable.AcquireBackend() + defer release() + hybrid, ok := backend.(*HybridBackend) if !ok { - t.Fatalf("active backend is %T, want *HybridBackend", swappable.Inner()) + t.Fatalf("active backend is %T, want *HybridBackend", backend) } if hybrid.TextBackend() != wantText { t.Fatalf("hybrid text backend is %T, want original %T", hybrid.TextBackend(), wantText) From 95e318209d1279935b9802bc8783e4ada85448db Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Fri, 14 Aug 2026 21:23:27 +0200 Subject: [PATCH 15/21] fix(search): preserve native FTS normalization across restarts --- cmd/gortex/eval_stdbench.go | 3 +- internal/graph/store.go | 10 + .../store_sqlite/fts_normalization_test.go | 202 +++++++++ internal/graph/store_sqlite/schema.go | 9 + .../schema_fts_state_migration_test.go | 94 +++++ internal/graph/store_sqlite/schema_version.go | 14 +- .../graph/store_sqlite/store_content_fts.go | 2 +- internal/graph/store_sqlite/store_fts.go | 15 +- .../graph/store_sqlite/store_fts_state.go | 40 ++ internal/graph/store_sqlite/store_purge.go | 13 + internal/indexer/fts_normalization.go | 80 ++++ .../fts_normalization_lifecycle_test.go | 386 ++++++++++++++++++ internal/indexer/indexer.go | 27 ++ .../search/fts_normalization_mode_test.go | 21 + internal/search/fts_normalize.go | 14 + 15 files changed, 922 insertions(+), 8 deletions(-) create mode 100644 internal/graph/store_sqlite/fts_normalization_test.go create mode 100644 internal/graph/store_sqlite/schema_fts_state_migration_test.go create mode 100644 internal/graph/store_sqlite/store_fts_state.go create mode 100644 internal/indexer/fts_normalization.go create mode 100644 internal/indexer/fts_normalization_lifecycle_test.go create mode 100644 internal/search/fts_normalization_mode_test.go diff --git a/cmd/gortex/eval_stdbench.go b/cmd/gortex/eval_stdbench.go index 6aba47c6d..aa91602a6 100644 --- a/cmd/gortex/eval_stdbench.go +++ b/cmd/gortex/eval_stdbench.go @@ -112,9 +112,10 @@ func runEvalStdbench(_ *cobra.Command, _ []string) error { // Mirrors the indexer's ftsTokensFor: the write side splits with // search.Tokenize so the read side's query tokenisation lands on // the same terms. + tokens := search.NormalizeFTSTokens(search.Tokenize(d.Text)) items = append(items, graph.SymbolFTSItem{ NodeID: d.ID, - Tokens: strings.Join(search.Tokenize(d.Text), " "), + Tokens: strings.Join(tokens, " "), }) } // One call, not a chunked loop: BulkUpsertSymbolFTS wipes the prefix diff --git a/internal/graph/store.go b/internal/graph/store.go index 4cc0c0e48..98507f827 100644 --- a/internal/graph/store.go +++ b/internal/graph/store.go @@ -574,6 +574,16 @@ type SymbolFTSCounter interface { SymbolFTSCount() (int, error) } +// SymbolFTSNormalizationState persists the normalization mode used to produce +// each repository's native symbol FTS corpus. An indexer must advance the +// marker only after an authoritative replacement succeeds; a crash between +// replacement and marker update is safe because the next warm reconcile +// repeats the idempotent rebuild. +type SymbolFTSNormalizationState interface { + GetSymbolFTSNormalization(repoPrefix string) (mode string, ok bool, err error) + SetSymbolFTSNormalization(repoPrefix, mode string) error +} + // SymbolFTSBatchUpserter is the optional incremental fast path. It is kept // separate from SymbolSearcher so alternate search implementations and test // doubles retain source compatibility. Indexing paths use this capability as diff --git a/internal/graph/store_sqlite/fts_normalization_test.go b/internal/graph/store_sqlite/fts_normalization_test.go new file mode 100644 index 000000000..1261f2dc8 --- /dev/null +++ b/internal/graph/store_sqlite/fts_normalization_test.go @@ -0,0 +1,202 @@ +package store_sqlite + +import ( + "os" + "os/exec" + "path/filepath" + "slices" + "strings" + "testing" + + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/search" +) + +const ftsNormalizationTestModeEnv = "GORTEX_FTS_NORMALIZATION_TEST_MODE" + +func TestSymbolFTSNormalizationStateLifecycle(t *testing.T) { + s, err := Open(filepath.Join(t.TempDir(), "fts-state.sqlite")) + if err != nil { + t.Fatalf("Open: %v", err) + } + t.Cleanup(func() { _ = s.Close() }) + + if mode, ok, err := s.GetSymbolFTSNormalization("repo-a"); err != nil || ok || mode != "" { + t.Fatalf("missing repo-a marker = (%q, %v, %v), want (\"\", false, nil)", mode, ok, err) + } + if err := s.SetSymbolFTSNormalization("repo-a", "porter-v1"); err != nil { + t.Fatalf("SetSymbolFTSNormalization(repo-a): %v", err) + } + if err := s.SetSymbolFTSNormalization("repo-b", ""); err != nil { + t.Fatalf("SetSymbolFTSNormalization(repo-b): %v", err) + } + const opaqueMode = "future/vendor-v7+opaque" + if err := s.SetSymbolFTSNormalization("repo-opaque", opaqueMode); err != nil { + t.Fatalf("SetSymbolFTSNormalization(repo-opaque): %v", err) + } + assertFTSNormalizationState(t, s, "repo-a", "porter-v1", true) + assertFTSNormalizationState(t, s, "repo-b", "", true) + assertFTSNormalizationState(t, s, "repo-opaque", opaqueMode, true) + + if err := s.SetSymbolFTSNormalization("repo-a", "porter-v2"); err != nil { + t.Fatalf("overwrite repo-a marker: %v", err) + } + assertFTSNormalizationState(t, s, "repo-a", "porter-v2", true) + assertFTSNormalizationState(t, s, "repo-b", "", true) + + if err := s.SetSymbolFTSNormalization("orphan", "porter-v1"); err != nil { + t.Fatalf("set orphan marker: %v", err) + } + if err := s.SetSymbolFTSNormalization("", "porter-v1"); err != nil { + t.Fatalf("set protected empty-prefix marker: %v", err) + } + orphans := s.OrphanRepoPrefixes([]string{"repo-a", "repo-b", "repo-opaque"}) + if !slices.Equal(orphans, []string{"orphan"}) { + t.Fatalf("OrphanRepoPrefixes = %v, want [orphan] (empty prefix must stay protected)", orphans) + } + + if err := s.PurgeRepo("repo-a"); err != nil { + t.Fatalf("PurgeRepo(repo-a): %v", err) + } + assertFTSNormalizationState(t, s, "repo-a", "", false) + assertFTSNormalizationState(t, s, "repo-b", "", true) + + if err := s.SetSymbolFTSNormalization("old", "porter-v1"); err != nil { + t.Fatalf("set old marker: %v", err) + } + if err := s.SetSymbolFTSNormalization("new", "stale-mode"); err != nil { + t.Fatalf("set destination marker: %v", err) + } + if err := s.RekeyRepoPrefix("old", "new"); err != nil { + t.Fatalf("RekeyRepoPrefix: %v", err) + } + // Rekey drops the symbol FTS corpus because node IDs change. Neither the + // source marker nor a stale destination marker may survive and falsely + // certify that the newly empty corpus was built in a particular mode. + assertFTSNormalizationState(t, s, "old", "", false) + assertFTSNormalizationState(t, s, "new", "", false) +} + +func assertFTSNormalizationState(t *testing.T, s *Store, repo, wantMode string, wantOK bool) { + t.Helper() + mode, ok, err := s.GetSymbolFTSNormalization(repo) + if err != nil { + t.Fatalf("GetSymbolFTSNormalization(%q): %v", repo, err) + } + if mode != wantMode || ok != wantOK { + t.Fatalf("GetSymbolFTSNormalization(%q) = (%q, %v), want (%q, %v)", repo, mode, ok, wantMode, wantOK) + } +} + +func TestFTSNormalizationWriteQueryAndContentParity(t *testing.T) { + if mode := os.Getenv(ftsNormalizationTestModeEnv); mode != "" { + runFTSNormalizationModeAssertions(t, mode == "enabled") + return + } + + for _, tc := range []struct { + name string + stemming string + }{ + {name: "disabled", stemming: "0"}, + {name: "enabled", stemming: "1"}, + } { + t.Run(tc.name, func(t *testing.T) { + cmd := exec.Command(os.Args[0], "-test.run=^TestFTSNormalizationWriteQueryAndContentParity$") + cmd.Env = ftsNormalizationTestEnv(tc.name, tc.stemming) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("subprocess failed: %v\n%s", err, out) + } + }) + } +} + +func ftsNormalizationTestEnv(mode, stemming string) []string { + env := make([]string, 0, len(os.Environ())+2) + for _, entry := range os.Environ() { + if strings.HasPrefix(entry, ftsNormalizationTestModeEnv+"=") || strings.HasPrefix(entry, "GORTEX_FTS_STEMMING=") { + continue + } + env = append(env, entry) + } + return append(env, ftsNormalizationTestModeEnv+"="+mode, "GORTEX_FTS_STEMMING="+stemming) +} + +func runFTSNormalizationModeAssertions(t *testing.T, enabled bool) { + wantMode := "" + wantSymbolMatch := `"the"* OR "manager"*` + if enabled { + wantMode = "porter-v1" + wantSymbolMatch = `"manag"*` + } + if got := search.FTSNormalizationMode(); got != wantMode { + t.Fatalf("FTSNormalizationMode = %q, want %q", got, wantMode) + } + + s, err := Open(filepath.Join(t.TempDir(), "fts-mode.sqlite")) + if err != nil { + t.Fatalf("Open: %v", err) + } + t.Cleanup(func() { _ = s.Close() }) + + if got := s.buildFTSMatch("the manager", true); got != wantSymbolMatch { + t.Fatalf("normalized symbol MATCH = %q, want %q", got, wantSymbolMatch) + } + if got := s.buildFTSMatch("the manager", false); got != `"the"* OR "manager"*` { + t.Fatalf("raw content MATCH = %q, want unnormalised terms", got) + } + + const symbolID = "repo/worker.go::WorkerCatalog" + s.AddNode(&graph.Node{ + ID: symbolID, Kind: graph.KindType, Name: "WorkerCatalog", + FilePath: "repo/worker.go", Language: "go", RepoPrefix: "repo", + }) + writeTokens := search.NormalizeFTSTokens(search.Tokenize("the manage")) + if err := s.UpsertSymbolFTS(symbolID, strings.Join(writeTokens, " ")); err != nil { + t.Fatalf("UpsertSymbolFTS: %v", err) + } + + hits, err := s.SearchSymbols("manager", 10) + if err != nil { + t.Fatalf("SearchSymbols(manager): %v", err) + } + if enabled && (len(hits) != 1 || hits[0].NodeID != symbolID) { + t.Fatalf("enabled stemming hits = %+v, want %q", hits, symbolID) + } + if !enabled && len(hits) != 0 { + t.Fatalf("disabled stemming hits = %+v, want no morphological match", hits) + } + + hits, err = s.SearchSymbols("the", 10) + if err != nil { + t.Fatalf("SearchSymbols(stopword): %v", err) + } + if enabled && len(hits) != 0 { + t.Fatalf("enabled stopword query hits = %+v, want none", hits) + } + if !enabled && (len(hits) != 1 || hits[0].NodeID != symbolID) { + t.Fatalf("disabled stopword query hits = %+v, want raw-token hit", hits) + } + + const contentBody = "the managers are running" + const contentID = "repo/readme.md::doc:section-0" + if err := s.AppendContent("repo", []graph.ContentFTSItem{{ + NodeID: contentID, FilePath: "repo/readme.md", Body: contentBody, + }}); err != nil { + t.Fatalf("AppendContent: %v", err) + } + contentHits, err := s.SearchContent("the", "repo", 10) + if err != nil { + t.Fatalf("SearchContent(stopword): %v", err) + } + if len(contentHits) != 1 || contentHits[0].NodeID != contentID { + t.Fatalf("content stopword hits = %+v, want raw content hit in both modes", contentHits) + } + var storedBody string + if err := s.db.QueryRow(`SELECT body FROM content_fts WHERE node_id = ?`, contentID).Scan(&storedBody); err != nil { + t.Fatalf("read raw content_fts body: %v", err) + } + if storedBody != contentBody { + t.Fatalf("content_fts body = %q, want raw %q", storedBody, contentBody) + } +} diff --git a/internal/graph/store_sqlite/schema.go b/internal/graph/store_sqlite/schema.go index 34b856178..4d0321881 100644 --- a/internal/graph/store_sqlite/schema.go +++ b/internal/graph/store_sqlite/schema.go @@ -768,6 +768,15 @@ CREATE INDEX IF NOT EXISTS blame_by_repo ON blame_enrichment(repo_prefix) WHERE -- on its next open + reindex. CREATE VIRTUAL TABLE IF NOT EXISTS symbol_fts USING fts5(node_id UNINDEXED, repo_prefix UNINDEXED, tokens); +-- symbol_fts_state records which deterministic token-normalization mode built +-- each repository's durable symbol corpus. The marker advances only after an +-- authoritative replacement succeeds, so a crash can cause a harmless repeat +-- rebuild but can never certify a corpus written with a different mode. +CREATE TABLE IF NOT EXISTS symbol_fts_state ( + repo_prefix TEXT PRIMARY KEY, + normalization TEXT NOT NULL DEFAULT '' +) WITHOUT ROWID; + -- symbol_fts_rowid maps a node_id to the rowid (FTS5 docid) of its row in -- symbol_fts. node_id is UNINDEXED in the FTS5 vtable, so deleting a node's -- prior row with "DELETE … WHERE node_id = ?" full-scans the entire index diff --git a/internal/graph/store_sqlite/schema_fts_state_migration_test.go b/internal/graph/store_sqlite/schema_fts_state_migration_test.go new file mode 100644 index 000000000..1897b21cf --- /dev/null +++ b/internal/graph/store_sqlite/schema_fts_state_migration_test.go @@ -0,0 +1,94 @@ +package store_sqlite + +import ( + "database/sql" + "path/filepath" + "testing" + + "github.com/zzet/gortex/internal/graph" +) + +func TestOpenV10AddsSymbolFTSStateWithoutRebuildingCorpus(t *testing.T) { + if currentSchemaVersion != 11 { + t.Fatalf("currentSchemaVersion = %d, want 11 for the symbol FTS state migration", currentSchemaVersion) + } + var v11 *schemaMigration + for i := range schemaMigrations { + if schemaMigrations[i].version == 11 { + v11 = &schemaMigrations[i] + break + } + } + if v11 == nil || v11.rebuild || v11.inPlace == nil { + t.Fatalf("v11 migration = %+v, want a registered in-place step", v11) + } + + path := filepath.Join(t.TempDir(), "v10.sqlite") + seed, err := Open(path) + if err != nil { + t.Fatalf("create current store: %v", err) + } + + const ( + symbolID = "repo/legacy.go::LegacyWidget" + callerID = "repo/caller.go::UseLegacyWidget" + ) + seed.AddBatch([]*graph.Node{ + {ID: symbolID, Kind: graph.KindType, Name: "LegacyWidget", FilePath: "repo/legacy.go", RepoPrefix: "repo"}, + {ID: callerID, Kind: graph.KindFunction, Name: "UseLegacyWidget", FilePath: "repo/caller.go", RepoPrefix: "repo"}, + }, []*graph.Edge{{ + From: callerID, To: symbolID, Kind: graph.EdgeCalls, FilePath: "repo/caller.go", Line: 7, + }}) + if err := seed.UpsertSymbolFTS(symbolID, "migration sentinel"); err != nil { + t.Fatalf("seed symbol FTS: %v", err) + } + if err := seed.Close(); err != nil { + t.Fatalf("close seed store: %v", err) + } + + // Recreate the exact v10 shape: the graph and native FTS corpus exist, but + // the per-repository normalization marker introduced by v11 does not. + withRawDB(t, path, func(db *sql.DB) { + if _, err := db.Exec(`DROP TABLE symbol_fts_state`); err != nil { + t.Fatalf("drop post-v10 marker table: %v", err) + } + if _, err := db.Exec(`PRAGMA user_version = 10`); err != nil { + t.Fatalf("stamp v10: %v", err) + } + }) + + migrated, err := Open(path) + if err != nil { + t.Fatalf("open v10 store: %v", err) + } + defer migrated.Close() + + if migrated.NeedsRebuild() { + t.Fatal("additive v10->v11 state-table migration requested a rebuild") + } + if version, err := readUserVersion(migrated.db); err != nil || version != currentSchemaVersion { + t.Fatalf("migrated user_version = %d (err %v), want %d", version, err, currentSchemaVersion) + } + var stateTables int + if err := migrated.db.QueryRow(`SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = 'symbol_fts_state'`).Scan(&stateTables); err != nil { + t.Fatalf("probe symbol_fts_state: %v", err) + } + if stateTables != 1 { + t.Fatalf("symbol_fts_state table count = %d, want 1", stateTables) + } + assertFTSNormalizationState(t, migrated, "repo", "", false) + + if got := migrated.NodeCount(); got != 2 { + t.Fatalf("node count after migration = %d, want 2", got) + } + if got := migrated.EdgeCount(); got != 1 { + t.Fatalf("edge count after migration = %d, want 1", got) + } + hits, err := migrated.SearchSymbols("sentinel", 10) + if err != nil { + t.Fatalf("search preserved v10 FTS corpus: %v", err) + } + if len(hits) != 1 || hits[0].NodeID != symbolID { + t.Fatalf("preserved FTS hits = %+v, want %q", hits, symbolID) + } +} diff --git a/internal/graph/store_sqlite/schema_version.go b/internal/graph/store_sqlite/schema_version.go index 9163a632c..ad5d11152 100644 --- a/internal/graph/store_sqlite/schema_version.go +++ b/internal/graph/store_sqlite/schema_version.go @@ -32,7 +32,7 @@ import ( // index changes in a way an old on-disk DB would not already have, and append a // matching schemaMigrations entry describing how to bring an older store // forward (in place, or by rebuild). -const currentSchemaVersion = 10 +const currentSchemaVersion = 11 // schemaMigration is one forward step. Exactly one strategy applies: // - rebuild=true: the change introduces structure/data that can only come @@ -78,6 +78,18 @@ var schemaMigrations = []schemaMigration{ // the legacy (node_id, dims, vec) rows for every ID shape. Rebuild only the // derived vector sidecar rather than discarding otherwise-valid topology. {version: 10, name: "rebuild vector corpus ownership and parents", inPlace: rebuildVectorCorpusSchema}, + {version: 11, name: "add symbol FTS normalization state", inPlace: createSymbolFTSNormalizationStateTable}, +} + +// createSymbolFTSNormalizationStateTable is the explicit v11 migration for +// existing stores. schemaSQL owns the canonical fresh-store definition; this +// idempotent step makes the additive table part of the versioned contract. +func createSymbolFTSNormalizationStateTable(tx *sql.Tx) error { + _, err := tx.Exec(`CREATE TABLE IF NOT EXISTS symbol_fts_state ( + repo_prefix TEXT PRIMARY KEY, + normalization TEXT NOT NULL DEFAULT '' + ) WITHOUT ROWID`) + return err } // dropUnusedSemanticPendingIndex removes an experimental index for a query diff --git a/internal/graph/store_sqlite/store_content_fts.go b/internal/graph/store_sqlite/store_content_fts.go index db194a659..a88f0304b 100644 --- a/internal/graph/store_sqlite/store_content_fts.go +++ b/internal/graph/store_sqlite/store_content_fts.go @@ -502,7 +502,7 @@ func (s *Store) SearchContent(query, repoPrefix string, limit int) ([]graph.Cont if limit <= 0 { limit = 20 } - match := s.buildFTSMatch(query) + match := s.buildFTSMatch(query, false) if match == "" { return nil, nil } diff --git a/internal/graph/store_sqlite/store_fts.go b/internal/graph/store_sqlite/store_fts.go index 34c141dfa..c351758e5 100644 --- a/internal/graph/store_sqlite/store_fts.go +++ b/internal/graph/store_sqlite/store_fts.go @@ -664,7 +664,7 @@ func (s *Store) SearchSymbolsRepoScoped(query string, repoAllow []string, limit } } - match := s.buildFTSMatch(query) + match := s.buildFTSMatch(query, true) if match == "" { return nil, nil } @@ -730,17 +730,22 @@ func tier0ShortCircuitKind(k graph.NodeKind) bool { // buildFTSMatch tokenises the query with the write-side splitter and // builds an FTS5 MATCH expression: each token becomes a quoted prefix // term ("tok"*) and the terms are OR-joined so any token match counts. +// normalize selects the symbol-FTS normalization pass. Content FTS keeps raw +// bodies for snippets and scans, so its query path must remain unnormalised. // Returns "" when the query degenerates to no tokens. -func (s *Store) buildFTSMatch(query string) string { +func (s *Store) buildFTSMatch(query string, normalize bool) string { tokens := search.Tokenize(query) if len(tokens) == 0 { // Fallback: when Tokenize drops everything (e.g. a single // sub-2-char token like "go"), use the looser query tokeniser so // the search still reaches the engine instead of returning empty. tokens = search.TokenizeQuery(query) - if len(tokens) == 0 { - return "" - } + } + if normalize { + tokens = search.NormalizeFTSTokens(tokens) + } + if len(tokens) == 0 { + return "" } parts := make([]string, 0, len(tokens)) for _, t := range tokens { diff --git a/internal/graph/store_sqlite/store_fts_state.go b/internal/graph/store_sqlite/store_fts_state.go new file mode 100644 index 000000000..65925e120 --- /dev/null +++ b/internal/graph/store_sqlite/store_fts_state.go @@ -0,0 +1,40 @@ +package store_sqlite + +import ( + "context" + "database/sql" + + "github.com/zzet/gortex/internal/graph" +) + +var _ graph.SymbolFTSNormalizationState = (*Store)(nil) + +// GetSymbolFTSNormalization returns the mode that produced repoPrefix's +// durable symbol FTS corpus. A missing row means the historical unnormalised +// mode; callers still receive ok=false so an enabled mode can trigger its +// one-time authoritative rebuild. +func (s *Store) GetSymbolFTSNormalization(repoPrefix string) (string, bool, error) { + var mode string + err := s.db.QueryRow(`SELECT normalization FROM symbol_fts_state WHERE repo_prefix = ?`, repoPrefix).Scan(&mode) + if err == sql.ErrNoRows { + return "", false, nil + } + if err != nil { + return "", false, err + } + return mode, true, nil +} + +// SetSymbolFTSNormalization advances repoPrefix's marker after the caller has +// committed an authoritative corpus replacement. Keeping this write separate +// and ordered after replacement is deliberately fail-closed: a crash between +// them leaves the old marker and makes the next warm reconcile repeat the +// idempotent rebuild. +func (s *Store) SetSymbolFTSNormalization(repoPrefix, mode string) error { + s.writeMu.Lock() + defer s.writeMu.Unlock() + _, err := s.execActiveWriteLocked(context.Background(), ` +INSERT INTO symbol_fts_state (repo_prefix, normalization) VALUES (?, ?) +ON CONFLICT(repo_prefix) DO UPDATE SET normalization = excluded.normalization`, repoPrefix, mode) + return err +} diff --git a/internal/graph/store_sqlite/store_purge.go b/internal/graph/store_sqlite/store_purge.go index fda669884..4ef3c154a 100644 --- a/internal/graph/store_sqlite/store_purge.go +++ b/internal/graph/store_sqlite/store_purge.go @@ -35,6 +35,7 @@ import ( var purgeSidecarTables = []string{ "file_mtimes", "repo_index_state", + "symbol_fts_state", "enrichment_state", "contract_state", "semantic_binding_types", @@ -143,6 +144,7 @@ var orphanScanTables = []string{ "nodes", "file_mtimes", "repo_index_state", + "symbol_fts_state", "enrichment_state", "files", "semantic_binding_types", @@ -303,6 +305,17 @@ func (s *Store) RekeyRepoPrefix(oldPrefix, newPrefix string) error { changed = true } } + // The symbol FTS corpus above is dropped rather than relabeled because its + // node IDs change. Invalidate both possible markers in the same transaction: + // moving the old marker would falsely certify the now-empty destination, + // while retaining a prior destination marker would do the same after merge. + stateRes, err := tx.Exec(`DELETE FROM symbol_fts_state WHERE repo_prefix IN (?, ?)`, oldPrefix, newPrefix) + if err != nil { + return fmt.Errorf("store_sqlite: RekeyRepoPrefix invalidate symbol FTS normalization: %w", err) + } + if n, rowsErr := stateRes.RowsAffected(); rowsErr == nil && n > 0 { + changed = true + } // Vectors are handled explicitly instead of joining rekeyDropTables because // that shared list is also used by the historical v6→v7 migration, whose // vector schema predates repo_prefix. At the current schema the old node and diff --git a/internal/indexer/fts_normalization.go b/internal/indexer/fts_normalization.go new file mode 100644 index 000000000..92783e44b --- /dev/null +++ b/internal/indexer/fts_normalization.go @@ -0,0 +1,80 @@ +package indexer + +import ( + "fmt" + + "go.uber.org/zap" + + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/progress" + "github.com/zzet/gortex/internal/search" +) + +const symbolFTSNormalizationRebuildPending = "__gortex_rebuild_pending__" + +// setSymbolFTSNormalization persists a mode marker for backends with a durable +// native symbol index. Other backends have no corpus whose mode can drift. +func (idx *Indexer) setSymbolFTSNormalization(target graph.Store, mode string) error { + state, ok := target.(graph.SymbolFTSNormalizationState) + if !ok { + return nil + } + if err := state.SetSymbolFTSNormalization(idx.RepoPrefix(), mode); err != nil { + return fmt.Errorf("persist symbol FTS normalization: %w", err) + } + return nil +} + +// markSymbolFTSNormalization records the current mode only after the caller has +// completed an authoritative symbol-FTS replacement and finalization. +func (idx *Indexer) markSymbolFTSNormalization(target graph.Store) error { + return idx.setSymbolFTSNormalization(target, search.FTSNormalizationMode()) +} + +// markSymbolFTSNormalizationPending invalidates any previously certified mode +// before a destructive replacement starts. The sentinel is deliberately not a +// valid normalization mode, so any failure forces the next warm reconcile to +// repeat the rebuild even when stemming is disabled. +func (idx *Indexer) markSymbolFTSNormalizationPending(target graph.Store) error { + return idx.setSymbolFTSNormalization(target, symbolFTSNormalizationRebuildPending) +} + +// reconcileSymbolFTSNormalization repairs a warm durable corpus whose +// persisted normalization marker differs from this process's immutable mode. +// The marker advances only after populateSymbolFTS atomically replaces the +// repository corpus. A crash between those operations leaves the old marker, +// so the next reconcile safely repeats the rebuild instead of serving a +// silently mismatched index. +func (idx *Indexer) reconcileSymbolFTSNormalization(reporter progress.Reporter) (bool, error) { + state, ok := idx.graph.(graph.SymbolFTSNormalizationState) + if !ok { + return false, nil + } + want := search.FTSNormalizationMode() + got, marked, err := state.GetSymbolFTSNormalization(idx.RepoPrefix()) + if err != nil { + return false, fmt.Errorf("read symbol FTS normalization: %w", err) + } + if marked && got == want { + return false, nil + } + // A missing marker is a corpus written before normalization existed. That + // historical corpus is already correct for the disabled (empty) mode. + if marked || want != "" { + if err := idx.markSymbolFTSNormalizationPending(idx.graph); err != nil { + return false, err + } + if err := idx.populateSymbolFTS(reporter); err != nil { + return false, err + } + } + if err := idx.markSymbolFTSNormalization(idx.graph); err != nil { + return false, err + } + idx.logger.Info("indexer: symbol FTS normalization reconciled", + zap.String("repo", idx.RepoPrefix()), + zap.String("from", got), + zap.String("to", want), + zap.Bool("rebuilt", marked || want != "")) + return marked || want != "", nil +} diff --git a/internal/indexer/fts_normalization_lifecycle_test.go b/internal/indexer/fts_normalization_lifecycle_test.go new file mode 100644 index 000000000..b59c33ff5 --- /dev/null +++ b/internal/indexer/fts_normalization_lifecycle_test.go @@ -0,0 +1,386 @@ +package indexer + +import ( + "bytes" + "context" + "errors" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/graph/store_sqlite" + "github.com/zzet/gortex/internal/search" +) + +var errFTSLifecycleWrite = errors.New("injected FTS replacement failure") +var errFTSLifecycleReset = errors.New("injected FTS reset failure") +var errFTSLifecycleBatch = errors.New("injected FTS batch failure") +var errFTSLifecycleFinalize = errors.New("injected FTS finalization failure") + +type ftsLifecycleProbeStore struct { + *store_sqlite.Store + replaceCalls int + markerCalls int + resetCalls int + batchCalls int + replaceErr error + resetErr error + batchFailAt int + finalizeErr error +} + +func (s *ftsLifecycleProbeStore) ReplaceSymbolFTS( + repoPrefix string, + produce func(emit func([]graph.SymbolFTSItem) error) error, +) error { + s.replaceCalls++ + if s.replaceErr != nil { + return s.replaceErr + } + return s.Store.ReplaceSymbolFTS(repoPrefix, produce) +} + +func (s *ftsLifecycleProbeStore) SetSymbolFTSNormalization(repoPrefix, mode string) error { + s.markerCalls++ + return s.Store.SetSymbolFTSNormalization(repoPrefix, mode) +} + +func (s *ftsLifecycleProbeStore) ResetSymbolFTS(repoPrefix string) error { + s.resetCalls++ + if s.resetErr != nil { + return s.resetErr + } + return s.Store.ResetSymbolFTS(repoPrefix) +} + +func (s *ftsLifecycleProbeStore) BatchUpsertSymbolFTS(items []graph.SymbolFTSItem) error { + s.batchCalls++ + if s.batchFailAt > 0 && s.batchCalls == s.batchFailAt { + return errFTSLifecycleBatch + } + return s.Store.BatchUpsertSymbolFTS(items) +} + +func (s *ftsLifecycleProbeStore) BuildSymbolIndex() error { + if s.finalizeErr != nil { + return s.finalizeErr + } + return s.Store.BuildSymbolIndex() +} + +func TestFTSNormalizationMarkerFollowsAuthoritativeIndexModes(t *testing.T) { + cases := []struct { + name string + shadowMaxFiles string + streaming string + wantReplacement bool + }{ + {name: "direct", shadowMaxFiles: "0", streaming: "0", wantReplacement: true}, + {name: "streaming", shadowMaxFiles: "0", streaming: "1", wantReplacement: true}, + {name: "cold_shadow", shadowMaxFiles: "1000000", streaming: "0"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Setenv("GORTEX_SHADOW_MAX_FILES", tc.shadowMaxFiles) + t.Setenv("GORTEX_STREAMING_FLUSH", tc.streaming) + t.Setenv("GORTEX_STREAMING_CHUNK_SIZE", "1") + + repo := t.TempDir() + writeFile(t, filepath.Join(repo, "orders.go"), `package shop + +func ReconcilePonies() int { return 1 } +`) + base := openFTSLifecycleStore(t) + probe := &ftsLifecycleProbeStore{Store: base} + idx := newTestIndexer(probe) + + _, err := idx.Index(repo) + require.NoError(t, err) + + got, ok, err := base.GetSymbolFTSNormalization(idx.RepoPrefix()) + require.NoError(t, err) + require.True(t, ok, "authoritative index did not persist an FTS normalization marker") + require.Equal(t, search.FTSNormalizationMode(), got) + require.Positive(t, probe.markerCalls) + if tc.wantReplacement { + require.Positive(t, probe.replaceCalls, "direct durable path must publish an authoritative corpus") + } + }) + } +} + +func TestFTSNormalizationCleanWarmReconcileIsNoOp(t *testing.T) { + base := openFTSLifecycleStore(t) + probe := &ftsLifecycleProbeStore{Store: base} + idx := newTestIndexer(probe) + require.NoError(t, base.SetSymbolFTSNormalization(idx.RepoPrefix(), search.FTSNormalizationMode())) + + _, err := idx.cleanCensusResult(context.Background(), 0, time.Now()) + require.NoError(t, err) + require.Zero(t, probe.replaceCalls, "a current warm corpus must not be rebuilt") + require.Zero(t, probe.markerCalls, "a current marker must not be rewritten") +} + +func TestFTSNormalizationAuthoritativeFailureLeavesPendingMarkerForRetry(t *testing.T) { + t.Setenv("GORTEX_SHADOW_MAX_FILES", "0") + t.Setenv("GORTEX_STREAMING_FLUSH", "0") + + repo := t.TempDir() + writeFile(t, filepath.Join(repo, "orders.go"), `package shop + +func ReconcilePonies() int { return 1 } +`) + base := openFTSLifecycleStore(t) + probe := &ftsLifecycleProbeStore{Store: base, finalizeErr: errFTSLifecycleFinalize} + idx := newTestIndexer(probe) + current := search.FTSNormalizationMode() + require.NoError(t, base.SetSymbolFTSNormalization(idx.RepoPrefix(), current)) + + _, err := idx.Index(repo) + require.ErrorIs(t, err, errFTSLifecycleFinalize) + pending, ok, readErr := base.GetSymbolFTSNormalization(idx.RepoPrefix()) + require.NoError(t, readErr) + require.True(t, ok) + require.NotEqual(t, current, pending, "failed authoritative write left a falsely-current marker") + + probe.finalizeErr = nil + probe.replaceCalls = 0 + probe.markerCalls = 0 + rebuilt, err := idx.reconcileSymbolFTSNormalization(nil) + require.NoError(t, err) + require.True(t, rebuilt, "pending marker did not force an authoritative retry") + require.Positive(t, probe.replaceCalls) + require.Positive(t, probe.markerCalls) + got, ok, readErr := base.GetSymbolFTSNormalization(idx.RepoPrefix()) + require.NoError(t, readErr) + require.True(t, ok) + require.Equal(t, current, got) +} + +func TestFTSNormalizationShadowFailureLeavesPendingMarkerForRetry(t *testing.T) { + t.Setenv("GORTEX_SHADOW_MAX_FILES", "1000000") + t.Setenv("GORTEX_STREAMING_FLUSH", "0") + + for _, tc := range []struct { + name string + resetErr error + batchFailAt int + finalizeErr error + wantErr error + }{ + {name: "reset", resetErr: errFTSLifecycleReset, wantErr: errFTSLifecycleReset}, + {name: "batch", batchFailAt: 1, wantErr: errFTSLifecycleBatch}, + {name: "finalization", finalizeErr: errFTSLifecycleFinalize, wantErr: errFTSLifecycleFinalize}, + } { + t.Run(tc.name, func(t *testing.T) { + repo := t.TempDir() + writeFile(t, filepath.Join(repo, "orders.go"), `package shop + +func ReconcilePonies() int { return 1 } +`) + base := openFTSLifecycleStore(t) + probe := &ftsLifecycleProbeStore{ + Store: base, + resetErr: tc.resetErr, + batchFailAt: tc.batchFailAt, + finalizeErr: tc.finalizeErr, + } + idx := newTestIndexer(probe) + current := search.FTSNormalizationMode() + require.NoError(t, base.SetSymbolFTSNormalization(idx.RepoPrefix(), current)) + + _, err := idx.Index(repo) + require.ErrorIs(t, err, tc.wantErr) + pending, ok, readErr := base.GetSymbolFTSNormalization(idx.RepoPrefix()) + require.NoError(t, readErr) + require.True(t, ok) + require.NotEqual(t, current, pending, "failed shadow publication left a falsely-current marker") + + probe.resetErr = nil + probe.batchFailAt = 0 + probe.finalizeErr = nil + probe.replaceCalls = 0 + probe.markerCalls = 0 + rebuilt, err := idx.reconcileSymbolFTSNormalization(nil) + require.NoError(t, err) + require.True(t, rebuilt, "pending shadow marker did not force an authoritative retry") + require.Positive(t, probe.replaceCalls) + got, ok, readErr := base.GetSymbolFTSNormalization(idx.RepoPrefix()) + require.NoError(t, readErr) + require.True(t, ok) + require.Equal(t, current, got) + }) + } +} + +func TestFTSNormalizationMarkerDoesNotAdvanceOnRebuildFailure(t *testing.T) { + cases := []struct { + name string + replaceErr error + finalizeErr error + wantErr error + }{ + {name: "write", replaceErr: errFTSLifecycleWrite, wantErr: errFTSLifecycleWrite}, + {name: "finalization", finalizeErr: errFTSLifecycleFinalize, wantErr: errFTSLifecycleFinalize}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + base := openFTSLifecycleStore(t) + probe := &ftsLifecycleProbeStore{ + Store: base, + replaceErr: tc.replaceErr, + finalizeErr: tc.finalizeErr, + } + idx := newTestIndexer(probe) + prior := mismatchedFTSNormalizationMode() + require.NoError(t, base.SetSymbolFTSNormalization(idx.RepoPrefix(), prior)) + + rebuilt, err := idx.reconcileSymbolFTSNormalization(nil) + require.ErrorIs(t, err, tc.wantErr) + require.False(t, rebuilt) + require.Positive(t, probe.markerCalls, "rebuild did not invalidate the old marker before writing") + got, ok, readErr := base.GetSymbolFTSNormalization(idx.RepoPrefix()) + require.NoError(t, readErr) + require.True(t, ok) + require.NotEqual(t, search.FTSNormalizationMode(), got, "incomplete rebuild left a falsely-current marker") + }) + } +} + +func TestFTSNormalizationReconcilesBeforeIncrementalMutation(t *testing.T) { + if os.Getenv("GORTEX_TEST_FTS_NORMALIZATION_CHILD") == "1" { + runFTSNormalizationIncrementalChild(t) + return + } + + exe, err := os.Executable() + require.NoError(t, err) + cmd := exec.Command(exe, "-test.run=^TestFTSNormalizationReconcilesBeforeIncrementalMutation$") + cmd.Env = append(envWithoutFTSLifecycleKeys(os.Environ()), + "GORTEX_TEST_FTS_NORMALIZATION_CHILD=1", + "GORTEX_FTS_STEMMING=1", + ) + var output bytes.Buffer + cmd.Stdout = &output + cmd.Stderr = &output + require.NoError(t, cmd.Run(), output.String()) +} + +func runFTSNormalizationIncrementalChild(t *testing.T) { + require.Equal(t, "porter-v1", search.FTSNormalizationMode()) + + for _, tc := range []struct { + name string + paths func(repo, changed string) []string + mode incrementalPathMode + }{ + { + name: "scoped", + paths: func(_ string, changed string) []string { return []string{changed} }, + mode: incrementalPathMode{forceExplicitFiles: true}, + }, + { + name: "full_root", + paths: func(_, _ string) []string { return nil }, + mode: incrementalPathMode{detectDeletions: true}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + t.Setenv("GORTEX_SHADOW_MAX_FILES", "1000000") + t.Setenv("GORTEX_STREAMING_FLUSH", "0") + + repo := t.TempDir() + changed := filepath.Join(repo, "first.go") + writeFile(t, changed, `package herd + +func PoniesFirst() int { return 1 } +`) + writeFile(t, filepath.Join(repo, "second.go"), `package herd + +func PoniesSecond() int { return 2 } +`) + + base := openFTSLifecycleStore(t) + probe := &ftsLifecycleProbeStore{Store: base} + idx := newTestIndexer(probe) + _, err := idx.Index(repo) + require.NoError(t, err) + + oldItems := make([]graph.SymbolFTSItem, 0, 2) + for _, file := range []string{"first.go", "second.go"} { + for _, node := range base.GetFileNodes(file) { + if node != nil && node.Kind == graph.KindFunction && strings.HasPrefix(node.Name, "Ponies") { + oldItems = append(oldItems, graph.SymbolFTSItem{NodeID: node.ID, Tokens: "ponies " + strings.ToLower(node.Name)}) + } + } + } + require.Len(t, oldItems, 2) + require.NoError(t, base.ReplaceSymbolFTS(idx.RepoPrefix(), func(emit func([]graph.SymbolFTSItem) error) error { + return emit(oldItems) + })) + require.NoError(t, base.BuildSymbolIndex()) + require.NoError(t, base.SetSymbolFTSNormalization(idx.RepoPrefix(), "")) + probe.replaceCalls = 0 + probe.markerCalls = 0 + + writeFile(t, changed, `package herd + +func PoniesFirst() int { return 11 } +`) + future := time.Now().Add(2 * time.Second) + require.NoError(t, os.Chtimes(changed, future, future)) + + result, err := idx.incrementalReindexPathsMode(repo, tc.paths(repo, changed), tc.mode) + require.NoError(t, err) + require.Equal(t, 1, result.StaleFileCount) + require.Positive(t, probe.replaceCalls, "mode mismatch was not repaired before the incremental patch") + + gotMode, ok, err := base.GetSymbolFTSNormalization(idx.RepoPrefix()) + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, search.FTSNormalizationMode(), gotMode) + + names := make([]string, 0, 2) + for _, hit := range searchAll(t, base, "pony") { + if node := base.GetNode(hit.NodeID); node != nil { + names = append(names, node.Name) + } + } + require.Contains(t, names, "PoniesFirst") + require.Contains(t, names, "PoniesSecond", "unchanged old-mode posting survived beside the patched corpus") + }) + } +} + +func openFTSLifecycleStore(t *testing.T) *store_sqlite.Store { + t.Helper() + store, err := store_sqlite.Open(filepath.Join(t.TempDir(), "fts-lifecycle.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, store.Close()) }) + return store +} + +func mismatchedFTSNormalizationMode() string { + if search.FTSNormalizationMode() == "" { + return "porter-v1" + } + return "" +} + +func envWithoutFTSLifecycleKeys(env []string) []string { + out := make([]string, 0, len(env)) + for _, entry := range env { + if strings.HasPrefix(entry, "GORTEX_TEST_FTS_NORMALIZATION_CHILD=") || + strings.HasPrefix(entry, "GORTEX_FTS_STEMMING=") { + continue + } + out = append(out, entry) + } + return out +} diff --git a/internal/indexer/indexer.go b/internal/indexer/indexer.go index 99eb59338..5954c2004 100644 --- a/internal/indexer/indexer.go +++ b/internal/indexer/indexer.go @@ -701,6 +701,7 @@ func ftsTokensFor(n *graph.Node, projectName string) string { } tokens = append(tokens, search.Tokenize(f)...) } + tokens = search.NormalizeFTSTokens(tokens) if len(tokens) == 0 { return "" } @@ -2646,6 +2647,9 @@ func (idx *Indexer) indexCtxRaw(ctx context.Context, root string) (result *Index // into the durable store, which is far cheaper than writing each node // and edge through as it is parsed. idx.indexCount.Add(1) + if err := idx.markSymbolFTSNormalizationPending(idx.graph); err != nil { + return nil, err + } diskTarget = idx.graph inMemShadow = idx.newStructuralIntegrityShadow(diskTarget, graph.StructuralPathShadowCold) idx.graph = inMemShadow @@ -2822,6 +2826,11 @@ func (idx *Indexer) indexCtxRaw(ctx context.Context, root string) (result *Index retErr = fmt.Errorf("indexer: finalize backend FTS: %w", ferr) } } + if retErr == nil && ftsReady { + if err := idx.markSymbolFTSNormalization(diskTarget); err != nil { + retErr = err + } + } reporter.Report("building symbol fts", 1, 1) } reporter.Report("persisting bulk graph", 1, 1) @@ -2875,8 +2884,16 @@ func (idx *Indexer) indexCtxRaw(ctx context.Context, root string) (result *Index if retErr != nil { return } + if err := idx.markSymbolFTSNormalizationPending(idx.graph); err != nil { + retErr = err + return + } if err := idx.populateSymbolFTS(reporter); err != nil { retErr = err + return + } + if err := idx.markSymbolFTSNormalization(idx.graph); err != nil { + retErr = err } }() } @@ -3989,6 +4006,9 @@ func (idx *Indexer) cleanCensusResult(ctx context.Context, detected int, started return nil, err } } + if _, err := idx.reconcileSymbolFTSNormalization(nil); err != nil { + return nil, err + } nodes, edges := idx.repoNodeEdgeCount() fileCount := idx.trackedFileCount() @@ -5673,6 +5693,13 @@ func (idx *Indexer) incrementalReindexPathsMode( } idx.storeRootPath(absRoot) + // Reconcile the complete durable corpus before any scoped mutation writes + // rows with this process's normalization mode. Doing this after a partial + // update would leave unchanged symbols in the previous mode. + if _, err := idx.reconcileSymbolFTSNormalization(nil); err != nil { + return nil, err + } + // scopeRels holds the repo-relative slash-paths the caller asked to // reindex — used both to drive the discovery walk and to bound // deletion detection to the scoped subtree. diff --git a/internal/search/fts_normalization_mode_test.go b/internal/search/fts_normalization_mode_test.go new file mode 100644 index 000000000..06656f9bf --- /dev/null +++ b/internal/search/fts_normalization_mode_test.go @@ -0,0 +1,21 @@ +package search + +import "testing" + +func TestFTSNormalizationModeTracksConfiguredCorpusMode(t *testing.T) { + for _, tc := range []struct { + name string + on bool + want string + }{ + {name: "disabled", on: false, want: ""}, + {name: "enabled", on: true, want: "porter-v1"}, + } { + t.Run(tc.name, func(t *testing.T) { + withStemming(t, tc.on) + if got := FTSNormalizationMode(); got != tc.want { + t.Fatalf("FTSNormalizationMode() = %q, want %q", got, tc.want) + } + }) + } +} diff --git a/internal/search/fts_normalize.go b/internal/search/fts_normalize.go index 2ced4e82a..9db38a483 100644 --- a/internal/search/fts_normalize.go +++ b/internal/search/fts_normalize.go @@ -24,6 +24,20 @@ import ( // two never disagree. var ftsStemmingEnabled = ftsStemmingFromEnv() +const ftsNormalizationPorterV1 = "porter-v1" + +// FTSNormalizationMode returns the stable identifier persisted beside each +// repository's native symbol FTS corpus. The empty string is the historical +// unnormalised mode, so stores created before the marker existed remain +// compatible while stemming is disabled. A non-empty change requires the +// indexer to rebuild the repository's symbol FTS before serving it. +func FTSNormalizationMode() string { + if ftsStemmingEnabled { + return ftsNormalizationPorterV1 + } + return "" +} + func ftsStemmingFromEnv() bool { switch strings.ToLower(strings.TrimSpace(os.Getenv("GORTEX_FTS_STEMMING"))) { case "1", "true", "yes", "on", "y": From aa2fcab3693b3debca8699f8b139aee8d7b933bf Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Fri, 14 Aug 2026 21:35:23 +0200 Subject: [PATCH 16/21] test(mcp): migrate rename recovery fixture off BM25 --- internal/mcp/rename_recovery_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/mcp/rename_recovery_test.go b/internal/mcp/rename_recovery_test.go index 6eafeafda..98783f09f 100644 --- a/internal/mcp/rename_recovery_test.go +++ b/internal/mcp/rename_recovery_test.go @@ -234,7 +234,7 @@ func setupMultiRepoRenameRecoveryServer( registry := testRegistry() store := graph.New() - multi := indexer.NewMultiIndexer(store, registry, search.NewBM25(), manager, zap.NewNop()) + multi := indexer.NewMultiIndexer(store, registry, search.NewNull(), manager, zap.NewNop()) _, err = multi.IndexAll() require.NoError(t, err) From c0d2d6fa8fb982edc3553eb1e5f12eb2be18ef98 Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Fri, 14 Aug 2026 21:35:24 +0200 Subject: [PATCH 17/21] docs(search): clarify native FTS evaluation behavior --- BENCHMARK.md | 5 ++++- cmd/gortex/eval_recall.go | 6 +++--- internal/search/fts_normalize.go | 5 +++-- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/BENCHMARK.md b/BENCHMARK.md index d75ad258e..c1d4c0756 100644 --- a/BENCHMARK.md +++ b/BENCHMARK.md @@ -233,7 +233,10 @@ joinable._ symbol-name queries — 3.2× ripgrep's R@5 floor. Enabling Porter stemming (`GORTEX_FTS_STEMMING=1`) trades a little exact-tier precision for breadth — R@20 +5.7pp, exact-tier R@5 −3.1pp — so it -ships opt-in. The `semantic` and `rrf` rankers require `--embeddings` +ships opt-in. The mode is read at process start; on a later restart, +gortex automatically rebuilds a repository's native symbol FTS when +its persisted normalization marker differs. The `semantic` and `rrf` +rankers require `--embeddings` and are omitted here; the `graph` ranker scores only graph-traversal fixtures. diff --git a/cmd/gortex/eval_recall.go b/cmd/gortex/eval_recall.go index 8af1d7b99..e36bc14f1 100644 --- a/cmd/gortex/eval_recall.go +++ b/cmd/gortex/eval_recall.go @@ -62,9 +62,9 @@ Rankers: By default every available ranker runs; narrow with --rankers bm25,rrf. -Without --embeddings, semantic and RRF degrade: semantic reports SKIPPED -and RRF falls back to BM25 inside HybridBackend.Search. Use --embeddings -to enable the built-in static (GloVe) provider, or --embeddings-url to +Without --embeddings, semantic and RRF both report SKIPPED. Use +--embeddings to enable the built-in static (GloVe) provider, or +--embeddings-url to point at an OpenAI-compatible API (e.g. Ollama).`, RunE: runEvalRecall, } diff --git a/internal/search/fts_normalize.go b/internal/search/fts_normalize.go index 9db38a483..01b2c497a 100644 --- a/internal/search/fts_normalize.go +++ b/internal/search/fts_normalize.go @@ -9,8 +9,9 @@ import ( ) // ftsStemmingEnabled gates the token-normalization pass — stopword -// removal plus Porter stemming — applied to the full-text-search index -// and query paths. Default OFF: on the recall fixture stemming trades +// removal plus Porter stemming — applied to the native symbol FTS index +// and query paths. Content FTS intentionally keeps raw body tokens. +// Default OFF: on the recall fixture stemming trades // exact-symbol-lookup precision (exact-tier R@5 −3.1pp) for broader // recall (R@20 +5.7pp), so it ships as an opt-in rather than quietly // reranking every identifier query. Enable it with From ff1ff1ef11387324938abc850b2f73c83127b11a Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Fri, 14 Aug 2026 21:40:08 +0200 Subject: [PATCH 18/21] docs(search): retire stale RRF and builder terminology --- cmd/gortex/eval_recall.go | 3 ++- internal/eval/recall/rankers.go | 6 +++--- internal/indexer/multi_test.go | 2 +- internal/query/engine.go | 2 +- internal/search/rerank/retriever.go | 3 ++- 5 files changed, 9 insertions(+), 7 deletions(-) diff --git a/cmd/gortex/eval_recall.go b/cmd/gortex/eval_recall.go index e36bc14f1..73e119ab0 100644 --- a/cmd/gortex/eval_recall.go +++ b/cmd/gortex/eval_recall.go @@ -56,7 +56,8 @@ Rankers: bm25 — lexical-only: the store-native symbol FTS queried through Engine.SearchSymbols (the search_symbols path) semantic — vector-only (requires --embeddings) - rrf — BM25 + vector fused via RRF (requires --embeddings) + rrf — native FTS + vector with adaptive alpha (historical key; + requires --embeddings) winnow — graph-aware constraint chain (MCP winnow_symbols scorer) ripgrep — rg --files-with-matches baseline ("retrieval floor") diff --git a/internal/eval/recall/rankers.go b/internal/eval/recall/rankers.go index 4a8133d33..18df3cbb7 100644 --- a/internal/eval/recall/rankers.go +++ b/internal/eval/recall/rankers.go @@ -84,9 +84,9 @@ func SemanticRanker(name string, vector *search.VectorBackend, embedder embeddin } } -// RRFRanker adapts a HybridBackend (BM25 + vector fused via RRF) to the -// Ranker shape. Uses Search() which runs both sides and fuses when the -// vector backend has data — otherwise it degrades to BM25 gracefully. +// RRFRanker preserves the historical eval row key while adapting the live +// HybridBackend (native symbol FTS + vector with adaptive-alpha fusion) to the +// Ranker shape. Callers install it only when the vector backend has data. func RRFRanker(name string, hybrid *search.HybridBackend) Ranker { return Ranker{ Name: name, diff --git a/internal/indexer/multi_test.go b/internal/indexer/multi_test.go index 8eec56345..e80357687 100644 --- a/internal/indexer/multi_test.go +++ b/internal/indexer/multi_test.go @@ -290,7 +290,7 @@ func TestMultiIndexer_TrackRepo(t *testing.T) { } // TestMultiIndexer_TrackRepo_SearchSpansAllRepos verifies that scoping -// buildSearchIndex to the current repo (the perf fix that drops the +// native symbol FTS writes to the current repo (the perf fix that drops the // O(N²) re-index of every prior repo's nodes on every TrackRepo call) // does not regress search recall. After three repos are tracked, the // shared search backend must still find symbols defined in the first, diff --git a/internal/query/engine.go b/internal/query/engine.go index 58b352a6d..acab97517 100644 --- a/internal/query/engine.go +++ b/internal/query/engine.go @@ -932,7 +932,7 @@ func (e *Engine) gatherBackendCandidates(query string, limit int, opts QueryOpti insert(id, -1, rank) } - // Stop early when the BM25 + vector union has already exceeded the + // Stop early when the native text + vector union has already exceeded the // requested width; the supplementary tiers below are a fill, not a // boost. if len(cands) >= limit*2 { diff --git a/internal/search/rerank/retriever.go b/internal/search/rerank/retriever.go index 28042e79c..33a43e34f 100644 --- a/internal/search/rerank/retriever.go +++ b/internal/search/rerank/retriever.go @@ -7,7 +7,8 @@ import ( ) // Retriever is the pluggable candidate-producer protocol. The existing -// hybrid pipeline (BM25 + vector + RRF) is one implementation; others +// hybrid pipeline (native symbol FTS + vector + adaptive fusion) is one +// implementation; others // — graph_completion, an in-house embedding model, a research-grade // LLM-as-retriever — plug in by implementing this interface. // From aa4989cb9a2f0971d0ebdf3617abeea545c02012 Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Fri, 14 Aug 2026 21:44:43 +0200 Subject: [PATCH 19/21] chore(deps): tidy checksums after search retirement --- go.sum | 39 +++------------------------------------ 1 file changed, 3 insertions(+), 36 deletions(-) diff --git a/go.sum b/go.sum index c57fded8b..b09cd001b 100644 --- a/go.sum +++ b/go.sum @@ -450,8 +450,6 @@ github.com/charmbracelet/colorprofile v0.4.3 h1:QPa1IWkYI+AOB+fE+mg/5/4HRMZcaXex github.com/charmbracelet/colorprofile v0.4.3/go.mod h1:/zT4BhpD5aGFpqQQqw7a+VtHCzu+zrQtt1zhMt9mR4Q= github.com/charmbracelet/lipgloss v1.1.0 h1:vYXsiLHVkK7fp74RkV7b2kq9+zDLoEU4MZoFqR/noCY= github.com/charmbracelet/lipgloss v1.1.0/go.mod h1:/6Q8FR2o+kj8rz4Dq0zQc3vYf7X+B0binUUBwA0aL30= -github.com/charmbracelet/x/ansi v0.11.7 h1:kzv1kJvjg2S3r9KHo8hDdHFQLEqn4RBCb39dAYC84jI= -github.com/charmbracelet/x/ansi v0.11.7/go.mod h1:9qGpnAVYz+8ACONkZBUWPtL7lulP9No6p1epAihUZwQ= github.com/charmbracelet/x/ansi v0.11.8 h1:JMFwp0CgDC2+jcOB162HH5k7I3FVbgFSMMYg7dSPBQQ= github.com/charmbracelet/x/ansi v0.11.8/go.mod h1:ZNN+3mXny/516oTQPLMPIBeSINvNJJQ8uQXDgbeJxY0= github.com/charmbracelet/x/cellbuf v0.0.15 h1:ur3pZy0o6z/R7EylET877CBxaiE1Sp1GMxoFPAIztPI= @@ -573,8 +571,6 @@ github.com/ledongthuc/pdf v0.0.0-20250511090121-5959a4027728 h1:QwWKgMY28TAXaDl+ github.com/ledongthuc/pdf v0.0.0-20250511090121-5959a4027728/go.mod h1:1fEHWurg7pvf5SG6XNE5Q8UZmOwex51Mkx3SLhrW5B4= github.com/lucasb-eyer/go-colorful v1.4.1 h1:1EO+WB73+EH8EVbzlrG3KLAfEypQWVHIBqlTf+2hNss= github.com/lucasb-eyer/go-colorful v1.4.1/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= -github.com/mark3labs/mcp-go v0.57.0 h1:jzWKyCzdWnwnZt05cvcQQ+ngiUl2RnixXJa7Kj4qP1E= -github.com/mark3labs/mcp-go v0.57.0/go.mod h1:+8WclSK1ZUweCP3hvktSji8n8ABG/95QaEkeVE/Uwas= github.com/mark3labs/mcp-go v0.58.0 h1:AWfBk8lgRR0KZYve7PaLbR2MIjpw1oK2eGpBApaNS+Q= github.com/mark3labs/mcp-go v0.58.0/go.mod h1:+8WclSK1ZUweCP3hvktSji8n8ABG/95QaEkeVE/Uwas= github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= @@ -617,8 +613,6 @@ github.com/sagikazarmark/locafero v0.12.0 h1:/NQhBAkUb4+fH1jivKHWusDYFjMOOKU88ee github.com/sagikazarmark/locafero v0.12.0/go.mod h1:sZh36u/YSZ918v0Io+U9ogLYQJ9tLLBmM4eneO6WwsI= github.com/sahilm/fuzzy v0.1.3 h1:juByESSS32nVD81vr6tHmKmA/8zde7gE+x5CLxrzXPU= github.com/sahilm/fuzzy v0.1.3/go.mod h1:au6//VbVSqu6DFrkL2CfjlJ5iURpNCPeE+1GwY3XsT8= -github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ= -github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= github.com/santhosh-tekuri/jsonschema/v6 v6.0.3 h1:1EYB5IzjZawrrnELUi78f9fPu57HuXjmddZPjrls/28= github.com/santhosh-tekuri/jsonschema/v6 v6.0.3/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= github.com/schollz/progressbar/v3 v3.19.0 h1:Ea18xuIRQXLAUidVDox3AbwfUhD0/1IvohyTutOIFoc= @@ -690,8 +684,6 @@ github.com/tree-sitter/tree-sitter-ruby v0.23.1 h1:T/NKHUA+iVbHM440hFx+lzVOzS4dV github.com/tree-sitter/tree-sitter-ruby v0.23.1/go.mod h1:kUS4kCCQloFcdX6sdpr8p6r2rogbM6ZjTox5ZOQy8cA= github.com/tree-sitter/tree-sitter-rust v0.24.2 h1:NL4nF67ib21RMzzfvkmXlVwe45vvhW10DVyO+D0z/W0= github.com/tree-sitter/tree-sitter-rust v0.24.2/go.mod h1:hfeGWic9BAfgTrc7Xf6FaOAguCFJRo3RBbs7QJ6D7MI= -github.com/tree-sitter/tree-sitter-scala v0.26.0 h1:hpn0hO6cGtAAC9aqyVlp9HDGq9EexUa+6IzcDP6wR44= -github.com/tree-sitter/tree-sitter-scala v0.26.0/go.mod h1:BmDV0f9rgsnGuG9QtKXQZnqJvECyR9fM8wVg984ulBo= github.com/tree-sitter/tree-sitter-scala v0.26.2 h1:ue3zsxnZzKvCAlBqOxIecqgI+NZjjdvrY+hQ8OA3K0Q= github.com/tree-sitter/tree-sitter-scala v0.26.2/go.mod h1:BmDV0f9rgsnGuG9QtKXQZnqJvECyR9fM8wVg984ulBo= github.com/tree-sitter/tree-sitter-typescript v0.23.2 h1:/Odvphn18PniVixb9e97X0DbNVsU6Qocv9mfkyzdXwU= @@ -702,12 +694,8 @@ github.com/viterin/partial v1.1.0 h1:iH1l1xqBlapXsYzADS1dcbizg3iQUKTU1rbwkHv/80E github.com/viterin/partial v1.1.0/go.mod h1:oKGAo7/wylWkJTLrWX8n+f4aDPtQMQ6VG4dd2qur5QA= github.com/viterin/vek v0.4.3 h1:cogdlNjd6EJYtNbmTN0lJCey2htrfSo1AHWpc6DVncQ= github.com/viterin/vek v0.4.3/go.mod h1:A4JRAe8OvbhdzBL5ofzjBS0J29FyUrf95tQogvtHHUc= -github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= -github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= github.com/xo/terminfo v1.0.0 h1:2ZpYzqWzyyytjk3TP6aJVDhkMAkc99/1xKQdA3TDTBY= github.com/xo/terminfo v1.0.0/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= -github.com/yalue/onnxruntime_go v1.32.0 h1:O4pPw3IT+46CRrfuT0lcHWkczVoFvtfq6kMAO/iIVKc= -github.com/yalue/onnxruntime_go v1.32.0/go.mod h1:b4X26A8pekNb1ACJ58wAXgNKeUCGEAQ9dmACut9Sm/4= github.com/yalue/onnxruntime_go v1.32.1 h1:bjvmp0ONztctKBOkaU6hgphVoKPH3GTZAJvFfPHdtcI= github.com/yalue/onnxruntime_go v1.32.1/go.mod h1:b4X26A8pekNb1ACJ58wAXgNKeUCGEAQ9dmACut9Sm/4= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= @@ -727,20 +715,12 @@ go.uber.org/zap v1.28.0/go.mod h1:rDLpOi171uODNm/mxFcuYWxDsqWSAVkFdX4XojSKg/Q= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= -golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= -golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= -golang.org/x/exp v0.0.0-20260727155853-b88d891fe743 h1:ex206bKw+v3K0dm3andkrIF+ijyQKJG1pLgwQ2PYdQM= -golang.org/x/exp v0.0.0-20260727155853-b88d891fe743/go.mod h1:EdfpwwqSu+0Li0mzskwHU6FWDV3t9Q+RZDo3QMUtL3Q= golang.org/x/exp v0.0.0-20260813180055-c1d0aacb2297 h1:YXnL44eJ77R+ji4/ooy8UsXIhz+lbi2Qgdlc8iRN0gY= golang.org/x/exp v0.0.0-20260813180055-c1d0aacb2297/go.mod h1:Mkmymgv+uMpSQ/XxJ/7GpdrdYoqm3u72jEbpCLiJmNk= -golang.org/x/image v0.44.0 h1:+tDekMZED9+LrtB3G5xzRggpVh9CARjZqROla3R3R+I= -golang.org/x/image v0.44.0/go.mod h1:V8K3KE9KKKE+pLpQDOeN18w9oacNSvy1tDOirTu4xtY= golang.org/x/image v0.45.0 h1:FMb1nTbH5H9vF55SriQHgFw5GnNL9Jg6L25BwXKzhB0= golang.org/x/image v0.45.0/go.mod h1:n62x/7RqlwXDvGsSU4u6IUTUf6KghUZ9Bt7cG/T9Fx4= -golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk= -golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40= golang.org/x/mod v0.40.0 h1:hUv+3cXcdRHz08UmSiOob7sadHig73uo5bkXxQ/tvUs= golang.org/x/mod v0.40.0/go.mod h1:0/weTWkPWGBikyTWAX3dkjVztMmBA5hM0DH6BElSupE= golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= @@ -750,18 +730,12 @@ golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= -golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= -golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= -golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE= -golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk= golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI= golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo= gonum.org/v1/plot v0.15.2 h1:Tlfh/jBk2tqjLZ4/P8ZIwGrLEWQSPDLRm/SNWKNXiGI= gonum.org/v1/plot v0.15.2/go.mod h1:DX+x+DWso3LTha+AdkJEv5Txvi+Tql3KAGkehP0/Ubg= -google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= -google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -772,29 +746,22 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc= k8s.io/klog/v2 v2.140.0/go.mod h1:o+/RWfJ6PwpnFn7OyAG3QnO47BFsymfEfrz6XyYSSp0= -modernc.org/cc/v4 v4.29.1 h1:MKgdCV3WykTSPqpVrnxdEDS0HEd2FHpKZDzxzU5LyeI= -modernc.org/cc/v4 v4.29.1/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= modernc.org/cc/v4 v4.29.2 h1:h6+9ciCnPKutf4I03CvheAvDLX7+IHlqR6Iy6J+cgd8= -modernc.org/ccgo/v4 v4.34.6 h1:sBgfIwyN0TQ9C5hwIeuqyeAKyMWnbvj2fvpF4L11uzU= -modernc.org/ccgo/v4 v4.34.6/go.mod h1:SZ8YcN9NG7XVsQYdm6jYBvi8PQP1qi+kqB6OhjqI3Fk= +modernc.org/cc/v4 v4.29.2/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= modernc.org/ccgo/v4 v4.35.0 h1:F+TUsmw09QxLzmi3aeYYGxjAXarmZaKgj3mKQHNaA8w= +modernc.org/ccgo/v4 v4.35.0/go.mod h1:qrVGs9S3Sr2Ztcg9ve+kTAYMp5a3YvWjo+SoN06kJ5I= modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= -modernc.org/gc/v3 v3.1.4 h1:2g65LGVSmFQrXeITAw97x7hCRvZFcyE1uDP+7Vng7JI= -modernc.org/gc/v3 v3.1.4/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= modernc.org/gc/v3 v3.1.5 h1:21ldfPfRYE31Tb7B3mwAK8gy1AxP4+dKjrOQPfqakoc= +modernc.org/gc/v3 v3.1.5/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= -modernc.org/libc v1.74.4 h1:fX1Omw4o2/1C2iRkkIsrQTasJQldLhRmuPreXLoWs9k= -modernc.org/libc v1.74.4/go.mod h1:eeQAS9W3sZeKYMFubydxJpII9ybHWshk+7or7bLG9co= modernc.org/libc v1.75.3 h1:vCqT5+R0jPXMnvMkGo0T2zXvFNth+lYXVCx5X7CCX/g= modernc.org/libc v1.75.3/go.mod h1:MjAX68G+0oufI+hNuh0QXcK+Ap+sL8bNPPcIC6EqOfo= modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= -modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= -modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= modernc.org/memory v1.12.0 h1:twkmYNkGXCvtYWzoux02jtK6eovjZbdI0uHFUYp6kuU= modernc.org/memory v1.12.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= From bf3a0471758075ad01a3c44eca002e3505ee3e80 Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Fri, 14 Aug 2026 21:45:45 +0200 Subject: [PATCH 20/21] style(mcp): format explore fixture --- internal/mcp/tools_explore_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/mcp/tools_explore_test.go b/internal/mcp/tools_explore_test.go index be91dd902..afa58ae98 100644 --- a/internal/mcp/tools_explore_test.go +++ b/internal/mcp/tools_explore_test.go @@ -710,7 +710,7 @@ func TestRenderExploreLimitsFullBodiesToTopTargets(t *testing.T) { targets = append(targets, exploreTarget{ node: &graph.Node{ Name: name, Kind: graph.KindFunction, - FilePath: fmt.Sprintf("cand_%02d.go", index), + FilePath: fmt.Sprintf("cand_%02d.go", index), StartLine: 1, EndLine: 4, Language: "go", }, source: fmt.Sprintf("func %s() int {\n\treturn 9900 + %d\n}", name, index), From d1ae1a299b73778c7dcef07e17c43b94622f668b Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Fri, 14 Aug 2026 22:09:06 +0200 Subject: [PATCH 21/21] test(daemon): expect authoritative native FTS counts --- cmd/gortex/daemon_search_backend_test.go | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/cmd/gortex/daemon_search_backend_test.go b/cmd/gortex/daemon_search_backend_test.go index 318f9c36b..d3ebe1a6c 100644 --- a/cmd/gortex/daemon_search_backend_test.go +++ b/cmd/gortex/daemon_search_backend_test.go @@ -40,23 +40,21 @@ func TestResolveSearchBackend_SymbolSearcherBackend(t *testing.T) { assert.Equal(t, "sqlite-fts5", info.Name) assert.True(t, info.DiskResident, "the FTS5 index lives inside the graph store, not in-process heap") assert.Zero(t, info.Bytes, "no fabricated byte count for a disk-resident backend") - // Count() is a since-construction Add/Remove delta, not a corpus size — - // it goes negative as soon as an eviction path drops more than the admit - // predicate ever added. A store that cannot answer the real count must - // leave the figure unreported rather than have the delta stand in for it. - assert.False(t, info.DocCountKnown, "the delta must never be reported as a document count") + // Add and Remove are no-ops because the native store owns the corpus. + // A store that cannot answer the authoritative count must leave the figure + // unreported rather than fabricate an adapter-local document count. + assert.False(t, info.DocCountKnown, "an unavailable count must remain unknown") assert.Zero(t, info.DocCount) } func TestResolveSearchBackend_SymbolSearcherBackend_CountFromIndex(t *testing.T) { - // Adds and removes move the adapter's delta. Count combines that delta - // with the persisted baseline, while status still reports the index's own - // authoritative count. + // Adds and removes are no-ops. Count and status both report the native + // index's authoritative corpus size without duplicating write-path deltas. b := search.NewSymbolSearcherBackend(countingSymbolSearcher{count: 48572}) b.Add("node-1") b.Remove("node-2") b.Remove("node-3") - assert.Equal(t, 48571, b.Count(), "Count must combine the persisted baseline with the adapter delta") + assert.Equal(t, 48572, b.Count(), "Count must come from the native index") info := resolveSearchBackend(b)